redcord 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85772724fda7e2a152acab1f04164ff1ee0146a15f13facebd7b5f7b18323aa9
-  data.tar.gz: e1527ca0cfe6e7cc0a0aef6346bdb055cd47e6e6d7ca22aa8488608a02e5eeaa
+  metadata.gz: 16cb660da0ea843f300ff18cc090ccfed7883c9d1ca48ef0bc5ed8d4e805854c
+  data.tar.gz: f53002cb43b3e5286ba1f914c41ffd6b8a9722d2d33d9764deb7bcba88d2659c
 SHA512:
-  metadata.gz: cd8b0d84aecba33d41fdad600f00c74a2005c566f439914b3a72f920effc87d344b6974fc18a236d4972fb6d21efa607f43ee121da2d5c70cc6d772a5b80e5c7
-  data.tar.gz: c16a9836a3556bbfdf38b04e78e66827733c7e40cfb002ec2744c9748453968571c110dff24f00838b3b2dfa64b658b79c378c214dd82c936df0459c303ab269
+  metadata.gz: db443ffbf21066c549b30daed2ad44f836aa4469065b7bd8b6d0c13f45597bd7bdbf6c7ef4202ddf8103a1c2fbcfca28ca60f9f59b90584d0ec718d380d0f908
+  data.tar.gz: 40ded5853307f453fe7c5ab2962e29cc081c05ab7f0c2cfddb898259e207ae9c343dfee6a67967449ff43c227c88db6f7a7922c425d8e6df92466b5ed778c5c2

data/lib/redcord/actions.rb

@@ -9,6 +9,7 @@ require 'redcord/relation'
 module Redcord
   # Raised by Model.find
   class RecordNotFound < StandardError; end
+  class InvalidAction < StandardError; end
 end
 
 module Redcord::Actions
@@ -24,15 +25,36 @@ module Redcord::Actions
   module ClassMethods
     extend T::Sig
 
+    sig { returns(Integer) }
+    def count
+      Redcord::Base.trace(
+       'redcord_actions_class_methods_count',
+        model_name: name,
+      ) do
+        res = 0
+        redis.scan_each_shard("#{model_key}:id:*") { res += 1 }
+        res
+      end
+    end
+
     sig { params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
     def create!(args)
       Redcord::Base.trace(
        'redcord_actions_class_methods_create!',
         model_name: name,
       ) do
+        self.props.keys.each { |attr_key| args[attr_key] = nil unless args.key?(attr_key) }
         args[:created_at] = args[:updated_at] = Time.zone.now
         instance = TypeCoerce[self].new.from(args)
-        id = redis.create_hash_returning_id(model_key, to_redis_hash(args))
+        id = redis.create_hash_returning_id(
+          model_key,
+          to_redis_hash(args),
+          ttl: _script_arg_ttl,
+          index_attrs: _script_arg_index_attrs,
+          range_index_attrs: _script_arg_range_index_attrs,
+          custom_index_attrs: _script_arg_custom_index_attrs,
+          hash_tag: instance.hash_tag,
+        )
         instance.send(:id=, id)
         instance
       end
@@ -75,7 +97,13 @@ module Redcord::Actions
        'redcord_actions_class_methods_destroy',
         model_name: name,
       ) do
-        redis.delete_hash(model_key, id) == 1
+        redis.delete_hash(
+          model_key,
+          id,
+          index_attrs: _script_arg_index_attrs,
+          range_index_attrs: _script_arg_range_index_attrs,
+          custom_index_attrs: _script_arg_custom_index_attrs,
+        ) == 1
       end
     end
   end
@@ -115,10 +143,19 @@ module Redcord::Actions
         self.updated_at = Time.zone.now
         _id = id
         if _id.nil?
+          serialized_instance = serialize
+          self.class.props.keys.each do |attr_key|
+            serialized_instance[attr_key.to_s] = nil unless serialized_instance.key?(attr_key.to_s) 
+          end
           self.created_at = T.must(self.updated_at)
           _id = redis.create_hash_returning_id(
             self.class.model_key,
-            self.class.to_redis_hash(serialize),
+            self.class.to_redis_hash(serialized_instance),
+            ttl: self.class._script_arg_ttl,
+            index_attrs: self.class._script_arg_index_attrs,
+            range_index_attrs: self.class._script_arg_range_index_attrs,
+            custom_index_attrs: self.class._script_arg_custom_index_attrs,
+            hash_tag: hash_tag,
           )
           send(:id=, _id)
         else
@@ -126,17 +163,37 @@ module Redcord::Actions
             self.class.model_key,
             _id,
             self.class.to_redis_hash(serialize),
+            ttl: self.class._script_arg_ttl,
+            index_attrs: self.class._script_arg_index_attrs,
+            range_index_attrs: self.class._script_arg_range_index_attrs,
+            custom_index_attrs: self.class._script_arg_custom_index_attrs,
+            hash_tag: hash_tag,
           )
         end
       end
     end
 
+    sig { returns(T::Boolean) }
+    def save
+      save!
+
+      true
+    rescue Redis::CommandError
+      # TODO: break down Redis::CommandError by parsing the error message
+      false
+    end
+
     sig { params(args: T::Hash[Symbol, T.untyped]).void }
-    def update!(args = {})
+    def update!(args)
       Redcord::Base.trace(
        'redcord_actions_instance_methods_update!',
         model_name: self.class.name,
       ) do
+        shard_by_attr = self.class.shard_by_attribute
+        if args.keys.include?(shard_by_attr)
+          raise Redcord::InvalidAction, "Cannot update shard_by attribute #{shard_by_attr}"
+        end
+
         _id = id
         if _id.nil?
           _set_args!(args)
@@ -148,11 +205,26 @@ module Redcord::Actions
             self.class.model_key,
             _id,
             self.class.to_redis_hash(args),
+            ttl: self.class._script_arg_ttl,
+            index_attrs: self.class._script_arg_index_attrs,
+            range_index_attrs: self.class._script_arg_range_index_attrs,
+            custom_index_attrs: self.class._script_arg_custom_index_attrs,
+            hash_tag: hash_tag,
           )
         end
       end
     end
 
+    sig { params(args: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
+    def update(args)
+      update!(args)
+
+      true
+    rescue Redis::CommandError
+      # TODO: break down Redis::CommandError by parsing the error message
+      false
+    end
+
     sig { returns(T::Boolean) }
     def destroy
       Redcord::Base.trace(
@@ -177,14 +249,14 @@ module Redcord::Actions
       end
     end
 
-    sig { returns(T.nilable(Integer)) }
+    sig { returns(T.nilable(String)) }
     def id
       instance_variable_get(:@_id)
     end
 
     private
 
-    sig { params(id: Integer).returns(Integer) }
+    sig { params(id: String).returns(String) }
     def id=(id)
       instance_variable_set(:@_id, id)
     end

data/lib/redcord/attribute.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 # typed: strict
+module Redcord
+  class InvalidAttribute < StandardError; end
+end
 
 module Redcord::Attribute
   extend T::Sig
@@ -10,18 +13,33 @@ module Redcord::Attribute
   # type.
   RangeIndexType = T.type_alias {
     T.any(
-      T.nilable(Float),
-      T.nilable(Integer),
-      T.nilable(Time),
+      Float,
+      Integer,
+      NilClass,
+      Numeric,
+      Time,
     )
   }
 
+  # Implicitly determine what data type can be a used in custom index on Redis based on Ruby type.
+  # Custom index currently supports positive integers with up to 19 characters in decimal notation, 
+  # will raise error in Lua if bigger numbers are used.
+  CustomIndexType = T.type_alias {
+    T.any(
+      Integer,
+      Time,
+    )
+  }
+  
   sig { params(klass: T.class_of(T::Struct)).void }
   def self.included(klass)
     klass.extend(ClassMethods)
+    klass.include(InstanceMethods)
     klass.class_variable_set(:@@index_attributes, Set.new)
     klass.class_variable_set(:@@range_index_attributes, Set.new)
+    klass.class_variable_set(:@@custom_index_attributes, Hash.new { |h, k| h[k] = [] })
     klass.class_variable_set(:@@ttl, nil)
+    klass.class_variable_set(:@@shard_by_attribute, nil)
   end
 
   module ClassMethods
@@ -46,30 +64,81 @@ module Redcord::Attribute
     def index_attribute(attr, type)
       if should_range_index?(type)
         class_variable_get(:@@range_index_attributes) << attr
-        sadd_proc_on_redis_connection('range_index_attrs', attr.to_s)
       else
         class_variable_get(:@@index_attributes) << attr
-        sadd_proc_on_redis_connection('index_attrs', attr.to_s)
       end
     end
+    
+    sig { params(index_name: Symbol, attrs: T::Array[Symbol]).void }
+    def custom_index(index_name, attrs)
+      attrs.each do |attr|
+        type = props[attr][:type]
+        if !can_custom_index?(type)
+          raise(Redcord::WrongAttributeType, "Custom index doesn't support '#{type}' attributes.")
+        end
+      end
+      shard_by_attr = class_variable_get(:@@shard_by_attribute)
+      if shard_by_attr and shard_by_attr != attrs.first
+        raise(
+          Redcord::CustomIndexInvalidDesign,
+          "shard_by attribute '#{shard_by_attr}' must be placed first in '#{index_name}' index"
+        )
+      end
+      class_variable_get(:@@custom_index_attributes)[index_name] = attrs
+    end
 
     sig { params(duration: T.nilable(ActiveSupport::Duration)).void }
     def ttl(duration)
       class_variable_set(:@@ttl, duration)
     end
 
-    private
+    def shard_by_attribute(attr=nil)
+      return class_variable_get(:@@shard_by_attribute) if attr.nil?
 
-    sig { params(redis_key: String, item_to_add: String).void }
-    def sadd_proc_on_redis_connection(redis_key, item_to_add)
-      # TODO: Currently we're setting indexed attributes through procs that are
-      # run when a RedisConnection is established. This should be replaced with
-      # migrations
-      Redcord::RedisConnection.procs_to_prepare << proc do |redis|
-        redis.sadd("#{model_key}:#{redis_key}", item_to_add)
+      # attr must be an non-index attribute (index: false)
+      if class_variable_get(:@@index_attributes).include?(attr) ||
+          class_variable_get(:@@range_index_attributes).include?(attr)
+        raise Redcord::InvalidAttribute, "Cannot shard by an index attribute '#{attr}'"
       end
+
+      class_variable_get(:@@custom_index_attributes).each do |index_name, attrs|
+        if attr != attrs.first
+          raise(
+            Redcord::CustomIndexInvalidDesign,
+            "shard_by attribute '#{attr}' must be placed first in '#{index_name}' index"
+          )
+        end
+
+        # Delete the shard_by_attribute since it would be a constant in the
+        # custom index set
+        attrs.shift
+      end
+
+      class_variable_set(:@@shard_by_attribute, attr)
     end
 
+    sig { returns(Integer) }
+    def _script_arg_ttl
+      class_variable_get(:@@ttl)&.to_i || -1
+    end
+
+    sig { returns(T::Array[Symbol]) }
+    def _script_arg_index_attrs
+      class_variable_get(:@@index_attributes).to_a
+    end
+
+    sig { returns(T::Array[Symbol]) }
+    def _script_arg_range_index_attrs
+      class_variable_get(:@@range_index_attributes).to_a
+    end
+
+    sig { returns(T::Hash[Symbol, T::Array]) }
+    def _script_arg_custom_index_attrs
+      class_variable_get(:@@custom_index_attributes)
+    end
+
+    private
+
     sig { params(type: T.any(Class, T::Types::Base)).returns(T::Boolean) }
     def should_range_index?(type)
       # Change Ruby raw type to Sorbet type in order to call subtype_of?
@@ -77,6 +146,34 @@ module Redcord::Attribute
 
       type.subtype_of?(RangeIndexType)
     end
+
+    sig { params(type: T.any(Class, T::Types::Base)).returns(T::Boolean) }
+    def can_custom_index?(type)
+      # Change Ruby raw type to Sorbet type in order to call subtype_of?
+      type = T::Types::Simple.new(type) if type.is_a?(Class)
+      type.subtype_of?(CustomIndexType)
+    end
+  end
+
+  module InstanceMethods
+    extend T::Sig
+
+    sig { returns(T.nilable(String)) }
+    def hash_tag
+      attr = self.class.class_variable_get(:@@shard_by_attribute)
+
+      return nil if attr.nil?
+
+      # A blank hash tag would cause MOVED error in cluster mode
+      tag = send(attr)
+      default_tag = '__redcord_hash_tag_null__'
+
+      if tag == default_tag
+        raise Redcord::InvalidAttribute, "#{attr}=#{default_tag} conflicts with default hash_tag value"
+      end
+
+      "{#{tag || default_tag}}"
+    end
   end
 
   mixes_in_class_methods(ClassMethods)

data/lib/redcord/migration/index.rb

@@ -7,32 +7,21 @@ module Redcord::Migration::Index
 
   sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
   def remove_index(model, index_name)
-    if model.redis.sismember("#{model.model_key}:index_attrs", index_name)
-      _remove_index_from_attr_set(
-        model: model,
-        attr_set_name: 'index_attrs',
-        index_name: index_name,
-      )
+    model.redis.scan_each_shard("#{model.model_key}:#{index_name}:*") { |key| _del_set(model, key) }
 
-      model.redis.scan_each(match: "#{model.model_key}:#{index_name}:*") { |key| _del_set(model, key) }
-    elsif model.redis.sismember("#{model.model_key}:range_index_attrs", index_name)
-      _remove_index_from_attr_set(
-        model: model,
-        attr_set_name: 'range_index_attrs',
-        index_name: index_name,
-      )
+    attr_set = "#{model.model_key}:#{index_name}"
+    nil_attr_set = "#{attr_set}:"
 
-      attr_set = "#{model.model_key}:#{index_name}"
-      nil_attr_set = "#{attr_set}:"
+    model.redis.scan_each_shard("#{nil_attr_set}*") { |key| _del_set(model, key) }
+    model.redis.scan_each_shard("#{attr_set}*") { |key| _del_zset(model, key) }
+  end
 
-      _del_set(model, nil_attr_set)
-      _del_zset(model, attr_set)
-    else
-      raise(
-        Redcord::AttributeNotIndexed,
-        "#{index_name} is not an indexed attribute.",
-      )
-    end
+  sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
+  def remove_custom_index(model, index_name)
+    index_key = "#{model.model_key}:custom_index:#{index_name}"
+    index_content_key = "#{model.model_key}:custom_index:#{index_name}_content"
+    model.redis.scan_each_shard("#{index_key}*") { |key| model.redis.unlink(key) }
+    model.redis.scan_each_shard("#{index_content_key}*") { |key| model.redis.unlink(key) }
   end
 
   sig {

data/lib/redcord/migration/ttl.rb

@@ -7,16 +7,9 @@ module Redcord::Migration::TTL
     model.class_variable_get(:@@ttl) || -1
   end
 
-  # This won't change ttl until we call update on a record
-  sig { params(model: T.class_of(Redcord::Base)).void }
-  def change_ttl_passive(model)
-    model.redis.set("#{model.model_key}:ttl", _get_ttl(model))
-  end
-
   sig { params(model: T.class_of(Redcord::Base)).void }
   def change_ttl_active(model)
-    change_ttl_passive(model)
-    model.redis.scan_each(match: "#{model.model_key}:id:*") do |key|
+    model.redis.scan_each_shard("#{model.model_key}:id:*") do |key|
       model.redis.expire(key, _get_ttl(model))
     end
   end

data/lib/redcord/railtie.rb

@@ -29,7 +29,6 @@ class Redcord::Railtie < Rails::Railtie
       )
     end
 
-    Redcord::PreparedRedis.load_server_scripts!
     Redcord._after_initialize!
   end
 end

data/lib/redcord/redis.rb

@@ -0,0 +1,225 @@
+# typed: true
+require 'digest'
+require 'redis'
+require 'securerandom'
+
+class Redcord::Redis < Redis
+  extend T::Sig
+
+  sig do
+    params(
+      key: T.any(String, Symbol),
+      args: T::Hash[T.untyped, T.untyped],
+      ttl: T.nilable(Integer),
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      custom_index_attrs: T::Hash[Symbol, T::Array],
+      hash_tag: T.nilable(String),
+    ).returns(String)
+  end
+  def create_hash_returning_id(key, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag: nil)
+    Redcord::Base.trace(
+      'redcord_redis_create_hash_returning_id',
+      model_name: key,
+    ) do
+      id = "#{SecureRandom.uuid}#{hash_tag}"
+      custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
+        result << index_name
+        result << attrs.size
+        result + attrs
+      end
+      run_script(
+        :create_hash,
+        keys: [id, hash_tag],
+        argv: [key, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] + 
+          index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
+      )
+      id
+    end
+  end
+
+  sig do
+    params(
+      model: String,
+      id: String,
+      args: T::Hash[T.untyped, T.untyped],
+      ttl: T.nilable(Integer),
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      custom_index_attrs: T::Hash[Symbol, T::Array],
+      hash_tag: T.nilable(String),
+    ).void
+  end
+  def update_hash(model, id, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag:)
+    Redcord::Base.trace(
+      'redcord_redis_update_hash',
+      model_name: model,
+    ) do
+      custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
+        if !(args.keys.to_set & attrs.to_set).empty?
+          result << index_name
+          result << attrs.size
+          result + attrs
+        else
+          result
+        end
+      end
+      run_script(
+        :update_hash,
+        keys: [id, hash_tag],
+        argv: [model, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
+          index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
+      )
+    end
+  end
+
+  sig do
+    params(
+      model: String,
+      id: String,
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      custom_index_attrs: T::Hash[Symbol, T::Array],
+    ).returns(Integer)
+  end
+  def delete_hash(model, id, index_attrs:, range_index_attrs:, custom_index_attrs:)
+    Redcord::Base.trace(
+      'redcord_redis_delete_hash',
+      model_name: model,
+    ) do
+      custom_index_names = custom_index_attrs.keys
+      run_script(
+        :delete_hash,
+        keys: [id, id.match(/\{.*\}$/)&.send(:[], 0)],
+        argv: [model, index_attrs.size, range_index_attrs.size] + index_attrs + range_index_attrs + custom_index_names,
+      )
+    end
+  end
+
+  sig do
+    params(
+      model: String,
+      query_conditions: T::Hash[T.untyped, T.untyped],
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      select_attrs: T::Set[Symbol],
+      custom_index_attrs: T::Array[Symbol],
+      hash_tag: T.nilable(String),
+      custom_index_name: T.nilable(Symbol),
+    ).returns(T::Hash[Integer, T::Hash[T.untyped, T.untyped]])
+  end
+  def find_by_attr(
+        model,
+        query_conditions,
+        select_attrs: Set.new,
+        index_attrs:,
+        range_index_attrs:,
+        custom_index_attrs: Array.new,
+        hash_tag: nil,
+        custom_index_name: nil
+      )
+    Redcord::Base.trace(
+      'redcord_redis_find_by_attr',
+      model_name: model,
+    ) do
+      conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
+      res = run_script(
+        :find_by_attr,
+        keys: [hash_tag],
+        argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size, conditions.size] + 
+          index_attrs + range_index_attrs + custom_index_attrs + conditions + select_attrs.to_a.flatten
+      )
+      # The Lua script will return this as a flattened array.
+      # Convert the result into a hash of {id -> model hash}
+      res_hash = res.each_slice(2)
+      res_hash.map { |key, val| [key, val.each_slice(2).to_h] }.to_h
+    end
+  end
+
+  sig do
+    params(
+      model: String,
+      query_conditions: T::Hash[T.untyped, T.untyped],
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      custom_index_attrs: T::Array[Symbol],
+      hash_tag: T.nilable(String),
+      custom_index_name: T.nilable(Symbol),
+    ).returns(Integer)
+  end
+  def find_by_attr_count(
+        model,
+        query_conditions,
+        index_attrs:,
+        range_index_attrs:,
+        custom_index_attrs: Array.new,
+        hash_tag: nil,
+        custom_index_name: nil
+      )
+    Redcord::Base.trace(
+      'redcord_redis_find_by_attr_count',
+      model_name: model,
+    ) do
+      conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
+      run_script(
+        :find_by_attr_count,
+        keys: [hash_tag],
+        argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size] +
+          index_attrs + range_index_attrs + custom_index_attrs + conditions
+      )
+    end
+  end
+
+  def scan_each_shard(key, &blk)
+    clients = instance_variable_get(:@client)
+      &.instance_variable_get(:@node)
+      &.instance_variable_get(:@clients)
+      &.values
+
+    if clients.nil?
+      scan_each(match: key, &blk)
+    else
+      clients.each do |client|
+        cursor = 0
+        loop do
+          cursor, keys = client.call([:scan, cursor, 'match', key])
+          keys.each(&blk)
+          break if cursor == "0"
+        end
+      end
+    end
+  end
+
+  private
+
+  def run_script(script_name, *args)
+    # Use EVAL when a redis shard has not loaded the script before
+    hash_var_name = :"@script_sha_#{script_name}"
+    hash = instance_variable_get(hash_var_name)
+
+    begin
+      return evalsha(hash, *args) if hash
+    rescue Redis::CommandError => e
+      if e.message != 'NOSCRIPT No matching script. Please use EVAL.'
+        raise e
+      end
+    end
+
+    script_content = Redcord::LuaScriptReader.read_lua_script(script_name.to_s)
+    instance_variable_set(hash_var_name, Digest::SHA1.hexdigest(script_content))
+    self.eval(script_content, *args)
+  end
+
+  # When using custom index: On Lua side script expects query conditions sorted 
+  # in the order of appearance of attributes in specified index
+  sig { params(query_conditions: T::Hash[T.untyped, T.untyped], partial_order: T::Array[Symbol]).returns(T::Array[T.untyped]) }
+  def flatten_with_partial_sort(query_conditions, partial_order)
+    conditions = partial_order.inject([]) do |result, attr|
+      if !query_conditions[attr].nil?
+        result << attr << query_conditions.delete(attr)
+      end
+      result.flatten
+    end
+    conditions += query_conditions.to_a.flatten
+  end
+end