redcord 0.0.2.alpha → 0.1.2

data/lib/redcord/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+#
 # typed: strict
 #
 # A Redis ORM API inspired by ActiveRecord:
@@ -11,6 +13,7 @@ require 'redcord/configurations'
 require 'redcord/logger'
 require 'redcord/redis_connection'
 require 'redcord/serializer'
+require 'redcord/tracer'
 
 module Redcord::Base
   extend T::Sig
@@ -23,6 +26,7 @@ module Redcord::Base
   include Redcord::Configurations
   include Redcord::Logger
   include Redcord::RedisConnection
+  include Redcord::Tracer
 
   abstract!
 
@@ -52,9 +56,19 @@ module Redcord::Base
       # coerced to the specified attribute types. Like ActiveRecord,
       # Redcord manages the created_at and updated_at fields behind the
       # scene.
-      attribute :id, T.nilable(Integer), index: true
-      attribute :created_at, T.nilable(Time), index: true
-      attribute :updated_at, T.nilable(Time), index: true
+      prop :created_at, T.nilable(Time)
+      prop :updated_at, T.nilable(Time)
     end
   end
+
+  sig { returns(T::Array[T.class_of(Redcord::Base)]) }
+  def self.descendants
+    descendants = []
+    # TODO: Use T::Struct instead of Class
+    ObjectSpace.each_object(Class) do |klass|
+      descendants << klass if klass < self
+    end
+    descendants
+  end
+
 end

data/lib/redcord/configurations.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+#
 # typed: strict
 #
 #  This allows us to configure Redis connections for Redcord. Redis
@@ -40,6 +42,8 @@
 #  ```
 #
 require 'redcord/redis_connection'
+require 'redcord/tracer'
+
 module Redcord::Configurations
   extend T::Sig
   extend T::Helpers

data/lib/redcord/logger.rb

@@ -12,7 +12,7 @@ module Redcord::Logger
   module ClassMethods
     extend T::Sig
 
-    @@logger = T.let(Rails.logger, T.untyped)
+    @@logger = T.let(nil, T.untyped)
 
     sig { returns(T.untyped) }
     def logger

data/lib/redcord/migration.rb

@@ -2,11 +2,13 @@
 class Redcord::Migration
 end
 
+require 'redcord/migration/index'
 require 'redcord/migration/ttl'
 
 class Redcord::Migration
   extend T::Sig
   extend T::Helpers
+  include Redcord::Migration::Index
   include Redcord::Migration::TTL
 
   abstract!

data/lib/redcord/migration/index.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# typed: strict
+
+module Redcord::Migration::Index
+  extend T::Sig
+
+  sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
+  def remove_index(model, index_name)
+    model.redis.scan_each_shard("#{model.model_key}:#{index_name}:*") { |key| _del_set(model, key) }
+
+    attr_set = "#{model.model_key}:#{index_name}"
+    nil_attr_set = "#{attr_set}:"
+
+    model.redis.scan_each_shard("#{nil_attr_set}*") { |key| _del_set(model, key) }
+    model.redis.scan_each_shard("#{attr_set}*") { |key| _del_zset(model, key) }
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
+  def remove_custom_index(model, index_name)
+    index_key = "#{model.model_key}:custom_index:#{index_name}"
+    index_content_key = "#{model.model_key}:custom_index:#{index_name}_content"
+    model.redis.scan_each_shard("#{index_key}*") { |key| model.redis.unlink(key) }
+    model.redis.scan_each_shard("#{index_content_key}*") { |key| model.redis.unlink(key) }
+  end
+
+  sig {
+    params(
+      model: T.class_of(Redcord::Base),
+      attr_set_name: String,
+      index_name: Symbol,
+    ).void
+  }
+  def _remove_index_from_attr_set(model:, attr_set_name:, index_name:)
+    model.redis.srem("#{model.model_key}:#{attr_set_name}", index_name)
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), key: String).void }
+  def _del_set(model, key)
+    # Use SPOP here to minimize blocking
+    loop do
+      break unless model.redis.spop(key)
+    end
+
+    model.redis.del(key)
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), key: String).void }
+  def _del_zset(model, key)
+    # ZPOPMIN might not be avaliable on old redis servers
+    model.redis.zscan_each(match: key) do |id, _|
+      model.redis.zrem(key, id)
+    end
+
+    model.redis.del(key)
+  end
+end

data/lib/redcord/migration/ttl.rb

@@ -2,10 +2,15 @@
 module Redcord::Migration::TTL
   extend T::Sig
 
-  # This won't change ttl until we call update on a record
+  sig { params(model: T.class_of(Redcord::Base)).returns(T.untyped) }
+  def _get_ttl(model)
+    model.class_variable_get(:@@ttl) || -1
+  end
+
   sig { params(model: T.class_of(Redcord::Base)).void }
-  def change_ttl_passive(model)
-    ttl = model.class_variable_get(:@@ttl)
-    model.redis.set("#{model.model_key}:ttl", ttl ? ttl : -1)
+  def change_ttl_active(model)
+    model.redis.scan_each_shard("#{model.model_key}:id:*") do |key|
+      model.redis.expire(key, _get_ttl(model))
+    end
   end
 end

data/lib/redcord/railtie.rb

@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+
 # typed: strict
+
 require 'rails'
+require 'yaml'
 
 class Redcord::Railtie < Rails::Railtie
   railtie_name 'redcord'
@@ -13,4 +17,18 @@ class Redcord::Railtie < Rails::Railtie
   config.before_configuration do
     require 'redcord/base'
   end
+
+  config.after_initialize do
+    Redcord::Base.logger = Rails.logger
+
+    config_file = 'config/redcord.yml'
+
+    if File.file?(config_file)
+      Redcord::Base.configurations = YAML.load(
+        ERB.new(File.read(config_file)).result
+      )
+    end
+
+    Redcord._after_initialize!
+  end
 end

data/lib/redcord/redis.rb

@@ -0,0 +1,200 @@
+# typed: true
+require 'digest'
+require 'redis'
+require 'securerandom'
+
+class Redcord::Redis < Redis
+  extend T::Sig
+
+  sig do
+    params(
+      key: T.any(String, Symbol),
+      args: T::Hash[T.untyped, T.untyped],
+      ttl: T.nilable(Integer),
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      custom_index_attrs: T::Hash[Symbol, T::Array],
+      hash_tag: T.nilable(String),
+    ).returns(String)
+  end
+  def create_hash_returning_id(key, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag: nil)
+    id = "#{SecureRandom.uuid}#{hash_tag}"
+    custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
+      result << index_name
+      result << attrs.size
+      result + attrs
+    end
+    run_script(
+      :create_hash,
+      keys: [id, hash_tag],
+      argv: [key, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
+        index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
+    )
+    id
+  end
+
+  sig do
+    params(
+      model: String,
+      id: String,
+      args: T::Hash[T.untyped, T.untyped],
+      ttl: T.nilable(Integer),
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      custom_index_attrs: T::Hash[Symbol, T::Array],
+      hash_tag: T.nilable(String),
+    ).void
+  end
+  def update_hash(model, id, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag:)
+    custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
+      if !(args.keys.to_set & attrs.to_set).empty?
+        result << index_name
+        result << attrs.size
+        result + attrs
+      else
+        result
+      end
+    end
+    run_script(
+      :update_hash,
+      keys: [id, hash_tag],
+      argv: [model, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
+        index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
+    )
+  end
+
+  sig do
+    params(
+      model: String,
+      id: String,
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      custom_index_attrs: T::Hash[Symbol, T::Array],
+    ).returns(Integer)
+  end
+  def delete_hash(model, id, index_attrs:, range_index_attrs:, custom_index_attrs:)
+    custom_index_names = custom_index_attrs.keys
+    run_script(
+      :delete_hash,
+      keys: [id, id.match(/\{.*\}$/)&.send(:[], 0)],
+      argv: [model, index_attrs.size, range_index_attrs.size] + index_attrs + range_index_attrs + custom_index_names,
+    )
+  end
+
+  sig do
+    params(
+      model: String,
+      query_conditions: T::Hash[T.untyped, T.untyped],
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      select_attrs: T::Set[Symbol],
+      custom_index_attrs: T::Array[Symbol],
+      hash_tag: T.nilable(String),
+      custom_index_name: T.nilable(Symbol),
+    ).returns(T::Hash[Integer, T::Hash[T.untyped, T.untyped]])
+  end
+  def find_by_attr(
+        model,
+        query_conditions,
+        select_attrs: Set.new,
+        index_attrs:,
+        range_index_attrs:,
+        custom_index_attrs: Array.new,
+        hash_tag: nil,
+        custom_index_name: nil
+      )
+    conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
+    res = run_script(
+      :find_by_attr,
+      keys: [hash_tag],
+      argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size, conditions.size] + 
+        index_attrs + range_index_attrs + custom_index_attrs + conditions + select_attrs.to_a.flatten
+    )
+    # The Lua script will return this as a flattened array.
+    # Convert the result into a hash of {id -> model hash}
+    res_hash = res.each_slice(2)
+    res_hash.map { |key, val| [key, val.each_slice(2).to_h] }.to_h
+  end
+
+  sig do
+    params(
+      model: String,
+      query_conditions: T::Hash[T.untyped, T.untyped],
+      index_attrs: T::Array[Symbol],
+      range_index_attrs: T::Array[Symbol],
+      custom_index_attrs: T::Array[Symbol],
+      hash_tag: T.nilable(String),
+      custom_index_name: T.nilable(Symbol),
+    ).returns(Integer)
+  end
+  def find_by_attr_count(
+        model,
+        query_conditions,
+        index_attrs:,
+        range_index_attrs:,
+        custom_index_attrs: Array.new,
+        hash_tag: nil,
+        custom_index_name: nil
+      )
+    conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
+    run_script(
+      :find_by_attr_count,
+      keys: [hash_tag],
+      argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size] +
+        index_attrs + range_index_attrs + custom_index_attrs + conditions
+    )
+  end
+
+  def scan_each_shard(key, count: 1000, &blk)
+    clients = instance_variable_get(:@client)
+      &.instance_variable_get(:@node)
+      &.instance_variable_get(:@clients)
+      &.values
+
+    if clients.nil?
+      scan_each(match: key, count: count, &blk)
+    else
+      clients.each do |client|
+        cursor = 0
+        loop do
+          cursor, keys = client.call([:scan, cursor, 'match', key, 'count', count])
+          keys.each(&blk)
+          break if cursor == "0"
+        end
+      end
+    end
+  end
+
+  private
+
+  def run_script(script_name, *args)
+    # Use EVAL when a redis shard has not loaded the script before
+    hash_var_name = :"@script_sha_#{script_name}"
+    hash = instance_variable_get(hash_var_name)
+
+    begin
+      return evalsha(hash, *args) if hash
+    rescue Redis::CommandError => e
+      if e.message != 'NOSCRIPT No matching script. Please use EVAL.'
+        raise e
+      end
+    end
+
+    script_content = Redcord::LuaScriptReader.read_lua_script(script_name.to_s)
+    instance_variable_set(hash_var_name, Digest::SHA1.hexdigest(script_content))
+    self.eval(script_content, *args)
+  end
+
+  # When using custom index: On Lua side script expects query conditions sorted 
+  # in the order of appearance of attributes in specified index
+  sig { params(query_conditions: T::Hash[T.untyped, T.untyped], partial_order: T::Array[Symbol]).returns(T::Array[T.untyped]) }
+  def flatten_with_partial_sort(query_conditions, partial_order)
+    conditions = partial_order.inject([]) do |result, attr|
+      if !query_conditions[attr].nil?
+        result << attr << query_conditions.delete(attr)
+      end
+      result.flatten
+    end
+    conditions += query_conditions.to_a.flatten
+  end
+end

data/lib/redcord/redis_connection.rb

@@ -5,7 +5,7 @@
 require 'rails'
 
 require 'redcord/lua_script_reader'
-require 'redcord/prepared_redis'
+require 'redcord/redis'
 
 module Redcord::RedisConnection
   extend T::Sig
@@ -29,17 +29,17 @@ module Redcord::RedisConnection
       (env_config[name.underscore] || env_config['default']).symbolize_keys
     end
 
-    sig { returns(Redcord::PreparedRedis) }
+    sig { returns(Redcord::Redis) }
     def redis
       Redcord::RedisConnection.connections[name.underscore] ||= prepare_redis!
     end
 
-    sig { returns(Redcord::PreparedRedis) }
+    sig { returns(Redcord::Redis) }
     def establish_connection
       Redcord::RedisConnection.connections[name.underscore] = prepare_redis!
     end
 
-    sig { params(redis: Redis).returns(Redcord::PreparedRedis) }
+    sig { params(redis: Redis).returns(Redcord::Redis) }
     def redis=(redis)
       Redcord::RedisConnection.connections[name.underscore] =
         prepare_redis!(redis)
@@ -50,11 +50,11 @@ module Redcord::RedisConnection
     # definitions in each Redis query.
     #
     # TODO: Replace this with Redcord migrations
-    sig { params(client: T.nilable(Redis)).returns(Redcord::PreparedRedis) }
+    sig { params(client: T.nilable(Redis)).returns(Redcord::Redis) }
     def prepare_redis!(client = nil)
-      return client if client.is_a?(Redcord::PreparedRedis)
+      return client if client.is_a?(Redcord::Redis)
 
-      client = Redcord::PreparedRedis.new(
+      client = Redcord::Redis.new(
         **(
           if client.nil?
             connection_config
@@ -65,23 +65,7 @@ module Redcord::RedisConnection
         logger: Redcord::Logger.proxy,
       )
 
-      client.pipelined do
-        Redcord::RedisConnection.procs_to_prepare.each do |proc_to_prepare|
-          proc_to_prepare.call(client)
-        end
-      end
-
-      script_names = Redcord::ServerScripts.instance_methods
-      res = client.pipelined do
-        script_names.each do |script_name|
-          client.script(
-            :load,
-            Redcord::LuaScriptReader.read_lua_script(script_name.to_s),
-          )
-        end
-      end
-
-      client.redcord_server_script_shas = script_names.zip(res).to_h
+      client.ping
       client
     end
   end
@@ -89,7 +73,7 @@ module Redcord::RedisConnection
   module InstanceMethods
     extend T::Sig
 
-    sig { returns(Redcord::PreparedRedis) }
+    sig { returns(Redcord::Redis) }
     def redis
       self.class.redis
     end
@@ -119,3 +103,10 @@ module Redcord::RedisConnection
 
   mixes_in_class_methods(ClassMethods)
 end
+
+module Redcord
+  sig { void }
+  def self.establish_connections
+    Redcord::Base.descendants.select(&:name).each(&:establish_connection)
+  end
+end