redcord 0.1.0 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16cb660da0ea843f300ff18cc090ccfed7883c9d1ca48ef0bc5ed8d4e805854c
-  data.tar.gz: f53002cb43b3e5286ba1f914c41ffd6b8a9722d2d33d9764deb7bcba88d2659c
+  metadata.gz: 134041206308ab5d1cb967dca5b5985b2bdd6b9c058601d7040d75d1119a2c98
+  data.tar.gz: d04f590786ab4cd9251ac32f5eea7d11e8103c448916c4aaee290db177370eca
 SHA512:
-  metadata.gz: db443ffbf21066c549b30daed2ad44f836aa4469065b7bd8b6d0c13f45597bd7bdbf6c7ef4202ddf8103a1c2fbcfca28ca60f9f59b90584d0ec718d380d0f908
-  data.tar.gz: 40ded5853307f453fe7c5ab2962e29cc081c05ab7f0c2cfddb898259e207ae9c343dfee6a67967449ff43c227c88db6f7a7922c425d8e6df92466b5ed778c5c2
+  metadata.gz: 374deab010a1f6b40e038adae19b4a7dee5485c5e17ff3a7e8fc92a4e839a2e9835166acd9b6369e7b628e206c1a280e943a28c040172d107c147d3be158fa8b
+  data.tar.gz: 28279ab008fcf6ba226d574250f882f35b05b69fc968794430c3d992c69fd686ec9c2f3fa7821aa6227426c7a027fda07c6e929624a533058c4a396c975f1140

data/lib/redcord/connection_pool.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+require 'connection_pool'
+require_relative 'redis'
+
+class Redcord::ConnectionPool
+  def initialize(pool_size:, timeout:, **client_options)
+    @connection_pool = ::ConnectionPool.new(size: pool_size, timeout: timeout) do
+      # Construct a new client every time the block gets called
+      Redcord::Redis.new(**client_options, logger: Redcord::Logger.proxy)
+    end
+  end
+
+  # Avoid method_missing when possible for better performance
+  methods = Set.new(Redcord::Redis.instance_methods(false) + Redis.instance_methods(false))
+  methods.each do |method_name|
+    define_method method_name do |*args, &blk|
+      @connection_pool.with do |redis|
+        redis.send(method_name, *args, &blk)
+      end
+    end
+  end
+
+  def method_missing(method_name, *args, &blk)
+    @connection_pool.with do |redis|
+      redis.send(method_name, *args, &blk)
+    end
+  end
+end

data/lib/redcord/redis.rb

@@ -18,24 +18,19 @@ class Redcord::Redis < Redis
     ).returns(String)
   end
   def create_hash_returning_id(key, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag: nil)
-    Redcord::Base.trace(
-      'redcord_redis_create_hash_returning_id',
-      model_name: key,
-    ) do
-      id = "#{SecureRandom.uuid}#{hash_tag}"
-      custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
-        result << index_name
-        result << attrs.size
-        result + attrs
-      end
-      run_script(
-        :create_hash,
-        keys: [id, hash_tag],
-        argv: [key, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] + 
-          index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
-      )
-      id
+    id = "#{SecureRandom.uuid}#{hash_tag}"
+    custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
+      result << index_name
+      result << attrs.size
+      result + attrs
     end
+    run_script(
+      :create_hash,
+      keys: [id, hash_tag],
+      argv: [key, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
+        index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
+    )
+    id
   end
 
   sig do
@@ -51,26 +46,21 @@ class Redcord::Redis < Redis
     ).void
   end
   def update_hash(model, id, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag:)
-    Redcord::Base.trace(
-      'redcord_redis_update_hash',
-      model_name: model,
-    ) do
-      custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
-        if !(args.keys.to_set & attrs.to_set).empty?
-          result << index_name
-          result << attrs.size
-          result + attrs
-        else
-          result
-        end
+    custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
+      if !(args.keys.to_set & attrs.to_set).empty?
+        result << index_name
+        result << attrs.size
+        result + attrs
+      else
+        result
       end
-      run_script(
-        :update_hash,
-        keys: [id, hash_tag],
-        argv: [model, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
-          index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
-      )
     end
+    run_script(
+      :update_hash,
+      keys: [id, hash_tag],
+      argv: [model, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
+        index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
+    )
   end
 
   sig do
@@ -83,17 +73,12 @@ class Redcord::Redis < Redis
     ).returns(Integer)
   end
   def delete_hash(model, id, index_attrs:, range_index_attrs:, custom_index_attrs:)
-    Redcord::Base.trace(
-      'redcord_redis_delete_hash',
-      model_name: model,
-    ) do
-      custom_index_names = custom_index_attrs.keys
-      run_script(
-        :delete_hash,
-        keys: [id, id.match(/\{.*\}$/)&.send(:[], 0)],
-        argv: [model, index_attrs.size, range_index_attrs.size] + index_attrs + range_index_attrs + custom_index_names,
-      )
-    end
+    custom_index_names = custom_index_attrs.keys
+    run_script(
+      :delete_hash,
+      keys: [id, id.match(/\{.*\}$/)&.send(:[], 0)],
+      argv: [model, index_attrs.size, range_index_attrs.size] + index_attrs + range_index_attrs + custom_index_names,
+    )
   end
 
   sig do
@@ -118,22 +103,17 @@ class Redcord::Redis < Redis
         hash_tag: nil,
         custom_index_name: nil
       )
-    Redcord::Base.trace(
-      'redcord_redis_find_by_attr',
-      model_name: model,
-    ) do
-      conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
-      res = run_script(
-        :find_by_attr,
-        keys: [hash_tag],
-        argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size, conditions.size] + 
-          index_attrs + range_index_attrs + custom_index_attrs + conditions + select_attrs.to_a.flatten
-      )
-      # The Lua script will return this as a flattened array.
-      # Convert the result into a hash of {id -> model hash}
-      res_hash = res.each_slice(2)
-      res_hash.map { |key, val| [key, val.each_slice(2).to_h] }.to_h
-    end
+    conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
+    res = run_script(
+      :find_by_attr,
+      keys: [hash_tag],
+      argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size, conditions.size] + 
+        index_attrs + range_index_attrs + custom_index_attrs + conditions + select_attrs.to_a.flatten
+    )
+    # The Lua script will return this as a flattened array.
+    # Convert the result into a hash of {id -> model hash}
+    res_hash = res.each_slice(2)
+    res_hash.map { |key, val| [key, val.each_slice(2).to_h] }.to_h
   end
 
   sig do
@@ -156,33 +136,28 @@ class Redcord::Redis < Redis
         hash_tag: nil,
         custom_index_name: nil
       )
-    Redcord::Base.trace(
-      'redcord_redis_find_by_attr_count',
-      model_name: model,
-    ) do
-      conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
-      run_script(
-        :find_by_attr_count,
-        keys: [hash_tag],
-        argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size] +
-          index_attrs + range_index_attrs + custom_index_attrs + conditions
-      )
-    end
+    conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
+    run_script(
+      :find_by_attr_count,
+      keys: [hash_tag],
+      argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size] +
+        index_attrs + range_index_attrs + custom_index_attrs + conditions
+    )
   end
 
-  def scan_each_shard(key, &blk)
+  def scan_each_shard(key, count: 1000, &blk)
     clients = instance_variable_get(:@client)
       &.instance_variable_get(:@node)
       &.instance_variable_get(:@clients)
       &.values
 
     if clients.nil?
-      scan_each(match: key, &blk)
+      scan_each(match: key, count: count, &blk)
     else
       clients.each do |client|
         cursor = 0
         loop do
-          cursor, keys = client.call([:scan, cursor, 'match', key])
+          cursor, keys = client.call([:scan, cursor, 'match', key, 'count', count])
           keys.each(&blk)
           break if cursor == "0"
         end

data/lib/redcord/redis_connection.rb

@@ -6,11 +6,14 @@ require 'rails'
 
 require 'redcord/lua_script_reader'
 require 'redcord/redis'
+require 'redcord/connection_pool'
 
 module Redcord::RedisConnection
   extend T::Sig
   extend T::Helpers
 
+  RedcordClientType = T.type_alias { T.any(Redcord::Redis, Redcord::ConnectionPool) }
+
   @connections = T.let(nil, T.nilable(T::Hash[String, T.untyped]))
   @procs_to_prepare = T.let([], T::Array[Proc])
 
@@ -29,17 +32,17 @@ module Redcord::RedisConnection
       (env_config[name.underscore] || env_config['default']).symbolize_keys
     end
 
-    sig { returns(Redcord::Redis) }
+    sig { returns(RedcordClientType) }
     def redis
       Redcord::RedisConnection.connections[name.underscore] ||= prepare_redis!
     end
 
-    sig { returns(Redcord::Redis) }
+    sig { returns(RedcordClientType) }
     def establish_connection
       Redcord::RedisConnection.connections[name.underscore] = prepare_redis!
     end
 
-    sig { params(redis: Redis).returns(Redcord::Redis) }
+    sig { params(redis: Redis).returns(RedcordClientType) }
     def redis=(redis)
       Redcord::RedisConnection.connections[name.underscore] =
         prepare_redis!(redis)
@@ -50,20 +53,21 @@ module Redcord::RedisConnection
     # definitions in each Redis query.
     #
     # TODO: Replace this with Redcord migrations
-    sig { params(client: T.nilable(Redis)).returns(Redcord::Redis) }
+    sig { params(client: T.nilable(Redis)).returns(RedcordClientType) }
     def prepare_redis!(client = nil)
-      return client if client.is_a?(Redcord::Redis)
-
-      client = Redcord::Redis.new(
-        **(
-          if client.nil?
-            connection_config
-          else
-            client.instance_variable_get(:@options)
-          end
-        ),
-        logger: Redcord::Logger.proxy,
-      )
+      return client if client.is_a?(Redcord::Redis) || client.is_a?(Redcord::ConnectionPool)
+
+      options = client.nil? ? connection_config : client.instance_variable_get(:@options)
+      client =
+        if options[:pool]
+          Redcord::ConnectionPool.new(
+            pool_size: options[:pool],
+            timeout: options[:connection_timeout] || 1.0,
+            **options
+          )
+        else
+          Redcord::Redis.new(**options, logger: Redcord::Logger.proxy)
+        end
 
       client.ping
       client
@@ -73,7 +77,7 @@ module Redcord::RedisConnection
   module InstanceMethods
     extend T::Sig
 
-    sig { returns(Redcord::Redis) }
+    sig { returns(RedcordClientType) }
     def redis
       self.class.redis
     end

data/lib/redcord/relation.rb

@@ -2,6 +2,7 @@
 
 # typed: strict
 
+require 'active_support'
 require 'active_support/core_ext/array'
 require 'active_support/core_ext/module'
 
@@ -88,16 +89,21 @@ class Redcord::Relation
 
   sig { returns(Integer) }
   def count
-    model.validate_index_attributes(query_conditions.keys, custom_index_name: custom_index_name)
-    redis.find_by_attr_count(
-      model.model_key,
-      extract_query_conditions!,
-      index_attrs: model._script_arg_index_attrs,
-      range_index_attrs: model._script_arg_range_index_attrs,
-      custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
-      hash_tag: extract_hash_tag!,
-      custom_index_name: custom_index_name
-    )
+    Redcord::Base.trace(
+     'redcord_relation_count',
+     model_name: model.name,
+    ) do
+      model.validate_index_attributes(query_conditions.keys, custom_index_name: custom_index_name)
+      redis.find_by_attr_count(
+        model.model_key,
+        extract_query_conditions!,
+        index_attrs: model._script_arg_index_attrs,
+        range_index_attrs: model._script_arg_range_index_attrs,
+        custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
+        hash_tag: extract_hash_tag!,
+        custom_index_name: custom_index_name
+      )
+    end
   end
 
   sig { params(index_name: T.nilable(Symbol)).returns(Redcord::Relation) }
@@ -243,7 +249,7 @@ class Redcord::Relation
     end
   end
 
-  sig { returns(Redcord::Redis) }
+  sig { returns(Redcord::RedisConnection::RedcordClientType) }
   def redis
     model.redis
   end

data/lib/redcord/server_scripts/find_by_attr.erb.lua

@@ -46,10 +46,11 @@ local attr_selection_pos = query_cond_pos + ARGV[6]
 
 -- Get all ids which have the corresponding attribute values.
 local ids_set = nil
+local index_sets, range_index_sets = {}, {}
 
 -- If custom index name is empty -> use single-attribute indices
 if index_name == '' then
-  local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(
+  index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(
     KEYS[1],
     model,
     to_set({unpack(ARGV, index_attr_pos, range_attr_pos - 1)}),

data/lib/redcord/vacuum_helper.rb

@@ -9,15 +9,15 @@ module Redcord::VacuumHelper
   sig { params(model: T.class_of(Redcord::Base)).void }
   def self.vacuum(model)
     model.class_variable_get(:@@index_attributes).each do |index_attr|
-      puts "Vacuuming index attribute: #{index_attr}"
+      puts "Vacuuming index attribute: #{index_attr} for model: #{model.name}"
       _vacuum_index_attribute(model, index_attr)
     end
     model.class_variable_get(:@@range_index_attributes).each do |range_index_attr|
-      puts "Vacuuming range index attribute: #{range_index_attr}"
+      puts "Vacuuming range index attribute: #{range_index_attr} for model: #{model.name}"
       _vacuum_range_index_attribute(model, range_index_attr)
     end
     model.class_variable_get(:@@custom_index_attributes).keys.each do |index_name|
-      puts "Vacuuming custom index: #{index_name}"
+      puts "Vacuuming custom index: #{index_name} for model: #{model.name}"
       _vacuum_custom_index(model, index_name)
     end
   end
@@ -32,24 +32,27 @@ module Redcord::VacuumHelper
 
   sig { params(model: T.class_of(Redcord::Base), range_index_attr: Symbol).void }
   def self._vacuum_range_index_attribute(model, range_index_attr)
+    key_suffix = model.shard_by_attribute.nil? ? nil : '{*}'
     range_index_set_key = "#{model.model_key}:#{range_index_attr}"
     range_index_set_nil_key = "#{range_index_set_key}:"
 
     # Handle nil values for range index attributes, which are stored in a normal
     # set at Redcord:Model:range_index_attr:
-    model.redis.scan_each_shard("#{range_index_set_nil_key}*") do |key|
+    model.redis.scan_each_shard("#{range_index_set_nil_key}#{key_suffix}") do |key|
       _remove_stale_ids_from_set(model, key)
     end
 
-    model.redis.scan_each_shard("#{range_index_set_key}*") do |key|
+    model.redis.scan_each_shard("#{range_index_set_key}#{key_suffix}") do |key|
       _remove_stale_ids_from_sorted_set(model, key)
     end
   end
 
   sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
   def self._vacuum_custom_index(model, index_name)
+    key_suffix = model.shard_by_attribute.nil? ? nil : '{*}'
     custom_index_content_key = "#{model.model_key}:custom_index:#{index_name}_content"
-    model.redis.scan_each_shard("#{custom_index_content_key}*") do |key|
+
+    model.redis.scan_each_shard("#{custom_index_content_key}#{key_suffix}") do |key|
       hash_tag = key.split(custom_index_content_key)[1] || ""
       _remove_stale_records_from_custom_index(model, hash_tag, index_name)
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redcord
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.4
 platform: ruby
 authors:
 - Chan Zuckerberg Initiative
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.3
 - !ruby/object:Gem::Dependency
   name: sorbet
   requirement: !ruby/object:Gem::Requirement
@@ -162,6 +176,7 @@ files:
 - lib/redcord/attribute.rb
 - lib/redcord/base.rb
 - lib/redcord/configurations.rb
+- lib/redcord/connection_pool.rb
 - lib/redcord/logger.rb
 - lib/redcord/lua_script_reader.rb
 - lib/redcord/migration.rb
@@ -205,7 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: A Ruby ORM like Active Record, but for Redis