redcord 0.0.4 → 0.1.0

data/lib/redcord/server_scripts/update_hash.erb.lua

@@ -16,21 +16,24 @@ nil
 -- accessed by Lua using the ARGV global variable, very similarly to what
 -- happens with keys (so ARGV[1], ARGV[2], ...).
 --
---   KEYS[1] = redcord_instance.class.name
---   KEYS[2] = redcord_instance.id
---   ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
+--   KEYS = redcord_instance.id hash_tag
+--   ARGV = Model.name ttl index_attr_size range_index_attr_size custom_index_attrs_flat_size [index_attr_key ...] [range_index_attr_key ...]
+--          [custom_index_name attrs_size [custom_index_attr_key ...] ...] attr_key attr_val [attr_key attr_val ..]
 <%= include_lua 'shared/lua_helper_methods' %>
 <%= include_lua 'shared/index_helper_methods' %>
 
 if #KEYS ~= 2 then
   error('Expected keys of be of size 2')
 end
-if #ARGV % 2 ~= 0 then
-  error('Expected an even number of arguments')
-end
 
-local model = KEYS[1]
-local id = KEYS[2]
+local model, ttl = unpack(ARGV)
+local id, hash_tag = unpack(KEYS)
+
+local index_attr_pos = 6
+local range_attr_pos = index_attr_pos + ARGV[3]
+local custom_attr_pos = range_attr_pos + ARGV[4]
+-- Starting position of the attr_key-attr_val pairs
+local attr_pos = custom_attr_pos + ARGV[5]
 
 -- key = "#{model}:id:{id}"
 local key = model .. ':id:' .. id
@@ -44,8 +47,8 @@ if redis.call('exists', key) == 0 then
 end
 
 -- Modify the id sets for any indexed attributes
-local attrs_hash = to_hash(ARGV)
-local indexed_attr_keys = redis.call('smembers', model .. ':index_attrs')
+local attrs_hash = to_hash(unpack(ARGV, attr_pos))
+local indexed_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
 if #indexed_attr_keys > 0 then
   -- Get the previous and new values for indexed attributes
   local prev_attrs = redis.call('hmget', key, unpack(indexed_attr_keys))
@@ -53,11 +56,11 @@ if #indexed_attr_keys > 0 then
     local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
     -- Skip attr values not present in the argument hash
     if curr_attr_val then
-      replace_id_in_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+      replace_id_in_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
     end
   end
 end
-local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
+local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}
 if #range_index_attr_keys > 0 then
   -- Get the previous and new values for indexed attributes
   local prev_attrs = redis.call('hmget', key, unpack(range_index_attr_keys))
@@ -65,27 +68,38 @@ if #range_index_attr_keys > 0 then
     local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
     -- Skip attr values not present in the argument hash
     if curr_attr_val then
-      replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+      replace_id_in_range_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
     end
   end
 end
 
 -- Forward the script arguments to the Redis command HSET and update the args.
 -- Call the Redis command: HSET key [field value ...]
-redis.call('hset', key, unpack(ARGV))
-
--- Call the Redis command: GET "#{Model.name}:ttl"
-local ttl = redis.call('get', model .. ':ttl')
+redis.call('hset', key, unpack(ARGV, attr_pos))
 
-if ttl then
-  if ttl == '-1' then
-    -- Persist the object if the ttl is set to -1
-    redis.call('persist', key)
-  else
-    -- Reset the TTL for this object. We do this manually becaues altering the
-    -- field value of a hash with HSET, etc. will leave the TTL
-    -- untouched: https://redis.io/commands/expire
-    redis.call('expire', key, ttl)
+-- Update custom indexes
+local updated_hash = to_hash(unpack(redis.call('hgetall', key)))
+local custom_index_attr_keys = {unpack(ARGV, custom_attr_pos, attr_pos - 1)}
+local i = 1
+while i < #custom_index_attr_keys do
+  local index_name, attrs_num = custom_index_attr_keys[i], custom_index_attr_keys[i+1]
+  local attr_values = {}
+  for j, attr_key in ipairs({unpack(custom_index_attr_keys, i + 2, i + attrs_num + 1)}) do
+    attr_values[j] = updated_hash[attr_key]
   end
+  delete_record_from_custom_index(hash_tag, model, index_name, id)
+  add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
+  i = i + 2 + attrs_num
+end
+
+-- Call the Redis command: GET "#{Model.name}:ttl"
+if ttl == '-1' then
+  -- Persist the object if the ttl is set to -1
+  redis.call('persist', key)
+else
+  -- Reset the TTL for this object. We do this manually becaues altering the
+  -- field value of a hash with HSET, etc. will leave the TTL
+  -- untouched: https://redis.io/commands/expire
+  redis.call('expire', key, ttl)
 end
 return nil

data/lib/redcord/vacuum_helper.rb

@@ -16,12 +16,16 @@ module Redcord::VacuumHelper
       puts "Vacuuming range index attribute: #{range_index_attr}"
       _vacuum_range_index_attribute(model, range_index_attr)
     end
+    model.class_variable_get(:@@custom_index_attributes).keys.each do |index_name|
+      puts "Vacuuming custom index: #{index_name}"
+      _vacuum_custom_index(model, index_name)
+    end
   end
 
   sig { params(model: T.class_of(Redcord::Base), index_attr: Symbol).void }
   def self._vacuum_index_attribute(model, index_attr)
     # Scan through all index attribute values by matching on Redcord:Model:index_attr:*
-    model.redis.scan_each(match: "#{model.model_key}:#{index_attr}:*") do |key|
+    model.redis.scan_each_shard("#{model.model_key}:#{index_attr}:*") do |key|
       _remove_stale_ids_from_set(model, key)
     end
   end
@@ -29,12 +33,26 @@ module Redcord::VacuumHelper
   sig { params(model: T.class_of(Redcord::Base), range_index_attr: Symbol).void }
   def self._vacuum_range_index_attribute(model, range_index_attr)
     range_index_set_key = "#{model.model_key}:#{range_index_attr}"
-    _remove_stale_ids_from_sorted_set(model, range_index_set_key)
+    range_index_set_nil_key = "#{range_index_set_key}:"
 
     # Handle nil values for range index attributes, which are stored in a normal
     # set at Redcord:Model:range_index_attr:
-    range_index_set_nil_key = "#{range_index_set_key}:"
-    _remove_stale_ids_from_set(model, range_index_set_nil_key)
+    model.redis.scan_each_shard("#{range_index_set_nil_key}*") do |key|
+      _remove_stale_ids_from_set(model, key)
+    end
+
+    model.redis.scan_each_shard("#{range_index_set_key}*") do |key|
+      _remove_stale_ids_from_sorted_set(model, key)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
+  def self._vacuum_custom_index(model, index_name)
+    custom_index_content_key = "#{model.model_key}:custom_index:#{index_name}_content"
+    model.redis.scan_each_shard("#{custom_index_content_key}*") do |key|
+      hash_tag = key.split(custom_index_content_key)[1] || ""
+      _remove_stale_records_from_custom_index(model, hash_tag, index_name)
+    end
   end
 
   sig { params(model: T.class_of(Redcord::Base), set_key: String).void }
@@ -54,4 +72,16 @@ module Redcord::VacuumHelper
       end
     end
   end
+
+  sig { params(model: T.class_of(Redcord::Base), hash_tag: String, index_name: Symbol).void }
+  def self._remove_stale_records_from_custom_index(model, hash_tag, index_name)
+    index_key = "#{model.model_key}:custom_index:#{index_name}#{hash_tag}"
+    index_content_key = "#{model.model_key}:custom_index:#{index_name}_content#{hash_tag}"
+    model.redis.hscan_each(index_content_key).each do |id, index_string|
+      if !model.redis.exists?("#{model.model_key}:id:#{id}")
+        model.redis.hdel(index_content_key, id)
+        model.redis.zremrangebylex(index_key, "[#{index_string}", "[#{index_string}")
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redcord
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - Chan Zuckerberg Initiative
@@ -169,13 +169,13 @@ files:
 - lib/redcord/migration/migrator.rb
 - lib/redcord/migration/ttl.rb
 - lib/redcord/migration/version.rb
-- lib/redcord/prepared_redis.rb
 - lib/redcord/railtie.rb
 - lib/redcord/range_interval.rb
+- lib/redcord/redis.rb
 - lib/redcord/redis_connection.rb
 - lib/redcord/relation.rb
 - lib/redcord/serializer.rb
-- lib/redcord/server_scripts/create_hash_returning_id.erb.lua
+- lib/redcord/server_scripts/create_hash.erb.lua
 - lib/redcord/server_scripts/delete_hash.erb.lua
 - lib/redcord/server_scripts/find_by_attr.erb.lua
 - lib/redcord/server_scripts/find_by_attr_count.erb.lua
@@ -205,8 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: A Ruby ORM like Active Record, but for Redis

data/lib/redcord/prepared_redis.rb

@@ -1,147 +0,0 @@
-# typed: strict
-require 'redis'
-
-# TODO: Rename Redcord::PreparedRedis -> Redcord::Redis
-class Redcord::PreparedRedis < Redis
-  extend T::Sig
-
-  sig do
-    params(
-      key: T.any(String, Symbol),
-      args: T::Hash[T.untyped, T.untyped],
-    ).returns(Integer)
-  end
-  def create_hash_returning_id(key, args)
-    Redcord::Base.trace(
-      'redcord_redis_create_hash_returning_id',
-      model_name: key,
-    ) do
-      evalsha(
-        self.class.server_script_shas[:create_hash_returning_id],
-        keys: [key],
-        argv: args.to_a.flatten,
-      ).to_i
-    end
-  end
-
-  sig do
-    params(
-      model: String,
-      id: Integer,
-      args: T::Hash[T.untyped, T.untyped],
-    ).void
-  end
-  def update_hash(model, id, args)
-    Redcord::Base.trace(
-      'redcord_redis_update_hash',
-      model_name: model,
-    ) do
-      evalsha(
-        self.class.server_script_shas[:update_hash],
-        keys: [model, id],
-        argv: args.to_a.flatten,
-      )
-    end
-  end
-
-  sig do
-    params(
-      model: String,
-      id: Integer
-    ).returns(Integer)
-  end
-  def delete_hash(model, id)
-    Redcord::Base.trace(
-      'redcord_redis_delete_hash',
-      model_name: model,
-    ) do
-      evalsha(
-        self.class.server_script_shas[:delete_hash],
-        keys: [model, id]
-      )
-    end
-  end
-
-  sig do
-    params(
-      model: String,
-      query_conditions: T::Hash[T.untyped, T.untyped],
-      select_attrs: T::Set[Symbol]
-    ).returns(T::Hash[Integer, T::Hash[T.untyped, T.untyped]])
-  end
-  def find_by_attr(model, query_conditions, select_attrs=Set.new)
-    Redcord::Base.trace(
-      'redcord_redis_find_by_attr',
-      model_name: model,
-    ) do
-      res = evalsha(
-        self.class.server_script_shas[:find_by_attr],
-        keys: [model] + query_conditions.to_a.flatten,
-        argv: select_attrs.to_a.flatten
-      )
-      # The Lua script will return this as a flattened array.
-      # Convert the result into a hash of {id -> model hash}
-      res_hash = res.each_slice(2)
-      res_hash.map { |key, val| [key.to_i, val.each_slice(2).to_h] }.to_h
-    end
-  end
-
-  sig do
-    params(
-      model: String,
-      query_conditions: T::Hash[T.untyped, T.untyped]
-    ).returns(Integer)
-  end
-  def find_by_attr_count(model, query_conditions)
-    Redcord::Base.trace(
-      'redcord_redis_find_by_attr_count',
-      model_name: model,
-    ) do
-      evalsha(
-        self.class.server_script_shas[:find_by_attr_count],
-        keys: [model] + query_conditions.to_a.flatten,
-      )
-    end
-  end
-
-  sig { void }
-  def load_server_scripts!
-    script_names = Dir[File.join(
-      __dir__,
-      'server_scripts/*.lua',
-    )].map do |filename|
-      # lib/redcord/server_scripts/find_by_attr.erb.lua -> find_by_attr
-      T.must(filename.split('/').last).split('.').first&.to_sym
-    end
-
-    res = pipelined do
-      script_names.each do |script_name|
-        script(
-          :load,
-          Redcord::LuaScriptReader.read_lua_script(script_name.to_s),
-        )
-      end
-    end
-
-    if self.class.class_variable_get(:@@server_script_shas).nil?
-      self.class.class_variable_set(
-        :@@server_script_shas,
-        script_names.zip(res).to_h
-      )
-    end
-  end
-
-  @@server_script_shas = T.let(nil, T.nilable(T::Hash[Symbol, String]))
-
-  sig { returns(T::Hash[Symbol, String]) }
-  def self.server_script_shas
-    T.must(@@server_script_shas)
-  end
-
-  sig { void }
-  def self.load_server_scripts!
-    Redcord::Base.configurations[Rails.env].each do |_, config|
-      new(**(config.symbolize_keys)).load_server_scripts!
-    end
-  end
-end

data/lib/redcord/server_scripts/create_hash_returning_id.erb.lua

@@ -1,68 +0,0 @@
---[[
-EVALSHA SHA1(__FILE__) [field value ...]
-> Time complexity: O(N) where N is the number of fields being set.
-
-Create a hash with the specified fields to their respective values stored at
-key when key does not exist.
-
-# Return value
-The id of the created hash as a string.
---]]
-
--- The arguments can be accessed by Lua using the KEYS global variable in the
--- form of a one-based array (so KEYS[1], KEYS[2], ...).
--- All the additional arguments should not represent key names and can be
--- accessed by Lua using the ARGV global variable, very similarly to what
--- happens with keys (so ARGV[1], ARGV[2], ...).
-
---   KEYS[1] = Model.name
---   ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
-<%= include_lua 'shared/lua_helper_methods' %>
-<%= include_lua 'shared/index_helper_methods' %>
-
--- Validate input to script before making Redis db calls
-if #KEYS ~= 1 then
-  error('Expected keys to be of size 1')
-end
-if #ARGV % 2 ~= 0 then
-  error('Expected an even number of arguments')
-end
-
-local model = KEYS[1]
-
--- Call the Redis command: INCR "#{Model.name}:id_seq". If "#{Model.name}:id_seq" does
--- not exist, the command returns 0. It errors if the id_seq overflows a 64 bit
--- signed integer.
-redis.call('incr', model .. ':id_seq')
-
--- The Lua version used by Redis does not support 64 bit integers:
---   https://github.com/antirez/redis/issues/5261
--- We ignore the integer response from INCR and use the string response from
--- the GET/MGET command.
-local id, ttl = unpack(redis.call('mget', model .. ':id_seq', model .. ':ttl'))
-local key = model .. ':id:' .. id
-
--- Forward the script arguments to the Redis command HSET.
--- Call the Redis command: HSET "#{Model.name}:id:#{id}" field value ...
-redis.call('hset', key, unpack(ARGV))
-
--- Set TTL on key
-if ttl and ttl ~= '-1' then
-  redis.call('expire', key, ttl)
-end
-
--- Add id value for any index and range index attributes
-local attrs_hash = to_hash(ARGV)
-local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
-if #index_attr_keys > 0 then
-  for _, attr_key in ipairs(index_attr_keys) do
-    add_id_to_index_attr(model, attr_key, attrs_hash[attr_key], id)
-  end
-end
-local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
-if #range_index_attr_keys > 0 then
-  for _, attr_key in ipairs(range_index_attr_keys) do
-    add_id_to_range_index_attr(model, attr_key, attrs_hash[attr_key], id)
-  end
-end
-return id