redcord 0.0.2.alpha → 0.1.2

data/lib/redcord/server_scripts/update_hash.erb.lua

@@ -16,21 +16,24 @@ nil
 -- accessed by Lua using the ARGV global variable, very similarly to what
 -- happens with keys (so ARGV[1], ARGV[2], ...).
 --
---   KEYS[1] = redcord_instance.class.name
---   KEYS[2] = redcord_instance.id
---   ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
+--   KEYS = redcord_instance.id hash_tag
+--   ARGV = Model.name ttl index_attr_size range_index_attr_size custom_index_attrs_flat_size [index_attr_key ...] [range_index_attr_key ...]
+--          [custom_index_name attrs_size [custom_index_attr_key ...] ...] attr_key attr_val [attr_key attr_val ..]
 <%= include_lua 'shared/lua_helper_methods' %>
 <%= include_lua 'shared/index_helper_methods' %>
 
 if #KEYS ~= 2 then
   error('Expected keys of be of size 2')
 end
-if #ARGV % 2 ~= 0 then
-  error('Expected an even number of arguments')
-end
 
-local model = KEYS[1]
-local id = KEYS[2]
+local model, ttl = unpack(ARGV)
+local id, hash_tag = unpack(KEYS)
+
+local index_attr_pos = 6
+local range_attr_pos = index_attr_pos + ARGV[3]
+local custom_attr_pos = range_attr_pos + ARGV[4]
+-- Starting position of the attr_key-attr_val pairs
+local attr_pos = custom_attr_pos + ARGV[5]
 
 -- key = "#{model}:id:{id}"
 local key = model .. ':id:' .. id
@@ -44,8 +47,8 @@ if redis.call('exists', key) == 0 then
 end
 
 -- Modify the id sets for any indexed attributes
-local attrs_hash = to_hash(ARGV)
-local indexed_attr_keys = redis.call('smembers', model .. ':index_attrs')
+local attrs_hash = to_hash(unpack(ARGV, attr_pos))
+local indexed_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
 if #indexed_attr_keys > 0 then
   -- Get the previous and new values for indexed attributes
   local prev_attrs = redis.call('hmget', key, unpack(indexed_attr_keys))
@@ -53,11 +56,11 @@ if #indexed_attr_keys > 0 then
     local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
     -- Skip attr values not present in the argument hash
     if curr_attr_val then
-      replace_id_in_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+      replace_id_in_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
     end
   end
 end
-local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
+local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}
 if #range_index_attr_keys > 0 then
   -- Get the previous and new values for indexed attributes
   local prev_attrs = redis.call('hmget', key, unpack(range_index_attr_keys))
@@ -65,27 +68,38 @@ if #range_index_attr_keys > 0 then
     local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
     -- Skip attr values not present in the argument hash
     if curr_attr_val then
-      replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+      replace_id_in_range_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
     end
   end
 end
 
 -- Forward the script arguments to the Redis command HSET and update the args.
 -- Call the Redis command: HSET key [field value ...]
-redis.call('hset', key, unpack(ARGV))
-
--- Call the Redis command: GET "#{Model.name}:ttl"
-local ttl = redis.call('get', model .. ':ttl')
+redis.call('hset', key, unpack(ARGV, attr_pos))
 
-if ttl then
-  if ttl == '-1' then
-    -- Persist the object if the ttl is set to -1
-    redis.call('persist', key)
-  else
-    -- Reset the TTL for this object. We do this manually becaues altering the
-    -- field value of a hash with HSET, etc. will leave the TTL
-    -- untouched: https://redis.io/commands/expire
-    redis.call('expire', key, ttl)
+-- Update custom indexes
+local updated_hash = to_hash(unpack(redis.call('hgetall', key)))
+local custom_index_attr_keys = {unpack(ARGV, custom_attr_pos, attr_pos - 1)}
+local i = 1
+while i < #custom_index_attr_keys do
+  local index_name, attrs_num = custom_index_attr_keys[i], custom_index_attr_keys[i+1]
+  local attr_values = {}
+  for j, attr_key in ipairs({unpack(custom_index_attr_keys, i + 2, i + attrs_num + 1)}) do
+    attr_values[j] = updated_hash[attr_key]
   end
+  delete_record_from_custom_index(hash_tag, model, index_name, id)
+  add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
+  i = i + 2 + attrs_num
+end
+
+-- Call the Redis command: GET "#{Model.name}:ttl"
+if ttl == '-1' then
+  -- Persist the object if the ttl is set to -1
+  redis.call('persist', key)
+else
+  -- Reset the TTL for this object. We do this manually becaues altering the
+  -- field value of a hash with HSET, etc. will leave the TTL
+  -- untouched: https://redis.io/commands/expire
+  redis.call('expire', key, ttl)
 end
 return nil

data/lib/redcord/tasks/redis.rake

@@ -1,6 +1,7 @@
 # typed: strict
 require 'redcord/migration/version'
 require 'redcord/migration/migrator'
+require 'redcord/vacuum_helper'
 
 db_namespace = namespace :redis do
   task migrate: :environment do
@@ -31,4 +32,18 @@ db_namespace = namespace :redis do
     migration_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     puts "\nFinished in #{(migration_end - migration_start).round(3)} seconds"
   end
+
+  task :vacuum, [:model_name] => :environment do |t, args|
+    desc "Vacuum index attributes for stale ids on a Redcord model"
+    $stdout.sync = true
+    model_name = args[:model_name]
+    puts "Attempting to vacuum the index attributes of the Redcord model: #{model_name}"
+    vacuum_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    Redcord::VacuumHelper.vacuum(Object.const_get(args[:model_name]))
+    vacuum_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    puts "Finished vacuuming #{model_name} in #{(vacuum_end - vacuum_start).round(3)} seconds"
+  rescue NameError => e
+    raise StandardError.new("#{args[:model_name]} is not a valid Redcord model.")
+  end
 end

data/lib/redcord/tracer.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# typed: strict
+
+module Redcord::Tracer
+  extend T::Sig
+  extend T::Helpers
+
+  sig { params(klass: Module).void }
+  def self.included(klass)
+    klass.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    include Kernel
+
+    extend T::Sig
+
+    @@tracer = T.let(nil, T.untyped)
+
+    sig {
+      params(
+        span_name: String,
+        model_name: String,
+        tags: T::Array[String],
+        blk: T.proc.returns(T.untyped),
+      ).returns(T.untyped)
+    }
+    def trace(span_name, model_name:, tags: [], &blk)
+      return blk.call if @@tracer.nil?
+
+      @@tracer.call.trace(
+        span_name,
+        resource: model_name,
+        service: 'redcord',
+        tags: tags,
+        &blk
+      )
+    end
+
+    sig { params(blk: T.proc.returns(T.untyped)).void }
+    def tracer(&blk)
+      @@tracer = blk
+    end
+  end
+
+  mixes_in_class_methods(ClassMethods)
+end

data/lib/redcord/vacuum_helper.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# typed: strict
+
+module Redcord::VacuumHelper
+  extend T::Sig
+  extend T::Helpers
+
+  sig { params(model: T.class_of(Redcord::Base)).void }
+  def self.vacuum(model)
+    model.class_variable_get(:@@index_attributes).each do |index_attr|
+      puts "Vacuuming index attribute: #{index_attr}"
+      _vacuum_index_attribute(model, index_attr)
+    end
+    model.class_variable_get(:@@range_index_attributes).each do |range_index_attr|
+      puts "Vacuuming range index attribute: #{range_index_attr}"
+      _vacuum_range_index_attribute(model, range_index_attr)
+    end
+    model.class_variable_get(:@@custom_index_attributes).keys.each do |index_name|
+      puts "Vacuuming custom index: #{index_name}"
+      _vacuum_custom_index(model, index_name)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), index_attr: Symbol).void }
+  def self._vacuum_index_attribute(model, index_attr)
+    # Scan through all index attribute values by matching on Redcord:Model:index_attr:*
+    model.redis.scan_each_shard("#{model.model_key}:#{index_attr}:*") do |key|
+      _remove_stale_ids_from_set(model, key)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), range_index_attr: Symbol).void }
+  def self._vacuum_range_index_attribute(model, range_index_attr)
+    key_suffix = model.shard_by_attribute.nil? ? nil : '{*}'
+    range_index_set_key = "#{model.model_key}:#{range_index_attr}"
+    range_index_set_nil_key = "#{range_index_set_key}:"
+
+    # Handle nil values for range index attributes, which are stored in a normal
+    # set at Redcord:Model:range_index_attr:
+    model.redis.scan_each_shard("#{range_index_set_nil_key}#{key_suffix}") do |key|
+      _remove_stale_ids_from_set(model, key)
+    end
+
+    model.redis.scan_each_shard("#{range_index_set_key}#{key_suffix}") do |key|
+      _remove_stale_ids_from_sorted_set(model, key)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
+  def self._vacuum_custom_index(model, index_name)
+    key_suffix = model.shard_by_attribute.nil? ? nil : '{*}'
+    custom_index_content_key = "#{model.model_key}:custom_index:#{index_name}_content"
+
+    model.redis.scan_each_shard("#{custom_index_content_key}#{key_suffix}") do |key|
+      hash_tag = key.split(custom_index_content_key)[1] || ""
+      _remove_stale_records_from_custom_index(model, hash_tag, index_name)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), set_key: String).void }
+  def self._remove_stale_ids_from_set(model, set_key)
+    model.redis.sscan_each(set_key) do |id|
+      if !model.redis.exists?("#{model.model_key}:id:#{id}")
+        model.redis.srem(set_key, id)
+      end
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), sorted_set_key: String).void }
+  def self._remove_stale_ids_from_sorted_set(model, sorted_set_key)
+    model.redis.zscan_each(sorted_set_key) do |id, _|
+      if !model.redis.exists?("#{model.model_key}:id:#{id}")
+        model.redis.zrem(sorted_set_key, id)
+      end
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), hash_tag: String, index_name: Symbol).void }
+  def self._remove_stale_records_from_custom_index(model, hash_tag, index_name)
+    index_key = "#{model.model_key}:custom_index:#{index_name}#{hash_tag}"
+    index_content_key = "#{model.model_key}:custom_index:#{index_name}_content#{hash_tag}"
+    model.redis.hscan_each(index_content_key).each do |id, index_string|
+      if !model.redis.exists?("#{model.model_key}:id:#{id}")
+        model.redis.hdel(index_content_key, id)
+        model.redis.zremrangebylex(index_key, "[#{index_string}", "[#{index_string}")
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redcord
 version: !ruby/object:Gem::Version
-  version: 0.0.2.alpha
+  version: 0.1.2
 platform: ruby
 authors:
 - Chan Zuckerberg Initiative
@@ -165,17 +165,17 @@ files:
 - lib/redcord/logger.rb
 - lib/redcord/lua_script_reader.rb
 - lib/redcord/migration.rb
+- lib/redcord/migration/index.rb
 - lib/redcord/migration/migrator.rb
 - lib/redcord/migration/ttl.rb
 - lib/redcord/migration/version.rb
-- lib/redcord/prepared_redis.rb
 - lib/redcord/railtie.rb
 - lib/redcord/range_interval.rb
+- lib/redcord/redis.rb
 - lib/redcord/redis_connection.rb
 - lib/redcord/relation.rb
 - lib/redcord/serializer.rb
-- lib/redcord/server_scripts.rb
-- lib/redcord/server_scripts/create_hash_returning_id.erb.lua
+- lib/redcord/server_scripts/create_hash.erb.lua
 - lib/redcord/server_scripts/delete_hash.erb.lua
 - lib/redcord/server_scripts/find_by_attr.erb.lua
 - lib/redcord/server_scripts/find_by_attr_count.erb.lua
@@ -184,6 +184,8 @@ files:
 - lib/redcord/server_scripts/shared/query_helper_methods.erb.lua
 - lib/redcord/server_scripts/update_hash.erb.lua
 - lib/redcord/tasks/redis.rake
+- lib/redcord/tracer.rb
+- lib/redcord/vacuum_helper.rb
 homepage: https://github.com/chanzuckerberg/redis-record
 licenses:
 - MIT
@@ -199,12 +201,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: A Ruby ORM like Active Record, but for Redis

data/lib/redcord/prepared_redis.rb

@@ -1,18 +0,0 @@
-# typed: strict
-require 'redis'
-require 'redcord/server_scripts'
-
-class Redcord::PreparedRedis < Redis
-  extend T::Sig
-  include Redcord::ServerScripts
-
-  sig { returns(T::Hash[Symbol, String]) }
-  def redcord_server_script_shas
-    instance_variable_get(:@_redcord_server_script_shas)
-  end
-
-  sig { params(shas: T::Hash[Symbol, String]).void }
-  def redcord_server_script_shas=(shas)
-    instance_variable_set(:@_redcord_server_script_shas, shas)
-  end
-end

data/lib/redcord/server_scripts.rb

@@ -1,78 +0,0 @@
-# typed: strict
-module Redcord::ServerScripts
-  extend T::Sig
-
-  sig do
-    params(
-      key: T.any(String, Symbol),
-      args: T::Hash[T.untyped, T.untyped],
-    ).returns(Integer)
-  end
-  def create_hash_returning_id(key, args)
-    evalsha(
-      T.must(redcord_server_script_shas[:create_hash_returning_id]),
-      keys: [key],
-      argv: args.to_a.flatten,
-    ).to_i
-  end
-
-  sig do
-    params(
-      model: String,
-      id: Integer,
-      args: T::Hash[T.untyped, T.untyped],
-    ).void
-  end
-  def update_hash(model, id, args)
-    evalsha(
-      T.must(redcord_server_script_shas[:update_hash]),
-      keys: [model, id],
-      argv: args.to_a.flatten,
-    )
-  end
-
-  sig do
-    params(
-      model: String,
-      id: Integer
-    ).returns(Integer)
-  end
-  def delete_hash(model, id)
-    evalsha(
-      T.must(redcord_server_script_shas[:delete_hash]),
-      keys: [model, id]
-    )
-  end
-
-  sig do
-    params(
-      model: String,
-      query_conditions: T::Hash[T.untyped, T.untyped],
-      select_attrs: T::Set[Symbol]
-    ).returns(T::Hash[Integer, T::Hash[T.untyped, T.untyped]])
-  end
-  def find_by_attr(model, query_conditions, select_attrs=Set.new)
-    res = evalsha(
-      T.must(redcord_server_script_shas[:find_by_attr]),
-      keys: [model] + query_conditions.to_a.flatten,
-      argv: select_attrs.to_a.flatten
-    )
-    # The Lua script will return this as a flattened array.
-    # Convert the result into a hash of {id -> model hash}
-    res_hash = res.each_slice(2)
-    res_hash.map { |key, val| [key.to_i, val.each_slice(2).to_h] }.to_h
-  end
-
-  sig do
-    params(
-      model: String,
-      query_conditions: T::Hash[T.untyped, T.untyped]
-    ).returns(Integer)
-  end
-  def find_by_attr_count(model, query_conditions)
-    evalsha(
-      T.must(redcord_server_script_shas[:find_by_attr_count]),
-      keys: [model] + query_conditions.to_a.flatten,
-    )
-  end
-end

data/lib/redcord/server_scripts/create_hash_returning_id.erb.lua

@@ -1,69 +0,0 @@
---[[
-EVALSHA SHA1(__FILE__) [field value ...]
-> Time complexity: O(N) where N is the number of fields being set.
-
-Create a hash with the specified fields to their respective values stored at
-key when key does not exist.
-
-# Return value
-The id of the created hash as a string.
---]]
-
--- The arguments can be accessed by Lua using the KEYS global variable in the
--- form of a one-based array (so KEYS[1], KEYS[2], ...).
--- All the additional arguments should not represent key names and can be
--- accessed by Lua using the ARGV global variable, very similarly to what
--- happens with keys (so ARGV[1], ARGV[2], ...).
-
---   KEYS[1] = Model.name
---   ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
-<%= include_lua 'shared/lua_helper_methods' %>
-<%= include_lua 'shared/index_helper_methods' %>
-
--- Validate input to script before making Redis db calls
-if #KEYS ~= 1 then
-  error('Expected keys to be of size 1')
-end
-if #ARGV % 2 ~= 0 then
-  error('Expected an even number of arguments')
-end
-
-local model = KEYS[1]
-
--- Call the Redis command: INCR "#{Model.name}:id_seq". If "#{Model.name}:id_seq" does
--- not exist, the command returns 0. It errors if the id_seq overflows a 64 bit
--- signed integer.
-redis.call('incr', model .. ':id_seq')
-
--- The Lua version used by Redis does not support 64 bit integers:
---   https://github.com/antirez/redis/issues/5261
--- We ignore the integer response from INCR and use the string response from
--- the GET/MGET command.
-local id, ttl = unpack(redis.call('mget', model .. ':id_seq', model .. ':ttl'))
-local key = model .. ':id:' .. id
-
--- Forward the script arguments to the Redis command HSET.
--- Call the Redis command: HSET "#{Model.name}:id:#{id}" field value ...
-redis.call('hset', key, unpack(ARGV))
-
--- Set TTL on key
-if ttl and ttl ~= '-1' then
-  redis.call('expire', key, ttl)
-end
-
--- Add id value for any index and range index attributes
-local attrs_hash = to_hash(ARGV)
-local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
-if #index_attr_keys > 0 then
-  for _, attr_key in ipairs(index_attr_keys) do
-    add_id_to_index_attr(model, attr_key, attrs_hash[attr_key], id)
-  end
-end
-local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
-attrs_hash['id'] = id
-if #range_index_attr_keys > 0 then
-  for _, attr_key in ipairs(range_index_attr_keys) do
-    add_id_to_range_index_attr(model, attr_key, attrs_hash[attr_key], id)
-  end
-end
-return id