redcord 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7612a04876a98367ab0b8a6d94a0254d0d79aa564b3bd43e5119785f114d962c
-  data.tar.gz: 9b679659f7087bfd65d202a58be3f0836b1a7e8bc889f16d34e423340ce8c4ed
+  metadata.gz: 85772724fda7e2a152acab1f04164ff1ee0146a15f13facebd7b5f7b18323aa9
+  data.tar.gz: e1527ca0cfe6e7cc0a0aef6346bdb055cd47e6e6d7ca22aa8488608a02e5eeaa
 SHA512:
-  metadata.gz: 44796213c268dfb63e59472c9b59470e3b01dd7a26b0acc63aa48e0f7ff537c25906957d3700b24639eb41d9526708898dc8ff82a4eeea193fc7ec1a3a5cfe43
-  data.tar.gz: d656f40756c62507113ac660429e9dc10751f40158c8768d47ccdb238ac2d392ccdfc055744ef68837608e7ed33f5673527178a9d0ef77a928d4cabd49da055f
+  metadata.gz: cd8b0d84aecba33d41fdad600f00c74a2005c566f439914b3a72f920effc87d344b6974fc18a236d4972fb6d21efa607f43ee121da2d5c70cc6d772a5b80e5c7
+  data.tar.gz: c16a9836a3556bbfdf38b04e78e66827733c7e40cfb002ec2744c9748453968571c110dff24f00838b3b2dfa64b658b79c378c214dd82c936df0459c303ab269

data/lib/redcord.rb

@@ -36,3 +36,4 @@ require 'redcord/migration'
 require 'redcord/migration/migrator'
 require 'redcord/migration/version'
 require 'redcord/railtie'
+require 'redcord/vacuum_helper'

data/lib/redcord/base.rb

@@ -56,9 +56,19 @@ module Redcord::Base
       # coerced to the specified attribute types. Like ActiveRecord,
       # Redcord manages the created_at and updated_at fields behind the
       # scene.
-      attribute :id, T.nilable(Integer), index: true
-      attribute :created_at, T.nilable(Time), index: true
-      attribute :updated_at, T.nilable(Time), index: true
+      prop :created_at, T.nilable(Time)
+      prop :updated_at, T.nilable(Time)
     end
   end
+
+  sig { returns(T::Array[T.class_of(Redcord::Base)]) }
+  def self.descendants
+    descendants = []
+    # TODO: Use T::Struct instead of Class
+    ObjectSpace.each_object(Class) do |klass|
+      descendants << klass if klass < self
+    end
+    descendants
+  end
+
 end

data/lib/redcord/migration.rb

@@ -2,11 +2,13 @@
 class Redcord::Migration
 end
 
+require 'redcord/migration/index'
 require 'redcord/migration/ttl'
 
 class Redcord::Migration
   extend T::Sig
   extend T::Helpers
+  include Redcord::Migration::Index
   include Redcord::Migration::TTL
 
   abstract!

data/lib/redcord/migration/index.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# typed: strict
+
+module Redcord::Migration::Index
+  extend T::Sig
+
+  sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
+  def remove_index(model, index_name)
+    if model.redis.sismember("#{model.model_key}:index_attrs", index_name)
+      _remove_index_from_attr_set(
+        model: model,
+        attr_set_name: 'index_attrs',
+        index_name: index_name,
+      )
+
+      model.redis.scan_each(match: "#{model.model_key}:#{index_name}:*") { |key| _del_set(model, key) }
+    elsif model.redis.sismember("#{model.model_key}:range_index_attrs", index_name)
+      _remove_index_from_attr_set(
+        model: model,
+        attr_set_name: 'range_index_attrs',
+        index_name: index_name,
+      )
+
+      attr_set = "#{model.model_key}:#{index_name}"
+      nil_attr_set = "#{attr_set}:"
+
+      _del_set(model, nil_attr_set)
+      _del_zset(model, attr_set)
+    else
+      raise(
+        Redcord::AttributeNotIndexed,
+        "#{index_name} is not an indexed attribute.",
+      )
+    end
+  end
+
+  sig {
+    params(
+      model: T.class_of(Redcord::Base),
+      attr_set_name: String,
+      index_name: Symbol,
+    ).void
+  }
+  def _remove_index_from_attr_set(model:, attr_set_name:, index_name:)
+    model.redis.srem("#{model.model_key}:#{attr_set_name}", index_name)
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), key: String).void }
+  def _del_set(model, key)
+    # Use SPOP here to minimize blocking
+    loop do
+      break unless model.redis.spop(key)
+    end
+
+    model.redis.del(key)
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), key: String).void }
+  def _del_zset(model, key)
+    # ZPOPMIN might not be avaliable on old redis servers
+    model.redis.zscan_each(match: key) do |id, _|
+      model.redis.zrem(key, id)
+    end
+
+    model.redis.del(key)
+  end
+end

data/lib/redcord/migration/ttl.rb

@@ -2,10 +2,22 @@
 module Redcord::Migration::TTL
   extend T::Sig
 
+  sig { params(model: T.class_of(Redcord::Base)).returns(T.untyped) }
+  def _get_ttl(model)
+    model.class_variable_get(:@@ttl) || -1
+  end
+
   # This won't change ttl until we call update on a record
   sig { params(model: T.class_of(Redcord::Base)).void }
   def change_ttl_passive(model)
-    ttl = model.class_variable_get(:@@ttl)
-    model.redis.set("#{model.model_key}:ttl", ttl ? ttl : -1)
+    model.redis.set("#{model.model_key}:ttl", _get_ttl(model))
+  end
+
+  sig { params(model: T.class_of(Redcord::Base)).void }
+  def change_ttl_active(model)
+    change_ttl_passive(model)
+    model.redis.scan_each(match: "#{model.model_key}:id:*") do |key|
+      model.redis.expire(key, _get_ttl(model))
+    end
   end
 end

data/lib/redcord/redis_connection.rb

@@ -108,3 +108,10 @@ module Redcord::RedisConnection
 
   mixes_in_class_methods(ClassMethods)
 end
+
+module Redcord
+  sig { void }
+  def self.establish_connections
+    Redcord::Base.descendants.select(&:name).each(&:establish_connection)
+  end
+end

data/lib/redcord/server_scripts/create_hash_returning_id.erb.lua

@@ -60,7 +60,6 @@ if #index_attr_keys > 0 then
   end
 end
 local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
-attrs_hash['id'] = id
 if #range_index_attr_keys > 0 then
   for _, attr_key in ipairs(range_index_attr_keys) do
     add_id_to_range_index_attr(model, attr_key, attrs_hash[attr_key], id)

data/lib/redcord/tasks/redis.rake

@@ -1,6 +1,7 @@
 # typed: strict
 require 'redcord/migration/version'
 require 'redcord/migration/migrator'
+require 'redcord/vacuum_helper'
 
 db_namespace = namespace :redis do
   task migrate: :environment do
@@ -31,4 +32,18 @@ db_namespace = namespace :redis do
     migration_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     puts "\nFinished in #{(migration_end - migration_start).round(3)} seconds"
   end
+
+  task :vacuum, [:model_name] => :environment do |t, args|
+    desc "Vacuum index attributes for stale ids on a Redcord model"
+    $stdout.sync = true
+    model_name = args[:model_name]
+    puts "Attempting to vacuum the index attributes of the Redcord model: #{model_name}"
+    vacuum_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    Redcord::VacuumHelper.vacuum(Object.const_get(args[:model_name]))
+    vacuum_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    puts "Finished vacuuming #{model_name} in #{(vacuum_end - vacuum_start).round(3)} seconds"
+  rescue NameError => e
+    raise StandardError.new("#{args[:model_name]} is not a valid Redcord model.")
+  end
 end

data/lib/redcord/vacuum_helper.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# typed: strict
+
+module Redcord::VacuumHelper
+  extend T::Sig
+  extend T::Helpers
+
+  sig { params(model: T.class_of(Redcord::Base)).void }
+  def self.vacuum(model)
+    model.class_variable_get(:@@index_attributes).each do |index_attr|
+      puts "Vacuuming index attribute: #{index_attr}"
+      _vacuum_index_attribute(model, index_attr)
+    end
+    model.class_variable_get(:@@range_index_attributes).each do |range_index_attr|
+      puts "Vacuuming range index attribute: #{range_index_attr}"
+      _vacuum_range_index_attribute(model, range_index_attr)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), index_attr: Symbol).void }
+  def self._vacuum_index_attribute(model, index_attr)
+    # Scan through all index attribute values by matching on Redcord:Model:index_attr:*
+    model.redis.scan_each(match: "#{model.model_key}:#{index_attr}:*") do |key|
+      _remove_stale_ids_from_set(model, key)
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), range_index_attr: Symbol).void }
+  def self._vacuum_range_index_attribute(model, range_index_attr)
+    range_index_set_key = "#{model.model_key}:#{range_index_attr}"
+    _remove_stale_ids_from_sorted_set(model, range_index_set_key)
+
+    # Handle nil values for range index attributes, which are stored in a normal
+    # set at Redcord:Model:range_index_attr:
+    range_index_set_nil_key = "#{range_index_set_key}:"
+    _remove_stale_ids_from_set(model, range_index_set_nil_key)
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), set_key: String).void }
+  def self._remove_stale_ids_from_set(model, set_key)
+    model.redis.sscan_each(set_key) do |id|
+      if !model.redis.exists?("#{model.model_key}:id:#{id}")
+        model.redis.srem(set_key, id)
+      end
+    end
+  end
+
+  sig { params(model: T.class_of(Redcord::Base), sorted_set_key: String).void }
+  def self._remove_stale_ids_from_sorted_set(model, sorted_set_key)
+    model.redis.zscan_each(sorted_set_key) do |id, _|
+      if !model.redis.exists?("#{model.model_key}:id:#{id}")
+        model.redis.zrem(sorted_set_key, id)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redcord
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Chan Zuckerberg Initiative
@@ -165,6 +165,7 @@ files:
 - lib/redcord/logger.rb
 - lib/redcord/lua_script_reader.rb
 - lib/redcord/migration.rb
+- lib/redcord/migration/index.rb
 - lib/redcord/migration/migrator.rb
 - lib/redcord/migration/ttl.rb
 - lib/redcord/migration/version.rb
@@ -184,6 +185,7 @@ files:
 - lib/redcord/server_scripts/update_hash.erb.lua
 - lib/redcord/tasks/redis.rake
 - lib/redcord/tracer.rb
+- lib/redcord/vacuum_helper.rb
 homepage: https://github.com/chanzuckerberg/redis-record
 licenses:
 - MIT