redcord 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7612a04876a98367ab0b8a6d94a0254d0d79aa564b3bd43e5119785f114d962c
4
- data.tar.gz: 9b679659f7087bfd65d202a58be3f0836b1a7e8bc889f16d34e423340ce8c4ed
3
+ metadata.gz: 85772724fda7e2a152acab1f04164ff1ee0146a15f13facebd7b5f7b18323aa9
4
+ data.tar.gz: e1527ca0cfe6e7cc0a0aef6346bdb055cd47e6e6d7ca22aa8488608a02e5eeaa
5
5
  SHA512:
6
- metadata.gz: 44796213c268dfb63e59472c9b59470e3b01dd7a26b0acc63aa48e0f7ff537c25906957d3700b24639eb41d9526708898dc8ff82a4eeea193fc7ec1a3a5cfe43
7
- data.tar.gz: d656f40756c62507113ac660429e9dc10751f40158c8768d47ccdb238ac2d392ccdfc055744ef68837608e7ed33f5673527178a9d0ef77a928d4cabd49da055f
6
+ metadata.gz: cd8b0d84aecba33d41fdad600f00c74a2005c566f439914b3a72f920effc87d344b6974fc18a236d4972fb6d21efa607f43ee121da2d5c70cc6d772a5b80e5c7
7
+ data.tar.gz: c16a9836a3556bbfdf38b04e78e66827733c7e40cfb002ec2744c9748453968571c110dff24f00838b3b2dfa64b658b79c378c214dd82c936df0459c303ab269
@@ -36,3 +36,4 @@ require 'redcord/migration'
36
36
  require 'redcord/migration/migrator'
37
37
  require 'redcord/migration/version'
38
38
  require 'redcord/railtie'
39
+ require 'redcord/vacuum_helper'
@@ -56,9 +56,19 @@ module Redcord::Base
56
56
  # coerced to the specified attribute types. Like ActiveRecord,
57
57
  # Redcord manages the created_at and updated_at fields behind the
58
58
  # scene.
59
- attribute :id, T.nilable(Integer), index: true
60
- attribute :created_at, T.nilable(Time), index: true
61
- attribute :updated_at, T.nilable(Time), index: true
59
+ prop :created_at, T.nilable(Time)
60
+ prop :updated_at, T.nilable(Time)
62
61
  end
63
62
  end
63
+
64
+ sig { returns(T::Array[T.class_of(Redcord::Base)]) }
65
+ def self.descendants
66
+ descendants = []
67
+ # TODO: Use T::Struct instead of Class
68
+ ObjectSpace.each_object(Class) do |klass|
69
+ descendants << klass if klass < self
70
+ end
71
+ descendants
72
+ end
73
+
64
74
  end
@@ -2,11 +2,13 @@
2
2
  class Redcord::Migration
3
3
  end
4
4
 
5
+ require 'redcord/migration/index'
5
6
  require 'redcord/migration/ttl'
6
7
 
7
8
  class Redcord::Migration
8
9
  extend T::Sig
9
10
  extend T::Helpers
11
+ include Redcord::Migration::Index
10
12
  include Redcord::Migration::TTL
11
13
 
12
14
  abstract!
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ # typed: strict
4
+
5
+ module Redcord::Migration::Index
6
+ extend T::Sig
7
+
8
+ sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
9
+ def remove_index(model, index_name)
10
+ if model.redis.sismember("#{model.model_key}:index_attrs", index_name)
11
+ _remove_index_from_attr_set(
12
+ model: model,
13
+ attr_set_name: 'index_attrs',
14
+ index_name: index_name,
15
+ )
16
+
17
+ model.redis.scan_each(match: "#{model.model_key}:#{index_name}:*") { |key| _del_set(model, key) }
18
+ elsif model.redis.sismember("#{model.model_key}:range_index_attrs", index_name)
19
+ _remove_index_from_attr_set(
20
+ model: model,
21
+ attr_set_name: 'range_index_attrs',
22
+ index_name: index_name,
23
+ )
24
+
25
+ attr_set = "#{model.model_key}:#{index_name}"
26
+ nil_attr_set = "#{attr_set}:"
27
+
28
+ _del_set(model, nil_attr_set)
29
+ _del_zset(model, attr_set)
30
+ else
31
+ raise(
32
+ Redcord::AttributeNotIndexed,
33
+ "#{index_name} is not an indexed attribute.",
34
+ )
35
+ end
36
+ end
37
+
38
+ sig {
39
+ params(
40
+ model: T.class_of(Redcord::Base),
41
+ attr_set_name: String,
42
+ index_name: Symbol,
43
+ ).void
44
+ }
45
+ def _remove_index_from_attr_set(model:, attr_set_name:, index_name:)
46
+ model.redis.srem("#{model.model_key}:#{attr_set_name}", index_name)
47
+ end
48
+
49
+ sig { params(model: T.class_of(Redcord::Base), key: String).void }
50
+ def _del_set(model, key)
51
+ # Use SPOP here to minimize blocking
52
+ loop do
53
+ break unless model.redis.spop(key)
54
+ end
55
+
56
+ model.redis.del(key)
57
+ end
58
+
59
+ sig { params(model: T.class_of(Redcord::Base), key: String).void }
60
+ def _del_zset(model, key)
61
+ # ZPOPMIN might not be avaliable on old redis servers
62
+ model.redis.zscan_each(match: key) do |id, _|
63
+ model.redis.zrem(key, id)
64
+ end
65
+
66
+ model.redis.del(key)
67
+ end
68
+ end
@@ -2,10 +2,22 @@
2
2
  module Redcord::Migration::TTL
3
3
  extend T::Sig
4
4
 
5
+ sig { params(model: T.class_of(Redcord::Base)).returns(T.untyped) }
6
+ def _get_ttl(model)
7
+ model.class_variable_get(:@@ttl) || -1
8
+ end
9
+
5
10
  # This won't change ttl until we call update on a record
6
11
  sig { params(model: T.class_of(Redcord::Base)).void }
7
12
  def change_ttl_passive(model)
8
- ttl = model.class_variable_get(:@@ttl)
9
- model.redis.set("#{model.model_key}:ttl", ttl ? ttl : -1)
13
+ model.redis.set("#{model.model_key}:ttl", _get_ttl(model))
14
+ end
15
+
16
+ sig { params(model: T.class_of(Redcord::Base)).void }
17
+ def change_ttl_active(model)
18
+ change_ttl_passive(model)
19
+ model.redis.scan_each(match: "#{model.model_key}:id:*") do |key|
20
+ model.redis.expire(key, _get_ttl(model))
21
+ end
10
22
  end
11
23
  end
@@ -108,3 +108,10 @@ module Redcord::RedisConnection
108
108
 
109
109
  mixes_in_class_methods(ClassMethods)
110
110
  end
111
+
112
+ module Redcord
113
+ sig { void }
114
+ def self.establish_connections
115
+ Redcord::Base.descendants.select(&:name).each(&:establish_connection)
116
+ end
117
+ end
@@ -60,7 +60,6 @@ if #index_attr_keys > 0 then
60
60
  end
61
61
  end
62
62
  local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
63
- attrs_hash['id'] = id
64
63
  if #range_index_attr_keys > 0 then
65
64
  for _, attr_key in ipairs(range_index_attr_keys) do
66
65
  add_id_to_range_index_attr(model, attr_key, attrs_hash[attr_key], id)
@@ -1,6 +1,7 @@
1
1
  # typed: strict
2
2
  require 'redcord/migration/version'
3
3
  require 'redcord/migration/migrator'
4
+ require 'redcord/vacuum_helper'
4
5
 
5
6
  db_namespace = namespace :redis do
6
7
  task migrate: :environment do
@@ -31,4 +32,18 @@ db_namespace = namespace :redis do
31
32
  migration_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
32
33
  puts "\nFinished in #{(migration_end - migration_start).round(3)} seconds"
33
34
  end
35
+
36
+ task :vacuum, [:model_name] => :environment do |t, args|
37
+ desc "Vacuum index attributes for stale ids on a Redcord model"
38
+ $stdout.sync = true
39
+ model_name = args[:model_name]
40
+ puts "Attempting to vacuum the index attributes of the Redcord model: #{model_name}"
41
+ vacuum_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
42
+
43
+ Redcord::VacuumHelper.vacuum(Object.const_get(args[:model_name]))
44
+ vacuum_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
45
+ puts "Finished vacuuming #{model_name} in #{(vacuum_end - vacuum_start).round(3)} seconds"
46
+ rescue NameError => e
47
+ raise StandardError.new("#{args[:model_name]} is not a valid Redcord model.")
48
+ end
34
49
  end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ # typed: strict
4
+
5
+ module Redcord::VacuumHelper
6
+ extend T::Sig
7
+ extend T::Helpers
8
+
9
+ sig { params(model: T.class_of(Redcord::Base)).void }
10
+ def self.vacuum(model)
11
+ model.class_variable_get(:@@index_attributes).each do |index_attr|
12
+ puts "Vacuuming index attribute: #{index_attr}"
13
+ _vacuum_index_attribute(model, index_attr)
14
+ end
15
+ model.class_variable_get(:@@range_index_attributes).each do |range_index_attr|
16
+ puts "Vacuuming range index attribute: #{range_index_attr}"
17
+ _vacuum_range_index_attribute(model, range_index_attr)
18
+ end
19
+ end
20
+
21
+ sig { params(model: T.class_of(Redcord::Base), index_attr: Symbol).void }
22
+ def self._vacuum_index_attribute(model, index_attr)
23
+ # Scan through all index attribute values by matching on Redcord:Model:index_attr:*
24
+ model.redis.scan_each(match: "#{model.model_key}:#{index_attr}:*") do |key|
25
+ _remove_stale_ids_from_set(model, key)
26
+ end
27
+ end
28
+
29
+ sig { params(model: T.class_of(Redcord::Base), range_index_attr: Symbol).void }
30
+ def self._vacuum_range_index_attribute(model, range_index_attr)
31
+ range_index_set_key = "#{model.model_key}:#{range_index_attr}"
32
+ _remove_stale_ids_from_sorted_set(model, range_index_set_key)
33
+
34
+ # Handle nil values for range index attributes, which are stored in a normal
35
+ # set at Redcord:Model:range_index_attr:
36
+ range_index_set_nil_key = "#{range_index_set_key}:"
37
+ _remove_stale_ids_from_set(model, range_index_set_nil_key)
38
+ end
39
+
40
+ sig { params(model: T.class_of(Redcord::Base), set_key: String).void }
41
+ def self._remove_stale_ids_from_set(model, set_key)
42
+ model.redis.sscan_each(set_key) do |id|
43
+ if !model.redis.exists?("#{model.model_key}:id:#{id}")
44
+ model.redis.srem(set_key, id)
45
+ end
46
+ end
47
+ end
48
+
49
+ sig { params(model: T.class_of(Redcord::Base), sorted_set_key: String).void }
50
+ def self._remove_stale_ids_from_sorted_set(model, sorted_set_key)
51
+ model.redis.zscan_each(sorted_set_key) do |id, _|
52
+ if !model.redis.exists?("#{model.model_key}:id:#{id}")
53
+ model.redis.zrem(sorted_set_key, id)
54
+ end
55
+ end
56
+ end
57
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: redcord
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chan Zuckerberg Initiative
@@ -165,6 +165,7 @@ files:
165
165
  - lib/redcord/logger.rb
166
166
  - lib/redcord/lua_script_reader.rb
167
167
  - lib/redcord/migration.rb
168
+ - lib/redcord/migration/index.rb
168
169
  - lib/redcord/migration/migrator.rb
169
170
  - lib/redcord/migration/ttl.rb
170
171
  - lib/redcord/migration/version.rb
@@ -184,6 +185,7 @@ files:
184
185
  - lib/redcord/server_scripts/update_hash.erb.lua
185
186
  - lib/redcord/tasks/redis.rake
186
187
  - lib/redcord/tracer.rb
188
+ - lib/redcord/vacuum_helper.rb
187
189
  homepage: https://github.com/chanzuckerberg/redis-record
188
190
  licenses:
189
191
  - MIT