redcord 0.0.2.alpha → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -16,21 +16,24 @@ nil
16
16
  -- accessed by Lua using the ARGV global variable, very similarly to what
17
17
  -- happens with keys (so ARGV[1], ARGV[2], ...).
18
18
  --
19
- -- KEYS[1] = redcord_instance.class.name
20
- -- KEYS[2] = redcord_instance.id
21
- -- ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
19
+ -- KEYS = redcord_instance.id hash_tag
20
+ -- ARGV = Model.name ttl index_attr_size range_index_attr_size custom_index_attrs_flat_size [index_attr_key ...] [range_index_attr_key ...]
21
+ -- [custom_index_name attrs_size [custom_index_attr_key ...] ...] attr_key attr_val [attr_key attr_val ..]
22
22
  <%= include_lua 'shared/lua_helper_methods' %>
23
23
  <%= include_lua 'shared/index_helper_methods' %>
24
24
 
25
25
  if #KEYS ~= 2 then
26
26
  error('Expected keys of be of size 2')
27
27
  end
28
- if #ARGV % 2 ~= 0 then
29
- error('Expected an even number of arguments')
30
- end
31
28
 
32
- local model = KEYS[1]
33
- local id = KEYS[2]
29
+ local model, ttl = unpack(ARGV)
30
+ local id, hash_tag = unpack(KEYS)
31
+
32
+ local index_attr_pos = 6
33
+ local range_attr_pos = index_attr_pos + ARGV[3]
34
+ local custom_attr_pos = range_attr_pos + ARGV[4]
35
+ -- Starting position of the attr_key-attr_val pairs
36
+ local attr_pos = custom_attr_pos + ARGV[5]
34
37
 
35
38
  -- key = "#{model}:id:{id}"
36
39
  local key = model .. ':id:' .. id
@@ -44,8 +47,8 @@ if redis.call('exists', key) == 0 then
44
47
  end
45
48
 
46
49
  -- Modify the id sets for any indexed attributes
47
- local attrs_hash = to_hash(ARGV)
48
- local indexed_attr_keys = redis.call('smembers', model .. ':index_attrs')
50
+ local attrs_hash = to_hash(unpack(ARGV, attr_pos))
51
+ local indexed_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
49
52
  if #indexed_attr_keys > 0 then
50
53
  -- Get the previous and new values for indexed attributes
51
54
  local prev_attrs = redis.call('hmget', key, unpack(indexed_attr_keys))
@@ -53,11 +56,11 @@ if #indexed_attr_keys > 0 then
53
56
  local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
54
57
  -- Skip attr values not present in the argument hash
55
58
  if curr_attr_val then
56
- replace_id_in_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
59
+ replace_id_in_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
57
60
  end
58
61
  end
59
62
  end
60
- local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
63
+ local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}
61
64
  if #range_index_attr_keys > 0 then
62
65
  -- Get the previous and new values for indexed attributes
63
66
  local prev_attrs = redis.call('hmget', key, unpack(range_index_attr_keys))
@@ -65,27 +68,38 @@ if #range_index_attr_keys > 0 then
65
68
  local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
66
69
  -- Skip attr values not present in the argument hash
67
70
  if curr_attr_val then
68
- replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
71
+ replace_id_in_range_index_attr(hash_tag, model, attr_key, prev_attr_val, curr_attr_val, id)
69
72
  end
70
73
  end
71
74
  end
72
75
 
73
76
  -- Forward the script arguments to the Redis command HSET and update the args.
74
77
  -- Call the Redis command: HSET key [field value ...]
75
- redis.call('hset', key, unpack(ARGV))
76
-
77
- -- Call the Redis command: GET "#{Model.name}:ttl"
78
- local ttl = redis.call('get', model .. ':ttl')
78
+ redis.call('hset', key, unpack(ARGV, attr_pos))
79
79
 
80
- if ttl then
81
- if ttl == '-1' then
82
- -- Persist the object if the ttl is set to -1
83
- redis.call('persist', key)
84
- else
85
- -- Reset the TTL for this object. We do this manually becaues altering the
86
- -- field value of a hash with HSET, etc. will leave the TTL
87
- -- untouched: https://redis.io/commands/expire
88
- redis.call('expire', key, ttl)
80
+ -- Update custom indexes
81
+ local updated_hash = to_hash(unpack(redis.call('hgetall', key)))
82
+ local custom_index_attr_keys = {unpack(ARGV, custom_attr_pos, attr_pos - 1)}
83
+ local i = 1
84
+ while i < #custom_index_attr_keys do
85
+ local index_name, attrs_num = custom_index_attr_keys[i], custom_index_attr_keys[i+1]
86
+ local attr_values = {}
87
+ for j, attr_key in ipairs({unpack(custom_index_attr_keys, i + 2, i + attrs_num + 1)}) do
88
+ attr_values[j] = updated_hash[attr_key]
89
89
  end
90
+ delete_record_from_custom_index(hash_tag, model, index_name, id)
91
+ add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
92
+ i = i + 2 + attrs_num
93
+ end
94
+
95
+ -- Call the Redis command: GET "#{Model.name}:ttl"
96
+ if ttl == '-1' then
97
+ -- Persist the object if the ttl is set to -1
98
+ redis.call('persist', key)
99
+ else
100
+ -- Reset the TTL for this object. We do this manually becaues altering the
101
+ -- field value of a hash with HSET, etc. will leave the TTL
102
+ -- untouched: https://redis.io/commands/expire
103
+ redis.call('expire', key, ttl)
90
104
  end
91
105
  return nil
@@ -1,6 +1,7 @@
1
1
  # typed: strict
2
2
  require 'redcord/migration/version'
3
3
  require 'redcord/migration/migrator'
4
+ require 'redcord/vacuum_helper'
4
5
 
5
6
  db_namespace = namespace :redis do
6
7
  task migrate: :environment do
@@ -31,4 +32,18 @@ db_namespace = namespace :redis do
31
32
  migration_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
32
33
  puts "\nFinished in #{(migration_end - migration_start).round(3)} seconds"
33
34
  end
35
+
36
+ task :vacuum, [:model_name] => :environment do |t, args|
37
+ desc "Vacuum index attributes for stale ids on a Redcord model"
38
+ $stdout.sync = true
39
+ model_name = args[:model_name]
40
+ puts "Attempting to vacuum the index attributes of the Redcord model: #{model_name}"
41
+ vacuum_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
42
+
43
+ Redcord::VacuumHelper.vacuum(Object.const_get(args[:model_name]))
44
+ vacuum_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
45
+ puts "Finished vacuuming #{model_name} in #{(vacuum_end - vacuum_start).round(3)} seconds"
46
+ rescue NameError => e
47
+ raise StandardError.new("#{args[:model_name]} is not a valid Redcord model.")
48
+ end
34
49
  end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ # typed: strict
4
+
5
+ module Redcord::Tracer
6
+ extend T::Sig
7
+ extend T::Helpers
8
+
9
+ sig { params(klass: Module).void }
10
+ def self.included(klass)
11
+ klass.extend(ClassMethods)
12
+ end
13
+
14
+ module ClassMethods
15
+ include Kernel
16
+
17
+ extend T::Sig
18
+
19
+ @@tracer = T.let(nil, T.untyped)
20
+
21
+ sig {
22
+ params(
23
+ span_name: String,
24
+ model_name: String,
25
+ tags: T::Array[String],
26
+ blk: T.proc.returns(T.untyped),
27
+ ).returns(T.untyped)
28
+ }
29
+ def trace(span_name, model_name:, tags: [], &blk)
30
+ return blk.call if @@tracer.nil?
31
+
32
+ @@tracer.call.trace(
33
+ span_name,
34
+ resource: model_name,
35
+ service: 'redcord',
36
+ tags: tags,
37
+ &blk
38
+ )
39
+ end
40
+
41
+ sig { params(blk: T.proc.returns(T.untyped)).void }
42
+ def tracer(&blk)
43
+ @@tracer = blk
44
+ end
45
+ end
46
+
47
+ mixes_in_class_methods(ClassMethods)
48
+ end
@@ -0,0 +1,90 @@
1
+ # frozen_string_literal: true
2
+
3
+ # typed: strict
4
+
5
+ module Redcord::VacuumHelper
6
+ extend T::Sig
7
+ extend T::Helpers
8
+
9
+ sig { params(model: T.class_of(Redcord::Base)).void }
10
+ def self.vacuum(model)
11
+ model.class_variable_get(:@@index_attributes).each do |index_attr|
12
+ puts "Vacuuming index attribute: #{index_attr}"
13
+ _vacuum_index_attribute(model, index_attr)
14
+ end
15
+ model.class_variable_get(:@@range_index_attributes).each do |range_index_attr|
16
+ puts "Vacuuming range index attribute: #{range_index_attr}"
17
+ _vacuum_range_index_attribute(model, range_index_attr)
18
+ end
19
+ model.class_variable_get(:@@custom_index_attributes).keys.each do |index_name|
20
+ puts "Vacuuming custom index: #{index_name}"
21
+ _vacuum_custom_index(model, index_name)
22
+ end
23
+ end
24
+
25
+ sig { params(model: T.class_of(Redcord::Base), index_attr: Symbol).void }
26
+ def self._vacuum_index_attribute(model, index_attr)
27
+ # Scan through all index attribute values by matching on Redcord:Model:index_attr:*
28
+ model.redis.scan_each_shard("#{model.model_key}:#{index_attr}:*") do |key|
29
+ _remove_stale_ids_from_set(model, key)
30
+ end
31
+ end
32
+
33
+ sig { params(model: T.class_of(Redcord::Base), range_index_attr: Symbol).void }
34
+ def self._vacuum_range_index_attribute(model, range_index_attr)
35
+ key_suffix = model.shard_by_attribute.nil? ? nil : '{*}'
36
+ range_index_set_key = "#{model.model_key}:#{range_index_attr}"
37
+ range_index_set_nil_key = "#{range_index_set_key}:"
38
+
39
+ # Handle nil values for range index attributes, which are stored in a normal
40
+ # set at Redcord:Model:range_index_attr:
41
+ model.redis.scan_each_shard("#{range_index_set_nil_key}#{key_suffix}") do |key|
42
+ _remove_stale_ids_from_set(model, key)
43
+ end
44
+
45
+ model.redis.scan_each_shard("#{range_index_set_key}#{key_suffix}") do |key|
46
+ _remove_stale_ids_from_sorted_set(model, key)
47
+ end
48
+ end
49
+
50
+ sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
51
+ def self._vacuum_custom_index(model, index_name)
52
+ key_suffix = model.shard_by_attribute.nil? ? nil : '{*}'
53
+ custom_index_content_key = "#{model.model_key}:custom_index:#{index_name}_content"
54
+
55
+ model.redis.scan_each_shard("#{custom_index_content_key}#{key_suffix}") do |key|
56
+ hash_tag = key.split(custom_index_content_key)[1] || ""
57
+ _remove_stale_records_from_custom_index(model, hash_tag, index_name)
58
+ end
59
+ end
60
+
61
+ sig { params(model: T.class_of(Redcord::Base), set_key: String).void }
62
+ def self._remove_stale_ids_from_set(model, set_key)
63
+ model.redis.sscan_each(set_key) do |id|
64
+ if !model.redis.exists?("#{model.model_key}:id:#{id}")
65
+ model.redis.srem(set_key, id)
66
+ end
67
+ end
68
+ end
69
+
70
+ sig { params(model: T.class_of(Redcord::Base), sorted_set_key: String).void }
71
+ def self._remove_stale_ids_from_sorted_set(model, sorted_set_key)
72
+ model.redis.zscan_each(sorted_set_key) do |id, _|
73
+ if !model.redis.exists?("#{model.model_key}:id:#{id}")
74
+ model.redis.zrem(sorted_set_key, id)
75
+ end
76
+ end
77
+ end
78
+
79
+ sig { params(model: T.class_of(Redcord::Base), hash_tag: String, index_name: Symbol).void }
80
+ def self._remove_stale_records_from_custom_index(model, hash_tag, index_name)
81
+ index_key = "#{model.model_key}:custom_index:#{index_name}#{hash_tag}"
82
+ index_content_key = "#{model.model_key}:custom_index:#{index_name}_content#{hash_tag}"
83
+ model.redis.hscan_each(index_content_key).each do |id, index_string|
84
+ if !model.redis.exists?("#{model.model_key}:id:#{id}")
85
+ model.redis.hdel(index_content_key, id)
86
+ model.redis.zremrangebylex(index_key, "[#{index_string}", "[#{index_string}")
87
+ end
88
+ end
89
+ end
90
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: redcord
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2.alpha
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chan Zuckerberg Initiative
@@ -165,17 +165,17 @@ files:
165
165
  - lib/redcord/logger.rb
166
166
  - lib/redcord/lua_script_reader.rb
167
167
  - lib/redcord/migration.rb
168
+ - lib/redcord/migration/index.rb
168
169
  - lib/redcord/migration/migrator.rb
169
170
  - lib/redcord/migration/ttl.rb
170
171
  - lib/redcord/migration/version.rb
171
- - lib/redcord/prepared_redis.rb
172
172
  - lib/redcord/railtie.rb
173
173
  - lib/redcord/range_interval.rb
174
+ - lib/redcord/redis.rb
174
175
  - lib/redcord/redis_connection.rb
175
176
  - lib/redcord/relation.rb
176
177
  - lib/redcord/serializer.rb
177
- - lib/redcord/server_scripts.rb
178
- - lib/redcord/server_scripts/create_hash_returning_id.erb.lua
178
+ - lib/redcord/server_scripts/create_hash.erb.lua
179
179
  - lib/redcord/server_scripts/delete_hash.erb.lua
180
180
  - lib/redcord/server_scripts/find_by_attr.erb.lua
181
181
  - lib/redcord/server_scripts/find_by_attr_count.erb.lua
@@ -184,6 +184,8 @@ files:
184
184
  - lib/redcord/server_scripts/shared/query_helper_methods.erb.lua
185
185
  - lib/redcord/server_scripts/update_hash.erb.lua
186
186
  - lib/redcord/tasks/redis.rake
187
+ - lib/redcord/tracer.rb
188
+ - lib/redcord/vacuum_helper.rb
187
189
  homepage: https://github.com/chanzuckerberg/redis-record
188
190
  licenses:
189
191
  - MIT
@@ -199,12 +201,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
199
201
  version: 2.5.0
200
202
  required_rubygems_version: !ruby/object:Gem::Requirement
201
203
  requirements:
202
- - - ">"
204
+ - - ">="
203
205
  - !ruby/object:Gem::Version
204
- version: 1.3.1
206
+ version: '0'
205
207
  requirements: []
206
- rubyforge_project:
207
- rubygems_version: 2.7.6.2
208
+ rubygems_version: 3.0.8
208
209
  signing_key:
209
210
  specification_version: 4
210
211
  summary: A Ruby ORM like Active Record, but for Redis
@@ -1,18 +0,0 @@
1
- # typed: strict
2
- require 'redis'
3
- require 'redcord/server_scripts'
4
-
5
- class Redcord::PreparedRedis < Redis
6
- extend T::Sig
7
- include Redcord::ServerScripts
8
-
9
- sig { returns(T::Hash[Symbol, String]) }
10
- def redcord_server_script_shas
11
- instance_variable_get(:@_redcord_server_script_shas)
12
- end
13
-
14
- sig { params(shas: T::Hash[Symbol, String]).void }
15
- def redcord_server_script_shas=(shas)
16
- instance_variable_set(:@_redcord_server_script_shas, shas)
17
- end
18
- end
@@ -1,78 +0,0 @@
1
- # typed: strict
2
- module Redcord::ServerScripts
3
- extend T::Sig
4
-
5
- sig do
6
- params(
7
- key: T.any(String, Symbol),
8
- args: T::Hash[T.untyped, T.untyped],
9
- ).returns(Integer)
10
- end
11
- def create_hash_returning_id(key, args)
12
- evalsha(
13
- T.must(redcord_server_script_shas[:create_hash_returning_id]),
14
- keys: [key],
15
- argv: args.to_a.flatten,
16
- ).to_i
17
- end
18
-
19
- sig do
20
- params(
21
- model: String,
22
- id: Integer,
23
- args: T::Hash[T.untyped, T.untyped],
24
- ).void
25
- end
26
- def update_hash(model, id, args)
27
- evalsha(
28
- T.must(redcord_server_script_shas[:update_hash]),
29
- keys: [model, id],
30
- argv: args.to_a.flatten,
31
- )
32
- end
33
-
34
- sig do
35
- params(
36
- model: String,
37
- id: Integer
38
- ).returns(Integer)
39
- end
40
- def delete_hash(model, id)
41
- evalsha(
42
- T.must(redcord_server_script_shas[:delete_hash]),
43
- keys: [model, id]
44
- )
45
- end
46
-
47
- sig do
48
- params(
49
- model: String,
50
- query_conditions: T::Hash[T.untyped, T.untyped],
51
- select_attrs: T::Set[Symbol]
52
- ).returns(T::Hash[Integer, T::Hash[T.untyped, T.untyped]])
53
- end
54
- def find_by_attr(model, query_conditions, select_attrs=Set.new)
55
- res = evalsha(
56
- T.must(redcord_server_script_shas[:find_by_attr]),
57
- keys: [model] + query_conditions.to_a.flatten,
58
- argv: select_attrs.to_a.flatten
59
- )
60
- # The Lua script will return this as a flattened array.
61
- # Convert the result into a hash of {id -> model hash}
62
- res_hash = res.each_slice(2)
63
- res_hash.map { |key, val| [key.to_i, val.each_slice(2).to_h] }.to_h
64
- end
65
-
66
- sig do
67
- params(
68
- model: String,
69
- query_conditions: T::Hash[T.untyped, T.untyped]
70
- ).returns(Integer)
71
- end
72
- def find_by_attr_count(model, query_conditions)
73
- evalsha(
74
- T.must(redcord_server_script_shas[:find_by_attr_count]),
75
- keys: [model] + query_conditions.to_a.flatten,
76
- )
77
- end
78
- end
@@ -1,69 +0,0 @@
1
- --[[
2
- EVALSHA SHA1(__FILE__) [field value ...]
3
- > Time complexity: O(N) where N is the number of fields being set.
4
-
5
- Create a hash with the specified fields to their respective values stored at
6
- key when key does not exist.
7
-
8
- # Return value
9
- The id of the created hash as a string.
10
- --]]
11
-
12
- -- The arguments can be accessed by Lua using the KEYS global variable in the
13
- -- form of a one-based array (so KEYS[1], KEYS[2], ...).
14
- -- All the additional arguments should not represent key names and can be
15
- -- accessed by Lua using the ARGV global variable, very similarly to what
16
- -- happens with keys (so ARGV[1], ARGV[2], ...).
17
-
18
- -- KEYS[1] = Model.name
19
- -- ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
20
- <%= include_lua 'shared/lua_helper_methods' %>
21
- <%= include_lua 'shared/index_helper_methods' %>
22
-
23
- -- Validate input to script before making Redis db calls
24
- if #KEYS ~= 1 then
25
- error('Expected keys to be of size 1')
26
- end
27
- if #ARGV % 2 ~= 0 then
28
- error('Expected an even number of arguments')
29
- end
30
-
31
- local model = KEYS[1]
32
-
33
- -- Call the Redis command: INCR "#{Model.name}:id_seq". If "#{Model.name}:id_seq" does
34
- -- not exist, the command returns 0. It errors if the id_seq overflows a 64 bit
35
- -- signed integer.
36
- redis.call('incr', model .. ':id_seq')
37
-
38
- -- The Lua version used by Redis does not support 64 bit integers:
39
- -- https://github.com/antirez/redis/issues/5261
40
- -- We ignore the integer response from INCR and use the string response from
41
- -- the GET/MGET command.
42
- local id, ttl = unpack(redis.call('mget', model .. ':id_seq', model .. ':ttl'))
43
- local key = model .. ':id:' .. id
44
-
45
- -- Forward the script arguments to the Redis command HSET.
46
- -- Call the Redis command: HSET "#{Model.name}:id:#{id}" field value ...
47
- redis.call('hset', key, unpack(ARGV))
48
-
49
- -- Set TTL on key
50
- if ttl and ttl ~= '-1' then
51
- redis.call('expire', key, ttl)
52
- end
53
-
54
- -- Add id value for any index and range index attributes
55
- local attrs_hash = to_hash(ARGV)
56
- local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
57
- if #index_attr_keys > 0 then
58
- for _, attr_key in ipairs(index_attr_keys) do
59
- add_id_to_index_attr(model, attr_key, attrs_hash[attr_key], id)
60
- end
61
- end
62
- local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
63
- attrs_hash['id'] = id
64
- if #range_index_attr_keys > 0 then
65
- for _, attr_key in ipairs(range_index_attr_keys) do
66
- add_id_to_range_index_attr(model, attr_key, attrs_hash[attr_key], id)
67
- end
68
- end
69
- return id