redcord 0.0.2.alpha → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/redcord.rb +30 -2
- data/lib/redcord.rbi +0 -16
- data/lib/redcord/actions.rb +152 -45
- data/lib/redcord/attribute.rb +110 -13
- data/lib/redcord/base.rb +17 -3
- data/lib/redcord/configurations.rb +4 -0
- data/lib/redcord/logger.rb +1 -1
- data/lib/redcord/migration.rb +2 -0
- data/lib/redcord/migration/index.rb +57 -0
- data/lib/redcord/migration/ttl.rb +9 -4
- data/lib/redcord/railtie.rb +18 -0
- data/lib/redcord/redis.rb +200 -0
- data/lib/redcord/redis_connection.rb +16 -25
- data/lib/redcord/relation.rb +141 -33
- data/lib/redcord/serializer.rb +84 -33
- data/lib/redcord/server_scripts/create_hash.erb.lua +81 -0
- data/lib/redcord/server_scripts/delete_hash.erb.lua +17 -8
- data/lib/redcord/server_scripts/find_by_attr.erb.lua +51 -16
- data/lib/redcord/server_scripts/find_by_attr_count.erb.lua +45 -14
- data/lib/redcord/server_scripts/shared/index_helper_methods.erb.lua +45 -16
- data/lib/redcord/server_scripts/shared/lua_helper_methods.erb.lua +20 -4
- data/lib/redcord/server_scripts/shared/query_helper_methods.erb.lua +81 -14
- data/lib/redcord/server_scripts/update_hash.erb.lua +40 -26
- data/lib/redcord/tasks/redis.rake +15 -0
- data/lib/redcord/tracer.rb +48 -0
- data/lib/redcord/vacuum_helper.rb +90 -0
- metadata +9 -8
- data/lib/redcord/prepared_redis.rb +0 -18
- data/lib/redcord/server_scripts.rb +0 -78
- data/lib/redcord/server_scripts/create_hash_returning_id.erb.lua +0 -69
data/lib/redcord/base.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
#
|
1
3
|
# typed: strict
|
2
4
|
#
|
3
5
|
# A Redis ORM API inspired by ActiveRecord:
|
@@ -11,6 +13,7 @@ require 'redcord/configurations'
|
|
11
13
|
require 'redcord/logger'
|
12
14
|
require 'redcord/redis_connection'
|
13
15
|
require 'redcord/serializer'
|
16
|
+
require 'redcord/tracer'
|
14
17
|
|
15
18
|
module Redcord::Base
|
16
19
|
extend T::Sig
|
@@ -23,6 +26,7 @@ module Redcord::Base
|
|
23
26
|
include Redcord::Configurations
|
24
27
|
include Redcord::Logger
|
25
28
|
include Redcord::RedisConnection
|
29
|
+
include Redcord::Tracer
|
26
30
|
|
27
31
|
abstract!
|
28
32
|
|
@@ -52,9 +56,19 @@ module Redcord::Base
|
|
52
56
|
# coerced to the specified attribute types. Like ActiveRecord,
|
53
57
|
# Redcord manages the created_at and updated_at fields behind the
|
54
58
|
# scene.
|
55
|
-
|
56
|
-
|
57
|
-
attribute :updated_at, T.nilable(Time), index: true
|
59
|
+
prop :created_at, T.nilable(Time)
|
60
|
+
prop :updated_at, T.nilable(Time)
|
58
61
|
end
|
59
62
|
end
|
63
|
+
|
64
|
+
sig { returns(T::Array[T.class_of(Redcord::Base)]) }
|
65
|
+
def self.descendants
|
66
|
+
descendants = []
|
67
|
+
# TODO: Use T::Struct instead of Class
|
68
|
+
ObjectSpace.each_object(Class) do |klass|
|
69
|
+
descendants << klass if klass < self
|
70
|
+
end
|
71
|
+
descendants
|
72
|
+
end
|
73
|
+
|
60
74
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
#
|
1
3
|
# typed: strict
|
2
4
|
#
|
3
5
|
# This allows us to configure Redis connections for Redcord. Redis
|
@@ -40,6 +42,8 @@
|
|
40
42
|
# ```
|
41
43
|
#
|
42
44
|
require 'redcord/redis_connection'
|
45
|
+
require 'redcord/tracer'
|
46
|
+
|
43
47
|
module Redcord::Configurations
|
44
48
|
extend T::Sig
|
45
49
|
extend T::Helpers
|
data/lib/redcord/logger.rb
CHANGED
data/lib/redcord/migration.rb
CHANGED
@@ -2,11 +2,13 @@
|
|
2
2
|
class Redcord::Migration
|
3
3
|
end
|
4
4
|
|
5
|
+
require 'redcord/migration/index'
|
5
6
|
require 'redcord/migration/ttl'
|
6
7
|
|
7
8
|
class Redcord::Migration
|
8
9
|
extend T::Sig
|
9
10
|
extend T::Helpers
|
11
|
+
include Redcord::Migration::Index
|
10
12
|
include Redcord::Migration::TTL
|
11
13
|
|
12
14
|
abstract!
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# typed: strict
|
4
|
+
|
5
|
+
module Redcord::Migration::Index
|
6
|
+
extend T::Sig
|
7
|
+
|
8
|
+
sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
|
9
|
+
def remove_index(model, index_name)
|
10
|
+
model.redis.scan_each_shard("#{model.model_key}:#{index_name}:*") { |key| _del_set(model, key) }
|
11
|
+
|
12
|
+
attr_set = "#{model.model_key}:#{index_name}"
|
13
|
+
nil_attr_set = "#{attr_set}:"
|
14
|
+
|
15
|
+
model.redis.scan_each_shard("#{nil_attr_set}*") { |key| _del_set(model, key) }
|
16
|
+
model.redis.scan_each_shard("#{attr_set}*") { |key| _del_zset(model, key) }
|
17
|
+
end
|
18
|
+
|
19
|
+
sig { params(model: T.class_of(Redcord::Base), index_name: Symbol).void }
|
20
|
+
def remove_custom_index(model, index_name)
|
21
|
+
index_key = "#{model.model_key}:custom_index:#{index_name}"
|
22
|
+
index_content_key = "#{model.model_key}:custom_index:#{index_name}_content"
|
23
|
+
model.redis.scan_each_shard("#{index_key}*") { |key| model.redis.unlink(key) }
|
24
|
+
model.redis.scan_each_shard("#{index_content_key}*") { |key| model.redis.unlink(key) }
|
25
|
+
end
|
26
|
+
|
27
|
+
sig {
|
28
|
+
params(
|
29
|
+
model: T.class_of(Redcord::Base),
|
30
|
+
attr_set_name: String,
|
31
|
+
index_name: Symbol,
|
32
|
+
).void
|
33
|
+
}
|
34
|
+
def _remove_index_from_attr_set(model:, attr_set_name:, index_name:)
|
35
|
+
model.redis.srem("#{model.model_key}:#{attr_set_name}", index_name)
|
36
|
+
end
|
37
|
+
|
38
|
+
sig { params(model: T.class_of(Redcord::Base), key: String).void }
|
39
|
+
def _del_set(model, key)
|
40
|
+
# Use SPOP here to minimize blocking
|
41
|
+
loop do
|
42
|
+
break unless model.redis.spop(key)
|
43
|
+
end
|
44
|
+
|
45
|
+
model.redis.del(key)
|
46
|
+
end
|
47
|
+
|
48
|
+
sig { params(model: T.class_of(Redcord::Base), key: String).void }
|
49
|
+
def _del_zset(model, key)
|
50
|
+
# ZPOPMIN might not be avaliable on old redis servers
|
51
|
+
model.redis.zscan_each(match: key) do |id, _|
|
52
|
+
model.redis.zrem(key, id)
|
53
|
+
end
|
54
|
+
|
55
|
+
model.redis.del(key)
|
56
|
+
end
|
57
|
+
end
|
@@ -2,10 +2,15 @@
|
|
2
2
|
module Redcord::Migration::TTL
|
3
3
|
extend T::Sig
|
4
4
|
|
5
|
-
|
5
|
+
sig { params(model: T.class_of(Redcord::Base)).returns(T.untyped) }
|
6
|
+
def _get_ttl(model)
|
7
|
+
model.class_variable_get(:@@ttl) || -1
|
8
|
+
end
|
9
|
+
|
6
10
|
sig { params(model: T.class_of(Redcord::Base)).void }
|
7
|
-
def
|
8
|
-
|
9
|
-
|
11
|
+
def change_ttl_active(model)
|
12
|
+
model.redis.scan_each_shard("#{model.model_key}:id:*") do |key|
|
13
|
+
model.redis.expire(key, _get_ttl(model))
|
14
|
+
end
|
10
15
|
end
|
11
16
|
end
|
data/lib/redcord/railtie.rb
CHANGED
@@ -1,5 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# typed: strict
|
4
|
+
|
2
5
|
require 'rails'
|
6
|
+
require 'yaml'
|
3
7
|
|
4
8
|
class Redcord::Railtie < Rails::Railtie
|
5
9
|
railtie_name 'redcord'
|
@@ -13,4 +17,18 @@ class Redcord::Railtie < Rails::Railtie
|
|
13
17
|
config.before_configuration do
|
14
18
|
require 'redcord/base'
|
15
19
|
end
|
20
|
+
|
21
|
+
config.after_initialize do
|
22
|
+
Redcord::Base.logger = Rails.logger
|
23
|
+
|
24
|
+
config_file = 'config/redcord.yml'
|
25
|
+
|
26
|
+
if File.file?(config_file)
|
27
|
+
Redcord::Base.configurations = YAML.load(
|
28
|
+
ERB.new(File.read(config_file)).result
|
29
|
+
)
|
30
|
+
end
|
31
|
+
|
32
|
+
Redcord._after_initialize!
|
33
|
+
end
|
16
34
|
end
|
@@ -0,0 +1,200 @@
|
|
1
|
+
# typed: true
|
2
|
+
require 'digest'
|
3
|
+
require 'redis'
|
4
|
+
require 'securerandom'
|
5
|
+
|
6
|
+
class Redcord::Redis < Redis
|
7
|
+
extend T::Sig
|
8
|
+
|
9
|
+
sig do
|
10
|
+
params(
|
11
|
+
key: T.any(String, Symbol),
|
12
|
+
args: T::Hash[T.untyped, T.untyped],
|
13
|
+
ttl: T.nilable(Integer),
|
14
|
+
index_attrs: T::Array[Symbol],
|
15
|
+
range_index_attrs: T::Array[Symbol],
|
16
|
+
custom_index_attrs: T::Hash[Symbol, T::Array],
|
17
|
+
hash_tag: T.nilable(String),
|
18
|
+
).returns(String)
|
19
|
+
end
|
20
|
+
def create_hash_returning_id(key, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag: nil)
|
21
|
+
id = "#{SecureRandom.uuid}#{hash_tag}"
|
22
|
+
custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
|
23
|
+
result << index_name
|
24
|
+
result << attrs.size
|
25
|
+
result + attrs
|
26
|
+
end
|
27
|
+
run_script(
|
28
|
+
:create_hash,
|
29
|
+
keys: [id, hash_tag],
|
30
|
+
argv: [key, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
|
31
|
+
index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
|
32
|
+
)
|
33
|
+
id
|
34
|
+
end
|
35
|
+
|
36
|
+
sig do
|
37
|
+
params(
|
38
|
+
model: String,
|
39
|
+
id: String,
|
40
|
+
args: T::Hash[T.untyped, T.untyped],
|
41
|
+
ttl: T.nilable(Integer),
|
42
|
+
index_attrs: T::Array[Symbol],
|
43
|
+
range_index_attrs: T::Array[Symbol],
|
44
|
+
custom_index_attrs: T::Hash[Symbol, T::Array],
|
45
|
+
hash_tag: T.nilable(String),
|
46
|
+
).void
|
47
|
+
end
|
48
|
+
def update_hash(model, id, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag:)
|
49
|
+
custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
|
50
|
+
if !(args.keys.to_set & attrs.to_set).empty?
|
51
|
+
result << index_name
|
52
|
+
result << attrs.size
|
53
|
+
result + attrs
|
54
|
+
else
|
55
|
+
result
|
56
|
+
end
|
57
|
+
end
|
58
|
+
run_script(
|
59
|
+
:update_hash,
|
60
|
+
keys: [id, hash_tag],
|
61
|
+
argv: [model, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
|
62
|
+
index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
|
63
|
+
)
|
64
|
+
end
|
65
|
+
|
66
|
+
sig do
|
67
|
+
params(
|
68
|
+
model: String,
|
69
|
+
id: String,
|
70
|
+
index_attrs: T::Array[Symbol],
|
71
|
+
range_index_attrs: T::Array[Symbol],
|
72
|
+
custom_index_attrs: T::Hash[Symbol, T::Array],
|
73
|
+
).returns(Integer)
|
74
|
+
end
|
75
|
+
def delete_hash(model, id, index_attrs:, range_index_attrs:, custom_index_attrs:)
|
76
|
+
custom_index_names = custom_index_attrs.keys
|
77
|
+
run_script(
|
78
|
+
:delete_hash,
|
79
|
+
keys: [id, id.match(/\{.*\}$/)&.send(:[], 0)],
|
80
|
+
argv: [model, index_attrs.size, range_index_attrs.size] + index_attrs + range_index_attrs + custom_index_names,
|
81
|
+
)
|
82
|
+
end
|
83
|
+
|
84
|
+
sig do
|
85
|
+
params(
|
86
|
+
model: String,
|
87
|
+
query_conditions: T::Hash[T.untyped, T.untyped],
|
88
|
+
index_attrs: T::Array[Symbol],
|
89
|
+
range_index_attrs: T::Array[Symbol],
|
90
|
+
select_attrs: T::Set[Symbol],
|
91
|
+
custom_index_attrs: T::Array[Symbol],
|
92
|
+
hash_tag: T.nilable(String),
|
93
|
+
custom_index_name: T.nilable(Symbol),
|
94
|
+
).returns(T::Hash[Integer, T::Hash[T.untyped, T.untyped]])
|
95
|
+
end
|
96
|
+
def find_by_attr(
|
97
|
+
model,
|
98
|
+
query_conditions,
|
99
|
+
select_attrs: Set.new,
|
100
|
+
index_attrs:,
|
101
|
+
range_index_attrs:,
|
102
|
+
custom_index_attrs: Array.new,
|
103
|
+
hash_tag: nil,
|
104
|
+
custom_index_name: nil
|
105
|
+
)
|
106
|
+
conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
|
107
|
+
res = run_script(
|
108
|
+
:find_by_attr,
|
109
|
+
keys: [hash_tag],
|
110
|
+
argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size, conditions.size] +
|
111
|
+
index_attrs + range_index_attrs + custom_index_attrs + conditions + select_attrs.to_a.flatten
|
112
|
+
)
|
113
|
+
# The Lua script will return this as a flattened array.
|
114
|
+
# Convert the result into a hash of {id -> model hash}
|
115
|
+
res_hash = res.each_slice(2)
|
116
|
+
res_hash.map { |key, val| [key, val.each_slice(2).to_h] }.to_h
|
117
|
+
end
|
118
|
+
|
119
|
+
sig do
|
120
|
+
params(
|
121
|
+
model: String,
|
122
|
+
query_conditions: T::Hash[T.untyped, T.untyped],
|
123
|
+
index_attrs: T::Array[Symbol],
|
124
|
+
range_index_attrs: T::Array[Symbol],
|
125
|
+
custom_index_attrs: T::Array[Symbol],
|
126
|
+
hash_tag: T.nilable(String),
|
127
|
+
custom_index_name: T.nilable(Symbol),
|
128
|
+
).returns(Integer)
|
129
|
+
end
|
130
|
+
def find_by_attr_count(
|
131
|
+
model,
|
132
|
+
query_conditions,
|
133
|
+
index_attrs:,
|
134
|
+
range_index_attrs:,
|
135
|
+
custom_index_attrs: Array.new,
|
136
|
+
hash_tag: nil,
|
137
|
+
custom_index_name: nil
|
138
|
+
)
|
139
|
+
conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
|
140
|
+
run_script(
|
141
|
+
:find_by_attr_count,
|
142
|
+
keys: [hash_tag],
|
143
|
+
argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size] +
|
144
|
+
index_attrs + range_index_attrs + custom_index_attrs + conditions
|
145
|
+
)
|
146
|
+
end
|
147
|
+
|
148
|
+
def scan_each_shard(key, count: 1000, &blk)
|
149
|
+
clients = instance_variable_get(:@client)
|
150
|
+
&.instance_variable_get(:@node)
|
151
|
+
&.instance_variable_get(:@clients)
|
152
|
+
&.values
|
153
|
+
|
154
|
+
if clients.nil?
|
155
|
+
scan_each(match: key, count: count, &blk)
|
156
|
+
else
|
157
|
+
clients.each do |client|
|
158
|
+
cursor = 0
|
159
|
+
loop do
|
160
|
+
cursor, keys = client.call([:scan, cursor, 'match', key, 'count', count])
|
161
|
+
keys.each(&blk)
|
162
|
+
break if cursor == "0"
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
private
|
169
|
+
|
170
|
+
def run_script(script_name, *args)
|
171
|
+
# Use EVAL when a redis shard has not loaded the script before
|
172
|
+
hash_var_name = :"@script_sha_#{script_name}"
|
173
|
+
hash = instance_variable_get(hash_var_name)
|
174
|
+
|
175
|
+
begin
|
176
|
+
return evalsha(hash, *args) if hash
|
177
|
+
rescue Redis::CommandError => e
|
178
|
+
if e.message != 'NOSCRIPT No matching script. Please use EVAL.'
|
179
|
+
raise e
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
script_content = Redcord::LuaScriptReader.read_lua_script(script_name.to_s)
|
184
|
+
instance_variable_set(hash_var_name, Digest::SHA1.hexdigest(script_content))
|
185
|
+
self.eval(script_content, *args)
|
186
|
+
end
|
187
|
+
|
188
|
+
# When using custom index: On Lua side script expects query conditions sorted
|
189
|
+
# in the order of appearance of attributes in specified index
|
190
|
+
sig { params(query_conditions: T::Hash[T.untyped, T.untyped], partial_order: T::Array[Symbol]).returns(T::Array[T.untyped]) }
|
191
|
+
def flatten_with_partial_sort(query_conditions, partial_order)
|
192
|
+
conditions = partial_order.inject([]) do |result, attr|
|
193
|
+
if !query_conditions[attr].nil?
|
194
|
+
result << attr << query_conditions.delete(attr)
|
195
|
+
end
|
196
|
+
result.flatten
|
197
|
+
end
|
198
|
+
conditions += query_conditions.to_a.flatten
|
199
|
+
end
|
200
|
+
end
|
@@ -5,7 +5,7 @@
|
|
5
5
|
require 'rails'
|
6
6
|
|
7
7
|
require 'redcord/lua_script_reader'
|
8
|
-
require 'redcord/
|
8
|
+
require 'redcord/redis'
|
9
9
|
|
10
10
|
module Redcord::RedisConnection
|
11
11
|
extend T::Sig
|
@@ -29,17 +29,17 @@ module Redcord::RedisConnection
|
|
29
29
|
(env_config[name.underscore] || env_config['default']).symbolize_keys
|
30
30
|
end
|
31
31
|
|
32
|
-
sig { returns(Redcord::
|
32
|
+
sig { returns(Redcord::Redis) }
|
33
33
|
def redis
|
34
34
|
Redcord::RedisConnection.connections[name.underscore] ||= prepare_redis!
|
35
35
|
end
|
36
36
|
|
37
|
-
sig { returns(Redcord::
|
37
|
+
sig { returns(Redcord::Redis) }
|
38
38
|
def establish_connection
|
39
39
|
Redcord::RedisConnection.connections[name.underscore] = prepare_redis!
|
40
40
|
end
|
41
41
|
|
42
|
-
sig { params(redis: Redis).returns(Redcord::
|
42
|
+
sig { params(redis: Redis).returns(Redcord::Redis) }
|
43
43
|
def redis=(redis)
|
44
44
|
Redcord::RedisConnection.connections[name.underscore] =
|
45
45
|
prepare_redis!(redis)
|
@@ -50,11 +50,11 @@ module Redcord::RedisConnection
|
|
50
50
|
# definitions in each Redis query.
|
51
51
|
#
|
52
52
|
# TODO: Replace this with Redcord migrations
|
53
|
-
sig { params(client: T.nilable(Redis)).returns(Redcord::
|
53
|
+
sig { params(client: T.nilable(Redis)).returns(Redcord::Redis) }
|
54
54
|
def prepare_redis!(client = nil)
|
55
|
-
return client if client.is_a?(Redcord::
|
55
|
+
return client if client.is_a?(Redcord::Redis)
|
56
56
|
|
57
|
-
client = Redcord::
|
57
|
+
client = Redcord::Redis.new(
|
58
58
|
**(
|
59
59
|
if client.nil?
|
60
60
|
connection_config
|
@@ -65,23 +65,7 @@ module Redcord::RedisConnection
|
|
65
65
|
logger: Redcord::Logger.proxy,
|
66
66
|
)
|
67
67
|
|
68
|
-
client.
|
69
|
-
Redcord::RedisConnection.procs_to_prepare.each do |proc_to_prepare|
|
70
|
-
proc_to_prepare.call(client)
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
script_names = Redcord::ServerScripts.instance_methods
|
75
|
-
res = client.pipelined do
|
76
|
-
script_names.each do |script_name|
|
77
|
-
client.script(
|
78
|
-
:load,
|
79
|
-
Redcord::LuaScriptReader.read_lua_script(script_name.to_s),
|
80
|
-
)
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
client.redcord_server_script_shas = script_names.zip(res).to_h
|
68
|
+
client.ping
|
85
69
|
client
|
86
70
|
end
|
87
71
|
end
|
@@ -89,7 +73,7 @@ module Redcord::RedisConnection
|
|
89
73
|
module InstanceMethods
|
90
74
|
extend T::Sig
|
91
75
|
|
92
|
-
sig { returns(Redcord::
|
76
|
+
sig { returns(Redcord::Redis) }
|
93
77
|
def redis
|
94
78
|
self.class.redis
|
95
79
|
end
|
@@ -119,3 +103,10 @@ module Redcord::RedisConnection
|
|
119
103
|
|
120
104
|
mixes_in_class_methods(ClassMethods)
|
121
105
|
end
|
106
|
+
|
107
|
+
module Redcord
|
108
|
+
sig { void }
|
109
|
+
def self.establish_connections
|
110
|
+
Redcord::Base.descendants.select(&:name).each(&:establish_connection)
|
111
|
+
end
|
112
|
+
end
|