redcord 0.0.3 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,10 +2,15 @@
2
2
  module Redcord::Migration::TTL
3
3
  extend T::Sig
4
4
 
5
- # This won't change ttl until we call update on a record
5
+ sig { params(model: T.class_of(Redcord::Base)).returns(T.untyped) }
6
+ def _get_ttl(model)
7
+ model.class_variable_get(:@@ttl) || -1
8
+ end
9
+
6
10
  sig { params(model: T.class_of(Redcord::Base)).void }
7
- def change_ttl_passive(model)
8
- ttl = model.class_variable_get(:@@ttl)
9
- model.redis.set("#{model.model_key}:ttl", ttl ? ttl : -1)
11
+ def change_ttl_active(model)
12
+ model.redis.scan_each_shard("#{model.model_key}:id:*") do |key|
13
+ model.redis.expire(key, _get_ttl(model))
14
+ end
10
15
  end
11
16
  end
@@ -29,7 +29,6 @@ class Redcord::Railtie < Rails::Railtie
29
29
  )
30
30
  end
31
31
 
32
- Redcord::PreparedRedis.load_server_scripts!
33
32
  Redcord._after_initialize!
34
33
  end
35
34
  end
@@ -0,0 +1,200 @@
1
+ # typed: true
2
+ require 'digest'
3
+ require 'redis'
4
+ require 'securerandom'
5
+
6
+ class Redcord::Redis < Redis
7
+ extend T::Sig
8
+
9
+ sig do
10
+ params(
11
+ key: T.any(String, Symbol),
12
+ args: T::Hash[T.untyped, T.untyped],
13
+ ttl: T.nilable(Integer),
14
+ index_attrs: T::Array[Symbol],
15
+ range_index_attrs: T::Array[Symbol],
16
+ custom_index_attrs: T::Hash[Symbol, T::Array],
17
+ hash_tag: T.nilable(String),
18
+ ).returns(String)
19
+ end
20
+ def create_hash_returning_id(key, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag: nil)
21
+ id = "#{SecureRandom.uuid}#{hash_tag}"
22
+ custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
23
+ result << index_name
24
+ result << attrs.size
25
+ result + attrs
26
+ end
27
+ run_script(
28
+ :create_hash,
29
+ keys: [id, hash_tag],
30
+ argv: [key, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
31
+ index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
32
+ )
33
+ id
34
+ end
35
+
36
+ sig do
37
+ params(
38
+ model: String,
39
+ id: String,
40
+ args: T::Hash[T.untyped, T.untyped],
41
+ ttl: T.nilable(Integer),
42
+ index_attrs: T::Array[Symbol],
43
+ range_index_attrs: T::Array[Symbol],
44
+ custom_index_attrs: T::Hash[Symbol, T::Array],
45
+ hash_tag: T.nilable(String),
46
+ ).void
47
+ end
48
+ def update_hash(model, id, args, ttl:, index_attrs:, range_index_attrs:, custom_index_attrs:, hash_tag:)
49
+ custom_index_attrs_flat = custom_index_attrs.inject([]) do |result, (index_name, attrs)|
50
+ if !(args.keys.to_set & attrs.to_set).empty?
51
+ result << index_name
52
+ result << attrs.size
53
+ result + attrs
54
+ else
55
+ result
56
+ end
57
+ end
58
+ run_script(
59
+ :update_hash,
60
+ keys: [id, hash_tag],
61
+ argv: [model, ttl, index_attrs.size, range_index_attrs.size, custom_index_attrs_flat.size] +
62
+ index_attrs + range_index_attrs + custom_index_attrs_flat + args.to_a.flatten,
63
+ )
64
+ end
65
+
66
+ sig do
67
+ params(
68
+ model: String,
69
+ id: String,
70
+ index_attrs: T::Array[Symbol],
71
+ range_index_attrs: T::Array[Symbol],
72
+ custom_index_attrs: T::Hash[Symbol, T::Array],
73
+ ).returns(Integer)
74
+ end
75
+ def delete_hash(model, id, index_attrs:, range_index_attrs:, custom_index_attrs:)
76
+ custom_index_names = custom_index_attrs.keys
77
+ run_script(
78
+ :delete_hash,
79
+ keys: [id, id.match(/\{.*\}$/)&.send(:[], 0)],
80
+ argv: [model, index_attrs.size, range_index_attrs.size] + index_attrs + range_index_attrs + custom_index_names,
81
+ )
82
+ end
83
+
84
+ sig do
85
+ params(
86
+ model: String,
87
+ query_conditions: T::Hash[T.untyped, T.untyped],
88
+ index_attrs: T::Array[Symbol],
89
+ range_index_attrs: T::Array[Symbol],
90
+ select_attrs: T::Set[Symbol],
91
+ custom_index_attrs: T::Array[Symbol],
92
+ hash_tag: T.nilable(String),
93
+ custom_index_name: T.nilable(Symbol),
94
+ ).returns(T::Hash[Integer, T::Hash[T.untyped, T.untyped]])
95
+ end
96
+ def find_by_attr(
97
+ model,
98
+ query_conditions,
99
+ select_attrs: Set.new,
100
+ index_attrs:,
101
+ range_index_attrs:,
102
+ custom_index_attrs: Array.new,
103
+ hash_tag: nil,
104
+ custom_index_name: nil
105
+ )
106
+ conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
107
+ res = run_script(
108
+ :find_by_attr,
109
+ keys: [hash_tag],
110
+ argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size, conditions.size] +
111
+ index_attrs + range_index_attrs + custom_index_attrs + conditions + select_attrs.to_a.flatten
112
+ )
113
+ # The Lua script will return this as a flattened array.
114
+ # Convert the result into a hash of {id -> model hash}
115
+ res_hash = res.each_slice(2)
116
+ res_hash.map { |key, val| [key, val.each_slice(2).to_h] }.to_h
117
+ end
118
+
119
+ sig do
120
+ params(
121
+ model: String,
122
+ query_conditions: T::Hash[T.untyped, T.untyped],
123
+ index_attrs: T::Array[Symbol],
124
+ range_index_attrs: T::Array[Symbol],
125
+ custom_index_attrs: T::Array[Symbol],
126
+ hash_tag: T.nilable(String),
127
+ custom_index_name: T.nilable(Symbol),
128
+ ).returns(Integer)
129
+ end
130
+ def find_by_attr_count(
131
+ model,
132
+ query_conditions,
133
+ index_attrs:,
134
+ range_index_attrs:,
135
+ custom_index_attrs: Array.new,
136
+ hash_tag: nil,
137
+ custom_index_name: nil
138
+ )
139
+ conditions = flatten_with_partial_sort(query_conditions.clone, custom_index_attrs)
140
+ run_script(
141
+ :find_by_attr_count,
142
+ keys: [hash_tag],
143
+ argv: [model, custom_index_name, index_attrs.size, range_index_attrs.size, custom_index_attrs.size] +
144
+ index_attrs + range_index_attrs + custom_index_attrs + conditions
145
+ )
146
+ end
147
+
148
+ def scan_each_shard(key, count: 1000, &blk)
149
+ clients = instance_variable_get(:@client)
150
+ &.instance_variable_get(:@node)
151
+ &.instance_variable_get(:@clients)
152
+ &.values
153
+
154
+ if clients.nil?
155
+ scan_each(match: key, count: count, &blk)
156
+ else
157
+ clients.each do |client|
158
+ cursor = 0
159
+ loop do
160
+ cursor, keys = client.call([:scan, cursor, 'match', key, 'count', count])
161
+ keys.each(&blk)
162
+ break if cursor == "0"
163
+ end
164
+ end
165
+ end
166
+ end
167
+
168
+ private
169
+
170
+ def run_script(script_name, *args)
171
+ # Use EVAL when a redis shard has not loaded the script before
172
+ hash_var_name = :"@script_sha_#{script_name}"
173
+ hash = instance_variable_get(hash_var_name)
174
+
175
+ begin
176
+ return evalsha(hash, *args) if hash
177
+ rescue Redis::CommandError => e
178
+ if e.message != 'NOSCRIPT No matching script. Please use EVAL.'
179
+ raise e
180
+ end
181
+ end
182
+
183
+ script_content = Redcord::LuaScriptReader.read_lua_script(script_name.to_s)
184
+ instance_variable_set(hash_var_name, Digest::SHA1.hexdigest(script_content))
185
+ self.eval(script_content, *args)
186
+ end
187
+
188
+ # When using custom index: On Lua side script expects query conditions sorted
189
+ # in the order of appearance of attributes in specified index
190
+ sig { params(query_conditions: T::Hash[T.untyped, T.untyped], partial_order: T::Array[Symbol]).returns(T::Array[T.untyped]) }
191
+ def flatten_with_partial_sort(query_conditions, partial_order)
192
+ conditions = partial_order.inject([]) do |result, attr|
193
+ if !query_conditions[attr].nil?
194
+ result << attr << query_conditions.delete(attr)
195
+ end
196
+ result.flatten
197
+ end
198
+ conditions += query_conditions.to_a.flatten
199
+ end
200
+ end
@@ -5,12 +5,15 @@
5
5
  require 'rails'
6
6
 
7
7
  require 'redcord/lua_script_reader'
8
- require 'redcord/prepared_redis'
8
+ require 'redcord/redis'
9
+ require 'redcord/connection_pool'
9
10
 
10
11
  module Redcord::RedisConnection
11
12
  extend T::Sig
12
13
  extend T::Helpers
13
14
 
15
+ RedcordClientType = T.type_alias { T.any(Redcord::Redis, Redcord::ConnectionPool) }
16
+
14
17
  @connections = T.let(nil, T.nilable(T::Hash[String, T.untyped]))
15
18
  @procs_to_prepare = T.let([], T::Array[Proc])
16
19
 
@@ -29,17 +32,17 @@ module Redcord::RedisConnection
29
32
  (env_config[name.underscore] || env_config['default']).symbolize_keys
30
33
  end
31
34
 
32
- sig { returns(Redcord::PreparedRedis) }
35
+ sig { returns(RedcordClientType) }
33
36
  def redis
34
37
  Redcord::RedisConnection.connections[name.underscore] ||= prepare_redis!
35
38
  end
36
39
 
37
- sig { returns(Redcord::PreparedRedis) }
40
+ sig { returns(RedcordClientType) }
38
41
  def establish_connection
39
42
  Redcord::RedisConnection.connections[name.underscore] = prepare_redis!
40
43
  end
41
44
 
42
- sig { params(redis: Redis).returns(Redcord::PreparedRedis) }
45
+ sig { params(redis: Redis).returns(RedcordClientType) }
43
46
  def redis=(redis)
44
47
  Redcord::RedisConnection.connections[name.underscore] =
45
48
  prepare_redis!(redis)
@@ -50,27 +53,23 @@ module Redcord::RedisConnection
50
53
  # definitions in each Redis query.
51
54
  #
52
55
  # TODO: Replace this with Redcord migrations
53
- sig { params(client: T.nilable(Redis)).returns(Redcord::PreparedRedis) }
56
+ sig { params(client: T.nilable(Redis)).returns(RedcordClientType) }
54
57
  def prepare_redis!(client = nil)
55
- return client if client.is_a?(Redcord::PreparedRedis)
56
-
57
- client = Redcord::PreparedRedis.new(
58
- **(
59
- if client.nil?
60
- connection_config
61
- else
62
- client.instance_variable_get(:@options)
63
- end
64
- ),
65
- logger: Redcord::Logger.proxy,
66
- )
67
-
68
- client.pipelined do
69
- Redcord::RedisConnection.procs_to_prepare.each do |proc_to_prepare|
70
- proc_to_prepare.call(client)
58
+ return client if client.is_a?(Redcord::Redis) || client.is_a?(Redcord::ConnectionPool)
59
+
60
+ options = client.nil? ? connection_config : client.instance_variable_get(:@options)
61
+ client =
62
+ if options[:pool]
63
+ Redcord::ConnectionPool.new(
64
+ pool_size: options[:pool],
65
+ timeout: options[:connection_timeout] || 1.0,
66
+ **options
67
+ )
68
+ else
69
+ Redcord::Redis.new(**options, logger: Redcord::Logger.proxy)
71
70
  end
72
- end
73
71
 
72
+ client.ping
74
73
  client
75
74
  end
76
75
  end
@@ -78,7 +77,7 @@ module Redcord::RedisConnection
78
77
  module InstanceMethods
79
78
  extend T::Sig
80
79
 
81
- sig { returns(Redcord::PreparedRedis) }
80
+ sig { returns(RedcordClientType) }
82
81
  def redis
83
82
  self.class.redis
84
83
  end
@@ -108,3 +107,10 @@ module Redcord::RedisConnection
108
107
 
109
108
  mixes_in_class_methods(ClassMethods)
110
109
  end
110
+
111
+ module Redcord
112
+ sig { void }
113
+ def self.establish_connections
114
+ Redcord::Base.descendants.select(&:name).each(&:establish_connection)
115
+ end
116
+ end
@@ -5,42 +5,62 @@
5
5
  require 'active_support/core_ext/array'
6
6
  require 'active_support/core_ext/module'
7
7
 
8
+ module Redcord
9
+ class InvalidQuery < StandardError; end
10
+ end
11
+
8
12
  class Redcord::Relation
9
13
  extend T::Sig
10
14
 
11
15
  sig { returns(T.class_of(Redcord::Base)) }
12
16
  attr_reader :model
13
17
 
14
- sig { returns(T::Hash[Symbol, T.untyped]) }
15
- attr_reader :query_conditions
16
-
17
18
  sig { returns(T::Set[Symbol]) }
18
19
  attr_reader :select_attrs
19
20
 
21
+ sig { returns(T.nilable(Symbol)) }
22
+ attr_reader :custom_index_name
23
+
24
+ sig { returns(T::Hash[Symbol, T.untyped]) }
25
+ attr_reader :regular_index_query_conditions
26
+
27
+ sig { returns(T::Hash[Symbol, T.untyped]) }
28
+ attr_reader :custom_index_query_conditions
29
+
20
30
  sig do
21
31
  params(
22
32
  model: T.class_of(Redcord::Base),
23
- query_conditions: T::Hash[Symbol, T.untyped],
33
+ regular_index_query_conditions: T::Hash[Symbol, T.untyped],
34
+ custom_index_query_conditions: T::Hash[Symbol, T.untyped],
24
35
  select_attrs: T::Set[Symbol],
36
+ custom_index_name: T.nilable(Symbol)
25
37
  ).void
26
38
  end
27
39
  def initialize(
28
40
  model,
29
- query_conditions = {},
30
- select_attrs = Set.new
41
+ regular_index_query_conditions = {},
42
+ custom_index_query_conditions = {},
43
+ select_attrs = Set.new,
44
+ custom_index_name: nil
31
45
  )
32
46
  @model = model
33
- @query_conditions = query_conditions
47
+ @regular_index_query_conditions = regular_index_query_conditions
48
+ @custom_index_query_conditions = custom_index_query_conditions
34
49
  @select_attrs = select_attrs
50
+ @custom_index_name = custom_index_name
35
51
  end
36
52
 
37
53
  sig { params(args: T::Hash[Symbol, T.untyped]).returns(Redcord::Relation) }
38
54
  def where(args)
39
55
  encoded_args = args.map do |attr_key, attr_val|
40
- encoded_val = model.validate_and_encode_query(attr_key, attr_val)
56
+ encoded_val = model.validate_types_and_encode_query(attr_key, attr_val)
41
57
  [attr_key, encoded_val]
42
58
  end
43
- query_conditions.merge!(encoded_args.to_h)
59
+
60
+ regular_index_query_conditions.merge!(encoded_args.to_h)
61
+ if custom_index_name
62
+ with_index(custom_index_name)
63
+ end
44
64
  self
45
65
  end
46
66
 
@@ -68,7 +88,29 @@ class Redcord::Relation
68
88
 
69
89
  sig { returns(Integer) }
70
90
  def count
71
- redis.find_by_attr_count(model.model_key, query_conditions)
91
+ Redcord::Base.trace(
92
+ 'redcord_relation_count',
93
+ model_name: model.name,
94
+ ) do
95
+ model.validate_index_attributes(query_conditions.keys, custom_index_name: custom_index_name)
96
+ redis.find_by_attr_count(
97
+ model.model_key,
98
+ extract_query_conditions!,
99
+ index_attrs: model._script_arg_index_attrs,
100
+ range_index_attrs: model._script_arg_range_index_attrs,
101
+ custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
102
+ hash_tag: extract_hash_tag!,
103
+ custom_index_name: custom_index_name
104
+ )
105
+ end
106
+ end
107
+
108
+ sig { params(index_name: T.nilable(Symbol)).returns(Redcord::Relation) }
109
+ def with_index(index_name)
110
+ @custom_index_name = index_name
111
+ adjusted_query_conditions = model.validate_and_adjust_custom_index_query_conditions(regular_index_query_conditions)
112
+ custom_index_query_conditions.merge!(adjusted_query_conditions)
113
+ self
72
114
  end
73
115
 
74
116
  delegate(
@@ -137,17 +179,52 @@ class Redcord::Relation
137
179
 
138
180
  private
139
181
 
182
+ sig { returns(T.nilable(String)) }
183
+ def extract_hash_tag!
184
+ attr = model.shard_by_attribute
185
+ return nil if attr.nil?
186
+
187
+ if !query_conditions.keys.include?(attr)
188
+ raise(
189
+ Redcord::InvalidQuery,
190
+ "Queries must contain attribute '#{attr}' since model #{model.name} is sharded by this attribute"
191
+ )
192
+ end
193
+
194
+ # Query conditions on custom index are always in form of range, even when query is by value condition is [value_x, value_x]
195
+ # When in fact query is by value, range is trasformed to a single value to pass the validation.
196
+ condition = query_conditions[attr]
197
+ if custom_index_name and condition.first == condition.last
198
+ condition = condition.first
199
+ end
200
+ case condition
201
+ when Integer, String
202
+ "{#{condition}}"
203
+ else
204
+ raise(
205
+ Redcord::InvalidQuery,
206
+ "Does not support query condition #{condition} on a Redis Cluster",
207
+ )
208
+ end
209
+ end
210
+
140
211
  sig { returns(T::Array[T.untyped]) }
141
212
  def execute_query
142
213
  Redcord::Base.trace(
143
214
  'redcord_relation_execute_query',
144
215
  model_name: model.name,
145
216
  ) do
217
+ model.validate_index_attributes(query_conditions.keys, custom_index_name: custom_index_name)
146
218
  if !select_attrs.empty?
147
219
  res_hash = redis.find_by_attr(
148
220
  model.model_key,
149
- query_conditions,
150
- select_attrs,
221
+ extract_query_conditions!,
222
+ select_attrs: select_attrs,
223
+ index_attrs: model._script_arg_index_attrs,
224
+ range_index_attrs: model._script_arg_range_index_attrs,
225
+ custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
226
+ hash_tag: extract_hash_tag!,
227
+ custom_index_name: custom_index_name
151
228
  )
152
229
 
153
230
  res_hash.map do |id, args|
@@ -158,7 +235,12 @@ class Redcord::Relation
158
235
  else
159
236
  res_hash = redis.find_by_attr(
160
237
  model.model_key,
161
- query_conditions,
238
+ extract_query_conditions!,
239
+ index_attrs: model._script_arg_index_attrs,
240
+ range_index_attrs: model._script_arg_range_index_attrs,
241
+ custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
242
+ hash_tag: extract_hash_tag!,
243
+ custom_index_name: custom_index_name
162
244
  )
163
245
 
164
246
  res_hash.map { |id, args| model.coerce_and_set_id(args, id) }
@@ -166,8 +248,24 @@ class Redcord::Relation
166
248
  end
167
249
  end
168
250
 
169
- sig { returns(Redcord::PreparedRedis) }
251
+ sig { returns(Redcord::RedisConnection::RedcordClientType) }
170
252
  def redis
171
253
  model.redis
172
254
  end
255
+
256
+ sig { returns(T::Hash[Symbol, T.untyped]) }
257
+ def query_conditions
258
+ custom_index_name ? custom_index_query_conditions : regular_index_query_conditions
259
+ end
260
+
261
+ sig { returns(T::Hash[Symbol, T.untyped]) }
262
+ def extract_query_conditions!
263
+ attr = model.shard_by_attribute
264
+ return query_conditions if attr.nil?
265
+
266
+ cond = query_conditions.reject { |key| key == attr }
267
+ raise Redcord::InvalidQuery, "Cannot query only by shard_by_attribute: #{attr}" if cond.empty?
268
+
269
+ cond
270
+ end
173
271
  end