redcord 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,7 +5,7 @@
5
5
  require 'rails'
6
6
 
7
7
  require 'redcord/lua_script_reader'
8
- require 'redcord/prepared_redis'
8
+ require 'redcord/redis'
9
9
 
10
10
  module Redcord::RedisConnection
11
11
  extend T::Sig
@@ -29,17 +29,17 @@ module Redcord::RedisConnection
29
29
  (env_config[name.underscore] || env_config['default']).symbolize_keys
30
30
  end
31
31
 
32
- sig { returns(Redcord::PreparedRedis) }
32
+ sig { returns(Redcord::Redis) }
33
33
  def redis
34
34
  Redcord::RedisConnection.connections[name.underscore] ||= prepare_redis!
35
35
  end
36
36
 
37
- sig { returns(Redcord::PreparedRedis) }
37
+ sig { returns(Redcord::Redis) }
38
38
  def establish_connection
39
39
  Redcord::RedisConnection.connections[name.underscore] = prepare_redis!
40
40
  end
41
41
 
42
- sig { params(redis: Redis).returns(Redcord::PreparedRedis) }
42
+ sig { params(redis: Redis).returns(Redcord::Redis) }
43
43
  def redis=(redis)
44
44
  Redcord::RedisConnection.connections[name.underscore] =
45
45
  prepare_redis!(redis)
@@ -50,11 +50,11 @@ module Redcord::RedisConnection
50
50
  # definitions in each Redis query.
51
51
  #
52
52
  # TODO: Replace this with Redcord migrations
53
- sig { params(client: T.nilable(Redis)).returns(Redcord::PreparedRedis) }
53
+ sig { params(client: T.nilable(Redis)).returns(Redcord::Redis) }
54
54
  def prepare_redis!(client = nil)
55
- return client if client.is_a?(Redcord::PreparedRedis)
55
+ return client if client.is_a?(Redcord::Redis)
56
56
 
57
- client = Redcord::PreparedRedis.new(
57
+ client = Redcord::Redis.new(
58
58
  **(
59
59
  if client.nil?
60
60
  connection_config
@@ -65,12 +65,7 @@ module Redcord::RedisConnection
65
65
  logger: Redcord::Logger.proxy,
66
66
  )
67
67
 
68
- client.pipelined do
69
- Redcord::RedisConnection.procs_to_prepare.each do |proc_to_prepare|
70
- proc_to_prepare.call(client)
71
- end
72
- end
73
-
68
+ client.ping
74
69
  client
75
70
  end
76
71
  end
@@ -78,7 +73,7 @@ module Redcord::RedisConnection
78
73
  module InstanceMethods
79
74
  extend T::Sig
80
75
 
81
- sig { returns(Redcord::PreparedRedis) }
76
+ sig { returns(Redcord::Redis) }
82
77
  def redis
83
78
  self.class.redis
84
79
  end
@@ -5,42 +5,62 @@
5
5
  require 'active_support/core_ext/array'
6
6
  require 'active_support/core_ext/module'
7
7
 
8
+ module Redcord
9
+ class InvalidQuery < StandardError; end
10
+ end
11
+
8
12
  class Redcord::Relation
9
13
  extend T::Sig
10
14
 
11
15
  sig { returns(T.class_of(Redcord::Base)) }
12
16
  attr_reader :model
13
17
 
14
- sig { returns(T::Hash[Symbol, T.untyped]) }
15
- attr_reader :query_conditions
16
-
17
18
  sig { returns(T::Set[Symbol]) }
18
19
  attr_reader :select_attrs
19
20
 
21
+ sig { returns(T.nilable(Symbol)) }
22
+ attr_reader :custom_index_name
23
+
24
+ sig { returns(T::Hash[Symbol, T.untyped]) }
25
+ attr_reader :regular_index_query_conditions
26
+
27
+ sig { returns(T::Hash[Symbol, T.untyped]) }
28
+ attr_reader :custom_index_query_conditions
29
+
20
30
  sig do
21
31
  params(
22
32
  model: T.class_of(Redcord::Base),
23
- query_conditions: T::Hash[Symbol, T.untyped],
33
+ regular_index_query_conditions: T::Hash[Symbol, T.untyped],
34
+ custom_index_query_conditions: T::Hash[Symbol, T.untyped],
24
35
  select_attrs: T::Set[Symbol],
36
+ custom_index_name: T.nilable(Symbol)
25
37
  ).void
26
38
  end
27
39
  def initialize(
28
40
  model,
29
- query_conditions = {},
30
- select_attrs = Set.new
41
+ regular_index_query_conditions = {},
42
+ custom_index_query_conditions = {},
43
+ select_attrs = Set.new,
44
+ custom_index_name: nil
31
45
  )
32
46
  @model = model
33
- @query_conditions = query_conditions
47
+ @regular_index_query_conditions = regular_index_query_conditions
48
+ @custom_index_query_conditions = custom_index_query_conditions
34
49
  @select_attrs = select_attrs
50
+ @custom_index_name = custom_index_name
35
51
  end
36
52
 
37
53
  sig { params(args: T::Hash[Symbol, T.untyped]).returns(Redcord::Relation) }
38
54
  def where(args)
39
55
  encoded_args = args.map do |attr_key, attr_val|
40
- encoded_val = model.validate_and_encode_query(attr_key, attr_val)
56
+ encoded_val = model.validate_types_and_encode_query(attr_key, attr_val)
41
57
  [attr_key, encoded_val]
42
58
  end
43
- query_conditions.merge!(encoded_args.to_h)
59
+
60
+ regular_index_query_conditions.merge!(encoded_args.to_h)
61
+ if custom_index_name
62
+ with_index(custom_index_name)
63
+ end
44
64
  self
45
65
  end
46
66
 
@@ -68,7 +88,24 @@ class Redcord::Relation
68
88
 
69
89
  sig { returns(Integer) }
70
90
  def count
71
- redis.find_by_attr_count(model.model_key, query_conditions)
91
+ model.validate_index_attributes(query_conditions.keys, custom_index_name: custom_index_name)
92
+ redis.find_by_attr_count(
93
+ model.model_key,
94
+ extract_query_conditions!,
95
+ index_attrs: model._script_arg_index_attrs,
96
+ range_index_attrs: model._script_arg_range_index_attrs,
97
+ custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
98
+ hash_tag: extract_hash_tag!,
99
+ custom_index_name: custom_index_name
100
+ )
101
+ end
102
+
103
+ sig { params(index_name: T.nilable(Symbol)).returns(Redcord::Relation) }
104
+ def with_index(index_name)
105
+ @custom_index_name = index_name
106
+ adjusted_query_conditions = model.validate_and_adjust_custom_index_query_conditions(regular_index_query_conditions)
107
+ custom_index_query_conditions.merge!(adjusted_query_conditions)
108
+ self
72
109
  end
73
110
 
74
111
  delegate(
@@ -137,17 +174,52 @@ class Redcord::Relation
137
174
 
138
175
  private
139
176
 
177
+ sig { returns(T.nilable(String)) }
178
+ def extract_hash_tag!
179
+ attr = model.shard_by_attribute
180
+ return nil if attr.nil?
181
+
182
+ if !query_conditions.keys.include?(attr)
183
+ raise(
184
+ Redcord::InvalidQuery,
185
+ "Queries must contain attribute '#{attr}' since model #{model.name} is sharded by this attribute"
186
+ )
187
+ end
188
+
189
+ # Query conditions on custom index are always in form of range, even when query is by value condition is [value_x, value_x]
190
+ # When in fact query is by value, range is trasformed to a single value to pass the validation.
191
+ condition = query_conditions[attr]
192
+ if custom_index_name and condition.first == condition.last
193
+ condition = condition.first
194
+ end
195
+ case condition
196
+ when Integer, String
197
+ "{#{condition}}"
198
+ else
199
+ raise(
200
+ Redcord::InvalidQuery,
201
+ "Does not support query condition #{condition} on a Redis Cluster",
202
+ )
203
+ end
204
+ end
205
+
140
206
  sig { returns(T::Array[T.untyped]) }
141
207
  def execute_query
142
208
  Redcord::Base.trace(
143
209
  'redcord_relation_execute_query',
144
210
  model_name: model.name,
145
211
  ) do
212
+ model.validate_index_attributes(query_conditions.keys, custom_index_name: custom_index_name)
146
213
  if !select_attrs.empty?
147
214
  res_hash = redis.find_by_attr(
148
215
  model.model_key,
149
- query_conditions,
150
- select_attrs,
216
+ extract_query_conditions!,
217
+ select_attrs: select_attrs,
218
+ index_attrs: model._script_arg_index_attrs,
219
+ range_index_attrs: model._script_arg_range_index_attrs,
220
+ custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
221
+ hash_tag: extract_hash_tag!,
222
+ custom_index_name: custom_index_name
151
223
  )
152
224
 
153
225
  res_hash.map do |id, args|
@@ -158,7 +230,12 @@ class Redcord::Relation
158
230
  else
159
231
  res_hash = redis.find_by_attr(
160
232
  model.model_key,
161
- query_conditions,
233
+ extract_query_conditions!,
234
+ index_attrs: model._script_arg_index_attrs,
235
+ range_index_attrs: model._script_arg_range_index_attrs,
236
+ custom_index_attrs: model._script_arg_custom_index_attrs[custom_index_name],
237
+ hash_tag: extract_hash_tag!,
238
+ custom_index_name: custom_index_name
162
239
  )
163
240
 
164
241
  res_hash.map { |id, args| model.coerce_and_set_id(args, id) }
@@ -166,8 +243,24 @@ class Redcord::Relation
166
243
  end
167
244
  end
168
245
 
169
- sig { returns(Redcord::PreparedRedis) }
246
+ sig { returns(Redcord::Redis) }
170
247
  def redis
171
248
  model.redis
172
249
  end
250
+
251
+ sig { returns(T::Hash[Symbol, T.untyped]) }
252
+ def query_conditions
253
+ custom_index_name ? custom_index_query_conditions : regular_index_query_conditions
254
+ end
255
+
256
+ sig { returns(T::Hash[Symbol, T.untyped]) }
257
+ def extract_query_conditions!
258
+ attr = model.shard_by_attribute
259
+ return query_conditions if attr.nil?
260
+
261
+ cond = query_conditions.reject { |key| key == attr }
262
+ raise Redcord::InvalidQuery, "Cannot query only by shard_by_attribute: #{attr}" if cond.empty?
263
+
264
+ cond
265
+ end
173
266
  end
@@ -8,6 +8,8 @@ module Redcord
8
8
  # Raised by Model.where
9
9
  class AttributeNotIndexed < StandardError; end
10
10
  class WrongAttributeType < TypeError; end
11
+ class CustomIndexInvalidQuery < StandardError; end
12
+ class CustomIndexInvalidDesign < StandardError; end
11
13
  end
12
14
 
13
15
  # This module defines various helper methods on Redcord for serialization
@@ -31,50 +33,36 @@ module Redcord::Serializer
31
33
  sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
32
34
  def encode_attr_value(attribute, val)
33
35
  if !val.blank? && TIME_TYPES.include?(props[attribute][:type])
34
- val = val.to_f
36
+ time_in_nano_sec = val.to_i * 1_000_000_000
37
+ time_in_nano_sec >= 0 ? time_in_nano_sec + val.nsec : time_in_nano_sec - val.nsec
38
+ elsif val.is_a?(Float)
39
+ # Encode as round-trippable float64
40
+ '%1.16e' % [val]
41
+ else
42
+ val
35
43
  end
36
-
37
- val
38
44
  end
39
45
 
40
46
  sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
41
47
  def decode_attr_value(attribute, val)
42
48
  if !val.blank? && TIME_TYPES.include?(props[attribute][:type])
43
- val = Time.zone.at(val.to_f)
44
- end
49
+ val = val.to_i
50
+ nsec = val >= 0 ? val % 1_000_000_000 : -val % 1_000_000_000
45
51
 
46
- val
52
+ Time.zone.at(val / 1_000_000_000).change(nsec: nsec)
53
+ else
54
+ val
55
+ end
47
56
  end
48
57
 
49
58
  sig { params(attr_key: Symbol, attr_val: T.untyped).returns(T.untyped)}
50
- def validate_and_encode_query(attr_key, attr_val)
51
- # Validate that attributes queried for are index attributes
52
- if !class_variable_get(:@@index_attributes).include?(attr_key) &&
53
- !class_variable_get(:@@range_index_attributes).include?(attr_key)
54
- raise(
55
- Redcord::AttributeNotIndexed,
56
- "#{attr_key} is not an indexed attribute.",
57
- )
58
- end
59
-
60
- # Validate attribute types for normal index attributes
59
+ def validate_types_and_encode_query(attr_key, attr_val)
60
+ # Validate attribute types for index attributes
61
61
  attr_type = get_attr_type(attr_key)
62
- if class_variable_get(:@@index_attributes).include?(attr_key)
62
+ if class_variable_get(:@@index_attributes).include?(attr_key) || attr_key == shard_by_attribute
63
63
  validate_attr_type(attr_val, attr_type)
64
64
  else
65
- # Validate attribute types for range index attributes
66
- if attr_val.is_a?(Redcord::RangeInterval)
67
- validate_attr_type(
68
- attr_val.min,
69
- T.cast(T.nilable(attr_type), T::Types::Base),
70
- )
71
- validate_attr_type(
72
- attr_val.max,
73
- T.cast(T.nilable(attr_type), T::Types::Base),
74
- )
75
- else
76
- validate_attr_type(attr_val, attr_type)
77
- end
65
+ validate_range_attr_types(attr_val, attr_type)
78
66
 
79
67
  # Range index attributes need to be further encoded into a format
80
68
  # understood by the Lua script.
@@ -82,10 +70,73 @@ module Redcord::Serializer
82
70
  attr_val = encode_range_index_attr_val(attr_key, attr_val)
83
71
  end
84
72
  end
85
-
86
73
  attr_val
87
74
  end
88
75
 
76
+ # Validate that attributes queried for are index attributes
77
+ # For custom index: validate that attributes are present in specified index
78
+ sig { params(attr_keys: T::Array[Symbol], custom_index_name: T.nilable(Symbol)).void}
79
+ def validate_index_attributes(attr_keys, custom_index_name: nil)
80
+ custom_index_attributes = class_variable_get(:@@custom_index_attributes)[custom_index_name]
81
+ attr_keys.each do |attr_key|
82
+ next if attr_key == shard_by_attribute
83
+
84
+ if !custom_index_attributes.empty?
85
+ if !custom_index_attributes.include?(attr_key)
86
+ raise(
87
+ Redcord::AttributeNotIndexed,
88
+ "#{attr_key} is not a part of #{custom_index_name} index.",
89
+ )
90
+ end
91
+ else
92
+ if !class_variable_get(:@@index_attributes).include?(attr_key) &&
93
+ !class_variable_get(:@@range_index_attributes).include?(attr_key)
94
+ raise(
95
+ Redcord::AttributeNotIndexed,
96
+ "#{attr_key} is not an indexed attribute.",
97
+ )
98
+ end
99
+ end
100
+ end
101
+ end
102
+
103
+ # Validate exclusive ranges not used; Change all query conditions to range form;
104
+ # The position of the attribute and type of query is validated on Lua side
105
+ sig { params(query_conditions: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped])}
106
+ def validate_and_adjust_custom_index_query_conditions(query_conditions)
107
+ adjusted_query_conditions = query_conditions.clone
108
+ query_conditions.each do |attr_key, condition|
109
+ if !condition.is_a?(Array)
110
+ adjusted_query_conditions[attr_key] = [condition, condition]
111
+ elsif condition[0].to_s[0] == '(' or condition[1].to_s[0] == '('
112
+ raise(Redcord::CustomIndexInvalidQuery, "Custom index doesn't support exclusive ranges")
113
+ end
114
+ end
115
+ adjusted_query_conditions
116
+ end
117
+
118
+ sig {
119
+ params(
120
+ attr_val: T.untyped,
121
+ attr_type: T.any(Class, T::Types::Base),
122
+ ).void
123
+ }
124
+ def validate_range_attr_types(attr_val, attr_type)
125
+ # Validate attribute types for range index attributes
126
+ if attr_val.is_a?(Redcord::RangeInterval)
127
+ validate_attr_type(
128
+ attr_val.min,
129
+ T.cast(T.nilable(attr_type), T::Types::Base),
130
+ )
131
+ validate_attr_type(
132
+ attr_val.max,
133
+ T.cast(T.nilable(attr_type), T::Types::Base),
134
+ )
135
+ else
136
+ validate_attr_type(attr_val, attr_type)
137
+ end
138
+ end
139
+
89
140
  sig {
90
141
  params(
91
142
  attr_val: T.untyped,
@@ -137,7 +188,7 @@ module Redcord::Serializer
137
188
  sig {
138
189
  params(
139
190
  redis_hash: T::Hash[T.untyped, T.untyped],
140
- id: Integer,
191
+ id: String,
141
192
  ).returns(T.untyped)
142
193
  }
143
194
  def coerce_and_set_id(redis_hash, id)
@@ -0,0 +1,81 @@
1
+ --[[
2
+ EVALSHA SHA1(__FILE__) [field value ...]
3
+ > Time complexity: O(N) where N is the number of fields being set.
4
+
5
+ Create a hash with the specified fields to their respective values stored at
6
+ key when key does not exist.
7
+
8
+ # Return value
9
+ The id of the created hash as a string.
10
+ --]]
11
+
12
+ -- The arguments can be accessed by Lua using the KEYS global variable in the
13
+ -- form of a one-based array (so KEYS[1], KEYS[2], ...).
14
+ -- All the additional arguments should not represent key names and can be
15
+ -- accessed by Lua using the ARGV global variable, very similarly to what
16
+ -- happens with keys (so ARGV[1], ARGV[2], ...).
17
+
18
+ -- KEYS = id hash_tag
19
+ -- ARGV = Model.name ttl index_attr_size range_index_attr_size custom_index_attrs_flat_size [index_attr_key ...] [range_index_attr_key ...]
20
+ -- [custom_index_name attrs_size [custom_index_attr_key ...] ...] attr_key attr_val [attr_key attr_val ..]
21
+ <%= include_lua 'shared/lua_helper_methods' %>
22
+ <%= include_lua 'shared/index_helper_methods' %>
23
+
24
+ -- Validate input to script before making Redis db calls
25
+ if #KEYS ~= 2 then
26
+ error('Expected keys to be of size 2')
27
+ end
28
+
29
+ local id, hash_tag = unpack(KEYS)
30
+ local model, ttl = unpack(ARGV)
31
+ local key = model .. ':id:' .. id
32
+
33
+ local index_attr_pos = 6
34
+ local range_attr_pos = index_attr_pos + ARGV[3]
35
+ local custom_attr_pos = range_attr_pos + ARGV[4]
36
+ -- Starting position of the attr_key-attr_val pairs
37
+ local attr_pos = custom_attr_pos + ARGV[5]
38
+
39
+
40
+ if redis.call('exists', key) ~= 0 then
41
+ error(key .. ' already exists')
42
+ end
43
+
44
+ -- Forward the script arguments to the Redis command HSET.
45
+ -- Call the Redis command: HSET "#{Model.name}:id:#{id}" field value ...
46
+ redis.call('hset', key, unpack(ARGV, attr_pos))
47
+
48
+ -- Set TTL on key
49
+ if ttl and ttl ~= '-1' then
50
+ redis.call('expire', key, ttl)
51
+ end
52
+
53
+ -- Add id value for any index and range index attributes
54
+ local attrs_hash = to_hash(unpack(ARGV, attr_pos))
55
+ local index_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
56
+ if #index_attr_keys > 0 then
57
+ for _, attr_key in ipairs(index_attr_keys) do
58
+ add_id_to_index_attr(hash_tag, model, attr_key, attrs_hash[attr_key], id)
59
+ end
60
+ end
61
+ local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}
62
+ if #range_index_attr_keys > 0 then
63
+ for _, attr_key in ipairs(range_index_attr_keys) do
64
+ add_id_to_range_index_attr(hash_tag, model, attr_key, attrs_hash[attr_key], id)
65
+ end
66
+ end
67
+
68
+ -- Add a record to every custom index
69
+ local custom_index_attr_keys = {unpack(ARGV, custom_attr_pos, attr_pos - 1)}
70
+ local i = 1
71
+ while i < #custom_index_attr_keys do
72
+ local index_name, attrs_num = custom_index_attr_keys[i], custom_index_attr_keys[i+1]
73
+ local attr_values = {}
74
+ for j, attr_key in ipairs({unpack(custom_index_attr_keys, i + 2, i + attrs_num + 1)}) do
75
+ attr_values[j] = attrs_hash[attr_key]
76
+ end
77
+ add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
78
+ i = i + 2 + attrs_num
79
+ end
80
+
81
+ return nil