redcord 0.0.3 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,6 +8,8 @@ module Redcord
8
8
  # Raised by Model.where
9
9
  class AttributeNotIndexed < StandardError; end
10
10
  class WrongAttributeType < TypeError; end
11
+ class CustomIndexInvalidQuery < StandardError; end
12
+ class CustomIndexInvalidDesign < StandardError; end
11
13
  end
12
14
 
13
15
  # This module defines various helper methods on Redcord for serialization
@@ -31,50 +33,36 @@ module Redcord::Serializer
31
33
  sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
32
34
  def encode_attr_value(attribute, val)
33
35
  if !val.blank? && TIME_TYPES.include?(props[attribute][:type])
34
- val = val.to_f
36
+ time_in_nano_sec = val.to_i * 1_000_000_000
37
+ time_in_nano_sec >= 0 ? time_in_nano_sec + val.nsec : time_in_nano_sec - val.nsec
38
+ elsif val.is_a?(Float)
39
+ # Encode as round-trippable float64
40
+ '%1.16e' % [val]
41
+ else
42
+ val
35
43
  end
36
-
37
- val
38
44
  end
39
45
 
40
46
  sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
41
47
  def decode_attr_value(attribute, val)
42
48
  if !val.blank? && TIME_TYPES.include?(props[attribute][:type])
43
- val = Time.zone.at(val.to_f)
44
- end
49
+ val = val.to_i
50
+ nsec = val >= 0 ? val % 1_000_000_000 : -val % 1_000_000_000
45
51
 
46
- val
52
+ Time.zone.at(val / 1_000_000_000).change(nsec: nsec)
53
+ else
54
+ val
55
+ end
47
56
  end
48
57
 
49
58
  sig { params(attr_key: Symbol, attr_val: T.untyped).returns(T.untyped)}
50
- def validate_and_encode_query(attr_key, attr_val)
51
- # Validate that attributes queried for are index attributes
52
- if !class_variable_get(:@@index_attributes).include?(attr_key) &&
53
- !class_variable_get(:@@range_index_attributes).include?(attr_key)
54
- raise(
55
- Redcord::AttributeNotIndexed,
56
- "#{attr_key} is not an indexed attribute.",
57
- )
58
- end
59
-
60
- # Validate attribute types for normal index attributes
59
+ def validate_types_and_encode_query(attr_key, attr_val)
60
+ # Validate attribute types for index attributes
61
61
  attr_type = get_attr_type(attr_key)
62
- if class_variable_get(:@@index_attributes).include?(attr_key)
62
+ if class_variable_get(:@@index_attributes).include?(attr_key) || attr_key == shard_by_attribute
63
63
  validate_attr_type(attr_val, attr_type)
64
64
  else
65
- # Validate attribute types for range index attributes
66
- if attr_val.is_a?(Redcord::RangeInterval)
67
- validate_attr_type(
68
- attr_val.min,
69
- T.cast(T.nilable(attr_type), T::Types::Base),
70
- )
71
- validate_attr_type(
72
- attr_val.max,
73
- T.cast(T.nilable(attr_type), T::Types::Base),
74
- )
75
- else
76
- validate_attr_type(attr_val, attr_type)
77
- end
65
+ validate_range_attr_types(attr_val, attr_type)
78
66
 
79
67
  # Range index attributes need to be further encoded into a format
80
68
  # understood by the Lua script.
@@ -82,10 +70,73 @@ module Redcord::Serializer
82
70
  attr_val = encode_range_index_attr_val(attr_key, attr_val)
83
71
  end
84
72
  end
85
-
86
73
  attr_val
87
74
  end
88
75
 
76
+ # Validate that attributes queried for are index attributes
77
+ # For custom index: validate that attributes are present in specified index
78
+ sig { params(attr_keys: T::Array[Symbol], custom_index_name: T.nilable(Symbol)).void}
79
+ def validate_index_attributes(attr_keys, custom_index_name: nil)
80
+ custom_index_attributes = class_variable_get(:@@custom_index_attributes)[custom_index_name]
81
+ attr_keys.each do |attr_key|
82
+ next if attr_key == shard_by_attribute
83
+
84
+ if !custom_index_attributes.empty?
85
+ if !custom_index_attributes.include?(attr_key)
86
+ raise(
87
+ Redcord::AttributeNotIndexed,
88
+ "#{attr_key} is not a part of #{custom_index_name} index.",
89
+ )
90
+ end
91
+ else
92
+ if !class_variable_get(:@@index_attributes).include?(attr_key) &&
93
+ !class_variable_get(:@@range_index_attributes).include?(attr_key)
94
+ raise(
95
+ Redcord::AttributeNotIndexed,
96
+ "#{attr_key} is not an indexed attribute.",
97
+ )
98
+ end
99
+ end
100
+ end
101
+ end
102
+
103
+ # Validate exclusive ranges not used; Change all query conditions to range form;
104
+ # The position of the attribute and type of query is validated on Lua side
105
+ sig { params(query_conditions: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped])}
106
+ def validate_and_adjust_custom_index_query_conditions(query_conditions)
107
+ adjusted_query_conditions = query_conditions.clone
108
+ query_conditions.each do |attr_key, condition|
109
+ if !condition.is_a?(Array)
110
+ adjusted_query_conditions[attr_key] = [condition, condition]
111
+ elsif condition[0].to_s[0] == '(' or condition[1].to_s[0] == '('
112
+ raise(Redcord::CustomIndexInvalidQuery, "Custom index doesn't support exclusive ranges")
113
+ end
114
+ end
115
+ adjusted_query_conditions
116
+ end
117
+
118
+ sig {
119
+ params(
120
+ attr_val: T.untyped,
121
+ attr_type: T.any(Class, T::Types::Base),
122
+ ).void
123
+ }
124
+ def validate_range_attr_types(attr_val, attr_type)
125
+ # Validate attribute types for range index attributes
126
+ if attr_val.is_a?(Redcord::RangeInterval)
127
+ validate_attr_type(
128
+ attr_val.min,
129
+ T.cast(T.nilable(attr_type), T::Types::Base),
130
+ )
131
+ validate_attr_type(
132
+ attr_val.max,
133
+ T.cast(T.nilable(attr_type), T::Types::Base),
134
+ )
135
+ else
136
+ validate_attr_type(attr_val, attr_type)
137
+ end
138
+ end
139
+
89
140
  sig {
90
141
  params(
91
142
  attr_val: T.untyped,
@@ -137,7 +188,7 @@ module Redcord::Serializer
137
188
  sig {
138
189
  params(
139
190
  redis_hash: T::Hash[T.untyped, T.untyped],
140
- id: Integer,
191
+ id: String,
141
192
  ).returns(T.untyped)
142
193
  }
143
194
  def coerce_and_set_id(redis_hash, id)
@@ -0,0 +1,81 @@
1
+ --[[
2
+ EVALSHA SHA1(__FILE__) [field value ...]
3
+ > Time complexity: O(N) where N is the number of fields being set.
4
+
5
+ Create a hash with the specified fields to their respective values stored at
6
+ key when key does not exist.
7
+
8
+ # Return value
9
+ The id of the created hash as a string.
10
+ --]]
11
+
12
+ -- The arguments can be accessed by Lua using the KEYS global variable in the
13
+ -- form of a one-based array (so KEYS[1], KEYS[2], ...).
14
+ -- All the additional arguments should not represent key names and can be
15
+ -- accessed by Lua using the ARGV global variable, very similarly to what
16
+ -- happens with keys (so ARGV[1], ARGV[2], ...).
17
+
18
+ -- KEYS = id hash_tag
19
+ -- ARGV = Model.name ttl index_attr_size range_index_attr_size custom_index_attrs_flat_size [index_attr_key ...] [range_index_attr_key ...]
20
+ -- [custom_index_name attrs_size [custom_index_attr_key ...] ...] attr_key attr_val [attr_key attr_val ..]
21
+ <%= include_lua 'shared/lua_helper_methods' %>
22
+ <%= include_lua 'shared/index_helper_methods' %>
23
+
24
+ -- Validate input to script before making Redis db calls
25
+ if #KEYS ~= 2 then
26
+ error('Expected keys to be of size 2')
27
+ end
28
+
29
+ local id, hash_tag = unpack(KEYS)
30
+ local model, ttl = unpack(ARGV)
31
+ local key = model .. ':id:' .. id
32
+
33
+ local index_attr_pos = 6
34
+ local range_attr_pos = index_attr_pos + ARGV[3]
35
+ local custom_attr_pos = range_attr_pos + ARGV[4]
36
+ -- Starting position of the attr_key-attr_val pairs
37
+ local attr_pos = custom_attr_pos + ARGV[5]
38
+
39
+
40
+ if redis.call('exists', key) ~= 0 then
41
+ error(key .. ' already exists')
42
+ end
43
+
44
+ -- Forward the script arguments to the Redis command HSET.
45
+ -- Call the Redis command: HSET "#{Model.name}:id:#{id}" field value ...
46
+ redis.call('hset', key, unpack(ARGV, attr_pos))
47
+
48
+ -- Set TTL on key
49
+ if ttl and ttl ~= '-1' then
50
+ redis.call('expire', key, ttl)
51
+ end
52
+
53
+ -- Add id value for any index and range index attributes
54
+ local attrs_hash = to_hash(unpack(ARGV, attr_pos))
55
+ local index_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
56
+ if #index_attr_keys > 0 then
57
+ for _, attr_key in ipairs(index_attr_keys) do
58
+ add_id_to_index_attr(hash_tag, model, attr_key, attrs_hash[attr_key], id)
59
+ end
60
+ end
61
+ local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}
62
+ if #range_index_attr_keys > 0 then
63
+ for _, attr_key in ipairs(range_index_attr_keys) do
64
+ add_id_to_range_index_attr(hash_tag, model, attr_key, attrs_hash[attr_key], id)
65
+ end
66
+ end
67
+
68
+ -- Add a record to every custom index
69
+ local custom_index_attr_keys = {unpack(ARGV, custom_attr_pos, attr_pos - 1)}
70
+ local i = 1
71
+ while i < #custom_index_attr_keys do
72
+ local index_name, attrs_num = custom_index_attr_keys[i], custom_index_attr_keys[i+1]
73
+ local attr_values = {}
74
+ for j, attr_key in ipairs({unpack(custom_index_attr_keys, i + 2, i + attrs_num + 1)}) do
75
+ attr_values[j] = attrs_hash[attr_key]
76
+ end
77
+ add_record_to_custom_index(hash_tag, model, index_name, attr_values, id)
78
+ i = i + 2 + attrs_num
79
+ end
80
+
81
+ return nil
@@ -12,8 +12,8 @@ The number of keys deleted from Redis
12
12
  -- The arguments can be accessed by Lua using the KEYS global variable in the
13
13
  -- form of a one-based array (so KEYS[1], KEYS[2], ...).
14
14
  --
15
- -- KEYS[1] = Model.name
16
- -- KEYS[2] = id
15
+ -- KEYS = id, hash_tag
16
+ -- ARGV = Model.name index_attr_size range_index_attr_size [index_attr_key ...] [range_index_attr_key ...] [custom_index_name ...]
17
17
  <%= include_lua 'shared/index_helper_methods' %>
18
18
 
19
19
  -- Validate input to script before making Redis db calls
@@ -21,28 +21,37 @@ if #KEYS ~= 2 then
21
21
  error('Expected keys of be of size 2')
22
22
  end
23
23
 
24
- local model = KEYS[1]
25
- local id = KEYS[2]
24
+ local model = ARGV[1]
25
+ local id, hash_tag = unpack(KEYS)
26
26
 
27
27
  -- key = "#{model}:id:{id}"
28
28
  local key = model .. ':id:' .. id
29
29
 
30
+ local index_attr_pos = 4
31
+ local range_attr_pos = index_attr_pos + ARGV[2]
32
+ local custom_index_pos = range_attr_pos + ARGV[3]
33
+
30
34
  -- Clean up id sets for both index and range index attributes
31
- local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
35
+ local index_attr_keys = {unpack(ARGV, index_attr_pos, range_attr_pos - 1)}
32
36
  if #index_attr_keys > 0 then
33
37
  -- Retrieve old index attr values so we can delete them in the attribute id sets
34
38
  local attr_vals = redis.call('hmget', key, unpack(index_attr_keys))
35
39
  for i=1, #index_attr_keys do
36
- delete_id_from_index_attr(model, index_attr_keys[i], attr_vals[i], id)
40
+ delete_id_from_index_attr(hash_tag, model, index_attr_keys[i], attr_vals[i], id)
37
41
  end
38
42
  end
39
- local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
43
+ local range_index_attr_keys = {unpack(ARGV, range_attr_pos, custom_index_pos - 1)}
40
44
  if #range_index_attr_keys > 0 then
41
45
  local attr_vals = redis.call('hmget', key, unpack(range_index_attr_keys))
42
46
  for i=1, #range_index_attr_keys do
43
- delete_id_from_range_index_attr(model, range_index_attr_keys[i], attr_vals[i], id)
47
+ delete_id_from_range_index_attr(hash_tag, model, range_index_attr_keys[i], attr_vals[i], id)
44
48
  end
45
49
  end
50
+ -- Delete record from custom indexes
51
+ local custom_index_names = {unpack(ARGV, custom_index_pos)}
52
+ for _, index_name in ipairs(custom_index_names) do
53
+ delete_record_from_custom_index(hash_tag, model, index_name, id)
54
+ end
46
55
 
47
56
  -- delete the actual key
48
57
  return redis.call('del', key)
@@ -15,9 +15,11 @@ A hash of id:model of all the ids that match the query conditions given.
15
15
  -- accessed by Lua using the ARGV global variable, very similarly to what
16
16
  -- happens with keys (so ARGV[1], ARGV[2], ...).
17
17
  --
18
- -- KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
19
- -- ARGV[1...N] = attr_key [attr_key ..]
20
- --
18
+ -- KEYS[1] = hash_tag
19
+ -- ARGV = Model.name custom_index_name num_index_attr num_range_index_attr num_custom_index_attr num_query_conditions
20
+ -- [index_attrs ...] [range_index_attrs ...] [custom_index_attrs ...] [query_conidtions ...] [attr_selections ...]
21
+ -- [query_conidtions ...]: [attr_key1 attr_val1 attr_key2 attr_val2 ...]
22
+ -- [attr_selections ...]: [attr_key1 attr_key2 ...]
21
23
  -- For equality query conditions, key value pairs are expected to appear in
22
24
  -- the KEYS array as [attr_key, attr_val]
23
25
  -- For range query conditions, key value pairs are expected to appear in the
@@ -29,27 +31,60 @@ A hash of id:model of all the ids that match the query conditions given.
29
31
  <%= include_lua 'shared/lua_helper_methods' %>
30
32
  <%= include_lua 'shared/query_helper_methods' %>
31
33
 
32
- if #KEYS < 3 then
33
- error('Expected keys to be at least of size 3')
34
+ if #KEYS ~=1 then
35
+ error('Expected keys to be of size 1')
34
36
  end
35
37
 
36
- local model = KEYS[1]
37
- local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
38
+ local model = ARGV[1]
39
+
40
+ local index_name = ARGV[2]
41
+ local index_attr_pos = 7
42
+ local range_attr_pos = index_attr_pos + ARGV[3]
43
+ local custom_attr_pos = range_attr_pos + ARGV[4]
44
+ local query_cond_pos = custom_attr_pos + ARGV[5]
45
+ local attr_selection_pos = query_cond_pos + ARGV[6]
38
46
 
39
47
  -- Get all ids which have the corresponding attribute values.
40
48
  local ids_set = nil
41
- -- For normal sets, Redis has SINTER built in to return the set intersection
42
- if #index_sets > 0 then
43
- ids_set = to_set(redis.call('sinter', unpack(index_sets)))
44
- end
45
- -- For sorted sets, call helper function zinter_zrangebyscore, which calls
46
- -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
47
- if #range_index_sets > 0 then
48
- ids_set = intersect_range_index_sets(ids_set, range_index_sets)
49
+ local index_sets, range_index_sets = {}, {}
50
+
51
+ -- If custom index name is empty -> use single-attribute indices
52
+ if index_name == '' then
53
+ index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(
54
+ KEYS[1],
55
+ model,
56
+ to_set({unpack(ARGV, index_attr_pos, range_attr_pos - 1)}),
57
+ to_set({unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}),
58
+ unpack(ARGV, query_cond_pos, attr_selection_pos - 1)
59
+ ))
60
+
61
+ -- For normal sets, Redis has SINTER built in to return the set intersection
62
+ if #index_sets > 0 then
63
+ ids_set = to_set(redis.call('sinter', unpack(index_sets)))
64
+ end
65
+ -- For sorted sets, call helper function zinter_zrangebyscore, which calls
66
+ -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
67
+ if #range_index_sets > 0 then
68
+ ids_set = intersect_range_index_sets(ids_set, range_index_sets)
69
+ end
70
+ else
71
+ local custom_index_attrs = {unpack(ARGV, custom_attr_pos, query_cond_pos - 1)}
72
+ local custom_index_query = validate_and_parse_query_conditions_custom(
73
+ KEYS[1],
74
+ model,
75
+ index_name,
76
+ custom_index_attrs,
77
+ {unpack(ARGV, query_cond_pos, attr_selection_pos - 1)}
78
+ )
79
+ if #custom_index_query > 0 then
80
+ ids_set = get_id_set_from_custom_index(ids_set, custom_index_query)
81
+ else
82
+ ids_set = {}
83
+ end
49
84
  end
50
85
 
51
86
  -- Query for the hashes for all ids in the set intersection
52
- local res, stale_ids = unpack(batch_hget(model, ids_set, ARGV))
87
+ local res, stale_ids = unpack(batch_hget(model, ids_set, unpack(ARGV, attr_selection_pos)))
53
88
 
54
89
  -- Delete any stale ids which are no longer in redis from the id sets.
55
90
  -- This can happen if an entry was auto expired due to ttl, but not removed up yet
@@ -15,7 +15,9 @@ An integer number of records that match the query conditions given.
15
15
  -- accessed by Lua using the ARGV global variable, very similarly to what
16
16
  -- happens with keys (so ARGV[1], ARGV[2], ...).
17
17
  --
18
- -- KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
18
+ -- KEYS[1] = hash_tag
19
+ -- ARGV = Model.name custom_index_name num_index_attr num_range_index_attr num_custom_index_attr
20
+ -- [index_attrs ...] [range_index_attrs ...] [custom_index_attrs ...] [query_conidtions ...]
19
21
  --
20
22
  -- For equality query conditions, key value pairs are expected to appear in
21
23
  -- the KEYS array as [attr_key, attr_val]
@@ -25,25 +27,54 @@ An integer number of records that match the query conditions given.
25
27
  <%= include_lua 'shared/lua_helper_methods' %>
26
28
  <%= include_lua 'shared/query_helper_methods' %>
27
29
 
28
- if #KEYS < 3 then
29
- error('Expected keys to be at least of size 3')
30
+ if #KEYS ~=1 then
31
+ error('Expected keys to be of size 1')
30
32
  end
31
33
 
32
- local model = KEYS[1]
33
- local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
34
+ local model = ARGV[1]
35
+
36
+ local index_name = ARGV[2]
37
+ local index_attr_pos = 6
38
+ local range_attr_pos = index_attr_pos + ARGV[3]
39
+ local custom_attr_pos = range_attr_pos + ARGV[4]
40
+ local query_cond_pos = custom_attr_pos + ARGV[5]
34
41
 
35
42
  -- Get all ids which have the corresponding attribute values.
36
43
  local ids_set = nil
37
- -- For normal sets, Redis has SINTER built in to return the set intersection
38
- if #index_sets > 0 then
39
- ids_set = to_set(redis.call('sinter', unpack(index_sets)))
40
- end
41
- -- For sorted sets, call helper function zinter_zrangebyscore, which calls
42
- -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
43
- if #range_index_sets > 0 then
44
- ids_set = intersect_range_index_sets(ids_set, range_index_sets)
45
- end
46
44
 
45
+ if index_name == '' then
46
+ local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(
47
+ KEYS[1],
48
+ model,
49
+ to_set({unpack(ARGV, index_attr_pos, range_attr_pos - 1)}),
50
+ to_set({unpack(ARGV, range_attr_pos, custom_attr_pos - 1)}),
51
+ unpack(ARGV, query_cond_pos)
52
+ ))
53
+
54
+ -- For normal sets, Redis has SINTER built in to return the set intersection
55
+ if #index_sets > 0 then
56
+ ids_set = to_set(redis.call('sinter', unpack(index_sets)))
57
+ end
58
+ -- For sorted sets, call helper function zinter_zrangebyscore, which calls
59
+ -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
60
+ if #range_index_sets > 0 then
61
+ ids_set = intersect_range_index_sets(ids_set, range_index_sets)
62
+ end
63
+ else
64
+ local custom_index_attrs = {unpack(ARGV, custom_attr_pos, query_cond_pos - 1)}
65
+ local custom_index_query = validate_and_parse_query_conditions_custom(
66
+ KEYS[1],
67
+ model,
68
+ index_name,
69
+ custom_index_attrs,
70
+ {unpack(ARGV, query_cond_pos)}
71
+ )
72
+ if #custom_index_query > 0 then
73
+ ids_set = get_id_set_from_custom_index(ids_set, custom_index_query)
74
+ else
75
+ ids_set = {}
76
+ end
77
+ end
47
78
  -- Get the number of records which satisfy the query conditions.
48
79
  -- We do not delete stale ids as part of this function call because we do not have
49
80
  -- the list of ids which don't exist. The Redis command EXISTS key [key ...] is an O(1)