redcord 0.0.1.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,129 @@
1
+ require 'redcord/range_interval'
2
+ # typed: strict
3
+ #
4
+ # This module defines various helper methods on Redcord for serialization between the
5
+ # Ruby client and Redis server.
6
+ module Redcord
7
+ # Raised by Model.where
8
+ class AttributeNotIndexed < StandardError; end
9
+ class WrongAttributeType < TypeError; end
10
+ end
11
+
12
+ module Redcord::Serializer
13
+ extend T::Sig
14
+
15
+ sig { params(klass: T.any(Module, T.class_of(T::Struct))).void }
16
+ def self.included(klass)
17
+ klass.extend(ClassMethods)
18
+ end
19
+
20
+ module ClassMethods
21
+ extend T::Sig
22
+
23
+ # Redis only allows range queries on floats. To allow range queries on the Ruby Time
24
+ # type, encode_attr_value and decode_attr_value will implicitly encode and decode
25
+ # Time attributes to a float.
26
+ TIME_TYPES = T.let(Set[Time, T.nilable(Time)], T::Set[T.untyped])
27
+ sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
28
+ def encode_attr_value(attribute, val)
29
+ if val && TIME_TYPES.include?(props[attribute][:type])
30
+ val = val.to_f
31
+ end
32
+ val
33
+ end
34
+
35
+ sig { params(attribute: Symbol, val: T.untyped).returns(T.untyped) }
36
+ def decode_attr_value(attribute, val)
37
+ if val && TIME_TYPES.include?(props[attribute][:type])
38
+ val = Time.zone.at(val.to_f)
39
+ end
40
+ val
41
+ end
42
+
43
+ sig { params(attr_key: Symbol, attr_val: T.untyped).returns(T.untyped)}
44
+ def validate_and_encode_query(attr_key, attr_val)
45
+ # Validate that attributes queried for are index attributes
46
+ if !class_variable_get(:@@index_attributes).include?(attr_key) &&
47
+ !class_variable_get(:@@range_index_attributes).include?(attr_key)
48
+ raise Redcord::AttributeNotIndexed.new(
49
+ "#{attr_key} is not an indexed attribute."
50
+ )
51
+ end
52
+ # Validate attribute types for normal index attributes
53
+ attr_type = get_attr_type(attr_key)
54
+ if class_variable_get(:@@index_attributes).include?(attr_key)
55
+ validate_attr_type(attr_val, attr_type)
56
+ else
57
+ # Validate attribute types for range index attributes
58
+ if attr_val.is_a?(Redcord::RangeInterval)
59
+ validate_attr_type(attr_val.min, T.cast(T.nilable(attr_type), T::Types::Base))
60
+ validate_attr_type(attr_val.max, T.cast(T.nilable(attr_type), T::Types::Base))
61
+ else
62
+ validate_attr_type(attr_val, attr_type)
63
+ end
64
+ # Range index attributes need to be further encoded into a format understood by the Lua script.
65
+ if attr_val != nil
66
+ attr_val = encode_range_index_attr_val(attr_key, attr_val)
67
+ end
68
+ end
69
+ attr_val
70
+ end
71
+
72
+ sig { params(attr_val: T.untyped, attr_type: T.any(Class, T::Types::Base)).void }
73
+ def validate_attr_type(attr_val, attr_type)
74
+ if (attr_type.is_a?(Class) && !attr_val.is_a?(attr_type)) ||
75
+ (attr_type.is_a?(T::Types::Base) && !attr_type.valid?(attr_val))
76
+ raise Redcord::WrongAttributeType.new(
77
+ "Expected type #{attr_type}, got #{attr_val.class}"
78
+ )
79
+ end
80
+ end
81
+
82
+ sig { params(attribute: Symbol, val: T.untyped).returns([T.untyped, T.untyped]) }
83
+ def encode_range_index_attr_val(attribute, val)
84
+ if val.is_a?(Redcord::RangeInterval)
85
+ # nil is treated as -inf and +inf. This is supported in Redis sorted sets
86
+ # so clients aren't required to know the highest and lowest scores in a range
87
+ min_val = !val.min ? '-inf' : encode_attr_value(attribute, val.min)
88
+ max_val = !val.max ? '+inf' : encode_attr_value(attribute, val.max)
89
+
90
+ # In Redis, by default min and max is closed. You can prefix the score with '(' to
91
+ # specify an open interval.
92
+ min_val = val.min_exclusive ? '(' + min_val.to_s : min_val.to_s
93
+ max_val = val.max_exclusive ? '(' + max_val.to_s : max_val.to_s
94
+ return [min_val, max_val]
95
+ else
96
+ # Equality queries for range indices are be passed to redis as a range [val, val].
97
+ encoded_val = encode_attr_value(attribute, val)
98
+ [encoded_val, encoded_val]
99
+ end
100
+ end
101
+
102
+ sig { params(attr_key: Symbol).returns(T.any(Class, T::Types::Base)) }
103
+ def get_attr_type(attr_key)
104
+ props[attr_key][:type_object]
105
+ end
106
+
107
+ sig { params(redis_hash: T::Hash[T.untyped, T.untyped], id: Integer).returns(T.untyped) }
108
+ def coerce_and_set_id(redis_hash, id)
109
+ # Coerce each serialized result returned from Redis back into Model instance
110
+ instance = TypeCoerce.send(:[], self).new.from(from_redis_hash(redis_hash))
111
+ instance.send(:id=, id)
112
+ instance
113
+ end
114
+ sig { returns(String) }
115
+ def model_key
116
+ "Redcord:#{name}"
117
+ end
118
+
119
+ sig { params(args: T::Hash[T.any(String, Symbol), T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
120
+ def to_redis_hash(args)
121
+ args.map { |key, val| [key.to_sym, encode_attr_value(key.to_sym, val)] }.to_h
122
+ end
123
+
124
+ sig { params(args: T::Hash[T.untyped, T.untyped]).returns(T::Hash[T.untyped, T.untyped]) }
125
+ def from_redis_hash(args)
126
+ args.map { |key, val| [key, decode_attr_value(key.to_sym, val)] }.to_h
127
+ end
128
+ end
129
+ end
@@ -0,0 +1,78 @@
1
+ # typed: strict
2
+ module Redcord::ServerScripts
3
+ extend T::Sig
4
+
5
+ sig do
6
+ params(
7
+ key: T.any(String, Symbol),
8
+ args: T::Hash[T.untyped, T.untyped],
9
+ ).returns(Integer)
10
+ end
11
+ def create_hash_returning_id(key, args)
12
+ evalsha(
13
+ T.must(redcord_server_script_shas[:create_hash_returning_id]),
14
+ keys: [key],
15
+ argv: args.to_a.flatten,
16
+ ).to_i
17
+ end
18
+
19
+ sig do
20
+ params(
21
+ model: String,
22
+ id: Integer,
23
+ args: T::Hash[T.untyped, T.untyped],
24
+ ).void
25
+ end
26
+ def update_hash(model, id, args)
27
+ evalsha(
28
+ T.must(redcord_server_script_shas[:update_hash]),
29
+ keys: [model, id],
30
+ argv: args.to_a.flatten,
31
+ )
32
+ end
33
+
34
+ sig do
35
+ params(
36
+ model: String,
37
+ id: Integer
38
+ ).returns(Integer)
39
+ end
40
+ def delete_hash(model, id)
41
+ evalsha(
42
+ T.must(redcord_server_script_shas[:delete_hash]),
43
+ keys: [model, id]
44
+ )
45
+ end
46
+
47
+ sig do
48
+ params(
49
+ model: String,
50
+ query_conditions: T::Hash[T.untyped, T.untyped],
51
+ select_attrs: T::Set[Symbol]
52
+ ).returns(T::Hash[Integer, T::Hash[T.untyped, T.untyped]])
53
+ end
54
+ def find_by_attr(model, query_conditions, select_attrs=Set.new)
55
+ res = evalsha(
56
+ T.must(redcord_server_script_shas[:find_by_attr]),
57
+ keys: [model] + query_conditions.to_a.flatten,
58
+ argv: select_attrs.to_a.flatten
59
+ )
60
+ # The Lua script will return this as a flattened array.
61
+ # Convert the result into a hash of {id -> model hash}
62
+ res_hash = res.each_slice(2)
63
+ res_hash.map { |key, val| [key.to_i, val.each_slice(2).to_h] }.to_h
64
+ end
65
+
66
+ sig do
67
+ params(
68
+ model: String,
69
+ query_conditions: T::Hash[T.untyped, T.untyped]
70
+ ).returns(Integer)
71
+ end
72
+ def find_by_attr_count(model, query_conditions)
73
+ evalsha(
74
+ T.must(redcord_server_script_shas[:find_by_attr_count]),
75
+ keys: [model] + query_conditions.to_a.flatten,
76
+ )
77
+ end
78
+ end
@@ -0,0 +1,68 @@
1
+ --[[
2
+ EVALSHA SHA1(__FILE__) [field value ...]
3
+ > Time complexity: O(N) where N is the number of fields being set.
4
+
5
+ Create a hash with the specified fields to their respective values stored at
6
+ key when key does not exist.
7
+
8
+ # Return value
9
+ The id of the created hash as a string.
10
+ --]]
11
+
12
+ -- The arguments can be accessed by Lua using the KEYS global variable in the
13
+ -- form of a one-based array (so KEYS[1], KEYS[2], ...).
14
+ -- All the additional arguments should not represent key names and can be
15
+ -- accessed by Lua using the ARGV global variable, very similarly to what
16
+ -- happens with keys (so ARGV[1], ARGV[2], ...).
17
+
18
+ -- KEYS[1] = Model.name
19
+ -- ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
20
+ <%= include_lua 'shared/lua_helper_methods' %>
21
+ <%= include_lua 'shared/index_helper_methods' %>
22
+
23
+ -- Validate input to script before making Redis db calls
24
+ if #KEYS ~= 1 then
25
+ error('Expected keys to be of size 1')
26
+ end
27
+ if #ARGV % 2 ~= 0 then
28
+ error('Expected an even number of arguments')
29
+ end
30
+
31
+ local model = KEYS[1]
32
+
33
+ -- Call the Redis command: INCR "#{Model.name}:id_seq". If "#{Model.name}:id_seq" does
34
+ -- not exist, the command returns 0. It errors if the id_seq overflows a 64 bit
35
+ -- signed integer.
36
+ redis.call('incr', model .. ':id_seq')
37
+
38
+ -- The Lua version used by Redis does not support 64 bit integers:
39
+ -- https://github.com/antirez/redis/issues/5261
40
+ -- We ignore the integer response from INCR and use the string response from
41
+ -- the GET/MGET command.
42
+ local id, ttl = unpack(redis.call('mget', model .. ':id_seq', model .. ':ttl'))
43
+ local key = model .. ':id:' .. id
44
+
45
+ -- Forward the script arguments to the Redis command HSET.
46
+ -- Call the Redis command: HSET "#{Model.name}:id:#{id}" field value ...
47
+ redis.call('hset', key, unpack(ARGV))
48
+
49
+ -- Set TTL on key
50
+ if ttl and ttl ~= '-1' then
51
+ redis.call('expire', key, ttl)
52
+ end
53
+
54
+ -- Add id value for any index and range index attributes
55
+ local attrs_hash = to_hash(ARGV)
56
+ local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
57
+ if #index_attr_keys > 0 then
58
+ for _, attr_key in ipairs(index_attr_keys) do
59
+ add_id_to_index_attr(model, attr_key, attrs_hash[attr_key], id)
60
+ end
61
+ end
62
+ local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
63
+ if #range_index_attr_keys > 0 then
64
+ for _, attr_key in ipairs(range_index_attr_keys) do
65
+ add_id_to_range_index_attr(model, attr_key, attrs_hash[attr_key], id)
66
+ end
67
+ end
68
+ return id
@@ -0,0 +1,48 @@
1
+ --[[
2
+ EVALSHA SHA1(__FILE__) model id
3
+ > Time complexity: O(1)
4
+
5
+ Delete a hash at "#model:id:#id", and the corresponding id from the indexed
6
+ attribute id sets.
7
+
8
+ # Return value
9
+ The number of keys deleted from Redis
10
+ --]]
11
+
12
+ -- The arguments can be accessed by Lua using the KEYS global variable in the
13
+ -- form of a one-based array (so KEYS[1], KEYS[2], ...).
14
+ --
15
+ -- KEYS[1] = Model.name
16
+ -- KEYS[2] = id
17
+ <%= include_lua 'shared/index_helper_methods' %>
18
+
19
+ -- Validate input to script before making Redis db calls
20
+ if #KEYS ~= 2 then
21
+ error('Expected keys of be of size 2')
22
+ end
23
+
24
+ local model = KEYS[1]
25
+ local id = KEYS[2]
26
+
27
+ -- key = "#{model}:id:{id}"
28
+ local key = model .. ':id:' .. id
29
+
30
+ -- Clean up id sets for both index and range index attributes
31
+ local index_attr_keys = redis.call('smembers', model .. ':index_attrs')
32
+ if #index_attr_keys > 0 then
33
+ -- Retrieve old index attr values so we can delete them in the attribute id sets
34
+ local attr_vals = redis.call('hmget', key, unpack(index_attr_keys))
35
+ for i=1, #index_attr_keys do
36
+ delete_id_from_index_attr(model, index_attr_keys[i], attr_vals[i], id)
37
+ end
38
+ end
39
+ local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
40
+ if #range_index_attr_keys > 0 then
41
+ local attr_vals = redis.call('hmget', key, unpack(range_index_attr_keys))
42
+ for i=1, #range_index_attr_keys do
43
+ delete_id_from_range_index_attr(model, range_index_attr_keys[i], attr_vals[i], id)
44
+ end
45
+ end
46
+
47
+ -- delete the actual key
48
+ return redis.call('del', key)
@@ -0,0 +1,67 @@
1
+ --[[
2
+ EVALSHA SHA1(__FILE__) model id
3
+ > Time complexity: O(N) where N is the number of ids with these attributes
4
+
5
+ Query for all model instances that have the given attribute values or value ranges.
6
+ Return an error if an attribute is not an index.
7
+
8
+ # Return value
9
+ A hash of id:model of all the ids that match the query conditions given.
10
+ --]]
11
+
12
+ -- The arguments can be accessed by Lua using the KEYS global variable in the
13
+ -- form of a one-based array (so KEYS[1], KEYS[2], ...).
14
+ -- All the additional arguments should not represent key names and can be
15
+ -- accessed by Lua using the ARGV global variable, very similarly to what
16
+ -- happens with keys (so ARGV[1], ARGV[2], ...).
17
+ --
18
+ -- KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
19
+ -- ARGV[1...N] = attr_key [attr_key ..]
20
+ --
21
+ -- For equality query conditions, key value pairs are expected to appear in
22
+ -- the KEYS array as [attr_key, attr_val]
23
+ -- For range query conditions, key value pairs are expected to appear in the
24
+ -- KEYS array as [key min_val max_val]
25
+ --
26
+ -- The ARGV array is used to specify specific fields to select for each record. If
27
+ -- the ARGV array is empty, then all fields will be retrieved.
28
+
29
+ <%= include_lua 'shared/lua_helper_methods' %>
30
+ <%= include_lua 'shared/query_helper_methods' %>
31
+
32
+ if #KEYS < 3 then
33
+ error('Expected keys to be at least of size 3')
34
+ end
35
+
36
+ local model = KEYS[1]
37
+ local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
38
+
39
+ -- Get all ids which have the corresponding attribute values.
40
+ local ids_set = nil
41
+ -- For normal sets, Redis has SINTER built in to return the set intersection
42
+ if #index_sets > 0 then
43
+ ids_set = to_set(redis.call('sinter', unpack(index_sets)))
44
+ end
45
+ -- For sorted sets, call helper function zinter_zrangebyscore, which calls
46
+ -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
47
+ if #range_index_sets > 0 then
48
+ ids_set = intersect_range_index_sets(ids_set, range_index_sets)
49
+ end
50
+
51
+ -- Query for the hashes for all ids in the set intersection
52
+ local res, stale_ids = unpack(batch_hget(model, ids_set, ARGV))
53
+
54
+ -- Delete any stale ids which are no longer in redis from the id sets.
55
+ -- This can happen if an entry was auto expired due to ttl, but not removed up yet
56
+ -- from the id sets.
57
+ if #stale_ids > 0 then
58
+ for _, key in ipairs(index_sets) do
59
+ redis.call('srem', key, unpack(stale_ids))
60
+ end
61
+ for _, key in ipairs(range_index_sets) do
62
+ local redis_key = key[1]
63
+ redis.call('zrem', redis_key, unpack(stale_ids))
64
+ end
65
+ end
66
+
67
+ return res
@@ -0,0 +1,52 @@
1
+ --[[
2
+ EVALSHA SHA1(__FILE__) model id
3
+ > Time complexity: O(N) where N is the number of ids with these attributes
4
+
5
+ Query for all model instances that have the given attribute values or value ranges.
6
+ Return an error if an attribute is not an index.
7
+
8
+ # Return value
9
+ An integer number of records that match the query conditions given.
10
+ --]]
11
+
12
+ -- The arguments can be accessed by Lua using the KEYS global variable in the
13
+ -- form of a one-based array (so KEYS[1], KEYS[2], ...).
14
+ -- All the additional arguments should not represent key names and can be
15
+ -- accessed by Lua using the ARGV global variable, very similarly to what
16
+ -- happens with keys (so ARGV[1], ARGV[2], ...).
17
+ --
18
+ -- KEYS[1] = Model.name attr_key attr_val [attr_key attr_val ..]
19
+ --
20
+ -- For equality query conditions, key value pairs are expected to appear in
21
+ -- the KEYS array as [attr_key, attr_val]
22
+ -- For range query conditions, key value pairs are expected to appear in the
23
+ -- KEYS array as [key min_val max_val]
24
+
25
+ <%= include_lua 'shared/lua_helper_methods' %>
26
+ <%= include_lua 'shared/query_helper_methods' %>
27
+
28
+ if #KEYS < 3 then
29
+ error('Expected keys to be at least of size 3')
30
+ end
31
+
32
+ local model = KEYS[1]
33
+ local index_sets, range_index_sets = unpack(validate_and_parse_query_conditions(model, KEYS))
34
+
35
+ -- Get all ids which have the corresponding attribute values.
36
+ local ids_set = nil
37
+ -- For normal sets, Redis has SINTER built in to return the set intersection
38
+ if #index_sets > 0 then
39
+ ids_set = to_set(redis.call('sinter', unpack(index_sets)))
40
+ end
41
+ -- For sorted sets, call helper function zinter_zrangebyscore, which calls
42
+ -- ZRANGEBYSCORE for each {redis_key, min, max} tuple and returns the set intersection
43
+ if #range_index_sets > 0 then
44
+ ids_set = intersect_range_index_sets(ids_set, range_index_sets)
45
+ end
46
+
47
+ -- Get the number of records which satisfy the query conditions.
48
+ -- We do not delete stale ids as part of this function call because we do not have
49
+ -- the list of ids which don't exist. The Redis command EXISTS key [key ...] is an O(1)
50
+ -- operation that only returns the count of ids that exist. Getting the list of ids that
51
+ -- don't exist would be an O(N) operation.
52
+ return batch_exists(model, ids_set)