redcord 0.0.1.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,61 @@
1
+ -- Add an id to the id set of the index attribute
2
+ local function add_id_to_index_attr(model, attr_key, attr_val, id)
3
+ if attr_val then
4
+ -- Call the Redis command: SADD "#{Model.name}:#{attr_name}:#{attr_val}" member ..
5
+ redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val, id)
6
+ end
7
+ end
8
+
9
+ -- Remove an id from the id set of the index attribute
10
+ local function delete_id_from_index_attr(model, attr_key, attr_val, id)
11
+ if attr_val then
12
+ -- Call the Redis command: SREM "#{Model.name}:#{attr_name}:#{attr_val}" member ..
13
+ redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val, id)
14
+ end
15
+ end
16
+
17
+ -- Move an id from one id set to another for the index attribute
18
+ local function replace_id_in_index_attr(model, attr_key, prev_attr_val,curr_attr_val, id)
19
+ -- If previous and new value differs, then modify the id sets accordingly
20
+ if prev_attr_val ~= curr_attr_val then
21
+ delete_id_from_index_attr(model, attr_key, prev_attr_val, id)
22
+ add_id_to_index_attr(model, attr_key, curr_attr_val, id)
23
+ end
24
+ end
25
+
26
+ -- Add an id to the sorted id set of the range index attribute
27
+ local function add_id_to_range_index_attr(model, attr_key, attr_val, id)
28
+ if attr_val then
29
+ -- Nil values of range indices are sent to Redis as an empty string. They are stored
30
+ -- as a regular set at key "#{Model.name}:#{attr_name}:"
31
+ if attr_val == "" then
32
+ redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val, id)
33
+ else
34
+ -- Call the Redis command: ZADD "#{Model.name}:#{attr_name}" #{attr_val} member ..,
35
+ -- where attr_val is the score of the sorted set
36
+ redis.call('zadd', model .. ':' .. attr_key, attr_val, id)
37
+ end
38
+ end
39
+ end
40
+
41
+ -- Remove an id from the sorted id set of the range index attribute
42
+ local function delete_id_from_range_index_attr(model, attr_key, attr_val, id)
43
+ if attr_val then
44
+ -- Nil values of range indices are sent to Redis as an empty string. They are stored
45
+ -- as a regular set at key "#{Model.name}:#{attr_name}:"
46
+ if attr_val == "" then
47
+ redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val, id)
48
+ else
49
+ -- Call the Redis command: ZREM "#{Model.name}:#{attr_name}:#{attr_val}" member ..
50
+ redis.call('zrem', model .. ':' .. attr_key, id)
51
+ end
52
+ end
53
+ end
54
+
55
+ -- Move an id from one sorted id set to another for the range index attribute
56
+ local function replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
57
+ if prev_attr_val ~= curr_attr_val then
58
+ delete_id_from_range_index_attr(model, attr_key, prev_attr_val, id)
59
+ add_id_to_range_index_attr(model, attr_key, curr_attr_val, id)
60
+ end
61
+ end
@@ -0,0 +1,33 @@
1
+ -- Helper function to convert argument array to hash set
2
+ local function to_hash(list)
3
+ local hash = {}
4
+ if not list then return hash end
5
+ for i=1, #list, 2 do
6
+ hash[list[i]] = list[i+1]
7
+ end
8
+ return hash
9
+ end
10
+
11
+ -- Helper function to convert list to set
12
+ local function to_set(list)
13
+ local set = {}
14
+ if not list then return set end
15
+ for _, item in ipairs(list) do
16
+ set[item] = true
17
+ end
18
+ return set
19
+ end
20
+
21
+ -- Helper function to compute the intersection of the given set and list.
22
+ local function set_list_intersect(set, list)
23
+ -- A nil set means that no items have been added to the set yet. If so,
24
+ -- we can just return the given list as a set
25
+ if not set then return to_set(list) end
26
+ local set_intersect = {}
27
+ for _, item in ipairs(list) do
28
+ if set[item] then
29
+ set_intersect[item] = true
30
+ end
31
+ end
32
+ return set_intersect
33
+ end
@@ -0,0 +1,105 @@
1
+ -- Calls the Redis command: ZRANGEBYSCORE key min max
2
+ -- for each {key, min, max} given in the input arguments. Returns
3
+ -- the set intersection of the results
4
+ local function intersect_range_index_sets(set, tuples)
5
+ for _, redis_key in ipairs(tuples) do
6
+ local key, min, max = unpack(redis_key)
7
+ local ids = redis.call('zrangebyscore', key, min, max)
8
+ set = set_list_intersect(set, ids)
9
+ end
10
+ return set
11
+ end
12
+
13
+ -- Gets the hash of all the ids given. Returns the results in a
14
+ -- table, as well as any ids not found in Redis as a separate table
15
+ local function batch_hget(model, ids_set, fields)
16
+ local res, stale_ids = {}, {}
17
+ for id, _ in pairs(ids_set) do
18
+ local instance = nil
19
+ if fields and #fields > 0 then
20
+ local values = redis.call('hmget', model .. ':id:' .. id, unpack(fields))
21
+ -- HMGET returns the value in the order of the fields given. Map back to
22
+ -- field value [field value ..]
23
+ instance = {}
24
+ for i, field in ipairs(fields) do
25
+ if not values[i] then
26
+ instance = nil
27
+ break
28
+ end
29
+ table.insert(instance, field)
30
+ table.insert(instance, values[i])
31
+ end
32
+ else
33
+ -- HGETALL returns the value as field value [field value ..]
34
+ instance = redis.call('hgetall', model .. ':id:' .. id)
35
+ end
36
+ -- Only add to result if entry is not stale (if query to hgetall is not empty)
37
+ if instance and #instance > 0 then
38
+ -- We cannot return a Lua table to Redis as a hash. Return result as a flattened
39
+ -- array instead
40
+ table.insert(res, id)
41
+ table.insert(res, instance)
42
+ else
43
+ table.insert(stale_ids, id)
44
+ end
45
+ end
46
+ return {res, stale_ids}
47
+ end
48
+
49
+ -- Returns the number of ids which exist in the given ids_set
50
+ local function batch_exists(model, ids_set)
51
+ local id_keys = {}
52
+ for id, _ in pairs(ids_set) do
53
+ table.insert(id_keys, model .. ':id:' .. id)
54
+ end
55
+ return redis.call('exists', unpack(id_keys))
56
+ end
57
+
58
+ -- Validate that each item in the attr_vals table is not nil
59
+ local function validate_attr_vals(attr_key, attr_vals)
60
+ if not attr_vals or #attr_vals == 0 then
61
+ error('Invalid value given for attribute : ' .. attr_key)
62
+ end
63
+ for _, val in ipairs(attr_vals) do
64
+ if not val then
65
+ error('Invalid value given for attribute : ' .. attr_key)
66
+ end
67
+ end
68
+ end
69
+
70
+ -- Validate the query conditions by checking if the attributes queried for are indexed
71
+ -- attributes. Parse query conditions into two separate tables:
72
+ -- 1. index_sets formatted as the id set keys in Redis '#{Model.name}:#{attr_key}:#{attr_val}'
73
+ -- 2. range_index_sets formatted as a tuple {id set key, min, max} => { '#{Model.name}:#{attr_key}' min max }
74
+ local function validate_and_parse_query_conditions(model, args)
75
+ local index_attrs = to_set(redis.call('smembers', model .. ':index_attrs'))
76
+ local range_index_attrs = to_set(redis.call('smembers', model .. ':range_index_attrs'))
77
+ -- Iterate through the arguments of the script to form the redis keys at which the
78
+ -- indexed id sets are stored.
79
+ local index_sets, range_index_sets = {}, {}
80
+ local i = 2
81
+ while i <= #args do
82
+ local attr_key, attr_val = args[i], args[i+1]
83
+ if index_attrs[attr_key] then
84
+ validate_attr_vals(attr_key, {attr_val})
85
+ -- For normal index attributes, keys are stored at "#{Model.name}:#{attr_key}:#{attr_val}"
86
+ table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val)
87
+ i = i + 2
88
+ elseif range_index_attrs[attr_key] then
89
+ -- For range attributes, nil values are stored as normal sets
90
+ if attr_val == "" then
91
+ table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val)
92
+ i = i + 2
93
+ else
94
+ local min, max = args[i+1], args[i+2]
95
+ validate_attr_vals(attr_key, {min, max})
96
+ -- For range index attributes, they are stored at "#{Model.name}:#{attr_key}"
97
+ table.insert(range_index_sets, {model .. ':' .. attr_key, min, max})
98
+ i = i + 3
99
+ end
100
+ else
101
+ error(attr_key .. ' is not an indexed attribute')
102
+ end
103
+ end
104
+ return {index_sets, range_index_sets}
105
+ end
@@ -0,0 +1,91 @@
1
+ --[[
2
+ EVALSHA SHA1(__FILE__) model id [field value ...]
3
+ > Time complexity: O(N) where N is the number of fields being set.
4
+
5
+ Update a hash with the specified fields to their respective values stored at
6
+ "model":id:"id", and modify the indexed attribute id sets accordingly. Refresh
7
+ the ttl if a model's ttl exists and is set to a value other than -1
8
+
9
+ # Return value
10
+ nil
11
+ --]]
12
+
13
+ -- The arguments can be accessed by Lua using the KEYS global variable in the
14
+ -- form of a one-based array (so KEYS[1], KEYS[2], ...).
15
+ -- All the additional arguments should not represent key names and can be
16
+ -- accessed by Lua using the ARGV global variable, very similarly to what
17
+ -- happens with keys (so ARGV[1], ARGV[2], ...).
18
+ --
19
+ -- KEYS[1] = redcord_instance.class.name
20
+ -- KEYS[2] = redcord_instance.id
21
+ -- ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
22
+ <%= include_lua 'shared/lua_helper_methods' %>
23
+ <%= include_lua 'shared/index_helper_methods' %>
24
+
25
+ if #KEYS ~= 2 then
26
+ error('Expected keys of be of size 2')
27
+ end
28
+ if #ARGV % 2 ~= 0 then
29
+ error('Expected an even number of arguments')
30
+ end
31
+
32
+ local model = KEYS[1]
33
+ local id = KEYS[2]
34
+
35
+ -- key = "#{model}:id:{id}"
36
+ local key = model .. ':id:' .. id
37
+
38
+ -- If there a delete operation (including expiring due to TTL) happened before
39
+ -- the update in another thread, the client might still send an update command
40
+ -- to the server. To avoid saving partial data, we reject this update call with
41
+ -- an error.
42
+ if redis.call('exists', key) == 0 then
43
+ error(key .. ' has been deleted')
44
+ end
45
+
46
+ -- Modify the id sets for any indexed attributes
47
+ local attrs_hash = to_hash(ARGV)
48
+ local indexed_attr_keys = redis.call('smembers', model .. ':index_attrs')
49
+ if #indexed_attr_keys > 0 then
50
+ -- Get the previous and new values for indexed attributes
51
+ local prev_attrs = redis.call('hmget', key, unpack(indexed_attr_keys))
52
+ for i, attr_key in ipairs(indexed_attr_keys) do
53
+ local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
54
+ -- Skip attr values not present in the argument hash
55
+ if curr_attr_val then
56
+ replace_id_in_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
57
+ end
58
+ end
59
+ end
60
+ local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
61
+ if #range_index_attr_keys > 0 then
62
+ -- Get the previous and new values for indexed attributes
63
+ local prev_attrs = redis.call('hmget', key, unpack(range_index_attr_keys))
64
+ for i, attr_key in ipairs(range_index_attr_keys) do
65
+ local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
66
+ -- Skip attr values not present in the argument hash
67
+ if curr_attr_val then
68
+ replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
69
+ end
70
+ end
71
+ end
72
+
73
+ -- Forward the script arguments to the Redis command HSET and update the args.
74
+ -- Call the Redis command: HSET key [field value ...]
75
+ redis.call('hset', key, unpack(ARGV))
76
+
77
+ -- Call the Redis command: GET "#{Model.name}:ttl"
78
+ local ttl = redis.call('get', model .. ':ttl')
79
+
80
+ if ttl then
81
+ if ttl == '-1' then
82
+ -- Persist the object if the ttl is set to -1
83
+ redis.call('persist', key)
84
+ else
85
+ -- Reset the TTL for this object. We do this manually becaues altering the
86
+ -- field value of a hash with HSET, etc. will leave the TTL
87
+ -- untouched: https://redis.io/commands/expire
88
+ redis.call('expire', key, ttl)
89
+ end
90
+ end
91
+ return nil
@@ -0,0 +1,34 @@
1
+ # typed: strict
2
+ require 'redcord/migration/version'
3
+ require 'redcord/migration/migrator'
4
+
5
+ db_namespace = namespace :redis do
6
+ task migrate: :environment do
7
+ $stdout.sync = true
8
+ migration_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
9
+
10
+ puts [
11
+ 'redis',
12
+ 'direction',
13
+ 'version',
14
+ 'migration',
15
+ 'duration',
16
+ ].map { |str| str.ljust(30) }.join("\t")
17
+
18
+ local_versions = Redcord::Migration::Version.new.all
19
+ Redcord::Base.configurations[Rails.env].each do |model, config|
20
+ redis = Redis.new(**(config.symbolize_keys))
21
+ remote_versions = Redcord::Migration::Version.new(redis: redis).all
22
+ (local_versions - remote_versions).sort.each do |version|
23
+ Redcord::Migration::Migrator.migrate(
24
+ redis: redis,
25
+ version: version,
26
+ direction: :up,
27
+ )
28
+ end
29
+ end
30
+
31
+ migration_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
32
+ puts "\nFinished in #{(migration_end - migration_start).round(3)} seconds"
33
+ end
34
+ end
metadata ADDED
@@ -0,0 +1,210 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: redcord
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1.alpha
5
+ platform: ruby
6
+ authors:
7
+ - Chan Zuckerberg Initiative
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-06-01 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: activesupport
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '5'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '5'
27
+ - !ruby/object:Gem::Dependency
28
+ name: railties
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '5'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '5'
41
+ - !ruby/object:Gem::Dependency
42
+ name: redis
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '4'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '4'
55
+ - !ruby/object:Gem::Dependency
56
+ name: sorbet
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: 0.4.4704
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: 0.4.4704
69
+ - !ruby/object:Gem::Dependency
70
+ name: sorbet-coerce
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: 0.2.7
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: 0.2.7
83
+ - !ruby/object:Gem::Dependency
84
+ name: sorbet-runtime
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: 0.4.4704
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: 0.4.4704
97
+ - !ruby/object:Gem::Dependency
98
+ name: sorbet-static
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: 0.4.4704
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: 0.4.4704
111
+ - !ruby/object:Gem::Dependency
112
+ name: codecov
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rspec
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '3.2'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '3.2'
139
+ - !ruby/object:Gem::Dependency
140
+ name: simplecov
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ description:
154
+ email: opensource@chanzuckerberg.com
155
+ executables: []
156
+ extensions: []
157
+ extra_rdoc_files: []
158
+ files:
159
+ - lib/redcord.rb
160
+ - lib/redcord.rbi
161
+ - lib/redcord/actions.rb
162
+ - lib/redcord/attribute.rb
163
+ - lib/redcord/base.rb
164
+ - lib/redcord/configurations.rb
165
+ - lib/redcord/logger.rb
166
+ - lib/redcord/lua_script_reader.rb
167
+ - lib/redcord/migration.rb
168
+ - lib/redcord/migration/migrator.rb
169
+ - lib/redcord/migration/ttl.rb
170
+ - lib/redcord/migration/version.rb
171
+ - lib/redcord/prepared_redis.rb
172
+ - lib/redcord/railtie.rb
173
+ - lib/redcord/range_interval.rb
174
+ - lib/redcord/redis_connection.rb
175
+ - lib/redcord/relation.rb
176
+ - lib/redcord/serializer.rb
177
+ - lib/redcord/server_scripts.rb
178
+ - lib/redcord/server_scripts/create_hash_returning_id.erb.lua
179
+ - lib/redcord/server_scripts/delete_hash.erb.lua
180
+ - lib/redcord/server_scripts/find_by_attr.erb.lua
181
+ - lib/redcord/server_scripts/find_by_attr_count.erb.lua
182
+ - lib/redcord/server_scripts/shared/index_helper_methods.erb.lua
183
+ - lib/redcord/server_scripts/shared/lua_helper_methods.erb.lua
184
+ - lib/redcord/server_scripts/shared/query_helper_methods.erb.lua
185
+ - lib/redcord/server_scripts/update_hash.erb.lua
186
+ - lib/redcord/tasks/redis.rake
187
+ homepage: https://github.com/chanzuckerberg/redis-record
188
+ licenses:
189
+ - MIT
190
+ metadata: {}
191
+ post_install_message:
192
+ rdoc_options: []
193
+ require_paths:
194
+ - lib
195
+ required_ruby_version: !ruby/object:Gem::Requirement
196
+ requirements:
197
+ - - ">="
198
+ - !ruby/object:Gem::Version
199
+ version: 2.5.0
200
+ required_rubygems_version: !ruby/object:Gem::Requirement
201
+ requirements:
202
+ - - ">"
203
+ - !ruby/object:Gem::Version
204
+ version: 1.3.1
205
+ requirements: []
206
+ rubygems_version: 3.1.3
207
+ signing_key:
208
+ specification_version: 4
209
+ summary: A Ruby ORM like Active Record, but for Redis
210
+ test_files: []