redcord 0.0.1.alpha

data/lib/redcord/server_scripts/shared/index_helper_methods.erb.lua

@@ -0,0 +1,61 @@
+-- Add an id to the id set of the index attribute
+local function add_id_to_index_attr(model, attr_key, attr_val, id)
+  if attr_val then
+    -- Call the Redis command: SADD "#{Model.name}:#{attr_name}:#{attr_val}" member ..
+    redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val, id)
+  end
+end
+
+-- Remove an id from the id set of the index attribute
+local function delete_id_from_index_attr(model, attr_key, attr_val, id)
+  if attr_val then
+    -- Call the Redis command: SREM "#{Model.name}:#{attr_name}:#{attr_val}" member ..
+    redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val, id)
+  end
+end
+
+-- Move an id from one id set to another for the index attribute
+local function replace_id_in_index_attr(model, attr_key, prev_attr_val,curr_attr_val, id)
+  -- If previous and new value differs, then modify the id sets accordingly
+  if prev_attr_val ~= curr_attr_val then
+    delete_id_from_index_attr(model, attr_key, prev_attr_val, id)
+    add_id_to_index_attr(model, attr_key, curr_attr_val, id)
+  end
+end
+
+-- Add an id to the sorted id set of the range index attribute
+local function add_id_to_range_index_attr(model, attr_key, attr_val, id)
+  if attr_val then
+    -- Nil values of range indices are sent to Redis as an empty string. They are stored
+    -- as a regular set at key "#{Model.name}:#{attr_name}:"
+    if attr_val == "" then
+      redis.call('sadd', model .. ':' .. attr_key .. ':' .. attr_val, id)
+    else
+      -- Call the Redis command: ZADD "#{Model.name}:#{attr_name}" #{attr_val} member ..,
+      -- where attr_val is the score of the sorted set
+      redis.call('zadd', model .. ':' .. attr_key, attr_val, id)
+    end
+  end
+end
+
+-- Remove an id from the sorted id set of the range index attribute
+local function delete_id_from_range_index_attr(model, attr_key, attr_val, id)
+  if attr_val then
+    -- Nil values of range indices are sent to Redis as an empty string. They are stored
+    -- as a regular set at key "#{Model.name}:#{attr_name}:"
+    if attr_val == "" then
+      redis.call('srem', model .. ':' .. attr_key .. ':' .. attr_val, id)
+    else
+      -- Call the Redis command: ZREM "#{Model.name}:#{attr_name}:#{attr_val}" member ..
+      redis.call('zrem', model .. ':' .. attr_key, id)
+    end
+  end
+end
+
+-- Move an id from one sorted id set to another for the range index attribute
+local function replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+  if prev_attr_val ~= curr_attr_val then
+    delete_id_from_range_index_attr(model, attr_key, prev_attr_val, id)
+    add_id_to_range_index_attr(model, attr_key, curr_attr_val, id)
+  end
+end

data/lib/redcord/server_scripts/shared/lua_helper_methods.erb.lua

@@ -0,0 +1,33 @@
+-- Helper function to convert argument array to hash set
+local function to_hash(list)
+  local hash = {}
+  if not list then return hash end
+  for i=1, #list, 2 do
+    hash[list[i]] = list[i+1]
+  end
+  return hash
+end
+
+-- Helper function to convert list to set
+local function to_set(list)
+  local set = {}
+  if not list then return set end
+  for _, item in ipairs(list) do
+    set[item] = true
+  end
+  return set
+end
+
+-- Helper function to compute the intersection of the given set and list.
+local function set_list_intersect(set, list)
+  -- A nil set means that no items have been added to the set yet. If so,
+  -- we can just return the given list as a set
+  if not set then return to_set(list) end
+  local set_intersect = {}
+  for _, item in ipairs(list) do
+    if set[item] then
+      set_intersect[item] = true
+    end
+  end
+  return set_intersect
+end

data/lib/redcord/server_scripts/shared/query_helper_methods.erb.lua

@@ -0,0 +1,105 @@
+-- Calls the Redis command: ZRANGEBYSCORE key min max
+-- for each {key, min, max} given in the input arguments. Returns
+-- the set intersection of the results
+local function intersect_range_index_sets(set, tuples)
+  for _, redis_key in ipairs(tuples) do
+    local key, min, max = unpack(redis_key)
+    local ids = redis.call('zrangebyscore', key, min, max)
+    set = set_list_intersect(set, ids)
+  end
+  return set
+end
+
+-- Gets the hash of all the ids given. Returns the results in a
+-- table, as well as any ids not found in Redis as a separate table
+local function batch_hget(model, ids_set, fields)
+  local res, stale_ids = {}, {}
+  for id, _ in pairs(ids_set) do
+    local instance = nil
+    if fields and #fields > 0 then
+      local values = redis.call('hmget', model .. ':id:' .. id, unpack(fields))
+      -- HMGET returns the value in the order of the fields given. Map back to
+      -- field value [field value ..]
+      instance = {}
+      for i, field in ipairs(fields) do
+        if not values[i] then
+          instance = nil
+          break
+        end
+        table.insert(instance, field)
+        table.insert(instance, values[i])
+      end
+    else
+      -- HGETALL returns the value as field value [field value ..]
+      instance = redis.call('hgetall', model .. ':id:' .. id)
+    end
+      -- Only add to result if entry is not stale (if query to hgetall is not empty)
+    if instance and #instance > 0 then
+      -- We cannot return a Lua table to Redis as a hash. Return result as a flattened
+      -- array instead
+      table.insert(res, id)
+      table.insert(res, instance)
+    else
+      table.insert(stale_ids, id)
+    end
+  end
+  return {res, stale_ids}
+end
+
+-- Returns the number of ids which exist in the given ids_set
+local function batch_exists(model, ids_set)
+  local id_keys = {}
+  for id, _ in pairs(ids_set) do
+    table.insert(id_keys, model .. ':id:' .. id)
+  end
+  return redis.call('exists', unpack(id_keys))
+end
+
+-- Validate that each item in the attr_vals table is not nil
+local function validate_attr_vals(attr_key, attr_vals)
+  if not attr_vals or #attr_vals == 0 then
+    error('Invalid value given for attribute : ' .. attr_key)
+  end
+  for _, val in ipairs(attr_vals) do
+    if not val then
+      error('Invalid value given for attribute : ' .. attr_key)
+    end
+  end
+end
+
+-- Validate the query conditions by checking if the attributes queried for are indexed
+-- attributes. Parse query conditions into two separate tables: 
+-- 1. index_sets formatted as the id set keys in Redis '#{Model.name}:#{attr_key}:#{attr_val}'
+-- 2. range_index_sets formatted as a tuple {id set key, min, max} => { '#{Model.name}:#{attr_key}' min max }
+local function validate_and_parse_query_conditions(model, args)
+  local index_attrs = to_set(redis.call('smembers', model .. ':index_attrs'))
+  local range_index_attrs = to_set(redis.call('smembers', model .. ':range_index_attrs'))
+  -- Iterate through the arguments of the script to form the redis keys at which the
+  -- indexed id sets are stored.
+  local index_sets, range_index_sets = {}, {}
+  local i = 2
+  while i <= #args do
+    local attr_key, attr_val = args[i], args[i+1]
+    if index_attrs[attr_key] then
+      validate_attr_vals(attr_key, {attr_val})
+      -- For normal index attributes, keys are stored at "#{Model.name}:#{attr_key}:#{attr_val}"
+      table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val)
+      i = i + 2
+    elseif range_index_attrs[attr_key] then
+      -- For range attributes, nil values are stored as normal sets
+      if attr_val == "" then
+        table.insert(index_sets, model .. ':' .. attr_key .. ':' .. attr_val)
+        i = i + 2
+      else
+        local min, max = args[i+1], args[i+2]
+        validate_attr_vals(attr_key, {min, max})
+        -- For range index attributes, they are stored at "#{Model.name}:#{attr_key}"
+        table.insert(range_index_sets, {model .. ':' .. attr_key, min, max})
+        i = i + 3
+      end
+    else
+      error(attr_key .. ' is not an indexed attribute')
+    end
+  end
+  return {index_sets, range_index_sets}
+end

data/lib/redcord/server_scripts/update_hash.erb.lua

@@ -0,0 +1,91 @@
+--[[
+EVALSHA SHA1(__FILE__) model id [field value ...]
+> Time complexity: O(N) where N is the number of fields being set.
+
+Update a hash with the specified fields to their respective values stored at
+"model":id:"id", and modify the indexed attribute id sets accordingly. Refresh 
+the ttl if a model's ttl exists and is set to a value other than -1
+
+# Return value
+nil
+--]]
+
+-- The arguments can be accessed by Lua using the KEYS global variable in the
+-- form of a one-based array (so KEYS[1], KEYS[2], ...).
+-- All the additional arguments should not represent key names and can be
+-- accessed by Lua using the ARGV global variable, very similarly to what
+-- happens with keys (so ARGV[1], ARGV[2], ...).
+--
+--   KEYS[1] = redcord_instance.class.name
+--   KEYS[2] = redcord_instance.id
+--   ARGV[1...2N] = attr_key attr_val [attr_key attr_val ..]
+<%= include_lua 'shared/lua_helper_methods' %>
+<%= include_lua 'shared/index_helper_methods' %>
+
+if #KEYS ~= 2 then
+  error('Expected keys of be of size 2')
+end
+if #ARGV % 2 ~= 0 then
+  error('Expected an even number of arguments')
+end
+
+local model = KEYS[1]
+local id = KEYS[2]
+
+-- key = "#{model}:id:{id}"
+local key = model .. ':id:' .. id
+
+-- If there a delete operation (including expiring due to TTL) happened before
+-- the update in another thread, the client might still send an update command
+-- to the server. To avoid saving partial data, we reject this update call with
+-- an error.
+if redis.call('exists', key) == 0 then
+  error(key .. ' has been deleted')
+end
+
+-- Modify the id sets for any indexed attributes
+local attrs_hash = to_hash(ARGV)
+local indexed_attr_keys = redis.call('smembers', model .. ':index_attrs')
+if #indexed_attr_keys > 0 then
+  -- Get the previous and new values for indexed attributes
+  local prev_attrs = redis.call('hmget', key, unpack(indexed_attr_keys))
+  for i, attr_key in ipairs(indexed_attr_keys) do
+    local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
+    -- Skip attr values not present in the argument hash
+    if curr_attr_val then
+      replace_id_in_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+    end
+  end
+end
+local range_index_attr_keys = redis.call('smembers', model .. ':range_index_attrs')
+if #range_index_attr_keys > 0 then
+  -- Get the previous and new values for indexed attributes
+  local prev_attrs = redis.call('hmget', key, unpack(range_index_attr_keys))
+  for i, attr_key in ipairs(range_index_attr_keys) do
+    local prev_attr_val, curr_attr_val = prev_attrs[i], attrs_hash[attr_key]
+    -- Skip attr values not present in the argument hash
+    if curr_attr_val then
+      replace_id_in_range_index_attr(model, attr_key, prev_attr_val, curr_attr_val, id)
+    end
+  end
+end
+
+-- Forward the script arguments to the Redis command HSET and update the args.
+-- Call the Redis command: HSET key [field value ...]
+redis.call('hset', key, unpack(ARGV))
+
+-- Call the Redis command: GET "#{Model.name}:ttl"
+local ttl = redis.call('get', model .. ':ttl')
+
+if ttl then
+  if ttl == '-1' then
+    -- Persist the object if the ttl is set to -1
+    redis.call('persist', key)
+  else
+    -- Reset the TTL for this object. We do this manually becaues altering the
+    -- field value of a hash with HSET, etc. will leave the TTL
+    -- untouched: https://redis.io/commands/expire
+    redis.call('expire', key, ttl)
+  end
+end
+return nil

data/lib/redcord/tasks/redis.rake

@@ -0,0 +1,34 @@
+# typed: strict
+require 'redcord/migration/version'
+require 'redcord/migration/migrator'
+
+db_namespace = namespace :redis do
+  task migrate: :environment do
+    $stdout.sync = true
+    migration_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    puts [
+      'redis',
+      'direction',
+      'version',
+      'migration',
+      'duration',
+    ].map { |str| str.ljust(30) }.join("\t")
+
+    local_versions = Redcord::Migration::Version.new.all
+    Redcord::Base.configurations[Rails.env].each do |model, config|
+      redis = Redis.new(**(config.symbolize_keys))
+      remote_versions = Redcord::Migration::Version.new(redis: redis).all
+      (local_versions - remote_versions).sort.each do |version|
+        Redcord::Migration::Migrator.migrate(
+          redis: redis,
+          version: version,
+          direction: :up,
+        )
+      end
+    end
+
+    migration_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    puts "\nFinished in #{(migration_end - migration_start).round(3)} seconds"
+  end
+end

metadata

@@ -0,0 +1,210 @@
+--- !ruby/object:Gem::Specification
+name: redcord
+version: !ruby/object:Gem::Version
+  version: 0.0.1.alpha
+platform: ruby
+authors:
+- Chan Zuckerberg Initiative
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-06-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: sorbet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4704
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4704
+- !ruby/object:Gem::Dependency
+  name: sorbet-coerce
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+- !ruby/object:Gem::Dependency
+  name: sorbet-runtime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4704
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4704
+- !ruby/object:Gem::Dependency
+  name: sorbet-static
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4704
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4704
+- !ruby/object:Gem::Dependency
+  name: codecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: opensource@chanzuckerberg.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/redcord.rb
+- lib/redcord.rbi
+- lib/redcord/actions.rb
+- lib/redcord/attribute.rb
+- lib/redcord/base.rb
+- lib/redcord/configurations.rb
+- lib/redcord/logger.rb
+- lib/redcord/lua_script_reader.rb
+- lib/redcord/migration.rb
+- lib/redcord/migration/migrator.rb
+- lib/redcord/migration/ttl.rb
+- lib/redcord/migration/version.rb
+- lib/redcord/prepared_redis.rb
+- lib/redcord/railtie.rb
+- lib/redcord/range_interval.rb
+- lib/redcord/redis_connection.rb
+- lib/redcord/relation.rb
+- lib/redcord/serializer.rb
+- lib/redcord/server_scripts.rb
+- lib/redcord/server_scripts/create_hash_returning_id.erb.lua
+- lib/redcord/server_scripts/delete_hash.erb.lua
+- lib/redcord/server_scripts/find_by_attr.erb.lua
+- lib/redcord/server_scripts/find_by_attr_count.erb.lua
+- lib/redcord/server_scripts/shared/index_helper_methods.erb.lua
+- lib/redcord/server_scripts/shared/lua_helper_methods.erb.lua
+- lib/redcord/server_scripts/shared/query_helper_methods.erb.lua
+- lib/redcord/server_scripts/update_hash.erb.lua
+- lib/redcord/tasks/redis.rake
+homepage: https://github.com/chanzuckerberg/redis-record
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.1.3
+signing_key: 
+specification_version: 4
+summary: A Ruby ORM like Active Record, but for Redis
+test_files: []