@rbxts/dataforge 0.1.0 → 0.4.1

package/src/profile/profile.luau

@@ -1,4 +1,6 @@
+local errors = require "../errors"
 local mutex = require "../mutex"
+local result = require "../result"
 local types = require "../types"
 local utils = require "../utils"
 local common = require "./common"
@@ -13,22 +15,57 @@ export type LoadOptions = types.LoadOptions
 local profile = {}
 profile.__index = profile
 
-local function adquire_lock(load_options: LoadOptions): types.DataRecord
+--- One counter per process: a lock id must be unique per acquisition, not per
+--- server, so two sessions of the same key on this server can tell their locks
+--- apart. `#` keeps them out of the transaction id namespace (`{job_id}-{n}`),
+--- which lockless writes compare by id.
+local lock_counter = utils.counter()
+
+--- Takes the session lock on `key`, waiting out (or stealing by expiry) any
+--- holder. Returns the locked record and the id this acquisition stamped on it.
+local function adquire_lock(load_options: LoadOptions): (types.DataRecord, string)
 	local config = load_options.config
 	local user_ids = load_options.user_ids
 	local datastore = load_options._datastore
-	local job_id = config._hook.job_id()
 	local key = load_options.key
+	-- minted once per acquisition, before the loop: waiting may write the lock
+	-- several times and every attempt must stamp the same id
+	local lock_id = `{config._hook.job_id()}#{lock_counter()}`
 
 	local scheduler = config._scheduler
 
 	local deadline = scheduler.now() + config.load_timeout
+	-- the last lock seen on the record, reported when the load times out
+	local last_lock: types.Lock? = nil
+	-- set by the transform: the record lists more migrations than this server
+	-- declares, so the lock must not be taken (the write is cancelled)
+	local outdated_migrations: { string }? = nil
+	-- set by the transform: the error the store's predicate rejected the record
+	-- with (or the one the predicate itself threw); the lock is not taken
+	local predicate_error: any = nil
 
 	while true do
 		local now = scheduler.now()
-		local lock_applied = utils.retry(config, function()
-			return datastore.update_async(key, function(old: types.DataRecord?): types.DataRecord?
-				if common.is_locked_for(old :: types.DataRecord, job_id, now) then
+		local lock_applied = utils.update_async(
+			config,
+			datastore,
+			key,
+			function(old: types.DataRecord?): types.DataRecord?
+				outdated_migrations = nil
+				predicate_error = nil
+				if common.is_outdated(old, config) then
+					outdated_migrations = (old :: types.DataRecord).migrations
+					return nil
+				end
+				-- before the lock check on purpose: a key this server must not
+				-- own is rejected right away instead of waiting out its holder
+				local stored = if old then old.data else nil
+				local passed, failure = common.run_predicate(config, key, stored)
+				if not passed then
+					predicate_error = failure or errors.predicate_failed(key, "load", stored)
+					return nil
+				end
+				if common.is_locked_for(old :: types.DataRecord, lock_id, now) then
 					return nil
 				end
 				local record: types.DataRecord = old or {} :: types.DataRecord
@@ -38,28 +75,35 @@ local function adquire_lock(load_options: LoadOptions): types.DataRecord
 					pending = record.pending,
 					migrations = record.migrations or {},
 					lock = {
-						id = job_id,
+						id = lock_id,
 						expires = now + config.lock_ttl,
 					},
 				}
-			end, user_ids)
-		end) :: types.DataRecord
+			end,
+			user_ids
+		) :: types.DataRecord
 
-		if common.is_lock_owned(lock_applied, job_id) then
-			return lock_applied
+		if outdated_migrations ~= nil then
+			error(errors.outdated(key, outdated_migrations, common.all_migration_names(config.migrations)), 0)
+		end
+		if predicate_error ~= nil then
+			error(predicate_error, 0)
 		end
+		if common.is_lock_owned(lock_applied, lock_id) then
+			return lock_applied, lock_id
+		end
+		last_lock = if lock_applied then lock_applied.lock else nil
 
 		while true do
 			if scheduler.now() >= deadline then
-				error(`datastore: timed out loading "{key}" (session locked by another server)`)
+				error(errors.timeout(key, last_lock), 0)
 			end
 			scheduler.wait(config.load_poll)
 
-			local current = utils.retry(config, function()
-				return datastore.get_async(key)
-			end) :: types.DataRecord?
+			local current = utils.get_async(config, datastore, key) :: types.DataRecord?
+			last_lock = if current then current.lock else nil
 
-			if not common.is_locked_for(current, job_id, scheduler.now()) then
+			if not common.is_locked_for(current, lock_id, scheduler.now()) then
 				break
 			end
 		end
@@ -85,9 +129,11 @@ local function run_autosave_loop(self: Profile)
 
 			-- a failed write must not kill the loop: the next tick retries, and
 			-- `write_data_record` gives the lock up after a full ttl of failures
-			local ok, err = utils.try(self.save, self)
+			local ok, res = utils.try(self.save, self)
 			if not ok then
-				config._hook.warn(`datastore: autosave failed for "{self.key}": {tostring(err)}`)
+				config._hook.warn(`datastore: autosave failed for "{self.key}": {tostring(res)}`)
+			elseif not res.success then
+				config._hook.warn(`datastore: autosave failed for "{self.key}": {tostring(res.error)}`)
 			end
 		end
 	end)
@@ -109,6 +155,18 @@ local function set_data(self: Profile, data: any)
 	fire_hooks(self, self._hooks.change, new, old)
 end
 
+--- Lets the store drop the key as soon as the profile stops being usable,
+--- without waiting for the final write to land. Idempotent: a lost lock
+--- followed by an unload must only detach once.
+local function detach(self: Profile)
+	local on_detach = self._on_detach
+	if on_detach == nil or self._detached then
+		return
+	end
+	self._detached = true
+	on_detach()
+end
+
 --- Another server took the key: the profile closes without a release write.
 --- Runs inside a write (mutex held by the caller), so it only closes the mutex:
 --- the holder finishes, everything waiting on it is rejected.
@@ -117,7 +175,8 @@ local function process_lock_lost(self: Profile)
 		return
 	end
 	self.open = false
-	self._mutex:close(`datastore: profile "{self.key}" lost its session lock`)
+	detach(self)
+	self._mutex:close(errors.profile_closed(self, "lock_lost"))
 	fire_hooks(self, self._hooks.lock_lost)
 	fire_hooks(self, self._hooks.closed)
 end
@@ -127,20 +186,40 @@ type WriteOptions = {
 	pending: types.Pending?,
 }
 
-local function write_data_record(self: Profile, options: WriteOptions): boolean
+--- Writes the data (and `pending`, if any) under our session lock, refreshing
+--- or releasing it. Returns true when written; false and the lock now on the
+--- record (nil when none) when the lock is not ours any more, in which case
+--- the profile is closed. Throws a `roblox` error when storage fails, or
+--- `predicate_failed` when the store's predicate rejected the stored data
+--- (nothing is written and the lock is kept).
+local function write_data_record(self: Profile, options: WriteOptions): (boolean, types.Lock?)
 	local config = self.config
 	local scheduler = config._scheduler
 	local datastore = self._datastore
 	local now = scheduler.now()
-	local job_id = config._hook.job_id()
+	-- this acquisition's id, not the job id: another session of this key on this
+	-- server holds a different lock and must not be written over
+	local lock_id = self.lock_id
 
 	-- set by the transform: whether the write went through (the transform may
-	-- run more than once when the storage retries it)
+	-- run more than once when the storage retries it), and the lock it found
 	local applied = false
+	local seen_lock: types.Lock? = nil
+	-- set by the transform: the store's predicate rejected the stored data (or
+	-- threw); the write is cancelled without the lock being lost
+	local predicate_error: any = nil
 
 	local function transform(old: any?): DataRecord?
 		applied = false
-		if not common.is_lock_owned(old, job_id) then
+		predicate_error = nil
+		seen_lock = if old then old.lock else nil
+		if not common.is_lock_owned(old, lock_id) then
+			return nil
+		end
+		local stored = if old then old.data else nil
+		local passed, failure = common.run_predicate(config, self.key, stored)
+		if not passed then
+			predicate_error = failure or errors.predicate_failed(self.key, "save", stored, self)
 			return nil
 		end
 		applied = true
@@ -148,7 +227,9 @@ local function write_data_record(self: Profile, options: WriteOptions): boolean
 		local new_lock: types.Lock? = nil
 		if not options.release then
 			new_lock = {
-				id = job_id,
+				-- the same id every time: a fresh one would look like a new lock
+				-- to everyone else
+				id = lock_id,
 				expires = now + config.lock_ttl,
 				tx = nil,
 			} :: types.Lock
@@ -163,9 +244,7 @@ local function write_data_record(self: Profile, options: WriteOptions): boolean
 	end
 
 	local success, err = pcall(function()
-		utils.retry(config, function()
-			return datastore.update_async(self.key, transform, self.user_ids)
-		end)
+		utils.update_async(config, datastore, self.key, transform, self.user_ids)
 	end)
 
 	if not success then
@@ -177,10 +256,16 @@ local function write_data_record(self: Profile, options: WriteOptions): boolean
 		error(err, 0)
 	end
 
+	if predicate_error ~= nil then
+		-- the predicate cancelled the write, not a lost lock: the profile keeps
+		-- its lock and stays open, so the next save may go through again
+		error(predicate_error, 0)
+	end
+
 	if not applied then
 		-- the transform bailed out: the lock is not ours any more
 		process_lock_lost(self)
-		return false
+		return false, seen_lock
 	end
 
 	self.last_write = now
@@ -197,31 +282,33 @@ local function write_data_record(self: Profile, options: WriteOptions): boolean
 	return true
 end
 
---- Gives the lock taken by `adquire_lock` back, leaving the record otherwise
---- untouched. Used when the load fails after locking; a failure here is only
---- warned about (the lock expires on its own).
-local function release_lock(load_options: LoadOptions)
+--- Gives the lock `lock_id` (taken by `adquire_lock`) back, leaving the record
+--- otherwise untouched. Used when the load fails after locking; a failure here
+--- is only warned about (the lock expires on its own). Takes the acquisition id
+--- so a failed load cannot clear a lock another local acquisition just took.
+local function release_lock(load_options: LoadOptions, lock_id: string)
 	local config = load_options.config
-	local job_id = config._hook.job_id()
 
 	local ok, err = utils.try(function()
-		utils.retry(config, function()
-			return load_options._datastore.update_async(load_options.key, function(old: DataRecord?): DataRecord?
-				if not common.is_lock_owned(old, job_id) then
-					return nil
-				end
-				local record = old :: DataRecord
-				return {
-					data = record.data,
-					pending = record.pending,
-					migrations = record.migrations,
-					lock = nil,
-				}
-			end, load_options.user_ids)
-		end)
+		utils.update_async(config, load_options._datastore, load_options.key, function(old: DataRecord?): DataRecord?
+			if not common.is_lock_owned(old, lock_id) then
+				return nil
+			end
+			local record = old :: DataRecord
+			return {
+				data = record.data,
+				pending = record.pending,
+				migrations = record.migrations,
+				lock = nil,
+			}
+		end, load_options.user_ids)
 	end)
 	if not ok then
-		config._hook.warn(`datastore: could not release the lock of "{load_options.key}" after a failed load (it will expire): {tostring(err)}`)
+		config._hook.warn(
+			`datastore: could not release the lock of "{load_options.key}" after a failed load (it will expire): {tostring(
+				err
+			)}`
+		)
 	end
 end
 
@@ -253,15 +340,15 @@ local function resolve_loaded(load_options: LoadOptions, record: DataRecord): (a
 		end
 		dirty = true
 	else
-		local result = apply_migrations(data, record.migrations or {}, migrations)
-		if not result.success then
-			error(`datastore: failed to apply migrations for "{load_options.key}": {result.err}`)
+		local migrated = apply_migrations(load_options.key, data, record.migrations or {}, migrations)
+		if not migrated.success then
+			error(migrated.err, 0)
 		end
 
-		data = result.result
-		new_migrations = result.new_migrations
+		data = migrated.result
+		new_migrations = migrated.new_migrations
 
-		if result.applied then
+		if migrated.applied then
 			dirty = true
 		end
 	end
@@ -271,19 +358,21 @@ end
 
 --- Takes the session lock and resolves the locked record into data. If the
 --- resolution fails the lock is given back before rethrowing.
-local function adquire_and_resolve(load_options: LoadOptions): (any, { string }, boolean)
-	local record = adquire_lock(load_options)
+local function adquire_and_resolve(load_options: LoadOptions): (any, { string }, boolean, string)
+	local record, lock_id = adquire_lock(load_options)
 	local config = load_options.config
 
 	-- from here on the key is locked: anything failing must give it back
 	local ok, data, new_migrations, dirty = pcall(resolve_loaded, load_options, record)
 	if not ok then
 		local err = data
-		config._hook.warn(`datastore: load of "{load_options.key}" failed after taking its lock, releasing it: {tostring(err)}`)
-		release_lock(load_options)
+		config._hook.warn(
+			`datastore: load of "{load_options.key}" failed after taking its lock, releasing it: {tostring(err)}`
+		)
+		release_lock(load_options, lock_id)
 		error(err, 0)
 	end
-	return data, new_migrations, dirty
+	return data, new_migrations, dirty, lock_id
 end
 
 local function load_options_of(self: Profile): LoadOptions
@@ -297,12 +386,12 @@ end
 
 local function assert_locked(self: Profile)
 	if not self.is_locked then
-		error(`datastore: profile "{self.key}" does not hold its session lock`, 3)
+		error(errors.released(self), 0)
 	end
 end
 
 function profile.load(load_options: LoadOptions): Profile
-	local data, new_migrations, dirty = adquire_and_resolve(load_options)
+	local data, new_migrations, dirty, lock_id = adquire_and_resolve(load_options)
 	local config = load_options.config
 
 	local self: Profile = {} :: any
@@ -312,6 +401,8 @@ function profile.load(load_options: LoadOptions): Profile
 	self._datastore = load_options._datastore
 	self.open = true
 	self.is_locked = true
+	self.lock_id = lock_id
+	self._detached = false
 
 	self.config = load_options.config
 	self.migrations = new_migrations
@@ -343,22 +434,26 @@ function profile.get_data(self: Profile)
 end
 
 --- Applies `dispatcher` to the current data. Returning nil or false leaves the
---- data untouched (returns false); anything else becomes the new data and is
---- persisted on the next save. Waits for any in-flight save on the mutex.
-function profile.update(self: Profile, dispatcher: (any) -> any): boolean
-	common.assert_open(self)
-	assert_locked(self)
-
-	return self._mutex:with(function()
-		-- a release may have run while this call waited on the mutex
+--- data untouched (Ok(false)); anything else becomes the new data and is
+--- persisted on the next save (Ok(true)). Waits for any in-flight save on the
+--- mutex. Fails with `profile_closed` / `released`; an error thrown by the
+--- dispatcher propagates as is.
+function profile.update(self: Profile, dispatcher: (any) -> any): types.Result<boolean>
+	return result.capture(function()
+		common.assert_open(self)
 		assert_locked(self)
-		local result = dispatcher(self._data)
-		if result == nil or result == false then
-			return false
-		end
-		set_data(self, result)
-		self._dirty = true
-		return true
+
+		return self._mutex:with(function()
+			-- a release may have run while this call waited on the mutex
+			assert_locked(self)
+			local new = dispatcher(self._data)
+			if new == nil or new == false then
+				return false
+			end
+			set_data(self, new)
+			self._dirty = true
+			return true
+		end)
 	end)
 end
 
@@ -382,61 +477,92 @@ function profile.on_lock_lost(self: Profile, callback: () -> ())
 	table.insert(self._hooks.lock_lost.hooks, callback)
 end
 
-function profile.save(self: Profile)
-	common.assert_open(self)
-	assert_locked(self)
-	-- before the mutex so hooks can still run updates that land in this save
-	fire_hooks(self, self._hooks.save)
-	common.assert_open(self)
-	assert_locked(self)
-	self._mutex:with(function()
-		-- without the lock the write would be cancelled and wrongly read as lost
+--- Writes the data, refreshing the session lock. Fails with `profile_closed`,
+--- `released`, `roblox`, `predicate_failed` (the store's predicate rejected
+--- the stored data; nothing is written and the lock is kept), or `lock_lost`
+--- (another server took the key; the profile is closed).
+function profile.save(self: Profile): types.Result<nil>
+	return result.capture(function()
+		common.assert_open(self)
+		assert_locked(self)
+		-- before the mutex so hooks can still run updates that land in this save
+		fire_hooks(self, self._hooks.save)
+		common.assert_open(self)
 		assert_locked(self)
-		write_data_record(self, { release = false })
+		self._mutex:with(function()
+			-- without the lock the write would be cancelled and wrongly read as lost
+			assert_locked(self)
+			local written, lock = write_data_record(self, { release = false })
+			if not written then
+				error(errors.lock_lost(self, lock), 0)
+			end
+		end)
+		return nil
 	end)
 end
 
 --- Writes the data and gives the session lock back. The profile stays open
---- (another server may load the key meanwhile); `update` and `save` throw
---- until `readquire`. No-op when already released.
-function profile.release(self: Profile)
-	common.assert_open(self)
-	if not self.is_locked then
-		return
-	end
-	self._mutex:with(function()
-		if not self.open or not self.is_locked then
-			return
+--- (another server may load the key meanwhile); `update` and `save` fail with
+--- `released` until `readquire`. No-op when already released. Fails with
+--- `profile_closed`, `roblox` or `predicate_failed` (the lock stays on the
+--- record, `is_locked` stays true), or `lock_lost` (the profile is closed).
+function profile.release(self: Profile): types.Result<nil>
+	return result.capture(function()
+		common.assert_open(self)
+		if not self.is_locked then
+			return nil
 		end
-		-- false: the lock was not ours any more, `process_lock_lost` closed the profile.
-		-- A storage error propagates; the lock is still on the record, so `is_locked` stays.
-		if write_data_record(self, { release = true }) then
+		self._mutex:with(function()
+			-- an unload queued behind this call has flipped `open`: its own write
+			-- gives the lock back
+			if not self.open or not self.is_locked then
+				return
+			end
+			local written, lock = write_data_record(self, { release = true })
+			if not written then
+				error(errors.lock_lost(self, lock), 0)
+			end
 			self.is_locked = false
-		end
+		end)
+		return nil
 	end)
 end
 
 --- Takes the session lock again, waiting for a holder like `load` does, and
 --- adopts the stored record (resolved pending, migrations) as the current
---- data. Throws on timeout, leaving the profile open and released. No-op when
---- already locked.
-function profile.readquire(self: Profile)
-	common.assert_open(self)
-	if self.is_locked then
-		return
-	end
-	self._mutex:with(function()
+--- data. Fails with `timeout` (the profile stays open and released),
+--- `profile_closed`, `roblox`, `outdated`, `predicate_failed`,
+--- `migration_mismatch` or `tx_marker_invalid`. No-op when already locked.
+function profile.readquire(self: Profile): types.Result<nil>
+	return result.capture(function()
 		common.assert_open(self)
 		if self.is_locked then
-			return
+			return nil
 		end
-		local data, new_migrations, dirty = adquire_and_resolve(load_options_of(self))
+		self._mutex:with(function()
+			common.assert_open(self)
+			if self.is_locked then
+				return
+			end
+			local ok, data, new_migrations, dirty, lock_id = pcall(adquire_and_resolve, load_options_of(self))
+			if not ok then
+				local err: any = data
+				if errors.is(err) and err.type == "outdated" then
+					err.profile = self
+				end
+				error(err, 0)
+			end
 
-		self.migrations = new_migrations
-		set_data(self, data)
-		self._dirty = dirty
-		self.last_write = self.config._scheduler.now()
-		self.is_locked = true
+			self.migrations = new_migrations
+			-- a fresh acquisition, so a fresh id (the released profile kept the
+			-- old one; nothing could be written with it while released)
+			self.lock_id = lock_id
+			set_data(self, data)
+			self._dirty = dirty
+			self.last_write = self.config._scheduler.now()
+			self.is_locked = true
+		end)
+		return nil
 	end)
 end
 
@@ -451,10 +577,14 @@ function profile.unload(self: Profile)
 	end
 
 	self.open = false
+	-- before the release write (which may take seconds under throttling): the
+	-- key is free to be loaded again, the new load waits on the record lock
+	detach(self)
 	self._mutex:lock()
-	self._mutex:close(`datastore: profile "{self.key}" is closed`)
+	self._mutex:close(errors.profile_closed(self, "unloaded"))
 	local ok, err = utils.try(function()
-		-- an already released profile has nothing on the record to give back
+		-- an already released profile has nothing on the record to give back;
+		-- a lost lock (false) has nothing to give back either
 		if self.is_locked then
 			write_data_record(self, { release = true })
 		end

package/src/result.luau

@@ -0,0 +1,61 @@
+--!strict
+
+local errors = require "./errors"
+local types = require "./types"
+local utils = require "./utils"
+
+--[[
+	`Result<T>`: what every fallible library call returns instead of throwing.
+
+	Internally the library keeps throwing, but it throws `Error` tables (see
+	`errors.luau`). `result.capture` at each public entry point turns an
+	`Error` into `Err`; anything else (a string thrown by a user callback, an
+	internal invariant) is rethrown untouched.
+]]
+
+export type Result<T> = types.Result<T>
+export type Error = types.Error
+
+local result_mt = {}
+result_mt.__index = result_mt
+
+function result_mt.unwrap(self: any): any
+	if self.success then
+		return self.value
+	end
+	error(self.error, 0)
+end
+
+local result = {}
+
+function result.ok<T>(value: T): Result<T>
+	return setmetatable({ success = true, value = value }, result_mt) :: any
+end
+
+function result.err(err: Error): Result<any>
+	assert(errors.is(err), "result.err expects an Error table")
+	return setmetatable({ success = false, error = err }, result_mt) :: any
+end
+
+function result.is(value: any): boolean
+	return type(value) == "table" and getmetatable(value) == result_mt
+end
+
+--- Turns a `pcall` outcome into a Result: `Ok(value)` on success, `Err` when
+--- the thrown value is an `Error` table, rethrown raw otherwise.
+function result.of<T>(ok: boolean, value: any): Result<T>
+	if ok then
+		return result.ok(value)
+	end
+	if errors.is(value) then
+		return result.err(value)
+	end
+	error(value, 0)
+end
+
+--- `pcall(fn, ...)` folded through `result.of`.
+function result.capture<T>(fn: (...any) -> T, ...: any): Result<T>
+	return result.of(utils.try(fn, ...))
+end
+
+return result