@rbxts/dataforge 0.1.0 → 0.4.1

package/src/store.luau

@@ -1,5 +1,7 @@
 --! strict
 
+local errors = require "./errors"
+local result = require "./result"
 local types = require "./types"
 local utils = require "./utils"
 local roblox_scheduler = require "./schedulers/roblox"
@@ -20,6 +22,7 @@ type Profile<T = any> = types.Profile<T>
 type LocklessProfile<T = any> = types.LocklessProfile<T>
 type PeekProfile<T = any> = types.PeekProfile<T>
 type HooksList<T... = ()> = types.HooksList<T...>
+type Result<T> = types.Result<T>
 
 export type Store<T = any> = {
 	_loading: { [string]: boolean },
@@ -43,31 +46,42 @@ export type Store<T = any> = {
 	},
 
 	config: ResolvedConfig<T>,
-	load: (self: Store<T>, key: string, user_ids: { number }) -> Profile<T>,
-	wait_loaded: (self: Store<T>, key: string, user_ids: { number }) -> Profile<T>,
+	--- Takes the session lock for `key` and loads it. Yields. Fails with
+	--- `store_closed`, `already_loaded`, `timeout`, `roblox`, `outdated` (a
+	--- newer server wrote the record; no lock is taken), `predicate_failed`
+	--- (`config.predicate` rejected the stored data; no lock is taken),
+	--- `migration_mismatch` or `tx_marker_invalid`.
+	load: (self: Store<T>, key: string, user_ids: { number }) -> Result<Profile<T>>,
+	--- Returns the profile for `key`, waiting for it to load or loading it if
+	--- nobody has yet. Waiters get the same Result as the load.
+	wait_loaded: (self: Store<T>, key: string, user_ids: { number }) -> Result<Profile<T>>,
 	get_loaded: (self: Store<T>, key: string) -> Profile<T>?,
 
 	--- Returns the lockless profile for `key`, creating it if needed. Never
-	--- yields: a lockless profile does no storage work until `fetch`.
+	--- yields: a lockless profile does no storage work until `fetch`. Throws a
+	--- `store_closed` error on a closed store.
 	get_lockless: (self: Store<T>, key: string, user_ids: { number }) -> LocklessProfile<T>,
 
 	--- Read-only snapshot of `key` via `GetAsync`: the current data and lock.
 	--- Yields. The result is not tracked by the store, takes no lock and cannot
-	--- join a transaction.
-	peek: (self: Store<T>, key: string) -> PeekProfile<T>,
+	--- join a transaction. Fails with `store_closed`, `roblox`, `outdated`,
+	--- `predicate_failed` or `migration_mismatch`.
+	peek: (self: Store<T>, key: string) -> Result<PeekProfile<T>>,
 
 	--- Atomically updates profiles of this store (locked or lockless). `transform`
 	--- gets their data in order and returns the new data in the same order, or
-	--- false to cancel. Returns false when cancelled, true when committed; throws
-	--- when aborted. `config` fills in what the store does not know (ttls, retry);
-	--- hook and scheduler default to the store's.
+	--- false to cancel. Ok(false) when cancelled, Ok(true) when committed; fails
+	--- like `dataforge.transaction`. `config` fills in what the store does not
+	--- know (ttls, retry); hook and scheduler default to the store's.
 	transaction: (
 		self: Store<T>,
 		profiles: Array<ProfileBase<T>>,
 		transform: (data: Array<T>) -> Array<T> | false,
 		config: types.TransactionConfig?
-	) -> boolean,
+	) -> Result<boolean>,
 
+	--- Fires for every data change of every profile of this store, including
+	--- the data a key loads with (`old == nil` marks that initial value).
 	on_change: (self: Store<T>, callback: (profile: Profile<T>, new: T, old: T?) -> ()) -> (),
 	on_save: (self: Store<T>, callback: (profile: Profile<T>) -> ()) -> (),
 	on_closing: (self: Store<T>, callback: (profile: Profile<T>) -> ()) -> (),
@@ -77,7 +91,9 @@ export type Store<T = any> = {
 	close: (self: Store<T>) -> (),
 }
 
-local function resume_load_locks(self: Store, key: string, ok: boolean, result: any)
+--- Resumes every `wait_loaded` caller with the load's Result, or with a raw
+--- error (`res == nil`) they rethrow.
+local function resume_load_locks(self: Store, key: string, res: Result<Profile>?, raw: any?)
 	local loading_locks = self._loading_locks[key]
 	if not loading_locks then
 		return
@@ -85,7 +101,7 @@ local function resume_load_locks(self: Store, key: string, ok: boolean, result:
 	self._loading_locks[key] = nil
 
 	for _, thread in loading_locks do
-		self.config._scheduler.spawn(thread, ok, result)
+		self.config._scheduler.spawn(thread, res, raw)
 	end
 end
 
@@ -100,6 +116,9 @@ local function insert_loaded_lock(self: Store, key: string)
 	table.insert(waiters, thread)
 end
 
+--- Drops `profile` from the registry, unless the key has already been taken by
+--- a newer one: with instant detach a new profile may be registered while the
+--- old one is still writing, and the old one's late unregister must not evict it.
 local function unregister_profile(self: Store, profile: Profile)
 	local key = profile.key
 	if self.profiles[key] == profile then
@@ -145,6 +164,10 @@ function store.create<T>(config: Config<T>): Store<T>
 	assert(type(config) == "table", "datastore: config must be a table")
 	assert(type(config.name) == "string" and #config.name > 0, "datastore: config.name is required")
 	assert(config.template ~= nil, "datastore: config.template is required")
+	assert(
+		config.predicate == nil or type(config.predicate) == "function",
+		"datastore: config.predicate must be a function"
+	)
 
 	local lock_ttl = config.lock_ttl or 60
 	local hook = config._hook
@@ -168,6 +191,7 @@ function store.create<T>(config: Config<T>): Store<T>
 		_scheduler = scheduler,
 
 		migrations = config.migrations or {},
+		predicate = config.predicate,
 
 		autosave_interval = config.autosave_interval or 30,
 		flush_interval = config.flush_interval or config.autosave_interval or 30,
@@ -202,16 +226,19 @@ function store.create<T>(config: Config<T>): Store<T>
 	return setmetatable(self, store) :: any
 end
 
-function store.load(self: Store, key: string, user_ids: { number }): Profile
+function store.load(self: Store, key: string, user_ids: { number }): Result<Profile>
 	if self.closed then
-		error "datastore: store is closed"
+		return result.err(errors.store_closed(self))
 	end
-	if self.profiles[key] ~= nil or self._loading[key] then
-		error(`datastore: "{key}" is already loaded in this store`)
+	-- a closing profile detaches before its final write, so it is normally gone
+	-- from the registry already; the `open` check is belt and braces
+	local existing = self.profiles[key]
+	if (existing ~= nil and existing.open) or self._loading[key] then
+		return result.err(errors.already_loaded(self, key))
 	end
 	self._loading[key] = true
 
-	local ok, profile = pcall(profile.load, {
+	local ok, loaded = utils.try(profile.load, {
 		key = key,
 		config = self.config,
 		user_ids = user_ids,
@@ -220,29 +247,58 @@ function store.load(self: Store, key: string, user_ids: { number }): Profile
 	self._loading[key] = nil
 
 	if not ok then
-		resume_load_locks(self, key, false, profile)
-		error(profile, 0)
+		if errors.is(loaded) then
+			local failure = result.err(loaded)
+			resume_load_locks(self, key, failure)
+			return failure
+		end
+		-- a migration threw: everyone waiting gets the raw error too
+		resume_load_locks(self, key, nil, loaded)
+		error(loaded, 0)
 	end
 
-	profile:on_closed(function()
-		unregister_profile(self, profile)
-	end)
-	forward_hooks(self, profile)
+	if self.closed then
+		-- the store closed while this load was in flight: give the lock back
+		-- right away instead of registering a profile nobody will unload
+		loaded:unload()
+		local failure = result.err(errors.store_closed(self))
+		resume_load_locks(self, key, failure)
+		return failure
+	end
 
-	self.profiles[key] = profile
-	resume_load_locks(self, key, ok, profile)
-	return profile
+	-- dropped from the registry the moment it stops being usable, so the key can
+	-- be loaded again while the old session's final write is still in flight
+	loaded._on_detach = function()
+		unregister_profile(self, loaded)
+	end
+	-- safety net: a profile that closes some other way still unregisters
+	loaded:on_closed(function()
+		unregister_profile(self, loaded)
+	end)
+	forward_hooks(self, loaded)
+
+	self.profiles[key] = loaded
+	-- a load's initial data never reaches the profile's own hooks (they are
+	-- attached after `profile.load` set it), so report it here: `old == nil`
+	-- marks a key that just appeared. Lockless profiles need no equivalent:
+	-- `get_lockless` forwards their hooks before any `fetch` sets data.
+	fire_hooks(self, self._hooks.change, loaded, loaded:get_data(), nil)
+
+	local success = result.ok(loaded)
+	resume_load_locks(self, key, success)
+	return success
 end
 
---- Returns the profile for `key`, waiting for it to load or loading it if nobody has yet.
-function store.wait_loaded(self: Store, key: string, user_ids: { number }): Profile
+--- Returns the profile for `key`, waiting for it to load or loading it if
+--- nobody has yet. Waiters get the same Result (or raw error) as the load.
+function store.wait_loaded(self: Store, key: string, user_ids: { number }): Result<Profile>
 	if self.closed then
-		error "datastore: store is closed"
+		return result.err(errors.store_closed(self))
 	end
 
 	local existing = self.profiles[key]
 	if existing ~= nil then
-		return existing
+		return result.ok(existing)
 	end
 
 	if not self._loading[key] then
@@ -250,12 +306,11 @@ function store.wait_loaded(self: Store, key: string, user_ids: { number }): Prof
 	end
 	insert_loaded_lock(self, key)
 
-	local ok, result = coroutine.yield()
-	if not ok then
-		error(result, 0)
+	local res, raw = coroutine.yield()
+	if res == nil then
+		error(raw, 0)
 	end
-
-	return result
+	return res
 end
 
 function store.get_loaded(self: Store, key: string): Profile?
@@ -264,7 +319,7 @@ end
 
 function store.get_lockless(self: Store, key: string, user_ids: { number }): LocklessProfile
 	if self.closed then
-		error "datastore: store is closed"
+		error(errors.store_closed(self), 0)
 	end
 
 	local existing = self.lockless_profiles[key]
@@ -292,9 +347,9 @@ end
 
 --- One `get_async` on `key`, no write. Never registered in the store: `close`
 --- does not know about it and hooks do not fire for it.
-function store.peek(self: Store, key: string): PeekProfile
+function store.peek(self: Store, key: string): Result<PeekProfile>
 	if self.closed then
-		error "datastore: store is closed"
+		return result.err(errors.store_closed(self))
 	end
 
 	return peek_profile.create {
@@ -317,9 +372,9 @@ function store.transaction(
 	profiles: Array<ProfileBase>,
 	transform: (data: Array<any>) -> Array<any> | false,
 	config: types.TransactionConfig?
-): boolean
+): Result<boolean>
 	if self.closed then
-		error "datastore: store is closed"
+		return result.err(errors.store_closed(self))
 	end
 
 	local resolved = tx.resolve_config(config, {
@@ -335,11 +390,11 @@ function store.transaction(
 			datas[index] = ctx:get(participant)
 		end
 
-		local result = transform(datas)
-		if result == false then
+		local transformed = transform(datas)
+		if transformed == false then
 			return false
 		end
-		local new_datas = result :: Array<any>
+		local new_datas = transformed :: Array<any>
 
 		for index, participant in profiles do
 			local new = new_datas[index]
@@ -353,7 +408,9 @@ function store.transaction(
 end
 
 --- Unloads every profile (locked and lockless) concurrently and waits for all
---- of them. Loads and transactions are rejected from the first line on.
+--- of them, and for any load still in flight (which fails with `store_closed`
+--- and releases its lock). Loads and transactions are rejected from the first
+--- line on.
 function store.close(self: Store)
 	if self.closed then
 		return
@@ -379,12 +436,40 @@ function store.close(self: Store)
 		end)
 	end
 
+	--- A load still in flight: `load` sees `closed` when it finishes and gives
+	--- the lock back itself; wait for that like for an unload.
+	local function wait_load(key: string)
+		remaining += 1
+		scheduler.spawn(function()
+			insert_loaded_lock(self, key)
+			coroutine.yield()
+			remaining -= 1
+			if remaining == 0 and waiting then
+				scheduler.spawn(thread)
+			end
+		end)
+	end
+
+	-- snapshots: `unload` spawns a thread that runs up to its first yield right
+	-- away, and a detaching profile removes itself from these tables
+	local locked: Array<Profile> = {}
 	for _, profile in self.profiles do
-		unload(profile)
+		table.insert(locked, profile)
 	end
+	local lockless: Array<LocklessProfile> = {}
 	for _, profile in self.lockless_profiles do
+		table.insert(lockless, profile)
+	end
+
+	for _, profile in locked do
 		unload(profile)
 	end
+	for _, profile in lockless do
+		unload(profile)
+	end
+	for key in self._loading do
+		wait_load(key)
+	end
 
 	if remaining > 0 then
 		waiting = true

package/src/tx/handle.luau

@@ -7,6 +7,8 @@
 	foreign `pending` entries.
 ]]
 
+local errors = require "../errors"
+local result = require "../result"
 local types = require "../types"
 local utils = require "../utils"
 local roblox_scheduler = require "../schedulers/roblox"
@@ -25,6 +27,7 @@ type ResolvedTransactionConfig = types.ResolvedTransactionConfig
 
 type TxOutcome = types.TxOutcome
 type TxPatch = types.TxPatch
+type Error = types.Error
 export type TxContext = types.TxContext
 
 local tx_handle = {}
@@ -50,19 +53,17 @@ local function resolve_marker_in(
 	id: string,
 	fallback: TxOutcome
 ): TxOutcome
-	local result = utils.retry(config, function()
-		return datastore.update_async(id, function(current: any?)
-			if current == "committed" or current == "aborted" then
-				return current
-			end
-			return fallback
-		end)
+	local outcome = utils.update_async(config, datastore, id, function(current: any?)
+		if current == "committed" or current == "aborted" then
+			return current
+		end
+		return fallback
 	end)
 
-	if result ~= "committed" and result ~= "aborted" then
-		error(`datastore: unexpected tx marker value for {id}: {tostring(result)}`)
+	if outcome ~= "committed" and outcome ~= "aborted" then
+		error(errors.tx_marker_invalid(id, outcome), 0)
 	end
-	return result :: TxOutcome
+	return outcome :: TxOutcome
 end
 
 --- Reads the marker for `id`, writing `fallback` if it has not been decided yet.
@@ -154,42 +155,47 @@ local function prepare_sort(a: TxPatch, b: TxPatch): boolean
 	return sort_key(a.profile) < sort_key(b.profile)
 end
 
---- Locks every profile mutex in order. The returned function releases them in
---- reverse and, for a lockless profile still holding its storage transaction
---- lock (the transaction ended before writing `pending`), gives the key back.
-local function lock_profiles(profiles: Array<ProfileBase>)
+--- Locks every profile mutex in order, appending each one taken to `locked`
+--- so a rejection midway (a mutex closed while waiting) leaves an exact list
+--- for `unlock_all`.
+local function lock_all(profiles: Array<ProfileBase>, locked: Array<ProfileBase>)
 	for _, participant in profiles do
 		participant._mutex:lock()
+		table.insert(locked, participant)
 	end
-	return function()
-		for index = #profiles, 1, -1 do
-			local participant = profiles[index]
-			local lockless = participant :: LocklessProfile
-			if participant.kind == "lockless" and lockless._tx_lock then
-				local ok, err = utils.try(function()
-					lockless:_write_queued {
-						old_tx_lock = lockless._tx_lock,
-					}
-				end)
-				if not ok then
-					participant.config._hook.warn(
-						`datastore: failed to release tx lock on "{participant.key}" (it will expire): {tostring(err)}`
-					)
-					lockless._tx_lock = nil
-				end
+end
+
+--- Releases the mutexes in reverse and, for a lockless profile still holding
+--- its storage transaction lock (the transaction ended before writing
+--- `pending`), gives the key back.
+local function unlock_all(locked: Array<ProfileBase>)
+	for index = #locked, 1, -1 do
+		local participant = locked[index]
+		local lockless = participant :: LocklessProfile
+		if participant.kind == "lockless" and lockless._tx_lock then
+			local ok, err = utils.try(function()
+				lockless:_write_queued {
+					old_tx_lock = lockless._tx_lock,
+				}
+			end)
+			if not ok then
+				participant.config._hook.warn(
+					`datastore: failed to release tx lock on "{participant.key}" (it will expire): {tostring(err)}`
+				)
+				lockless._tx_lock = nil
 			end
-			participant._mutex:unlock()
 		end
+		participant._mutex:unlock()
 	end
 end
 
 local function assert_open_profiles(profiles: Array<ProfileBase>)
 	for _, participant in profiles do
 		if not participant.open then
-			error(`datastore: transaction failed, profile "{participant.key}" is closed`)
+			error(errors.closed(participant), 0)
 		end
 		if participant.kind == "locked" and not (participant :: Profile).is_locked then
-			error(`datastore: transaction failed, profile "{participant.key}" does not hold its session lock`)
+			error(errors.released(participant), 0)
 		end
 	end
 end
@@ -230,7 +236,10 @@ local function locked_patch(participant: Profile, new_data: any): TxPatch
 	patch.prepared = false
 
 	function patch.prepare(pending)
-		return participant:_write_record { release = false, pending = pending }
+		local written, lock = participant:_write_record { release = false, pending = pending }
+		if not written then
+			error(errors.lock_lost(participant, lock), 0)
+		end
 	end
 	function patch.commit(operation)
 		table.insert(participant._cleanup_txs, operation)
@@ -256,13 +265,13 @@ local function lockless_patch(participant: LocklessProfile, new_data: any): TxPa
 	patch.prepared = false
 
 	function patch.prepare(pending)
-		-- releases the transaction lock taken in `adquire_lockless_lock`; fails
-		-- (nothing written) if that lock is gone or someone else's by now
+		-- releases the transaction lock taken in `adquire_lockless_lock`; throws
+		-- `tx_lock_lost` (nothing written) if that lock is gone or someone
+		-- else's by now
 		participant:_write_queued {
 			pending = pending,
 			old_tx_lock = participant._tx_lock,
 		}
-		return true
 	end
 	function patch.commit(operation)
 		participant._known_outcomes[operation.id] = "committed"
@@ -287,13 +296,16 @@ local function make_patch(participant: ProfileBase, new_data: any): TxPatch
 end
 
 --- Runs `process(ctx)` over the given profiles (any store, locked or lockless)
---- and commits every `ctx:set` atomically. Returns false when `process`
---- returns false. Throws when the transaction aborts.
+--- and commits every `ctx:set` atomically. Ok(false) when `process` returns
+--- false, Ok(true) when committed. Fails with `tx_aborted` (its `cause` says
+--- why phase 1 gave up), `profile_closed`, `released`, or what flushing a
+--- lockless participant raised (`lockless_locked`, `outdated`, `roblox`, ...).
+--- Argument misuse throws; an error thrown by `process` propagates as is.
 function tx_handle.run(
 	config: ResolvedTransactionConfig,
 	profiles: Array<ProfileBase>,
 	process: (ctx: TxContext) -> boolean?
-): boolean
+): types.Result<boolean>
 	local seen: { [ProfileBase]: boolean } = {}
 	local profile_list: Array<ProfileBase> = {}
 
@@ -301,18 +313,18 @@ function tx_handle.run(
 		if not is_profile(profile) then
 			error "datastore: transaction expects profiles"
 		end
-		if not profile.open then
-			error "datastore: all profiles in a transaction must be open"
-		end
-		if profile.kind == "locked" and not (profile :: Profile).is_locked then
-			error(`datastore: transaction failed, profile "{profile.key}" does not hold its session lock`)
-		end
 		if seen[profile] then
 			error "datastore: a profile was provided twice in a transaction"
 		end
 		if profile.config._hook ~= config._hook then
 			error "datastore: all profiles in a transaction must share the transaction's hook"
 		end
+		if not profile.open then
+			return result.err(errors.closed(profile))
+		end
+		if profile.kind == "locked" and not (profile :: Profile).is_locked then
+			return result.err(errors.released(profile))
+		end
 
 		seen[profile] = true
 		table.insert(profile_list, profile)
@@ -323,9 +335,12 @@ function tx_handle.run(
 	end
 	profile_list = utils.sorted(profile_list, profile_sort)
 
-	local unlock_all = lock_profiles(profile_list)
+	local locked: Array<ProfileBase> = {}
 
-	local ok, result: boolean = utils.try(function()
+	local ok, value = utils.try(function()
+		-- inside the protected region: a mutex closed while waiting rejects the
+		-- lock, and the ones already taken must still be given back
+		lock_all(profile_list, locked)
 		assert_open_profiles(profile_list)
 		-- kind of an oxymoron
 		adquire_lockless_lock(config, profile_list)
@@ -385,7 +400,10 @@ function tx_handle.run(
 		-- Phase 1: every changed participant persists its post-tx data as `pending`.
 		local order = utils.sorted(patches, prepare_sort)
 		local desired: TxOutcome = "committed"
-		local reason = ""
+		-- why phase 1 gave up: a library error (reported as the abort's cause)
+		-- or a raw one (an internal invariant; rethrown as is after settling)
+		local cause: Error? = nil
+		local raw: any = nil
 		local expires = config._scheduler.now() + config.tx_ttl
 		local store_name = config._datastore.name
 
@@ -396,13 +414,16 @@ function tx_handle.run(
 				expires = expires,
 				data = patch.new_data,
 			}
-			local write_ok, prepared = utils.try(patch.prepare, pending)
-			if write_ok and prepared == true then
+			local write_ok, err = utils.try(patch.prepare, pending)
+			if write_ok then
 				patch.prepared = true
 			else
 				desired = "aborted"
-				local key = patch.profile.key
-				reason = if not write_ok then tostring(prepared) else `lock lost for "{key}"`
+				if errors.is(err) then
+					cause = err
+				else
+					raw = err
+				end
 				break
 			end
 		end
@@ -414,14 +435,16 @@ function tx_handle.run(
 		if outcome == "committed" then
 			return true
 		end
-		error(`datastore: transaction aborted: {reason}`)
+		if raw ~= nil then
+			error(raw, 0)
+		end
+		-- no cause: every prepare went through but the marker was already
+		-- resolved as aborted by another server (the pending's grace expired)
+		error(errors.tx_aborted(id, cause), 0)
 	end)
 
-	unlock_all()
-	if not ok then
-		error(result)
-	end
-	return result
+	unlock_all(locked)
+	return result.of(ok, value)
 end
 
 return tx_handle

package/src/tx/operation.luau

@@ -1,3 +1,4 @@
+local errors = require "../errors"
 local types = require "../types"
 
 export type TxOperation = types.TxOperation
@@ -23,14 +24,16 @@ local function resolve_marker(self: TxOperation): TxOutcome
 	end)
 
 	if result ~= "committed" and result ~= "aborted" then
-		error(`datastore: unexpected tx marker value for {id}: {tostring(result)}`)
+		error(errors.tx_marker_invalid(id, result), 0)
 	end
 	return result :: TxOutcome
 end
 
 --- Resolves the marker (blocking, retrying with backoff until the storage
---- answers), then applies the outcome to every patch exactly once. Safe to call
---- from several coroutines: later callers wait for the first resolution.
+--- answers with a valid outcome), then applies the outcome to every patch
+--- exactly once. Never fails: every participant mutex is held meanwhile, so
+--- giving up would leave them without an outcome. Safe to call from several
+--- coroutines: later callers wait for the first resolution.
 function tx_operation.settle(self: TxOperation): TxOutcome
 	local scheduler = self.config._scheduler