@quenty/datastore 13.40.0 → 13.42.0

package/src/Server/DataStore.TwoServerLock.spec.lua

@@ -0,0 +1,190 @@
+--!nonstrict
+--[[
+	Two servers fighting over one player's session-locked key: two DataStore objects (server A and
+	server B) share one DataStoreMock with distinct SessionIds, exactly like two live game servers.
+	This exercises the full cross-server lock lifecycle -- clean handoff via close, crash recovery via
+	stale-lock steal, a concurrent-load race, the MessagingService graceful-close handshake, and
+	session-stolen data integrity.
+
+	@class DataStoreTwoServerLock.spec.lua
+]]
+local require = require(script.Parent.loader).load(script)
+
+local DataStoreTestUtils = require("DataStoreTestUtils")
+local Jest = require("Jest")
+local PromiseTestUtils = require("PromiseTestUtils")
+
+local describe = Jest.Globals.describe
+local expect = Jest.Globals.expect
+local it = Jest.Globals.it
+
+local KEY = "player_1"
+
+describe("two servers: clean handoff and crash recovery", function()
+	it("hands a player off cleanly: A saves and closes, B loads A's data and owns the lock", function()
+		local controller = DataStoreTestUtils.setup()
+
+		local serverA = controller.newServer()
+		expect(controller.awaitOwn(serverA)).toEqual(true)
+		serverA:Store("coins", 42)
+		if not PromiseTestUtils.awaitSettled(serverA:SaveAndCloseSession(), 10) then
+			expect("A close hung").toEqual("settled")
+			controller:destroy()
+			return
+		end
+
+		local serverB = controller.newServer()
+		local coins = serverB:Load("coins")
+		if not PromiseTestUtils.awaitSettled(coins, 10) then
+			expect("B load hung").toEqual("settled")
+			controller:destroy()
+			return
+		end
+		expect((coins:Wait())).toEqual(42)
+		expect(controller.mock:GetRaw(KEY).lock.ActiveSession.SessionId).toEqual(serverB:GetSessionId())
+
+		controller:destroy()
+	end)
+
+	it("recovers a crashed server's saved data by stealing its stale lock", function()
+		local controller = DataStoreTestUtils.setup()
+
+		-- A acquires, saves coins under its lock, then "crashes" (no clean close).
+		local serverA = controller.newServer()
+		expect(controller.awaitOwn(serverA)).toEqual(true)
+		serverA:Store("coins", 7)
+		if not PromiseTestUtils.awaitSettled(serverA:Save(), 10) then
+			expect("A save hung").toEqual("settled")
+			controller:destroy()
+			return
+		end
+
+		-- Age A's lock so it looks like a long-dead (crashed) server, then abandon A.
+		local raw = controller.mock:GetRaw(KEY)
+		raw.lock.LastUpdateTime = os.time() - 1000000
+		controller.mock:SetRaw(KEY, raw)
+
+		-- B loads: steals the stale lock and recovers A's saved coins.
+		local serverB = controller.newServer()
+		local coins = serverB:Load("coins")
+		if not PromiseTestUtils.awaitSettled(coins, 10) then
+			expect("B load hung").toEqual("settled")
+			controller:destroy()
+			return
+		end
+		expect((coins:Wait())).toEqual(7)
+		expect(controller.mock:GetRaw(KEY).lock.ActiveSession.SessionId).toEqual(serverB:GetSessionId())
+
+		controller:destroy()
+	end)
+
+	it("lets exactly one of two concurrent loads acquire the lock; the other stays blocked", function()
+		local controller = DataStoreTestUtils.setup()
+
+		local serverA = controller.newServer()
+		local serverB = controller.newServer()
+
+		local loadA = serverA:PromiseLoadSuccessful()
+		local loadB = serverB:PromiseLoadSuccessful()
+
+		-- One acquires quickly; the other is blocked (retrying against the winner's fresh lock).
+		expect(PromiseTestUtils.awaitValue(function()
+			return not loadA:IsPending() or not loadB:IsPending()
+		end, 5)).toEqual(true)
+
+		local aSettled = not loadA:IsPending()
+		local bSettled = not loadB:IsPending()
+		expect(aSettled ~= bSettled).toEqual(true) -- exactly one
+
+		local owner = controller.mock:GetRaw(KEY).lock.ActiveSession.SessionId
+		expect(owner == serverA:GetSessionId() or owner == serverB:GetSessionId()).toEqual(true)
+
+		controller:destroy()
+	end)
+
+	it("cancels the loser's write when a session is stolen (no duplication)", function()
+		local controller = DataStoreTestUtils.setup()
+
+		local serverA = controller.newServer()
+		expect(controller.awaitOwn(serverA)).toEqual(true)
+
+		-- B takes over by writing its own lock + data directly (as if it stole the session).
+		controller.mock:SetRaw(KEY, {
+			coins = 100,
+			lock = {
+				LastUpdateTime = os.time(),
+				ActiveSession = { SessionId = "server-b", PlaceId = 222, JobId = "job-b" },
+			},
+		})
+
+		-- A tries to save its own value; the write is cancelled (SessionStolen), B's data survives.
+		local stolen = nil
+		serverA.SessionStolen:Connect(function(session)
+			stolen = session
+		end)
+		serverA:Store("coins", 5)
+		if not PromiseTestUtils.awaitSettled(serverA:Save(), 10) then
+			expect("A save hung").toEqual("settled")
+			controller:destroy()
+			return
+		end
+
+		expect(stolen).never.toBeNil()
+		expect(controller.mock:GetRaw(KEY).coins).toEqual(100)
+
+		controller:destroy()
+	end)
+end)
+
+describe("two servers: MessagingService graceful close", function()
+	it("completes the close handshake: B asks A to close, A closes and releases the lock", function()
+		local controller = DataStoreTestUtils.setup()
+
+		local serverA = controller.newServer({ messaging = true, autoCloseOnRequest = true })
+		expect(controller.awaitOwn(serverA)).toEqual(true)
+
+		local _serverB, helperB = controller.newServer({ messaging = true })
+
+		-- B gracefully asks A (the lock holder) to close.
+		local graceful = helperB:PromiseCloseSessionGraceful(game.PlaceId, game.JobId, serverA:GetSessionId())
+		if not PromiseTestUtils.awaitSettled(graceful, 15) then
+			expect("graceful close hung").toEqual("resolved")
+			controller:destroy()
+			return
+		end
+		expect((graceful:Yield())).toEqual(true)
+
+		-- A honored the request and released its lock.
+		expect(PromiseTestUtils.awaitValue(function()
+			local raw = controller.mock:GetRaw(KEY)
+			return raw ~= nil and raw.lock == nil
+		end, 5)).toEqual(true)
+
+		controller:destroy()
+	end, 30000) -- MessagingService round-trip, beyond jest's 5s default
+
+	it("evicts the holder during a messaging-enabled load and then acquires (production flow)", function()
+		local controller = DataStoreTestUtils.setup()
+
+		-- A holds the lock and will honor a graceful close request.
+		local serverA = controller.newServer({ messaging = true, autoCloseOnRequest = true })
+		expect(controller.awaitOwn(serverA)).toEqual(true)
+
+		-- B loads with messaging enabled: blocked by A's fresh lock, its load asks A to close, then
+		-- (after the propagation delay) retries and acquires. This is the real cross-server handoff.
+		-- Use a tiny propagation delay so the test does not wait the production 5s.
+		local serverB = controller.newServer({ messaging = true })
+		serverB:SetSessionMessagingCloseDelaySeconds(0.1)
+		local loadB = serverB:PromiseLoadSuccessful()
+
+		if not PromiseTestUtils.awaitSettled(loadB, 8) then
+			expect("B messaging load hung").toEqual("settled")
+			controller:destroy()
+			return
+		end
+		expect((loadB:Wait())).toEqual(true)
+		expect(controller.mock:GetRaw(KEY).lock.ActiveSession.SessionId).toEqual(serverB:GetSessionId())
+
+		controller:destroy()
+	end)
+end)

package/src/Server/DataStore.lua

@@ -72,6 +72,7 @@ local HttpService = game:GetService("HttpService")
 local DataStoreDeleteToken = require("DataStoreDeleteToken")
 local DataStoreLockHelper = require("DataStoreLockHelper")
 local DataStoreMessageHelper = require("DataStoreMessageHelper")
+local DataStoreNonRetryableLoadError = require("DataStoreNonRetryableLoadError")
 local DataStorePromises = require("DataStorePromises")
 local DataStoreStage = require("DataStoreStage")
 local Maid = require("Maid")
@@ -92,6 +93,18 @@ local DEFAULT_DEBUG_WRITING = false
 local DEFAULT_AUTO_SAVE_TIME_SECONDS = 60 * 5
 local DEFAULT_JITTER_PROPORTION = 0.1 -- Randomly assign jitter so if a ton of players join at once we don't hit the datastore at once
 
+-- Retry backoff for the session-locked load (~49s total); tunable so tests need not wait it out.
+-- https://exponentialbackoffcalculator.com/
+local DEFAULT_LOAD_RETRY_OPTIONS = {
+	exponential = 1.25,
+	initialWaitTime = 6,
+	maxAttempts = 6,
+	printWarning = true,
+}
+
+-- After a graceful session-close request we wait for Roblox to replicate the release before retrying.
+local DEFAULT_SESSION_MESSAGING_CLOSE_DELAY_SECONDS = 5
+
 local DataStore = setmetatable({}, DataStoreStage)
 DataStore.ClassName = "DataStore"
 DataStore.__index = DataStore
@@ -113,6 +126,8 @@ export type DataStore =
 			_loadedOk: ValueObject.ValueObject<boolean>,
 			_firstLoadPromise: Promise.Promise<()>,
 			_promiseSessionLockingFailed: Promise.Promise<()>,
+			_loadRetryOptions: PromiseRetryUtils.RetryOptions,
+			_sessionMessagingCloseDelaySeconds: number,
 
 			-- Events
 			Saving: Signal.Signal<Promise.Promise<()>>,
@@ -147,6 +162,8 @@ function DataStore.new(robloxDataStore: DataStorePromises.RobloxDataStore, key:
 	self._syncOnSave = self._maid:Add(ValueObject.new(false, "boolean"))
 	self._loadedOk = self._maid:Add(ValueObject.new(false, "boolean"))
 	self._promiseSessionLockingFailed = self._maid:Add(Promise.new())
+	self._loadRetryOptions = DEFAULT_LOAD_RETRY_OPTIONS
+	self._sessionMessagingCloseDelaySeconds = DEFAULT_SESSION_MESSAGING_CLOSE_DELAY_SECONDS
 
 	self._userIdList = nil
 
@@ -314,6 +331,32 @@ function DataStore.SetSyncOnSave(self: DataStore, syncEnabled: boolean)
 	self._syncOnSave.Value = syncEnabled
 end
 
+--[=[
+	Overrides the retry backoff used while acquiring a session-locked load. Defaults to ~49s of
+	exponential backoff. Mainly useful for tests, which set a tiny backoff so they need not wait it
+	out. Must be set before the datastore loads.
+
+	@param options { exponential: number?, initialWaitTime: number, maxAttempts: number, printWarning: boolean }
+]=]
+function DataStore.SetLoadRetryOptions(self: DataStore, options: PromiseRetryUtils.RetryOptions): ()
+	assert(not self._firstLoadPromise, "Must set load retry options before the datastore is loaded")
+	assert(type(options) == "table", "Bad options")
+
+	self._loadRetryOptions = options
+end
+
+--[=[
+	Overrides how long to wait for Roblox to replicate a released lock after a graceful session-close
+	request before retrying the load. Defaults to 5 seconds. Mainly useful for tests.
+
+	@param seconds number
+]=]
+function DataStore.SetSessionMessagingCloseDelaySeconds(self: DataStore, seconds: number): ()
+	assert(type(seconds) == "number" and seconds >= 0, "Bad seconds")
+
+	self._sessionMessagingCloseDelaySeconds = seconds
+end
+
 --[=[
 	Returns whether the datastore failed.
 	@return boolean
@@ -405,9 +448,13 @@ function DataStore.PromiseViewUpToDate(self: DataStore): Promise.Promise<()>
 	local promise = self:_promiseGetAsyncNoCache()
 	self._firstLoadPromise = promise
 
-	promise:Tap(function()
-		self._loadedOk.Value = true
-	end)
+	-- Fire-and-forget side effect; Catch so a failed/cancelled load is not surfaced as an uncaught
+	-- rejection on this discarded branch (the load result itself is handled by callers).
+	promise
+		:Tap(function()
+			self._loadedOk.Value = true
+		end)
+		:Catch(function() end)
 
 	return promise
 end
@@ -599,10 +646,12 @@ function DataStore._doDataSync(
 		)
 	end
 
-	promise:Tap(nil, function(err)
-		-- Might be caused by Maid rejecting state
-		warn("[DataStore] - Failed to sync data", err)
-	end)
+	promise
+		:Tap(nil, function(err)
+			-- Might be caused by Maid rejecting state
+			warn("[DataStore] - Failed to sync data", err)
+		end)
+		:Catch(function() end)
 
 	self._maid._saveMaid = maid
 
@@ -653,7 +702,9 @@ function DataStore._promiseGetAsyncNoCache(self: DataStore): Promise.Promise<()>
 									:Then(function()
 										-- Give enough time for Roblox to replicate changes
 										-- We probably could bump back to the loop but this has slightly better error messages
-										return maid:GivePromise(PromiseUtils.delayed(5))
+										return maid:GivePromise(
+											PromiseUtils.delayed(self._sessionMessagingCloseDelaySeconds)
+										)
 									end)
 									:Then(function()
 										return maid:GivePromise(promiseLoadUnlockedProfile(canStealLock, false))
@@ -682,7 +733,13 @@ function DataStore._promiseGetAsyncNoCache(self: DataStore): Promise.Promise<()>
 
 						return lockResult.lockedProfile, userIdList, metadata
 					end)
-				)
+				):Catch(function(opError)
+					-- The datastore operation itself failed (e.g. 509), which is NOT lock contention and
+					-- will not resolve by retrying. Fail the load fast, preserving the original error.
+					if loadPromise:IsPending() then
+						loadPromise:Reject(DataStoreNonRetryableLoadError.new(opError))
+					end
+				end)
 			end)
 
 			loadPromise:Finally(function()
@@ -692,18 +749,27 @@ function DataStore._promiseGetAsyncNoCache(self: DataStore): Promise.Promise<()>
 			return loadPromise
 		end
 
+		local retryOptions = {
+			exponential = self._loadRetryOptions.exponential,
+			initialWaitTime = self._loadRetryOptions.initialWaitTime,
+			maxAttempts = self._loadRetryOptions.maxAttempts,
+			printWarning = self._loadRetryOptions.printWarning,
+			-- Retry lock contention (the holder may release); fail fast on a fatal op failure.
+			shouldRetry = function(err)
+				return not DataStoreNonRetryableLoadError.isNonRetryableLoadError(err)
+			end,
+		}
+
 		promise = self._maid
 			:Add(PromiseRetryUtils.retry(function()
 				return promiseLoadUnlockedProfile(false, true)
-			end, {
-				-- https://exponentialbackoffcalculator.com/
-				-- 49.242 seconds
-				exponential = 1.25,
-				initialWaitTime = 6,
-				maxAttempts = 6,
-				printWarning = true,
-			}))
+			end, retryOptions))
 			:Catch(function(err)
+				-- A fatal op failure is not made loadable by stealing the lock; propagate it.
+				if DataStoreNonRetryableLoadError.isNonRetryableLoadError(err) then
+					return Promise.rejected(err)
+				end
+
 				warn(
 					string.format(
 						"DataStorePromises.updateAsync(%q) (will steal session) -> warning - %s",
@@ -724,7 +790,7 @@ function DataStore._promiseGetAsyncNoCache(self: DataStore): Promise.Promise<()>
 				)
 
 				self._promiseSessionLockingFailed:Resolve()
-				return Promise.rejected(err)
+				return Promise.rejected(DataStoreNonRetryableLoadError.unwrapLoadError(err))
 			end)
 	else
 		promise = self._maid
@@ -758,6 +824,11 @@ function DataStore._promiseGetAsyncNoCache(self: DataStore): Promise.Promise<()>
 			)
 			-- print(string.format("DataStorePromises.getAsync(%q) -> Got ", self._key), data)
 		end
+	end, function(err)
+		-- Propagate the load failure. The explicit onRejected also marks the upstream promise as
+		-- consumed, so a cancellation rejection (e.g. the store destroyed mid-load) is not surfaced
+		-- as an uncaught exception on this branch.
+		return Promise.rejected(err)
 	end)
 end
 

package/src/Server/DataStore.spec.lua

@@ -0,0 +1,237 @@
+--!nonstrict
+--[[
+	Integration coverage for DataStore against a mocked Roblox datastore: the load/save round-trip
+	and how it behaves when datastore operations fail (e.g. the 509 Personal-RCC block).
+
+	@class DataStore.spec.lua
+]]
+local require = require(script.Parent.loader).load(script)
+
+local DataStoreMock = require("DataStoreMock")
+local DataStoreTestUtils = require("DataStoreTestUtils")
+local Jest = require("Jest")
+local PromiseTestUtils = require("PromiseTestUtils")
+
+local describe = Jest.Globals.describe
+local expect = Jest.Globals.expect
+local it = Jest.Globals.it
+
+-- Asserts the promise settled within the timeout, so a hung promise fails the test (here) instead of
+-- freezing the runner on the following :Yield().
+local function expectSettled(promise, timeout: number?)
+	expect(PromiseTestUtils.awaitSettled(promise, timeout)).toEqual(true)
+end
+
+describe("DataStore without session locking", function()
+	it("should load the default value when the key is empty", function()
+		local controller = DataStoreTestUtils.setup()
+		local dataStore = controller.newDataStore()
+
+		local promise = dataStore:Load("coins", 99)
+		expectSettled(promise)
+
+		local ok, value = promise:Yield()
+		expect(ok).toEqual(true)
+		expect(value).toEqual(99)
+
+		controller:destroy()
+	end)
+
+	it("should round-trip a stored value through the datastore", function()
+		local controller = DataStoreTestUtils.setup()
+
+		local writer = controller.newDataStore()
+		writer:Store("coins", 5)
+
+		local savePromise = writer:Save()
+		expectSettled(savePromise)
+		expect((savePromise:Yield())).toEqual(true)
+
+		local reader = controller.newDataStore()
+		local loadPromise = reader:Load("coins")
+		expectSettled(loadPromise)
+
+		local ok, value = loadPromise:Yield()
+		expect(ok).toEqual(true)
+		expect(value).toEqual(5)
+
+		controller:destroy()
+	end)
+
+	it("should round-trip multiple keys and load defaults for missing ones", function()
+		local controller = DataStoreTestUtils.setup()
+
+		local writer = controller.newDataStore()
+		writer:Store("coins", 5)
+		writer:Store("gems", 10)
+		expectSettled(writer:Save())
+
+		local reader = controller.newDataStore()
+		local promise = reader:LoadAll()
+		expectSettled(promise)
+
+		local ok, all = promise:Yield()
+		expect(ok).toEqual(true)
+		expect(all.coins).toEqual(5)
+		expect(all.gems).toEqual(10)
+
+		local missingPromise = reader:Load("missing", "default")
+		expectSettled(missingPromise)
+		expect((missingPromise:Wait())).toEqual("default")
+
+		controller:destroy()
+	end)
+
+	it("should round-trip substore values", function()
+		local controller = DataStoreTestUtils.setup()
+
+		local writer = controller.newDataStore()
+		writer:GetSubStore("inventory"):Store("sword", true)
+		expectSettled(writer:Save())
+
+		local reader = controller.newDataStore()
+		local promise = reader:GetSubStore("inventory"):Load("sword")
+		expectSettled(promise)
+
+		local ok, value = promise:Yield()
+		expect(ok).toEqual(true)
+		expect(value).toEqual(true)
+
+		controller:destroy()
+	end)
+
+	it("should delete a key so it no longer loads", function()
+		local controller = DataStoreTestUtils.setup()
+
+		local writer = controller.newDataStore()
+		writer:Store("a", 1)
+		writer:Store("b", 2)
+		expectSettled(writer:Save())
+
+		writer:Delete("a")
+		expectSettled(writer:Save())
+
+		local reader = controller.newDataStore()
+		local promise = reader:LoadAll()
+		expectSettled(promise)
+
+		local ok, all = promise:Yield()
+		expect(ok).toEqual(true)
+		expect(all.a).toEqual(nil)
+		expect(all.b).toEqual(2)
+
+		controller:destroy()
+	end)
+
+	it("should resolve a save when nothing is staged", function()
+		local controller = DataStoreTestUtils.setup()
+		local dataStore = controller.newDataStore()
+
+		expectSettled(dataStore:Load("x"))
+
+		local savePromise = dataStore:Save()
+		expectSettled(savePromise)
+		expect((savePromise:Yield())).toEqual(true)
+
+		controller:destroy()
+	end)
+
+	it("should report load failure (not hang) when the datastore is unavailable", function()
+		local controller = DataStoreTestUtils.setup()
+		controller.mock:FailAllRequests()
+		local dataStore = controller.newDataStore()
+
+		local promise = dataStore:PromiseLoadSuccessful()
+		expectSettled(promise, 5)
+
+		local ok, loadedOk = promise:Yield()
+		expect(ok).toEqual(true)
+		expect(loadedOk).toEqual(false)
+
+		controller:destroy()
+	end)
+
+	it("should reject a save when the datastore goes down after loading", function()
+		local controller = DataStoreTestUtils.setup()
+		local dataStore = controller.newDataStore()
+
+		expectSettled(dataStore:Load("x"))
+
+		controller.mock:FailAllRequests()
+		dataStore:Store("x", 1)
+
+		local savePromise = dataStore:Save()
+		expectSettled(savePromise, 5)
+		expect((savePromise:Yield())).toEqual(false)
+
+		controller:destroy()
+	end)
+
+	it("should mark DidLoadFail after a failed load", function()
+		local controller = DataStoreTestUtils.setup()
+		controller.mock:FailAllRequests()
+		local dataStore = controller.newDataStore()
+
+		local promise = dataStore:PromiseLoadSuccessful()
+		expectSettled(promise, 5)
+
+		expect(dataStore:DidLoadFail()).toEqual(true)
+
+		controller:destroy()
+	end)
+end)
+
+describe("DataStore with session locking", function()
+	it("should load successfully against a healthy datastore", function()
+		local controller = DataStoreTestUtils.setup()
+
+		local dataStore = controller.newDataStore()
+		dataStore:SetSessionLockingEnabled(true)
+		dataStore:SetUserIdList({ 1 })
+
+		local promise = dataStore:PromiseLoadSuccessful()
+		expectSettled(promise, 10)
+
+		local ok, loadedOk = promise:Yield()
+		expect(ok).toEqual(true)
+		expect(loadedOk).toEqual(true)
+
+		controller:destroy()
+	end)
+
+	it("surfaces a persistent datastore failure fast instead of hanging", function()
+		local controller = DataStoreTestUtils.setup()
+		controller.mock:FailAllRequests()
+
+		local dataStore = controller.newDataStore()
+		dataStore:SetSessionLockingEnabled(true)
+		dataStore:SetUserIdList({ 1 })
+
+		local promise = dataStore:PromiseLoadSuccessful()
+		expectSettled(promise, 5)
+
+		local ok, loadedOk = promise:Yield()
+		expect(ok).toEqual(true)
+		expect(loadedOk).toEqual(false)
+
+		controller:destroy()
+	end)
+
+	it("rejects a failed session-locked load with the preserved datastore error", function()
+		local controller = DataStoreTestUtils.setup()
+		controller.mock:FailAllRequests(DataStoreMock.OPERATION_NOT_ALLOWED_509)
+
+		local dataStore = controller.newDataStore()
+		dataStore:SetSessionLockingEnabled(true)
+		dataStore:SetUserIdList({ 1 })
+
+		-- PromiseLoadSuccessful maps the error to a boolean, so read the rejection from
+		-- PromiseViewUpToDate to assert the underlying datastore error propagates.
+		local outcome, err = PromiseTestUtils.awaitOutcome(dataStore:PromiseViewUpToDate())
+
+		expect(outcome).toEqual("rejected")
+		expect(string.find(tostring(err), "509", 1, true) ~= nil).toEqual(true)
+
+		controller:destroy()
+	end)
+end)