@rbxts/scribe 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/LICENSE +21 -0
  2. package/NOTICE +21 -0
  3. package/README.md +100 -0
  4. package/Scribe-source-map.json +1 -0
  5. package/licenses/ProfileStore-Apache-2.0.txt +201 -0
  6. package/package.json +32 -0
  7. package/src/Client/DebugHook.luau +457 -0
  8. package/src/Client/init.luau +1607 -0
  9. package/src/Internal/Addons.luau +68 -0
  10. package/src/Internal/Big.luau +582 -0
  11. package/src/Internal/Budget.luau +377 -0
  12. package/src/Internal/Clock.luau +75 -0
  13. package/src/Internal/Config.luau +17 -0
  14. package/src/Internal/Conform.luau +400 -0
  15. package/src/Internal/Datatypes.luau +653 -0
  16. package/src/Internal/Defaults.luau +12 -0
  17. package/src/Internal/Derived.luau +428 -0
  18. package/src/Internal/Exchange.luau +556 -0
  19. package/src/Internal/ExchangeLeg.luau +1368 -0
  20. package/src/Internal/ExchangeVerdict.luau +179 -0
  21. package/src/Internal/Failure.luau +148 -0
  22. package/src/Internal/Health.luau +320 -0
  23. package/src/Internal/LeaderboardDiagnostics.luau +36 -0
  24. package/src/Internal/Log.luau +378 -0
  25. package/src/Internal/Marker.luau +76 -0
  26. package/src/Internal/Metrics.luau +152 -0
  27. package/src/Internal/Node.luau +3954 -0
  28. package/src/Internal/Profile.luau +69 -0
  29. package/src/Internal/Rng.luau +83 -0
  30. package/src/Internal/Sched.luau +56 -0
  31. package/src/Internal/Signal.luau +199 -0
  32. package/src/Internal/Store/ProfileStore.luau +3019 -0
  33. package/src/Internal/Store/init.luau +18 -0
  34. package/src/Internal/StudioCodec.luau +155 -0
  35. package/src/Internal/Template.luau +1848 -0
  36. package/src/Internal/Transport.luau +169 -0
  37. package/src/Internal/Util.luau +517 -0
  38. package/src/Internal/Version.luau +4 -0
  39. package/src/Internal/Wire.luau +1283 -0
  40. package/src/Server/Analytics.luau +223 -0
  41. package/src/Server/Commands.luau +728 -0
  42. package/src/Server/DebugHook.luau +1490 -0
  43. package/src/Server/Exchanges.luau +737 -0
  44. package/src/Server/Leaderboards.luau +2143 -0
  45. package/src/Server/Monetization.luau +3212 -0
  46. package/src/Server/Persistence.luau +3775 -0
  47. package/src/Server/Replication.luau +1253 -0
  48. package/src/Server/ServerStore.luau +81 -0
  49. package/src/Server/Timed.luau +579 -0
  50. package/src/Server/init.luau +672 -0
  51. package/src/Types.luau +480 -0
  52. package/src/init.luau +2231 -0
  53. package/types/api.d.ts +252 -0
  54. package/types/common.d.ts +355 -0
  55. package/types/index.d.ts +226 -0
  56. package/types/schema.d.ts +276 -0
@@ -0,0 +1,3775 @@
1
+ --!strict
2
+ -- Persistence on ProfileStore: session lifecycle, fail-closed
3
+ -- migrations, offline access, version history/restore, save observability,
4
+ -- wipe guard, and BindToClose.
5
+
6
+ local Players = game:GetService("Players")
7
+ local HttpService = game:GetService("HttpService")
8
+
9
+ local Internal = script.Parent.Parent.Internal
10
+ local Types = require(script.Parent.Parent.Types)
11
+ local Util = require(Internal.Util)
12
+ local Node = require(Internal.Node)
13
+ local Template = require(Internal.Template)
14
+ local Defaults = require(Internal.Defaults)
15
+ local Conform = require(Internal.Conform)
16
+ local DerivedModule = require(Internal.Derived)
17
+ local Log = require(Internal.Log)
18
+ local Metrics = require(Internal.Metrics)
19
+ local Health = require(Internal.Health)
20
+ local Signal = require(Internal.Signal)
21
+ local Config = require(Internal.Config)
22
+ local Clock = require(Internal.Clock)
23
+ local Sched = require(Internal.Sched)
24
+ local Profile = require(Internal.Profile)
25
+ local Failure = require(Internal.Failure)
26
+ local Budget = require(Internal.Budget)
27
+ local Rng = require(Internal.Rng)
28
+
29
+ local Persistence = {}
30
+
31
+ export type Entry = {
32
+ Player: Player,
33
+ UserId: number,
34
+ Key: string,
35
+ JoinedAt: number?, -- monotonic, stamped at the join; drives LoadDuration
36
+ State: "Loading" | "Ready" | "SessionEnded",
37
+ FailReason: string?,
38
+ Profile: any?,
39
+ Tree: any?,
40
+ SessionStore: { [string]: any },
41
+ LiveData: { [string]: any }?,
42
+ StateChanged: any,
43
+ SaveInfo: { LastSaveAt: number?, LastResult: ("Ok" | "Fail")?, Dirty: boolean },
44
+ SaveT0: number?,
45
+ -- Saves on the wire right now. The store permits N concurrent saves, so a single
46
+ -- slot cannot attribute a failure that arrives after the first completion.
47
+ SavesInFlight: number,
48
+ -- Dirtiness as it stood before ANY in-flight save snapshotted its payload, so the
49
+ -- OnError handler can restore it when one fails. OR in at every snapshot and clear
50
+ -- only when SavesInFlight drains to zero; a per-save capture lets a later save
51
+ -- overwrite an earlier one's value. nil and false both mean "nothing says dirty", so
52
+ -- a store exposing no OnSettled never clears this and can only ever over-report.
53
+ DirtyAtSaveStart: boolean?,
54
+ Blocked: boolean,
55
+ ForceSaveOnce: boolean,
56
+ WipeGuardRebaselinePending: boolean?,
57
+ LocalSessionEnd: boolean?, -- set before every locally-initiated EndSession
58
+ -- Timed's live _Scribe.Timed dict while a save is in flight, held so the restore
59
+ -- can run from OnAfterSave or, if the save fails, from the OnError handler.
60
+ TimedHeld: { [string]: number }?,
61
+ TimedLive: { [string]: number }?, -- same table, read by Timed's own store lookup
62
+ TimedSaves: number?, -- open save windows; saves are not serialized per profile
63
+ LeavingHookRan: boolean?, -- OnPlayerLeaving fires at most once per session
64
+ StopLeavingHookWatchdog: (() -> ())?,
65
+ -- Cross-server messages handed to Data.OnMessage whose FireTracked has not settled.
66
+ -- Nonzero at session end means a handler never returned; teardown reports it.
67
+ UnsettledMessages: number?,
68
+ SessionStolen: boolean?, -- OnSessionEnd fired that we did not initiate (external steal)
69
+ LastSavedSize: number?,
70
+ -- Top-level keys of the last successful save. The wipe guard's baseline, kept
71
+ -- here so the store does not have to keep a second copy of the whole profile.
72
+ LastSavedKeys: { [string]: true }?,
73
+ LiveSize: number?, -- approx bytes measured at the start of the last save attempt
74
+ SizeWarned: boolean,
75
+ TornDown: boolean,
76
+ -- Replication state (owned by Replication.luau)
77
+ HelloReceived: boolean,
78
+ -- Two flags, deliberately: QueueOpen means a snapshot has been taken, so a write
79
+ -- belongs in the queue; HandshakeReady means the snapshot is on the wire, so this
80
+ -- client may be sent frames. One flag left a gap between those two moments in
81
+ -- which a write belonged nowhere and was silently dropped.
82
+ QueueOpen: boolean,
83
+ HandshakeReady: boolean,
84
+ InitDelivered: boolean, -- a snapshot has reached this client
85
+ LastInitAt: number,
86
+ -- A frame did not reach the transport, so this client's mirror is stale and it
87
+ -- cannot ask for a repair (a loaded client never re-Hellos). The flush loop
88
+ -- re-sends the whole snapshot; LastResyncAt throttles repeats so a permanently
89
+ -- broken transport cannot cost one encode per frame.
90
+ ResyncPending: boolean?,
91
+ LastResyncAt: number?,
92
+ Queue: { any },
93
+ QueueIndex: { [string]: number },
94
+ SharedQueue: { any },
95
+ MalformedCount: number,
96
+ MalformedBlocked: boolean?, -- crossed the malformed-frame limit; inbound frames ignored
97
+ LastMalformedLog: number?, -- os.clock of the last emitted malformed-frame log/anomaly
98
+ OversizeCount: number, -- own budget: an oversize frame is never read, a malformed one was
99
+ OversizeBlocked: boolean?, -- crossed the oversize limit; inbound frames ignored
100
+ LastOversizeLog: number?, -- os.clock of the last emitted INBOUND_OVERSIZE
101
+ -- Raw inbound frame budget: every frame the player sends, whatever its type, size
102
+ -- or validity, spent before the frame is looked at at all.
103
+ FrameTokens: number,
104
+ FrameLast: number,
105
+ LastInboundRateLog: number?, -- os.clock of the last emitted INBOUND_RATE_LIMITED
106
+ -- Commands state
107
+ RateTokens: number,
108
+ RateLast: number,
109
+ RateLimitedCount: number?, -- consecutive rate-limited frames (decays when quiet)
110
+ LastRateLimitAt: number?, -- os.clock of the last rate-limited frame
111
+ LastRateLimitLog: number?, -- os.clock of the last emitted COMMAND_RATE_LIMITED
112
+ -- Why the store last failed for this key. StartSessionAsync answers nil with no reason
113
+ -- of its own, so this is the only way a load failure can tell a throttle apart.
114
+ LastStoreErrorClass: Failure.Class?,
115
+ LastStoreErrorAt: number?, -- monotonic
116
+ -- A parallel field, not a fourth State, so nothing switching on State changes
117
+ -- meaning. nil once Ready.
118
+ LoadPhase: Types.LoadPhase?,
119
+ SlowLoadWarned: boolean?,
120
+ StopLoadWatchdog: (() -> ())?,
121
+ -- Monetization state
122
+ GiftLastAt: number,
123
+ OwnsLast: { [string]: boolean }?, -- last reported effective ownership, for change detection
124
+ OwnsWatchers: { () -> () }?, -- Perks/OwnsCache listeners driving OnOwnershipChanged
125
+ }
126
+
127
+ local SIZE_WARN_BYTES = 3.5 * 1024 * 1024 -- warn approaching the ~4 MB DataStore ceiling
128
+ -- Past this a join is something the player notices. Well under LoadTimeout, where they
129
+ -- are kicked: a warn firing only at the timeout would only ever report failures.
130
+ local SLOW_LOAD_SECONDS = 10
131
+ local SLOW_LEAVING_HOOK_SECONDS = 10
132
+ -- An import is unbounded by design and holds the session lock throughout. Well past
133
+ -- SLOW_LOAD, because waiting on budget is the point.
134
+ local SLOW_IMPORT_SECONDS = 30
135
+ local IMPORT_SLOT_POLL = 0.25
136
+ local DEFAULT_MIGRATION_CONCURRENCY = 2
137
+ -- Held back from migrations. Nothing in a join CALLS GetAsync (ProfileStore loads
138
+ -- through UpdateAsync), but an UpdateAsync charges the read pool too, so a join spends
139
+ -- this anyway: one read and one write per join, measured in SimBudget. That dual charge
140
+ -- is undocumented (Uncertain.luau U12), so a join-rate derivation would rest on it.
141
+ local GET_RESERVE = 2
142
+ -- Ceiling on the GetSortedAsync reserve, which follows the declared board count: one
143
+ -- read per board lets a refresh tick through while a migration drains the pool. Capped
144
+ -- because boards come due on their own intervals, not together.
145
+ local SORTED_RESERVE_CAP = 4
146
+ local MIN_SAVE_INTERVAL = 15 -- seconds; below this, per-key DataStore throttling causes save failures
147
+ local WIPE_GUARD_RATIO_DEFAULT = 0.6 -- a profile shrinking past 60% looks like a wipe, not like spending
148
+
149
+ -- Matches the store's ASSUME_DEAD. Its own constant rather than read from the module,
150
+ -- so the operator paths behave identically against an injected store that lacks it.
151
+ local ASSUME_DEAD_SECONDS = 630
152
+
153
+ -- Share of the shutdown budget the OnPlayerLeaving phase may spend. Hooks come first
154
+ -- because their writes are the only thing here that cannot be rebuilt; the remainder
155
+ -- is a floor under the leaderboard flush, not a cap on it.
156
+ local HOOK_PHASE_SHARE = 0.75
157
+ local HOOK_JOIN_POLL = 0.05 -- seconds between checks that the parked hooks have returned
158
+
159
+ local function watchdog(seconds: number, callback: () -> ()): () -> ()
160
+ local pending: (() -> ())? = callback
161
+ Clock.Delay(seconds, function()
162
+ local run = pending
163
+ pending = nil
164
+ if run then
165
+ run()
166
+ end
167
+ end)
168
+ return function()
169
+ pending = nil -- release the entry without cancelling a game callback
170
+ end
171
+ end
172
+
173
+ --- A copy of the exchange ledger, for comparing across a hook that writes raw.
174
+ ---
175
+ --- OnPlayerInit and Scribe.Dynamic factories are handed `profile.Data` DIRECTLY, before
176
+ --- the tree exists, so the accessor refusal that protects the reserved root has nothing
177
+ --- to refuse through. A starter kit that clears `_Scribe` to "start clean" would destroy
178
+ --- an in-flight exchange with no report at all.
179
+ local function exchangeSnapshot(data: any): any
180
+ local reserved = if type(data) == "table" then data[Template.RESERVED_ROOT] else nil
181
+ if type(reserved) ~= "table" then
182
+ return nil
183
+ end
184
+ return {
185
+ Exchange = if type(reserved.Exchange) == "table" then Util.deepClone(reserved.Exchange) else nil,
186
+ ExchangeInbox = if type(reserved.ExchangeInbox) == "table" then Util.deepClone(reserved.ExchangeInbox) else nil,
187
+ }
188
+ end
189
+
190
+ --- Deep equality over two exchange snapshots, tolerating either side being absent.
191
+ local function sameExchangeState(a: any, b: any): boolean
192
+ local function eq(x: any, y: any): boolean
193
+ if type(x) ~= type(y) then
194
+ return false
195
+ end
196
+ if type(x) ~= "table" then
197
+ return x == y
198
+ end
199
+ for key, value in x do
200
+ if not eq(value, y[key]) then
201
+ return false
202
+ end
203
+ end
204
+ for key in y do
205
+ if x[key] == nil then
206
+ return false
207
+ end
208
+ end
209
+ return true
210
+ end
211
+ return eq(a, b)
212
+ end
213
+
214
+ --- Does this stored profile hold exchange state that a wipe or a rollback would split?
215
+ --- Value in flight sits in the reserved root while the matching hole sits in game data,
216
+ --- so rolling one half back leaves the item in the inventory AND in the escrow, or takes
217
+ --- it out of the inventory with the inbox entry already cleared.
218
+ local function exchangeStateOf(data: any): (number, number)
219
+ local reserved = if type(data) == "table" then data[Template.RESERVED_ROOT] else nil
220
+ if type(reserved) ~= "table" then
221
+ return 0, 0
222
+ end
223
+ local staked, parked = 0, 0
224
+ if type(reserved.Exchange) == "table" then
225
+ for _ in reserved.Exchange do
226
+ staked += 1
227
+ end
228
+ end
229
+ if type(reserved.ExchangeInbox) == "table" then
230
+ for _ in reserved.ExchangeInbox do
231
+ parked += 1
232
+ end
233
+ end
234
+ return staked, parked
235
+ end
236
+
237
+ -- Hand the store module Scribe's error classifier: the fork has no `require` of its own,
238
+ -- and MessageAsync needs it to answer whether an attempt inside its retry loop could have
239
+ -- written (the value a gift-credit spend refunds on). A fork that finds no classifier
240
+ -- assumes every failure might have written.
241
+ local function installErrorClassifier(store: any): any
242
+ if type(store) == "table" then
243
+ store.ClassifyError = Failure.Classify
244
+ end
245
+ return store
246
+ end
247
+
248
+ local function findProfileStoreModule(injected: any): any
249
+ if type(injected) == "table" then
250
+ return installErrorClassifier(injected)
251
+ end
252
+ if typeof(injected) == "Instance" and (injected :: Instance):IsA("ModuleScript") then
253
+ return installErrorClassifier(require(injected) :: any)
254
+ end
255
+ -- Scribe ships its own store (src/Internal/Store, a vendored fork of ProfileStore
256
+ -- 1.0.3), so there is nothing to search for. Do not restore the old
257
+ -- ServerPackages/Packages hunt: it would bind to an unpatched build whose Save()
258
+ -- cannot report an outcome.
259
+ return installErrorClassifier(require(Internal.Store) :: any)
260
+ end
261
+
262
+ function Persistence.new(ctx: any)
263
+ local self = {} :: any
264
+ local options = ctx.Options
265
+
266
+ -- Player-facing kick text, by cause. Two families, each with its own catch-all:
267
+ -- Fallback is the family option, so setting LoadFailureMessage or SessionEndMessage
268
+ -- still covers everything in that family, and naming one cause overrides only it.
269
+ local KICK_TEXT: { [string]: { Option: string, Fallback: string?, Default: string } } = {
270
+ -- Load family.
271
+ LegacyImport = {
272
+ Option = "LegacyImportFailureMessage",
273
+ Fallback = "LoadFailureMessage",
274
+ Default = "We couldn't check your existing data, and we didn't want to start you from scratch. Please rejoin!",
275
+ },
276
+ Migration = {
277
+ Option = "MigrationFailureMessage",
278
+ Fallback = "LoadFailureMessage",
279
+ Default = "We couldn't update your data to this version of the game. Please rejoin!",
280
+ },
281
+ VersionAhead = {
282
+ Option = "VersionAheadMessage",
283
+ Fallback = "LoadFailureMessage",
284
+ Default = "Your data is from a newer version of the game than this server. Please rejoin!",
285
+ },
286
+ Schema = {
287
+ Option = "SchemaFailureMessage",
288
+ Fallback = "LoadFailureMessage",
289
+ Default = "Your saved data doesn't match this version of the game. Please rejoin!",
290
+ },
291
+ RateLimited = {
292
+ Option = "RateLimitedMessage",
293
+ Fallback = "LoadFailureMessage",
294
+ Default = "Roblox's data service is busy right now. Please rejoin in a moment!",
295
+ },
296
+ Erasing = {
297
+ Option = "ErasingMessage",
298
+ Default = "Your data is being reset. Please rejoin in a few minutes!",
299
+ },
300
+ Load = {
301
+ Option = "LoadFailureMessage",
302
+ Default = "We couldn't load your data. Please rejoin!",
303
+ },
304
+ -- Session-end family.
305
+ SessionStolen = {
306
+ Option = "SessionStolenMessage",
307
+ Fallback = "SessionEndMessage",
308
+ Default = "Your data was opened on another server, so this session ended. Please rejoin!",
309
+ },
310
+ SessionInterrupted = {
311
+ Option = "SessionInterruptedMessage",
312
+ Fallback = "SessionEndMessage",
313
+ Default = "Your data session ended before it finished loading. Please rejoin!",
314
+ },
315
+ SessionEnd = {
316
+ Option = "SessionEndMessage",
317
+ Default = "Your data session has ended. Please rejoin!",
318
+ },
319
+ }
320
+
321
+ local function kickText(reason: string): string
322
+ local spec = KICK_TEXT[reason]
323
+ local specific = options[spec.Option]
324
+ if type(specific) == "string" then
325
+ return specific
326
+ end
327
+ if spec.Fallback ~= nil then
328
+ local family = options[spec.Fallback]
329
+ if type(family) == "string" then
330
+ return family
331
+ end
332
+ end
333
+ return spec.Default
334
+ end
335
+
336
+ -- Windowed: the class is stamped whenever the store errors on this key, and an error
337
+ -- from minutes ago says nothing about the attempt that just failed.
338
+ local THROTTLE_ATTRIBUTION_WINDOW = 15
339
+ local function loadKickText(entry: Entry): string
340
+ local at = entry.LastStoreErrorAt
341
+ if
342
+ entry.LastStoreErrorClass == "Throttled"
343
+ and at ~= nil
344
+ and (Clock.Monotonic() - at) <= THROTTLE_ATTRIBUTION_WINDOW
345
+ then
346
+ return kickText("RateLimited")
347
+ end
348
+ return kickText("Load")
349
+ end
350
+
351
+ -- Bound on ONE StartSessionAsync attempt. ProfileStore's own START_SESSION_TIMEOUT
352
+ -- applies only `if params.Cancel == nil` and Scribe always passes a Cancel, so
353
+ -- without this the load never returns during a DataStore outage and everything below
354
+ -- it (PROFILE_LOAD_FAIL, LoadFailurePolicy) is unreachable.
355
+ -- Floored at 60s: ProfileStore steals a held session after SESSION_STEAL (40s), so a
356
+ -- shorter deadline aborts every legitimate cross-server handoff.
357
+ local loadTimeout: number = math.max(60, tonumber(options.LoadTimeout) or 120)
358
+ local eraseJoinTimeout: number = math.max(0.1, tonumber(options.EraseJoinTimeout) or 10)
359
+ local compiled: Template.Compiled = ctx.Compiled
360
+
361
+ -- Storage location is explicit and never defaulted (see Scribe() guard).
362
+ local storeIndex = options.ProfileStoreIndex
363
+ if type(storeIndex) ~= "string" or storeIndex == "" then
364
+ error("Scribe: ProfileStoreIndex is required: name your game's DataStore explicitly", 0)
365
+ end
366
+ -- "" is allowed and means bare user-id keys (see the Scribe() guard); only an
367
+ -- absent or non-string prefix is an error.
368
+ if type(options.ProfileKeyPrefix) ~= "string" then
369
+ error(
370
+ 'Scribe: ProfileKeyPrefix is required: set a per-player key prefix (e.g. "PLAYER_"), or "" for bare user-id keys',
371
+ 0
372
+ )
373
+ end
374
+
375
+ local ProfileStore = findProfileStoreModule(options.ProfileStore)
376
+
377
+ -- Take shutdown off the store and drive it from Scribe's own drain. Roblox runs bound
378
+ -- BindToClose callbacks concurrently in an undocumented order, and the store's
379
+ -- releases every profile before its first yield, while Scribe's needs them still
380
+ -- active to run each player's OnPlayerLeaving hook.
381
+ if type(ProfileStore) == "table" and ProfileStore.ExternalShutdown ~= nil then
382
+ ProfileStore.ExternalShutdown = true
383
+ end
384
+ -- The PLAYER view throughout: a store root must never reach a profile payload, a
385
+ -- per-player session store, or the per-player snapshot.
386
+ local playerCompiled = compiled.Views.Player
387
+ local persistentDefaults = Template.BuildDefaults(playerCompiled, "Persistent")
388
+ local baseStore = ProfileStore.New(storeIndex, persistentDefaults)
389
+ -- Persistence mode: which store to read, whether to hold a session, whether anything
390
+ -- saves.
391
+ -- Live real store, exclusive session, saves (the default)
392
+ -- Mock ProfileStore's mock store, saves only there
393
+ -- NoSave real profile read as a snapshot, no session, never saves
394
+ -- Mode wins over the legacy flags when it is set; absent, they resolve to the
395
+ -- equivalent mode, so existing configs are untouched.
396
+ local VALID_MODES = { Live = true, Mock = true, NoSave = true }
397
+ local mode: string
398
+ local useMockStore: boolean
399
+ local targetUserId: number? = options.TargetUserId
400
+
401
+ if options.Mode ~= nil then
402
+ if type(options.Mode) ~= "string" or not VALID_MODES[options.Mode] then
403
+ error(`Scribe: Mode must be "Live", "Mock" or "NoSave", got {tostring(options.Mode)}`, 0)
404
+ end
405
+ mode = options.Mode
406
+ useMockStore = mode == "Mock"
407
+ local legacy: { string } = {}
408
+ for _, name in { "UseMock", "DontSave", "ViewedUserId", "OverriddenUserId" } do
409
+ if (options :: any)[name] ~= nil then
410
+ table.insert(legacy, name)
411
+ end
412
+ end
413
+ if #legacy > 0 then
414
+ Log.Warn(
415
+ "Persistence",
416
+ "MODE_OVERRIDES_LEGACY",
417
+ `Mode = "{mode}" is set, so the legacy flag(s) {table.concat(legacy, ", ")} are ignored`,
418
+ { Mode = mode, Ignored = table.concat(legacy, ", ") }
419
+ )
420
+ end
421
+ else
422
+ -- Legacy resolution. The old flags controlled two independent axes: the store
423
+ -- came from UseMock/DontSave, the view branch from ViewedUserId. Keep them
424
+ -- independent, or an existing config silently changes which store it reads.
425
+ useMockStore = (options.UseMock or options.DontSave) == true
426
+ if options.ViewedUserId ~= nil then
427
+ mode = "NoSave"
428
+ targetUserId = options.ViewedUserId
429
+ elseif useMockStore then
430
+ mode = "Mock"
431
+ else
432
+ mode = "Live"
433
+ targetUserId = targetUserId or options.OverriddenUserId
434
+ end
435
+ end
436
+
437
+ self.Mode = mode
438
+ self.TargetUserId = targetUserId
439
+
440
+ -- The offline verbs write the real key, which a NoSave bundle promises never to touch.
441
+ local function refuseNoSave(verb: string): string
442
+ local reason = `Mode = "NoSave" never writes the store, so {verb} is refused`
443
+ Log.Warn("Persistence", "NOSAVE_WRITE_REFUSED", reason, { Verb = verb })
444
+ return reason
445
+ end
446
+ self.RefuseNoSave = refuseNoSave
447
+
448
+ local store = if useMockStore then baseStore.Mock else baseStore
449
+
450
+ -- Lower ProfileStore's autosave cadence so less progress is at risk on an unclean
451
+ -- exit. AUTO_SAVE_PERIOD is a ProfileStore GLOBAL, so this affects every Scribe
452
+ -- bundle and any direct ProfileStore use in the game; the last value wins. Floored
453
+ -- at 15s because Roblox throttles per-key writes to roughly one per 6s.
454
+ local function applyAutoSaveInterval(interval: number, source: string)
455
+ if interval < MIN_SAVE_INTERVAL then
456
+ Log.Warn(
457
+ "Persistence",
458
+ "SAVE_INTERVAL_CLAMPED",
459
+ `autosave interval below the {MIN_SAVE_INTERVAL}s floor (DataStore write throttling); clamping`,
460
+ { Requested = interval, Applied = MIN_SAVE_INTERVAL }
461
+ )
462
+ interval = MIN_SAVE_INTERVAL
463
+ end
464
+ -- Two bundles asking for different cadences is a real conflict; unreported it
465
+ -- resolves as last-writer-wins, with the winner decided by require order.
466
+ if Config.AutoSaveInterval ~= nil and Config.AutoSaveInterval ~= interval then
467
+ Log.Warn(
468
+ "Persistence",
469
+ "SAVE_INTERVAL_CONFLICT",
470
+ "autosave interval is a ProfileStore process global and was already set to a different value; "
471
+ .. "the later value wins. Set it once with Scribe.Configure instead.",
472
+ { Existing = Config.AutoSaveInterval, Requested = interval, Source = source }
473
+ )
474
+ end
475
+ if type(ProfileStore.SetConstant) == "function" then
476
+ local ok, err = pcall(ProfileStore.SetConstant, "AUTO_SAVE_PERIOD", interval)
477
+ if ok then
478
+ Config.AutoSaveInterval = interval
479
+ Config.AutoSaveSource = source
480
+ Log.Info("Persistence", "SAVE_INTERVAL_SET", "autosave interval configured", {
481
+ Seconds = interval,
482
+ Source = source,
483
+ })
484
+ else
485
+ Log.Warn("Persistence", "SAVE_INTERVAL_FAIL", tostring(err), {})
486
+ end
487
+ end
488
+ end
489
+
490
+ -- Scribe.Configure runs before any bundle exists, so apply its value here,
491
+ -- where ProfileStore has been resolved.
492
+ if Config.AutoSaveSource == "Configure" and Config.AutoSaveInterval ~= nil then
493
+ local pending = Config.AutoSaveInterval
494
+ Config.AutoSaveInterval = nil -- let applyAutoSaveInterval own the record
495
+ applyAutoSaveInterval(pending, "Configure")
496
+ end
497
+
498
+ if options.SaveInterval ~= nil then
499
+ local interval = options.SaveInterval
500
+ if type(interval) ~= "number" or interval ~= interval or interval <= 0 then
501
+ error("Scribe: SaveInterval must be a positive number of seconds", 0)
502
+ end
503
+ applyAutoSaveInterval(interval, "SaveInterval")
504
+ end
505
+
506
+ self.Store = store
507
+ -- Connections to PROCESS-GLOBAL signals: they outlive the bundle unless released, so
508
+ -- Stop() disconnects them. Per-profile signals die with the profile.
509
+ local connections: { any } = {}
510
+ self.Entries = {} :: { [Player]: Entry }
511
+ -- Maintained alongside Entries rather than recomputed: an O(N) dict walk on every
512
+ -- teardown makes draining a full server O(N^2) in session count.
513
+ self.EntryCount = 0
514
+ -- userId -> Entry, so a lookup by userId is not a linear scan over Entries.
515
+ self.EntriesByUserId = {} :: { [number]: Entry }
516
+ self.Closing = false
517
+ -- Set by Data.Stop, deliberately not the same flag as Closing: Closing also aborts an
518
+ -- in-flight load through the Cancel hook, marks a session end as ours rather than a
519
+ -- steal, and quiets teardown, none of which apply to a Stop. It exists for the
520
+ -- LoadFailurePolicy = "Wait" retry loop, which never gives up and whose only other
521
+ -- exit is Closing.
522
+ self.Stopped = false
523
+ -- Why the last session ended, for WaitForData callers arriving after a
524
+ -- synchronous teardown (weak keys: entries vanish with the Player).
525
+ self.EndReasons = setmetatable({}, { __mode = "k" }) :: any
526
+
527
+ local keyPrefix: string = options.ProfileKeyPrefix
528
+ local migrations: { [number]: (data: any) -> () } = options.Migrations or {}
529
+ local currentVersion = 1
530
+ for version in migrations do
531
+ if type(version) ~= "number" or version % 1 ~= 0 or version < 2 then
532
+ error("Scribe: Migrations keys must be integers >= 2", 0)
533
+ end
534
+ currentVersion = math.max(currentVersion, version)
535
+ end
536
+ for version = 2, currentVersion do
537
+ if type(migrations[version]) ~= "function" then
538
+ error(`Scribe: Migrations table is missing step {version}`, 0)
539
+ end
540
+ end
541
+ self.CurrentVersion = currentVersion
542
+
543
+ -- ProfileStore health signals (member names verified against v1.0.3).
544
+ local baseStoreName = storeIndex
545
+ local okErr = pcall(function()
546
+ connections[#connections + 1] = ProfileStore.OnError:Connect(
547
+ function(message: string, storeName: string, profileKey: string)
548
+ -- OnError is module-global and fires for every DataStore error, other
549
+ -- stores included. Only our store counts toward Health, and only an
550
+ -- in-flight SAVE marks a save failure: a throttled version read for an
551
+ -- online player must not fail their receipts and save state.
552
+ if storeName ~= baseStoreName then
553
+ return
554
+ end
555
+ -- Classified here rather than in Failure.Report: this is the one site
556
+ -- that publishes the verdict and the raw code in its own log context, so
557
+ -- an error family Roblox adds later reads as unclassified rather than
558
+ -- silently transient.
559
+ local class = Failure.Classify(message)
560
+ -- DataStoreErrors counts per ATTEMPT, so one failing save produces N
561
+ -- errors here. SavesFailed is per-operation and stays behind the
562
+ -- in-flight guard below. Health is fed every error, throttles included.
563
+ Failure.Report({
564
+ Subsystem = "ProfileStore",
565
+ -- Keyed, so the store's own retries of one failing profile collapse
566
+ -- into one problem. Unkeyed, a single oversize profile drove the whole
567
+ -- server to Outage, which refuses Robux receipts for everyone on it.
568
+ Key = profileKey,
569
+ Error = message,
570
+ Class = class,
571
+ Metric = "DataStoreErrors",
572
+ Category = "Persistence",
573
+ Code = "PROFILE_STORE_ERROR",
574
+ Message = tostring(message),
575
+ Context = {
576
+ Store = storeName,
577
+ Key = profileKey,
578
+ Class = class,
579
+ -- nil when the message carried no code: the signal that either
580
+ -- Roblox changed the shape or this is not a store error at all.
581
+ Code = Failure.CodeOf(message),
582
+ },
583
+ })
584
+ for _, entry in self.Entries :: { [Player]: Entry } do
585
+ if entry.Key == profileKey then
586
+ -- Outside the in-flight guard: a load has no save on the wire, and
587
+ -- the load path is what needs the distinction. Dies with the entry.
588
+ entry.LastStoreErrorClass = class
589
+ entry.LastStoreErrorAt = Clock.Monotonic()
590
+ end
591
+ if entry.Key == profileKey and entry.SavesInFlight > 0 then
592
+ -- Attribute this failure to a save genuinely on the wire. A
593
+ -- single-slot SaveT0 let an overlapping success clear it first, so
594
+ -- LastResult stayed "Ok" and the Dirty restore never ran, while
595
+ -- Monetization reads both to decide whether a duplicate receipt may
596
+ -- be acknowledged. OnSettled fires exactly once per save call.
597
+ entry.SaveInfo.LastResult = "Fail"
598
+ -- A profile over the size-warning threshold fails EVERY UpdateAsync
599
+ -- until it is trimmed, which is not a transient throttle.
600
+ if entry.LiveSize and entry.LiveSize > SIZE_WARN_BYTES then
601
+ Log.Error(
602
+ "Persistence",
603
+ "PROFILE_TOO_LARGE",
604
+ `save failed and the profile is {entry.LiveSize} bytes (~4 MB DataStore ceiling); this will not recover until the profile is trimmed`,
605
+ { Player = entry.Player, Bytes = entry.LiveSize }
606
+ )
607
+ end
608
+ -- The failed save's payload never persisted: restore the dirtiness
609
+ -- captured at its start (plus anything written since).
610
+ entry.SaveInfo.Dirty = entry.DirtyAtSaveStart or entry.SaveInfo.Dirty
611
+ -- Deliberately NOT closing the Timed save window here. OnError cannot
612
+ -- prove the failing call was this entry's save, and closing it on an
613
+ -- unrelated error points the profile back at live DEADLINES that the
614
+ -- pending save then serializes as remaining seconds, so a rejoin
615
+ -- reads a cooldown decades out. The next BeforeSave re-derives the
616
+ -- persisted copy from the live table anyway.
617
+ ctx.Signals.OnSave:Fire({ Player = entry.Player, Ok = false, Duration = nil, At = Clock.Now() })
618
+ Metrics.Add("SavesFailed")
619
+ self.UpdateSaveInfoMirror(entry)
620
+ end
621
+ end
622
+ end
623
+ )
624
+ end)
625
+ if not okErr then
626
+ Log.Warn(
627
+ "Persistence",
628
+ "PROFILE_STORE_SIGNAL_MISSING",
629
+ "ProfileStore.OnError not connectable; save-failure observability disabled",
630
+ {}
631
+ )
632
+ end
633
+ -- ProfileStore replaces a stored value it cannot recognise as a profile with a fresh
634
+ -- template and fires OnOverwrite. Silent, irreversible data loss: the overwritten
635
+ -- profile comes back at SessionLoadCount 1, so isNew is true, migrations are skipped
636
+ -- and OnPlayerInit runs as if for a first-time player.
637
+ local okOverwrite = pcall(function()
638
+ connections[#connections + 1] = ProfileStore.OnOverwrite:Connect(function(storeName: string, profileKey: string)
639
+ -- OnOverwrite is module-global; only our store is ours to report on.
640
+ if storeName ~= baseStoreName then
641
+ return
642
+ end
643
+ -- The entry may not exist yet (the overwrite happens during the load that
644
+ -- creates it), so the player is best-effort and the key is the reliable id.
645
+ local entry: Entry? = nil
646
+ for _, candidate in self.Entries :: { [Player]: Entry } do
647
+ if candidate.Key == profileKey then
648
+ entry = candidate
649
+ break
650
+ end
651
+ end
652
+ Failure.Report({
653
+ Subsystem = "ProfileOverwrite",
654
+ Key = profileKey,
655
+ -- No Error: OnOverwrite carries no message to classify.
656
+ Metric = "ProfilesOverwritten",
657
+ Level = "Error",
658
+ Category = "Integrity",
659
+ Code = "PROFILE_OVERWRITTEN",
660
+ Message = "stored data for this key was not a valid profile and has been REPLACED with a fresh "
661
+ .. "template by ProfileStore; this session starts from defaults and the previous data is gone. "
662
+ .. "Restore from version history before the player saves over it",
663
+ Context = { Player = if entry then entry.Player else nil, Store = storeName, Key = profileKey },
664
+ })
665
+ ctx.Signals.OnAnomaly:Fire({
666
+ Player = if entry then entry.Player else nil,
667
+ Path = {},
668
+ Value = nil,
669
+ Reason = "ProfileOverwritten",
670
+ })
671
+ end)
672
+ end)
673
+ if not okOverwrite then
674
+ Log.Warn(
675
+ "Persistence",
676
+ "PROFILE_STORE_SIGNAL_MISSING",
677
+ "ProfileStore.OnOverwrite not connectable; silent corruption-overwrite detection disabled",
678
+ {}
679
+ )
680
+ end
681
+ local okCrit = pcall(function()
682
+ connections[#connections + 1] = ProfileStore.OnCriticalToggle:Connect(function(isCritical: boolean)
683
+ Health.SetCritical(isCritical)
684
+ if isCritical then
685
+ Log.Error("Persistence", "DATASTORE_CRITICAL", "ProfileStore entered critical state", {})
686
+ else
687
+ Log.Info("Persistence", "DATASTORE_RECOVERED", "ProfileStore left critical state", {})
688
+ end
689
+ end)
690
+ end)
691
+ if not okCrit then
692
+ Log.Warn("Persistence", "PROFILE_STORE_SIGNAL_MISSING", "ProfileStore.OnCriticalToggle not connectable", {})
693
+ end
694
+
695
+ function self.KeyFor(userId: number): string
696
+ return keyPrefix .. tostring(userId)
697
+ end
698
+
699
+ function self.EntryFor(player: Player): Entry?
700
+ return self.Entries[player]
701
+ end
702
+
703
+ function self.EntryForUserId(userId: number): Entry?
704
+ return self.EntriesByUserId[userId]
705
+ end
706
+
707
+ local function warnSlowLoad(entry: Entry)
708
+ if self.Stopped or entry.SlowLoadWarned or entry.State ~= "Loading" or entry.JoinedAt == nil then
709
+ return
710
+ end
711
+ local duration = Clock.Monotonic() - entry.JoinedAt
712
+ if duration < SLOW_LOAD_SECONDS then
713
+ return
714
+ end
715
+ entry.SlowLoadWarned = true
716
+ Log.Warn("Persistence", "SLOW_LOAD", "profile load exceeded 10s; inspect its phase and store errors", {
717
+ Player = entry.Player,
718
+ UserId = entry.UserId,
719
+ Phase = entry.LoadPhase or "Initializing",
720
+ Duration = duration,
721
+ })
722
+ end
723
+
724
+ local function setState(entry: Entry, state: "Loading" | "Ready" | "SessionEnded", reason: string?)
725
+ if state ~= "Loading" then
726
+ warnSlowLoad(entry)
727
+ if entry.StopLoadWatchdog then
728
+ entry.StopLoadWatchdog()
729
+ entry.StopLoadWatchdog = nil
730
+ end
731
+ entry.LoadPhase = nil
732
+ end
733
+ entry.State = state
734
+ entry.FailReason = reason
735
+ entry.StateChanged:Fire(state, reason)
736
+ end
737
+
738
+ --- Fires StateChanged so a WaitForData caller wakes on the transition.
739
+ local function setPhase(entry: Entry, phase: Types.LoadPhase?)
740
+ if entry.LoadPhase == phase then
741
+ return
742
+ end
743
+ entry.LoadPhase = phase
744
+ entry.StateChanged:Fire(entry.State, entry.FailReason)
745
+ end
746
+
747
+ -- AwaitBudget is cooperative (a hook that never calls it is unpaced), so the cap is
748
+ -- the only hard bound on what a migrating server spends.
749
+ local importsRunning = 0
750
+
751
+ --- Wait for a slot, then hand back a release. The caller must run it on every exit
752
+ --- or the cap leaks a slot per import.
753
+ local function takeImportSlot(entry: Entry): () -> ()
754
+ local cap = math.max(1, tonumber(options.MigrationConcurrency) or DEFAULT_MIGRATION_CONCURRENCY)
755
+ while importsRunning >= cap and not entry.TornDown do
756
+ setPhase(entry, "ImportThrottled")
757
+ Clock.Wait(IMPORT_SLOT_POLL)
758
+ end
759
+ importsRunning += 1
760
+ setPhase(entry, "Importing")
761
+ local released = false
762
+ return function()
763
+ if not released then
764
+ released = true
765
+ importsRunning -= 1
766
+ end
767
+ end
768
+ end
769
+
770
+ --- What ImportLegacyData and OnPlayerInit receive. The wait reports itself, so a
771
+ --- throttled migration is visible without the hook doing anything beyond calling it.
772
+ local function migrationContext(entry: Entry)
773
+ return {
774
+ AwaitBudget = function(requestType: string, count: number?, timeout: number?): (boolean, () -> ())
775
+ if type(requestType) ~= "string" then
776
+ error(
777
+ `Scribe: migration.AwaitBudget(requestType, count?, timeout?): requestType must be a `
778
+ .. `string, got {typeof(requestType)}`,
779
+ 2
780
+ )
781
+ end
782
+ return Budget.Await(requestType, count, timeout, function(waiting)
783
+ setPhase(entry, if waiting then "ImportThrottled" else "Importing")
784
+ end, function()
785
+ return entry.TornDown
786
+ end)
787
+ end,
788
+ }
789
+ end
790
+
791
+ --- UpdateAsync tracks the session count because its worst case is the shutdown
792
+ --- drain: every online session saving at once. Process-wide, so a stopped bundle
793
+ --- leaves them alone.
794
+ local function syncBudgetReserves()
795
+ if self.Stopped then
796
+ return
797
+ end
798
+ Budget.SetReserve("UpdateAsync", self.EntryCount)
799
+ Budget.SetReserve("GetAsync", GET_RESERVE)
800
+ local boards = if ctx.Leaderboards ~= nil then ctx.Leaderboards.BoardCount or 0 else 0
801
+ Budget.SetReserve("GetSortedAsync", math.min(boards, SORTED_RESERVE_CAP))
802
+ end
803
+
804
+ function self.UpdateSaveInfoMirror(entry: Entry)
805
+ if entry.Tree and not entry.Tree.Dead then
806
+ local accessor = entry.Tree.Accessor[Template.SESSION_ROOT].SaveInfo
807
+ accessor.Set({
808
+ LastSaveAt = entry.SaveInfo.LastSaveAt,
809
+ LastResult = entry.SaveInfo.LastResult,
810
+ Dirty = entry.SaveInfo.Dirty,
811
+ Size = entry.LastSavedSize,
812
+ })
813
+ end
814
+ end
815
+
816
+ -- Deep-compare stored profile data against the current template defaults, skipping
817
+ -- Scribe.Dynamic leaves (their samples differ per server boot) and comparing packed
818
+ -- buffers by content. Detects a profile whose first session crashed before its
819
+ -- first save: still template-shaped, but SessionLoadCount already past 1.
820
+ local dynamicPathKeys: { [string]: boolean } = {}
821
+ for _, node in compiled.DynamicNodes do
822
+ dynamicPathKeys[Util.pathKey(node.Path :: any)] = true
823
+ end
824
+ local function templateEquals(a: any, b: any, pathKey: string?): boolean
825
+ if pathKey ~= nil and dynamicPathKeys[pathKey] then
826
+ return true
827
+ end
828
+ if type(a) == "buffer" and type(b) == "buffer" then
829
+ return buffer.tostring(a) == buffer.tostring(b)
830
+ end
831
+ if type(a) ~= type(b) then
832
+ return false
833
+ end
834
+ if type(a) ~= "table" then
835
+ return a == b
836
+ end
837
+ for k, v in a do
838
+ local childKey = if pathKey then pathKey .. "\31" .. tostring(k) else tostring(k)
839
+ if not templateEquals(v, (b :: { [any]: any })[k], childKey) then
840
+ return false
841
+ end
842
+ end
843
+ for k in b :: { [any]: any } do
844
+ if (a :: { [any]: any })[k] == nil then
845
+ return false
846
+ end
847
+ end
848
+ return true
849
+ end
850
+ local function equalsTemplateDefaults(data: { [string]: any }): boolean
851
+ return templateEquals(data, persistentDefaults, nil)
852
+ end
853
+
854
+ -- Commit a view-mode profile and report whether the write actually landed.
855
+ -- Profile:SetAsync reports nothing on failure: ProfileStore's own pcall swallows the
856
+ -- fault and the call returns normally having written nothing, so the offline receipt
857
+ -- path turns that into PurchaseGranted for Robux that were silently consumed.
858
+ -- RegisterError fires OnError synchronously before UpdateAsync returns, so a
859
+ -- connection held across the call is the only available failure signal.
860
+ local function commitViewProfile(profile: any, key: string): (boolean, string?)
861
+ local failure: string? = nil
862
+ local ok, conn = pcall(function()
863
+ return ProfileStore.OnError:Connect(function(message: string, _storeName: string, profileKey: string)
864
+ if profileKey == key and failure == nil then
865
+ failure = tostring(message)
866
+ end
867
+ end)
868
+ end)
869
+ local setOk, setErr = pcall(function()
870
+ profile:SetAsync()
871
+ end)
872
+ if ok and conn then
873
+ pcall(function()
874
+ conn:Disconnect()
875
+ end)
876
+ end
877
+ if not setOk then
878
+ return false, tostring(setErr) -- a real misuse error, not infrastructure
879
+ end
880
+ if failure ~= nil then
881
+ return false, `DataStore error during save: {failure}`
882
+ end
883
+ if not ok then
884
+ -- Could not observe OnError at all, so success here is unverified. Say so
885
+ -- rather than reporting a commit we cannot stand behind.
886
+ return false, "could not observe ProfileStore.OnError; save result unverifiable"
887
+ end
888
+ return true, nil
889
+ end
890
+
891
+ -- Asymmetric diff for the migration shadow in finishLoad. Reports a path differing on
892
+ -- both sides, or present only on the shadow side; keys only on the REAL side are
893
+ -- Reconcile's backfill and would fire for every legacy profile. RESERVED_ROOT is
894
+ -- skipped whole: runMigrations stamps its Version and the shadow loop never does.
895
+ local function divergences(shadow: any, real: any, path: { string | number }, out: { string })
896
+ if #out >= 5 then
897
+ return
898
+ end
899
+ if type(shadow) == "buffer" and type(real) == "buffer" then
900
+ if buffer.tostring(shadow) ~= buffer.tostring(real) then
901
+ table.insert(out, Util.pathString(path))
902
+ end
903
+ return
904
+ end
905
+ if type(shadow) ~= type(real) then
906
+ table.insert(out, Util.pathString(path))
907
+ return
908
+ end
909
+ if type(shadow) ~= "table" then
910
+ if shadow ~= real then
911
+ table.insert(out, Util.pathString(path))
912
+ end
913
+ return
914
+ end
915
+ for k, v in shadow do
916
+ if #path == 0 and k == Template.RESERVED_ROOT then
917
+ continue
918
+ end
919
+ table.insert(path, tostring(k))
920
+ divergences(v, (real :: any)[k], path, out)
921
+ table.remove(path)
922
+ if #out >= 5 then
923
+ return
924
+ end
925
+ end
926
+ end
927
+
928
+ -- Bounds on how far the profile may shrink before the wipe guard calls it a wipe.
929
+ -- Both ends fail quietly: at 0 any shrink trips the guard, and under WipeGuardPolicy
930
+ -- = "Block" it then keeps tripping, so live data never persists again. At 1 or above
931
+ -- it can never fire, and negative makes (1 - ratio) exceed 1, so a profile that GREW
932
+ -- trips it. The floor is not near-zero: a few percent of churn is ordinary gameplay.
933
+ local WIPE_GUARD_RATIO_MIN = 0.05
934
+ local WIPE_GUARD_RATIO_MAX = 0.95
935
+ -- Block policy writes the last-good BYTES back as the save payload, so it is the one
936
+ -- setting that needs the store to keep its full copy. Same variable here and at the
937
+ -- retention opt-out in finishLoad: if the two disagree, the guard persists an EMPTY
938
+ -- profile, which is the wipe it exists to prevent.
939
+ local wipeGuardBlocks: boolean = (options.WipeGuardPolicy or "Warn") == "Block"
940
+ local wipeGuardRatio = WIPE_GUARD_RATIO_DEFAULT
941
+ do
942
+ local configured = options.WipeGuardShrinkRatio
943
+ if configured ~= nil then
944
+ local asNumber = tonumber(configured)
945
+ local clamped = if asNumber == nil or asNumber ~= asNumber
946
+ then WIPE_GUARD_RATIO_DEFAULT
947
+ else math.clamp(asNumber, WIPE_GUARD_RATIO_MIN, WIPE_GUARD_RATIO_MAX)
948
+ if clamped ~= configured then
949
+ Log.Warn(
950
+ "Persistence",
951
+ "WIPE_GUARD_RATIO_CLAMPED",
952
+ `WipeGuardShrinkRatio must be between {WIPE_GUARD_RATIO_MIN} and {WIPE_GUARD_RATIO_MAX}: `
953
+ .. "at or below zero every shrink trips the guard (and under Block the profile never "
954
+ .. "saves again), at or above one it can never trip and the guard is off",
955
+ { Requested = configured, Applied = clamped }
956
+ )
957
+ end
958
+ wipeGuardRatio = clamped
959
+ end
960
+ end
961
+
962
+ -- Wipe guard: compare against the last successful save. `liveSize` is passed in
963
+ -- because onProfileSaving measures the same table in the same synchronous frame.
964
+ local function wipeGuardCheck(entry: Entry, liveSize: number?): (boolean, string?)
965
+ local profile = entry.Profile
966
+ local live = entry.LiveData
967
+ if not profile or not live then
968
+ return false, nil
969
+ end
970
+ -- The baseline is a KEY SET and a size, not the bytes, so the store need not hold
971
+ -- a second full copy of the profile. The store's copy is still the baseline in
972
+ -- the two cases the summary cannot be: before the first save of a session, and
973
+ -- under WipeGuardPolicy = "Block", which writes those bytes back as the payload.
974
+ local keys: any = entry.LastSavedKeys
975
+ local oldSize = entry.LastSavedSize
976
+ if keys == nil then
977
+ local lastSaved = profile.LastSavedData
978
+ if type(lastSaved) ~= "table" or next(lastSaved) == nil then
979
+ return false, nil
980
+ end
981
+ keys = lastSaved
982
+ oldSize = oldSize or Util.approxSize(lastSaved)
983
+ end
984
+ local missing = {}
985
+ for key in keys do
986
+ if live[key] == nil then
987
+ table.insert(missing, tostring(key))
988
+ end
989
+ end
990
+ if #missing > 0 then
991
+ return true, `top-level keys vanished: {table.concat(missing, ", ")}`
992
+ end
993
+ local ratio = wipeGuardRatio
994
+ local newSize = liveSize or Util.approxSize(live)
995
+ if oldSize and newSize and oldSize > 1024 and newSize < oldSize * (1 - ratio) then
996
+ return true, `serialized size collapsed {oldSize} -> {newSize} bytes`
997
+ end
998
+ return false, nil
999
+ end
1000
+
1001
+ -- Wrapped rather than edited in place: the body returns early on the ForceSaveOnce
1002
+ -- and rebaseline paths, so a capture at the bottom of it would miss them. Reading
1003
+ -- entry.Blocked here runs synchronously inside the save, before it yields, so the
1004
+ -- flag still describes this payload rather than a later save's.
1005
+ local onProfileSavingInner: (entry: Entry) -> ()
1006
+
1007
+ local function onProfileSaving(entry: Entry, saveContext: { [string]: any }?)
1008
+ onProfileSavingInner(entry)
1009
+ if saveContext then
1010
+ saveContext.Blocked = entry.Blocked == true
1011
+ end
1012
+ end
1013
+
1014
+ function onProfileSavingInner(entry: Entry)
1015
+ -- The EARLIEST live start, not this save's. The slot is shared by every save in
1016
+ -- flight, so assigning would let a later save overwrite an earlier one's t0 and
1017
+ -- under-report its duration. Cleared on the drain to zero in OnSettled.
1018
+ entry.SaveT0 = entry.SaveT0 or Clock.Monotonic()
1019
+ entry.SavesInFlight += 1
1020
+ -- This save's payload is serialized NOW: reset Dirty so it re-accumulates from
1021
+ -- writes landing after the snapshot, and keep the pre-snapshot state so a failed
1022
+ -- save can restore it (OnError path). ACCUMULATED, not assigned: with N concurrent
1023
+ -- saves a second one would read back the false the first wrote, and the first
1024
+ -- one's failure would restore clean over an unstored write. Cleared on the drain.
1025
+ entry.DirtyAtSaveStart = entry.DirtyAtSaveStart or entry.SaveInfo.Dirty
1026
+ entry.SaveInfo.Dirty = false
1027
+ local profile = entry.Profile
1028
+ if not profile or not entry.LiveData then
1029
+ return
1030
+ end
1031
+
1032
+ -- Measure the payload on the way IN, not only on save success: a profile past
1033
+ -- the ~4 MB DataStore ceiling makes every UpdateAsync throw, so the
1034
+ -- success-path check below would never run for it and the developer would see
1035
+ -- only generic retryable-looking failures. Latched to once per crossing.
1036
+ local liveSize = Util.approxSize(entry.LiveData)
1037
+ if liveSize then
1038
+ entry.LiveSize = liveSize
1039
+ if liveSize > SIZE_WARN_BYTES and not entry.SizeWarned then
1040
+ entry.SizeWarned = true
1041
+ Log.Warn(
1042
+ "Persistence",
1043
+ "PROFILE_SIZE",
1044
+ `profile approaching the ~4 MB DataStore ceiling ({liveSize} bytes); trim it or saves will start failing`,
1045
+ { Player = entry.Player, Bytes = liveSize }
1046
+ )
1047
+ ctx.Signals.OnAnomaly:Fire({ Player = entry.Player, Path = {}, Value = nil, Reason = "ProfileSize" })
1048
+ elseif liveSize <= SIZE_WARN_BYTES then
1049
+ entry.SizeWarned = false
1050
+ end
1051
+ end
1052
+
1053
+ if entry.ForceSaveOnce then
1054
+ entry.ForceSaveOnce = false
1055
+ if entry.Blocked then
1056
+ entry.Blocked = false
1057
+ profile:SetSaveSource(entry.LiveData)
1058
+ Log.Warn("Integrity", "WIPE_GUARD_FORCED", "blocked save forced through", { Player = entry.Player })
1059
+ end
1060
+ return
1061
+ end
1062
+
1063
+ if entry.WipeGuardRebaselinePending then
1064
+ -- First save after a reshaping migration: the LastSavedData baseline predates
1065
+ -- it, so a comparison would false-trip on any key the migration removed.
1066
+ -- Cleared on save SUCCESS (onProfileSaved), not here: this hook fires per
1067
+ -- ATTEMPT, and clearing on a failed one re-arms the guard against the stale
1068
+ -- baseline, which under Block policy stalls the migration permanently.
1069
+ return
1070
+ end
1071
+
1072
+ local tripped, reason = wipeGuardCheck(entry, liveSize)
1073
+ if tripped then
1074
+ Log.Error("Integrity", "WIPE_GUARD_TRIPPED", reason :: string, { Player = entry.Player })
1075
+ Metrics.Add("WipeGuardTrips")
1076
+ ctx.Signals.OnAnomaly:Fire({
1077
+ Player = entry.Player,
1078
+ Path = {},
1079
+ Value = nil,
1080
+ Reason = "WipeGuard: " .. (reason :: string),
1081
+ })
1082
+ if wipeGuardBlocks then
1083
+ -- Persist the last good snapshot instead; the live tree keeps
1084
+ -- its own root table. Unblocked when the guard passes again or
1085
+ -- via Data.Flush(player, { Force = true }).
1086
+ if not entry.Blocked then
1087
+ entry.Blocked = true
1088
+ end
1089
+ profile:SetSaveSource(Util.deepClone(profile.LastSavedData))
1090
+ Log.Warn("Integrity", "WIPE_GUARD_BLOCKED", "save held; persisting last good snapshot", {
1091
+ Player = entry.Player,
1092
+ })
1093
+ end
1094
+ elseif entry.Blocked then
1095
+ entry.Blocked = false
1096
+ profile:SetSaveSource(entry.LiveData)
1097
+ Log.Info("Integrity", "WIPE_GUARD_CLEARED", "wipe guard cleared; live data persists again", {
1098
+ Player = entry.Player,
1099
+ })
1100
+ end
1101
+ end
1102
+
1103
+ local function onProfileSaved(entry: Entry, lastSavedData: any)
1104
+ -- ProfileStore fires OnAfterSave even when the session was STOLEN and nothing
1105
+ -- local persisted. Skip the success bookkeeping only for steals: a
1106
+ -- locally-initiated final save (clean leave, shutdown drain, migration
1107
+ -- fail-closed) genuinely persisted, and dropping it would skew Health.
1108
+ if entry.TornDown and entry.SessionStolen then
1109
+ return
1110
+ end
1111
+ local duration = if entry.SaveT0 then Clock.Monotonic() - entry.SaveT0 else nil
1112
+ entry.SaveInfo.LastSaveAt = Clock.Now()
1113
+ entry.SaveInfo.LastResult = "Ok"
1114
+ -- Keep any dirtiness accumulated from writes landing AFTER this save's payload
1115
+ -- snapshot (onProfileSaving reset the flag at that moment); blocked saves
1116
+ -- persisted old data, so live data stays unsaved regardless.
1117
+ entry.SaveInfo.Dirty = entry.SaveInfo.Dirty or entry.Blocked
1118
+ -- A successful, unblocked save re-established the wipe-guard baseline with the
1119
+ -- migrated shape; the one-time post-migration skip is complete.
1120
+ if not entry.Blocked then
1121
+ entry.WipeGuardRebaselinePending = false
1122
+ end
1123
+ Metrics.Add("SavesOk")
1124
+ if duration then
1125
+ Metrics.Observe("SaveDuration", duration)
1126
+ end
1127
+ Health.ReportSuccess("Saves")
1128
+ ctx.Signals.OnSave:Fire({ Player = entry.Player, Ok = true, Duration = duration, At = Clock.Now() })
1129
+
1130
+ -- Record the size that actually persisted (the warning + anomaly are owned by
1131
+ -- the pre-save check in onProfileSaving, which also fires when a save fails).
1132
+ local size = Util.approxSize(lastSavedData)
1133
+ if size then
1134
+ entry.LastSavedSize = size
1135
+ entry.LiveSize = size
1136
+ Metrics.Observe("ProfileSize", size)
1137
+ end
1138
+ -- The other half of the wipe guard's baseline, from the payload that actually
1139
+ -- persisted rather than from live data: a blocked save stores the last-good
1140
+ -- snapshot, and the baseline has to describe what is on the key.
1141
+ if type(lastSavedData) == "table" then
1142
+ local keys: { [string]: true } = {}
1143
+ for key in lastSavedData do
1144
+ keys[key :: string] = true
1145
+ end
1146
+ entry.LastSavedKeys = keys
1147
+ end
1148
+
1149
+ self.UpdateSaveInfoMirror(entry)
1150
+ end
1151
+
1152
+ -- Does this stored session belong to a server that is still alive?
1153
+ -- `profile.Session` is a straight read of MetaData.ActiveSession with no staleness
1154
+ -- allowance, and a server that dies before its final save leaves it set forever. The
1155
+ -- store steals a session whose LastUpdate is older than ASSUME_DEAD, so mirroring
1156
+ -- that threshold keeps RestoreVersion and Erase from staying refused permanently for
1157
+ -- the profile that most needs them. No usable LastUpdate reads as LIVE, the safe
1158
+ -- direction.
1159
+ local function sessionLooksLive(profile: any): boolean
1160
+ if not profile or not profile.Session then
1161
+ return false
1162
+ end
1163
+ local lastUpdate = profile.LastUpdate
1164
+ if type(lastUpdate) ~= "number" or lastUpdate <= 0 then
1165
+ return true
1166
+ end
1167
+ return (Clock.Now() - lastUpdate) <= ASSUME_DEAD_SECONDS
1168
+ end
1169
+
1170
+ -- Release the lock WITHOUT writing Data back. Every fail-closed load path promises the
1171
+ -- stored bytes are left as found, and EndSession is an unconditional save: by the time
1172
+ -- those paths run, profile.Data has been through CoerceMapKeys, Reconcile, the version
1173
+ -- stamp and the migration chain. Falls back to a plain EndSession for an injected
1174
+ -- store without the primitive: unrejoinable is worse than rewritten.
1175
+ local function releaseWithoutWriting(profile: { EndSessionWithoutSaving: ((any) -> ())?, EndSession: (any) -> () })
1176
+ if type(profile.EndSessionWithoutSaving) == "function" then
1177
+ profile:EndSessionWithoutSaving()
1178
+ else
1179
+ profile:EndSession()
1180
+ end
1181
+ end
1182
+
1183
+ -- Deep structural comparison for the reserved-root discard check in runMigrations.
1184
+ -- Nothing existing fits: Util.leafEquals is false for tables, Node's entryEquals is
1185
+ -- schema-driven, and templateEquals above skips nodes by root-relative path key, so it
1186
+ -- would skip a reserved child sharing a name with a Dynamic root field. Buffers
1187
+ -- compare by identity under `==`, hence the byte compare.
1188
+ local function deepSame(a: any, b: any): boolean
1189
+ if type(a) == "buffer" and type(b) == "buffer" then
1190
+ return Util.bufferEquals(a :: buffer, b :: buffer)
1191
+ end
1192
+ if type(a) ~= type(b) then
1193
+ return false
1194
+ end
1195
+ if type(a) ~= "table" then
1196
+ return a == b
1197
+ end
1198
+ for k, v in a :: { [any]: any } do
1199
+ if not deepSame(v, (b :: { [any]: any })[k]) then
1200
+ return false
1201
+ end
1202
+ end
1203
+ for k in b :: { [any]: any } do
1204
+ if (a :: { [any]: any })[k] == nil then
1205
+ return false
1206
+ end
1207
+ end
1208
+ return true
1209
+ end
1210
+
1211
+ -- Do these two reserved roots agree? `compareVersion` decides whether the TOP-LEVEL
1212
+ -- `Version` key counts, and the two call sites want opposite answers:
1213
+ -- * MIGRATION passes `false`: the library re-stamps Version right after the chain.
1214
+ -- * RESTORE passes `true`: there is no re-stamp, so Version alone decides whether
1215
+ -- the chain re-runs against the older-shaped data the restore just wrote.
1216
+ -- A `Version` key one level down always counts, in both modes.
1217
+ local function reservedUnchanged(stored: { [string]: any }, other: any, compareVersion: boolean): boolean
1218
+ if type(other) ~= "table" then
1219
+ return false
1220
+ end
1221
+ for k, v in stored do
1222
+ if (compareVersion or k ~= "Version") and not deepSame(v, (other :: { [string]: any })[k]) then
1223
+ return false
1224
+ end
1225
+ end
1226
+ for k in other :: { [string]: any } do
1227
+ if (compareVersion or k ~= "Version") and stored[k] == nil then
1228
+ return false
1229
+ end
1230
+ end
1231
+ return true
1232
+ end
1233
+
1234
+ local function runMigrationsInner(entry: Entry, data: { [string]: any }): (boolean, string?)
1235
+ local reserved = data[Template.RESERVED_ROOT]
1236
+ local stored = if type(reserved) == "table" and type(reserved.Version) == "number" then reserved.Version else 1
1237
+ if stored > currentVersion then
1238
+ -- Migrated by a newer deploy this server's code has never seen. Fail closed
1239
+ -- by default (VersionAheadPolicy "Kick") so an old server can't corrupt
1240
+ -- newer-shaped data; "Allow" opts into running old code against it
1241
+ -- (documented staged-deploy hazard).
1242
+ local policy = options.VersionAheadPolicy or "Kick"
1243
+ if policy == "Kick" then
1244
+ Log.Error(
1245
+ "Persistence",
1246
+ "PROFILE_VERSION_AHEAD",
1247
+ "stored version is newer than this server's; failing closed",
1248
+ {
1249
+ Player = entry.Player,
1250
+ Stored = stored,
1251
+ Current = currentVersion,
1252
+ }
1253
+ )
1254
+ -- Not a migration failure: nothing ran, this server is just older than the data.
1255
+ return false, "version-ahead"
1256
+ end
1257
+ Log.Warn(
1258
+ "Persistence",
1259
+ "PROFILE_VERSION_AHEAD",
1260
+ "stored version is newer than this server's (VersionAheadPolicy=Allow)",
1261
+ {
1262
+ Player = entry.Player,
1263
+ Stored = stored,
1264
+ Current = currentVersion,
1265
+ }
1266
+ )
1267
+ return true
1268
+ end
1269
+ if stored == currentVersion then
1270
+ return true
1271
+ end
1272
+ -- Run migrations on a working clone; only commit on full success so a
1273
+ -- half-migrated profile can never persist.
1274
+ local work = Util.deepClone(data)
1275
+ for version = stored + 1, currentVersion do
1276
+ local ok, err = xpcall(function()
1277
+ migrations[version](work)
1278
+ end, function(e)
1279
+ return debug.traceback(tostring(e), 2)
1280
+ end)
1281
+ if not ok then
1282
+ Log.Error("Persistence", "MIGRATION_FAIL", `migration step {version} failed: {err}`, {
1283
+ Player = entry.Player,
1284
+ Step = version,
1285
+ })
1286
+ Metrics.Add("MigrationsFailed")
1287
+ return false
1288
+ end
1289
+ end
1290
+ -- PRESERVE THE RESERVED ROOT. Reconcile has already run, so nothing the library
1291
+ -- has gained since this profile was written is missing from the stored root.
1292
+ -- `_Scribe` is library-owned state (receipt dedupe
1293
+ -- ring, gift escrow, credits, perks, purchase logs, Timed deadlines) and the
1294
+ -- migration chain is entirely game-supplied, so a step that rebuilds the profile
1295
+ -- from its own key whitelist destroys receipt idempotency and paid-for gifts.
1296
+ -- Assigned BY REFERENCE: table.clear(data) below does not recurse, so this table
1297
+ -- survives the copy loop as the same object, and the migration was handed `work`,
1298
+ -- a deep clone, so nothing aliases. Ordered BEFORE the unpersistable scan so the
1299
+ -- scan sees exactly what commits.
1300
+ local storedReserved = data[Template.RESERVED_ROOT]
1301
+ local migratedReserved = work[Template.RESERVED_ROOT]
1302
+ if type(storedReserved) == "table" then
1303
+ work[Template.RESERVED_ROOT] = storedReserved
1304
+ -- Reverting a developer's edit in silence is its own trap, so a discard is
1305
+ -- reported at Error, which also fires OnIssue. `false`: the stamp below
1306
+ -- rewrites `Version` unconditionally.
1307
+ if not reservedUnchanged(storedReserved, migratedReserved, false) then
1308
+ Log.Error(
1309
+ "Persistence",
1310
+ "MIGRATION_RESERVED_DISCARDED",
1311
+ `a migration step changed the reserved "{Template.RESERVED_ROOT}" root; the change was DISCARDED `
1312
+ .. `and the stored reserved state kept (it is library-owned: receipt dedupe, gift escrow, `
1313
+ .. `perks, purchase logs, Timed) -- migrate your own keys only`,
1314
+ { Player = entry.Player, Path = Template.RESERVED_ROOT, Stored = stored, Current = currentVersion }
1315
+ )
1316
+ end
1317
+ elseif type(migratedReserved) ~= "table" then
1318
+ -- Nothing stored to preserve (a legacy or imported profile that never had a
1319
+ -- reserved root) and the chain left none either, so make somewhere for the
1320
+ -- stamp below to go. A root a migration CREATED is kept as it built it:
1321
+ -- nothing was destroyed, so nothing is reported.
1322
+ work[Template.RESERVED_ROOT] = {}
1323
+ end
1324
+
1325
+ -- Migration functions mutate `work` raw, bypassing Node.validateWrite, so a step
1326
+ -- introducing unpersistable data (invalid-UTF-8 string/key, or a
1327
+ -- userdata/Instance/function value) would fail every save. Scan BEFORE committing
1328
+ -- into `data`, so failure takes the fail-closed path below and the unmodified
1329
+ -- original persists.
1330
+ local badMigrated = Util.findUnpersistable(work)
1331
+ if badMigrated ~= nil then
1332
+ Log.Error(
1333
+ "Persistence",
1334
+ "MIGRATION_FAIL",
1335
+ `migration produced unpersistable data ({badMigrated.Detail}) at "{Util.pathString(badMigrated.Path)}"`,
1336
+ { Player = entry.Player, Path = Util.pathString(badMigrated.Path) }
1337
+ )
1338
+ Metrics.Add("MigrationsFailed")
1339
+ return false
1340
+ end
1341
+
1342
+ -- The stamp goes HERE, below every `return false` above. On the preserve path
1343
+ -- `work[RESERVED_ROOT]` IS `data[RESERVED_ROOT]`, so this writes straight into the
1344
+ -- live profile.Data. Stamped earlier, a rejected migration leaves the caller's
1345
+ -- all-default carve-out comparing against a profile already marked migrated, and
1346
+ -- it kicks the player that carve-out exists to load.
1347
+ work[Template.RESERVED_ROOT].Version = currentVersion
1348
+ table.clear(data)
1349
+ for k, v in work do
1350
+ data[k] = v
1351
+ end
1352
+ Log.Info("Persistence", "MIGRATED", `profile migrated {stored} -> {currentVersion}`, { Player = entry.Player })
1353
+ -- A migration may legitimately remove or rename top-level keys, but the wipe
1354
+ -- guard's baseline still holds the pre-migration shape until the first save
1355
+ -- commits. Skip the guard for that one save, or under WipeGuardPolicy = "Block"
1356
+ -- it re-persists the old data and stalls the migration permanently.
1357
+ entry.WipeGuardRebaselinePending = true
1358
+ return true
1359
+ end
1360
+
1361
+ --- Not spanned, deliberately: a migration is developer code that may yield, and an
1362
+ --- annotation does not survive a yield. LoadDuration already covers what it costs.
1363
+ local function runMigrations(entry: Entry, data: { [string]: any }): (boolean, string?)
1364
+ return runMigrationsInner(entry, data)
1365
+ end
1366
+
1367
+ local function buildTree(entry: Entry)
1368
+ local profile = entry.Profile
1369
+ entry.LiveData = profile.Data
1370
+ entry.SessionStore = Template.BuildDefaults(playerCompiled, "Session")
1371
+
1372
+ entry.Tree = Node.NewTree({
1373
+ Compiled = playerCompiled,
1374
+ Stores = { Persist = profile.Data, Session = entry.SessionStore },
1375
+ IsServer = true,
1376
+ Label = entry.Player.Name,
1377
+ OnOp = function(op: Node.Op, schema: Template.SchemaNode?)
1378
+ local persists = if schema then schema.Persist else Template.PersistOf(compiled, op.Path)
1379
+ if persists and not entry.SaveInfo.Dirty then
1380
+ entry.SaveInfo.Dirty = true
1381
+ Sched.Defer(function()
1382
+ if entry.Tree and not entry.Tree.Dead and entry.SaveInfo.Dirty then
1383
+ self.UpdateSaveInfoMirror(entry)
1384
+ end
1385
+ end)
1386
+ end
1387
+ ctx.Replication.PushOp(entry, op, schema)
1388
+ end,
1389
+ OnAnomaly = function(path, value, reason)
1390
+ Metrics.Add("Anomalies")
1391
+ Log.Warn("Integrity", "ANOMALY", `implausible write at "{Util.pathString(path)}": {reason}`, {
1392
+ Player = entry.Player,
1393
+ Path = Util.pathString(path),
1394
+ Reason = reason,
1395
+ })
1396
+ ctx.Signals.OnAnomaly:Fire({ Player = entry.Player, Path = path, Value = value, Reason = reason })
1397
+ end,
1398
+ BoundsPolicy = options.BoundsPolicy or "Clamp",
1399
+ DevMode = ctx.DevMode,
1400
+ TimedHooks = ctx.Timed.HooksFor(entry),
1401
+ OnEconomy = function(path, delta, newValue, tags)
1402
+ ctx.Analytics.Economy(entry.Player, path, delta, newValue, tags)
1403
+ end,
1404
+ LogError = function(message)
1405
+ Log.Error("Lifecycle", "LISTENER_ERROR", message, { Player = entry.Player })
1406
+ end,
1407
+ LogWarn = function(message)
1408
+ Log.Warn("Lifecycle", "DEV_WARNING", message, { Player = entry.Player })
1409
+ end,
1410
+ OnBatchEnd = function()
1411
+ ctx.Replication.FlushEntry(entry)
1412
+ end,
1413
+ OnDerivedIssue = function(code, field, detail)
1414
+ -- MISMATCH is a dev-mode purity report, not a live fault; the other
1415
+ -- two mean a field is serving a value its inputs no longer justify.
1416
+ local report: typeof(Log.Warn) = if code == "DERIVED_MISMATCH" then Log.Warn else Log.Error
1417
+ report("Derived", code :: any, `"{field}": {detail}`, { Player = entry.Player, Field = field })
1418
+ end,
1419
+ })
1420
+ end
1421
+
1422
+ -- Run the game's OnPlayerLeaving hook. Runs BEFORE EndSession, which performs the
1423
+ -- final save, so anything written here lands. Errors are contained: a throwing
1424
+ -- hook must not skip the save it was meant to contribute to.
1425
+ local function runLeavingHook(entry: Entry, reason: string)
1426
+ local hook = ctx.Options.OnPlayerLeaving
1427
+ -- The hook runs under a plain pcall, so one that yields parks the whole leave
1428
+ -- with the entry still in self.Entries and the shutdown drain then walks it
1429
+ -- again. A double fire double-counts the hook's writes into one final save.
1430
+ if hook == nil or entry.LeavingHookRan then
1431
+ return
1432
+ end
1433
+ entry.LeavingHookRan = true
1434
+ if entry.State ~= "Ready" or entry.Tree == nil then
1435
+ return -- never loaded, so there is nothing coherent to hand it
1436
+ end
1437
+ local started = Clock.Monotonic()
1438
+ local warned = false
1439
+ local function warnSlowHook()
1440
+ local duration = Clock.Monotonic() - started
1441
+ if self.Stopped or warned or duration < SLOW_LEAVING_HOOK_SECONDS then
1442
+ return
1443
+ end
1444
+ warned = true
1445
+ Log.Warn("Lifecycle", "PLAYER_LEAVING_HOOK_SLOW", "OnPlayerLeaving exceeded 10s before the final save", {
1446
+ Player = entry.Player,
1447
+ UserId = entry.UserId,
1448
+ Reason = reason,
1449
+ Duration = duration,
1450
+ })
1451
+ end
1452
+ local stopWatchdog = watchdog(SLOW_LEAVING_HOOK_SECONDS, warnSlowHook)
1453
+ entry.StopLeavingHookWatchdog = stopWatchdog
1454
+ local ok, err = pcall(hook, entry.Player, entry.Tree.Accessor, reason)
1455
+ stopWatchdog()
1456
+ entry.StopLeavingHookWatchdog = nil
1457
+ Metrics.Observe("PlayerLeavingHookDuration", Clock.Monotonic() - started)
1458
+ warnSlowHook()
1459
+ if not ok then
1460
+ Log.Error(
1461
+ "Lifecycle",
1462
+ "LISTENER_ERROR",
1463
+ `OnPlayerLeaving errored; the save continues without its writes: {tostring(err)}`,
1464
+ { Player = entry.Player }
1465
+ )
1466
+ end
1467
+ end
1468
+
1469
+ local function teardown(entry: Entry, reason: string?)
1470
+ if entry.TornDown then
1471
+ return
1472
+ end
1473
+ entry.TornDown = true
1474
+ if entry.StopLeavingHookWatchdog then
1475
+ entry.StopLeavingHookWatchdog()
1476
+ entry.StopLeavingHookWatchdog = nil
1477
+ end
1478
+ -- One canonical reason reaches every consumer: the SessionEnded signal, the stored
1479
+ -- end reason, and WaitForData's second return (see Types.LifecycleReason).
1480
+ local canonical: string = reason or "session-ended"
1481
+ -- Before setState overwrites it: a mid-load kick never had a session to lose.
1482
+ local wasLoading = entry.State == "Loading"
1483
+ self.EndReasons[entry.Player] = canonical
1484
+
1485
+ -- A Data.OnMessage handler that never returns holds its FireTracked open forever,
1486
+ -- so no ack is emitted and the message is re-offered on every load, climbing
1487
+ -- toward MESSAGE_QUEUE_FULL. Reported at session end rather than on a timer: no
1488
+ -- threshold is right, since a handler spending a throttled round trip is slow
1489
+ -- rather than stuck. Warn, not Error: a player leaving while a handler is
1490
+ -- legitimately mid-yield reads identically, and only repetition tells them apart.
1491
+ local unsettled = entry.UnsettledMessages or 0
1492
+ if unsettled > 0 then
1493
+ Log.Warn(
1494
+ "Persistence",
1495
+ "MESSAGE_HANDLER_STALLED",
1496
+ "a Data.OnMessage handler had not returned when the session ended; the message was NOT "
1497
+ .. "acknowledged and will be redelivered on the next load",
1498
+ { Player = entry.Player, Outstanding = unsettled }
1499
+ )
1500
+ end
1501
+
1502
+ -- SessionEnded fires before the (default-on) kick.
1503
+ ctx.Signals.SessionEnded:Fire(entry.Player, canonical)
1504
+ setState(entry, "SessionEnded", canonical)
1505
+
1506
+ ctx.Replication.OnEntryRemoved(entry)
1507
+ ctx.Leaderboards.OnUnload(entry)
1508
+ ctx.Monetization.OnUnload(entry)
1509
+ ctx.Timed.OnUnload(entry)
1510
+ ctx.Exchanges.OnUnload(entry)
1511
+
1512
+ if entry.Tree then
1513
+ entry.Tree:Destroy()
1514
+ end
1515
+
1516
+ if
1517
+ options.KickOnSessionEnd ~= false
1518
+ and not self.Closing
1519
+ -- ...and not because this bundle was released. A Stop is not a session ending
1520
+ -- badly, so kicking would be a lie told by a bundle that no longer exists.
1521
+ and not self.Stopped
1522
+ and canonical ~= "player-left"
1523
+ and entry.Player.Parent == Players
1524
+ then
1525
+ -- Nearly always another server holding the profile now.
1526
+ local why = if entry.SessionStolen
1527
+ then "SessionStolen"
1528
+ elseif wasLoading then "SessionInterrupted"
1529
+ else "SessionEnd"
1530
+ entry.Player:Kick(kickText(why))
1531
+ end
1532
+
1533
+ -- Only adjust the maintained counters if this entry was still the live one
1534
+ -- for its player: teardown can run more than once for a session, and a
1535
+ -- rejoin can already have installed a NEWER entry under the same Player.
1536
+ if self.Entries[entry.Player] == entry then
1537
+ self.Entries[entry.Player] = nil
1538
+ self.EntryCount -= 1
1539
+ syncBudgetReserves()
1540
+ end
1541
+ if self.EntriesByUserId[entry.UserId] == entry then
1542
+ self.EntriesByUserId[entry.UserId] = nil
1543
+ end
1544
+ Metrics.Set("ActiveSessions", self.EntryCount)
1545
+ end
1546
+
1547
+ local function finishLoad(entry: Entry)
1548
+ local profile = entry.Profile :: any
1549
+ local player = entry.Player
1550
+
1551
+ -- Marked for erasure: release without saving.
1552
+ local markedReserved = if type(profile.Data) == "table" then profile.Data[Template.RESERVED_ROOT] else nil
1553
+ if type(markedReserved) == "table" and type(markedReserved.Erase) == "table" then
1554
+ entry.LocalSessionEnd = true
1555
+ releaseWithoutWriting(profile)
1556
+ if player.Parent == Players then
1557
+ player:Kick(kickText("Erasing"))
1558
+ end
1559
+ teardown(entry, "erasing")
1560
+ return
1561
+ end
1562
+
1563
+ -- Stop the store holding a second full copy of this profile all session. The wipe
1564
+ -- guard's top-level key set and its size are recorded on the entry by each
1565
+ -- successful save. Not under Block policy, which persists those bytes back. This
1566
+ -- nils profile.LastSavedData, not the OnAfterSave argument: the store still hands
1567
+ -- the real table to that handler, which is where the summary is taken.
1568
+ if not wipeGuardBlocks and type(profile.SetRetainLastSaved) == "function" then
1569
+ profile:SetRetainLastSaved(false)
1570
+ end
1571
+
1572
+ -- A brand-new profile's very first session ever. In-place ProfileStore adoption
1573
+ -- keeps the session count from the old game, so returning players still migrate.
1574
+ -- SessionsStarted rather than the raw SessionLoadCount: a view is fetched with
1575
+ -- GetAsync, which never runs the session-start increment, so read raw a dry run of
1576
+ -- an already-loaded profile is judged brand new and skips every migration.
1577
+ local isNew = profile:SessionsStarted() == 1
1578
+
1579
+ -- Snapshot BEFORE AddUserId mutates the in-memory list. The template-fresh check
1580
+ -- further down needs it: without it a returning player who never earned anything
1581
+ -- is indistinguishable from a crashed first session, and their migrations are
1582
+ -- skipped. Negative ids are excluded because AddUserId no-ops for them outside
1583
+ -- mock, so the signal can never accumulate.
1584
+ local hadPriorSave = entry.UserId >= 0 and profile:HasCompletedSave()
1585
+
1586
+ -- Decided once, so a refused reset loads like an ordinary join.
1587
+ local resetStaked, resetParked = 0, 0
1588
+ local willReset = false
1589
+ if options.ResetData then
1590
+ resetStaked, resetParked = exchangeStateOf(profile.Data)
1591
+ willReset = resetStaked == 0 and resetParked == 0
1592
+ end
1593
+
1594
+ -- Legacy import ---------------------------------------------------------
1595
+ -- Adopt a player's data from another library, ONCE, before Scribe has written for
1596
+ -- them. Gated on hadPriorSave rather than `isNew`: SessionLoadCount advances when
1597
+ -- a session STARTS, so a first join whose save never committed leaves the player
1598
+ -- non-new with nothing stored, and `isNew` would refuse the import while the empty
1599
+ -- profile Scribe then saved became canonical. Skipped under NoSave, where the dry
1600
+ -- run cannot persist the result anyway.
1601
+ local imported = false
1602
+ if
1603
+ options.ImportLegacyData ~= nil
1604
+ and not hadPriorSave
1605
+ and not options.ResetData
1606
+ and mode ~= "NoSave"
1607
+ and not entry.TornDown
1608
+ then
1609
+ -- YIELDS: the hook reads another library's key and everything below assumes a
1610
+ -- live session, so the liveness re-check moves up here.
1611
+ local importStarted = Clock.Monotonic()
1612
+ local releaseSlot = takeImportSlot(entry)
1613
+ if entry.TornDown then
1614
+ -- Waiting for a slot is a window the outer guard could not see, and teardown
1615
+ -- has already run.
1616
+ releaseSlot()
1617
+ setPhase(entry, nil)
1618
+ return
1619
+ end
1620
+ local ok, legacy = xpcall(options.ImportLegacyData, function(e)
1621
+ return debug.traceback(tostring(e), 2)
1622
+ end, player, entry.UserId, migrationContext(entry))
1623
+ releaseSlot()
1624
+ setPhase(entry, "Initializing")
1625
+ local importSeconds = Clock.Monotonic() - importStarted
1626
+ if importSeconds >= SLOW_IMPORT_SECONDS then
1627
+ -- Without this, "waiting on budget" and "hung" look identical in the log.
1628
+ Log.Warn(
1629
+ "Persistence",
1630
+ "SLOW_IMPORT",
1631
+ `ImportLegacyData took {string.format("%.1f", importSeconds)}s, past the {SLOW_IMPORT_SECONDS}s mark`,
1632
+ { Player = player }
1633
+ )
1634
+ end
1635
+
1636
+ if not ok then
1637
+ -- Fail closed, the same shape as a failed migration: a hook that threw
1638
+ -- cannot say whether legacy data exists, and loading the player on an
1639
+ -- empty profile makes emptiness canonical at their first save, after which
1640
+ -- hadPriorSave is true forever. The release below writes template defaults,
1641
+ -- which must not count as a prior save; it does not, because AddUserId runs
1642
+ -- further down, so the UserIds list it persists is still empty. Hoisting
1643
+ -- AddUserId above the import would make a retryable outage permanent.
1644
+ Metrics.Add("LegacyImportFailures")
1645
+ Log.Error(
1646
+ "Persistence",
1647
+ "LEGACY_IMPORT_FAIL",
1648
+ `ImportLegacyData threw, so it is unknown whether this player has data to carry over. `
1649
+ .. `Refusing the load rather than starting them empty: {legacy}`,
1650
+ { Player = player, UserId = entry.UserId }
1651
+ )
1652
+ entry.LocalSessionEnd = true
1653
+ releaseWithoutWriting(profile)
1654
+ if player.Parent == Players then
1655
+ player:Kick(kickText("LegacyImport"))
1656
+ end
1657
+ teardown(entry, "load-failed")
1658
+ return
1659
+ end
1660
+
1661
+ -- The hook yielded. A player who left in that window would leave Reconcile, the
1662
+ -- migration chain, the schema check and OnPlayerInit running against a session
1663
+ -- already gone. The guard further down catches later yields.
1664
+ if not profile:IsActive() then
1665
+ teardown(entry, if player.Parent == Players then "session-ended" else "player-left")
1666
+ return
1667
+ end
1668
+
1669
+ if type(legacy) == "table" then
1670
+ -- A returned table means "this player is an import", even an empty one; nil
1671
+ -- means "nothing to carry over". A hook must THROW rather than return nil
1672
+ -- when it could not find out: the two are indistinguishable here.
1673
+ local adopted = 0
1674
+ for key, value in legacy do
1675
+ -- Scribe's own roots are never adopted; the prefix test catches both
1676
+ -- ("_ScribeSession" starts with "_Scribe"), which is intended. An
1677
+ -- import writing into _Scribe would hand the old library control of
1678
+ -- purchase-dedupe ids, granted perks and cooldowns.
1679
+ if
1680
+ type(key) == "string"
1681
+ and string.sub(key, 1, #Template.RESERVED_ROOT) == Template.RESERVED_ROOT
1682
+ then
1683
+ continue
1684
+ end
1685
+ profile.Data[key] = value
1686
+ adopted += 1
1687
+ end
1688
+ imported = true
1689
+ Metrics.Add("LegacyImports")
1690
+ Log.Info(
1691
+ "Persistence",
1692
+ "LEGACY_IMPORTED",
1693
+ `adopted {adopted} top-level key(s) from ImportLegacyData; the profile will run the full `
1694
+ .. "migration chain from version 1",
1695
+ { Player = player, UserId = entry.UserId, Keys = adopted }
1696
+ )
1697
+ end
1698
+ end
1699
+
1700
+ -- Decide which Scribe.Dynamic fields to seed BEFORE Reconcile backfills their
1701
+ -- baseline: all of them on a brand-new profile, plus any the STORED data lacks.
1702
+ -- Reading the stored data first means a saved value is present, and so is never
1703
+ -- flagged or overwritten.
1704
+ local dynamicToSeed: { any } = {}
1705
+ if #compiled.DynamicNodes > 0 then
1706
+ for _, node in compiled.DynamicNodes do
1707
+ local present = true
1708
+ local at: any = profile.Data
1709
+ for _, seg in node.Path do
1710
+ if type(at) ~= "table" or at[seg] == nil then
1711
+ present = false
1712
+ break
1713
+ end
1714
+ at = at[seg]
1715
+ end
1716
+ -- ResetData wipes to persistentDefaults, which carry each Dynamic field's
1717
+ -- frozen declaration-time sample, so a reset must re-seed exactly like a
1718
+ -- new profile or the player keeps the shared sample.
1719
+ -- `isNew and not imported`: an imported profile is new to Scribe but not
1720
+ -- to the player, so a Dynamic value the import carried is left alone.
1721
+ if (isNew and not imported) or willReset or not present then
1722
+ table.insert(dynamicToSeed, node)
1723
+ end
1724
+ end
1725
+ end
1726
+
1727
+ profile:AddUserId(entry.UserId)
1728
+
1729
+ -- Keep the RAW pre-Reconcile bytes so the shadow run below can show what the
1730
+ -- migrations would have seen without the backfill. Opt-in and NOT gated on Studio
1731
+ -- or DevMode: the shadow re-EXECUTES the migration bodies against a clone, so any
1732
+ -- side effect in them fires a second time. Gated on the stored version here too,
1733
+ -- or a mature game pays a full profile deep clone on every join for a shadow run
1734
+ -- that then declines to happen; this read is conservative (no marker reads as nil
1735
+ -- and still clones) and the post-Reconcile read below stays authoritative.
1736
+ local preReconcileReserved = profile.Data[Template.RESERVED_ROOT]
1737
+ local preReconcileVersion = if type(preReconcileReserved) == "table"
1738
+ and type(preReconcileReserved.Version) == "number"
1739
+ then preReconcileReserved.Version
1740
+ else nil
1741
+ -- `or imported`: adopted data enters at version 1 and runs the whole chain against
1742
+ -- real stored values, which is the run an audit most wants to shadow.
1743
+ local rawStored: { [string]: any }? = if options.MigrationShadow == true
1744
+ and (not isNew or imported)
1745
+ and not willReset
1746
+ and (preReconcileVersion == nil or preReconcileVersion < currentVersion)
1747
+ then Util.deepClone(profile.Data)
1748
+ else nil
1749
+
1750
+ -- Before Reconcile: integer map keys arrive from the DataStore as JSON
1751
+ -- strings, and every read after this point expects the declared key type.
1752
+ Template.CoerceMapKeys(compiled, profile.Data)
1753
+
1754
+ profile:Reconcile()
1755
+
1756
+ -- A hard crash inside the very first session persists template-shaped data with
1757
+ -- the default Version while SessionLoadCount advanced past 1. Detect that shape
1758
+ -- and treat the profile as fresh, or every migration runs against new-player
1759
+ -- defaults. Gated to migrating profiles, so the deep compare is rare. Hoisted to
1760
+ -- function scope: the fail-closed carve-out below needs it, and re-reading it
1761
+ -- there would yield currentVersion after the stamp.
1762
+ local storedReserved = profile.Data[Template.RESERVED_ROOT]
1763
+ local storedVersion = if type(storedReserved) == "table" and type(storedReserved.Version) == "number"
1764
+ then storedReserved.Version
1765
+ else 1
1766
+
1767
+ local freshAsTemplate = false
1768
+ if not isNew and not options.ResetData then
1769
+ if storedVersion < currentVersion then
1770
+ -- Only claim "template-fresh" when no completed save is on record. A
1771
+ -- returning player who never earned anything has identical bytes, and
1772
+ -- treating them as new skipped every migration for them.
1773
+ freshAsTemplate = not hadPriorSave and equalsTemplateDefaults(profile.Data)
1774
+ if freshAsTemplate then
1775
+ Log.Info(
1776
+ "Persistence",
1777
+ "PROFILE_LOADED",
1778
+ "stored data is template-fresh and no completed save is recorded; treating as new",
1779
+ { Player = player }
1780
+ )
1781
+ -- Effectively a brand-new player: re-seed every Dynamic field too
1782
+ -- (the stored values are a dead boot's declaration-time samples).
1783
+ table.clear(dynamicToSeed)
1784
+ for _, node in compiled.DynamicNodes do
1785
+ table.insert(dynamicToSeed, node)
1786
+ end
1787
+ end
1788
+ end
1789
+ end
1790
+
1791
+ if options.ResetData and not willReset then
1792
+ -- The wipe would take the reserved root with it, and with it the only record
1793
+ -- that the staked value ever existed. Refused rather than reported: there is
1794
+ -- nothing to report to afterwards.
1795
+ Log.Error(
1796
+ "Persistence",
1797
+ "EXCHANGE_RESET_REFUSED",
1798
+ `ResetData refused: profile holds {resetStaked} in-flight and {resetParked} undelivered exchange(s); `
1799
+ .. `resolve or discard them before resetting`,
1800
+ { Player = player, Staked = resetStaked, Parked = resetParked }
1801
+ )
1802
+ elseif willReset then
1803
+ local fresh = Util.deepClone(persistentDefaults)
1804
+ table.clear(profile.Data)
1805
+ for k, v in fresh do
1806
+ profile.Data[k] = v
1807
+ end
1808
+ Log.Warn("Persistence", "PROFILE_RESET", "profile reset via ResetData option", { Player = player })
1809
+ end
1810
+
1811
+ -- Fresh data (a brand-new profile, or a ResetData wipe) is built entirely from the
1812
+ -- CURRENT template, so stamp it and let runMigrations no-op. Run against fresh
1813
+ -- defaults, migrations double-apply: a one-time compensation grant handed to a new
1814
+ -- player, or a rename step overwriting a field's default with nil. `not imported`:
1815
+ -- adopted data has never seen a Scribe migration, so it enters at version 1 and
1816
+ -- runs the entire chain, and stamping it current here would skip every step.
1817
+ if (isNew or willReset or freshAsTemplate) and not imported then
1818
+ local reserved = profile.Data[Template.RESERVED_ROOT]
1819
+ if type(reserved) == "table" then
1820
+ reserved.Version = currentVersion
1821
+ end
1822
+ end
1823
+
1824
+ local migrated, refusal = runMigrations(entry, profile.Data)
1825
+ if not migrated then
1826
+ -- Carve-out for a returning player whose stored data is still nothing but
1827
+ -- current-template defaults. Fail-closed protects real stored bytes and this
1828
+ -- profile has none, so kicking would manufacture a permanent outage for
1829
+ -- players who load fine today. Load them, but do NOT stamp the version, so the
1830
+ -- chain retries every join and lands once the migration is fixed.
1831
+ local allDefaults = not freshAsTemplate
1832
+ and storedVersion < currentVersion
1833
+ and equalsTemplateDefaults(profile.Data)
1834
+ if allDefaults then
1835
+ Log.Warn(
1836
+ "Persistence",
1837
+ "MIGRATION_FAIL",
1838
+ "migration failed against an all-default returning profile; loading it un-migrated "
1839
+ .. "and UNSTAMPED (nothing but template defaults is at risk). It retries on every "
1840
+ .. "join until the migration is fixed.",
1841
+ { Player = player }
1842
+ )
1843
+ else
1844
+ -- Fail-closed: never persist a half-migrated profile. Migrations ran on a
1845
+ -- clone and the reserved Version is stamped only after the last rejection,
1846
+ -- so profile.Data is untouched on every path that returns false.
1847
+ entry.LocalSessionEnd = true
1848
+ releaseWithoutWriting(profile)
1849
+ if player.Parent == Players then
1850
+ -- Their data is newer, not broken: the migration wording is backwards here.
1851
+ player:Kick(kickText(if refusal == "version-ahead" then "VersionAhead" else "Migration"))
1852
+ end
1853
+ teardown(entry, "migration-failed")
1854
+ return
1855
+ end
1856
+ end
1857
+
1858
+ -- Reconcile runs BEFORE migrations, so a migration body reaching through a static
1859
+ -- sub-record or a scalar default reads a value Reconcile manufactured rather than
1860
+ -- one the player stored, and the two commonest guarded shapes silently no-op.
1861
+ -- Reordering is not the fix (it nil-indexes raw data and fail-closed kicks every
1862
+ -- legacy player), so re-run the chain against the raw bytes and report a diff.
1863
+ if rawStored ~= nil and not freshAsTemplate and storedVersion < currentVersion then
1864
+ local shadowOk = true
1865
+ for version = storedVersion + 1, currentVersion do
1866
+ local step = migrations[version]
1867
+ if step then
1868
+ local ok, err = xpcall(step, function(e)
1869
+ return debug.traceback(tostring(e), 2)
1870
+ end, rawStored)
1871
+ if not ok then
1872
+ shadowOk = false
1873
+ Log.Warn(
1874
+ "Persistence",
1875
+ "MIGRATION_RECONCILE_DEPENDENT",
1876
+ `migration step {version} succeeds only because Reconcile backfilled a template `
1877
+ .. `key before it ran. Against the raw stored data it throws: {err}`,
1878
+ { Player = player, Step = version }
1879
+ )
1880
+ break
1881
+ end
1882
+ end
1883
+ end
1884
+ if shadowOk then
1885
+ local diffs = {}
1886
+ divergences(rawStored, profile.Data, {}, diffs)
1887
+ if #diffs > 0 then
1888
+ Log.Warn(
1889
+ "Persistence",
1890
+ "MIGRATION_RECONCILE_DEPENDENT",
1891
+ "migrations produce a different result against the raw stored data (a guarded step "
1892
+ .. "saw a Reconcile-backfilled default instead of nil): "
1893
+ .. table.concat(diffs, ", "),
1894
+ { Player = player }
1895
+ )
1896
+ end
1897
+ end
1898
+ end
1899
+
1900
+ -- Reconcile only fills STATIC template keys; ArrayOf/DictOf element fields hang off
1901
+ -- AnyChild and would never reach stored entries. After migrations for two reasons:
1902
+ -- a migration must observe the raw stored element data (a guarded rename would
1903
+ -- otherwise see a backfilled default and destroy the real value), and the
1904
+ -- version-ahead fail-closed path above must leave the stored bytes untouched.
1905
+ if Template.ReconcileElements(compiled, profile.Data) then
1906
+ entry.SaveInfo.Dirty = true
1907
+ end
1908
+
1909
+ -- Stored-shape check, opt-in. After Reconcile and ReconcileElements on purpose:
1910
+ -- before them every field the template has gained since this profile was written
1911
+ -- is still absent, and each would be reported on a profile about to be brought up
1912
+ -- to date anyway. Defaults to "Warn" in DevMode and off on live: turned on live it
1913
+ -- is worse than useless on an upgraded game, since ordinary template evolutions
1914
+ -- report and UnknownKey fires once per stored element, so one retired field on a
1915
+ -- 40-item inventory fills the whole limit. An explicit SchemaPolicy always wins.
1916
+ local schemaPolicy = options.SchemaPolicy
1917
+ if schemaPolicy == nil and ctx.DevMode then
1918
+ schemaPolicy = "Warn"
1919
+ end
1920
+ if schemaPolicy ~= nil then
1921
+ local violations = Conform.Check(compiled, profile.Data)
1922
+ if #violations > 0 then
1923
+ -- ADVISORY VIOLATIONS NEVER END A SESSION, even under "Reject"; the long
1924
+ -- form is on the Violation type in Conform. Only a table mixing array
1925
+ -- indices with string keys actually fails the DataStore save. The caps in
1926
+ -- particular fire on a state Scribe's own write path calls legal, and
1927
+ -- rejecting them kicked the player on every rejoin forever, since
1928
+ -- releaseWithoutWriting leaves the bytes as they are. ANY non-advisory
1929
+ -- violation rejects, however many advisory findings sit beside it.
1930
+ local reject = false
1931
+ if schemaPolicy == "Reject" then
1932
+ for _, violation in violations do
1933
+ if not violation.Advisory then
1934
+ reject = true
1935
+ break
1936
+ end
1937
+ end
1938
+ end
1939
+ for _, violation in violations do
1940
+ Metrics.Add("SchemaViolations")
1941
+ -- Advisory findings stay at Warn even on a rejecting load, so the
1942
+ -- level says which one actually cost the player the session.
1943
+ local report = if reject and not violation.Advisory then Log.Error else Log.Warn
1944
+ report("Persistence", "PROFILE_SCHEMA_VIOLATION", violation.Detail, {
1945
+ Player = player,
1946
+ Path = table.concat(violation.Path :: any, "."),
1947
+ Kind = violation.Kind,
1948
+ Advisory = violation.Advisory == true,
1949
+ })
1950
+ ctx.Signals.OnAnomaly:Fire({
1951
+ Player = player,
1952
+ Path = violation.Path,
1953
+ Reason = violation.Kind,
1954
+ })
1955
+ end
1956
+
1957
+ if reject then
1958
+ -- Fail-closed, the same shape as the migration failure above: the
1959
+ -- session ends rather than serving a profile that does not match the
1960
+ -- template, and the stored bytes are left alone (see
1961
+ -- releaseWithoutWriting for why EndSession will not do).
1962
+ entry.LocalSessionEnd = true
1963
+ releaseWithoutWriting(profile)
1964
+ if player.Parent == Players then
1965
+ player:Kick(kickText("Schema"))
1966
+ end
1967
+ teardown(entry, "load-failed")
1968
+ return
1969
+ end
1970
+ end
1971
+ end
1972
+
1973
+ -- Seed the flagged Scribe.Dynamic fields now that Reconcile has created any parent
1974
+ -- containers, before OnPlayerInit so it can still override. Written raw into
1975
+ -- profile.Data (the factory already packs datatypes); the tree wraps this same
1976
+ -- table, so the values are live and will save.
1977
+ if #dynamicToSeed > 0 then
1978
+ for _, node in dynamicToSeed do
1979
+ local ok, value = pcall(node.Factory :: () -> any)
1980
+ if not ok then
1981
+ Log.Error(
1982
+ "Lifecycle",
1983
+ "DYNAMIC_DEFAULT_FAILED",
1984
+ `Scribe.Dynamic factory for "{Util.pathString(node.Path)}" threw: {value}`,
1985
+ { Player = player, Path = Util.pathString(node.Path) }
1986
+ )
1987
+ else
1988
+ local container: any = profile.Data
1989
+ for i = 1, #node.Path - 1 do
1990
+ container = if type(container) == "table" then container[node.Path[i]] else nil
1991
+ end
1992
+ if type(container) == "table" then
1993
+ container[node.Path[#node.Path]] = value
1994
+ end
1995
+ end
1996
+ end
1997
+ entry.SaveInfo.Dirty = true
1998
+ end
1999
+
2000
+ if ctx.DevMode then
2001
+ local unknown = Template.UnknownRootKeys(compiled, profile.Data)
2002
+ if #unknown > 0 then
2003
+ Log.Warn(
2004
+ "Persistence",
2005
+ "UNKNOWN_ROOT_KEYS",
2006
+ "profile contains top-level keys absent from the template "
2007
+ .. "(removing keys is a migration's job)",
2008
+ {
2009
+ Player = player,
2010
+ Keys = table.concat(unknown, ", "),
2011
+ }
2012
+ )
2013
+ end
2014
+ end
2015
+
2016
+ -- Taken before BOTH hooks: Dynamic factories run above this point too.
2017
+ local exchangeStateBefore = exchangeSnapshot(profile.Data)
2018
+
2019
+ if options.OnPlayerInit then
2020
+ -- A brand-new profile, a ResetData wipe, or a first-session crash recovery all
2021
+ -- mean "no prior saved data", which is what starter kits need. An IMPORTED
2022
+ -- profile does not: they arrive carrying a full inventory, and the starter kit
2023
+ -- on top is the exact double-grant this flag prevents elsewhere.
2024
+ local isNewProfile = (isNew or willReset or freshAsTemplate) and not imported
2025
+ -- ImportLegacyData only fires for a player who has never saved, so a game
2026
+ -- already on Scribe finishes moving stores here instead. NOT behind the import
2027
+ -- cap: this runs on every join, and serialising those throttles ordinary logins.
2028
+ Util.safeCall(function(err)
2029
+ Log.Error("Lifecycle", "ON_PLAYER_INIT_ERROR", err, { Player = player })
2030
+ end, options.OnPlayerInit, player, profile.Data, isNewProfile, migrationContext(entry))
2031
+ end
2032
+
2033
+ -- REVERTED rather than merely reported: a half-edited exchange record is worse than
2034
+ -- either the original or a clean one, and the original is still in hand.
2035
+ if exchangeStateBefore ~= nil or exchangeSnapshot(profile.Data) ~= nil then
2036
+ local after = exchangeSnapshot(profile.Data)
2037
+ if not sameExchangeState(exchangeStateBefore, after) then
2038
+ local reserved = profile.Data[Template.RESERVED_ROOT]
2039
+ if type(reserved) == "table" then
2040
+ reserved.Exchange = if exchangeStateBefore then exchangeStateBefore.Exchange else {}
2041
+ reserved.ExchangeInbox = if exchangeStateBefore then exchangeStateBefore.ExchangeInbox else {}
2042
+ end
2043
+ Log.Error(
2044
+ "Lifecycle",
2045
+ "EXCHANGE_INIT_TAMPER",
2046
+ "OnPlayerInit or a Scribe.Dynamic factory changed the in-flight exchange ledger under "
2047
+ .. `"{Template.RESERVED_ROOT}"; the change has been reverted. That state is Scribe's, `
2048
+ .. "and editing it can duplicate or destroy an exchange in progress",
2049
+ { Player = player }
2050
+ )
2051
+ end
2052
+ end
2053
+
2054
+ -- Scribe.Dynamic factories and OnPlayerInit write raw into profile.Data without
2055
+ -- passing Node.validateWrite, and unpersistable data introduced there would fail
2056
+ -- the entire next save opaquely, with no path in the log. Non-fatal: the library
2057
+ -- cannot safely rewrite developer data.
2058
+ local depthStats = { MaxDepth = 0 }
2059
+ local badInit = Util.findUnpersistable(profile.Data, nil, nil, depthStats)
2060
+ if badInit ~= nil then
2061
+ local where = Util.pathString(badInit.Path)
2062
+ Log.Error(
2063
+ "Lifecycle",
2064
+ "PROFILE_UNPERSISTABLE",
2065
+ `unpersistable data written during load at "{where}" ({badInit.Detail}) -- this WILL fail the `
2066
+ .. `DataStore save; fix the value set by OnPlayerInit or a Scribe.Dynamic factory`,
2067
+ { Player = player, Path = where }
2068
+ )
2069
+ ctx.Signals.OnAnomaly:Fire({ Player = player, Path = badInit.Path, Value = nil, Reason = badInit.Kind })
2070
+ end
2071
+ -- Stored data may legally be nested deeper than a write is allowed to reach: it
2072
+ -- loads and reads fine, then refuses every write to its deepest leaves at the
2073
+ -- first Set, with nothing at load having hinted at it.
2074
+ if depthStats.MaxDepth > Util.MAX_VALUE_DEPTH then
2075
+ Log.Warn(
2076
+ "Integrity",
2077
+ "PROFILE_TOO_DEEP",
2078
+ `stored data is nested {depthStats.MaxDepth} levels deep, past the {Util.MAX_VALUE_DEPTH}-level `
2079
+ .. "write limit; those leaves load and read but every write to them will be refused. "
2080
+ .. "Flatten the shape or migrate it",
2081
+ { Player = player, Depth = depthStats.MaxDepth, Limit = Util.MAX_VALUE_DEPTH }
2082
+ )
2083
+ end
2084
+
2085
+ profile.OnSave:Connect(function(saveContext: any)
2086
+ -- Point the profile at a copy holding online-only cooldowns in their persisted
2087
+ -- form and keep the live deadlines aside. OnSave fires before the store reads
2088
+ -- profile.Data, OnAfterSave once it has, OnError when it never gets there.
2089
+ entry.TimedHeld = ctx.Timed.BeforeSave(entry)
2090
+ onProfileSaving(entry, saveContext)
2091
+ end)
2092
+ -- Exactly one fire per save call, success or failure, so this is where the
2093
+ -- in-flight count comes back down. OnAfterSave cannot do it: it fires only on
2094
+ -- success, so a failing save would leak a count forever and every later error on
2095
+ -- this key would be re-attributed to it. The fire's second argument, this call's
2096
+ -- own outcome, is deliberately unbound here: it is consumed as SaveAwait's return
2097
+ -- value below, and its one interesting class, "NotOwned", only happens as the
2098
+ -- session ends, by which time teardown has detached this entry from Entries.
2099
+ if type((profile :: any).OnSettled) == "table" then
2100
+ (profile :: any).OnSettled:Connect(function(saveContext: any)
2101
+ entry.SavesInFlight = math.max(0, entry.SavesInFlight - 1)
2102
+ if entry.SavesInFlight == 0 then
2103
+ entry.SaveT0 = nil
2104
+ -- ...and with it the pre-snapshot dirtiness the OnError restore reads.
2105
+ -- On the DRAIN, not once per settle: the slot is shared by every save
2106
+ -- in flight, so clearing it at the first of two completions hands the
2107
+ -- survivor a nil and its failure restores clean over an unstored write.
2108
+ -- Erring dirty costs one redundant save; the other direction costs a
2109
+ -- receipt answered PurchaseGranted for a grant that exists only in RAM.
2110
+ entry.DirtyAtSaveStart = nil
2111
+ end
2112
+ -- A release-only save wrote MetaData and left the stored bytes as it found
2113
+ -- them, so everything live is still unsaved. It reaches OnAfterSave like
2114
+ -- any successful write, so onProfileSaved has just cleared Dirty and
2115
+ -- stamped "Ok", which is exactly what isDurable reads: without this,
2116
+ -- Data.Flush short-circuits true for a profile whose data has never been
2117
+ -- persisted. OnSettled fires after OnAfterSave, so this is not undone.
2118
+ if type(saveContext) == "table" and saveContext.ReleaseOnly == true then
2119
+ entry.SaveInfo.Dirty = true
2120
+ self.UpdateSaveInfoMirror(entry)
2121
+ end
2122
+ end)
2123
+ end
2124
+
2125
+ profile.OnAfterSave:Connect(function(lastSavedData: any)
2126
+ ctx.Timed.AfterSave(entry, entry.TimedHeld)
2127
+ entry.TimedHeld = nil
2128
+ onProfileSaved(entry, lastSavedData)
2129
+ end)
2130
+ profile.OnSessionEnd:Connect(function()
2131
+ -- A session end WE did not initiate (and not the shutdown drain) is an
2132
+ -- external steal: its final save persists nothing locally, so save
2133
+ -- bookkeeping must not report success for it (onProfileSaved checks this).
2134
+ entry.SessionStolen = not entry.LocalSessionEnd and not self.Closing
2135
+ teardown(entry, if player.Parent == Players then "session-ended" else "player-left")
2136
+ end)
2137
+
2138
+ -- If the player left (or the session was stolen) during the yielding
2139
+ -- migration/OnPlayerInit above, OnSessionEnd already fired with no listener
2140
+ -- attached, so tear down rather than leaking a Ready session. A NoSave load holds
2141
+ -- a VIEW, which is never IsActive() by design, so the guard must not apply to it.
2142
+ if mode ~= "NoSave" and not profile:IsActive() then
2143
+ teardown(entry, if player.Parent == Players then "session-ended" else "player-left")
2144
+ return
2145
+ end
2146
+
2147
+ buildTree(entry)
2148
+ -- The session store was built from template defaults, so a derived field's default
2149
+ -- came from its inputs' defaults rather than the data that just loaded. Recompute
2150
+ -- before Ready, before Leaderboards.OnLoad (which enqueues an initial score), and
2151
+ -- before Replication.OnEntryReady (whose Init snapshot must carry real values).
2152
+ entry.Tree:SeedDerived()
2153
+ entry.SaveInfo.Dirty = false
2154
+ self.UpdateSaveInfoMirror(entry)
2155
+
2156
+ -- Subsystem hooks are isolated so one failing hook cannot abort the others
2157
+ -- (or strand the load thread).
2158
+ local function runHook(name: string, hook: (any) -> ...any)
2159
+ local ok, err = pcall(hook, entry)
2160
+ if not ok then
2161
+ Log.Error(
2162
+ "Lifecycle",
2163
+ "SUBSYSTEM_HOOK_ERROR",
2164
+ `{name} hook failed: {tostring(err)}`,
2165
+ { Player = player }
2166
+ )
2167
+ end
2168
+ end
2169
+
2170
+ -- BEFORE Ready, unlike the hooks below, and that is a correctness requirement. An
2171
+ -- online-only cooldown is stored as REMAINING seconds and only becomes a deadline
2172
+ -- here, so after Ready the documented PlayerAdded -> WaitForData -> read path
2173
+ -- reads a remaining value as a deadline, the gate reads expired, and re-arming in
2174
+ -- that window compounds into a permanent lockout. Must also stay BELOW
2175
+ -- `Dirty = false`: the load sweep's clears write through setTimed.
2176
+ runHook("Timed", ctx.Timed.OnLoad)
2177
+
2178
+ setState(entry, "Ready", nil)
2179
+ Metrics.Add("ProfilesLoaded")
2180
+ Metrics.Set("ActiveSessions", self.EntryCount)
2181
+
2182
+ -- The DataStore round trip is network time, not what the player experienced.
2183
+ -- Observed as a distribution so the TAIL is readable: a p50 join says nothing
2184
+ -- about the player who waited twenty seconds.
2185
+ local loadSeconds = if entry.JoinedAt then Clock.Monotonic() - entry.JoinedAt else nil
2186
+ if loadSeconds then
2187
+ Metrics.Observe("LoadDuration", loadSeconds)
2188
+ end
2189
+ Log.Info("Persistence", "PROFILE_LOADED", "profile session started", {
2190
+ Player = player,
2191
+ LoadSeconds = loadSeconds,
2192
+ })
2193
+
2194
+ -- Register the cross-server gift handler FIRST and independently: it only
2195
+ -- needs the tree, and it must not be lost if a subsystem hook below
2196
+ -- throws (e.g. a misconfigured leaderboard Stat path).
2197
+ profile:MessageHandler(function(message: any, processed: () -> ())
2198
+ ctx.Monetization.OnProfileMessage(entry, message, processed)
2199
+ -- User-level cross-server messages (Data.SendMessage) ride the same channel,
2200
+ -- tagged distinctly so they never collide with gift delivery.
2201
+ -- ACKNOWLEDGE ONLY WHAT A HANDLER ACTUALLY OBSERVED. Fire tells the caller
2202
+ -- nothing: it returns at the handler's first yield, and returns the same when
2203
+ -- no handler is connected. processed() only flags locked_global_updates, and
2204
+ -- the store removes the update inside the same UpdateAsync that writes
2205
+ -- profile.Data, so ack once the handler has RETURNED, the rule the gift arm
2206
+ -- already follows. Not acking is at-least-once, and bounded: an unacked
2207
+ -- message returns on the next load, not once per save.
2208
+ if type(message) == "table" and message.ScribeMessage == true then
2209
+ -- Counted before the fire and uncounted on settle, so teardown can report a
2210
+ -- handler that never returned. FireTracked settles on the LAST handler, so
2211
+ -- one parked listener holds the whole fire open.
2212
+ entry.UnsettledMessages = (entry.UnsettledMessages or 0) + 1
2213
+ ctx.Signals.OnMessage:FireTracked(function(ran: number, ok: boolean, err: any)
2214
+ entry.UnsettledMessages = (entry.UnsettledMessages or 1) - 1
2215
+ if ran == 0 then
2216
+ -- Nobody was connected. Common in a startup race where the profile
2217
+ -- loads before the game binds its handler; keeping the message
2218
+ -- costs a redelivery, dropping it costs the message.
2219
+ Log.Warn(
2220
+ "Persistence",
2221
+ "MESSAGE_NO_LISTENER",
2222
+ "a cross-server message arrived with nothing connected to Data.OnMessage; "
2223
+ .. "it was NOT acknowledged and will be redelivered on the next load",
2224
+ { Player = player }
2225
+ )
2226
+ return
2227
+ end
2228
+ if not ok then
2229
+ -- Fire let a handler error reach the scheduler; FireTracked pcalls it,
2230
+ -- so this is where it has to surface or it surfaces nowhere.
2231
+ Log.Error(
2232
+ "Persistence",
2233
+ "MESSAGE_HANDLER_ERROR",
2234
+ `a Data.OnMessage handler raised: {tostring(err)}; the message was NOT `
2235
+ .. "acknowledged and will be redelivered on the next load",
2236
+ { Player = player }
2237
+ )
2238
+ return
2239
+ end
2240
+ processed()
2241
+ end, entry.Player, message.Data)
2242
+ end
2243
+ end)
2244
+
2245
+ -- The rest run post-Ready; only Timed has to beat the state change. An exchange
2246
+ -- that resolves a moment later is invisible to the player, where a load that
2247
+ -- waited on the verdict round trip would not be. Guarded because a bundle
2248
+ -- constructed before this subsystem existed must still load profiles.
2249
+ if ctx.Exchanges then
2250
+ -- SPAWNED, unlike every other hook here. Resolution makes a DataStore round
2251
+ -- trip on the verdict key, and run inline that trip sits in front of
2252
+ -- everything after it, including the replication entry-ready that puts the
2253
+ -- player's first snapshot on the wire. Safe to detach: nothing below depends
2254
+ -- on an exchange being resolved, and Resolve re-reads the tree for death on
2255
+ -- entry and again per exchange, so a released profile stops rather than
2256
+ -- writing to a dead one.
2257
+ Sched.Spawn(function()
2258
+ runHook("Exchanges", ctx.Exchanges.OnLoad)
2259
+ end)
2260
+ end
2261
+ runHook("Monetization", ctx.Monetization.OnLoad)
2262
+ runHook("Leaderboards", ctx.Leaderboards.OnLoad)
2263
+ runHook("Replication", ctx.Replication.OnEntryReady)
2264
+ end
2265
+
2266
+ -- Erasures in flight, per user.
2267
+ local erasing: { [number]: number } = {}
2268
+ local NO_PROFILE = "profile does not exist"
2269
+ local LIVE_ELSEWHERE = "profile has an active session on another server"
2270
+
2271
+ local function loadProfile(entry: Entry)
2272
+ local player = entry.Player
2273
+
2274
+ if mode == "NoSave" then
2275
+ setPhase(entry, "ReadingProfile")
2276
+ -- Snapshot of the real profile with NO session: nothing is saved, because
2277
+ -- a view is never IsActive() and every save path checks that first.
2278
+ local started = Clock.Monotonic()
2279
+ local ok, profile = pcall(function()
2280
+ return store:GetAsync(entry.Key)
2281
+ end)
2282
+ Metrics.Observe("ProfileLoadDuration", Clock.Monotonic() - started)
2283
+ if ok and profile then
2284
+ entry.Profile = profile
2285
+ setPhase(entry, "Initializing")
2286
+ finishLoad(entry)
2287
+ else
2288
+ Log.Error(
2289
+ "Persistence",
2290
+ "PROFILE_LOAD_FAIL",
2291
+ `Mode = "NoSave": no stored profile to read for this key`,
2292
+ { Player = player, UserId = entry.UserId }
2293
+ )
2294
+ teardown(entry, "load-failed")
2295
+ end
2296
+ return
2297
+ end
2298
+
2299
+ local heldSince = Clock.Monotonic()
2300
+ while (erasing[entry.UserId] or 0) > 0 and not entry.TornDown and not self.Stopped do
2301
+ setPhase(entry, "WaitingForErase")
2302
+ if Clock.Monotonic() - heldSince >= eraseJoinTimeout then
2303
+ if player.Parent == Players then
2304
+ player:Kick(kickText("Erasing"))
2305
+ end
2306
+ teardown(entry, "erasing")
2307
+ return
2308
+ end
2309
+ Clock.Wait(0.1)
2310
+ end
2311
+
2312
+ local attempt = 0
2313
+ while not self.Closing and not self.Stopped do
2314
+ attempt += 1
2315
+ setPhase(entry, "AcquiringSession")
2316
+ -- Captured PER ATTEMPT. ProfileStore latches the first `true` its Cancel
2317
+ -- returns and never asks again, so a deadline computed once outside this loop
2318
+ -- would abort every later attempt the instant it began.
2319
+ local deadline = Clock.Monotonic() + loadTimeout
2320
+ local started = Clock.Monotonic()
2321
+ local ok, profile = pcall(function()
2322
+ return store:StartSessionAsync(entry.Key, {
2323
+ Cancel = function()
2324
+ -- Runs INSIDE a DataStore UpdateAsync transform, where
2325
+ -- yielding is forbidden, so this stays arithmetic and field
2326
+ -- reads. Monotonic, never Now: wall time can jump.
2327
+ return player.Parent ~= Players or self.Closing or Clock.Monotonic() >= deadline
2328
+ end,
2329
+ })
2330
+ end)
2331
+ Metrics.Observe("ProfileLoadDuration", Clock.Monotonic() - started)
2332
+
2333
+ if entry.TornDown then
2334
+ -- Torn down while the load was in flight (shutdown drain or a leave
2335
+ -- processed elsewhere): release any session the racing load just won
2336
+ -- and stop -- finishing a load for a dead entry would leak the lock.
2337
+ if ok and profile then
2338
+ profile:EndSession()
2339
+ end
2340
+ return
2341
+ end
2342
+
2343
+ if player.Parent ~= Players then
2344
+ if ok and profile then
2345
+ profile:EndSession()
2346
+ end
2347
+ teardown(entry, "player-left")
2348
+ return
2349
+ end
2350
+
2351
+ if ok and profile then
2352
+ entry.Profile = profile
2353
+ Health.ReportSuccess("ProfileLoad")
2354
+ setPhase(entry, "Initializing")
2355
+ finishLoad(entry)
2356
+ return
2357
+ end
2358
+
2359
+ -- A nil profile during shutdown is a Cancel-abort, not a DataStore
2360
+ -- failure: tear down quietly (BindToClose disconnects everyone) so we
2361
+ -- don't kick with a data-loss message or poison Health/metrics.
2362
+ if self.Closing then
2363
+ teardown(entry, "shutdown")
2364
+ return
2365
+ end
2366
+
2367
+ -- Keyed for the same reason as the save path: under LoadFailurePolicy = "Wait"
2368
+ -- this fires once per retry for the same profile, so folding keeps a retry
2369
+ -- cycle from emitting one Error per attempt.
2370
+ Failure.Report({
2371
+ Subsystem = "ProfileLoad",
2372
+ Key = entry.Key,
2373
+ -- StartSessionAsync answers nil rather than a message; the underlying
2374
+ -- DataStore error, if there was one, already went through OnError above.
2375
+ Metric = "ProfileLoadFailures",
2376
+ Level = "Error",
2377
+ Category = "Persistence",
2378
+ Code = "PROFILE_LOAD_FAIL",
2379
+ Message = `profile failed to load (attempt {attempt})`,
2380
+ Context = { Player = player, Attempt = attempt },
2381
+ })
2382
+
2383
+ if (options.LoadFailurePolicy or "Kick") == "Kick" then
2384
+ player:Kick(loadKickText(entry))
2385
+ teardown(entry, "load-failed")
2386
+ return
2387
+ end
2388
+ -- "Wait": stay in Loading and keep retrying with backoff; never fall back
2389
+ -- to template data on a live account. Jittered so a fleet that lost the
2390
+ -- DataStore at one instant does not come back at one instant.
2391
+ setPhase(entry, "Retrying")
2392
+ Clock.Wait(Rng.Jitter(math.min(5 * attempt, 60)))
2393
+ end
2394
+ -- Loop exited because Closing or Stopped flipped during a "Wait" backoff: tear the
2395
+ -- stranded Loading entry down so the shutdown drain isn't held to its full budget.
2396
+ if not entry.TornDown then
2397
+ teardown(entry, "shutdown")
2398
+ end
2399
+ end
2400
+
2401
+ function self.OnPlayerAdded(player: Player)
2402
+ if self.Entries[player] or self.Closing then
2403
+ return
2404
+ end
2405
+ self.EndReasons[player] = nil
2406
+ local userId = self.TargetUserId or player.UserId
2407
+ local entry: Entry = {
2408
+ Player = player,
2409
+ UserId = userId,
2410
+ Key = self.KeyFor(userId),
2411
+ State = "Loading",
2412
+ -- Monotonic, and stamped at the JOIN rather than at the store call: what
2413
+ -- matters is what the player waited for, retries and migration included.
2414
+ JoinedAt = Clock.Monotonic(),
2415
+ FailReason = nil,
2416
+ Profile = nil,
2417
+ Tree = nil,
2418
+ SessionStore = {},
2419
+ LiveData = nil,
2420
+ StateChanged = Signal.new(),
2421
+ SaveInfo = { LastSaveAt = nil, LastResult = nil, Dirty = false },
2422
+ SaveT0 = nil,
2423
+ SavesInFlight = 0,
2424
+ DirtyAtSaveStart = nil,
2425
+ Blocked = false,
2426
+ ForceSaveOnce = false,
2427
+ LastSavedSize = nil,
2428
+ LastSavedKeys = nil,
2429
+ SizeWarned = false,
2430
+ TornDown = false,
2431
+ UnsettledMessages = 0,
2432
+ HelloReceived = false,
2433
+ QueueOpen = false,
2434
+ HandshakeReady = false,
2435
+ InitDelivered = false,
2436
+ LastInitAt = 0,
2437
+ ResyncPending = false,
2438
+ Queue = {},
2439
+ QueueIndex = {},
2440
+ SharedQueue = {},
2441
+ MalformedCount = 0,
2442
+ OversizeCount = 0,
2443
+ FrameTokens = ctx.InboundFrameRate,
2444
+ FrameLast = Clock.Monotonic(),
2445
+ RateTokens = options.CommandRateLimit or 20,
2446
+ RateLast = Clock.Monotonic(),
2447
+ GiftLastAt = 0,
2448
+ }
2449
+ self.Entries[player] = entry
2450
+ self.EntryCount += 1
2451
+ syncBudgetReserves()
2452
+ self.EntriesByUserId[entry.UserId] = entry
2453
+ entry.StopLoadWatchdog = watchdog(SLOW_LOAD_SECONDS, function()
2454
+ entry.StopLoadWatchdog = nil
2455
+ if not entry.TornDown then
2456
+ warnSlowLoad(entry)
2457
+ end
2458
+ end)
2459
+ Sched.Spawn(function()
2460
+ -- Its OWN thread, so everything the load allocates, the profile copy and the
2461
+ -- whole accessor tree, is charged here. A memory category is per-thread and
2462
+ -- survives a yield, where a profiler span would not: the load waits on the
2463
+ -- DataStore, and an annotation does not survive that.
2464
+ Profile.MemoryCategory("Scribe")
2465
+ loadProfile(entry)
2466
+ end)
2467
+ end
2468
+
2469
+ function self.OnPlayerRemoving(player: Player)
2470
+ local entry = self.Entries[player]
2471
+ if not entry then
2472
+ return
2473
+ end
2474
+ if entry.Profile and entry.Profile:IsActive() then
2475
+ runLeavingHook(entry, if player.Parent == Players then "session-ended" else "player-left")
2476
+ entry.LocalSessionEnd = true
2477
+ entry.Profile:EndSession() -- final save; OnSessionEnd triggers teardown
2478
+ else
2479
+ -- INCLUDING Loading. The StartSessionAsync Cancel guard is enough for the
2480
+ -- default policy, but under LoadFailurePolicy = "Wait" the retry loop spends
2481
+ -- its time in the BACKOFF between attempts, where there is no Cancel hook, so
2482
+ -- the entry outlives the player by up to a backoff. A rejoin inside that
2483
+ -- window gets a new entry, and the old entry's late teardown broadcasts its
2484
+ -- Shared-root removal, dropping every bystander's mirror of a live Ready
2485
+ -- player. Safe for a load that IS in flight: teardown sets TornDown, and the
2486
+ -- loader checks it the moment StartSessionAsync returns.
2487
+ teardown(entry, "player-left")
2488
+ end
2489
+ end
2490
+
2491
+ -- Public API ------------------------------------------------------------------
2492
+
2493
+ -- Wait for the next StateChanged fire, resuming anyway once the deadline passes:
2494
+ -- under LoadFailurePolicy = "Wait" an entry can sit in Loading for an entire DataStore
2495
+ -- outage without a state transition, and WaitForData must still honor its timeout.
2496
+ local function waitStateChangedUntil(entry: Entry, deadline: number)
2497
+ local thread = coroutine.running()
2498
+ local settled = false
2499
+ local function resume()
2500
+ if settled then
2501
+ return
2502
+ end
2503
+ -- Claimed synchronously, resumed later. Two fires racing would otherwise
2504
+ -- both queue a wake before either set the flag.
2505
+ settled = true
2506
+ -- Deferred, not inline: an inline resume returns control from the middle of
2507
+ -- finishLoad, before the hooks have run, so the caller sees a session that
2508
+ -- reports Ready but is not. It also stops an inline Clock.Delay from resuming
2509
+ -- this thread before it has yielded.
2510
+ Sched.Defer(function()
2511
+ if coroutine.status(thread) == "suspended" then
2512
+ Sched.Resume(thread)
2513
+ end
2514
+ end)
2515
+ end
2516
+ local connection = entry.StateChanged:Connect(resume)
2517
+ Clock.Delay(math.max(deadline - Clock.Monotonic(), 0), resume)
2518
+ coroutine.yield()
2519
+ connection:Disconnect()
2520
+ end
2521
+
2522
+ function self.WaitForData(player: Player, timeout: number?): (any?, string?)
2523
+ local deadline = Clock.Monotonic() + (timeout or Defaults.ServerWaitForData)
2524
+ while true do
2525
+ local entry = self.Entries[player]
2526
+ if entry then
2527
+ if entry.State == "Ready" then
2528
+ return entry.Tree.Accessor, nil
2529
+ elseif entry.State == "SessionEnded" then
2530
+ return nil, entry.FailReason or "session-ended"
2531
+ end
2532
+ if Clock.Monotonic() >= deadline then
2533
+ -- Necessarily Loading, so the load is still in flight and will often
2534
+ -- succeed. Reported separately from "timeout" because the default wait
2535
+ -- (60s) is shorter than a legitimate slow load, and a caller that wants
2536
+ -- to keep waiting can pass its own timeout.
2537
+ return nil, "still-loading"
2538
+ end
2539
+ waitStateChangedUntil(entry, deadline)
2540
+ else
2541
+ local endedReason = self.EndReasons[player]
2542
+ if endedReason then
2543
+ return nil, endedReason
2544
+ end
2545
+ if player.Parent ~= Players then
2546
+ return nil, "player-left"
2547
+ end
2548
+ if Clock.Monotonic() >= deadline then
2549
+ -- No entry, no stored end reason, and the player is still here: Scribe
2550
+ -- has no session for them at all, which usually means the bundle was
2551
+ -- constructed after they joined and OnPlayerAdded never ran.
2552
+ return nil, "timeout"
2553
+ end
2554
+ Clock.Wait(0.1)
2555
+ end
2556
+ end
2557
+ end
2558
+
2559
+ --- The phase is a SECOND return, not a fourth state: every existing single-value
2560
+ --- caller keeps working and nothing switching on the state changes meaning.
2561
+ function self.GetState(player: Player): (string, string?)
2562
+ local entry = self.Entries[player]
2563
+ if entry == nil then
2564
+ return "SessionEnded", nil
2565
+ end
2566
+ return entry.State, entry.LoadPhase
2567
+ end
2568
+
2569
+ --- Did THIS save write something other than the live tree?
2570
+ ---
2571
+ --- The wipe guard's Block policy turns a save into a rewrite of the last-good snapshot
2572
+ --- by re-pointing profile.Data (SetSaveSource), which is per-PROFILE and affects every
2573
+ --- transform that has not run yet, so a save that snapshotted clean and was still out
2574
+ --- when a later save tripped the guard wrote the snapshot and told its caller true.
2575
+ --- `SavedSource` is the identity of the table the transform actually serialized, so
2576
+ --- this compares what WENT TO THE KEY against what the tree writes into. The
2577
+ --- snapshot-time flag is the fallback for a store that reports no SavedSource.
2578
+ local function blockedForSave(entry: Entry, saveContext: { [string]: any }): boolean
2579
+ local source = saveContext.SavedSource
2580
+ if source ~= nil and entry.LiveData ~= nil then
2581
+ return source ~= entry.LiveData
2582
+ end
2583
+ return saveContext.Blocked == true
2584
+ end
2585
+
2586
+ --- All three conditions are load-bearing: `Dirty` is cleared at SNAPSHOT time so it
2587
+ --- cannot answer alone; `SavesInFlight` stops a caller being told mid-flight writes are
2588
+ --- durable; and `LastResult` must be "Ok" rather than merely not "Fail", because
2589
+ --- finishLoad resets Dirty after the migration chain and OnPlayerInit have written.
2590
+ --- Deliberately no `Blocked` check: onProfileSaved already keeps a blocked entry dirty.
2591
+ local function isDurable(entry: Entry): boolean
2592
+ return entry.SaveInfo.Dirty == false and (entry.SavesInFlight or 0) == 0 and entry.SaveInfo.LastResult == "Ok"
2593
+ end
2594
+
2595
+ -- Force an immediate save. Returns true once the save is confirmed.
2596
+ function self.Flush(player: Player, opts: { Force: boolean?, Timeout: number? }?): boolean
2597
+ local entry = self.Entries[player]
2598
+ if not entry or not entry.Profile or entry.State ~= "Ready" then
2599
+ return false
2600
+ end
2601
+ if opts and opts.Force then
2602
+ -- Force exists to push a wipe-guard-BLOCKED save through, so it must always
2603
+ -- reach the store rather than share the short circuit below.
2604
+ entry.ForceSaveOnce = true
2605
+ elseif isDurable(entry) then
2606
+ -- Already on disk: answer honestly and spend nothing, since a save here would
2607
+ -- be a full UpdateAsync round trip writing bytes that are already there. This
2608
+ -- is the only safe part of "dirty-aware saving", because it removes a write
2609
+ -- rather than deferring one. Skipping AUTOSAVES is a different thing wearing
2610
+ -- the same words: the autosave also refreshes MetaData.LastUpdate, which
2611
+ -- another server reads to decide whether this session belongs to a crashed
2612
+ -- machine, and it is what delivers cross-server messages and gifts.
2613
+ Metrics.Add("FlushesAlreadyDurable")
2614
+ return true
2615
+ end
2616
+ return self.AwaitSave(entry, (opts and opts.Timeout) or 15)
2617
+ end
2618
+
2619
+ -- Save now, and report what happened to THIS save. The store answers for itself rather
2620
+ -- than correlating completions with a generation counter, which regularly answered a
2621
+ -- waiter whose write the completing save had never serialized. Monetization returns
2622
+ -- PurchaseGranted on this boolean.
2623
+ function self.AwaitSave(entry: Entry, timeout: number): boolean
2624
+ local profile = entry.Profile
2625
+ if not profile or not profile:IsActive() then
2626
+ return false
2627
+ end
2628
+ local thread = coroutine.running()
2629
+ -- Claimed synchronously by whichever path arrives first, so a completion landing
2630
+ -- after this caller has given up cannot answer it a second time: by then the thread
2631
+ -- is usually parked on something else, and a stray resume would hand it this save's
2632
+ -- boolean in place of the value it asked for.
2633
+ local settled = false
2634
+ -- The verdict, left where the caller can READ it rather than be woken with it.
2635
+ -- With a synchronous store (the fake in tests, a fast final save in production)
2636
+ -- SaveAwait returns on this caller's own stack, before it has reached its yield.
2637
+ -- Deferring the wake until the caller parked was the bug: Sched.Resume is
2638
+ -- task.spawn, so a thread woken from inside a deferred callback inherits that
2639
+ -- callback's re-entrancy generation and climbs one rung per AwaitSave round trip.
2640
+ -- At 80, task.defer silently DROPS the callback while still returning success, so
2641
+ -- the waiter hangs for the life of the server. A thread keeps the rung it climbed
2642
+ -- until its own next task.wait, so only a run of wakes with no wait compounds.
2643
+ local result: boolean? = nil
2644
+ -- Hand the verdict to whichever state the caller is in. Only the path that
2645
+ -- claimed `settled` gets here, so the two writes cannot race.
2646
+ local function answer(ok: boolean)
2647
+ result = ok
2648
+ -- "suspended" is the only status that may be resumed. "normal" is the
2649
+ -- synchronous store above and "running" is a Clock.Delay firing on this very
2650
+ -- stack; both are answered by `result`, and resuming either would error.
2651
+ if coroutine.status(thread) == "suspended" then
2652
+ Sched.Resume(thread, ok)
2653
+ end
2654
+ end
2655
+
2656
+ Sched.Spawn(function()
2657
+ local outcome, saveContext = profile:SaveAwait()
2658
+ if settled then
2659
+ return
2660
+ end
2661
+ settled = true
2662
+ -- "Saved" is the only outcome that puts these bytes on disk, and even then not
2663
+ -- if the wipe guard exchanged in the last-good snapshot, nor if this was a
2664
+ -- ReleaseOnly write: a fail-closed release writes MetaData and deliberately
2665
+ -- leaves the stored bytes untouched. All three terms are per-save rather than
2666
+ -- read back off shared entry state.
2667
+ answer(outcome == "Saved" and not blockedForSave(entry, saveContext) and saveContext.ReleaseOnly ~= true)
2668
+ end)
2669
+
2670
+ Clock.Delay(timeout, function()
2671
+ if settled then
2672
+ return
2673
+ end
2674
+ settled = true
2675
+ -- Unresolved, not failed: the save is very often still in flight and lands
2676
+ -- seconds later, so it is deliberately not counted as SavesFailed.
2677
+ Metrics.Add("SaveWaitTimeouts")
2678
+ Log.Warn(
2679
+ "Persistence",
2680
+ "SAVE_WAIT_TIMEOUT",
2681
+ `gave up waiting {timeout}s for a save to confirm; it may still be in flight, so this is unresolved rather than failed`,
2682
+ { Player = entry.Player, Timeout = timeout }
2683
+ )
2684
+ answer(false)
2685
+ end)
2686
+
2687
+ -- Answered without this thread ever parking: the store settled inline, or an inline
2688
+ -- Clock.Delay timed the wait out on this stack. Nothing can run between Sched.Spawn
2689
+ -- returning and this test, so reading `result` is not a race. Tested against nil
2690
+ -- rather than truthiness: a `false` verdict is an ANSWER, and falling through to
2691
+ -- the yield would park a caller both wakers have retired.
2692
+ if result ~= nil then
2693
+ return result
2694
+ end
2695
+ return coroutine.yield() == true
2696
+ end
2697
+
2698
+ function self.GetSaveInfo(
2699
+ player: Player
2700
+ ): { LastSaveAt: number?, LastResult: ("Ok" | "Fail")?, Dirty: boolean, Size: number? }
2701
+ local entry = self.Entries[player]
2702
+ if not entry then
2703
+ return { LastSaveAt = nil, LastResult = nil, Dirty = false, Size = nil }
2704
+ end
2705
+ local info = table.clone(entry.SaveInfo)
2706
+ info.Size = entry.LastSavedSize
2707
+ return info
2708
+ end
2709
+
2710
+ -- Offline access ----------------------------------------------------------
2711
+
2712
+ -- Derived fields are computed, so stored data lacks them. Both branches of GetOffline
2713
+ -- fill in the root-level fields whose every input persists (Derived.OfflineComputable);
2714
+ -- anything reading a Session field is left out rather than invented.
2715
+ local function deriveOffline(snapshot: { [string]: any })
2716
+ if #compiled.DerivedNodes == 0 then
2717
+ return
2718
+ end
2719
+ DerivedModule.EvaluateInto(compiled, snapshot, function(node)
2720
+ return node.Derived.OfflineComputable
2721
+ end)
2722
+ end
2723
+
2724
+ function self.GetOffline(userId: number): { [string]: any }?
2725
+ local entry = self.EntryForUserId(userId)
2726
+ if entry and entry.State == "Ready" and entry.LiveData then
2727
+ local live = Util.deepClone(entry.LiveData)
2728
+ deriveOffline(live)
2729
+ return live
2730
+ end
2731
+ local ok, profile = pcall(function()
2732
+ return store:GetAsync(self.KeyFor(userId))
2733
+ end)
2734
+ if not ok then
2735
+ -- Distinguish a read error (retryable) from a missing profile
2736
+ -- (returns nil below with no failure report).
2737
+ Failure.Report({
2738
+ Subsystem = "OfflineRead",
2739
+ Key = tostring(userId),
2740
+ Error = profile,
2741
+ Category = "Persistence",
2742
+ Code = "OFFLINE_READ_FAIL",
2743
+ Message = tostring(profile),
2744
+ Context = { UserId = userId },
2745
+ })
2746
+ return nil
2747
+ end
2748
+ Health.ReportSuccess("OfflineRead")
2749
+ if not profile then
2750
+ return nil
2751
+ end
2752
+ -- A DataStore round trip turns a Scribe.MapOf("integer") key into its string
2753
+ -- form, and only the session load undid that, so an offline reader saw the raw
2754
+ -- strings. Coerce the copy, not the store's table.
2755
+ local snapshot = Util.deepClone(profile.Data)
2756
+ Template.CoerceMapKeys(compiled, snapshot)
2757
+ deriveOffline(snapshot)
2758
+ return snapshot
2759
+ end
2760
+
2761
+ -- Serialize same-user offline read-modify-writes on THIS server. A view-mode SetAsync
2762
+ -- writes the whole GetAsync snapshot back, so two interleaved UpdateOffline cycles for
2763
+ -- one userId have the second clobber the first, silently dropping a grant and its
2764
+ -- dedupe marker while both report success. Concurrent offline writes from a DIFFERENT
2765
+ -- server remain a view-mode limitation.
2766
+ local offlineLocks: { [number]: { thread } } = {}
2767
+ local function acquireOffline(userId: number)
2768
+ local waiters = offlineLocks[userId]
2769
+ if waiters == nil then
2770
+ offlineLocks[userId] = {} -- lock now held, no waiters
2771
+ return
2772
+ end
2773
+ table.insert(waiters, coroutine.running())
2774
+ coroutine.yield() -- resumed by releaseOffline when it is our turn
2775
+ end
2776
+ local function releaseOffline(userId: number)
2777
+ local waiters = offlineLocks[userId]
2778
+ if waiters and #waiters > 0 then
2779
+ Sched.Resume(table.remove(waiters, 1) :: thread) -- next waiter takes the lock
2780
+ else
2781
+ offlineLocks[userId] = nil
2782
+ end
2783
+ end
2784
+
2785
+ --- Turn an UpdateOfflineAsync outcome into Scribe's (ok, reason, dataStoreFailed).
2786
+ --- One table for updateOfflineProfile and RestoreVersion, so the two cannot come to
2787
+ --- disagree about which outcomes are infrastructure failures. Only "Failed" is.
2788
+ local function mapOfflineOutcome(outcome: string?, what: string): (boolean, string?, boolean)
2789
+ if outcome == "Written" then
2790
+ return true, nil, false
2791
+ elseif outcome == "SessionActive" then
2792
+ return false, "profile has an active session on another server", false
2793
+ elseif outcome == "Changed" then
2794
+ -- Somebody else wrote between the read and here. Nothing was written, so a
2795
+ -- caller that cares can simply try again against fresh data.
2796
+ return false, `profile changed while the {what} was being prepared`, false
2797
+ elseif outcome == "Missing" then
2798
+ return false, "profile does not exist", false
2799
+ end
2800
+ -- "Failed": the DataStore call itself did not land. Fail closed; an offline write
2801
+ -- we cannot stand behind is what the receipt path turns into a granted-but-unsaved
2802
+ -- purchase.
2803
+ return false, `offline {what} did not complete`, true
2804
+ end
2805
+
2806
+ -- Applies `mutate` to an offline profile and saves it, returning
2807
+ -- (ok, reason, dataStoreFailed). Only a failing GetAsync/SetAsync sets the third: a
2808
+ -- precondition refusal or a throwing callback is a LOGIC error, and reporting those to
2809
+ -- Health would let ordinary refusals flip the server to Outage. `mutate` runs against
2810
+ -- a CLONE, so a callback that errors partway commits nothing. The active-session check
2811
+ -- Validate judges the change, not the profile; at the cap the two scans cannot be compared.
2812
+ local VALIDATE_LIMIT = 256
2813
+ local TOO_MANY_VIOLATIONS =
2814
+ `the profile already carries {VALIDATE_LIMIT} or more template violations, too many for validation to tell the change apart; repair it or write raw`
2815
+
2816
+ -- Length-prefixed, so a dotted key and a nested path differ.
2817
+ local function violationKey(violation: Conform.Violation): string
2818
+ local parts = table.create(#violation.Path + 1)
2819
+ for i, segment in violation.Path do
2820
+ parts[i] = if type(segment) == "number" then `#{segment}` else `{#segment}:{segment}`
2821
+ end
2822
+ parts[#parts + 1] = violation.Kind
2823
+ return table.concat(parts, "/")
2824
+ end
2825
+
2826
+ local function readPath(data: any, path: { string | number }): any
2827
+ local value = data
2828
+ for _, segment in path do
2829
+ if type(value) ~= "table" then
2830
+ return nil
2831
+ end
2832
+ value = value[segment]
2833
+ end
2834
+ return value
2835
+ end
2836
+
2837
+ local function schemaAt(path: { string | number }): Template.SchemaNode?
2838
+ local node: Template.SchemaNode? = compiled.Root
2839
+ for _, segment in path do
2840
+ if node == nil then
2841
+ return nil
2842
+ end
2843
+ node = Template.ChildSchema(node, segment)
2844
+ end
2845
+ return node
2846
+ end
2847
+
2848
+ -- A set counts distinct members, as its cap does.
2849
+ local function entryCount(value: any, node: Template.SchemaNode?): number
2850
+ local n = 0
2851
+ if node ~= nil and node.Unique then
2852
+ local seen: { [any]: boolean } = {}
2853
+ for _, item in value :: { any } do
2854
+ if not seen[item] then
2855
+ seen[item] = true
2856
+ n += 1
2857
+ end
2858
+ end
2859
+ return n
2860
+ end
2861
+ for _ in value :: { [any]: any } do
2862
+ n += 1
2863
+ end
2864
+ return n
2865
+ end
2866
+
2867
+ -- An over-cap container keeps its size (edited, not grown); everything else a copy of its value.
2868
+ local function violationFootprint(draft: { [string]: any }, violation: Conform.Violation): any
2869
+ local value = readPath(draft, violation.Path)
2870
+ if violation.Kind == "OverMaxLength" and type(value) == "table" then
2871
+ return { Count = entryCount(value, schemaAt(violation.Path)) }
2872
+ end
2873
+ return { Value = Util.deepClone(value) }
2874
+ end
2875
+
2876
+ local function stillTheSame(before: any, after: any): boolean
2877
+ if before.Count ~= nil then
2878
+ return after.Count ~= nil and after.Count <= before.Count
2879
+ end
2880
+ return after.Count == nil and deepSame(before.Value, after.Value)
2881
+ end
2882
+
2883
+ -- Before the callback: an old-shape profile would report every difference as its doing.
2884
+ local function offlineVersionRefusal(draft: { [string]: any }): string?
2885
+ local reserved = draft[Template.RESERVED_ROOT]
2886
+ local stored = if type(reserved) == "table" and type(reserved.Version) == "number" then reserved.Version else 1
2887
+ if stored < currentVersion then
2888
+ return `profile is at schema version {stored} and this build is {currentVersion}; load it in a session to migrate it first`
2889
+ elseif stored > currentVersion then
2890
+ return `profile is at schema version {stored}, newer than this build's {currentVersion}; update the place before editing it`
2891
+ end
2892
+ return nil
2893
+ end
2894
+
2895
+ type OfflineBaseline = {
2896
+ Violations: { [string]: any },
2897
+ Full: boolean,
2898
+ Reserved: any,
2899
+ Derived: { [string]: any },
2900
+ Session: { [string]: any },
2901
+ }
2902
+
2903
+ local function offlineBaseline(draft: { [string]: any }): OfflineBaseline
2904
+ local derived = {}
2905
+ for _, node in compiled.DerivedNodes do
2906
+ if #node.Path == 1 then
2907
+ derived[node.Path[1]] = draft[node.Path[1]]
2908
+ end
2909
+ end
2910
+ local session = {}
2911
+ for name, child in compiled.Root.Children or {} do
2912
+ if not child.Persist then
2913
+ session[name] = { Value = Util.deepClone(draft[name]) }
2914
+ end
2915
+ end
2916
+ local found = Conform.Check(compiled, draft, { Limit = VALIDATE_LIMIT })
2917
+ local violations = {}
2918
+ for _, violation in found do
2919
+ violations[violationKey(violation)] = violationFootprint(draft, violation)
2920
+ end
2921
+ return {
2922
+ Violations = violations,
2923
+ Full = #found >= VALIDATE_LIMIT,
2924
+ Reserved = Util.deepClone(draft[Template.RESERVED_ROOT]),
2925
+ Derived = derived,
2926
+ Session = session,
2927
+ }
2928
+ end
2929
+
2930
+ local function offlineChangeRefusal(
2931
+ draft: { [string]: any },
2932
+ baseline: OfflineBaseline
2933
+ ): (string?, { Conform.Violation }?)
2934
+ if baseline.Full then
2935
+ return TOO_MANY_VIOLATIONS, nil
2936
+ end
2937
+ if not deepSame(draft[Template.RESERVED_ROOT], baseline.Reserved) then
2938
+ return `the change touches "{Template.RESERVED_ROOT}", which Scribe owns and an offline edit may not change`,
2939
+ nil
2940
+ end
2941
+ for name, was in baseline.Derived do
2942
+ if draft[name] ~= was then
2943
+ return `the change writes "{name}", a derived field that is computed and never stored`, nil
2944
+ end
2945
+ end
2946
+ for _, node in compiled.DerivedNodes do
2947
+ if #node.Path == 1 and baseline.Derived[node.Path[1]] == nil and draft[node.Path[1]] ~= nil then
2948
+ return `the change writes "{node.Path[1]}", a derived field that is computed and never stored`, nil
2949
+ end
2950
+ end
2951
+ for name, before in baseline.Session do
2952
+ if not deepSame(draft[name], before.Value) then
2953
+ return `the change writes "{name}", a Session field that is never stored`, nil
2954
+ end
2955
+ end
2956
+ local found = Conform.Check(compiled, draft, { Limit = VALIDATE_LIMIT })
2957
+ if #found >= VALIDATE_LIMIT then
2958
+ return TOO_MANY_VIOLATIONS, nil
2959
+ end
2960
+ local introduced = {}
2961
+ for _, violation in found do
2962
+ local before = baseline.Violations[violationKey(violation)]
2963
+ if before == nil or not stillTheSame(before, violationFootprint(draft, violation)) then
2964
+ table.insert(introduced, violation)
2965
+ end
2966
+ end
2967
+ if #introduced > 0 then
2968
+ local first = introduced[1]
2969
+ return `the change does not match the template at "{Util.pathString(first.Path)}": {first.Detail}`,
2970
+ introduced
2971
+ end
2972
+ return nil, nil
2973
+ end
2974
+
2975
+ -- is a pure read, deliberately: ProfileStore has no non-stealing acquire, and the
2976
+ -- first StartSessionAsync attempt writes ForceLoadSession and kicks an online player
2977
+ -- off another server. Losing the read-then-write race costs one clobbered write;
2978
+ -- closing it that way costs evicted players and eaten Robux.
2979
+ local function updateOfflineProfile(
2980
+ userId: number,
2981
+ mutate: (data: { [string]: any }) -> (),
2982
+ validate: boolean,
2983
+ createIfMissing: boolean?
2984
+ ): (boolean, string?, boolean, { Conform.Violation }?)
2985
+ local readOk, profile = pcall(function()
2986
+ return store:GetAsync(self.KeyFor(userId))
2987
+ end)
2988
+ if not readOk then
2989
+ return false, tostring(profile), true
2990
+ end
2991
+ if not profile then
2992
+ if not createIfMissing then
2993
+ return false, NO_PROFILE, false
2994
+ end
2995
+ elseif sessionLooksLive(profile) then
2996
+ -- A read, not an acquire: StartSessionAsync writes ForceLoadSession and would kick
2997
+ -- a live player, and EndSession only task.spawns its save, so acquiring would also
2998
+ -- cost this path its promise that the write landed.
2999
+ return false, LIVE_ELSEWHERE, false
3000
+ end
3001
+
3002
+ local draft = if profile then Util.deepClone(profile.Data) else {}
3003
+ -- Same integer-key coercion the session load does, so the callback sees the keys
3004
+ -- its template declares rather than the DataStore's string forms. The commit
3005
+ -- writes them back as numbers, which the store re-encodes as for a live save.
3006
+ Template.CoerceMapKeys(compiled, draft)
3007
+ local baseline: OfflineBaseline? = nil
3008
+ if validate then
3009
+ local refusal = offlineVersionRefusal(draft)
3010
+ if refusal ~= nil then
3011
+ return false, refusal, false
3012
+ end
3013
+ baseline = offlineBaseline(draft)
3014
+ end
3015
+ local mutateOk, mutateErr = pcall(mutate, draft)
3016
+ if not mutateOk then
3017
+ return false, tostring(mutateErr), false
3018
+ end
3019
+ -- The callback mutates raw data, bypassing Node.validateWrite, so unpersistable
3020
+ -- data (invalid UTF-8, a userdata/Instance value) would fail the whole profile
3021
+ -- save opaquely. Reject before committing.
3022
+ local bad = Util.findUnpersistable(draft)
3023
+ if bad ~= nil then
3024
+ return false, `unpersistable data ({bad.Detail}) at "{Util.pathString(bad.Path)}"`, false
3025
+ end
3026
+ if baseline ~= nil then
3027
+ local refusal, introduced = offlineChangeRefusal(draft, baseline)
3028
+ if refusal ~= nil then
3029
+ return false, refusal, false, introduced
3030
+ end
3031
+ end
3032
+
3033
+ -- Commit conditionally, in ONE call. `mutate` above is developer code and may yield
3034
+ -- for as long as it likes, so a player can join another server inside that window.
3035
+ -- Getting it wrong is worse than a lost write: a view SetAsync sets
3036
+ -- session_owns_profile and clears ActiveSession, so committing over a live session
3037
+ -- DISOWNS the server holding the key, and everything it wrote since its last save
3038
+ -- is gone. The compare-and-set makes the check and the write one DataStore call.
3039
+ if type((store :: any).UpdateOfflineAsync) == "function" then
3040
+ -- LastUpdate has one-second granularity and a view-mode SetAsync moves neither
3041
+ -- counter, hence WriteIndex; a recreated key can repeat all three, hence Generation.
3042
+ -- An empty fingerprint matches no stored key.
3043
+ local expected = if profile
3044
+ then {
3045
+ SessionLoadCount = profile.SessionLoadCount,
3046
+ LastUpdate = profile.LastUpdate,
3047
+ WriteIndex = profile.WriteIndex,
3048
+ Generation = profile.Generation,
3049
+ }
3050
+ else {}
3051
+ local casOk, outcome = pcall(function()
3052
+ return (store :: any):UpdateOfflineAsync(self.KeyFor(userId), expected, draft, {
3053
+ -- The same dead-session age sessionLooksLive uses, and the one
3054
+ -- StartSessionAsync steals at. Without it the CAS refuses on
3055
+ -- ActiveSession ~= nil at ANY age, so a crashed server locks
3056
+ -- UpdateOffline out forever.
3057
+ StaleSessionAfter = ASSUME_DEAD_SECONDS,
3058
+ -- Only when the read found nothing, so a key erased since stays erased.
3059
+ CreateIfMissing = createIfMissing == true and profile == nil,
3060
+ })
3061
+ end)
3062
+ if not casOk then
3063
+ return false, tostring(outcome), true
3064
+ end
3065
+ return mapOfflineOutcome(outcome, "update")
3066
+ end
3067
+
3068
+ -- Fallback for an injected store without the compare-and-set: re-read, then
3069
+ -- write, with the window that leaves. It cannot create a key.
3070
+ if profile == nil then
3071
+ return false, NO_PROFILE, false
3072
+ end
3073
+ local recheckOk, latest = pcall(function()
3074
+ return store:GetAsync(self.KeyFor(userId))
3075
+ end)
3076
+ if not recheckOk then
3077
+ return false, tostring(latest), true
3078
+ end
3079
+ if sessionLooksLive(latest) then
3080
+ return false, "profile has an active session on another server", false
3081
+ end
3082
+
3083
+ table.clear(profile.Data)
3084
+ for key, value in draft do
3085
+ profile.Data[key] = value
3086
+ end
3087
+ local saveOk, saveErr = commitViewProfile(profile, self.KeyFor(userId))
3088
+ if not saveOk then
3089
+ return false, saveErr, true
3090
+ end
3091
+ return true, nil, false
3092
+ end
3093
+
3094
+ function self.UpdateOffline(
3095
+ userId: number,
3096
+ fn: (data: { [string]: any }) -> (),
3097
+ opts: { Validate: boolean? }?
3098
+ ): (boolean, string?, { Conform.Violation }?)
3099
+ if mode == "NoSave" then
3100
+ return false, refuseNoSave("UpdateOffline")
3101
+ end
3102
+ if self.EntryForUserId(userId) then
3103
+ return false, "user has an active session on this server"
3104
+ end
3105
+ local validate = opts ~= nil and opts.Validate == true
3106
+ acquireOffline(userId)
3107
+ -- Still pcalled so an unexpected internal error cannot skip releaseOffline;
3108
+ -- such an error is not a DataStore failure either, so it does not report.
3109
+ local callOk, ok, reason, dataStoreFailed, violations = pcall(updateOfflineProfile, userId, fn, validate)
3110
+ releaseOffline(userId)
3111
+ if not callOk then
3112
+ return false, tostring(ok)
3113
+ end
3114
+ if not ok then
3115
+ if dataStoreFailed then
3116
+ -- Only a DataStore failure reports. A refusal (a live session, a changed
3117
+ -- key, a missing profile) is the compare-and-set working, and counting
3118
+ -- it against Health would make ordinary contention look like an outage.
3119
+ Failure.Report({
3120
+ Subsystem = "OfflineWrite",
3121
+ Key = tostring(userId),
3122
+ Error = reason,
3123
+ Category = "Persistence",
3124
+ Code = "OFFLINE_WRITE_FAIL",
3125
+ Message = tostring(reason),
3126
+ Context = { UserId = userId },
3127
+ })
3128
+ end
3129
+ return false, reason, violations
3130
+ end
3131
+ Health.ReportSuccess("OfflineWrite")
3132
+ return true, nil
3133
+ end
3134
+
3135
+ -- Version history and restore -----------------------------------------------
3136
+
3137
+ function self.ListVersions(
3138
+ userId: number,
3139
+ limit: number?
3140
+ ): { { VersionId: string, CreatedAt: number, Size: number? } }
3141
+ local out = {}
3142
+ local ok, err = pcall(function()
3143
+ local query = store:VersionQuery(self.KeyFor(userId), Enum.SortDirection.Descending)
3144
+ for _ = 1, limit or 25 do
3145
+ local profile = query:NextAsync()
3146
+ if not profile then
3147
+ break
3148
+ end
3149
+ table.insert(out, {
3150
+ VersionId = profile.KeyInfo.Version,
3151
+ CreatedAt = profile.KeyInfo.CreatedTime / 1000,
3152
+ Size = Util.approxSize(profile.Data),
3153
+ })
3154
+ end
3155
+ end)
3156
+ if not ok then
3157
+ Log.Warn("Persistence", "VERSION_QUERY_FAIL", tostring(err), { UserId = userId })
3158
+ end
3159
+ return out
3160
+ end
3161
+
3162
+ function self.GetVersion(userId: number, versionId: string): { [string]: any }?
3163
+ local ok, profile = pcall(function()
3164
+ return store:GetAsync(self.KeyFor(userId), versionId)
3165
+ end)
3166
+ if not ok then
3167
+ -- A read ERROR is retryable and must not be mistaken for a missing
3168
+ -- version during a manual restore.
3169
+ Failure.Report({
3170
+ Subsystem = "VersionRead",
3171
+ Key = tostring(userId),
3172
+ Error = profile,
3173
+ Category = "Persistence",
3174
+ Code = "VERSION_READ_FAIL",
3175
+ Message = tostring(profile),
3176
+ Context = { UserId = userId, VersionId = versionId },
3177
+ })
3178
+ return nil
3179
+ end
3180
+ Health.ReportSuccess("VersionRead")
3181
+ if not profile then
3182
+ return nil -- version genuinely not found
3183
+ end
3184
+ -- Same integer-key coercion and derived fill as GetOffline: this is a raw
3185
+ -- DataStore read, so it carries stored fields only.
3186
+ local snapshot = Util.deepClone(profile.Data)
3187
+ Template.CoerceMapKeys(compiled, snapshot)
3188
+ deriveOffline(snapshot)
3189
+ return snapshot
3190
+ end
3191
+
3192
+ function self.RestoreVersion(
3193
+ userId: number,
3194
+ versionId: string,
3195
+ opts: { RollBackReserved: boolean? }?
3196
+ ): (boolean, string?)
3197
+ if mode == "NoSave" then
3198
+ return false, refuseNoSave("RestoreVersion")
3199
+ end
3200
+ if Health.GetStatus() == "Outage" then
3201
+ return false, "data services are experiencing an outage"
3202
+ end
3203
+ if self.EntryForUserId(userId) then
3204
+ return false, "user has an active session on this server; restore requires no active session"
3205
+ end
3206
+ -- Both read by the logging below, so they live outside the pcall. `differed` is set
3207
+ -- only on a path that goes on to commit, so a failed restore logs nothing about a
3208
+ -- preservation that did not happen.
3209
+ local reservedRolledBack = opts ~= nil and opts.RollBackReserved == true
3210
+ local reservedDiffered = false
3211
+ -- The schema version the restore COMMITTED, and the one it replaced. Only set
3212
+ -- on the preserve path, and only used by the log below.
3213
+ local restoredSchemaVersion: number? = nil
3214
+ local previousSchemaVersion: number? = nil
3215
+ local ok, err = pcall(function()
3216
+ local live = store:GetAsync(self.KeyFor(userId))
3217
+ if sessionLooksLive(live) then
3218
+ error("profile has an active session on another server")
3219
+ end
3220
+ if not live then
3221
+ -- No live key to restore over (restoring after an Erase hits this).
3222
+ -- Profile:SetAsync onto an empty key throws inside ProfileStore's own
3223
+ -- transform and that error is swallowed, so we would report a successful
3224
+ -- restore for a write that never landed.
3225
+ error("no live profile exists for this user to restore over (was it erased?)")
3226
+ end
3227
+ -- A restore rolls GAME DATA back and deliberately preserves the live reserved
3228
+ -- root, which is right for receipts and perks and wrong in both directions
3229
+ -- for an exchange. Free to check: `live` is the read already taken above.
3230
+ local staked, parked = exchangeStateOf(live.Data)
3231
+ if staked > 0 or parked > 0 then
3232
+ error(
3233
+ `profile holds {staked} in-flight and {parked} undelivered exchange(s); a restore would `
3234
+ .. `roll game data back while the exchange ledger stayed live, duplicating a stake or `
3235
+ .. `destroying a delivery. Resolve them first`
3236
+ )
3237
+ end
3238
+ -- The fingerprint, taken from the SAME read that just cleared the liveness
3239
+ -- check. Everything below is preparation; the commit compares against this.
3240
+ local expected = {
3241
+ SessionLoadCount = live.SessionLoadCount,
3242
+ LastUpdate = live.LastUpdate,
3243
+ WriteIndex = live.WriteIndex,
3244
+ Generation = live.Generation,
3245
+ }
3246
+ local versionProfile = store:GetAsync(self.KeyFor(userId), versionId)
3247
+ if not versionProfile then
3248
+ error("version not found")
3249
+ end
3250
+ local data = versionProfile.Data
3251
+ -- THE RESERVED ROOT IS NOT PART OF THE ROLLBACK. `_Scribe` is the ledger of
3252
+ -- things that happened in the real world: receipts granted, gifts paid for,
3253
+ -- entitlements bought, cooldowns started. None of those un-happen because an
3254
+ -- operator restored a backup, so rolling them back lets one PurchaseId grant
3255
+ -- twice, resurrects delivered gifts and strips paid credits and perks.
3256
+ -- DEEP-CLONED, unlike runMigrations: the `RestoredFrom` stamp below lands
3257
+ -- before the commit can still be refused, and an injected store whose GetAsync
3258
+ -- hands back its own backing table would turn that stamp into a live write.
3259
+ if not reservedRolledBack then
3260
+ local liveReserved = live.Data[Template.RESERVED_ROOT]
3261
+ if type(liveReserved) == "table" then
3262
+ local snapshotReserved = data[Template.RESERVED_ROOT]
3263
+ -- `Version` is the one member of this root that travels with the game
3264
+ -- data: it describes the SHAPE of the data the lines above just
3265
+ -- replaced with an older snapshot. Carrying the live number onto
3266
+ -- restored old-shaped data makes runMigrations return early, so the
3267
+ -- chain never re-runs and the profile is stuck at the pre-migration
3268
+ -- shape marked current; under a staged deploy an older server also
3269
+ -- fails closed on PROFILE_VERSION_AHEAD and kick-loops the player.
3270
+ -- ABSENT MEANS 1, not "keep the live one": a snapshot with no reserved
3271
+ -- root is a LEGACY profile, the reading runMigrations gives it above.
3272
+ local snapshotVersion = if type(snapshotReserved) == "table"
3273
+ and type(snapshotReserved.Version) == "number"
3274
+ then snapshotReserved.Version
3275
+ else 1
3276
+ -- Version-aware ON PURPOSE (the migration call site is not; see
3277
+ -- reservedUnchanged). Without it the common case, a pre-deploy snapshot
3278
+ -- for a player who has bought nothing, reports no difference and hides
3279
+ -- the only change that alters what the next load does.
3280
+ reservedDiffered = not reservedUnchanged(liveReserved, snapshotReserved, true)
3281
+ data[Template.RESERVED_ROOT] = Util.deepClone(liveReserved)
3282
+ data[Template.RESERVED_ROOT].Version = snapshotVersion
3283
+ restoredSchemaVersion = snapshotVersion
3284
+ previousSchemaVersion = if type(liveReserved.Version) == "number" then liveReserved.Version else 1
3285
+ end
3286
+ -- No live reserved root at all (a legacy or imported profile that never had
3287
+ -- one): nothing real-world to preserve, so the snapshot's own root, or the
3288
+ -- empty fallback below, stands.
3289
+ end
3290
+ if type(data[Template.RESERVED_ROOT]) ~= "table" then
3291
+ data[Template.RESERVED_ROOT] = {}
3292
+ end
3293
+ data[Template.RESERVED_ROOT].RestoredFrom = { VersionId = versionId, At = Clock.Now() }
3294
+ -- Commit through the SAME compare-and-set updateOfflineProfile uses, in one
3295
+ -- DataStore call. Re-reading the lock and then writing is two round trips,
3296
+ -- which narrows the window without closing it, and a session landing between
3297
+ -- them is DISOWNED: the view write clears ActiveSession, so the holder keeps
3298
+ -- accepting writes it can no longer persist.
3299
+ if type((store :: any).UpdateOfflineAsync) ~= "function" then
3300
+ -- An injected store without the compare-and-set keeps the old shape,
3301
+ -- window and all; it is the only thing available.
3302
+ local latest = store:GetAsync(self.KeyFor(userId))
3303
+ if sessionLooksLive(latest) then
3304
+ error("profile has an active session on another server")
3305
+ end
3306
+ local committed, commitErr = commitViewProfile(versionProfile, self.KeyFor(userId))
3307
+ if not committed then
3308
+ error(commitErr)
3309
+ end
3310
+ return
3311
+ end
3312
+ local outcome = (store :: any):UpdateOfflineAsync(self.KeyFor(userId), expected, data, {
3313
+ StaleSessionAfter = ASSUME_DEAD_SECONDS,
3314
+ })
3315
+ local restored, reason = mapOfflineOutcome(outcome, "restore")
3316
+ if not restored then
3317
+ error(
3318
+ if outcome == "Missing"
3319
+ then "no live profile exists for this user to restore over (was it erased?)"
3320
+ else reason
3321
+ )
3322
+ end
3323
+ end)
3324
+ if not ok then
3325
+ -- Warn rather than Error deliberately: the caller is already being told, so
3326
+ -- this is a record rather than an alert.
3327
+ Failure.Report({
3328
+ Subsystem = "Restore",
3329
+ Key = tostring(userId),
3330
+ Error = err,
3331
+ Category = "Persistence",
3332
+ Code = "PROFILE_RESTORE_FAIL",
3333
+ Message = tostring(err),
3334
+ Context = { UserId = userId, VersionId = versionId },
3335
+ })
3336
+ return false, tostring(err)
3337
+ end
3338
+ Health.ReportSuccess("Restore")
3339
+ Log.Warn("Persistence", "PROFILE_RESTORED", "profile restored from version history", {
3340
+ UserId = userId,
3341
+ VersionId = versionId,
3342
+ PlaceId = game.PlaceId,
3343
+ JobId = game.JobId,
3344
+ ReservedRolledBack = reservedRolledBack,
3345
+ })
3346
+ -- Only when it actually DIFFERED, and only after the commit landed: an operator
3347
+ -- running a restore must not have to guess whether the money moved with it.
3348
+ if reservedDiffered then
3349
+ Log.Warn(
3350
+ "Persistence",
3351
+ "RESTORE_RESERVED_PRESERVED",
3352
+ `the reserved "{Template.RESERVED_ROOT}" root differed from the restored snapshot and was PRESERVED, not `
3353
+ .. `rolled back: receipt dedupe, gift escrow, gift credits, perks, purchase logs and Timed are still `
3354
+ .. `as they were before the restore, because those record events that already happened. The schema `
3355
+ .. `Version is the one exception -- it describes the SHAPE of the game data, so it travels with it: `
3356
+ .. `now {restoredSchemaVersion}, was {previousSchemaVersion}, and any migration steps above `
3357
+ .. `{restoredSchemaVersion} re-run on the next load. Pass RollBackReserved = true only if that root `
3358
+ .. `is itself what you are repairing`,
3359
+ {
3360
+ UserId = userId,
3361
+ VersionId = versionId,
3362
+ PlaceId = game.PlaceId,
3363
+ JobId = game.JobId,
3364
+ SchemaVersion = restoredSchemaVersion,
3365
+ PreviousSchemaVersion = previousSchemaVersion,
3366
+ }
3367
+ )
3368
+ end
3369
+ return true, nil
3370
+ end
3371
+
3372
+ -- GDPR ---------------------------------------------------------------------
3373
+
3374
+ local ERASE_LOST = "the erase was finished or replaced elsewhere"
3375
+ local ERASE_BUSY = "another erase of this profile is still running"
3376
+ -- Longer than a budget wait plus a request; a crashed worker blocks a retry this long.
3377
+ local ERASE_LEASE = 300
3378
+
3379
+ local function leaseLive(lease: any): boolean
3380
+ return type(lease) == "table" and type(lease.Until) == "number" and lease.Until > Clock.Now()
3381
+ end
3382
+
3383
+ --- nil while `profile` carries a marker leased to `leaseId`, else why this attempt lost it.
3384
+ local function markerLost(profile: any, leaseId: string): string?
3385
+ if profile == nil then
3386
+ return NO_PROFILE
3387
+ end
3388
+ local data = profile.Data
3389
+ local reserved = if type(data) == "table" then data[Template.RESERVED_ROOT] else nil
3390
+ local stored = if type(reserved) == "table" then reserved.Erase else nil
3391
+ if type(stored) ~= "table" then
3392
+ return "the profile was recreated since this erase began"
3393
+ end
3394
+ if type(stored.Lease) ~= "table" or stored.Lease.Id ~= leaseId then
3395
+ return "another erase replaced this one"
3396
+ end
3397
+ return nil
3398
+ end
3399
+
3400
+ --- Takes the erase lease in the same compare-and-set as the read it decides on; a user with
3401
+ --- no profile gets a placeholder holding only the marker. Returns the marker, the board
3402
+ --- ranges and whether an earlier attempt left the marker. Errors with the refusal.
3403
+ local function claimErase(userId: number, leaseId: string): (any, any, boolean)
3404
+ local marker: any = nil
3405
+ local ranges: any = nil
3406
+ local resumed = false
3407
+ local wrote, why = updateOfflineProfile(userId, function(draft)
3408
+ local reserved = draft[Template.RESERVED_ROOT]
3409
+ if type(reserved) ~= "table" then
3410
+ reserved = {}
3411
+ draft[Template.RESERVED_ROOT] = reserved
3412
+ end
3413
+ ranges = if type(reserved.Boards) == "table" then Util.deepClone(reserved.Boards) else nil
3414
+ local existing = reserved.Erase
3415
+ if type(existing) == "table" then
3416
+ if leaseLive(existing.Lease) then
3417
+ error(ERASE_BUSY, 0)
3418
+ end
3419
+ resumed = true
3420
+ marker = {
3421
+ StartedAt = existing.StartedAt,
3422
+ Done = if type(existing.Done) == "table" then Util.deepClone(existing.Done) else {},
3423
+ Cursors = if type(existing.Cursors) == "table" then Util.deepClone(existing.Cursors) else {},
3424
+ }
3425
+ else
3426
+ marker = { StartedAt = Clock.Now(), Done = {}, Cursors = {} }
3427
+ end
3428
+ marker.Lease = { Id = leaseId, Until = Clock.Now() + ERASE_LEASE }
3429
+ reserved.Erase = Util.deepClone(marker)
3430
+ end, false, true)
3431
+ if wrote then
3432
+ return marker, ranges, resumed
3433
+ end
3434
+ if why == ERASE_BUSY or why == LIVE_ELSEWHERE then
3435
+ error(why, 0)
3436
+ end
3437
+ error(`could not mark the profile for erasure: {tostring(why)}`, 0)
3438
+ end
3439
+
3440
+ --- Writes the sweep's progress and renews the lease, or with `release` drops it. The third
3441
+ --- return is true once the key no longer carries this attempt's lease.
3442
+ local function renewErase(
3443
+ userId: number,
3444
+ marker: any,
3445
+ leaseId: string,
3446
+ release: boolean
3447
+ ): (boolean, string?, boolean)
3448
+ local wrote, why = updateOfflineProfile(userId, function(draft)
3449
+ local reserved = draft[Template.RESERVED_ROOT]
3450
+ local existing = if type(reserved) == "table" then reserved.Erase else nil
3451
+ if type(existing) ~= "table" or type(existing.Lease) ~= "table" or existing.Lease.Id ~= leaseId then
3452
+ error(ERASE_LOST, 0)
3453
+ end
3454
+ marker.Lease = if release then nil else { Id = leaseId, Until = Clock.Now() + ERASE_LEASE }
3455
+ reserved.Erase = Util.deepClone(marker)
3456
+ end, false)
3457
+ if wrote then
3458
+ return true, nil, false
3459
+ end
3460
+ if why == NO_PROFILE or why == ERASE_LOST then
3461
+ return false, why, true
3462
+ end
3463
+ -- A live session refuses the write before the marker is seen; only a read can tell.
3464
+ if why == LIVE_ELSEWHERE then
3465
+ local readOk, stored = pcall(function()
3466
+ return store:GetAsync(self.KeyFor(userId))
3467
+ end)
3468
+ if readOk and markerLost(stored, leaseId) ~= nil then
3469
+ return false, ERASE_LOST, true
3470
+ end
3471
+ end
3472
+ return false, why, false
3473
+ end
3474
+
3475
+ local function eraseHeld(userId: number): (boolean, string?)
3476
+ local leaseId = Util.guid()
3477
+ local marker: any = nil
3478
+ local ranges: any = nil
3479
+ local ok, err = pcall(function()
3480
+ local resumed
3481
+ marker, ranges, resumed = claimErase(userId, leaseId)
3482
+ if resumed then
3483
+ Log.Info("Persistence", "PROFILE_ERASE_RESUMED", "resuming an erase that did not finish", {
3484
+ UserId = userId,
3485
+ StartedAt = marker.StartedAt,
3486
+ })
3487
+ end
3488
+ end)
3489
+ local progress: any = nil
3490
+ if ok then
3491
+ progress = { Done = marker.Done, Cursors = marker.Cursors, Lost = false, Stopped = nil }
3492
+ -- Past a failed renewal the key may be another attempt's, so the sweep stops.
3493
+ progress.Save = function(): boolean
3494
+ local why
3495
+ for attempt = 1, 2 do
3496
+ local saved, reason, lost = renewErase(userId, marker, leaseId, false)
3497
+ if lost then
3498
+ progress.Lost = true
3499
+ return false
3500
+ end
3501
+ if saved then
3502
+ return true
3503
+ end
3504
+ why = reason
3505
+ if attempt == 1 then
3506
+ Clock.Wait(1)
3507
+ end
3508
+ end
3509
+ Log.Warn(
3510
+ "Persistence",
3511
+ "PROFILE_ERASE_PROGRESS_FAIL",
3512
+ "erase progress was not saved; the sweep stops and a retry resumes it",
3513
+ { UserId = userId, Reason = why }
3514
+ )
3515
+ progress.Stopped = tostring(why)
3516
+ return false
3517
+ end
3518
+ local lbOk, lbErr = ctx.Leaderboards.EraseUser(userId, ranges, progress)
3519
+ -- A lost erase is judged by the key below, not by the sweep it cut short.
3520
+ if progress.Stopped ~= nil then
3521
+ ok, err = false, `the erase lease could not be renewed: {progress.Stopped}`
3522
+ elseif not lbOk and not progress.Lost then
3523
+ ok, err = false, `leaderboard entries remain: {tostring(lbErr)}`
3524
+ end
3525
+ end
3526
+ local finishedElsewhere = false
3527
+ if ok then
3528
+ ok, err = pcall(function()
3529
+ -- The sweep yields, so the key may have changed hands since the claim.
3530
+ local stored = store:GetAsync(self.KeyFor(userId))
3531
+ if stored == nil then
3532
+ finishedElsewhere = true
3533
+ return
3534
+ end
3535
+ if sessionLooksLive(stored) then
3536
+ error(LIVE_ELSEWHERE, 0)
3537
+ end
3538
+ local lost = markerLost(stored, leaseId)
3539
+ if lost ~= nil then
3540
+ error(lost, 0)
3541
+ end
3542
+ if not store:RemoveAsync(self.KeyFor(userId)) then
3543
+ error("RemoveAsync returned false")
3544
+ end
3545
+ end)
3546
+ end
3547
+ if not ok then
3548
+ -- Drop the lease so a retry need not wait for it; a refused drop leaves it to lapse.
3549
+ if marker ~= nil and not (progress ~= nil and progress.Lost) then
3550
+ renewErase(userId, marker, leaseId, true)
3551
+ end
3552
+ -- Same as Restore: without this a failed GDPR erase reaches no sink at all.
3553
+ Failure.Report({
3554
+ Subsystem = "Erase",
3555
+ Key = tostring(userId),
3556
+ Error = err,
3557
+ Category = "Persistence",
3558
+ Code = "PROFILE_ERASE_FAIL",
3559
+ Message = tostring(err),
3560
+ Context = { UserId = userId },
3561
+ })
3562
+ return false, tostring(err)
3563
+ end
3564
+ if finishedElsewhere then
3565
+ return true, nil
3566
+ end
3567
+ Log.Warn("Persistence", "PROFILE_ERASED", "profile erased (GDPR)", {
3568
+ UserId = userId,
3569
+ PlaceId = game.PlaceId,
3570
+ JobId = game.JobId,
3571
+ })
3572
+ return true, nil
3573
+ end
3574
+
3575
+ function self.Erase(userId: number): (boolean, string?)
3576
+ if mode == "NoSave" then
3577
+ return false, refuseNoSave("Erase")
3578
+ end
3579
+ if Health.GetStatus() == "Outage" then
3580
+ return false, "data services are experiencing an outage"
3581
+ end
3582
+ if self.EntryForUserId(userId) then
3583
+ return false, "user has an active session on this server"
3584
+ end
3585
+ erasing[userId] = (erasing[userId] or 0) + 1
3586
+ local callOk, ok, reason = pcall(eraseHeld, userId)
3587
+ erasing[userId] = if erasing[userId] > 1 then erasing[userId] - 1 else nil
3588
+ if not callOk then
3589
+ return false, tostring(ok)
3590
+ end
3591
+ return ok, reason
3592
+ end
3593
+
3594
+ local function jsonSafe(value: any): any
3595
+ local t = type(value)
3596
+ if t == "buffer" then
3597
+ return "base64:" .. Util.base64FromBuffer(value)
3598
+ elseif t == "table" then
3599
+ local out = {}
3600
+ for k, v in value do
3601
+ out[k] = jsonSafe(v)
3602
+ end
3603
+ return out
3604
+ end
3605
+ return value
3606
+ end
3607
+
3608
+ function self.Export(userId: number): string?
3609
+ local data = self.GetOffline(userId)
3610
+ if data == nil then
3611
+ return nil
3612
+ end
3613
+ local ok, encoded = pcall(function()
3614
+ return HttpService:JSONEncode(jsonSafe(data))
3615
+ end)
3616
+ if not ok then
3617
+ Log.Warn("Persistence", "EXPORT_ENCODE_FAIL", tostring(encoded), { UserId = userId })
3618
+ return nil
3619
+ end
3620
+ return encoded
3621
+ end
3622
+
3623
+ -- Shutdown -------------------------------------------------------------------
3624
+
3625
+ -- The BindToClose drain: stop accepting new sessions, flush the
3626
+ -- leaderboard queue, end every profile session, and wait within the
3627
+ -- budget. Exposed separately so the test harness can simulate shutdown.
3628
+ function self.Shutdown(budget: number?)
3629
+ self.Closing = true
3630
+ -- Before anything else: stop the store starting work this drain would then have to
3631
+ -- wait for. Its own BindToClose stands down (ExternalShutdown), so unless this
3632
+ -- runs, IsClosing never gets set and MessageAsync keeps retrying through the
3633
+ -- shutdown window.
3634
+ if type((ProfileStore :: any).BeginClose) == "function" then
3635
+ pcall((ProfileStore :: any).BeginClose)
3636
+ end
3637
+ local sessionCount = self.EntryCount
3638
+ Log.Info("Persistence", "SHUTDOWN_FLUSH", "shutting down; draining saves and leaderboards", {
3639
+ Sessions = sessionCount,
3640
+ })
3641
+ -- Single deadline shared by both phases so a slow leaderboard flush can never
3642
+ -- squeeze profile saves past Roblox's ~30s BindToClose window. Profile saves go
3643
+ -- first; the leaderboard queue is rebuildable and gets whatever time remains.
3644
+ local deadline = Clock.Monotonic() + (budget or 25)
3645
+ -- Tear down profile-less Loading entries up front: one parked in a
3646
+ -- LoadFailurePolicy = "Wait" backoff sleeps up to 60s before noticing Closing and
3647
+ -- would hold the drain loop below to its full budget. Their StartSessionAsync
3648
+ -- cannot succeed anymore (the Cancel hook sees Closing), and the loader no-ops on
3649
+ -- TornDown when it wakes.
3650
+ local strandedLoading: { Entry } = {}
3651
+ for _, entry in self.Entries :: { [Player]: Entry } do
3652
+ if entry.State == "Loading" and entry.Profile == nil then
3653
+ table.insert(strandedLoading, entry)
3654
+ end
3655
+ end
3656
+ for _, entry in strandedLoading do
3657
+ teardown(entry, "shutdown")
3658
+ end
3659
+ -- Snapshot before running anything, the same two-phase shape as the stranded pass
3660
+ -- above. A hook gets the real accessor and can kick a player or end another
3661
+ -- session, and clearing keys mid-traversal is legal while ADDING one is undefined.
3662
+ local draining: { Entry } = {}
3663
+ for _, entry in self.Entries :: { [Player]: Entry } do
3664
+ if entry.Profile and entry.Profile:IsActive() then
3665
+ table.insert(draining, entry)
3666
+ end
3667
+ end
3668
+
3669
+ -- Idempotent, because the cut-off below ends a parked hook's session on this
3670
+ -- thread and the hook then resumes into its own call to this.
3671
+ local function endSession(entry: Entry)
3672
+ if entry.LocalSessionEnd or not entry.Profile then
3673
+ return
3674
+ end
3675
+ entry.LocalSessionEnd = true
3676
+ entry.Profile:EndSession()
3677
+ end
3678
+
3679
+ local hookDeadline = Clock.Monotonic() + math.max(0, deadline - Clock.Monotonic()) * HOOK_PHASE_SHARE
3680
+ local pending = 0
3681
+ local hooksSkipped = 0
3682
+ for _, entry in draining do
3683
+ if not (entry.Profile and entry.Profile:IsActive()) then
3684
+ continue -- a hook that already ran ended this one for us
3685
+ end
3686
+ if Clock.Monotonic() >= hookDeadline then
3687
+ -- Out of hook time. The save is not optional; the hook is.
3688
+ hooksSkipped += 1
3689
+ endSession(entry)
3690
+ continue
3691
+ end
3692
+ pending += 1
3693
+ Sched.Spawn(function()
3694
+ -- Per entry, the hook still runs strictly before ITS OWN EndSession: that
3695
+ -- ordering is the documented guarantee, and losing it is silent because
3696
+ -- EndSession destroys the tree synchronously and every later hook write
3697
+ -- raises into the contained LISTENER_ERROR path. What is relaxed is the
3698
+ -- ordering BETWEEN entries; Spawn runs inline until the hook yields, so a
3699
+ -- non-yielding hook behaves exactly as it did as straight-line code.
3700
+ runLeavingHook(entry, "shutdown")
3701
+ pending -= 1
3702
+ endSession(entry)
3703
+ end)
3704
+ end
3705
+ while pending > 0 and Clock.Monotonic() < hookDeadline do
3706
+ Clock.Wait(HOOK_JOIN_POLL)
3707
+ end
3708
+ -- Whoever is still parked has run out of budget, and Luau cannot interrupt them.
3709
+ -- What they wrote before this point is already in the profile table and persists,
3710
+ -- so a timed-out hook is not an atomic one.
3711
+ local hooksTimedOut = 0
3712
+ for _, entry in draining do
3713
+ if entry.Profile and entry.Profile:IsActive() and not entry.LocalSessionEnd then
3714
+ hooksTimedOut += 1
3715
+ endSession(entry)
3716
+ end
3717
+ end
3718
+ if hooksSkipped > 0 then
3719
+ Metrics.Add("LeavingHooksSkipped", hooksSkipped)
3720
+ end
3721
+ if hooksTimedOut > 0 then
3722
+ Metrics.Add("LeavingHooksTimedOut", hooksTimedOut)
3723
+ end
3724
+
3725
+ local lbBudget = math.max(0, math.min(deadline - Clock.Monotonic(), 20))
3726
+ ctx.Leaderboards.FlushAllBlocking(lbBudget)
3727
+ while next(self.Entries) ~= nil and Clock.Monotonic() < deadline do
3728
+ Clock.Wait(0.2)
3729
+ end
3730
+ -- Everything Scribe owns is released by now, so this is the job-drain wait:
3731
+ -- it keeps the process alive until the saves those releases started have
3732
+ -- actually landed, and sweeps up any profile Scribe does not know about.
3733
+ if type((ProfileStore :: any).AwaitClose) == "function" then
3734
+ pcall((ProfileStore :: any).AwaitClose)
3735
+ end
3736
+ Log.Info("Persistence", "SHUTDOWN_DONE", "shutdown drain complete", {
3737
+ Remaining = self.EntryCount,
3738
+ HooksSkipped = hooksSkipped,
3739
+ HooksTimedOut = hooksTimedOut,
3740
+ })
3741
+ end
3742
+
3743
+ function self.BindShutdown()
3744
+ game:BindToClose(function()
3745
+ self.Shutdown(25)
3746
+ end)
3747
+ end
3748
+
3749
+ -- Release the process-global signal connections this bundle installed.
3750
+ function self.Stop()
3751
+ -- FIRST, before anything is disconnected: the loader reads it on its next turn,
3752
+ -- and a flag set after the yield-heavy teardown would let another round through.
3753
+ self.Stopped = true
3754
+ for _, entry in self.Entries do
3755
+ if entry.StopLoadWatchdog then
3756
+ entry.StopLoadWatchdog()
3757
+ entry.StopLoadWatchdog = nil
3758
+ end
3759
+ if entry.StopLeavingHookWatchdog then
3760
+ entry.StopLeavingHookWatchdog()
3761
+ entry.StopLeavingHookWatchdog = nil
3762
+ end
3763
+ end
3764
+ for _, connection in connections do
3765
+ pcall(function()
3766
+ connection:Disconnect()
3767
+ end)
3768
+ end
3769
+ table.clear(connections)
3770
+ end
3771
+
3772
+ return self
3773
+ end
3774
+
3775
+ return Persistence