@rbxts/scribe 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +21 -0
- package/README.md +100 -0
- package/Scribe-source-map.json +1 -0
- package/licenses/ProfileStore-Apache-2.0.txt +201 -0
- package/package.json +32 -0
- package/src/Client/DebugHook.luau +457 -0
- package/src/Client/init.luau +1607 -0
- package/src/Internal/Addons.luau +68 -0
- package/src/Internal/Big.luau +582 -0
- package/src/Internal/Budget.luau +377 -0
- package/src/Internal/Clock.luau +75 -0
- package/src/Internal/Config.luau +17 -0
- package/src/Internal/Conform.luau +400 -0
- package/src/Internal/Datatypes.luau +653 -0
- package/src/Internal/Defaults.luau +12 -0
- package/src/Internal/Derived.luau +428 -0
- package/src/Internal/Exchange.luau +556 -0
- package/src/Internal/ExchangeLeg.luau +1368 -0
- package/src/Internal/ExchangeVerdict.luau +179 -0
- package/src/Internal/Failure.luau +148 -0
- package/src/Internal/Health.luau +320 -0
- package/src/Internal/LeaderboardDiagnostics.luau +36 -0
- package/src/Internal/Log.luau +378 -0
- package/src/Internal/Marker.luau +76 -0
- package/src/Internal/Metrics.luau +152 -0
- package/src/Internal/Node.luau +3954 -0
- package/src/Internal/Profile.luau +69 -0
- package/src/Internal/Rng.luau +83 -0
- package/src/Internal/Sched.luau +56 -0
- package/src/Internal/Signal.luau +199 -0
- package/src/Internal/Store/ProfileStore.luau +3019 -0
- package/src/Internal/Store/init.luau +18 -0
- package/src/Internal/StudioCodec.luau +155 -0
- package/src/Internal/Template.luau +1848 -0
- package/src/Internal/Transport.luau +169 -0
- package/src/Internal/Util.luau +517 -0
- package/src/Internal/Version.luau +4 -0
- package/src/Internal/Wire.luau +1283 -0
- package/src/Server/Analytics.luau +223 -0
- package/src/Server/Commands.luau +728 -0
- package/src/Server/DebugHook.luau +1490 -0
- package/src/Server/Exchanges.luau +737 -0
- package/src/Server/Leaderboards.luau +2143 -0
- package/src/Server/Monetization.luau +3212 -0
- package/src/Server/Persistence.luau +3775 -0
- package/src/Server/Replication.luau +1253 -0
- package/src/Server/ServerStore.luau +81 -0
- package/src/Server/Timed.luau +579 -0
- package/src/Server/init.luau +672 -0
- package/src/Types.luau +480 -0
- package/src/init.luau +2231 -0
- package/types/api.d.ts +252 -0
- package/types/common.d.ts +355 -0
- package/types/index.d.ts +226 -0
- package/types/schema.d.ts +276 -0
|
@@ -0,0 +1,3775 @@
|
|
|
1
|
+
--!strict
|
|
2
|
+
-- Persistence on ProfileStore: session lifecycle, fail-closed
|
|
3
|
+
-- migrations, offline access, version history/restore, save observability,
|
|
4
|
+
-- wipe guard, and BindToClose.
|
|
5
|
+
|
|
6
|
+
local Players = game:GetService("Players")
|
|
7
|
+
local HttpService = game:GetService("HttpService")
|
|
8
|
+
|
|
9
|
+
local Internal = script.Parent.Parent.Internal
|
|
10
|
+
local Types = require(script.Parent.Parent.Types)
|
|
11
|
+
local Util = require(Internal.Util)
|
|
12
|
+
local Node = require(Internal.Node)
|
|
13
|
+
local Template = require(Internal.Template)
|
|
14
|
+
local Defaults = require(Internal.Defaults)
|
|
15
|
+
local Conform = require(Internal.Conform)
|
|
16
|
+
local DerivedModule = require(Internal.Derived)
|
|
17
|
+
local Log = require(Internal.Log)
|
|
18
|
+
local Metrics = require(Internal.Metrics)
|
|
19
|
+
local Health = require(Internal.Health)
|
|
20
|
+
local Signal = require(Internal.Signal)
|
|
21
|
+
local Config = require(Internal.Config)
|
|
22
|
+
local Clock = require(Internal.Clock)
|
|
23
|
+
local Sched = require(Internal.Sched)
|
|
24
|
+
local Profile = require(Internal.Profile)
|
|
25
|
+
local Failure = require(Internal.Failure)
|
|
26
|
+
local Budget = require(Internal.Budget)
|
|
27
|
+
local Rng = require(Internal.Rng)
|
|
28
|
+
|
|
29
|
+
local Persistence = {}
|
|
30
|
+
|
|
31
|
+
export type Entry = {
|
|
32
|
+
Player: Player,
|
|
33
|
+
UserId: number,
|
|
34
|
+
Key: string,
|
|
35
|
+
JoinedAt: number?, -- monotonic, stamped at the join; drives LoadDuration
|
|
36
|
+
State: "Loading" | "Ready" | "SessionEnded",
|
|
37
|
+
FailReason: string?,
|
|
38
|
+
Profile: any?,
|
|
39
|
+
Tree: any?,
|
|
40
|
+
SessionStore: { [string]: any },
|
|
41
|
+
LiveData: { [string]: any }?,
|
|
42
|
+
StateChanged: any,
|
|
43
|
+
SaveInfo: { LastSaveAt: number?, LastResult: ("Ok" | "Fail")?, Dirty: boolean },
|
|
44
|
+
SaveT0: number?,
|
|
45
|
+
-- Saves on the wire right now. The store permits N concurrent saves, so a single
|
|
46
|
+
-- slot cannot attribute a failure that arrives after the first completion.
|
|
47
|
+
SavesInFlight: number,
|
|
48
|
+
-- Dirtiness as it stood before ANY in-flight save snapshotted its payload, so the
|
|
49
|
+
-- OnError handler can restore it when one fails. OR in at every snapshot and clear
|
|
50
|
+
-- only when SavesInFlight drains to zero; a per-save capture lets a later save
|
|
51
|
+
-- overwrite an earlier one's value. nil and false both mean "nothing says dirty", so
|
|
52
|
+
-- a store exposing no OnSettled never clears this and can only ever over-report.
|
|
53
|
+
DirtyAtSaveStart: boolean?,
|
|
54
|
+
Blocked: boolean,
|
|
55
|
+
ForceSaveOnce: boolean,
|
|
56
|
+
WipeGuardRebaselinePending: boolean?,
|
|
57
|
+
LocalSessionEnd: boolean?, -- set before every locally-initiated EndSession
|
|
58
|
+
-- Timed's live _Scribe.Timed dict while a save is in flight, held so the restore
|
|
59
|
+
-- can run from OnAfterSave or, if the save fails, from the OnError handler.
|
|
60
|
+
TimedHeld: { [string]: number }?,
|
|
61
|
+
TimedLive: { [string]: number }?, -- same table, read by Timed's own store lookup
|
|
62
|
+
TimedSaves: number?, -- open save windows; saves are not serialized per profile
|
|
63
|
+
LeavingHookRan: boolean?, -- OnPlayerLeaving fires at most once per session
|
|
64
|
+
StopLeavingHookWatchdog: (() -> ())?,
|
|
65
|
+
-- Cross-server messages handed to Data.OnMessage whose FireTracked has not settled.
|
|
66
|
+
-- Nonzero at session end means a handler never returned; teardown reports it.
|
|
67
|
+
UnsettledMessages: number?,
|
|
68
|
+
SessionStolen: boolean?, -- OnSessionEnd fired that we did not initiate (external steal)
|
|
69
|
+
LastSavedSize: number?,
|
|
70
|
+
-- Top-level keys of the last successful save. The wipe guard's baseline, kept
|
|
71
|
+
-- here so the store does not have to keep a second copy of the whole profile.
|
|
72
|
+
LastSavedKeys: { [string]: true }?,
|
|
73
|
+
LiveSize: number?, -- approx bytes measured at the start of the last save attempt
|
|
74
|
+
SizeWarned: boolean,
|
|
75
|
+
TornDown: boolean,
|
|
76
|
+
-- Replication state (owned by Replication.luau)
|
|
77
|
+
HelloReceived: boolean,
|
|
78
|
+
-- Two flags, deliberately: QueueOpen means a snapshot has been taken, so a write
|
|
79
|
+
-- belongs in the queue; HandshakeReady means the snapshot is on the wire, so this
|
|
80
|
+
-- client may be sent frames. One flag left a gap between those two moments in
|
|
81
|
+
-- which a write belonged nowhere and was silently dropped.
|
|
82
|
+
QueueOpen: boolean,
|
|
83
|
+
HandshakeReady: boolean,
|
|
84
|
+
InitDelivered: boolean, -- a snapshot has reached this client
|
|
85
|
+
LastInitAt: number,
|
|
86
|
+
-- A frame did not reach the transport, so this client's mirror is stale and it
|
|
87
|
+
-- cannot ask for a repair (a loaded client never re-Hellos). The flush loop
|
|
88
|
+
-- re-sends the whole snapshot; LastResyncAt throttles repeats so a permanently
|
|
89
|
+
-- broken transport cannot cost one encode per frame.
|
|
90
|
+
ResyncPending: boolean?,
|
|
91
|
+
LastResyncAt: number?,
|
|
92
|
+
Queue: { any },
|
|
93
|
+
QueueIndex: { [string]: number },
|
|
94
|
+
SharedQueue: { any },
|
|
95
|
+
MalformedCount: number,
|
|
96
|
+
MalformedBlocked: boolean?, -- crossed the malformed-frame limit; inbound frames ignored
|
|
97
|
+
LastMalformedLog: number?, -- os.clock of the last emitted malformed-frame log/anomaly
|
|
98
|
+
OversizeCount: number, -- own budget: an oversize frame is never read, a malformed one was
|
|
99
|
+
OversizeBlocked: boolean?, -- crossed the oversize limit; inbound frames ignored
|
|
100
|
+
LastOversizeLog: number?, -- os.clock of the last emitted INBOUND_OVERSIZE
|
|
101
|
+
-- Raw inbound frame budget: every frame the player sends, whatever its type, size
|
|
102
|
+
-- or validity, spent before the frame is looked at at all.
|
|
103
|
+
FrameTokens: number,
|
|
104
|
+
FrameLast: number,
|
|
105
|
+
LastInboundRateLog: number?, -- os.clock of the last emitted INBOUND_RATE_LIMITED
|
|
106
|
+
-- Commands state
|
|
107
|
+
RateTokens: number,
|
|
108
|
+
RateLast: number,
|
|
109
|
+
RateLimitedCount: number?, -- consecutive rate-limited frames (decays when quiet)
|
|
110
|
+
LastRateLimitAt: number?, -- os.clock of the last rate-limited frame
|
|
111
|
+
LastRateLimitLog: number?, -- os.clock of the last emitted COMMAND_RATE_LIMITED
|
|
112
|
+
-- Why the store last failed for this key. StartSessionAsync answers nil with no reason
|
|
113
|
+
-- of its own, so this is the only way a load failure can tell a throttle apart.
|
|
114
|
+
LastStoreErrorClass: Failure.Class?,
|
|
115
|
+
LastStoreErrorAt: number?, -- monotonic
|
|
116
|
+
-- A parallel field, not a fourth State, so nothing switching on State changes
|
|
117
|
+
-- meaning. nil once Ready.
|
|
118
|
+
LoadPhase: Types.LoadPhase?,
|
|
119
|
+
SlowLoadWarned: boolean?,
|
|
120
|
+
StopLoadWatchdog: (() -> ())?,
|
|
121
|
+
-- Monetization state
|
|
122
|
+
GiftLastAt: number,
|
|
123
|
+
OwnsLast: { [string]: boolean }?, -- last reported effective ownership, for change detection
|
|
124
|
+
OwnsWatchers: { () -> () }?, -- Perks/OwnsCache listeners driving OnOwnershipChanged
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
local SIZE_WARN_BYTES = 3.5 * 1024 * 1024 -- warn approaching the ~4 MB DataStore ceiling
|
|
128
|
+
-- Past this a join is something the player notices. Well under LoadTimeout, where they
|
|
129
|
+
-- are kicked: a warn firing only at the timeout would only ever report failures.
|
|
130
|
+
local SLOW_LOAD_SECONDS = 10
|
|
131
|
+
local SLOW_LEAVING_HOOK_SECONDS = 10
|
|
132
|
+
-- An import is unbounded by design and holds the session lock throughout. Well past
|
|
133
|
+
-- SLOW_LOAD, because waiting on budget is the point.
|
|
134
|
+
local SLOW_IMPORT_SECONDS = 30
|
|
135
|
+
local IMPORT_SLOT_POLL = 0.25
|
|
136
|
+
local DEFAULT_MIGRATION_CONCURRENCY = 2
|
|
137
|
+
-- Held back from migrations. Nothing in a join CALLS GetAsync (ProfileStore loads
|
|
138
|
+
-- through UpdateAsync), but an UpdateAsync charges the read pool too, so a join spends
|
|
139
|
+
-- this anyway: one read and one write per join, measured in SimBudget. That dual charge
|
|
140
|
+
-- is undocumented (Uncertain.luau U12), so a join-rate derivation would rest on it.
|
|
141
|
+
local GET_RESERVE = 2
|
|
142
|
+
-- Ceiling on the GetSortedAsync reserve, which follows the declared board count: one
|
|
143
|
+
-- read per board lets a refresh tick through while a migration drains the pool. Capped
|
|
144
|
+
-- because boards come due on their own intervals, not together.
|
|
145
|
+
local SORTED_RESERVE_CAP = 4
|
|
146
|
+
local MIN_SAVE_INTERVAL = 15 -- seconds; below this, per-key DataStore throttling causes save failures
|
|
147
|
+
local WIPE_GUARD_RATIO_DEFAULT = 0.6 -- a profile shrinking past 60% looks like a wipe, not like spending
|
|
148
|
+
|
|
149
|
+
-- Matches the store's ASSUME_DEAD. Its own constant rather than read from the module,
|
|
150
|
+
-- so the operator paths behave identically against an injected store that lacks it.
|
|
151
|
+
local ASSUME_DEAD_SECONDS = 630
|
|
152
|
+
|
|
153
|
+
-- Share of the shutdown budget the OnPlayerLeaving phase may spend. Hooks come first
|
|
154
|
+
-- because their writes are the only thing here that cannot be rebuilt; the remainder
|
|
155
|
+
-- is a floor under the leaderboard flush, not a cap on it.
|
|
156
|
+
local HOOK_PHASE_SHARE = 0.75
|
|
157
|
+
local HOOK_JOIN_POLL = 0.05 -- seconds between checks that the parked hooks have returned
|
|
158
|
+
|
|
159
|
+
local function watchdog(seconds: number, callback: () -> ()): () -> ()
|
|
160
|
+
local pending: (() -> ())? = callback
|
|
161
|
+
Clock.Delay(seconds, function()
|
|
162
|
+
local run = pending
|
|
163
|
+
pending = nil
|
|
164
|
+
if run then
|
|
165
|
+
run()
|
|
166
|
+
end
|
|
167
|
+
end)
|
|
168
|
+
return function()
|
|
169
|
+
pending = nil -- release the entry without cancelling a game callback
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
--- A copy of the exchange ledger, for comparing across a hook that writes raw.
|
|
174
|
+
---
|
|
175
|
+
--- OnPlayerInit and Scribe.Dynamic factories are handed `profile.Data` DIRECTLY, before
|
|
176
|
+
--- the tree exists, so the accessor refusal that protects the reserved root has nothing
|
|
177
|
+
--- to refuse through. A starter kit that clears `_Scribe` to "start clean" would destroy
|
|
178
|
+
--- an in-flight exchange with no report at all.
|
|
179
|
+
local function exchangeSnapshot(data: any): any
|
|
180
|
+
local reserved = if type(data) == "table" then data[Template.RESERVED_ROOT] else nil
|
|
181
|
+
if type(reserved) ~= "table" then
|
|
182
|
+
return nil
|
|
183
|
+
end
|
|
184
|
+
return {
|
|
185
|
+
Exchange = if type(reserved.Exchange) == "table" then Util.deepClone(reserved.Exchange) else nil,
|
|
186
|
+
ExchangeInbox = if type(reserved.ExchangeInbox) == "table" then Util.deepClone(reserved.ExchangeInbox) else nil,
|
|
187
|
+
}
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
--- Deep equality over two exchange snapshots, tolerating either side being absent.
|
|
191
|
+
local function sameExchangeState(a: any, b: any): boolean
|
|
192
|
+
local function eq(x: any, y: any): boolean
|
|
193
|
+
if type(x) ~= type(y) then
|
|
194
|
+
return false
|
|
195
|
+
end
|
|
196
|
+
if type(x) ~= "table" then
|
|
197
|
+
return x == y
|
|
198
|
+
end
|
|
199
|
+
for key, value in x do
|
|
200
|
+
if not eq(value, y[key]) then
|
|
201
|
+
return false
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
for key in y do
|
|
205
|
+
if x[key] == nil then
|
|
206
|
+
return false
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
return true
|
|
210
|
+
end
|
|
211
|
+
return eq(a, b)
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
--- Does this stored profile hold exchange state that a wipe or a rollback would split?
|
|
215
|
+
--- Value in flight sits in the reserved root while the matching hole sits in game data,
|
|
216
|
+
--- so rolling one half back leaves the item in the inventory AND in the escrow, or takes
|
|
217
|
+
--- it out of the inventory with the inbox entry already cleared.
|
|
218
|
+
local function exchangeStateOf(data: any): (number, number)
|
|
219
|
+
local reserved = if type(data) == "table" then data[Template.RESERVED_ROOT] else nil
|
|
220
|
+
if type(reserved) ~= "table" then
|
|
221
|
+
return 0, 0
|
|
222
|
+
end
|
|
223
|
+
local staked, parked = 0, 0
|
|
224
|
+
if type(reserved.Exchange) == "table" then
|
|
225
|
+
for _ in reserved.Exchange do
|
|
226
|
+
staked += 1
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
if type(reserved.ExchangeInbox) == "table" then
|
|
230
|
+
for _ in reserved.ExchangeInbox do
|
|
231
|
+
parked += 1
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
return staked, parked
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
-- Hand the store module Scribe's error classifier: the fork has no `require` of its own,
|
|
238
|
+
-- and MessageAsync needs it to answer whether an attempt inside its retry loop could have
|
|
239
|
+
-- written (the value a gift-credit spend refunds on). A fork that finds no classifier
|
|
240
|
+
-- assumes every failure might have written.
|
|
241
|
+
local function installErrorClassifier(store: any): any
|
|
242
|
+
if type(store) == "table" then
|
|
243
|
+
store.ClassifyError = Failure.Classify
|
|
244
|
+
end
|
|
245
|
+
return store
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
local function findProfileStoreModule(injected: any): any
|
|
249
|
+
if type(injected) == "table" then
|
|
250
|
+
return installErrorClassifier(injected)
|
|
251
|
+
end
|
|
252
|
+
if typeof(injected) == "Instance" and (injected :: Instance):IsA("ModuleScript") then
|
|
253
|
+
return installErrorClassifier(require(injected) :: any)
|
|
254
|
+
end
|
|
255
|
+
-- Scribe ships its own store (src/Internal/Store, a vendored fork of ProfileStore
|
|
256
|
+
-- 1.0.3), so there is nothing to search for. Do not restore the old
|
|
257
|
+
-- ServerPackages/Packages hunt: it would bind to an unpatched build whose Save()
|
|
258
|
+
-- cannot report an outcome.
|
|
259
|
+
return installErrorClassifier(require(Internal.Store) :: any)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
function Persistence.new(ctx: any)
|
|
263
|
+
local self = {} :: any
|
|
264
|
+
local options = ctx.Options
|
|
265
|
+
|
|
266
|
+
-- Player-facing kick text, by cause. Two families, each with its own catch-all:
|
|
267
|
+
-- Fallback is the family option, so setting LoadFailureMessage or SessionEndMessage
|
|
268
|
+
-- still covers everything in that family, and naming one cause overrides only it.
|
|
269
|
+
local KICK_TEXT: { [string]: { Option: string, Fallback: string?, Default: string } } = {
|
|
270
|
+
-- Load family.
|
|
271
|
+
LegacyImport = {
|
|
272
|
+
Option = "LegacyImportFailureMessage",
|
|
273
|
+
Fallback = "LoadFailureMessage",
|
|
274
|
+
Default = "We couldn't check your existing data, and we didn't want to start you from scratch. Please rejoin!",
|
|
275
|
+
},
|
|
276
|
+
Migration = {
|
|
277
|
+
Option = "MigrationFailureMessage",
|
|
278
|
+
Fallback = "LoadFailureMessage",
|
|
279
|
+
Default = "We couldn't update your data to this version of the game. Please rejoin!",
|
|
280
|
+
},
|
|
281
|
+
VersionAhead = {
|
|
282
|
+
Option = "VersionAheadMessage",
|
|
283
|
+
Fallback = "LoadFailureMessage",
|
|
284
|
+
Default = "Your data is from a newer version of the game than this server. Please rejoin!",
|
|
285
|
+
},
|
|
286
|
+
Schema = {
|
|
287
|
+
Option = "SchemaFailureMessage",
|
|
288
|
+
Fallback = "LoadFailureMessage",
|
|
289
|
+
Default = "Your saved data doesn't match this version of the game. Please rejoin!",
|
|
290
|
+
},
|
|
291
|
+
RateLimited = {
|
|
292
|
+
Option = "RateLimitedMessage",
|
|
293
|
+
Fallback = "LoadFailureMessage",
|
|
294
|
+
Default = "Roblox's data service is busy right now. Please rejoin in a moment!",
|
|
295
|
+
},
|
|
296
|
+
Erasing = {
|
|
297
|
+
Option = "ErasingMessage",
|
|
298
|
+
Default = "Your data is being reset. Please rejoin in a few minutes!",
|
|
299
|
+
},
|
|
300
|
+
Load = {
|
|
301
|
+
Option = "LoadFailureMessage",
|
|
302
|
+
Default = "We couldn't load your data. Please rejoin!",
|
|
303
|
+
},
|
|
304
|
+
-- Session-end family.
|
|
305
|
+
SessionStolen = {
|
|
306
|
+
Option = "SessionStolenMessage",
|
|
307
|
+
Fallback = "SessionEndMessage",
|
|
308
|
+
Default = "Your data was opened on another server, so this session ended. Please rejoin!",
|
|
309
|
+
},
|
|
310
|
+
SessionInterrupted = {
|
|
311
|
+
Option = "SessionInterruptedMessage",
|
|
312
|
+
Fallback = "SessionEndMessage",
|
|
313
|
+
Default = "Your data session ended before it finished loading. Please rejoin!",
|
|
314
|
+
},
|
|
315
|
+
SessionEnd = {
|
|
316
|
+
Option = "SessionEndMessage",
|
|
317
|
+
Default = "Your data session has ended. Please rejoin!",
|
|
318
|
+
},
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
local function kickText(reason: string): string
|
|
322
|
+
local spec = KICK_TEXT[reason]
|
|
323
|
+
local specific = options[spec.Option]
|
|
324
|
+
if type(specific) == "string" then
|
|
325
|
+
return specific
|
|
326
|
+
end
|
|
327
|
+
if spec.Fallback ~= nil then
|
|
328
|
+
local family = options[spec.Fallback]
|
|
329
|
+
if type(family) == "string" then
|
|
330
|
+
return family
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
return spec.Default
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
-- Windowed: the class is stamped whenever the store errors on this key, and an error
|
|
337
|
+
-- from minutes ago says nothing about the attempt that just failed.
|
|
338
|
+
local THROTTLE_ATTRIBUTION_WINDOW = 15
|
|
339
|
+
local function loadKickText(entry: Entry): string
|
|
340
|
+
local at = entry.LastStoreErrorAt
|
|
341
|
+
if
|
|
342
|
+
entry.LastStoreErrorClass == "Throttled"
|
|
343
|
+
and at ~= nil
|
|
344
|
+
and (Clock.Monotonic() - at) <= THROTTLE_ATTRIBUTION_WINDOW
|
|
345
|
+
then
|
|
346
|
+
return kickText("RateLimited")
|
|
347
|
+
end
|
|
348
|
+
return kickText("Load")
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
-- Bound on ONE StartSessionAsync attempt. ProfileStore's own START_SESSION_TIMEOUT
|
|
352
|
+
-- applies only `if params.Cancel == nil` and Scribe always passes a Cancel, so
|
|
353
|
+
-- without this the load never returns during a DataStore outage and everything below
|
|
354
|
+
-- it (PROFILE_LOAD_FAIL, LoadFailurePolicy) is unreachable.
|
|
355
|
+
-- Floored at 60s: ProfileStore steals a held session after SESSION_STEAL (40s), so a
|
|
356
|
+
-- shorter deadline aborts every legitimate cross-server handoff.
|
|
357
|
+
local loadTimeout: number = math.max(60, tonumber(options.LoadTimeout) or 120)
|
|
358
|
+
local eraseJoinTimeout: number = math.max(0.1, tonumber(options.EraseJoinTimeout) or 10)
|
|
359
|
+
local compiled: Template.Compiled = ctx.Compiled
|
|
360
|
+
|
|
361
|
+
-- Storage location is explicit and never defaulted (see Scribe() guard).
|
|
362
|
+
local storeIndex = options.ProfileStoreIndex
|
|
363
|
+
if type(storeIndex) ~= "string" or storeIndex == "" then
|
|
364
|
+
error("Scribe: ProfileStoreIndex is required: name your game's DataStore explicitly", 0)
|
|
365
|
+
end
|
|
366
|
+
-- "" is allowed and means bare user-id keys (see the Scribe() guard); only an
|
|
367
|
+
-- absent or non-string prefix is an error.
|
|
368
|
+
if type(options.ProfileKeyPrefix) ~= "string" then
|
|
369
|
+
error(
|
|
370
|
+
'Scribe: ProfileKeyPrefix is required: set a per-player key prefix (e.g. "PLAYER_"), or "" for bare user-id keys',
|
|
371
|
+
0
|
|
372
|
+
)
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
local ProfileStore = findProfileStoreModule(options.ProfileStore)
|
|
376
|
+
|
|
377
|
+
-- Take shutdown off the store and drive it from Scribe's own drain. Roblox runs bound
|
|
378
|
+
-- BindToClose callbacks concurrently in an undocumented order, and the store's
|
|
379
|
+
-- releases every profile before its first yield, while Scribe's needs them still
|
|
380
|
+
-- active to run each player's OnPlayerLeaving hook.
|
|
381
|
+
if type(ProfileStore) == "table" and ProfileStore.ExternalShutdown ~= nil then
|
|
382
|
+
ProfileStore.ExternalShutdown = true
|
|
383
|
+
end
|
|
384
|
+
-- The PLAYER view throughout: a store root must never reach a profile payload, a
|
|
385
|
+
-- per-player session store, or the per-player snapshot.
|
|
386
|
+
local playerCompiled = compiled.Views.Player
|
|
387
|
+
local persistentDefaults = Template.BuildDefaults(playerCompiled, "Persistent")
|
|
388
|
+
local baseStore = ProfileStore.New(storeIndex, persistentDefaults)
|
|
389
|
+
-- Persistence mode: which store to read, whether to hold a session, whether anything
|
|
390
|
+
-- saves.
|
|
391
|
+
-- Live real store, exclusive session, saves (the default)
|
|
392
|
+
-- Mock ProfileStore's mock store, saves only there
|
|
393
|
+
-- NoSave real profile read as a snapshot, no session, never saves
|
|
394
|
+
-- Mode wins over the legacy flags when it is set; absent, they resolve to the
|
|
395
|
+
-- equivalent mode, so existing configs are untouched.
|
|
396
|
+
local VALID_MODES = { Live = true, Mock = true, NoSave = true }
|
|
397
|
+
local mode: string
|
|
398
|
+
local useMockStore: boolean
|
|
399
|
+
local targetUserId: number? = options.TargetUserId
|
|
400
|
+
|
|
401
|
+
if options.Mode ~= nil then
|
|
402
|
+
if type(options.Mode) ~= "string" or not VALID_MODES[options.Mode] then
|
|
403
|
+
error(`Scribe: Mode must be "Live", "Mock" or "NoSave", got {tostring(options.Mode)}`, 0)
|
|
404
|
+
end
|
|
405
|
+
mode = options.Mode
|
|
406
|
+
useMockStore = mode == "Mock"
|
|
407
|
+
local legacy: { string } = {}
|
|
408
|
+
for _, name in { "UseMock", "DontSave", "ViewedUserId", "OverriddenUserId" } do
|
|
409
|
+
if (options :: any)[name] ~= nil then
|
|
410
|
+
table.insert(legacy, name)
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
if #legacy > 0 then
|
|
414
|
+
Log.Warn(
|
|
415
|
+
"Persistence",
|
|
416
|
+
"MODE_OVERRIDES_LEGACY",
|
|
417
|
+
`Mode = "{mode}" is set, so the legacy flag(s) {table.concat(legacy, ", ")} are ignored`,
|
|
418
|
+
{ Mode = mode, Ignored = table.concat(legacy, ", ") }
|
|
419
|
+
)
|
|
420
|
+
end
|
|
421
|
+
else
|
|
422
|
+
-- Legacy resolution. The old flags controlled two independent axes: the store
|
|
423
|
+
-- came from UseMock/DontSave, the view branch from ViewedUserId. Keep them
|
|
424
|
+
-- independent, or an existing config silently changes which store it reads.
|
|
425
|
+
useMockStore = (options.UseMock or options.DontSave) == true
|
|
426
|
+
if options.ViewedUserId ~= nil then
|
|
427
|
+
mode = "NoSave"
|
|
428
|
+
targetUserId = options.ViewedUserId
|
|
429
|
+
elseif useMockStore then
|
|
430
|
+
mode = "Mock"
|
|
431
|
+
else
|
|
432
|
+
mode = "Live"
|
|
433
|
+
targetUserId = targetUserId or options.OverriddenUserId
|
|
434
|
+
end
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
self.Mode = mode
|
|
438
|
+
self.TargetUserId = targetUserId
|
|
439
|
+
|
|
440
|
+
-- The offline verbs write the real key, which a NoSave bundle promises never to touch.
|
|
441
|
+
local function refuseNoSave(verb: string): string
|
|
442
|
+
local reason = `Mode = "NoSave" never writes the store, so {verb} is refused`
|
|
443
|
+
Log.Warn("Persistence", "NOSAVE_WRITE_REFUSED", reason, { Verb = verb })
|
|
444
|
+
return reason
|
|
445
|
+
end
|
|
446
|
+
self.RefuseNoSave = refuseNoSave
|
|
447
|
+
|
|
448
|
+
local store = if useMockStore then baseStore.Mock else baseStore
|
|
449
|
+
|
|
450
|
+
-- Lower ProfileStore's autosave cadence so less progress is at risk on an unclean
|
|
451
|
+
-- exit. AUTO_SAVE_PERIOD is a ProfileStore GLOBAL, so this affects every Scribe
|
|
452
|
+
-- bundle and any direct ProfileStore use in the game; the last value wins. Floored
|
|
453
|
+
-- at 15s because Roblox throttles per-key writes to roughly one per 6s.
|
|
454
|
+
local function applyAutoSaveInterval(interval: number, source: string)
|
|
455
|
+
if interval < MIN_SAVE_INTERVAL then
|
|
456
|
+
Log.Warn(
|
|
457
|
+
"Persistence",
|
|
458
|
+
"SAVE_INTERVAL_CLAMPED",
|
|
459
|
+
`autosave interval below the {MIN_SAVE_INTERVAL}s floor (DataStore write throttling); clamping`,
|
|
460
|
+
{ Requested = interval, Applied = MIN_SAVE_INTERVAL }
|
|
461
|
+
)
|
|
462
|
+
interval = MIN_SAVE_INTERVAL
|
|
463
|
+
end
|
|
464
|
+
-- Two bundles asking for different cadences is a real conflict; unreported it
|
|
465
|
+
-- resolves as last-writer-wins, with the winner decided by require order.
|
|
466
|
+
if Config.AutoSaveInterval ~= nil and Config.AutoSaveInterval ~= interval then
|
|
467
|
+
Log.Warn(
|
|
468
|
+
"Persistence",
|
|
469
|
+
"SAVE_INTERVAL_CONFLICT",
|
|
470
|
+
"autosave interval is a ProfileStore process global and was already set to a different value; "
|
|
471
|
+
.. "the later value wins. Set it once with Scribe.Configure instead.",
|
|
472
|
+
{ Existing = Config.AutoSaveInterval, Requested = interval, Source = source }
|
|
473
|
+
)
|
|
474
|
+
end
|
|
475
|
+
if type(ProfileStore.SetConstant) == "function" then
|
|
476
|
+
local ok, err = pcall(ProfileStore.SetConstant, "AUTO_SAVE_PERIOD", interval)
|
|
477
|
+
if ok then
|
|
478
|
+
Config.AutoSaveInterval = interval
|
|
479
|
+
Config.AutoSaveSource = source
|
|
480
|
+
Log.Info("Persistence", "SAVE_INTERVAL_SET", "autosave interval configured", {
|
|
481
|
+
Seconds = interval,
|
|
482
|
+
Source = source,
|
|
483
|
+
})
|
|
484
|
+
else
|
|
485
|
+
Log.Warn("Persistence", "SAVE_INTERVAL_FAIL", tostring(err), {})
|
|
486
|
+
end
|
|
487
|
+
end
|
|
488
|
+
end
|
|
489
|
+
|
|
490
|
+
-- Scribe.Configure runs before any bundle exists, so apply its value here,
|
|
491
|
+
-- where ProfileStore has been resolved.
|
|
492
|
+
if Config.AutoSaveSource == "Configure" and Config.AutoSaveInterval ~= nil then
|
|
493
|
+
local pending = Config.AutoSaveInterval
|
|
494
|
+
Config.AutoSaveInterval = nil -- let applyAutoSaveInterval own the record
|
|
495
|
+
applyAutoSaveInterval(pending, "Configure")
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
if options.SaveInterval ~= nil then
|
|
499
|
+
local interval = options.SaveInterval
|
|
500
|
+
if type(interval) ~= "number" or interval ~= interval or interval <= 0 then
|
|
501
|
+
error("Scribe: SaveInterval must be a positive number of seconds", 0)
|
|
502
|
+
end
|
|
503
|
+
applyAutoSaveInterval(interval, "SaveInterval")
|
|
504
|
+
end
|
|
505
|
+
|
|
506
|
+
self.Store = store
|
|
507
|
+
-- Connections to PROCESS-GLOBAL signals: they outlive the bundle unless released, so
|
|
508
|
+
-- Stop() disconnects them. Per-profile signals die with the profile.
|
|
509
|
+
local connections: { any } = {}
|
|
510
|
+
self.Entries = {} :: { [Player]: Entry }
|
|
511
|
+
-- Maintained alongside Entries rather than recomputed: an O(N) dict walk on every
|
|
512
|
+
-- teardown makes draining a full server O(N^2) in session count.
|
|
513
|
+
self.EntryCount = 0
|
|
514
|
+
-- userId -> Entry, so a lookup by userId is not a linear scan over Entries.
|
|
515
|
+
self.EntriesByUserId = {} :: { [number]: Entry }
|
|
516
|
+
self.Closing = false
|
|
517
|
+
-- Set by Data.Stop, deliberately not the same flag as Closing: Closing also aborts an
|
|
518
|
+
-- in-flight load through the Cancel hook, marks a session end as ours rather than a
|
|
519
|
+
-- steal, and quiets teardown, none of which apply to a Stop. It exists for the
|
|
520
|
+
-- LoadFailurePolicy = "Wait" retry loop, which never gives up and whose only other
|
|
521
|
+
-- exit is Closing.
|
|
522
|
+
self.Stopped = false
|
|
523
|
+
-- Why the last session ended, for WaitForData callers arriving after a
|
|
524
|
+
-- synchronous teardown (weak keys: entries vanish with the Player).
|
|
525
|
+
self.EndReasons = setmetatable({}, { __mode = "k" }) :: any
|
|
526
|
+
|
|
527
|
+
local keyPrefix: string = options.ProfileKeyPrefix
|
|
528
|
+
local migrations: { [number]: (data: any) -> () } = options.Migrations or {}
|
|
529
|
+
local currentVersion = 1
|
|
530
|
+
for version in migrations do
|
|
531
|
+
if type(version) ~= "number" or version % 1 ~= 0 or version < 2 then
|
|
532
|
+
error("Scribe: Migrations keys must be integers >= 2", 0)
|
|
533
|
+
end
|
|
534
|
+
currentVersion = math.max(currentVersion, version)
|
|
535
|
+
end
|
|
536
|
+
for version = 2, currentVersion do
|
|
537
|
+
if type(migrations[version]) ~= "function" then
|
|
538
|
+
error(`Scribe: Migrations table is missing step {version}`, 0)
|
|
539
|
+
end
|
|
540
|
+
end
|
|
541
|
+
self.CurrentVersion = currentVersion
|
|
542
|
+
|
|
543
|
+
-- ProfileStore health signals (member names verified against v1.0.3).
|
|
544
|
+
local baseStoreName = storeIndex
|
|
545
|
+
local okErr = pcall(function()
|
|
546
|
+
connections[#connections + 1] = ProfileStore.OnError:Connect(
|
|
547
|
+
function(message: string, storeName: string, profileKey: string)
|
|
548
|
+
-- OnError is module-global and fires for every DataStore error, other
|
|
549
|
+
-- stores included. Only our store counts toward Health, and only an
|
|
550
|
+
-- in-flight SAVE marks a save failure: a throttled version read for an
|
|
551
|
+
-- online player must not fail their receipts and save state.
|
|
552
|
+
if storeName ~= baseStoreName then
|
|
553
|
+
return
|
|
554
|
+
end
|
|
555
|
+
-- Classified here rather than in Failure.Report: this is the one site
|
|
556
|
+
-- that publishes the verdict and the raw code in its own log context, so
|
|
557
|
+
-- an error family Roblox adds later reads as unclassified rather than
|
|
558
|
+
-- silently transient.
|
|
559
|
+
local class = Failure.Classify(message)
|
|
560
|
+
-- DataStoreErrors counts per ATTEMPT, so one failing save produces N
|
|
561
|
+
-- errors here. SavesFailed is per-operation and stays behind the
|
|
562
|
+
-- in-flight guard below. Health is fed every error, throttles included.
|
|
563
|
+
Failure.Report({
|
|
564
|
+
Subsystem = "ProfileStore",
|
|
565
|
+
-- Keyed, so the store's own retries of one failing profile collapse
|
|
566
|
+
-- into one problem. Unkeyed, a single oversize profile drove the whole
|
|
567
|
+
-- server to Outage, which refuses Robux receipts for everyone on it.
|
|
568
|
+
Key = profileKey,
|
|
569
|
+
Error = message,
|
|
570
|
+
Class = class,
|
|
571
|
+
Metric = "DataStoreErrors",
|
|
572
|
+
Category = "Persistence",
|
|
573
|
+
Code = "PROFILE_STORE_ERROR",
|
|
574
|
+
Message = tostring(message),
|
|
575
|
+
Context = {
|
|
576
|
+
Store = storeName,
|
|
577
|
+
Key = profileKey,
|
|
578
|
+
Class = class,
|
|
579
|
+
-- nil when the message carried no code: the signal that either
|
|
580
|
+
-- Roblox changed the shape or this is not a store error at all.
|
|
581
|
+
Code = Failure.CodeOf(message),
|
|
582
|
+
},
|
|
583
|
+
})
|
|
584
|
+
for _, entry in self.Entries :: { [Player]: Entry } do
|
|
585
|
+
if entry.Key == profileKey then
|
|
586
|
+
-- Outside the in-flight guard: a load has no save on the wire, and
|
|
587
|
+
-- the load path is what needs the distinction. Dies with the entry.
|
|
588
|
+
entry.LastStoreErrorClass = class
|
|
589
|
+
entry.LastStoreErrorAt = Clock.Monotonic()
|
|
590
|
+
end
|
|
591
|
+
if entry.Key == profileKey and entry.SavesInFlight > 0 then
|
|
592
|
+
-- Attribute this failure to a save genuinely on the wire. A
|
|
593
|
+
-- single-slot SaveT0 let an overlapping success clear it first, so
|
|
594
|
+
-- LastResult stayed "Ok" and the Dirty restore never ran, while
|
|
595
|
+
-- Monetization reads both to decide whether a duplicate receipt may
|
|
596
|
+
-- be acknowledged. OnSettled fires exactly once per save call.
|
|
597
|
+
entry.SaveInfo.LastResult = "Fail"
|
|
598
|
+
-- A profile over the size-warning threshold fails EVERY UpdateAsync
|
|
599
|
+
-- until it is trimmed, which is not a transient throttle.
|
|
600
|
+
if entry.LiveSize and entry.LiveSize > SIZE_WARN_BYTES then
|
|
601
|
+
Log.Error(
|
|
602
|
+
"Persistence",
|
|
603
|
+
"PROFILE_TOO_LARGE",
|
|
604
|
+
`save failed and the profile is {entry.LiveSize} bytes (~4 MB DataStore ceiling); this will not recover until the profile is trimmed`,
|
|
605
|
+
{ Player = entry.Player, Bytes = entry.LiveSize }
|
|
606
|
+
)
|
|
607
|
+
end
|
|
608
|
+
-- The failed save's payload never persisted: restore the dirtiness
|
|
609
|
+
-- captured at its start (plus anything written since).
|
|
610
|
+
entry.SaveInfo.Dirty = entry.DirtyAtSaveStart or entry.SaveInfo.Dirty
|
|
611
|
+
-- Deliberately NOT closing the Timed save window here. OnError cannot
|
|
612
|
+
-- prove the failing call was this entry's save, and closing it on an
|
|
613
|
+
-- unrelated error points the profile back at live DEADLINES that the
|
|
614
|
+
-- pending save then serializes as remaining seconds, so a rejoin
|
|
615
|
+
-- reads a cooldown decades out. The next BeforeSave re-derives the
|
|
616
|
+
-- persisted copy from the live table anyway.
|
|
617
|
+
ctx.Signals.OnSave:Fire({ Player = entry.Player, Ok = false, Duration = nil, At = Clock.Now() })
|
|
618
|
+
Metrics.Add("SavesFailed")
|
|
619
|
+
self.UpdateSaveInfoMirror(entry)
|
|
620
|
+
end
|
|
621
|
+
end
|
|
622
|
+
end
|
|
623
|
+
)
|
|
624
|
+
end)
|
|
625
|
+
if not okErr then
|
|
626
|
+
Log.Warn(
|
|
627
|
+
"Persistence",
|
|
628
|
+
"PROFILE_STORE_SIGNAL_MISSING",
|
|
629
|
+
"ProfileStore.OnError not connectable; save-failure observability disabled",
|
|
630
|
+
{}
|
|
631
|
+
)
|
|
632
|
+
end
|
|
633
|
+
-- ProfileStore replaces a stored value it cannot recognise as a profile with a fresh
|
|
634
|
+
-- template and fires OnOverwrite. Silent, irreversible data loss: the overwritten
|
|
635
|
+
-- profile comes back at SessionLoadCount 1, so isNew is true, migrations are skipped
|
|
636
|
+
-- and OnPlayerInit runs as if for a first-time player.
|
|
637
|
+
local okOverwrite = pcall(function()
|
|
638
|
+
connections[#connections + 1] = ProfileStore.OnOverwrite:Connect(function(storeName: string, profileKey: string)
|
|
639
|
+
-- OnOverwrite is module-global; only our store is ours to report on.
|
|
640
|
+
if storeName ~= baseStoreName then
|
|
641
|
+
return
|
|
642
|
+
end
|
|
643
|
+
-- The entry may not exist yet (the overwrite happens during the load that
|
|
644
|
+
-- creates it), so the player is best-effort and the key is the reliable id.
|
|
645
|
+
local entry: Entry? = nil
|
|
646
|
+
for _, candidate in self.Entries :: { [Player]: Entry } do
|
|
647
|
+
if candidate.Key == profileKey then
|
|
648
|
+
entry = candidate
|
|
649
|
+
break
|
|
650
|
+
end
|
|
651
|
+
end
|
|
652
|
+
Failure.Report({
|
|
653
|
+
Subsystem = "ProfileOverwrite",
|
|
654
|
+
Key = profileKey,
|
|
655
|
+
-- No Error: OnOverwrite carries no message to classify.
|
|
656
|
+
Metric = "ProfilesOverwritten",
|
|
657
|
+
Level = "Error",
|
|
658
|
+
Category = "Integrity",
|
|
659
|
+
Code = "PROFILE_OVERWRITTEN",
|
|
660
|
+
Message = "stored data for this key was not a valid profile and has been REPLACED with a fresh "
|
|
661
|
+
.. "template by ProfileStore; this session starts from defaults and the previous data is gone. "
|
|
662
|
+
.. "Restore from version history before the player saves over it",
|
|
663
|
+
Context = { Player = if entry then entry.Player else nil, Store = storeName, Key = profileKey },
|
|
664
|
+
})
|
|
665
|
+
ctx.Signals.OnAnomaly:Fire({
|
|
666
|
+
Player = if entry then entry.Player else nil,
|
|
667
|
+
Path = {},
|
|
668
|
+
Value = nil,
|
|
669
|
+
Reason = "ProfileOverwritten",
|
|
670
|
+
})
|
|
671
|
+
end)
|
|
672
|
+
end)
|
|
673
|
+
if not okOverwrite then
|
|
674
|
+
Log.Warn(
|
|
675
|
+
"Persistence",
|
|
676
|
+
"PROFILE_STORE_SIGNAL_MISSING",
|
|
677
|
+
"ProfileStore.OnOverwrite not connectable; silent corruption-overwrite detection disabled",
|
|
678
|
+
{}
|
|
679
|
+
)
|
|
680
|
+
end
|
|
681
|
+
local okCrit = pcall(function()
|
|
682
|
+
connections[#connections + 1] = ProfileStore.OnCriticalToggle:Connect(function(isCritical: boolean)
|
|
683
|
+
Health.SetCritical(isCritical)
|
|
684
|
+
if isCritical then
|
|
685
|
+
Log.Error("Persistence", "DATASTORE_CRITICAL", "ProfileStore entered critical state", {})
|
|
686
|
+
else
|
|
687
|
+
Log.Info("Persistence", "DATASTORE_RECOVERED", "ProfileStore left critical state", {})
|
|
688
|
+
end
|
|
689
|
+
end)
|
|
690
|
+
end)
|
|
691
|
+
if not okCrit then
|
|
692
|
+
Log.Warn("Persistence", "PROFILE_STORE_SIGNAL_MISSING", "ProfileStore.OnCriticalToggle not connectable", {})
|
|
693
|
+
end
|
|
694
|
+
|
|
695
|
+
function self.KeyFor(userId: number): string
|
|
696
|
+
return keyPrefix .. tostring(userId)
|
|
697
|
+
end
|
|
698
|
+
|
|
699
|
+
function self.EntryFor(player: Player): Entry?
|
|
700
|
+
return self.Entries[player]
|
|
701
|
+
end
|
|
702
|
+
|
|
703
|
+
function self.EntryForUserId(userId: number): Entry?
|
|
704
|
+
return self.EntriesByUserId[userId]
|
|
705
|
+
end
|
|
706
|
+
|
|
707
|
+
local function warnSlowLoad(entry: Entry)
|
|
708
|
+
if self.Stopped or entry.SlowLoadWarned or entry.State ~= "Loading" or entry.JoinedAt == nil then
|
|
709
|
+
return
|
|
710
|
+
end
|
|
711
|
+
local duration = Clock.Monotonic() - entry.JoinedAt
|
|
712
|
+
if duration < SLOW_LOAD_SECONDS then
|
|
713
|
+
return
|
|
714
|
+
end
|
|
715
|
+
entry.SlowLoadWarned = true
|
|
716
|
+
Log.Warn("Persistence", "SLOW_LOAD", "profile load exceeded 10s; inspect its phase and store errors", {
|
|
717
|
+
Player = entry.Player,
|
|
718
|
+
UserId = entry.UserId,
|
|
719
|
+
Phase = entry.LoadPhase or "Initializing",
|
|
720
|
+
Duration = duration,
|
|
721
|
+
})
|
|
722
|
+
end
|
|
723
|
+
|
|
724
|
+
local function setState(entry: Entry, state: "Loading" | "Ready" | "SessionEnded", reason: string?)
|
|
725
|
+
if state ~= "Loading" then
|
|
726
|
+
warnSlowLoad(entry)
|
|
727
|
+
if entry.StopLoadWatchdog then
|
|
728
|
+
entry.StopLoadWatchdog()
|
|
729
|
+
entry.StopLoadWatchdog = nil
|
|
730
|
+
end
|
|
731
|
+
entry.LoadPhase = nil
|
|
732
|
+
end
|
|
733
|
+
entry.State = state
|
|
734
|
+
entry.FailReason = reason
|
|
735
|
+
entry.StateChanged:Fire(state, reason)
|
|
736
|
+
end
|
|
737
|
+
|
|
738
|
+
--- Fires StateChanged so a WaitForData caller wakes on the transition.
|
|
739
|
+
local function setPhase(entry: Entry, phase: Types.LoadPhase?)
|
|
740
|
+
if entry.LoadPhase == phase then
|
|
741
|
+
return
|
|
742
|
+
end
|
|
743
|
+
entry.LoadPhase = phase
|
|
744
|
+
entry.StateChanged:Fire(entry.State, entry.FailReason)
|
|
745
|
+
end
|
|
746
|
+
|
|
747
|
+
-- AwaitBudget is cooperative (a hook that never calls it is unpaced), so the cap is
|
|
748
|
+
-- the only hard bound on what a migrating server spends.
|
|
749
|
+
local importsRunning = 0
|
|
750
|
+
|
|
751
|
+
--- Wait for a slot, then hand back a release. The caller must run it on every exit
|
|
752
|
+
--- or the cap leaks a slot per import.
|
|
753
|
+
local function takeImportSlot(entry: Entry): () -> ()
|
|
754
|
+
local cap = math.max(1, tonumber(options.MigrationConcurrency) or DEFAULT_MIGRATION_CONCURRENCY)
|
|
755
|
+
while importsRunning >= cap and not entry.TornDown do
|
|
756
|
+
setPhase(entry, "ImportThrottled")
|
|
757
|
+
Clock.Wait(IMPORT_SLOT_POLL)
|
|
758
|
+
end
|
|
759
|
+
importsRunning += 1
|
|
760
|
+
setPhase(entry, "Importing")
|
|
761
|
+
local released = false
|
|
762
|
+
return function()
|
|
763
|
+
if not released then
|
|
764
|
+
released = true
|
|
765
|
+
importsRunning -= 1
|
|
766
|
+
end
|
|
767
|
+
end
|
|
768
|
+
end
|
|
769
|
+
|
|
770
|
+
--- What ImportLegacyData and OnPlayerInit receive. The wait reports itself, so a
|
|
771
|
+
--- throttled migration is visible without the hook doing anything beyond calling it.
|
|
772
|
+
local function migrationContext(entry: Entry)
|
|
773
|
+
return {
|
|
774
|
+
AwaitBudget = function(requestType: string, count: number?, timeout: number?): (boolean, () -> ())
|
|
775
|
+
if type(requestType) ~= "string" then
|
|
776
|
+
error(
|
|
777
|
+
`Scribe: migration.AwaitBudget(requestType, count?, timeout?): requestType must be a `
|
|
778
|
+
.. `string, got {typeof(requestType)}`,
|
|
779
|
+
2
|
|
780
|
+
)
|
|
781
|
+
end
|
|
782
|
+
return Budget.Await(requestType, count, timeout, function(waiting)
|
|
783
|
+
setPhase(entry, if waiting then "ImportThrottled" else "Importing")
|
|
784
|
+
end, function()
|
|
785
|
+
return entry.TornDown
|
|
786
|
+
end)
|
|
787
|
+
end,
|
|
788
|
+
}
|
|
789
|
+
end
|
|
790
|
+
|
|
791
|
+
--- UpdateAsync tracks the session count because its worst case is the shutdown
|
|
792
|
+
--- drain: every online session saving at once. Process-wide, so a stopped bundle
|
|
793
|
+
--- leaves them alone.
|
|
794
|
+
local function syncBudgetReserves()
|
|
795
|
+
if self.Stopped then
|
|
796
|
+
return
|
|
797
|
+
end
|
|
798
|
+
Budget.SetReserve("UpdateAsync", self.EntryCount)
|
|
799
|
+
Budget.SetReserve("GetAsync", GET_RESERVE)
|
|
800
|
+
local boards = if ctx.Leaderboards ~= nil then ctx.Leaderboards.BoardCount or 0 else 0
|
|
801
|
+
Budget.SetReserve("GetSortedAsync", math.min(boards, SORTED_RESERVE_CAP))
|
|
802
|
+
end
|
|
803
|
+
|
|
804
|
+
function self.UpdateSaveInfoMirror(entry: Entry)
|
|
805
|
+
if entry.Tree and not entry.Tree.Dead then
|
|
806
|
+
local accessor = entry.Tree.Accessor[Template.SESSION_ROOT].SaveInfo
|
|
807
|
+
accessor.Set({
|
|
808
|
+
LastSaveAt = entry.SaveInfo.LastSaveAt,
|
|
809
|
+
LastResult = entry.SaveInfo.LastResult,
|
|
810
|
+
Dirty = entry.SaveInfo.Dirty,
|
|
811
|
+
Size = entry.LastSavedSize,
|
|
812
|
+
})
|
|
813
|
+
end
|
|
814
|
+
end
|
|
815
|
+
|
|
816
|
+
-- Deep-compare stored profile data against the current template defaults, skipping
|
|
817
|
+
-- Scribe.Dynamic leaves (their samples differ per server boot) and comparing packed
|
|
818
|
+
-- buffers by content. Detects a profile whose first session crashed before its
|
|
819
|
+
-- first save: still template-shaped, but SessionLoadCount already past 1.
|
|
820
|
+
local dynamicPathKeys: { [string]: boolean } = {}
|
|
821
|
+
for _, node in compiled.DynamicNodes do
|
|
822
|
+
dynamicPathKeys[Util.pathKey(node.Path :: any)] = true
|
|
823
|
+
end
|
|
824
|
+
local function templateEquals(a: any, b: any, pathKey: string?): boolean
|
|
825
|
+
if pathKey ~= nil and dynamicPathKeys[pathKey] then
|
|
826
|
+
return true
|
|
827
|
+
end
|
|
828
|
+
if type(a) == "buffer" and type(b) == "buffer" then
|
|
829
|
+
return buffer.tostring(a) == buffer.tostring(b)
|
|
830
|
+
end
|
|
831
|
+
if type(a) ~= type(b) then
|
|
832
|
+
return false
|
|
833
|
+
end
|
|
834
|
+
if type(a) ~= "table" then
|
|
835
|
+
return a == b
|
|
836
|
+
end
|
|
837
|
+
for k, v in a do
|
|
838
|
+
local childKey = if pathKey then pathKey .. "\31" .. tostring(k) else tostring(k)
|
|
839
|
+
if not templateEquals(v, (b :: { [any]: any })[k], childKey) then
|
|
840
|
+
return false
|
|
841
|
+
end
|
|
842
|
+
end
|
|
843
|
+
for k in b :: { [any]: any } do
|
|
844
|
+
if (a :: { [any]: any })[k] == nil then
|
|
845
|
+
return false
|
|
846
|
+
end
|
|
847
|
+
end
|
|
848
|
+
return true
|
|
849
|
+
end
|
|
850
|
+
local function equalsTemplateDefaults(data: { [string]: any }): boolean
|
|
851
|
+
return templateEquals(data, persistentDefaults, nil)
|
|
852
|
+
end
|
|
853
|
+
|
|
854
|
+
-- Commit a view-mode profile and report whether the write actually landed.
|
|
855
|
+
-- Profile:SetAsync reports nothing on failure: ProfileStore's own pcall swallows the
|
|
856
|
+
-- fault and the call returns normally having written nothing, so the offline receipt
|
|
857
|
+
-- path turns that into PurchaseGranted for Robux that were silently consumed.
|
|
858
|
+
-- RegisterError fires OnError synchronously before UpdateAsync returns, so a
|
|
859
|
+
-- connection held across the call is the only available failure signal.
|
|
860
|
+
local function commitViewProfile(profile: any, key: string): (boolean, string?)
|
|
861
|
+
local failure: string? = nil
|
|
862
|
+
local ok, conn = pcall(function()
|
|
863
|
+
return ProfileStore.OnError:Connect(function(message: string, _storeName: string, profileKey: string)
|
|
864
|
+
if profileKey == key and failure == nil then
|
|
865
|
+
failure = tostring(message)
|
|
866
|
+
end
|
|
867
|
+
end)
|
|
868
|
+
end)
|
|
869
|
+
local setOk, setErr = pcall(function()
|
|
870
|
+
profile:SetAsync()
|
|
871
|
+
end)
|
|
872
|
+
if ok and conn then
|
|
873
|
+
pcall(function()
|
|
874
|
+
conn:Disconnect()
|
|
875
|
+
end)
|
|
876
|
+
end
|
|
877
|
+
if not setOk then
|
|
878
|
+
return false, tostring(setErr) -- a real misuse error, not infrastructure
|
|
879
|
+
end
|
|
880
|
+
if failure ~= nil then
|
|
881
|
+
return false, `DataStore error during save: {failure}`
|
|
882
|
+
end
|
|
883
|
+
if not ok then
|
|
884
|
+
-- Could not observe OnError at all, so success here is unverified. Say so
|
|
885
|
+
-- rather than reporting a commit we cannot stand behind.
|
|
886
|
+
return false, "could not observe ProfileStore.OnError; save result unverifiable"
|
|
887
|
+
end
|
|
888
|
+
return true, nil
|
|
889
|
+
end
|
|
890
|
+
|
|
891
|
+
-- Asymmetric diff for the migration shadow in finishLoad. Reports a path differing on
|
|
892
|
+
-- both sides, or present only on the shadow side; keys only on the REAL side are
|
|
893
|
+
-- Reconcile's backfill and would fire for every legacy profile. RESERVED_ROOT is
|
|
894
|
+
-- skipped whole: runMigrations stamps its Version and the shadow loop never does.
|
|
895
|
+
local function divergences(shadow: any, real: any, path: { string | number }, out: { string })
|
|
896
|
+
if #out >= 5 then
|
|
897
|
+
return
|
|
898
|
+
end
|
|
899
|
+
if type(shadow) == "buffer" and type(real) == "buffer" then
|
|
900
|
+
if buffer.tostring(shadow) ~= buffer.tostring(real) then
|
|
901
|
+
table.insert(out, Util.pathString(path))
|
|
902
|
+
end
|
|
903
|
+
return
|
|
904
|
+
end
|
|
905
|
+
if type(shadow) ~= type(real) then
|
|
906
|
+
table.insert(out, Util.pathString(path))
|
|
907
|
+
return
|
|
908
|
+
end
|
|
909
|
+
if type(shadow) ~= "table" then
|
|
910
|
+
if shadow ~= real then
|
|
911
|
+
table.insert(out, Util.pathString(path))
|
|
912
|
+
end
|
|
913
|
+
return
|
|
914
|
+
end
|
|
915
|
+
for k, v in shadow do
|
|
916
|
+
if #path == 0 and k == Template.RESERVED_ROOT then
|
|
917
|
+
continue
|
|
918
|
+
end
|
|
919
|
+
table.insert(path, tostring(k))
|
|
920
|
+
divergences(v, (real :: any)[k], path, out)
|
|
921
|
+
table.remove(path)
|
|
922
|
+
if #out >= 5 then
|
|
923
|
+
return
|
|
924
|
+
end
|
|
925
|
+
end
|
|
926
|
+
end
|
|
927
|
+
|
|
928
|
+
-- Bounds on how far the profile may shrink before the wipe guard calls it a wipe.
|
|
929
|
+
-- Both ends fail quietly: at 0 any shrink trips the guard, and under WipeGuardPolicy
|
|
930
|
+
-- = "Block" it then keeps tripping, so live data never persists again. At 1 or above
|
|
931
|
+
-- it can never fire, and negative makes (1 - ratio) exceed 1, so a profile that GREW
|
|
932
|
+
-- trips it. The floor is not near-zero: a few percent of churn is ordinary gameplay.
|
|
933
|
+
local WIPE_GUARD_RATIO_MIN = 0.05
|
|
934
|
+
local WIPE_GUARD_RATIO_MAX = 0.95
|
|
935
|
+
-- Block policy writes the last-good BYTES back as the save payload, so it is the one
|
|
936
|
+
-- setting that needs the store to keep its full copy. Same variable here and at the
|
|
937
|
+
-- retention opt-out in finishLoad: if the two disagree, the guard persists an EMPTY
|
|
938
|
+
-- profile, which is the wipe it exists to prevent.
|
|
939
|
+
local wipeGuardBlocks: boolean = (options.WipeGuardPolicy or "Warn") == "Block"
|
|
940
|
+
local wipeGuardRatio = WIPE_GUARD_RATIO_DEFAULT
|
|
941
|
+
do
|
|
942
|
+
local configured = options.WipeGuardShrinkRatio
|
|
943
|
+
if configured ~= nil then
|
|
944
|
+
local asNumber = tonumber(configured)
|
|
945
|
+
local clamped = if asNumber == nil or asNumber ~= asNumber
|
|
946
|
+
then WIPE_GUARD_RATIO_DEFAULT
|
|
947
|
+
else math.clamp(asNumber, WIPE_GUARD_RATIO_MIN, WIPE_GUARD_RATIO_MAX)
|
|
948
|
+
if clamped ~= configured then
|
|
949
|
+
Log.Warn(
|
|
950
|
+
"Persistence",
|
|
951
|
+
"WIPE_GUARD_RATIO_CLAMPED",
|
|
952
|
+
`WipeGuardShrinkRatio must be between {WIPE_GUARD_RATIO_MIN} and {WIPE_GUARD_RATIO_MAX}: `
|
|
953
|
+
.. "at or below zero every shrink trips the guard (and under Block the profile never "
|
|
954
|
+
.. "saves again), at or above one it can never trip and the guard is off",
|
|
955
|
+
{ Requested = configured, Applied = clamped }
|
|
956
|
+
)
|
|
957
|
+
end
|
|
958
|
+
wipeGuardRatio = clamped
|
|
959
|
+
end
|
|
960
|
+
end
|
|
961
|
+
|
|
962
|
+
-- Wipe guard: compare against the last successful save. `liveSize` is passed in
|
|
963
|
+
-- because onProfileSaving measures the same table in the same synchronous frame.
|
|
964
|
+
local function wipeGuardCheck(entry: Entry, liveSize: number?): (boolean, string?)
|
|
965
|
+
local profile = entry.Profile
|
|
966
|
+
local live = entry.LiveData
|
|
967
|
+
if not profile or not live then
|
|
968
|
+
return false, nil
|
|
969
|
+
end
|
|
970
|
+
-- The baseline is a KEY SET and a size, not the bytes, so the store need not hold
|
|
971
|
+
-- a second full copy of the profile. The store's copy is still the baseline in
|
|
972
|
+
-- the two cases the summary cannot be: before the first save of a session, and
|
|
973
|
+
-- under WipeGuardPolicy = "Block", which writes those bytes back as the payload.
|
|
974
|
+
local keys: any = entry.LastSavedKeys
|
|
975
|
+
local oldSize = entry.LastSavedSize
|
|
976
|
+
if keys == nil then
|
|
977
|
+
local lastSaved = profile.LastSavedData
|
|
978
|
+
if type(lastSaved) ~= "table" or next(lastSaved) == nil then
|
|
979
|
+
return false, nil
|
|
980
|
+
end
|
|
981
|
+
keys = lastSaved
|
|
982
|
+
oldSize = oldSize or Util.approxSize(lastSaved)
|
|
983
|
+
end
|
|
984
|
+
local missing = {}
|
|
985
|
+
for key in keys do
|
|
986
|
+
if live[key] == nil then
|
|
987
|
+
table.insert(missing, tostring(key))
|
|
988
|
+
end
|
|
989
|
+
end
|
|
990
|
+
if #missing > 0 then
|
|
991
|
+
return true, `top-level keys vanished: {table.concat(missing, ", ")}`
|
|
992
|
+
end
|
|
993
|
+
local ratio = wipeGuardRatio
|
|
994
|
+
local newSize = liveSize or Util.approxSize(live)
|
|
995
|
+
if oldSize and newSize and oldSize > 1024 and newSize < oldSize * (1 - ratio) then
|
|
996
|
+
return true, `serialized size collapsed {oldSize} -> {newSize} bytes`
|
|
997
|
+
end
|
|
998
|
+
return false, nil
|
|
999
|
+
end
|
|
1000
|
+
|
|
1001
|
+
-- Wrapped rather than edited in place: the body returns early on the ForceSaveOnce
|
|
1002
|
+
-- and rebaseline paths, so a capture at the bottom of it would miss them. Reading
|
|
1003
|
+
-- entry.Blocked here runs synchronously inside the save, before it yields, so the
|
|
1004
|
+
-- flag still describes this payload rather than a later save's.
|
|
1005
|
+
local onProfileSavingInner: (entry: Entry) -> ()
|
|
1006
|
+
|
|
1007
|
+
local function onProfileSaving(entry: Entry, saveContext: { [string]: any }?)
|
|
1008
|
+
onProfileSavingInner(entry)
|
|
1009
|
+
if saveContext then
|
|
1010
|
+
saveContext.Blocked = entry.Blocked == true
|
|
1011
|
+
end
|
|
1012
|
+
end
|
|
1013
|
+
|
|
1014
|
+
function onProfileSavingInner(entry: Entry)
|
|
1015
|
+
-- The EARLIEST live start, not this save's. The slot is shared by every save in
|
|
1016
|
+
-- flight, so assigning would let a later save overwrite an earlier one's t0 and
|
|
1017
|
+
-- under-report its duration. Cleared on the drain to zero in OnSettled.
|
|
1018
|
+
entry.SaveT0 = entry.SaveT0 or Clock.Monotonic()
|
|
1019
|
+
entry.SavesInFlight += 1
|
|
1020
|
+
-- This save's payload is serialized NOW: reset Dirty so it re-accumulates from
|
|
1021
|
+
-- writes landing after the snapshot, and keep the pre-snapshot state so a failed
|
|
1022
|
+
-- save can restore it (OnError path). ACCUMULATED, not assigned: with N concurrent
|
|
1023
|
+
-- saves a second one would read back the false the first wrote, and the first
|
|
1024
|
+
-- one's failure would restore clean over an unstored write. Cleared on the drain.
|
|
1025
|
+
entry.DirtyAtSaveStart = entry.DirtyAtSaveStart or entry.SaveInfo.Dirty
|
|
1026
|
+
entry.SaveInfo.Dirty = false
|
|
1027
|
+
local profile = entry.Profile
|
|
1028
|
+
if not profile or not entry.LiveData then
|
|
1029
|
+
return
|
|
1030
|
+
end
|
|
1031
|
+
|
|
1032
|
+
-- Measure the payload on the way IN, not only on save success: a profile past
|
|
1033
|
+
-- the ~4 MB DataStore ceiling makes every UpdateAsync throw, so the
|
|
1034
|
+
-- success-path check below would never run for it and the developer would see
|
|
1035
|
+
-- only generic retryable-looking failures. Latched to once per crossing.
|
|
1036
|
+
local liveSize = Util.approxSize(entry.LiveData)
|
|
1037
|
+
if liveSize then
|
|
1038
|
+
entry.LiveSize = liveSize
|
|
1039
|
+
if liveSize > SIZE_WARN_BYTES and not entry.SizeWarned then
|
|
1040
|
+
entry.SizeWarned = true
|
|
1041
|
+
Log.Warn(
|
|
1042
|
+
"Persistence",
|
|
1043
|
+
"PROFILE_SIZE",
|
|
1044
|
+
`profile approaching the ~4 MB DataStore ceiling ({liveSize} bytes); trim it or saves will start failing`,
|
|
1045
|
+
{ Player = entry.Player, Bytes = liveSize }
|
|
1046
|
+
)
|
|
1047
|
+
ctx.Signals.OnAnomaly:Fire({ Player = entry.Player, Path = {}, Value = nil, Reason = "ProfileSize" })
|
|
1048
|
+
elseif liveSize <= SIZE_WARN_BYTES then
|
|
1049
|
+
entry.SizeWarned = false
|
|
1050
|
+
end
|
|
1051
|
+
end
|
|
1052
|
+
|
|
1053
|
+
if entry.ForceSaveOnce then
|
|
1054
|
+
entry.ForceSaveOnce = false
|
|
1055
|
+
if entry.Blocked then
|
|
1056
|
+
entry.Blocked = false
|
|
1057
|
+
profile:SetSaveSource(entry.LiveData)
|
|
1058
|
+
Log.Warn("Integrity", "WIPE_GUARD_FORCED", "blocked save forced through", { Player = entry.Player })
|
|
1059
|
+
end
|
|
1060
|
+
return
|
|
1061
|
+
end
|
|
1062
|
+
|
|
1063
|
+
if entry.WipeGuardRebaselinePending then
|
|
1064
|
+
-- First save after a reshaping migration: the LastSavedData baseline predates
|
|
1065
|
+
-- it, so a comparison would false-trip on any key the migration removed.
|
|
1066
|
+
-- Cleared on save SUCCESS (onProfileSaved), not here: this hook fires per
|
|
1067
|
+
-- ATTEMPT, and clearing on a failed one re-arms the guard against the stale
|
|
1068
|
+
-- baseline, which under Block policy stalls the migration permanently.
|
|
1069
|
+
return
|
|
1070
|
+
end
|
|
1071
|
+
|
|
1072
|
+
local tripped, reason = wipeGuardCheck(entry, liveSize)
|
|
1073
|
+
if tripped then
|
|
1074
|
+
Log.Error("Integrity", "WIPE_GUARD_TRIPPED", reason :: string, { Player = entry.Player })
|
|
1075
|
+
Metrics.Add("WipeGuardTrips")
|
|
1076
|
+
ctx.Signals.OnAnomaly:Fire({
|
|
1077
|
+
Player = entry.Player,
|
|
1078
|
+
Path = {},
|
|
1079
|
+
Value = nil,
|
|
1080
|
+
Reason = "WipeGuard: " .. (reason :: string),
|
|
1081
|
+
})
|
|
1082
|
+
if wipeGuardBlocks then
|
|
1083
|
+
-- Persist the last good snapshot instead; the live tree keeps
|
|
1084
|
+
-- its own root table. Unblocked when the guard passes again or
|
|
1085
|
+
-- via Data.Flush(player, { Force = true }).
|
|
1086
|
+
if not entry.Blocked then
|
|
1087
|
+
entry.Blocked = true
|
|
1088
|
+
end
|
|
1089
|
+
profile:SetSaveSource(Util.deepClone(profile.LastSavedData))
|
|
1090
|
+
Log.Warn("Integrity", "WIPE_GUARD_BLOCKED", "save held; persisting last good snapshot", {
|
|
1091
|
+
Player = entry.Player,
|
|
1092
|
+
})
|
|
1093
|
+
end
|
|
1094
|
+
elseif entry.Blocked then
|
|
1095
|
+
entry.Blocked = false
|
|
1096
|
+
profile:SetSaveSource(entry.LiveData)
|
|
1097
|
+
Log.Info("Integrity", "WIPE_GUARD_CLEARED", "wipe guard cleared; live data persists again", {
|
|
1098
|
+
Player = entry.Player,
|
|
1099
|
+
})
|
|
1100
|
+
end
|
|
1101
|
+
end
|
|
1102
|
+
|
|
1103
|
+
local function onProfileSaved(entry: Entry, lastSavedData: any)
|
|
1104
|
+
-- ProfileStore fires OnAfterSave even when the session was STOLEN and nothing
|
|
1105
|
+
-- local persisted. Skip the success bookkeeping only for steals: a
|
|
1106
|
+
-- locally-initiated final save (clean leave, shutdown drain, migration
|
|
1107
|
+
-- fail-closed) genuinely persisted, and dropping it would skew Health.
|
|
1108
|
+
if entry.TornDown and entry.SessionStolen then
|
|
1109
|
+
return
|
|
1110
|
+
end
|
|
1111
|
+
local duration = if entry.SaveT0 then Clock.Monotonic() - entry.SaveT0 else nil
|
|
1112
|
+
entry.SaveInfo.LastSaveAt = Clock.Now()
|
|
1113
|
+
entry.SaveInfo.LastResult = "Ok"
|
|
1114
|
+
-- Keep any dirtiness accumulated from writes landing AFTER this save's payload
|
|
1115
|
+
-- snapshot (onProfileSaving reset the flag at that moment); blocked saves
|
|
1116
|
+
-- persisted old data, so live data stays unsaved regardless.
|
|
1117
|
+
entry.SaveInfo.Dirty = entry.SaveInfo.Dirty or entry.Blocked
|
|
1118
|
+
-- A successful, unblocked save re-established the wipe-guard baseline with the
|
|
1119
|
+
-- migrated shape; the one-time post-migration skip is complete.
|
|
1120
|
+
if not entry.Blocked then
|
|
1121
|
+
entry.WipeGuardRebaselinePending = false
|
|
1122
|
+
end
|
|
1123
|
+
Metrics.Add("SavesOk")
|
|
1124
|
+
if duration then
|
|
1125
|
+
Metrics.Observe("SaveDuration", duration)
|
|
1126
|
+
end
|
|
1127
|
+
Health.ReportSuccess("Saves")
|
|
1128
|
+
ctx.Signals.OnSave:Fire({ Player = entry.Player, Ok = true, Duration = duration, At = Clock.Now() })
|
|
1129
|
+
|
|
1130
|
+
-- Record the size that actually persisted (the warning + anomaly are owned by
|
|
1131
|
+
-- the pre-save check in onProfileSaving, which also fires when a save fails).
|
|
1132
|
+
local size = Util.approxSize(lastSavedData)
|
|
1133
|
+
if size then
|
|
1134
|
+
entry.LastSavedSize = size
|
|
1135
|
+
entry.LiveSize = size
|
|
1136
|
+
Metrics.Observe("ProfileSize", size)
|
|
1137
|
+
end
|
|
1138
|
+
-- The other half of the wipe guard's baseline, from the payload that actually
|
|
1139
|
+
-- persisted rather than from live data: a blocked save stores the last-good
|
|
1140
|
+
-- snapshot, and the baseline has to describe what is on the key.
|
|
1141
|
+
if type(lastSavedData) == "table" then
|
|
1142
|
+
local keys: { [string]: true } = {}
|
|
1143
|
+
for key in lastSavedData do
|
|
1144
|
+
keys[key :: string] = true
|
|
1145
|
+
end
|
|
1146
|
+
entry.LastSavedKeys = keys
|
|
1147
|
+
end
|
|
1148
|
+
|
|
1149
|
+
self.UpdateSaveInfoMirror(entry)
|
|
1150
|
+
end
|
|
1151
|
+
|
|
1152
|
+
-- Does this stored session belong to a server that is still alive?
|
|
1153
|
+
-- `profile.Session` is a straight read of MetaData.ActiveSession with no staleness
|
|
1154
|
+
-- allowance, and a server that dies before its final save leaves it set forever. The
|
|
1155
|
+
-- store steals a session whose LastUpdate is older than ASSUME_DEAD, so mirroring
|
|
1156
|
+
-- that threshold keeps RestoreVersion and Erase from staying refused permanently for
|
|
1157
|
+
-- the profile that most needs them. No usable LastUpdate reads as LIVE, the safe
|
|
1158
|
+
-- direction.
|
|
1159
|
+
local function sessionLooksLive(profile: any): boolean
|
|
1160
|
+
if not profile or not profile.Session then
|
|
1161
|
+
return false
|
|
1162
|
+
end
|
|
1163
|
+
local lastUpdate = profile.LastUpdate
|
|
1164
|
+
if type(lastUpdate) ~= "number" or lastUpdate <= 0 then
|
|
1165
|
+
return true
|
|
1166
|
+
end
|
|
1167
|
+
return (Clock.Now() - lastUpdate) <= ASSUME_DEAD_SECONDS
|
|
1168
|
+
end
|
|
1169
|
+
|
|
1170
|
+
-- Release the lock WITHOUT writing Data back. Every fail-closed load path promises the
|
|
1171
|
+
-- stored bytes are left as found, and EndSession is an unconditional save: by the time
|
|
1172
|
+
-- those paths run, profile.Data has been through CoerceMapKeys, Reconcile, the version
|
|
1173
|
+
-- stamp and the migration chain. Falls back to a plain EndSession for an injected
|
|
1174
|
+
-- store without the primitive: unrejoinable is worse than rewritten.
|
|
1175
|
+
local function releaseWithoutWriting(profile: { EndSessionWithoutSaving: ((any) -> ())?, EndSession: (any) -> () })
|
|
1176
|
+
if type(profile.EndSessionWithoutSaving) == "function" then
|
|
1177
|
+
profile:EndSessionWithoutSaving()
|
|
1178
|
+
else
|
|
1179
|
+
profile:EndSession()
|
|
1180
|
+
end
|
|
1181
|
+
end
|
|
1182
|
+
|
|
1183
|
+
-- Deep structural comparison for the reserved-root discard check in runMigrations.
|
|
1184
|
+
-- Nothing existing fits: Util.leafEquals is false for tables, Node's entryEquals is
|
|
1185
|
+
-- schema-driven, and templateEquals above skips nodes by root-relative path key, so it
|
|
1186
|
+
-- would skip a reserved child sharing a name with a Dynamic root field. Buffers
|
|
1187
|
+
-- compare by identity under `==`, hence the byte compare.
|
|
1188
|
+
local function deepSame(a: any, b: any): boolean
|
|
1189
|
+
if type(a) == "buffer" and type(b) == "buffer" then
|
|
1190
|
+
return Util.bufferEquals(a :: buffer, b :: buffer)
|
|
1191
|
+
end
|
|
1192
|
+
if type(a) ~= type(b) then
|
|
1193
|
+
return false
|
|
1194
|
+
end
|
|
1195
|
+
if type(a) ~= "table" then
|
|
1196
|
+
return a == b
|
|
1197
|
+
end
|
|
1198
|
+
for k, v in a :: { [any]: any } do
|
|
1199
|
+
if not deepSame(v, (b :: { [any]: any })[k]) then
|
|
1200
|
+
return false
|
|
1201
|
+
end
|
|
1202
|
+
end
|
|
1203
|
+
for k in b :: { [any]: any } do
|
|
1204
|
+
if (a :: { [any]: any })[k] == nil then
|
|
1205
|
+
return false
|
|
1206
|
+
end
|
|
1207
|
+
end
|
|
1208
|
+
return true
|
|
1209
|
+
end
|
|
1210
|
+
|
|
1211
|
+
-- Do these two reserved roots agree? `compareVersion` decides whether the TOP-LEVEL
|
|
1212
|
+
-- `Version` key counts, and the two call sites want opposite answers:
|
|
1213
|
+
-- * MIGRATION passes `false`: the library re-stamps Version right after the chain.
|
|
1214
|
+
-- * RESTORE passes `true`: there is no re-stamp, so Version alone decides whether
|
|
1215
|
+
-- the chain re-runs against the older-shaped data the restore just wrote.
|
|
1216
|
+
-- A `Version` key one level down always counts, in both modes.
|
|
1217
|
+
local function reservedUnchanged(stored: { [string]: any }, other: any, compareVersion: boolean): boolean
|
|
1218
|
+
if type(other) ~= "table" then
|
|
1219
|
+
return false
|
|
1220
|
+
end
|
|
1221
|
+
for k, v in stored do
|
|
1222
|
+
if (compareVersion or k ~= "Version") and not deepSame(v, (other :: { [string]: any })[k]) then
|
|
1223
|
+
return false
|
|
1224
|
+
end
|
|
1225
|
+
end
|
|
1226
|
+
for k in other :: { [string]: any } do
|
|
1227
|
+
if (compareVersion or k ~= "Version") and stored[k] == nil then
|
|
1228
|
+
return false
|
|
1229
|
+
end
|
|
1230
|
+
end
|
|
1231
|
+
return true
|
|
1232
|
+
end
|
|
1233
|
+
|
|
1234
|
+
local function runMigrationsInner(entry: Entry, data: { [string]: any }): (boolean, string?)
|
|
1235
|
+
local reserved = data[Template.RESERVED_ROOT]
|
|
1236
|
+
local stored = if type(reserved) == "table" and type(reserved.Version) == "number" then reserved.Version else 1
|
|
1237
|
+
if stored > currentVersion then
|
|
1238
|
+
-- Migrated by a newer deploy this server's code has never seen. Fail closed
|
|
1239
|
+
-- by default (VersionAheadPolicy "Kick") so an old server can't corrupt
|
|
1240
|
+
-- newer-shaped data; "Allow" opts into running old code against it
|
|
1241
|
+
-- (documented staged-deploy hazard).
|
|
1242
|
+
local policy = options.VersionAheadPolicy or "Kick"
|
|
1243
|
+
if policy == "Kick" then
|
|
1244
|
+
Log.Error(
|
|
1245
|
+
"Persistence",
|
|
1246
|
+
"PROFILE_VERSION_AHEAD",
|
|
1247
|
+
"stored version is newer than this server's; failing closed",
|
|
1248
|
+
{
|
|
1249
|
+
Player = entry.Player,
|
|
1250
|
+
Stored = stored,
|
|
1251
|
+
Current = currentVersion,
|
|
1252
|
+
}
|
|
1253
|
+
)
|
|
1254
|
+
-- Not a migration failure: nothing ran, this server is just older than the data.
|
|
1255
|
+
return false, "version-ahead"
|
|
1256
|
+
end
|
|
1257
|
+
Log.Warn(
|
|
1258
|
+
"Persistence",
|
|
1259
|
+
"PROFILE_VERSION_AHEAD",
|
|
1260
|
+
"stored version is newer than this server's (VersionAheadPolicy=Allow)",
|
|
1261
|
+
{
|
|
1262
|
+
Player = entry.Player,
|
|
1263
|
+
Stored = stored,
|
|
1264
|
+
Current = currentVersion,
|
|
1265
|
+
}
|
|
1266
|
+
)
|
|
1267
|
+
return true
|
|
1268
|
+
end
|
|
1269
|
+
if stored == currentVersion then
|
|
1270
|
+
return true
|
|
1271
|
+
end
|
|
1272
|
+
-- Run migrations on a working clone; only commit on full success so a
|
|
1273
|
+
-- half-migrated profile can never persist.
|
|
1274
|
+
local work = Util.deepClone(data)
|
|
1275
|
+
for version = stored + 1, currentVersion do
|
|
1276
|
+
local ok, err = xpcall(function()
|
|
1277
|
+
migrations[version](work)
|
|
1278
|
+
end, function(e)
|
|
1279
|
+
return debug.traceback(tostring(e), 2)
|
|
1280
|
+
end)
|
|
1281
|
+
if not ok then
|
|
1282
|
+
Log.Error("Persistence", "MIGRATION_FAIL", `migration step {version} failed: {err}`, {
|
|
1283
|
+
Player = entry.Player,
|
|
1284
|
+
Step = version,
|
|
1285
|
+
})
|
|
1286
|
+
Metrics.Add("MigrationsFailed")
|
|
1287
|
+
return false
|
|
1288
|
+
end
|
|
1289
|
+
end
|
|
1290
|
+
-- PRESERVE THE RESERVED ROOT. Reconcile has already run, so nothing the library
|
|
1291
|
+
-- has gained since this profile was written is missing from the stored root.
|
|
1292
|
+
-- `_Scribe` is library-owned state (receipt dedupe
|
|
1293
|
+
-- ring, gift escrow, credits, perks, purchase logs, Timed deadlines) and the
|
|
1294
|
+
-- migration chain is entirely game-supplied, so a step that rebuilds the profile
|
|
1295
|
+
-- from its own key whitelist destroys receipt idempotency and paid-for gifts.
|
|
1296
|
+
-- Assigned BY REFERENCE: table.clear(data) below does not recurse, so this table
|
|
1297
|
+
-- survives the copy loop as the same object, and the migration was handed `work`,
|
|
1298
|
+
-- a deep clone, so nothing aliases. Ordered BEFORE the unpersistable scan so the
|
|
1299
|
+
-- scan sees exactly what commits.
|
|
1300
|
+
local storedReserved = data[Template.RESERVED_ROOT]
|
|
1301
|
+
local migratedReserved = work[Template.RESERVED_ROOT]
|
|
1302
|
+
if type(storedReserved) == "table" then
|
|
1303
|
+
work[Template.RESERVED_ROOT] = storedReserved
|
|
1304
|
+
-- Reverting a developer's edit in silence is its own trap, so a discard is
|
|
1305
|
+
-- reported at Error, which also fires OnIssue. `false`: the stamp below
|
|
1306
|
+
-- rewrites `Version` unconditionally.
|
|
1307
|
+
if not reservedUnchanged(storedReserved, migratedReserved, false) then
|
|
1308
|
+
Log.Error(
|
|
1309
|
+
"Persistence",
|
|
1310
|
+
"MIGRATION_RESERVED_DISCARDED",
|
|
1311
|
+
`a migration step changed the reserved "{Template.RESERVED_ROOT}" root; the change was DISCARDED `
|
|
1312
|
+
.. `and the stored reserved state kept (it is library-owned: receipt dedupe, gift escrow, `
|
|
1313
|
+
.. `perks, purchase logs, Timed) -- migrate your own keys only`,
|
|
1314
|
+
{ Player = entry.Player, Path = Template.RESERVED_ROOT, Stored = stored, Current = currentVersion }
|
|
1315
|
+
)
|
|
1316
|
+
end
|
|
1317
|
+
elseif type(migratedReserved) ~= "table" then
|
|
1318
|
+
-- Nothing stored to preserve (a legacy or imported profile that never had a
|
|
1319
|
+
-- reserved root) and the chain left none either, so make somewhere for the
|
|
1320
|
+
-- stamp below to go. A root a migration CREATED is kept as it built it:
|
|
1321
|
+
-- nothing was destroyed, so nothing is reported.
|
|
1322
|
+
work[Template.RESERVED_ROOT] = {}
|
|
1323
|
+
end
|
|
1324
|
+
|
|
1325
|
+
-- Migration functions mutate `work` raw, bypassing Node.validateWrite, so a step
|
|
1326
|
+
-- introducing unpersistable data (invalid-UTF-8 string/key, or a
|
|
1327
|
+
-- userdata/Instance/function value) would fail every save. Scan BEFORE committing
|
|
1328
|
+
-- into `data`, so failure takes the fail-closed path below and the unmodified
|
|
1329
|
+
-- original persists.
|
|
1330
|
+
local badMigrated = Util.findUnpersistable(work)
|
|
1331
|
+
if badMigrated ~= nil then
|
|
1332
|
+
Log.Error(
|
|
1333
|
+
"Persistence",
|
|
1334
|
+
"MIGRATION_FAIL",
|
|
1335
|
+
`migration produced unpersistable data ({badMigrated.Detail}) at "{Util.pathString(badMigrated.Path)}"`,
|
|
1336
|
+
{ Player = entry.Player, Path = Util.pathString(badMigrated.Path) }
|
|
1337
|
+
)
|
|
1338
|
+
Metrics.Add("MigrationsFailed")
|
|
1339
|
+
return false
|
|
1340
|
+
end
|
|
1341
|
+
|
|
1342
|
+
-- The stamp goes HERE, below every `return false` above. On the preserve path
|
|
1343
|
+
-- `work[RESERVED_ROOT]` IS `data[RESERVED_ROOT]`, so this writes straight into the
|
|
1344
|
+
-- live profile.Data. Stamped earlier, a rejected migration leaves the caller's
|
|
1345
|
+
-- all-default carve-out comparing against a profile already marked migrated, and
|
|
1346
|
+
-- it kicks the player that carve-out exists to load.
|
|
1347
|
+
work[Template.RESERVED_ROOT].Version = currentVersion
|
|
1348
|
+
table.clear(data)
|
|
1349
|
+
for k, v in work do
|
|
1350
|
+
data[k] = v
|
|
1351
|
+
end
|
|
1352
|
+
Log.Info("Persistence", "MIGRATED", `profile migrated {stored} -> {currentVersion}`, { Player = entry.Player })
|
|
1353
|
+
-- A migration may legitimately remove or rename top-level keys, but the wipe
|
|
1354
|
+
-- guard's baseline still holds the pre-migration shape until the first save
|
|
1355
|
+
-- commits. Skip the guard for that one save, or under WipeGuardPolicy = "Block"
|
|
1356
|
+
-- it re-persists the old data and stalls the migration permanently.
|
|
1357
|
+
entry.WipeGuardRebaselinePending = true
|
|
1358
|
+
return true
|
|
1359
|
+
end
|
|
1360
|
+
|
|
1361
|
+
--- Not spanned, deliberately: a migration is developer code that may yield, and an
|
|
1362
|
+
--- annotation does not survive a yield. LoadDuration already covers what it costs.
|
|
1363
|
+
local function runMigrations(entry: Entry, data: { [string]: any }): (boolean, string?)
|
|
1364
|
+
return runMigrationsInner(entry, data)
|
|
1365
|
+
end
|
|
1366
|
+
|
|
1367
|
+
local function buildTree(entry: Entry)
|
|
1368
|
+
local profile = entry.Profile
|
|
1369
|
+
entry.LiveData = profile.Data
|
|
1370
|
+
entry.SessionStore = Template.BuildDefaults(playerCompiled, "Session")
|
|
1371
|
+
|
|
1372
|
+
entry.Tree = Node.NewTree({
|
|
1373
|
+
Compiled = playerCompiled,
|
|
1374
|
+
Stores = { Persist = profile.Data, Session = entry.SessionStore },
|
|
1375
|
+
IsServer = true,
|
|
1376
|
+
Label = entry.Player.Name,
|
|
1377
|
+
OnOp = function(op: Node.Op, schema: Template.SchemaNode?)
|
|
1378
|
+
local persists = if schema then schema.Persist else Template.PersistOf(compiled, op.Path)
|
|
1379
|
+
if persists and not entry.SaveInfo.Dirty then
|
|
1380
|
+
entry.SaveInfo.Dirty = true
|
|
1381
|
+
Sched.Defer(function()
|
|
1382
|
+
if entry.Tree and not entry.Tree.Dead and entry.SaveInfo.Dirty then
|
|
1383
|
+
self.UpdateSaveInfoMirror(entry)
|
|
1384
|
+
end
|
|
1385
|
+
end)
|
|
1386
|
+
end
|
|
1387
|
+
ctx.Replication.PushOp(entry, op, schema)
|
|
1388
|
+
end,
|
|
1389
|
+
OnAnomaly = function(path, value, reason)
|
|
1390
|
+
Metrics.Add("Anomalies")
|
|
1391
|
+
Log.Warn("Integrity", "ANOMALY", `implausible write at "{Util.pathString(path)}": {reason}`, {
|
|
1392
|
+
Player = entry.Player,
|
|
1393
|
+
Path = Util.pathString(path),
|
|
1394
|
+
Reason = reason,
|
|
1395
|
+
})
|
|
1396
|
+
ctx.Signals.OnAnomaly:Fire({ Player = entry.Player, Path = path, Value = value, Reason = reason })
|
|
1397
|
+
end,
|
|
1398
|
+
BoundsPolicy = options.BoundsPolicy or "Clamp",
|
|
1399
|
+
DevMode = ctx.DevMode,
|
|
1400
|
+
TimedHooks = ctx.Timed.HooksFor(entry),
|
|
1401
|
+
OnEconomy = function(path, delta, newValue, tags)
|
|
1402
|
+
ctx.Analytics.Economy(entry.Player, path, delta, newValue, tags)
|
|
1403
|
+
end,
|
|
1404
|
+
LogError = function(message)
|
|
1405
|
+
Log.Error("Lifecycle", "LISTENER_ERROR", message, { Player = entry.Player })
|
|
1406
|
+
end,
|
|
1407
|
+
LogWarn = function(message)
|
|
1408
|
+
Log.Warn("Lifecycle", "DEV_WARNING", message, { Player = entry.Player })
|
|
1409
|
+
end,
|
|
1410
|
+
OnBatchEnd = function()
|
|
1411
|
+
ctx.Replication.FlushEntry(entry)
|
|
1412
|
+
end,
|
|
1413
|
+
OnDerivedIssue = function(code, field, detail)
|
|
1414
|
+
-- MISMATCH is a dev-mode purity report, not a live fault; the other
|
|
1415
|
+
-- two mean a field is serving a value its inputs no longer justify.
|
|
1416
|
+
local report: typeof(Log.Warn) = if code == "DERIVED_MISMATCH" then Log.Warn else Log.Error
|
|
1417
|
+
report("Derived", code :: any, `"{field}": {detail}`, { Player = entry.Player, Field = field })
|
|
1418
|
+
end,
|
|
1419
|
+
})
|
|
1420
|
+
end
|
|
1421
|
+
|
|
1422
|
+
-- Run the game's OnPlayerLeaving hook. Runs BEFORE EndSession, which performs the
|
|
1423
|
+
-- final save, so anything written here lands. Errors are contained: a throwing
|
|
1424
|
+
-- hook must not skip the save it was meant to contribute to.
|
|
1425
|
+
local function runLeavingHook(entry: Entry, reason: string)
|
|
1426
|
+
local hook = ctx.Options.OnPlayerLeaving
|
|
1427
|
+
-- The hook runs under a plain pcall, so one that yields parks the whole leave
|
|
1428
|
+
-- with the entry still in self.Entries and the shutdown drain then walks it
|
|
1429
|
+
-- again. A double fire double-counts the hook's writes into one final save.
|
|
1430
|
+
if hook == nil or entry.LeavingHookRan then
|
|
1431
|
+
return
|
|
1432
|
+
end
|
|
1433
|
+
entry.LeavingHookRan = true
|
|
1434
|
+
if entry.State ~= "Ready" or entry.Tree == nil then
|
|
1435
|
+
return -- never loaded, so there is nothing coherent to hand it
|
|
1436
|
+
end
|
|
1437
|
+
local started = Clock.Monotonic()
|
|
1438
|
+
local warned = false
|
|
1439
|
+
local function warnSlowHook()
|
|
1440
|
+
local duration = Clock.Monotonic() - started
|
|
1441
|
+
if self.Stopped or warned or duration < SLOW_LEAVING_HOOK_SECONDS then
|
|
1442
|
+
return
|
|
1443
|
+
end
|
|
1444
|
+
warned = true
|
|
1445
|
+
Log.Warn("Lifecycle", "PLAYER_LEAVING_HOOK_SLOW", "OnPlayerLeaving exceeded 10s before the final save", {
|
|
1446
|
+
Player = entry.Player,
|
|
1447
|
+
UserId = entry.UserId,
|
|
1448
|
+
Reason = reason,
|
|
1449
|
+
Duration = duration,
|
|
1450
|
+
})
|
|
1451
|
+
end
|
|
1452
|
+
local stopWatchdog = watchdog(SLOW_LEAVING_HOOK_SECONDS, warnSlowHook)
|
|
1453
|
+
entry.StopLeavingHookWatchdog = stopWatchdog
|
|
1454
|
+
local ok, err = pcall(hook, entry.Player, entry.Tree.Accessor, reason)
|
|
1455
|
+
stopWatchdog()
|
|
1456
|
+
entry.StopLeavingHookWatchdog = nil
|
|
1457
|
+
Metrics.Observe("PlayerLeavingHookDuration", Clock.Monotonic() - started)
|
|
1458
|
+
warnSlowHook()
|
|
1459
|
+
if not ok then
|
|
1460
|
+
Log.Error(
|
|
1461
|
+
"Lifecycle",
|
|
1462
|
+
"LISTENER_ERROR",
|
|
1463
|
+
`OnPlayerLeaving errored; the save continues without its writes: {tostring(err)}`,
|
|
1464
|
+
{ Player = entry.Player }
|
|
1465
|
+
)
|
|
1466
|
+
end
|
|
1467
|
+
end
|
|
1468
|
+
|
|
1469
|
+
local function teardown(entry: Entry, reason: string?)
|
|
1470
|
+
if entry.TornDown then
|
|
1471
|
+
return
|
|
1472
|
+
end
|
|
1473
|
+
entry.TornDown = true
|
|
1474
|
+
if entry.StopLeavingHookWatchdog then
|
|
1475
|
+
entry.StopLeavingHookWatchdog()
|
|
1476
|
+
entry.StopLeavingHookWatchdog = nil
|
|
1477
|
+
end
|
|
1478
|
+
-- One canonical reason reaches every consumer: the SessionEnded signal, the stored
|
|
1479
|
+
-- end reason, and WaitForData's second return (see Types.LifecycleReason).
|
|
1480
|
+
local canonical: string = reason or "session-ended"
|
|
1481
|
+
-- Before setState overwrites it: a mid-load kick never had a session to lose.
|
|
1482
|
+
local wasLoading = entry.State == "Loading"
|
|
1483
|
+
self.EndReasons[entry.Player] = canonical
|
|
1484
|
+
|
|
1485
|
+
-- A Data.OnMessage handler that never returns holds its FireTracked open forever,
|
|
1486
|
+
-- so no ack is emitted and the message is re-offered on every load, climbing
|
|
1487
|
+
-- toward MESSAGE_QUEUE_FULL. Reported at session end rather than on a timer: no
|
|
1488
|
+
-- threshold is right, since a handler spending a throttled round trip is slow
|
|
1489
|
+
-- rather than stuck. Warn, not Error: a player leaving while a handler is
|
|
1490
|
+
-- legitimately mid-yield reads identically, and only repetition tells them apart.
|
|
1491
|
+
local unsettled = entry.UnsettledMessages or 0
|
|
1492
|
+
if unsettled > 0 then
|
|
1493
|
+
Log.Warn(
|
|
1494
|
+
"Persistence",
|
|
1495
|
+
"MESSAGE_HANDLER_STALLED",
|
|
1496
|
+
"a Data.OnMessage handler had not returned when the session ended; the message was NOT "
|
|
1497
|
+
.. "acknowledged and will be redelivered on the next load",
|
|
1498
|
+
{ Player = entry.Player, Outstanding = unsettled }
|
|
1499
|
+
)
|
|
1500
|
+
end
|
|
1501
|
+
|
|
1502
|
+
-- SessionEnded fires before the (default-on) kick.
|
|
1503
|
+
ctx.Signals.SessionEnded:Fire(entry.Player, canonical)
|
|
1504
|
+
setState(entry, "SessionEnded", canonical)
|
|
1505
|
+
|
|
1506
|
+
ctx.Replication.OnEntryRemoved(entry)
|
|
1507
|
+
ctx.Leaderboards.OnUnload(entry)
|
|
1508
|
+
ctx.Monetization.OnUnload(entry)
|
|
1509
|
+
ctx.Timed.OnUnload(entry)
|
|
1510
|
+
ctx.Exchanges.OnUnload(entry)
|
|
1511
|
+
|
|
1512
|
+
if entry.Tree then
|
|
1513
|
+
entry.Tree:Destroy()
|
|
1514
|
+
end
|
|
1515
|
+
|
|
1516
|
+
if
|
|
1517
|
+
options.KickOnSessionEnd ~= false
|
|
1518
|
+
and not self.Closing
|
|
1519
|
+
-- ...and not because this bundle was released. A Stop is not a session ending
|
|
1520
|
+
-- badly, so kicking would be a lie told by a bundle that no longer exists.
|
|
1521
|
+
and not self.Stopped
|
|
1522
|
+
and canonical ~= "player-left"
|
|
1523
|
+
and entry.Player.Parent == Players
|
|
1524
|
+
then
|
|
1525
|
+
-- Nearly always another server holding the profile now.
|
|
1526
|
+
local why = if entry.SessionStolen
|
|
1527
|
+
then "SessionStolen"
|
|
1528
|
+
elseif wasLoading then "SessionInterrupted"
|
|
1529
|
+
else "SessionEnd"
|
|
1530
|
+
entry.Player:Kick(kickText(why))
|
|
1531
|
+
end
|
|
1532
|
+
|
|
1533
|
+
-- Only adjust the maintained counters if this entry was still the live one
|
|
1534
|
+
-- for its player: teardown can run more than once for a session, and a
|
|
1535
|
+
-- rejoin can already have installed a NEWER entry under the same Player.
|
|
1536
|
+
if self.Entries[entry.Player] == entry then
|
|
1537
|
+
self.Entries[entry.Player] = nil
|
|
1538
|
+
self.EntryCount -= 1
|
|
1539
|
+
syncBudgetReserves()
|
|
1540
|
+
end
|
|
1541
|
+
if self.EntriesByUserId[entry.UserId] == entry then
|
|
1542
|
+
self.EntriesByUserId[entry.UserId] = nil
|
|
1543
|
+
end
|
|
1544
|
+
Metrics.Set("ActiveSessions", self.EntryCount)
|
|
1545
|
+
end
|
|
1546
|
+
|
|
1547
|
+
local function finishLoad(entry: Entry)
|
|
1548
|
+
local profile = entry.Profile :: any
|
|
1549
|
+
local player = entry.Player
|
|
1550
|
+
|
|
1551
|
+
-- Marked for erasure: release without saving.
|
|
1552
|
+
local markedReserved = if type(profile.Data) == "table" then profile.Data[Template.RESERVED_ROOT] else nil
|
|
1553
|
+
if type(markedReserved) == "table" and type(markedReserved.Erase) == "table" then
|
|
1554
|
+
entry.LocalSessionEnd = true
|
|
1555
|
+
releaseWithoutWriting(profile)
|
|
1556
|
+
if player.Parent == Players then
|
|
1557
|
+
player:Kick(kickText("Erasing"))
|
|
1558
|
+
end
|
|
1559
|
+
teardown(entry, "erasing")
|
|
1560
|
+
return
|
|
1561
|
+
end
|
|
1562
|
+
|
|
1563
|
+
-- Stop the store holding a second full copy of this profile all session. The wipe
|
|
1564
|
+
-- guard's top-level key set and its size are recorded on the entry by each
|
|
1565
|
+
-- successful save. Not under Block policy, which persists those bytes back. This
|
|
1566
|
+
-- nils profile.LastSavedData, not the OnAfterSave argument: the store still hands
|
|
1567
|
+
-- the real table to that handler, which is where the summary is taken.
|
|
1568
|
+
if not wipeGuardBlocks and type(profile.SetRetainLastSaved) == "function" then
|
|
1569
|
+
profile:SetRetainLastSaved(false)
|
|
1570
|
+
end
|
|
1571
|
+
|
|
1572
|
+
-- A brand-new profile's very first session ever. In-place ProfileStore adoption
|
|
1573
|
+
-- keeps the session count from the old game, so returning players still migrate.
|
|
1574
|
+
-- SessionsStarted rather than the raw SessionLoadCount: a view is fetched with
|
|
1575
|
+
-- GetAsync, which never runs the session-start increment, so read raw a dry run of
|
|
1576
|
+
-- an already-loaded profile is judged brand new and skips every migration.
|
|
1577
|
+
local isNew = profile:SessionsStarted() == 1
|
|
1578
|
+
|
|
1579
|
+
-- Snapshot BEFORE AddUserId mutates the in-memory list. The template-fresh check
|
|
1580
|
+
-- further down needs it: without it a returning player who never earned anything
|
|
1581
|
+
-- is indistinguishable from a crashed first session, and their migrations are
|
|
1582
|
+
-- skipped. Negative ids are excluded because AddUserId no-ops for them outside
|
|
1583
|
+
-- mock, so the signal can never accumulate.
|
|
1584
|
+
local hadPriorSave = entry.UserId >= 0 and profile:HasCompletedSave()
|
|
1585
|
+
|
|
1586
|
+
-- Decided once, so a refused reset loads like an ordinary join.
|
|
1587
|
+
local resetStaked, resetParked = 0, 0
|
|
1588
|
+
local willReset = false
|
|
1589
|
+
if options.ResetData then
|
|
1590
|
+
resetStaked, resetParked = exchangeStateOf(profile.Data)
|
|
1591
|
+
willReset = resetStaked == 0 and resetParked == 0
|
|
1592
|
+
end
|
|
1593
|
+
|
|
1594
|
+
-- Legacy import ---------------------------------------------------------
|
|
1595
|
+
-- Adopt a player's data from another library, ONCE, before Scribe has written for
|
|
1596
|
+
-- them. Gated on hadPriorSave rather than `isNew`: SessionLoadCount advances when
|
|
1597
|
+
-- a session STARTS, so a first join whose save never committed leaves the player
|
|
1598
|
+
-- non-new with nothing stored, and `isNew` would refuse the import while the empty
|
|
1599
|
+
-- profile Scribe then saved became canonical. Skipped under NoSave, where the dry
|
|
1600
|
+
-- run cannot persist the result anyway.
|
|
1601
|
+
local imported = false
|
|
1602
|
+
if
|
|
1603
|
+
options.ImportLegacyData ~= nil
|
|
1604
|
+
and not hadPriorSave
|
|
1605
|
+
and not options.ResetData
|
|
1606
|
+
and mode ~= "NoSave"
|
|
1607
|
+
and not entry.TornDown
|
|
1608
|
+
then
|
|
1609
|
+
-- YIELDS: the hook reads another library's key and everything below assumes a
|
|
1610
|
+
-- live session, so the liveness re-check moves up here.
|
|
1611
|
+
local importStarted = Clock.Monotonic()
|
|
1612
|
+
local releaseSlot = takeImportSlot(entry)
|
|
1613
|
+
if entry.TornDown then
|
|
1614
|
+
-- Waiting for a slot is a window the outer guard could not see, and teardown
|
|
1615
|
+
-- has already run.
|
|
1616
|
+
releaseSlot()
|
|
1617
|
+
setPhase(entry, nil)
|
|
1618
|
+
return
|
|
1619
|
+
end
|
|
1620
|
+
local ok, legacy = xpcall(options.ImportLegacyData, function(e)
|
|
1621
|
+
return debug.traceback(tostring(e), 2)
|
|
1622
|
+
end, player, entry.UserId, migrationContext(entry))
|
|
1623
|
+
releaseSlot()
|
|
1624
|
+
setPhase(entry, "Initializing")
|
|
1625
|
+
local importSeconds = Clock.Monotonic() - importStarted
|
|
1626
|
+
if importSeconds >= SLOW_IMPORT_SECONDS then
|
|
1627
|
+
-- Without this, "waiting on budget" and "hung" look identical in the log.
|
|
1628
|
+
Log.Warn(
|
|
1629
|
+
"Persistence",
|
|
1630
|
+
"SLOW_IMPORT",
|
|
1631
|
+
`ImportLegacyData took {string.format("%.1f", importSeconds)}s, past the {SLOW_IMPORT_SECONDS}s mark`,
|
|
1632
|
+
{ Player = player }
|
|
1633
|
+
)
|
|
1634
|
+
end
|
|
1635
|
+
|
|
1636
|
+
if not ok then
|
|
1637
|
+
-- Fail closed, the same shape as a failed migration: a hook that threw
|
|
1638
|
+
-- cannot say whether legacy data exists, and loading the player on an
|
|
1639
|
+
-- empty profile makes emptiness canonical at their first save, after which
|
|
1640
|
+
-- hadPriorSave is true forever. The release below writes template defaults,
|
|
1641
|
+
-- which must not count as a prior save; it does not, because AddUserId runs
|
|
1642
|
+
-- further down, so the UserIds list it persists is still empty. Hoisting
|
|
1643
|
+
-- AddUserId above the import would make a retryable outage permanent.
|
|
1644
|
+
Metrics.Add("LegacyImportFailures")
|
|
1645
|
+
Log.Error(
|
|
1646
|
+
"Persistence",
|
|
1647
|
+
"LEGACY_IMPORT_FAIL",
|
|
1648
|
+
`ImportLegacyData threw, so it is unknown whether this player has data to carry over. `
|
|
1649
|
+
.. `Refusing the load rather than starting them empty: {legacy}`,
|
|
1650
|
+
{ Player = player, UserId = entry.UserId }
|
|
1651
|
+
)
|
|
1652
|
+
entry.LocalSessionEnd = true
|
|
1653
|
+
releaseWithoutWriting(profile)
|
|
1654
|
+
if player.Parent == Players then
|
|
1655
|
+
player:Kick(kickText("LegacyImport"))
|
|
1656
|
+
end
|
|
1657
|
+
teardown(entry, "load-failed")
|
|
1658
|
+
return
|
|
1659
|
+
end
|
|
1660
|
+
|
|
1661
|
+
-- The hook yielded. A player who left in that window would leave Reconcile, the
|
|
1662
|
+
-- migration chain, the schema check and OnPlayerInit running against a session
|
|
1663
|
+
-- already gone. The guard further down catches later yields.
|
|
1664
|
+
if not profile:IsActive() then
|
|
1665
|
+
teardown(entry, if player.Parent == Players then "session-ended" else "player-left")
|
|
1666
|
+
return
|
|
1667
|
+
end
|
|
1668
|
+
|
|
1669
|
+
if type(legacy) == "table" then
|
|
1670
|
+
-- A returned table means "this player is an import", even an empty one; nil
|
|
1671
|
+
-- means "nothing to carry over". A hook must THROW rather than return nil
|
|
1672
|
+
-- when it could not find out: the two are indistinguishable here.
|
|
1673
|
+
local adopted = 0
|
|
1674
|
+
for key, value in legacy do
|
|
1675
|
+
-- Scribe's own roots are never adopted; the prefix test catches both
|
|
1676
|
+
-- ("_ScribeSession" starts with "_Scribe"), which is intended. An
|
|
1677
|
+
-- import writing into _Scribe would hand the old library control of
|
|
1678
|
+
-- purchase-dedupe ids, granted perks and cooldowns.
|
|
1679
|
+
if
|
|
1680
|
+
type(key) == "string"
|
|
1681
|
+
and string.sub(key, 1, #Template.RESERVED_ROOT) == Template.RESERVED_ROOT
|
|
1682
|
+
then
|
|
1683
|
+
continue
|
|
1684
|
+
end
|
|
1685
|
+
profile.Data[key] = value
|
|
1686
|
+
adopted += 1
|
|
1687
|
+
end
|
|
1688
|
+
imported = true
|
|
1689
|
+
Metrics.Add("LegacyImports")
|
|
1690
|
+
Log.Info(
|
|
1691
|
+
"Persistence",
|
|
1692
|
+
"LEGACY_IMPORTED",
|
|
1693
|
+
`adopted {adopted} top-level key(s) from ImportLegacyData; the profile will run the full `
|
|
1694
|
+
.. "migration chain from version 1",
|
|
1695
|
+
{ Player = player, UserId = entry.UserId, Keys = adopted }
|
|
1696
|
+
)
|
|
1697
|
+
end
|
|
1698
|
+
end
|
|
1699
|
+
|
|
1700
|
+
-- Decide which Scribe.Dynamic fields to seed BEFORE Reconcile backfills their
|
|
1701
|
+
-- baseline: all of them on a brand-new profile, plus any the STORED data lacks.
|
|
1702
|
+
-- Reading the stored data first means a saved value is present, and so is never
|
|
1703
|
+
-- flagged or overwritten.
|
|
1704
|
+
local dynamicToSeed: { any } = {}
|
|
1705
|
+
if #compiled.DynamicNodes > 0 then
|
|
1706
|
+
for _, node in compiled.DynamicNodes do
|
|
1707
|
+
local present = true
|
|
1708
|
+
local at: any = profile.Data
|
|
1709
|
+
for _, seg in node.Path do
|
|
1710
|
+
if type(at) ~= "table" or at[seg] == nil then
|
|
1711
|
+
present = false
|
|
1712
|
+
break
|
|
1713
|
+
end
|
|
1714
|
+
at = at[seg]
|
|
1715
|
+
end
|
|
1716
|
+
-- ResetData wipes to persistentDefaults, which carry each Dynamic field's
|
|
1717
|
+
-- frozen declaration-time sample, so a reset must re-seed exactly like a
|
|
1718
|
+
-- new profile or the player keeps the shared sample.
|
|
1719
|
+
-- `isNew and not imported`: an imported profile is new to Scribe but not
|
|
1720
|
+
-- to the player, so a Dynamic value the import carried is left alone.
|
|
1721
|
+
if (isNew and not imported) or willReset or not present then
|
|
1722
|
+
table.insert(dynamicToSeed, node)
|
|
1723
|
+
end
|
|
1724
|
+
end
|
|
1725
|
+
end
|
|
1726
|
+
|
|
1727
|
+
profile:AddUserId(entry.UserId)
|
|
1728
|
+
|
|
1729
|
+
-- Keep the RAW pre-Reconcile bytes so the shadow run below can show what the
|
|
1730
|
+
-- migrations would have seen without the backfill. Opt-in and NOT gated on Studio
|
|
1731
|
+
-- or DevMode: the shadow re-EXECUTES the migration bodies against a clone, so any
|
|
1732
|
+
-- side effect in them fires a second time. Gated on the stored version here too,
|
|
1733
|
+
-- or a mature game pays a full profile deep clone on every join for a shadow run
|
|
1734
|
+
-- that then declines to happen; this read is conservative (no marker reads as nil
|
|
1735
|
+
-- and still clones) and the post-Reconcile read below stays authoritative.
|
|
1736
|
+
local preReconcileReserved = profile.Data[Template.RESERVED_ROOT]
|
|
1737
|
+
local preReconcileVersion = if type(preReconcileReserved) == "table"
|
|
1738
|
+
and type(preReconcileReserved.Version) == "number"
|
|
1739
|
+
then preReconcileReserved.Version
|
|
1740
|
+
else nil
|
|
1741
|
+
-- `or imported`: adopted data enters at version 1 and runs the whole chain against
|
|
1742
|
+
-- real stored values, which is the run an audit most wants to shadow.
|
|
1743
|
+
local rawStored: { [string]: any }? = if options.MigrationShadow == true
|
|
1744
|
+
and (not isNew or imported)
|
|
1745
|
+
and not willReset
|
|
1746
|
+
and (preReconcileVersion == nil or preReconcileVersion < currentVersion)
|
|
1747
|
+
then Util.deepClone(profile.Data)
|
|
1748
|
+
else nil
|
|
1749
|
+
|
|
1750
|
+
-- Before Reconcile: integer map keys arrive from the DataStore as JSON
|
|
1751
|
+
-- strings, and every read after this point expects the declared key type.
|
|
1752
|
+
Template.CoerceMapKeys(compiled, profile.Data)
|
|
1753
|
+
|
|
1754
|
+
profile:Reconcile()
|
|
1755
|
+
|
|
1756
|
+
-- A hard crash inside the very first session persists template-shaped data with
|
|
1757
|
+
-- the default Version while SessionLoadCount advanced past 1. Detect that shape
|
|
1758
|
+
-- and treat the profile as fresh, or every migration runs against new-player
|
|
1759
|
+
-- defaults. Gated to migrating profiles, so the deep compare is rare. Hoisted to
|
|
1760
|
+
-- function scope: the fail-closed carve-out below needs it, and re-reading it
|
|
1761
|
+
-- there would yield currentVersion after the stamp.
|
|
1762
|
+
local storedReserved = profile.Data[Template.RESERVED_ROOT]
|
|
1763
|
+
local storedVersion = if type(storedReserved) == "table" and type(storedReserved.Version) == "number"
|
|
1764
|
+
then storedReserved.Version
|
|
1765
|
+
else 1
|
|
1766
|
+
|
|
1767
|
+
local freshAsTemplate = false
|
|
1768
|
+
if not isNew and not options.ResetData then
|
|
1769
|
+
if storedVersion < currentVersion then
|
|
1770
|
+
-- Only claim "template-fresh" when no completed save is on record. A
|
|
1771
|
+
-- returning player who never earned anything has identical bytes, and
|
|
1772
|
+
-- treating them as new skipped every migration for them.
|
|
1773
|
+
freshAsTemplate = not hadPriorSave and equalsTemplateDefaults(profile.Data)
|
|
1774
|
+
if freshAsTemplate then
|
|
1775
|
+
Log.Info(
|
|
1776
|
+
"Persistence",
|
|
1777
|
+
"PROFILE_LOADED",
|
|
1778
|
+
"stored data is template-fresh and no completed save is recorded; treating as new",
|
|
1779
|
+
{ Player = player }
|
|
1780
|
+
)
|
|
1781
|
+
-- Effectively a brand-new player: re-seed every Dynamic field too
|
|
1782
|
+
-- (the stored values are a dead boot's declaration-time samples).
|
|
1783
|
+
table.clear(dynamicToSeed)
|
|
1784
|
+
for _, node in compiled.DynamicNodes do
|
|
1785
|
+
table.insert(dynamicToSeed, node)
|
|
1786
|
+
end
|
|
1787
|
+
end
|
|
1788
|
+
end
|
|
1789
|
+
end
|
|
1790
|
+
|
|
1791
|
+
if options.ResetData and not willReset then
|
|
1792
|
+
-- The wipe would take the reserved root with it, and with it the only record
|
|
1793
|
+
-- that the staked value ever existed. Refused rather than reported: there is
|
|
1794
|
+
-- nothing to report to afterwards.
|
|
1795
|
+
Log.Error(
|
|
1796
|
+
"Persistence",
|
|
1797
|
+
"EXCHANGE_RESET_REFUSED",
|
|
1798
|
+
`ResetData refused: profile holds {resetStaked} in-flight and {resetParked} undelivered exchange(s); `
|
|
1799
|
+
.. `resolve or discard them before resetting`,
|
|
1800
|
+
{ Player = player, Staked = resetStaked, Parked = resetParked }
|
|
1801
|
+
)
|
|
1802
|
+
elseif willReset then
|
|
1803
|
+
local fresh = Util.deepClone(persistentDefaults)
|
|
1804
|
+
table.clear(profile.Data)
|
|
1805
|
+
for k, v in fresh do
|
|
1806
|
+
profile.Data[k] = v
|
|
1807
|
+
end
|
|
1808
|
+
Log.Warn("Persistence", "PROFILE_RESET", "profile reset via ResetData option", { Player = player })
|
|
1809
|
+
end
|
|
1810
|
+
|
|
1811
|
+
-- Fresh data (a brand-new profile, or a ResetData wipe) is built entirely from the
|
|
1812
|
+
-- CURRENT template, so stamp it and let runMigrations no-op. Run against fresh
|
|
1813
|
+
-- defaults, migrations double-apply: a one-time compensation grant handed to a new
|
|
1814
|
+
-- player, or a rename step overwriting a field's default with nil. `not imported`:
|
|
1815
|
+
-- adopted data has never seen a Scribe migration, so it enters at version 1 and
|
|
1816
|
+
-- runs the entire chain, and stamping it current here would skip every step.
|
|
1817
|
+
if (isNew or willReset or freshAsTemplate) and not imported then
|
|
1818
|
+
local reserved = profile.Data[Template.RESERVED_ROOT]
|
|
1819
|
+
if type(reserved) == "table" then
|
|
1820
|
+
reserved.Version = currentVersion
|
|
1821
|
+
end
|
|
1822
|
+
end
|
|
1823
|
+
|
|
1824
|
+
local migrated, refusal = runMigrations(entry, profile.Data)
|
|
1825
|
+
if not migrated then
|
|
1826
|
+
-- Carve-out for a returning player whose stored data is still nothing but
|
|
1827
|
+
-- current-template defaults. Fail-closed protects real stored bytes and this
|
|
1828
|
+
-- profile has none, so kicking would manufacture a permanent outage for
|
|
1829
|
+
-- players who load fine today. Load them, but do NOT stamp the version, so the
|
|
1830
|
+
-- chain retries every join and lands once the migration is fixed.
|
|
1831
|
+
local allDefaults = not freshAsTemplate
|
|
1832
|
+
and storedVersion < currentVersion
|
|
1833
|
+
and equalsTemplateDefaults(profile.Data)
|
|
1834
|
+
if allDefaults then
|
|
1835
|
+
Log.Warn(
|
|
1836
|
+
"Persistence",
|
|
1837
|
+
"MIGRATION_FAIL",
|
|
1838
|
+
"migration failed against an all-default returning profile; loading it un-migrated "
|
|
1839
|
+
.. "and UNSTAMPED (nothing but template defaults is at risk). It retries on every "
|
|
1840
|
+
.. "join until the migration is fixed.",
|
|
1841
|
+
{ Player = player }
|
|
1842
|
+
)
|
|
1843
|
+
else
|
|
1844
|
+
-- Fail-closed: never persist a half-migrated profile. Migrations ran on a
|
|
1845
|
+
-- clone and the reserved Version is stamped only after the last rejection,
|
|
1846
|
+
-- so profile.Data is untouched on every path that returns false.
|
|
1847
|
+
entry.LocalSessionEnd = true
|
|
1848
|
+
releaseWithoutWriting(profile)
|
|
1849
|
+
if player.Parent == Players then
|
|
1850
|
+
-- Their data is newer, not broken: the migration wording is backwards here.
|
|
1851
|
+
player:Kick(kickText(if refusal == "version-ahead" then "VersionAhead" else "Migration"))
|
|
1852
|
+
end
|
|
1853
|
+
teardown(entry, "migration-failed")
|
|
1854
|
+
return
|
|
1855
|
+
end
|
|
1856
|
+
end
|
|
1857
|
+
|
|
1858
|
+
-- Reconcile runs BEFORE migrations, so a migration body reaching through a static
|
|
1859
|
+
-- sub-record or a scalar default reads a value Reconcile manufactured rather than
|
|
1860
|
+
-- one the player stored, and the two commonest guarded shapes silently no-op.
|
|
1861
|
+
-- Reordering is not the fix (it nil-indexes raw data and fail-closed kicks every
|
|
1862
|
+
-- legacy player), so re-run the chain against the raw bytes and report a diff.
|
|
1863
|
+
if rawStored ~= nil and not freshAsTemplate and storedVersion < currentVersion then
|
|
1864
|
+
local shadowOk = true
|
|
1865
|
+
for version = storedVersion + 1, currentVersion do
|
|
1866
|
+
local step = migrations[version]
|
|
1867
|
+
if step then
|
|
1868
|
+
local ok, err = xpcall(step, function(e)
|
|
1869
|
+
return debug.traceback(tostring(e), 2)
|
|
1870
|
+
end, rawStored)
|
|
1871
|
+
if not ok then
|
|
1872
|
+
shadowOk = false
|
|
1873
|
+
Log.Warn(
|
|
1874
|
+
"Persistence",
|
|
1875
|
+
"MIGRATION_RECONCILE_DEPENDENT",
|
|
1876
|
+
`migration step {version} succeeds only because Reconcile backfilled a template `
|
|
1877
|
+
.. `key before it ran. Against the raw stored data it throws: {err}`,
|
|
1878
|
+
{ Player = player, Step = version }
|
|
1879
|
+
)
|
|
1880
|
+
break
|
|
1881
|
+
end
|
|
1882
|
+
end
|
|
1883
|
+
end
|
|
1884
|
+
if shadowOk then
|
|
1885
|
+
local diffs = {}
|
|
1886
|
+
divergences(rawStored, profile.Data, {}, diffs)
|
|
1887
|
+
if #diffs > 0 then
|
|
1888
|
+
Log.Warn(
|
|
1889
|
+
"Persistence",
|
|
1890
|
+
"MIGRATION_RECONCILE_DEPENDENT",
|
|
1891
|
+
"migrations produce a different result against the raw stored data (a guarded step "
|
|
1892
|
+
.. "saw a Reconcile-backfilled default instead of nil): "
|
|
1893
|
+
.. table.concat(diffs, ", "),
|
|
1894
|
+
{ Player = player }
|
|
1895
|
+
)
|
|
1896
|
+
end
|
|
1897
|
+
end
|
|
1898
|
+
end
|
|
1899
|
+
|
|
1900
|
+
-- Reconcile only fills STATIC template keys; ArrayOf/DictOf element fields hang off
|
|
1901
|
+
-- AnyChild and would never reach stored entries. After migrations for two reasons:
|
|
1902
|
+
-- a migration must observe the raw stored element data (a guarded rename would
|
|
1903
|
+
-- otherwise see a backfilled default and destroy the real value), and the
|
|
1904
|
+
-- version-ahead fail-closed path above must leave the stored bytes untouched.
|
|
1905
|
+
if Template.ReconcileElements(compiled, profile.Data) then
|
|
1906
|
+
entry.SaveInfo.Dirty = true
|
|
1907
|
+
end
|
|
1908
|
+
|
|
1909
|
+
-- Stored-shape check, opt-in. After Reconcile and ReconcileElements on purpose:
|
|
1910
|
+
-- before them every field the template has gained since this profile was written
|
|
1911
|
+
-- is still absent, and each would be reported on a profile about to be brought up
|
|
1912
|
+
-- to date anyway. Defaults to "Warn" in DevMode and off on live: turned on live it
|
|
1913
|
+
-- is worse than useless on an upgraded game, since ordinary template evolutions
|
|
1914
|
+
-- report and UnknownKey fires once per stored element, so one retired field on a
|
|
1915
|
+
-- 40-item inventory fills the whole limit. An explicit SchemaPolicy always wins.
|
|
1916
|
+
local schemaPolicy = options.SchemaPolicy
|
|
1917
|
+
if schemaPolicy == nil and ctx.DevMode then
|
|
1918
|
+
schemaPolicy = "Warn"
|
|
1919
|
+
end
|
|
1920
|
+
if schemaPolicy ~= nil then
|
|
1921
|
+
local violations = Conform.Check(compiled, profile.Data)
|
|
1922
|
+
if #violations > 0 then
|
|
1923
|
+
-- ADVISORY VIOLATIONS NEVER END A SESSION, even under "Reject"; the long
|
|
1924
|
+
-- form is on the Violation type in Conform. Only a table mixing array
|
|
1925
|
+
-- indices with string keys actually fails the DataStore save. The caps in
|
|
1926
|
+
-- particular fire on a state Scribe's own write path calls legal, and
|
|
1927
|
+
-- rejecting them kicked the player on every rejoin forever, since
|
|
1928
|
+
-- releaseWithoutWriting leaves the bytes as they are. ANY non-advisory
|
|
1929
|
+
-- violation rejects, however many advisory findings sit beside it.
|
|
1930
|
+
local reject = false
|
|
1931
|
+
if schemaPolicy == "Reject" then
|
|
1932
|
+
for _, violation in violations do
|
|
1933
|
+
if not violation.Advisory then
|
|
1934
|
+
reject = true
|
|
1935
|
+
break
|
|
1936
|
+
end
|
|
1937
|
+
end
|
|
1938
|
+
end
|
|
1939
|
+
for _, violation in violations do
|
|
1940
|
+
Metrics.Add("SchemaViolations")
|
|
1941
|
+
-- Advisory findings stay at Warn even on a rejecting load, so the
|
|
1942
|
+
-- level says which one actually cost the player the session.
|
|
1943
|
+
local report = if reject and not violation.Advisory then Log.Error else Log.Warn
|
|
1944
|
+
report("Persistence", "PROFILE_SCHEMA_VIOLATION", violation.Detail, {
|
|
1945
|
+
Player = player,
|
|
1946
|
+
Path = table.concat(violation.Path :: any, "."),
|
|
1947
|
+
Kind = violation.Kind,
|
|
1948
|
+
Advisory = violation.Advisory == true,
|
|
1949
|
+
})
|
|
1950
|
+
ctx.Signals.OnAnomaly:Fire({
|
|
1951
|
+
Player = player,
|
|
1952
|
+
Path = violation.Path,
|
|
1953
|
+
Reason = violation.Kind,
|
|
1954
|
+
})
|
|
1955
|
+
end
|
|
1956
|
+
|
|
1957
|
+
if reject then
|
|
1958
|
+
-- Fail-closed, the same shape as the migration failure above: the
|
|
1959
|
+
-- session ends rather than serving a profile that does not match the
|
|
1960
|
+
-- template, and the stored bytes are left alone (see
|
|
1961
|
+
-- releaseWithoutWriting for why EndSession will not do).
|
|
1962
|
+
entry.LocalSessionEnd = true
|
|
1963
|
+
releaseWithoutWriting(profile)
|
|
1964
|
+
if player.Parent == Players then
|
|
1965
|
+
player:Kick(kickText("Schema"))
|
|
1966
|
+
end
|
|
1967
|
+
teardown(entry, "load-failed")
|
|
1968
|
+
return
|
|
1969
|
+
end
|
|
1970
|
+
end
|
|
1971
|
+
end
|
|
1972
|
+
|
|
1973
|
+
-- Seed the flagged Scribe.Dynamic fields now that Reconcile has created any parent
|
|
1974
|
+
-- containers, before OnPlayerInit so it can still override. Written raw into
|
|
1975
|
+
-- profile.Data (the factory already packs datatypes); the tree wraps this same
|
|
1976
|
+
-- table, so the values are live and will save.
|
|
1977
|
+
if #dynamicToSeed > 0 then
|
|
1978
|
+
for _, node in dynamicToSeed do
|
|
1979
|
+
local ok, value = pcall(node.Factory :: () -> any)
|
|
1980
|
+
if not ok then
|
|
1981
|
+
Log.Error(
|
|
1982
|
+
"Lifecycle",
|
|
1983
|
+
"DYNAMIC_DEFAULT_FAILED",
|
|
1984
|
+
`Scribe.Dynamic factory for "{Util.pathString(node.Path)}" threw: {value}`,
|
|
1985
|
+
{ Player = player, Path = Util.pathString(node.Path) }
|
|
1986
|
+
)
|
|
1987
|
+
else
|
|
1988
|
+
local container: any = profile.Data
|
|
1989
|
+
for i = 1, #node.Path - 1 do
|
|
1990
|
+
container = if type(container) == "table" then container[node.Path[i]] else nil
|
|
1991
|
+
end
|
|
1992
|
+
if type(container) == "table" then
|
|
1993
|
+
container[node.Path[#node.Path]] = value
|
|
1994
|
+
end
|
|
1995
|
+
end
|
|
1996
|
+
end
|
|
1997
|
+
entry.SaveInfo.Dirty = true
|
|
1998
|
+
end
|
|
1999
|
+
|
|
2000
|
+
if ctx.DevMode then
|
|
2001
|
+
local unknown = Template.UnknownRootKeys(compiled, profile.Data)
|
|
2002
|
+
if #unknown > 0 then
|
|
2003
|
+
Log.Warn(
|
|
2004
|
+
"Persistence",
|
|
2005
|
+
"UNKNOWN_ROOT_KEYS",
|
|
2006
|
+
"profile contains top-level keys absent from the template "
|
|
2007
|
+
.. "(removing keys is a migration's job)",
|
|
2008
|
+
{
|
|
2009
|
+
Player = player,
|
|
2010
|
+
Keys = table.concat(unknown, ", "),
|
|
2011
|
+
}
|
|
2012
|
+
)
|
|
2013
|
+
end
|
|
2014
|
+
end
|
|
2015
|
+
|
|
2016
|
+
-- Taken before BOTH hooks: Dynamic factories run above this point too.
|
|
2017
|
+
local exchangeStateBefore = exchangeSnapshot(profile.Data)
|
|
2018
|
+
|
|
2019
|
+
if options.OnPlayerInit then
|
|
2020
|
+
-- A brand-new profile, a ResetData wipe, or a first-session crash recovery all
|
|
2021
|
+
-- mean "no prior saved data", which is what starter kits need. An IMPORTED
|
|
2022
|
+
-- profile does not: they arrive carrying a full inventory, and the starter kit
|
|
2023
|
+
-- on top is the exact double-grant this flag prevents elsewhere.
|
|
2024
|
+
local isNewProfile = (isNew or willReset or freshAsTemplate) and not imported
|
|
2025
|
+
-- ImportLegacyData only fires for a player who has never saved, so a game
|
|
2026
|
+
-- already on Scribe finishes moving stores here instead. NOT behind the import
|
|
2027
|
+
-- cap: this runs on every join, and serialising those throttles ordinary logins.
|
|
2028
|
+
Util.safeCall(function(err)
|
|
2029
|
+
Log.Error("Lifecycle", "ON_PLAYER_INIT_ERROR", err, { Player = player })
|
|
2030
|
+
end, options.OnPlayerInit, player, profile.Data, isNewProfile, migrationContext(entry))
|
|
2031
|
+
end
|
|
2032
|
+
|
|
2033
|
+
-- REVERTED rather than merely reported: a half-edited exchange record is worse than
|
|
2034
|
+
-- either the original or a clean one, and the original is still in hand.
|
|
2035
|
+
if exchangeStateBefore ~= nil or exchangeSnapshot(profile.Data) ~= nil then
|
|
2036
|
+
local after = exchangeSnapshot(profile.Data)
|
|
2037
|
+
if not sameExchangeState(exchangeStateBefore, after) then
|
|
2038
|
+
local reserved = profile.Data[Template.RESERVED_ROOT]
|
|
2039
|
+
if type(reserved) == "table" then
|
|
2040
|
+
reserved.Exchange = if exchangeStateBefore then exchangeStateBefore.Exchange else {}
|
|
2041
|
+
reserved.ExchangeInbox = if exchangeStateBefore then exchangeStateBefore.ExchangeInbox else {}
|
|
2042
|
+
end
|
|
2043
|
+
Log.Error(
|
|
2044
|
+
"Lifecycle",
|
|
2045
|
+
"EXCHANGE_INIT_TAMPER",
|
|
2046
|
+
"OnPlayerInit or a Scribe.Dynamic factory changed the in-flight exchange ledger under "
|
|
2047
|
+
.. `"{Template.RESERVED_ROOT}"; the change has been reverted. That state is Scribe's, `
|
|
2048
|
+
.. "and editing it can duplicate or destroy an exchange in progress",
|
|
2049
|
+
{ Player = player }
|
|
2050
|
+
)
|
|
2051
|
+
end
|
|
2052
|
+
end
|
|
2053
|
+
|
|
2054
|
+
-- Scribe.Dynamic factories and OnPlayerInit write raw into profile.Data without
|
|
2055
|
+
-- passing Node.validateWrite, and unpersistable data introduced there would fail
|
|
2056
|
+
-- the entire next save opaquely, with no path in the log. Non-fatal: the library
|
|
2057
|
+
-- cannot safely rewrite developer data.
|
|
2058
|
+
local depthStats = { MaxDepth = 0 }
|
|
2059
|
+
local badInit = Util.findUnpersistable(profile.Data, nil, nil, depthStats)
|
|
2060
|
+
if badInit ~= nil then
|
|
2061
|
+
local where = Util.pathString(badInit.Path)
|
|
2062
|
+
Log.Error(
|
|
2063
|
+
"Lifecycle",
|
|
2064
|
+
"PROFILE_UNPERSISTABLE",
|
|
2065
|
+
`unpersistable data written during load at "{where}" ({badInit.Detail}) -- this WILL fail the `
|
|
2066
|
+
.. `DataStore save; fix the value set by OnPlayerInit or a Scribe.Dynamic factory`,
|
|
2067
|
+
{ Player = player, Path = where }
|
|
2068
|
+
)
|
|
2069
|
+
ctx.Signals.OnAnomaly:Fire({ Player = player, Path = badInit.Path, Value = nil, Reason = badInit.Kind })
|
|
2070
|
+
end
|
|
2071
|
+
-- Stored data may legally be nested deeper than a write is allowed to reach: it
|
|
2072
|
+
-- loads and reads fine, then refuses every write to its deepest leaves at the
|
|
2073
|
+
-- first Set, with nothing at load having hinted at it.
|
|
2074
|
+
if depthStats.MaxDepth > Util.MAX_VALUE_DEPTH then
|
|
2075
|
+
Log.Warn(
|
|
2076
|
+
"Integrity",
|
|
2077
|
+
"PROFILE_TOO_DEEP",
|
|
2078
|
+
`stored data is nested {depthStats.MaxDepth} levels deep, past the {Util.MAX_VALUE_DEPTH}-level `
|
|
2079
|
+
.. "write limit; those leaves load and read but every write to them will be refused. "
|
|
2080
|
+
.. "Flatten the shape or migrate it",
|
|
2081
|
+
{ Player = player, Depth = depthStats.MaxDepth, Limit = Util.MAX_VALUE_DEPTH }
|
|
2082
|
+
)
|
|
2083
|
+
end
|
|
2084
|
+
|
|
2085
|
+
profile.OnSave:Connect(function(saveContext: any)
|
|
2086
|
+
-- Point the profile at a copy holding online-only cooldowns in their persisted
|
|
2087
|
+
-- form and keep the live deadlines aside. OnSave fires before the store reads
|
|
2088
|
+
-- profile.Data, OnAfterSave once it has, OnError when it never gets there.
|
|
2089
|
+
entry.TimedHeld = ctx.Timed.BeforeSave(entry)
|
|
2090
|
+
onProfileSaving(entry, saveContext)
|
|
2091
|
+
end)
|
|
2092
|
+
-- Exactly one fire per save call, success or failure, so this is where the
|
|
2093
|
+
-- in-flight count comes back down. OnAfterSave cannot do it: it fires only on
|
|
2094
|
+
-- success, so a failing save would leak a count forever and every later error on
|
|
2095
|
+
-- this key would be re-attributed to it. The fire's second argument, this call's
|
|
2096
|
+
-- own outcome, is deliberately unbound here: it is consumed as SaveAwait's return
|
|
2097
|
+
-- value below, and its one interesting class, "NotOwned", only happens as the
|
|
2098
|
+
-- session ends, by which time teardown has detached this entry from Entries.
|
|
2099
|
+
if type((profile :: any).OnSettled) == "table" then
|
|
2100
|
+
(profile :: any).OnSettled:Connect(function(saveContext: any)
|
|
2101
|
+
entry.SavesInFlight = math.max(0, entry.SavesInFlight - 1)
|
|
2102
|
+
if entry.SavesInFlight == 0 then
|
|
2103
|
+
entry.SaveT0 = nil
|
|
2104
|
+
-- ...and with it the pre-snapshot dirtiness the OnError restore reads.
|
|
2105
|
+
-- On the DRAIN, not once per settle: the slot is shared by every save
|
|
2106
|
+
-- in flight, so clearing it at the first of two completions hands the
|
|
2107
|
+
-- survivor a nil and its failure restores clean over an unstored write.
|
|
2108
|
+
-- Erring dirty costs one redundant save; the other direction costs a
|
|
2109
|
+
-- receipt answered PurchaseGranted for a grant that exists only in RAM.
|
|
2110
|
+
entry.DirtyAtSaveStart = nil
|
|
2111
|
+
end
|
|
2112
|
+
-- A release-only save wrote MetaData and left the stored bytes as it found
|
|
2113
|
+
-- them, so everything live is still unsaved. It reaches OnAfterSave like
|
|
2114
|
+
-- any successful write, so onProfileSaved has just cleared Dirty and
|
|
2115
|
+
-- stamped "Ok", which is exactly what isDurable reads: without this,
|
|
2116
|
+
-- Data.Flush short-circuits true for a profile whose data has never been
|
|
2117
|
+
-- persisted. OnSettled fires after OnAfterSave, so this is not undone.
|
|
2118
|
+
if type(saveContext) == "table" and saveContext.ReleaseOnly == true then
|
|
2119
|
+
entry.SaveInfo.Dirty = true
|
|
2120
|
+
self.UpdateSaveInfoMirror(entry)
|
|
2121
|
+
end
|
|
2122
|
+
end)
|
|
2123
|
+
end
|
|
2124
|
+
|
|
2125
|
+
profile.OnAfterSave:Connect(function(lastSavedData: any)
|
|
2126
|
+
ctx.Timed.AfterSave(entry, entry.TimedHeld)
|
|
2127
|
+
entry.TimedHeld = nil
|
|
2128
|
+
onProfileSaved(entry, lastSavedData)
|
|
2129
|
+
end)
|
|
2130
|
+
profile.OnSessionEnd:Connect(function()
|
|
2131
|
+
-- A session end WE did not initiate (and not the shutdown drain) is an
|
|
2132
|
+
-- external steal: its final save persists nothing locally, so save
|
|
2133
|
+
-- bookkeeping must not report success for it (onProfileSaved checks this).
|
|
2134
|
+
entry.SessionStolen = not entry.LocalSessionEnd and not self.Closing
|
|
2135
|
+
teardown(entry, if player.Parent == Players then "session-ended" else "player-left")
|
|
2136
|
+
end)
|
|
2137
|
+
|
|
2138
|
+
-- If the player left (or the session was stolen) during the yielding
|
|
2139
|
+
-- migration/OnPlayerInit above, OnSessionEnd already fired with no listener
|
|
2140
|
+
-- attached, so tear down rather than leaking a Ready session. A NoSave load holds
|
|
2141
|
+
-- a VIEW, which is never IsActive() by design, so the guard must not apply to it.
|
|
2142
|
+
if mode ~= "NoSave" and not profile:IsActive() then
|
|
2143
|
+
teardown(entry, if player.Parent == Players then "session-ended" else "player-left")
|
|
2144
|
+
return
|
|
2145
|
+
end
|
|
2146
|
+
|
|
2147
|
+
buildTree(entry)
|
|
2148
|
+
-- The session store was built from template defaults, so a derived field's default
|
|
2149
|
+
-- came from its inputs' defaults rather than the data that just loaded. Recompute
|
|
2150
|
+
-- before Ready, before Leaderboards.OnLoad (which enqueues an initial score), and
|
|
2151
|
+
-- before Replication.OnEntryReady (whose Init snapshot must carry real values).
|
|
2152
|
+
entry.Tree:SeedDerived()
|
|
2153
|
+
entry.SaveInfo.Dirty = false
|
|
2154
|
+
self.UpdateSaveInfoMirror(entry)
|
|
2155
|
+
|
|
2156
|
+
-- Subsystem hooks are isolated so one failing hook cannot abort the others
|
|
2157
|
+
-- (or strand the load thread).
|
|
2158
|
+
local function runHook(name: string, hook: (any) -> ...any)
|
|
2159
|
+
local ok, err = pcall(hook, entry)
|
|
2160
|
+
if not ok then
|
|
2161
|
+
Log.Error(
|
|
2162
|
+
"Lifecycle",
|
|
2163
|
+
"SUBSYSTEM_HOOK_ERROR",
|
|
2164
|
+
`{name} hook failed: {tostring(err)}`,
|
|
2165
|
+
{ Player = player }
|
|
2166
|
+
)
|
|
2167
|
+
end
|
|
2168
|
+
end
|
|
2169
|
+
|
|
2170
|
+
-- BEFORE Ready, unlike the hooks below, and that is a correctness requirement. An
|
|
2171
|
+
-- online-only cooldown is stored as REMAINING seconds and only becomes a deadline
|
|
2172
|
+
-- here, so after Ready the documented PlayerAdded -> WaitForData -> read path
|
|
2173
|
+
-- reads a remaining value as a deadline, the gate reads expired, and re-arming in
|
|
2174
|
+
-- that window compounds into a permanent lockout. Must also stay BELOW
|
|
2175
|
+
-- `Dirty = false`: the load sweep's clears write through setTimed.
|
|
2176
|
+
runHook("Timed", ctx.Timed.OnLoad)
|
|
2177
|
+
|
|
2178
|
+
setState(entry, "Ready", nil)
|
|
2179
|
+
Metrics.Add("ProfilesLoaded")
|
|
2180
|
+
Metrics.Set("ActiveSessions", self.EntryCount)
|
|
2181
|
+
|
|
2182
|
+
-- The DataStore round trip is network time, not what the player experienced.
|
|
2183
|
+
-- Observed as a distribution so the TAIL is readable: a p50 join says nothing
|
|
2184
|
+
-- about the player who waited twenty seconds.
|
|
2185
|
+
local loadSeconds = if entry.JoinedAt then Clock.Monotonic() - entry.JoinedAt else nil
|
|
2186
|
+
if loadSeconds then
|
|
2187
|
+
Metrics.Observe("LoadDuration", loadSeconds)
|
|
2188
|
+
end
|
|
2189
|
+
Log.Info("Persistence", "PROFILE_LOADED", "profile session started", {
|
|
2190
|
+
Player = player,
|
|
2191
|
+
LoadSeconds = loadSeconds,
|
|
2192
|
+
})
|
|
2193
|
+
|
|
2194
|
+
-- Register the cross-server gift handler FIRST and independently: it only
|
|
2195
|
+
-- needs the tree, and it must not be lost if a subsystem hook below
|
|
2196
|
+
-- throws (e.g. a misconfigured leaderboard Stat path).
|
|
2197
|
+
profile:MessageHandler(function(message: any, processed: () -> ())
|
|
2198
|
+
ctx.Monetization.OnProfileMessage(entry, message, processed)
|
|
2199
|
+
-- User-level cross-server messages (Data.SendMessage) ride the same channel,
|
|
2200
|
+
-- tagged distinctly so they never collide with gift delivery.
|
|
2201
|
+
-- ACKNOWLEDGE ONLY WHAT A HANDLER ACTUALLY OBSERVED. Fire tells the caller
|
|
2202
|
+
-- nothing: it returns at the handler's first yield, and returns the same when
|
|
2203
|
+
-- no handler is connected. processed() only flags locked_global_updates, and
|
|
2204
|
+
-- the store removes the update inside the same UpdateAsync that writes
|
|
2205
|
+
-- profile.Data, so ack once the handler has RETURNED, the rule the gift arm
|
|
2206
|
+
-- already follows. Not acking is at-least-once, and bounded: an unacked
|
|
2207
|
+
-- message returns on the next load, not once per save.
|
|
2208
|
+
if type(message) == "table" and message.ScribeMessage == true then
|
|
2209
|
+
-- Counted before the fire and uncounted on settle, so teardown can report a
|
|
2210
|
+
-- handler that never returned. FireTracked settles on the LAST handler, so
|
|
2211
|
+
-- one parked listener holds the whole fire open.
|
|
2212
|
+
entry.UnsettledMessages = (entry.UnsettledMessages or 0) + 1
|
|
2213
|
+
ctx.Signals.OnMessage:FireTracked(function(ran: number, ok: boolean, err: any)
|
|
2214
|
+
entry.UnsettledMessages = (entry.UnsettledMessages or 1) - 1
|
|
2215
|
+
if ran == 0 then
|
|
2216
|
+
-- Nobody was connected. Common in a startup race where the profile
|
|
2217
|
+
-- loads before the game binds its handler; keeping the message
|
|
2218
|
+
-- costs a redelivery, dropping it costs the message.
|
|
2219
|
+
Log.Warn(
|
|
2220
|
+
"Persistence",
|
|
2221
|
+
"MESSAGE_NO_LISTENER",
|
|
2222
|
+
"a cross-server message arrived with nothing connected to Data.OnMessage; "
|
|
2223
|
+
.. "it was NOT acknowledged and will be redelivered on the next load",
|
|
2224
|
+
{ Player = player }
|
|
2225
|
+
)
|
|
2226
|
+
return
|
|
2227
|
+
end
|
|
2228
|
+
if not ok then
|
|
2229
|
+
-- Fire let a handler error reach the scheduler; FireTracked pcalls it,
|
|
2230
|
+
-- so this is where it has to surface or it surfaces nowhere.
|
|
2231
|
+
Log.Error(
|
|
2232
|
+
"Persistence",
|
|
2233
|
+
"MESSAGE_HANDLER_ERROR",
|
|
2234
|
+
`a Data.OnMessage handler raised: {tostring(err)}; the message was NOT `
|
|
2235
|
+
.. "acknowledged and will be redelivered on the next load",
|
|
2236
|
+
{ Player = player }
|
|
2237
|
+
)
|
|
2238
|
+
return
|
|
2239
|
+
end
|
|
2240
|
+
processed()
|
|
2241
|
+
end, entry.Player, message.Data)
|
|
2242
|
+
end
|
|
2243
|
+
end)
|
|
2244
|
+
|
|
2245
|
+
-- The rest run post-Ready; only Timed has to beat the state change. An exchange
|
|
2246
|
+
-- that resolves a moment later is invisible to the player, where a load that
|
|
2247
|
+
-- waited on the verdict round trip would not be. Guarded because a bundle
|
|
2248
|
+
-- constructed before this subsystem existed must still load profiles.
|
|
2249
|
+
if ctx.Exchanges then
|
|
2250
|
+
-- SPAWNED, unlike every other hook here. Resolution makes a DataStore round
|
|
2251
|
+
-- trip on the verdict key, and run inline that trip sits in front of
|
|
2252
|
+
-- everything after it, including the replication entry-ready that puts the
|
|
2253
|
+
-- player's first snapshot on the wire. Safe to detach: nothing below depends
|
|
2254
|
+
-- on an exchange being resolved, and Resolve re-reads the tree for death on
|
|
2255
|
+
-- entry and again per exchange, so a released profile stops rather than
|
|
2256
|
+
-- writing to a dead one.
|
|
2257
|
+
Sched.Spawn(function()
|
|
2258
|
+
runHook("Exchanges", ctx.Exchanges.OnLoad)
|
|
2259
|
+
end)
|
|
2260
|
+
end
|
|
2261
|
+
runHook("Monetization", ctx.Monetization.OnLoad)
|
|
2262
|
+
runHook("Leaderboards", ctx.Leaderboards.OnLoad)
|
|
2263
|
+
runHook("Replication", ctx.Replication.OnEntryReady)
|
|
2264
|
+
end
|
|
2265
|
+
|
|
2266
|
+
-- Erasures in flight, per user.
|
|
2267
|
+
local erasing: { [number]: number } = {}
|
|
2268
|
+
local NO_PROFILE = "profile does not exist"
|
|
2269
|
+
local LIVE_ELSEWHERE = "profile has an active session on another server"
|
|
2270
|
+
|
|
2271
|
+
local function loadProfile(entry: Entry)
|
|
2272
|
+
local player = entry.Player
|
|
2273
|
+
|
|
2274
|
+
if mode == "NoSave" then
|
|
2275
|
+
setPhase(entry, "ReadingProfile")
|
|
2276
|
+
-- Snapshot of the real profile with NO session: nothing is saved, because
|
|
2277
|
+
-- a view is never IsActive() and every save path checks that first.
|
|
2278
|
+
local started = Clock.Monotonic()
|
|
2279
|
+
local ok, profile = pcall(function()
|
|
2280
|
+
return store:GetAsync(entry.Key)
|
|
2281
|
+
end)
|
|
2282
|
+
Metrics.Observe("ProfileLoadDuration", Clock.Monotonic() - started)
|
|
2283
|
+
if ok and profile then
|
|
2284
|
+
entry.Profile = profile
|
|
2285
|
+
setPhase(entry, "Initializing")
|
|
2286
|
+
finishLoad(entry)
|
|
2287
|
+
else
|
|
2288
|
+
Log.Error(
|
|
2289
|
+
"Persistence",
|
|
2290
|
+
"PROFILE_LOAD_FAIL",
|
|
2291
|
+
`Mode = "NoSave": no stored profile to read for this key`,
|
|
2292
|
+
{ Player = player, UserId = entry.UserId }
|
|
2293
|
+
)
|
|
2294
|
+
teardown(entry, "load-failed")
|
|
2295
|
+
end
|
|
2296
|
+
return
|
|
2297
|
+
end
|
|
2298
|
+
|
|
2299
|
+
local heldSince = Clock.Monotonic()
|
|
2300
|
+
while (erasing[entry.UserId] or 0) > 0 and not entry.TornDown and not self.Stopped do
|
|
2301
|
+
setPhase(entry, "WaitingForErase")
|
|
2302
|
+
if Clock.Monotonic() - heldSince >= eraseJoinTimeout then
|
|
2303
|
+
if player.Parent == Players then
|
|
2304
|
+
player:Kick(kickText("Erasing"))
|
|
2305
|
+
end
|
|
2306
|
+
teardown(entry, "erasing")
|
|
2307
|
+
return
|
|
2308
|
+
end
|
|
2309
|
+
Clock.Wait(0.1)
|
|
2310
|
+
end
|
|
2311
|
+
|
|
2312
|
+
local attempt = 0
|
|
2313
|
+
while not self.Closing and not self.Stopped do
|
|
2314
|
+
attempt += 1
|
|
2315
|
+
setPhase(entry, "AcquiringSession")
|
|
2316
|
+
-- Captured PER ATTEMPT. ProfileStore latches the first `true` its Cancel
|
|
2317
|
+
-- returns and never asks again, so a deadline computed once outside this loop
|
|
2318
|
+
-- would abort every later attempt the instant it began.
|
|
2319
|
+
local deadline = Clock.Monotonic() + loadTimeout
|
|
2320
|
+
local started = Clock.Monotonic()
|
|
2321
|
+
local ok, profile = pcall(function()
|
|
2322
|
+
return store:StartSessionAsync(entry.Key, {
|
|
2323
|
+
Cancel = function()
|
|
2324
|
+
-- Runs INSIDE a DataStore UpdateAsync transform, where
|
|
2325
|
+
-- yielding is forbidden, so this stays arithmetic and field
|
|
2326
|
+
-- reads. Monotonic, never Now: wall time can jump.
|
|
2327
|
+
return player.Parent ~= Players or self.Closing or Clock.Monotonic() >= deadline
|
|
2328
|
+
end,
|
|
2329
|
+
})
|
|
2330
|
+
end)
|
|
2331
|
+
Metrics.Observe("ProfileLoadDuration", Clock.Monotonic() - started)
|
|
2332
|
+
|
|
2333
|
+
if entry.TornDown then
|
|
2334
|
+
-- Torn down while the load was in flight (shutdown drain or a leave
|
|
2335
|
+
-- processed elsewhere): release any session the racing load just won
|
|
2336
|
+
-- and stop -- finishing a load for a dead entry would leak the lock.
|
|
2337
|
+
if ok and profile then
|
|
2338
|
+
profile:EndSession()
|
|
2339
|
+
end
|
|
2340
|
+
return
|
|
2341
|
+
end
|
|
2342
|
+
|
|
2343
|
+
if player.Parent ~= Players then
|
|
2344
|
+
if ok and profile then
|
|
2345
|
+
profile:EndSession()
|
|
2346
|
+
end
|
|
2347
|
+
teardown(entry, "player-left")
|
|
2348
|
+
return
|
|
2349
|
+
end
|
|
2350
|
+
|
|
2351
|
+
if ok and profile then
|
|
2352
|
+
entry.Profile = profile
|
|
2353
|
+
Health.ReportSuccess("ProfileLoad")
|
|
2354
|
+
setPhase(entry, "Initializing")
|
|
2355
|
+
finishLoad(entry)
|
|
2356
|
+
return
|
|
2357
|
+
end
|
|
2358
|
+
|
|
2359
|
+
-- A nil profile during shutdown is a Cancel-abort, not a DataStore
|
|
2360
|
+
-- failure: tear down quietly (BindToClose disconnects everyone) so we
|
|
2361
|
+
-- don't kick with a data-loss message or poison Health/metrics.
|
|
2362
|
+
if self.Closing then
|
|
2363
|
+
teardown(entry, "shutdown")
|
|
2364
|
+
return
|
|
2365
|
+
end
|
|
2366
|
+
|
|
2367
|
+
-- Keyed for the same reason as the save path: under LoadFailurePolicy = "Wait"
|
|
2368
|
+
-- this fires once per retry for the same profile, so folding keeps a retry
|
|
2369
|
+
-- cycle from emitting one Error per attempt.
|
|
2370
|
+
Failure.Report({
|
|
2371
|
+
Subsystem = "ProfileLoad",
|
|
2372
|
+
Key = entry.Key,
|
|
2373
|
+
-- StartSessionAsync answers nil rather than a message; the underlying
|
|
2374
|
+
-- DataStore error, if there was one, already went through OnError above.
|
|
2375
|
+
Metric = "ProfileLoadFailures",
|
|
2376
|
+
Level = "Error",
|
|
2377
|
+
Category = "Persistence",
|
|
2378
|
+
Code = "PROFILE_LOAD_FAIL",
|
|
2379
|
+
Message = `profile failed to load (attempt {attempt})`,
|
|
2380
|
+
Context = { Player = player, Attempt = attempt },
|
|
2381
|
+
})
|
|
2382
|
+
|
|
2383
|
+
if (options.LoadFailurePolicy or "Kick") == "Kick" then
|
|
2384
|
+
player:Kick(loadKickText(entry))
|
|
2385
|
+
teardown(entry, "load-failed")
|
|
2386
|
+
return
|
|
2387
|
+
end
|
|
2388
|
+
-- "Wait": stay in Loading and keep retrying with backoff; never fall back
|
|
2389
|
+
-- to template data on a live account. Jittered so a fleet that lost the
|
|
2390
|
+
-- DataStore at one instant does not come back at one instant.
|
|
2391
|
+
setPhase(entry, "Retrying")
|
|
2392
|
+
Clock.Wait(Rng.Jitter(math.min(5 * attempt, 60)))
|
|
2393
|
+
end
|
|
2394
|
+
-- Loop exited because Closing or Stopped flipped during a "Wait" backoff: tear the
|
|
2395
|
+
-- stranded Loading entry down so the shutdown drain isn't held to its full budget.
|
|
2396
|
+
if not entry.TornDown then
|
|
2397
|
+
teardown(entry, "shutdown")
|
|
2398
|
+
end
|
|
2399
|
+
end
|
|
2400
|
+
|
|
2401
|
+
function self.OnPlayerAdded(player: Player)
|
|
2402
|
+
if self.Entries[player] or self.Closing then
|
|
2403
|
+
return
|
|
2404
|
+
end
|
|
2405
|
+
self.EndReasons[player] = nil
|
|
2406
|
+
local userId = self.TargetUserId or player.UserId
|
|
2407
|
+
local entry: Entry = {
|
|
2408
|
+
Player = player,
|
|
2409
|
+
UserId = userId,
|
|
2410
|
+
Key = self.KeyFor(userId),
|
|
2411
|
+
State = "Loading",
|
|
2412
|
+
-- Monotonic, and stamped at the JOIN rather than at the store call: what
|
|
2413
|
+
-- matters is what the player waited for, retries and migration included.
|
|
2414
|
+
JoinedAt = Clock.Monotonic(),
|
|
2415
|
+
FailReason = nil,
|
|
2416
|
+
Profile = nil,
|
|
2417
|
+
Tree = nil,
|
|
2418
|
+
SessionStore = {},
|
|
2419
|
+
LiveData = nil,
|
|
2420
|
+
StateChanged = Signal.new(),
|
|
2421
|
+
SaveInfo = { LastSaveAt = nil, LastResult = nil, Dirty = false },
|
|
2422
|
+
SaveT0 = nil,
|
|
2423
|
+
SavesInFlight = 0,
|
|
2424
|
+
DirtyAtSaveStart = nil,
|
|
2425
|
+
Blocked = false,
|
|
2426
|
+
ForceSaveOnce = false,
|
|
2427
|
+
LastSavedSize = nil,
|
|
2428
|
+
LastSavedKeys = nil,
|
|
2429
|
+
SizeWarned = false,
|
|
2430
|
+
TornDown = false,
|
|
2431
|
+
UnsettledMessages = 0,
|
|
2432
|
+
HelloReceived = false,
|
|
2433
|
+
QueueOpen = false,
|
|
2434
|
+
HandshakeReady = false,
|
|
2435
|
+
InitDelivered = false,
|
|
2436
|
+
LastInitAt = 0,
|
|
2437
|
+
ResyncPending = false,
|
|
2438
|
+
Queue = {},
|
|
2439
|
+
QueueIndex = {},
|
|
2440
|
+
SharedQueue = {},
|
|
2441
|
+
MalformedCount = 0,
|
|
2442
|
+
OversizeCount = 0,
|
|
2443
|
+
FrameTokens = ctx.InboundFrameRate,
|
|
2444
|
+
FrameLast = Clock.Monotonic(),
|
|
2445
|
+
RateTokens = options.CommandRateLimit or 20,
|
|
2446
|
+
RateLast = Clock.Monotonic(),
|
|
2447
|
+
GiftLastAt = 0,
|
|
2448
|
+
}
|
|
2449
|
+
self.Entries[player] = entry
|
|
2450
|
+
self.EntryCount += 1
|
|
2451
|
+
syncBudgetReserves()
|
|
2452
|
+
self.EntriesByUserId[entry.UserId] = entry
|
|
2453
|
+
entry.StopLoadWatchdog = watchdog(SLOW_LOAD_SECONDS, function()
|
|
2454
|
+
entry.StopLoadWatchdog = nil
|
|
2455
|
+
if not entry.TornDown then
|
|
2456
|
+
warnSlowLoad(entry)
|
|
2457
|
+
end
|
|
2458
|
+
end)
|
|
2459
|
+
Sched.Spawn(function()
|
|
2460
|
+
-- Its OWN thread, so everything the load allocates, the profile copy and the
|
|
2461
|
+
-- whole accessor tree, is charged here. A memory category is per-thread and
|
|
2462
|
+
-- survives a yield, where a profiler span would not: the load waits on the
|
|
2463
|
+
-- DataStore, and an annotation does not survive that.
|
|
2464
|
+
Profile.MemoryCategory("Scribe")
|
|
2465
|
+
loadProfile(entry)
|
|
2466
|
+
end)
|
|
2467
|
+
end
|
|
2468
|
+
|
|
2469
|
+
function self.OnPlayerRemoving(player: Player)
|
|
2470
|
+
local entry = self.Entries[player]
|
|
2471
|
+
if not entry then
|
|
2472
|
+
return
|
|
2473
|
+
end
|
|
2474
|
+
if entry.Profile and entry.Profile:IsActive() then
|
|
2475
|
+
runLeavingHook(entry, if player.Parent == Players then "session-ended" else "player-left")
|
|
2476
|
+
entry.LocalSessionEnd = true
|
|
2477
|
+
entry.Profile:EndSession() -- final save; OnSessionEnd triggers teardown
|
|
2478
|
+
else
|
|
2479
|
+
-- INCLUDING Loading. The StartSessionAsync Cancel guard is enough for the
|
|
2480
|
+
-- default policy, but under LoadFailurePolicy = "Wait" the retry loop spends
|
|
2481
|
+
-- its time in the BACKOFF between attempts, where there is no Cancel hook, so
|
|
2482
|
+
-- the entry outlives the player by up to a backoff. A rejoin inside that
|
|
2483
|
+
-- window gets a new entry, and the old entry's late teardown broadcasts its
|
|
2484
|
+
-- Shared-root removal, dropping every bystander's mirror of a live Ready
|
|
2485
|
+
-- player. Safe for a load that IS in flight: teardown sets TornDown, and the
|
|
2486
|
+
-- loader checks it the moment StartSessionAsync returns.
|
|
2487
|
+
teardown(entry, "player-left")
|
|
2488
|
+
end
|
|
2489
|
+
end
|
|
2490
|
+
|
|
2491
|
+
-- Public API ------------------------------------------------------------------
|
|
2492
|
+
|
|
2493
|
+
-- Wait for the next StateChanged fire, resuming anyway once the deadline passes:
|
|
2494
|
+
-- under LoadFailurePolicy = "Wait" an entry can sit in Loading for an entire DataStore
|
|
2495
|
+
-- outage without a state transition, and WaitForData must still honor its timeout.
|
|
2496
|
+
local function waitStateChangedUntil(entry: Entry, deadline: number)
|
|
2497
|
+
local thread = coroutine.running()
|
|
2498
|
+
local settled = false
|
|
2499
|
+
local function resume()
|
|
2500
|
+
if settled then
|
|
2501
|
+
return
|
|
2502
|
+
end
|
|
2503
|
+
-- Claimed synchronously, resumed later. Two fires racing would otherwise
|
|
2504
|
+
-- both queue a wake before either set the flag.
|
|
2505
|
+
settled = true
|
|
2506
|
+
-- Deferred, not inline: an inline resume returns control from the middle of
|
|
2507
|
+
-- finishLoad, before the hooks have run, so the caller sees a session that
|
|
2508
|
+
-- reports Ready but is not. It also stops an inline Clock.Delay from resuming
|
|
2509
|
+
-- this thread before it has yielded.
|
|
2510
|
+
Sched.Defer(function()
|
|
2511
|
+
if coroutine.status(thread) == "suspended" then
|
|
2512
|
+
Sched.Resume(thread)
|
|
2513
|
+
end
|
|
2514
|
+
end)
|
|
2515
|
+
end
|
|
2516
|
+
local connection = entry.StateChanged:Connect(resume)
|
|
2517
|
+
Clock.Delay(math.max(deadline - Clock.Monotonic(), 0), resume)
|
|
2518
|
+
coroutine.yield()
|
|
2519
|
+
connection:Disconnect()
|
|
2520
|
+
end
|
|
2521
|
+
|
|
2522
|
+
function self.WaitForData(player: Player, timeout: number?): (any?, string?)
|
|
2523
|
+
local deadline = Clock.Monotonic() + (timeout or Defaults.ServerWaitForData)
|
|
2524
|
+
while true do
|
|
2525
|
+
local entry = self.Entries[player]
|
|
2526
|
+
if entry then
|
|
2527
|
+
if entry.State == "Ready" then
|
|
2528
|
+
return entry.Tree.Accessor, nil
|
|
2529
|
+
elseif entry.State == "SessionEnded" then
|
|
2530
|
+
return nil, entry.FailReason or "session-ended"
|
|
2531
|
+
end
|
|
2532
|
+
if Clock.Monotonic() >= deadline then
|
|
2533
|
+
-- Necessarily Loading, so the load is still in flight and will often
|
|
2534
|
+
-- succeed. Reported separately from "timeout" because the default wait
|
|
2535
|
+
-- (60s) is shorter than a legitimate slow load, and a caller that wants
|
|
2536
|
+
-- to keep waiting can pass its own timeout.
|
|
2537
|
+
return nil, "still-loading"
|
|
2538
|
+
end
|
|
2539
|
+
waitStateChangedUntil(entry, deadline)
|
|
2540
|
+
else
|
|
2541
|
+
local endedReason = self.EndReasons[player]
|
|
2542
|
+
if endedReason then
|
|
2543
|
+
return nil, endedReason
|
|
2544
|
+
end
|
|
2545
|
+
if player.Parent ~= Players then
|
|
2546
|
+
return nil, "player-left"
|
|
2547
|
+
end
|
|
2548
|
+
if Clock.Monotonic() >= deadline then
|
|
2549
|
+
-- No entry, no stored end reason, and the player is still here: Scribe
|
|
2550
|
+
-- has no session for them at all, which usually means the bundle was
|
|
2551
|
+
-- constructed after they joined and OnPlayerAdded never ran.
|
|
2552
|
+
return nil, "timeout"
|
|
2553
|
+
end
|
|
2554
|
+
Clock.Wait(0.1)
|
|
2555
|
+
end
|
|
2556
|
+
end
|
|
2557
|
+
end
|
|
2558
|
+
|
|
2559
|
+
--- The phase is a SECOND return, not a fourth state: every existing single-value
|
|
2560
|
+
--- caller keeps working and nothing switching on the state changes meaning.
|
|
2561
|
+
function self.GetState(player: Player): (string, string?)
|
|
2562
|
+
local entry = self.Entries[player]
|
|
2563
|
+
if entry == nil then
|
|
2564
|
+
return "SessionEnded", nil
|
|
2565
|
+
end
|
|
2566
|
+
return entry.State, entry.LoadPhase
|
|
2567
|
+
end
|
|
2568
|
+
|
|
2569
|
+
--- Did THIS save write something other than the live tree?
|
|
2570
|
+
---
|
|
2571
|
+
--- The wipe guard's Block policy turns a save into a rewrite of the last-good snapshot
|
|
2572
|
+
--- by re-pointing profile.Data (SetSaveSource), which is per-PROFILE and affects every
|
|
2573
|
+
--- transform that has not run yet, so a save that snapshotted clean and was still out
|
|
2574
|
+
--- when a later save tripped the guard wrote the snapshot and told its caller true.
|
|
2575
|
+
--- `SavedSource` is the identity of the table the transform actually serialized, so
|
|
2576
|
+
--- this compares what WENT TO THE KEY against what the tree writes into. The
|
|
2577
|
+
--- snapshot-time flag is the fallback for a store that reports no SavedSource.
|
|
2578
|
+
local function blockedForSave(entry: Entry, saveContext: { [string]: any }): boolean
|
|
2579
|
+
local source = saveContext.SavedSource
|
|
2580
|
+
if source ~= nil and entry.LiveData ~= nil then
|
|
2581
|
+
return source ~= entry.LiveData
|
|
2582
|
+
end
|
|
2583
|
+
return saveContext.Blocked == true
|
|
2584
|
+
end
|
|
2585
|
+
|
|
2586
|
+
--- All three conditions are load-bearing: `Dirty` is cleared at SNAPSHOT time so it
|
|
2587
|
+
--- cannot answer alone; `SavesInFlight` stops a caller being told mid-flight writes are
|
|
2588
|
+
--- durable; and `LastResult` must be "Ok" rather than merely not "Fail", because
|
|
2589
|
+
--- finishLoad resets Dirty after the migration chain and OnPlayerInit have written.
|
|
2590
|
+
--- Deliberately no `Blocked` check: onProfileSaved already keeps a blocked entry dirty.
|
|
2591
|
+
local function isDurable(entry: Entry): boolean
|
|
2592
|
+
return entry.SaveInfo.Dirty == false and (entry.SavesInFlight or 0) == 0 and entry.SaveInfo.LastResult == "Ok"
|
|
2593
|
+
end
|
|
2594
|
+
|
|
2595
|
+
-- Force an immediate save. Returns true once the save is confirmed.
|
|
2596
|
+
function self.Flush(player: Player, opts: { Force: boolean?, Timeout: number? }?): boolean
|
|
2597
|
+
local entry = self.Entries[player]
|
|
2598
|
+
if not entry or not entry.Profile or entry.State ~= "Ready" then
|
|
2599
|
+
return false
|
|
2600
|
+
end
|
|
2601
|
+
if opts and opts.Force then
|
|
2602
|
+
-- Force exists to push a wipe-guard-BLOCKED save through, so it must always
|
|
2603
|
+
-- reach the store rather than share the short circuit below.
|
|
2604
|
+
entry.ForceSaveOnce = true
|
|
2605
|
+
elseif isDurable(entry) then
|
|
2606
|
+
-- Already on disk: answer honestly and spend nothing, since a save here would
|
|
2607
|
+
-- be a full UpdateAsync round trip writing bytes that are already there. This
|
|
2608
|
+
-- is the only safe part of "dirty-aware saving", because it removes a write
|
|
2609
|
+
-- rather than deferring one. Skipping AUTOSAVES is a different thing wearing
|
|
2610
|
+
-- the same words: the autosave also refreshes MetaData.LastUpdate, which
|
|
2611
|
+
-- another server reads to decide whether this session belongs to a crashed
|
|
2612
|
+
-- machine, and it is what delivers cross-server messages and gifts.
|
|
2613
|
+
Metrics.Add("FlushesAlreadyDurable")
|
|
2614
|
+
return true
|
|
2615
|
+
end
|
|
2616
|
+
return self.AwaitSave(entry, (opts and opts.Timeout) or 15)
|
|
2617
|
+
end
|
|
2618
|
+
|
|
2619
|
+
-- Save now, and report what happened to THIS save. The store answers for itself rather
|
|
2620
|
+
-- than correlating completions with a generation counter, which regularly answered a
|
|
2621
|
+
-- waiter whose write the completing save had never serialized. Monetization returns
|
|
2622
|
+
-- PurchaseGranted on this boolean.
|
|
2623
|
+
function self.AwaitSave(entry: Entry, timeout: number): boolean
|
|
2624
|
+
local profile = entry.Profile
|
|
2625
|
+
if not profile or not profile:IsActive() then
|
|
2626
|
+
return false
|
|
2627
|
+
end
|
|
2628
|
+
local thread = coroutine.running()
|
|
2629
|
+
-- Claimed synchronously by whichever path arrives first, so a completion landing
|
|
2630
|
+
-- after this caller has given up cannot answer it a second time: by then the thread
|
|
2631
|
+
-- is usually parked on something else, and a stray resume would hand it this save's
|
|
2632
|
+
-- boolean in place of the value it asked for.
|
|
2633
|
+
local settled = false
|
|
2634
|
+
-- The verdict, left where the caller can READ it rather than be woken with it.
|
|
2635
|
+
-- With a synchronous store (the fake in tests, a fast final save in production)
|
|
2636
|
+
-- SaveAwait returns on this caller's own stack, before it has reached its yield.
|
|
2637
|
+
-- Deferring the wake until the caller parked was the bug: Sched.Resume is
|
|
2638
|
+
-- task.spawn, so a thread woken from inside a deferred callback inherits that
|
|
2639
|
+
-- callback's re-entrancy generation and climbs one rung per AwaitSave round trip.
|
|
2640
|
+
-- At 80, task.defer silently DROPS the callback while still returning success, so
|
|
2641
|
+
-- the waiter hangs for the life of the server. A thread keeps the rung it climbed
|
|
2642
|
+
-- until its own next task.wait, so only a run of wakes with no wait compounds.
|
|
2643
|
+
local result: boolean? = nil
|
|
2644
|
+
-- Hand the verdict to whichever state the caller is in. Only the path that
|
|
2645
|
+
-- claimed `settled` gets here, so the two writes cannot race.
|
|
2646
|
+
local function answer(ok: boolean)
|
|
2647
|
+
result = ok
|
|
2648
|
+
-- "suspended" is the only status that may be resumed. "normal" is the
|
|
2649
|
+
-- synchronous store above and "running" is a Clock.Delay firing on this very
|
|
2650
|
+
-- stack; both are answered by `result`, and resuming either would error.
|
|
2651
|
+
if coroutine.status(thread) == "suspended" then
|
|
2652
|
+
Sched.Resume(thread, ok)
|
|
2653
|
+
end
|
|
2654
|
+
end
|
|
2655
|
+
|
|
2656
|
+
Sched.Spawn(function()
|
|
2657
|
+
local outcome, saveContext = profile:SaveAwait()
|
|
2658
|
+
if settled then
|
|
2659
|
+
return
|
|
2660
|
+
end
|
|
2661
|
+
settled = true
|
|
2662
|
+
-- "Saved" is the only outcome that puts these bytes on disk, and even then not
|
|
2663
|
+
-- if the wipe guard exchanged in the last-good snapshot, nor if this was a
|
|
2664
|
+
-- ReleaseOnly write: a fail-closed release writes MetaData and deliberately
|
|
2665
|
+
-- leaves the stored bytes untouched. All three terms are per-save rather than
|
|
2666
|
+
-- read back off shared entry state.
|
|
2667
|
+
answer(outcome == "Saved" and not blockedForSave(entry, saveContext) and saveContext.ReleaseOnly ~= true)
|
|
2668
|
+
end)
|
|
2669
|
+
|
|
2670
|
+
Clock.Delay(timeout, function()
|
|
2671
|
+
if settled then
|
|
2672
|
+
return
|
|
2673
|
+
end
|
|
2674
|
+
settled = true
|
|
2675
|
+
-- Unresolved, not failed: the save is very often still in flight and lands
|
|
2676
|
+
-- seconds later, so it is deliberately not counted as SavesFailed.
|
|
2677
|
+
Metrics.Add("SaveWaitTimeouts")
|
|
2678
|
+
Log.Warn(
|
|
2679
|
+
"Persistence",
|
|
2680
|
+
"SAVE_WAIT_TIMEOUT",
|
|
2681
|
+
`gave up waiting {timeout}s for a save to confirm; it may still be in flight, so this is unresolved rather than failed`,
|
|
2682
|
+
{ Player = entry.Player, Timeout = timeout }
|
|
2683
|
+
)
|
|
2684
|
+
answer(false)
|
|
2685
|
+
end)
|
|
2686
|
+
|
|
2687
|
+
-- Answered without this thread ever parking: the store settled inline, or an inline
|
|
2688
|
+
-- Clock.Delay timed the wait out on this stack. Nothing can run between Sched.Spawn
|
|
2689
|
+
-- returning and this test, so reading `result` is not a race. Tested against nil
|
|
2690
|
+
-- rather than truthiness: a `false` verdict is an ANSWER, and falling through to
|
|
2691
|
+
-- the yield would park a caller both wakers have retired.
|
|
2692
|
+
if result ~= nil then
|
|
2693
|
+
return result
|
|
2694
|
+
end
|
|
2695
|
+
return coroutine.yield() == true
|
|
2696
|
+
end
|
|
2697
|
+
|
|
2698
|
+
function self.GetSaveInfo(
|
|
2699
|
+
player: Player
|
|
2700
|
+
): { LastSaveAt: number?, LastResult: ("Ok" | "Fail")?, Dirty: boolean, Size: number? }
|
|
2701
|
+
local entry = self.Entries[player]
|
|
2702
|
+
if not entry then
|
|
2703
|
+
return { LastSaveAt = nil, LastResult = nil, Dirty = false, Size = nil }
|
|
2704
|
+
end
|
|
2705
|
+
local info = table.clone(entry.SaveInfo)
|
|
2706
|
+
info.Size = entry.LastSavedSize
|
|
2707
|
+
return info
|
|
2708
|
+
end
|
|
2709
|
+
|
|
2710
|
+
-- Offline access ----------------------------------------------------------
|
|
2711
|
+
|
|
2712
|
+
-- Derived fields are computed, so stored data lacks them. Both branches of GetOffline
|
|
2713
|
+
-- fill in the root-level fields whose every input persists (Derived.OfflineComputable);
|
|
2714
|
+
-- anything reading a Session field is left out rather than invented.
|
|
2715
|
+
local function deriveOffline(snapshot: { [string]: any })
|
|
2716
|
+
if #compiled.DerivedNodes == 0 then
|
|
2717
|
+
return
|
|
2718
|
+
end
|
|
2719
|
+
DerivedModule.EvaluateInto(compiled, snapshot, function(node)
|
|
2720
|
+
return node.Derived.OfflineComputable
|
|
2721
|
+
end)
|
|
2722
|
+
end
|
|
2723
|
+
|
|
2724
|
+
function self.GetOffline(userId: number): { [string]: any }?
|
|
2725
|
+
local entry = self.EntryForUserId(userId)
|
|
2726
|
+
if entry and entry.State == "Ready" and entry.LiveData then
|
|
2727
|
+
local live = Util.deepClone(entry.LiveData)
|
|
2728
|
+
deriveOffline(live)
|
|
2729
|
+
return live
|
|
2730
|
+
end
|
|
2731
|
+
local ok, profile = pcall(function()
|
|
2732
|
+
return store:GetAsync(self.KeyFor(userId))
|
|
2733
|
+
end)
|
|
2734
|
+
if not ok then
|
|
2735
|
+
-- Distinguish a read error (retryable) from a missing profile
|
|
2736
|
+
-- (returns nil below with no failure report).
|
|
2737
|
+
Failure.Report({
|
|
2738
|
+
Subsystem = "OfflineRead",
|
|
2739
|
+
Key = tostring(userId),
|
|
2740
|
+
Error = profile,
|
|
2741
|
+
Category = "Persistence",
|
|
2742
|
+
Code = "OFFLINE_READ_FAIL",
|
|
2743
|
+
Message = tostring(profile),
|
|
2744
|
+
Context = { UserId = userId },
|
|
2745
|
+
})
|
|
2746
|
+
return nil
|
|
2747
|
+
end
|
|
2748
|
+
Health.ReportSuccess("OfflineRead")
|
|
2749
|
+
if not profile then
|
|
2750
|
+
return nil
|
|
2751
|
+
end
|
|
2752
|
+
-- A DataStore round trip turns a Scribe.MapOf("integer") key into its string
|
|
2753
|
+
-- form, and only the session load undid that, so an offline reader saw the raw
|
|
2754
|
+
-- strings. Coerce the copy, not the store's table.
|
|
2755
|
+
local snapshot = Util.deepClone(profile.Data)
|
|
2756
|
+
Template.CoerceMapKeys(compiled, snapshot)
|
|
2757
|
+
deriveOffline(snapshot)
|
|
2758
|
+
return snapshot
|
|
2759
|
+
end
|
|
2760
|
+
|
|
2761
|
+
-- Serialize same-user offline read-modify-writes on THIS server. A view-mode SetAsync
|
|
2762
|
+
-- writes the whole GetAsync snapshot back, so two interleaved UpdateOffline cycles for
|
|
2763
|
+
-- one userId have the second clobber the first, silently dropping a grant and its
|
|
2764
|
+
-- dedupe marker while both report success. Concurrent offline writes from a DIFFERENT
|
|
2765
|
+
-- server remain a view-mode limitation.
|
|
2766
|
+
local offlineLocks: { [number]: { thread } } = {}
|
|
2767
|
+
local function acquireOffline(userId: number)
|
|
2768
|
+
local waiters = offlineLocks[userId]
|
|
2769
|
+
if waiters == nil then
|
|
2770
|
+
offlineLocks[userId] = {} -- lock now held, no waiters
|
|
2771
|
+
return
|
|
2772
|
+
end
|
|
2773
|
+
table.insert(waiters, coroutine.running())
|
|
2774
|
+
coroutine.yield() -- resumed by releaseOffline when it is our turn
|
|
2775
|
+
end
|
|
2776
|
+
local function releaseOffline(userId: number)
|
|
2777
|
+
local waiters = offlineLocks[userId]
|
|
2778
|
+
if waiters and #waiters > 0 then
|
|
2779
|
+
Sched.Resume(table.remove(waiters, 1) :: thread) -- next waiter takes the lock
|
|
2780
|
+
else
|
|
2781
|
+
offlineLocks[userId] = nil
|
|
2782
|
+
end
|
|
2783
|
+
end
|
|
2784
|
+
|
|
2785
|
+
--- Turn an UpdateOfflineAsync outcome into Scribe's (ok, reason, dataStoreFailed).
|
|
2786
|
+
--- One table for updateOfflineProfile and RestoreVersion, so the two cannot come to
|
|
2787
|
+
--- disagree about which outcomes are infrastructure failures. Only "Failed" is.
|
|
2788
|
+
local function mapOfflineOutcome(outcome: string?, what: string): (boolean, string?, boolean)
|
|
2789
|
+
if outcome == "Written" then
|
|
2790
|
+
return true, nil, false
|
|
2791
|
+
elseif outcome == "SessionActive" then
|
|
2792
|
+
return false, "profile has an active session on another server", false
|
|
2793
|
+
elseif outcome == "Changed" then
|
|
2794
|
+
-- Somebody else wrote between the read and here. Nothing was written, so a
|
|
2795
|
+
-- caller that cares can simply try again against fresh data.
|
|
2796
|
+
return false, `profile changed while the {what} was being prepared`, false
|
|
2797
|
+
elseif outcome == "Missing" then
|
|
2798
|
+
return false, "profile does not exist", false
|
|
2799
|
+
end
|
|
2800
|
+
-- "Failed": the DataStore call itself did not land. Fail closed; an offline write
|
|
2801
|
+
-- we cannot stand behind is what the receipt path turns into a granted-but-unsaved
|
|
2802
|
+
-- purchase.
|
|
2803
|
+
return false, `offline {what} did not complete`, true
|
|
2804
|
+
end
|
|
2805
|
+
|
|
2806
|
+
-- Applies `mutate` to an offline profile and saves it, returning
|
|
2807
|
+
-- (ok, reason, dataStoreFailed). Only a failing GetAsync/SetAsync sets the third: a
|
|
2808
|
+
-- precondition refusal or a throwing callback is a LOGIC error, and reporting those to
|
|
2809
|
+
-- Health would let ordinary refusals flip the server to Outage. `mutate` runs against
|
|
2810
|
+
-- a CLONE, so a callback that errors partway commits nothing. The active-session check
|
|
2811
|
+
-- Validate judges the change, not the profile; at the cap the two scans cannot be compared.
|
|
2812
|
+
local VALIDATE_LIMIT = 256
|
|
2813
|
+
local TOO_MANY_VIOLATIONS =
|
|
2814
|
+
`the profile already carries {VALIDATE_LIMIT} or more template violations, too many for validation to tell the change apart; repair it or write raw`
|
|
2815
|
+
|
|
2816
|
+
-- Length-prefixed, so a dotted key and a nested path differ.
|
|
2817
|
+
local function violationKey(violation: Conform.Violation): string
|
|
2818
|
+
local parts = table.create(#violation.Path + 1)
|
|
2819
|
+
for i, segment in violation.Path do
|
|
2820
|
+
parts[i] = if type(segment) == "number" then `#{segment}` else `{#segment}:{segment}`
|
|
2821
|
+
end
|
|
2822
|
+
parts[#parts + 1] = violation.Kind
|
|
2823
|
+
return table.concat(parts, "/")
|
|
2824
|
+
end
|
|
2825
|
+
|
|
2826
|
+
local function readPath(data: any, path: { string | number }): any
|
|
2827
|
+
local value = data
|
|
2828
|
+
for _, segment in path do
|
|
2829
|
+
if type(value) ~= "table" then
|
|
2830
|
+
return nil
|
|
2831
|
+
end
|
|
2832
|
+
value = value[segment]
|
|
2833
|
+
end
|
|
2834
|
+
return value
|
|
2835
|
+
end
|
|
2836
|
+
|
|
2837
|
+
local function schemaAt(path: { string | number }): Template.SchemaNode?
|
|
2838
|
+
local node: Template.SchemaNode? = compiled.Root
|
|
2839
|
+
for _, segment in path do
|
|
2840
|
+
if node == nil then
|
|
2841
|
+
return nil
|
|
2842
|
+
end
|
|
2843
|
+
node = Template.ChildSchema(node, segment)
|
|
2844
|
+
end
|
|
2845
|
+
return node
|
|
2846
|
+
end
|
|
2847
|
+
|
|
2848
|
+
-- A set counts distinct members, as its cap does.
|
|
2849
|
+
local function entryCount(value: any, node: Template.SchemaNode?): number
|
|
2850
|
+
local n = 0
|
|
2851
|
+
if node ~= nil and node.Unique then
|
|
2852
|
+
local seen: { [any]: boolean } = {}
|
|
2853
|
+
for _, item in value :: { any } do
|
|
2854
|
+
if not seen[item] then
|
|
2855
|
+
seen[item] = true
|
|
2856
|
+
n += 1
|
|
2857
|
+
end
|
|
2858
|
+
end
|
|
2859
|
+
return n
|
|
2860
|
+
end
|
|
2861
|
+
for _ in value :: { [any]: any } do
|
|
2862
|
+
n += 1
|
|
2863
|
+
end
|
|
2864
|
+
return n
|
|
2865
|
+
end
|
|
2866
|
+
|
|
2867
|
+
-- An over-cap container keeps its size (edited, not grown); everything else a copy of its value.
|
|
2868
|
+
local function violationFootprint(draft: { [string]: any }, violation: Conform.Violation): any
|
|
2869
|
+
local value = readPath(draft, violation.Path)
|
|
2870
|
+
if violation.Kind == "OverMaxLength" and type(value) == "table" then
|
|
2871
|
+
return { Count = entryCount(value, schemaAt(violation.Path)) }
|
|
2872
|
+
end
|
|
2873
|
+
return { Value = Util.deepClone(value) }
|
|
2874
|
+
end
|
|
2875
|
+
|
|
2876
|
+
local function stillTheSame(before: any, after: any): boolean
|
|
2877
|
+
if before.Count ~= nil then
|
|
2878
|
+
return after.Count ~= nil and after.Count <= before.Count
|
|
2879
|
+
end
|
|
2880
|
+
return after.Count == nil and deepSame(before.Value, after.Value)
|
|
2881
|
+
end
|
|
2882
|
+
|
|
2883
|
+
-- Before the callback: an old-shape profile would report every difference as its doing.
|
|
2884
|
+
local function offlineVersionRefusal(draft: { [string]: any }): string?
|
|
2885
|
+
local reserved = draft[Template.RESERVED_ROOT]
|
|
2886
|
+
local stored = if type(reserved) == "table" and type(reserved.Version) == "number" then reserved.Version else 1
|
|
2887
|
+
if stored < currentVersion then
|
|
2888
|
+
return `profile is at schema version {stored} and this build is {currentVersion}; load it in a session to migrate it first`
|
|
2889
|
+
elseif stored > currentVersion then
|
|
2890
|
+
return `profile is at schema version {stored}, newer than this build's {currentVersion}; update the place before editing it`
|
|
2891
|
+
end
|
|
2892
|
+
return nil
|
|
2893
|
+
end
|
|
2894
|
+
|
|
2895
|
+
type OfflineBaseline = {
|
|
2896
|
+
Violations: { [string]: any },
|
|
2897
|
+
Full: boolean,
|
|
2898
|
+
Reserved: any,
|
|
2899
|
+
Derived: { [string]: any },
|
|
2900
|
+
Session: { [string]: any },
|
|
2901
|
+
}
|
|
2902
|
+
|
|
2903
|
+
local function offlineBaseline(draft: { [string]: any }): OfflineBaseline
|
|
2904
|
+
local derived = {}
|
|
2905
|
+
for _, node in compiled.DerivedNodes do
|
|
2906
|
+
if #node.Path == 1 then
|
|
2907
|
+
derived[node.Path[1]] = draft[node.Path[1]]
|
|
2908
|
+
end
|
|
2909
|
+
end
|
|
2910
|
+
local session = {}
|
|
2911
|
+
for name, child in compiled.Root.Children or {} do
|
|
2912
|
+
if not child.Persist then
|
|
2913
|
+
session[name] = { Value = Util.deepClone(draft[name]) }
|
|
2914
|
+
end
|
|
2915
|
+
end
|
|
2916
|
+
local found = Conform.Check(compiled, draft, { Limit = VALIDATE_LIMIT })
|
|
2917
|
+
local violations = {}
|
|
2918
|
+
for _, violation in found do
|
|
2919
|
+
violations[violationKey(violation)] = violationFootprint(draft, violation)
|
|
2920
|
+
end
|
|
2921
|
+
return {
|
|
2922
|
+
Violations = violations,
|
|
2923
|
+
Full = #found >= VALIDATE_LIMIT,
|
|
2924
|
+
Reserved = Util.deepClone(draft[Template.RESERVED_ROOT]),
|
|
2925
|
+
Derived = derived,
|
|
2926
|
+
Session = session,
|
|
2927
|
+
}
|
|
2928
|
+
end
|
|
2929
|
+
|
|
2930
|
+
local function offlineChangeRefusal(
|
|
2931
|
+
draft: { [string]: any },
|
|
2932
|
+
baseline: OfflineBaseline
|
|
2933
|
+
): (string?, { Conform.Violation }?)
|
|
2934
|
+
if baseline.Full then
|
|
2935
|
+
return TOO_MANY_VIOLATIONS, nil
|
|
2936
|
+
end
|
|
2937
|
+
if not deepSame(draft[Template.RESERVED_ROOT], baseline.Reserved) then
|
|
2938
|
+
return `the change touches "{Template.RESERVED_ROOT}", which Scribe owns and an offline edit may not change`,
|
|
2939
|
+
nil
|
|
2940
|
+
end
|
|
2941
|
+
for name, was in baseline.Derived do
|
|
2942
|
+
if draft[name] ~= was then
|
|
2943
|
+
return `the change writes "{name}", a derived field that is computed and never stored`, nil
|
|
2944
|
+
end
|
|
2945
|
+
end
|
|
2946
|
+
for _, node in compiled.DerivedNodes do
|
|
2947
|
+
if #node.Path == 1 and baseline.Derived[node.Path[1]] == nil and draft[node.Path[1]] ~= nil then
|
|
2948
|
+
return `the change writes "{node.Path[1]}", a derived field that is computed and never stored`, nil
|
|
2949
|
+
end
|
|
2950
|
+
end
|
|
2951
|
+
for name, before in baseline.Session do
|
|
2952
|
+
if not deepSame(draft[name], before.Value) then
|
|
2953
|
+
return `the change writes "{name}", a Session field that is never stored`, nil
|
|
2954
|
+
end
|
|
2955
|
+
end
|
|
2956
|
+
local found = Conform.Check(compiled, draft, { Limit = VALIDATE_LIMIT })
|
|
2957
|
+
if #found >= VALIDATE_LIMIT then
|
|
2958
|
+
return TOO_MANY_VIOLATIONS, nil
|
|
2959
|
+
end
|
|
2960
|
+
local introduced = {}
|
|
2961
|
+
for _, violation in found do
|
|
2962
|
+
local before = baseline.Violations[violationKey(violation)]
|
|
2963
|
+
if before == nil or not stillTheSame(before, violationFootprint(draft, violation)) then
|
|
2964
|
+
table.insert(introduced, violation)
|
|
2965
|
+
end
|
|
2966
|
+
end
|
|
2967
|
+
if #introduced > 0 then
|
|
2968
|
+
local first = introduced[1]
|
|
2969
|
+
return `the change does not match the template at "{Util.pathString(first.Path)}": {first.Detail}`,
|
|
2970
|
+
introduced
|
|
2971
|
+
end
|
|
2972
|
+
return nil, nil
|
|
2973
|
+
end
|
|
2974
|
+
|
|
2975
|
+
-- is a pure read, deliberately: ProfileStore has no non-stealing acquire, and the
|
|
2976
|
+
-- first StartSessionAsync attempt writes ForceLoadSession and kicks an online player
|
|
2977
|
+
-- off another server. Losing the read-then-write race costs one clobbered write;
|
|
2978
|
+
-- closing it that way costs evicted players and eaten Robux.
|
|
2979
|
+
local function updateOfflineProfile(
|
|
2980
|
+
userId: number,
|
|
2981
|
+
mutate: (data: { [string]: any }) -> (),
|
|
2982
|
+
validate: boolean,
|
|
2983
|
+
createIfMissing: boolean?
|
|
2984
|
+
): (boolean, string?, boolean, { Conform.Violation }?)
|
|
2985
|
+
local readOk, profile = pcall(function()
|
|
2986
|
+
return store:GetAsync(self.KeyFor(userId))
|
|
2987
|
+
end)
|
|
2988
|
+
if not readOk then
|
|
2989
|
+
return false, tostring(profile), true
|
|
2990
|
+
end
|
|
2991
|
+
if not profile then
|
|
2992
|
+
if not createIfMissing then
|
|
2993
|
+
return false, NO_PROFILE, false
|
|
2994
|
+
end
|
|
2995
|
+
elseif sessionLooksLive(profile) then
|
|
2996
|
+
-- A read, not an acquire: StartSessionAsync writes ForceLoadSession and would kick
|
|
2997
|
+
-- a live player, and EndSession only task.spawns its save, so acquiring would also
|
|
2998
|
+
-- cost this path its promise that the write landed.
|
|
2999
|
+
return false, LIVE_ELSEWHERE, false
|
|
3000
|
+
end
|
|
3001
|
+
|
|
3002
|
+
local draft = if profile then Util.deepClone(profile.Data) else {}
|
|
3003
|
+
-- Same integer-key coercion the session load does, so the callback sees the keys
|
|
3004
|
+
-- its template declares rather than the DataStore's string forms. The commit
|
|
3005
|
+
-- writes them back as numbers, which the store re-encodes as for a live save.
|
|
3006
|
+
Template.CoerceMapKeys(compiled, draft)
|
|
3007
|
+
local baseline: OfflineBaseline? = nil
|
|
3008
|
+
if validate then
|
|
3009
|
+
local refusal = offlineVersionRefusal(draft)
|
|
3010
|
+
if refusal ~= nil then
|
|
3011
|
+
return false, refusal, false
|
|
3012
|
+
end
|
|
3013
|
+
baseline = offlineBaseline(draft)
|
|
3014
|
+
end
|
|
3015
|
+
local mutateOk, mutateErr = pcall(mutate, draft)
|
|
3016
|
+
if not mutateOk then
|
|
3017
|
+
return false, tostring(mutateErr), false
|
|
3018
|
+
end
|
|
3019
|
+
-- The callback mutates raw data, bypassing Node.validateWrite, so unpersistable
|
|
3020
|
+
-- data (invalid UTF-8, a userdata/Instance value) would fail the whole profile
|
|
3021
|
+
-- save opaquely. Reject before committing.
|
|
3022
|
+
local bad = Util.findUnpersistable(draft)
|
|
3023
|
+
if bad ~= nil then
|
|
3024
|
+
return false, `unpersistable data ({bad.Detail}) at "{Util.pathString(bad.Path)}"`, false
|
|
3025
|
+
end
|
|
3026
|
+
if baseline ~= nil then
|
|
3027
|
+
local refusal, introduced = offlineChangeRefusal(draft, baseline)
|
|
3028
|
+
if refusal ~= nil then
|
|
3029
|
+
return false, refusal, false, introduced
|
|
3030
|
+
end
|
|
3031
|
+
end
|
|
3032
|
+
|
|
3033
|
+
-- Commit conditionally, in ONE call. `mutate` above is developer code and may yield
|
|
3034
|
+
-- for as long as it likes, so a player can join another server inside that window.
|
|
3035
|
+
-- Getting it wrong is worse than a lost write: a view SetAsync sets
|
|
3036
|
+
-- session_owns_profile and clears ActiveSession, so committing over a live session
|
|
3037
|
+
-- DISOWNS the server holding the key, and everything it wrote since its last save
|
|
3038
|
+
-- is gone. The compare-and-set makes the check and the write one DataStore call.
|
|
3039
|
+
if type((store :: any).UpdateOfflineAsync) == "function" then
|
|
3040
|
+
-- LastUpdate has one-second granularity and a view-mode SetAsync moves neither
|
|
3041
|
+
-- counter, hence WriteIndex; a recreated key can repeat all three, hence Generation.
|
|
3042
|
+
-- An empty fingerprint matches no stored key.
|
|
3043
|
+
local expected = if profile
|
|
3044
|
+
then {
|
|
3045
|
+
SessionLoadCount = profile.SessionLoadCount,
|
|
3046
|
+
LastUpdate = profile.LastUpdate,
|
|
3047
|
+
WriteIndex = profile.WriteIndex,
|
|
3048
|
+
Generation = profile.Generation,
|
|
3049
|
+
}
|
|
3050
|
+
else {}
|
|
3051
|
+
local casOk, outcome = pcall(function()
|
|
3052
|
+
return (store :: any):UpdateOfflineAsync(self.KeyFor(userId), expected, draft, {
|
|
3053
|
+
-- The same dead-session age sessionLooksLive uses, and the one
|
|
3054
|
+
-- StartSessionAsync steals at. Without it the CAS refuses on
|
|
3055
|
+
-- ActiveSession ~= nil at ANY age, so a crashed server locks
|
|
3056
|
+
-- UpdateOffline out forever.
|
|
3057
|
+
StaleSessionAfter = ASSUME_DEAD_SECONDS,
|
|
3058
|
+
-- Only when the read found nothing, so a key erased since stays erased.
|
|
3059
|
+
CreateIfMissing = createIfMissing == true and profile == nil,
|
|
3060
|
+
})
|
|
3061
|
+
end)
|
|
3062
|
+
if not casOk then
|
|
3063
|
+
return false, tostring(outcome), true
|
|
3064
|
+
end
|
|
3065
|
+
return mapOfflineOutcome(outcome, "update")
|
|
3066
|
+
end
|
|
3067
|
+
|
|
3068
|
+
-- Fallback for an injected store without the compare-and-set: re-read, then
|
|
3069
|
+
-- write, with the window that leaves. It cannot create a key.
|
|
3070
|
+
if profile == nil then
|
|
3071
|
+
return false, NO_PROFILE, false
|
|
3072
|
+
end
|
|
3073
|
+
local recheckOk, latest = pcall(function()
|
|
3074
|
+
return store:GetAsync(self.KeyFor(userId))
|
|
3075
|
+
end)
|
|
3076
|
+
if not recheckOk then
|
|
3077
|
+
return false, tostring(latest), true
|
|
3078
|
+
end
|
|
3079
|
+
if sessionLooksLive(latest) then
|
|
3080
|
+
return false, "profile has an active session on another server", false
|
|
3081
|
+
end
|
|
3082
|
+
|
|
3083
|
+
table.clear(profile.Data)
|
|
3084
|
+
for key, value in draft do
|
|
3085
|
+
profile.Data[key] = value
|
|
3086
|
+
end
|
|
3087
|
+
local saveOk, saveErr = commitViewProfile(profile, self.KeyFor(userId))
|
|
3088
|
+
if not saveOk then
|
|
3089
|
+
return false, saveErr, true
|
|
3090
|
+
end
|
|
3091
|
+
return true, nil, false
|
|
3092
|
+
end
|
|
3093
|
+
|
|
3094
|
+
function self.UpdateOffline(
|
|
3095
|
+
userId: number,
|
|
3096
|
+
fn: (data: { [string]: any }) -> (),
|
|
3097
|
+
opts: { Validate: boolean? }?
|
|
3098
|
+
): (boolean, string?, { Conform.Violation }?)
|
|
3099
|
+
if mode == "NoSave" then
|
|
3100
|
+
return false, refuseNoSave("UpdateOffline")
|
|
3101
|
+
end
|
|
3102
|
+
if self.EntryForUserId(userId) then
|
|
3103
|
+
return false, "user has an active session on this server"
|
|
3104
|
+
end
|
|
3105
|
+
local validate = opts ~= nil and opts.Validate == true
|
|
3106
|
+
acquireOffline(userId)
|
|
3107
|
+
-- Still pcalled so an unexpected internal error cannot skip releaseOffline;
|
|
3108
|
+
-- such an error is not a DataStore failure either, so it does not report.
|
|
3109
|
+
local callOk, ok, reason, dataStoreFailed, violations = pcall(updateOfflineProfile, userId, fn, validate)
|
|
3110
|
+
releaseOffline(userId)
|
|
3111
|
+
if not callOk then
|
|
3112
|
+
return false, tostring(ok)
|
|
3113
|
+
end
|
|
3114
|
+
if not ok then
|
|
3115
|
+
if dataStoreFailed then
|
|
3116
|
+
-- Only a DataStore failure reports. A refusal (a live session, a changed
|
|
3117
|
+
-- key, a missing profile) is the compare-and-set working, and counting
|
|
3118
|
+
-- it against Health would make ordinary contention look like an outage.
|
|
3119
|
+
Failure.Report({
|
|
3120
|
+
Subsystem = "OfflineWrite",
|
|
3121
|
+
Key = tostring(userId),
|
|
3122
|
+
Error = reason,
|
|
3123
|
+
Category = "Persistence",
|
|
3124
|
+
Code = "OFFLINE_WRITE_FAIL",
|
|
3125
|
+
Message = tostring(reason),
|
|
3126
|
+
Context = { UserId = userId },
|
|
3127
|
+
})
|
|
3128
|
+
end
|
|
3129
|
+
return false, reason, violations
|
|
3130
|
+
end
|
|
3131
|
+
Health.ReportSuccess("OfflineWrite")
|
|
3132
|
+
return true, nil
|
|
3133
|
+
end
|
|
3134
|
+
|
|
3135
|
+
-- Version history and restore -----------------------------------------------
|
|
3136
|
+
|
|
3137
|
+
function self.ListVersions(
|
|
3138
|
+
userId: number,
|
|
3139
|
+
limit: number?
|
|
3140
|
+
): { { VersionId: string, CreatedAt: number, Size: number? } }
|
|
3141
|
+
local out = {}
|
|
3142
|
+
local ok, err = pcall(function()
|
|
3143
|
+
local query = store:VersionQuery(self.KeyFor(userId), Enum.SortDirection.Descending)
|
|
3144
|
+
for _ = 1, limit or 25 do
|
|
3145
|
+
local profile = query:NextAsync()
|
|
3146
|
+
if not profile then
|
|
3147
|
+
break
|
|
3148
|
+
end
|
|
3149
|
+
table.insert(out, {
|
|
3150
|
+
VersionId = profile.KeyInfo.Version,
|
|
3151
|
+
CreatedAt = profile.KeyInfo.CreatedTime / 1000,
|
|
3152
|
+
Size = Util.approxSize(profile.Data),
|
|
3153
|
+
})
|
|
3154
|
+
end
|
|
3155
|
+
end)
|
|
3156
|
+
if not ok then
|
|
3157
|
+
Log.Warn("Persistence", "VERSION_QUERY_FAIL", tostring(err), { UserId = userId })
|
|
3158
|
+
end
|
|
3159
|
+
return out
|
|
3160
|
+
end
|
|
3161
|
+
|
|
3162
|
+
function self.GetVersion(userId: number, versionId: string): { [string]: any }?
|
|
3163
|
+
local ok, profile = pcall(function()
|
|
3164
|
+
return store:GetAsync(self.KeyFor(userId), versionId)
|
|
3165
|
+
end)
|
|
3166
|
+
if not ok then
|
|
3167
|
+
-- A read ERROR is retryable and must not be mistaken for a missing
|
|
3168
|
+
-- version during a manual restore.
|
|
3169
|
+
Failure.Report({
|
|
3170
|
+
Subsystem = "VersionRead",
|
|
3171
|
+
Key = tostring(userId),
|
|
3172
|
+
Error = profile,
|
|
3173
|
+
Category = "Persistence",
|
|
3174
|
+
Code = "VERSION_READ_FAIL",
|
|
3175
|
+
Message = tostring(profile),
|
|
3176
|
+
Context = { UserId = userId, VersionId = versionId },
|
|
3177
|
+
})
|
|
3178
|
+
return nil
|
|
3179
|
+
end
|
|
3180
|
+
Health.ReportSuccess("VersionRead")
|
|
3181
|
+
if not profile then
|
|
3182
|
+
return nil -- version genuinely not found
|
|
3183
|
+
end
|
|
3184
|
+
-- Same integer-key coercion and derived fill as GetOffline: this is a raw
|
|
3185
|
+
-- DataStore read, so it carries stored fields only.
|
|
3186
|
+
local snapshot = Util.deepClone(profile.Data)
|
|
3187
|
+
Template.CoerceMapKeys(compiled, snapshot)
|
|
3188
|
+
deriveOffline(snapshot)
|
|
3189
|
+
return snapshot
|
|
3190
|
+
end
|
|
3191
|
+
|
|
3192
|
+
function self.RestoreVersion(
|
|
3193
|
+
userId: number,
|
|
3194
|
+
versionId: string,
|
|
3195
|
+
opts: { RollBackReserved: boolean? }?
|
|
3196
|
+
): (boolean, string?)
|
|
3197
|
+
if mode == "NoSave" then
|
|
3198
|
+
return false, refuseNoSave("RestoreVersion")
|
|
3199
|
+
end
|
|
3200
|
+
if Health.GetStatus() == "Outage" then
|
|
3201
|
+
return false, "data services are experiencing an outage"
|
|
3202
|
+
end
|
|
3203
|
+
if self.EntryForUserId(userId) then
|
|
3204
|
+
return false, "user has an active session on this server; restore requires no active session"
|
|
3205
|
+
end
|
|
3206
|
+
-- Both read by the logging below, so they live outside the pcall. `differed` is set
|
|
3207
|
+
-- only on a path that goes on to commit, so a failed restore logs nothing about a
|
|
3208
|
+
-- preservation that did not happen.
|
|
3209
|
+
local reservedRolledBack = opts ~= nil and opts.RollBackReserved == true
|
|
3210
|
+
local reservedDiffered = false
|
|
3211
|
+
-- The schema version the restore COMMITTED, and the one it replaced. Only set
|
|
3212
|
+
-- on the preserve path, and only used by the log below.
|
|
3213
|
+
local restoredSchemaVersion: number? = nil
|
|
3214
|
+
local previousSchemaVersion: number? = nil
|
|
3215
|
+
local ok, err = pcall(function()
|
|
3216
|
+
local live = store:GetAsync(self.KeyFor(userId))
|
|
3217
|
+
if sessionLooksLive(live) then
|
|
3218
|
+
error("profile has an active session on another server")
|
|
3219
|
+
end
|
|
3220
|
+
if not live then
|
|
3221
|
+
-- No live key to restore over (restoring after an Erase hits this).
|
|
3222
|
+
-- Profile:SetAsync onto an empty key throws inside ProfileStore's own
|
|
3223
|
+
-- transform and that error is swallowed, so we would report a successful
|
|
3224
|
+
-- restore for a write that never landed.
|
|
3225
|
+
error("no live profile exists for this user to restore over (was it erased?)")
|
|
3226
|
+
end
|
|
3227
|
+
-- A restore rolls GAME DATA back and deliberately preserves the live reserved
|
|
3228
|
+
-- root, which is right for receipts and perks and wrong in both directions
|
|
3229
|
+
-- for an exchange. Free to check: `live` is the read already taken above.
|
|
3230
|
+
local staked, parked = exchangeStateOf(live.Data)
|
|
3231
|
+
if staked > 0 or parked > 0 then
|
|
3232
|
+
error(
|
|
3233
|
+
`profile holds {staked} in-flight and {parked} undelivered exchange(s); a restore would `
|
|
3234
|
+
.. `roll game data back while the exchange ledger stayed live, duplicating a stake or `
|
|
3235
|
+
.. `destroying a delivery. Resolve them first`
|
|
3236
|
+
)
|
|
3237
|
+
end
|
|
3238
|
+
-- The fingerprint, taken from the SAME read that just cleared the liveness
|
|
3239
|
+
-- check. Everything below is preparation; the commit compares against this.
|
|
3240
|
+
local expected = {
|
|
3241
|
+
SessionLoadCount = live.SessionLoadCount,
|
|
3242
|
+
LastUpdate = live.LastUpdate,
|
|
3243
|
+
WriteIndex = live.WriteIndex,
|
|
3244
|
+
Generation = live.Generation,
|
|
3245
|
+
}
|
|
3246
|
+
local versionProfile = store:GetAsync(self.KeyFor(userId), versionId)
|
|
3247
|
+
if not versionProfile then
|
|
3248
|
+
error("version not found")
|
|
3249
|
+
end
|
|
3250
|
+
local data = versionProfile.Data
|
|
3251
|
+
-- THE RESERVED ROOT IS NOT PART OF THE ROLLBACK. `_Scribe` is the ledger of
|
|
3252
|
+
-- things that happened in the real world: receipts granted, gifts paid for,
|
|
3253
|
+
-- entitlements bought, cooldowns started. None of those un-happen because an
|
|
3254
|
+
-- operator restored a backup, so rolling them back lets one PurchaseId grant
|
|
3255
|
+
-- twice, resurrects delivered gifts and strips paid credits and perks.
|
|
3256
|
+
-- DEEP-CLONED, unlike runMigrations: the `RestoredFrom` stamp below lands
|
|
3257
|
+
-- before the commit can still be refused, and an injected store whose GetAsync
|
|
3258
|
+
-- hands back its own backing table would turn that stamp into a live write.
|
|
3259
|
+
if not reservedRolledBack then
|
|
3260
|
+
local liveReserved = live.Data[Template.RESERVED_ROOT]
|
|
3261
|
+
if type(liveReserved) == "table" then
|
|
3262
|
+
local snapshotReserved = data[Template.RESERVED_ROOT]
|
|
3263
|
+
-- `Version` is the one member of this root that travels with the game
|
|
3264
|
+
-- data: it describes the SHAPE of the data the lines above just
|
|
3265
|
+
-- replaced with an older snapshot. Carrying the live number onto
|
|
3266
|
+
-- restored old-shaped data makes runMigrations return early, so the
|
|
3267
|
+
-- chain never re-runs and the profile is stuck at the pre-migration
|
|
3268
|
+
-- shape marked current; under a staged deploy an older server also
|
|
3269
|
+
-- fails closed on PROFILE_VERSION_AHEAD and kick-loops the player.
|
|
3270
|
+
-- ABSENT MEANS 1, not "keep the live one": a snapshot with no reserved
|
|
3271
|
+
-- root is a LEGACY profile, the reading runMigrations gives it above.
|
|
3272
|
+
local snapshotVersion = if type(snapshotReserved) == "table"
|
|
3273
|
+
and type(snapshotReserved.Version) == "number"
|
|
3274
|
+
then snapshotReserved.Version
|
|
3275
|
+
else 1
|
|
3276
|
+
-- Version-aware ON PURPOSE (the migration call site is not; see
|
|
3277
|
+
-- reservedUnchanged). Without it the common case, a pre-deploy snapshot
|
|
3278
|
+
-- for a player who has bought nothing, reports no difference and hides
|
|
3279
|
+
-- the only change that alters what the next load does.
|
|
3280
|
+
reservedDiffered = not reservedUnchanged(liveReserved, snapshotReserved, true)
|
|
3281
|
+
data[Template.RESERVED_ROOT] = Util.deepClone(liveReserved)
|
|
3282
|
+
data[Template.RESERVED_ROOT].Version = snapshotVersion
|
|
3283
|
+
restoredSchemaVersion = snapshotVersion
|
|
3284
|
+
previousSchemaVersion = if type(liveReserved.Version) == "number" then liveReserved.Version else 1
|
|
3285
|
+
end
|
|
3286
|
+
-- No live reserved root at all (a legacy or imported profile that never had
|
|
3287
|
+
-- one): nothing real-world to preserve, so the snapshot's own root, or the
|
|
3288
|
+
-- empty fallback below, stands.
|
|
3289
|
+
end
|
|
3290
|
+
if type(data[Template.RESERVED_ROOT]) ~= "table" then
|
|
3291
|
+
data[Template.RESERVED_ROOT] = {}
|
|
3292
|
+
end
|
|
3293
|
+
data[Template.RESERVED_ROOT].RestoredFrom = { VersionId = versionId, At = Clock.Now() }
|
|
3294
|
+
-- Commit through the SAME compare-and-set updateOfflineProfile uses, in one
|
|
3295
|
+
-- DataStore call. Re-reading the lock and then writing is two round trips,
|
|
3296
|
+
-- which narrows the window without closing it, and a session landing between
|
|
3297
|
+
-- them is DISOWNED: the view write clears ActiveSession, so the holder keeps
|
|
3298
|
+
-- accepting writes it can no longer persist.
|
|
3299
|
+
if type((store :: any).UpdateOfflineAsync) ~= "function" then
|
|
3300
|
+
-- An injected store without the compare-and-set keeps the old shape,
|
|
3301
|
+
-- window and all; it is the only thing available.
|
|
3302
|
+
local latest = store:GetAsync(self.KeyFor(userId))
|
|
3303
|
+
if sessionLooksLive(latest) then
|
|
3304
|
+
error("profile has an active session on another server")
|
|
3305
|
+
end
|
|
3306
|
+
local committed, commitErr = commitViewProfile(versionProfile, self.KeyFor(userId))
|
|
3307
|
+
if not committed then
|
|
3308
|
+
error(commitErr)
|
|
3309
|
+
end
|
|
3310
|
+
return
|
|
3311
|
+
end
|
|
3312
|
+
local outcome = (store :: any):UpdateOfflineAsync(self.KeyFor(userId), expected, data, {
|
|
3313
|
+
StaleSessionAfter = ASSUME_DEAD_SECONDS,
|
|
3314
|
+
})
|
|
3315
|
+
local restored, reason = mapOfflineOutcome(outcome, "restore")
|
|
3316
|
+
if not restored then
|
|
3317
|
+
error(
|
|
3318
|
+
if outcome == "Missing"
|
|
3319
|
+
then "no live profile exists for this user to restore over (was it erased?)"
|
|
3320
|
+
else reason
|
|
3321
|
+
)
|
|
3322
|
+
end
|
|
3323
|
+
end)
|
|
3324
|
+
if not ok then
|
|
3325
|
+
-- Warn rather than Error deliberately: the caller is already being told, so
|
|
3326
|
+
-- this is a record rather than an alert.
|
|
3327
|
+
Failure.Report({
|
|
3328
|
+
Subsystem = "Restore",
|
|
3329
|
+
Key = tostring(userId),
|
|
3330
|
+
Error = err,
|
|
3331
|
+
Category = "Persistence",
|
|
3332
|
+
Code = "PROFILE_RESTORE_FAIL",
|
|
3333
|
+
Message = tostring(err),
|
|
3334
|
+
Context = { UserId = userId, VersionId = versionId },
|
|
3335
|
+
})
|
|
3336
|
+
return false, tostring(err)
|
|
3337
|
+
end
|
|
3338
|
+
Health.ReportSuccess("Restore")
|
|
3339
|
+
Log.Warn("Persistence", "PROFILE_RESTORED", "profile restored from version history", {
|
|
3340
|
+
UserId = userId,
|
|
3341
|
+
VersionId = versionId,
|
|
3342
|
+
PlaceId = game.PlaceId,
|
|
3343
|
+
JobId = game.JobId,
|
|
3344
|
+
ReservedRolledBack = reservedRolledBack,
|
|
3345
|
+
})
|
|
3346
|
+
-- Only when it actually DIFFERED, and only after the commit landed: an operator
|
|
3347
|
+
-- running a restore must not have to guess whether the money moved with it.
|
|
3348
|
+
if reservedDiffered then
|
|
3349
|
+
Log.Warn(
|
|
3350
|
+
"Persistence",
|
|
3351
|
+
"RESTORE_RESERVED_PRESERVED",
|
|
3352
|
+
`the reserved "{Template.RESERVED_ROOT}" root differed from the restored snapshot and was PRESERVED, not `
|
|
3353
|
+
.. `rolled back: receipt dedupe, gift escrow, gift credits, perks, purchase logs and Timed are still `
|
|
3354
|
+
.. `as they were before the restore, because those record events that already happened. The schema `
|
|
3355
|
+
.. `Version is the one exception -- it describes the SHAPE of the game data, so it travels with it: `
|
|
3356
|
+
.. `now {restoredSchemaVersion}, was {previousSchemaVersion}, and any migration steps above `
|
|
3357
|
+
.. `{restoredSchemaVersion} re-run on the next load. Pass RollBackReserved = true only if that root `
|
|
3358
|
+
.. `is itself what you are repairing`,
|
|
3359
|
+
{
|
|
3360
|
+
UserId = userId,
|
|
3361
|
+
VersionId = versionId,
|
|
3362
|
+
PlaceId = game.PlaceId,
|
|
3363
|
+
JobId = game.JobId,
|
|
3364
|
+
SchemaVersion = restoredSchemaVersion,
|
|
3365
|
+
PreviousSchemaVersion = previousSchemaVersion,
|
|
3366
|
+
}
|
|
3367
|
+
)
|
|
3368
|
+
end
|
|
3369
|
+
return true, nil
|
|
3370
|
+
end
|
|
3371
|
+
|
|
3372
|
+
-- GDPR ---------------------------------------------------------------------
|
|
3373
|
+
|
|
3374
|
+
local ERASE_LOST = "the erase was finished or replaced elsewhere"
|
|
3375
|
+
local ERASE_BUSY = "another erase of this profile is still running"
|
|
3376
|
+
-- Longer than a budget wait plus a request; a crashed worker blocks a retry this long.
|
|
3377
|
+
local ERASE_LEASE = 300
|
|
3378
|
+
|
|
3379
|
+
local function leaseLive(lease: any): boolean
|
|
3380
|
+
return type(lease) == "table" and type(lease.Until) == "number" and lease.Until > Clock.Now()
|
|
3381
|
+
end
|
|
3382
|
+
|
|
3383
|
+
--- nil while `profile` carries a marker leased to `leaseId`, else why this attempt lost it.
|
|
3384
|
+
local function markerLost(profile: any, leaseId: string): string?
|
|
3385
|
+
if profile == nil then
|
|
3386
|
+
return NO_PROFILE
|
|
3387
|
+
end
|
|
3388
|
+
local data = profile.Data
|
|
3389
|
+
local reserved = if type(data) == "table" then data[Template.RESERVED_ROOT] else nil
|
|
3390
|
+
local stored = if type(reserved) == "table" then reserved.Erase else nil
|
|
3391
|
+
if type(stored) ~= "table" then
|
|
3392
|
+
return "the profile was recreated since this erase began"
|
|
3393
|
+
end
|
|
3394
|
+
if type(stored.Lease) ~= "table" or stored.Lease.Id ~= leaseId then
|
|
3395
|
+
return "another erase replaced this one"
|
|
3396
|
+
end
|
|
3397
|
+
return nil
|
|
3398
|
+
end
|
|
3399
|
+
|
|
3400
|
+
--- Takes the erase lease in the same compare-and-set as the read it decides on; a user with
|
|
3401
|
+
--- no profile gets a placeholder holding only the marker. Returns the marker, the board
|
|
3402
|
+
--- ranges and whether an earlier attempt left the marker. Errors with the refusal.
|
|
3403
|
+
local function claimErase(userId: number, leaseId: string): (any, any, boolean)
|
|
3404
|
+
local marker: any = nil
|
|
3405
|
+
local ranges: any = nil
|
|
3406
|
+
local resumed = false
|
|
3407
|
+
local wrote, why = updateOfflineProfile(userId, function(draft)
|
|
3408
|
+
local reserved = draft[Template.RESERVED_ROOT]
|
|
3409
|
+
if type(reserved) ~= "table" then
|
|
3410
|
+
reserved = {}
|
|
3411
|
+
draft[Template.RESERVED_ROOT] = reserved
|
|
3412
|
+
end
|
|
3413
|
+
ranges = if type(reserved.Boards) == "table" then Util.deepClone(reserved.Boards) else nil
|
|
3414
|
+
local existing = reserved.Erase
|
|
3415
|
+
if type(existing) == "table" then
|
|
3416
|
+
if leaseLive(existing.Lease) then
|
|
3417
|
+
error(ERASE_BUSY, 0)
|
|
3418
|
+
end
|
|
3419
|
+
resumed = true
|
|
3420
|
+
marker = {
|
|
3421
|
+
StartedAt = existing.StartedAt,
|
|
3422
|
+
Done = if type(existing.Done) == "table" then Util.deepClone(existing.Done) else {},
|
|
3423
|
+
Cursors = if type(existing.Cursors) == "table" then Util.deepClone(existing.Cursors) else {},
|
|
3424
|
+
}
|
|
3425
|
+
else
|
|
3426
|
+
marker = { StartedAt = Clock.Now(), Done = {}, Cursors = {} }
|
|
3427
|
+
end
|
|
3428
|
+
marker.Lease = { Id = leaseId, Until = Clock.Now() + ERASE_LEASE }
|
|
3429
|
+
reserved.Erase = Util.deepClone(marker)
|
|
3430
|
+
end, false, true)
|
|
3431
|
+
if wrote then
|
|
3432
|
+
return marker, ranges, resumed
|
|
3433
|
+
end
|
|
3434
|
+
if why == ERASE_BUSY or why == LIVE_ELSEWHERE then
|
|
3435
|
+
error(why, 0)
|
|
3436
|
+
end
|
|
3437
|
+
error(`could not mark the profile for erasure: {tostring(why)}`, 0)
|
|
3438
|
+
end
|
|
3439
|
+
|
|
3440
|
+
--- Writes the sweep's progress and renews the lease, or with `release` drops it. The third
|
|
3441
|
+
--- return is true once the key no longer carries this attempt's lease.
|
|
3442
|
+
local function renewErase(
|
|
3443
|
+
userId: number,
|
|
3444
|
+
marker: any,
|
|
3445
|
+
leaseId: string,
|
|
3446
|
+
release: boolean
|
|
3447
|
+
): (boolean, string?, boolean)
|
|
3448
|
+
local wrote, why = updateOfflineProfile(userId, function(draft)
|
|
3449
|
+
local reserved = draft[Template.RESERVED_ROOT]
|
|
3450
|
+
local existing = if type(reserved) == "table" then reserved.Erase else nil
|
|
3451
|
+
if type(existing) ~= "table" or type(existing.Lease) ~= "table" or existing.Lease.Id ~= leaseId then
|
|
3452
|
+
error(ERASE_LOST, 0)
|
|
3453
|
+
end
|
|
3454
|
+
marker.Lease = if release then nil else { Id = leaseId, Until = Clock.Now() + ERASE_LEASE }
|
|
3455
|
+
reserved.Erase = Util.deepClone(marker)
|
|
3456
|
+
end, false)
|
|
3457
|
+
if wrote then
|
|
3458
|
+
return true, nil, false
|
|
3459
|
+
end
|
|
3460
|
+
if why == NO_PROFILE or why == ERASE_LOST then
|
|
3461
|
+
return false, why, true
|
|
3462
|
+
end
|
|
3463
|
+
-- A live session refuses the write before the marker is seen; only a read can tell.
|
|
3464
|
+
if why == LIVE_ELSEWHERE then
|
|
3465
|
+
local readOk, stored = pcall(function()
|
|
3466
|
+
return store:GetAsync(self.KeyFor(userId))
|
|
3467
|
+
end)
|
|
3468
|
+
if readOk and markerLost(stored, leaseId) ~= nil then
|
|
3469
|
+
return false, ERASE_LOST, true
|
|
3470
|
+
end
|
|
3471
|
+
end
|
|
3472
|
+
return false, why, false
|
|
3473
|
+
end
|
|
3474
|
+
|
|
3475
|
+
local function eraseHeld(userId: number): (boolean, string?)
|
|
3476
|
+
local leaseId = Util.guid()
|
|
3477
|
+
local marker: any = nil
|
|
3478
|
+
local ranges: any = nil
|
|
3479
|
+
local ok, err = pcall(function()
|
|
3480
|
+
local resumed
|
|
3481
|
+
marker, ranges, resumed = claimErase(userId, leaseId)
|
|
3482
|
+
if resumed then
|
|
3483
|
+
Log.Info("Persistence", "PROFILE_ERASE_RESUMED", "resuming an erase that did not finish", {
|
|
3484
|
+
UserId = userId,
|
|
3485
|
+
StartedAt = marker.StartedAt,
|
|
3486
|
+
})
|
|
3487
|
+
end
|
|
3488
|
+
end)
|
|
3489
|
+
local progress: any = nil
|
|
3490
|
+
if ok then
|
|
3491
|
+
progress = { Done = marker.Done, Cursors = marker.Cursors, Lost = false, Stopped = nil }
|
|
3492
|
+
-- Past a failed renewal the key may be another attempt's, so the sweep stops.
|
|
3493
|
+
progress.Save = function(): boolean
|
|
3494
|
+
local why
|
|
3495
|
+
for attempt = 1, 2 do
|
|
3496
|
+
local saved, reason, lost = renewErase(userId, marker, leaseId, false)
|
|
3497
|
+
if lost then
|
|
3498
|
+
progress.Lost = true
|
|
3499
|
+
return false
|
|
3500
|
+
end
|
|
3501
|
+
if saved then
|
|
3502
|
+
return true
|
|
3503
|
+
end
|
|
3504
|
+
why = reason
|
|
3505
|
+
if attempt == 1 then
|
|
3506
|
+
Clock.Wait(1)
|
|
3507
|
+
end
|
|
3508
|
+
end
|
|
3509
|
+
Log.Warn(
|
|
3510
|
+
"Persistence",
|
|
3511
|
+
"PROFILE_ERASE_PROGRESS_FAIL",
|
|
3512
|
+
"erase progress was not saved; the sweep stops and a retry resumes it",
|
|
3513
|
+
{ UserId = userId, Reason = why }
|
|
3514
|
+
)
|
|
3515
|
+
progress.Stopped = tostring(why)
|
|
3516
|
+
return false
|
|
3517
|
+
end
|
|
3518
|
+
local lbOk, lbErr = ctx.Leaderboards.EraseUser(userId, ranges, progress)
|
|
3519
|
+
-- A lost erase is judged by the key below, not by the sweep it cut short.
|
|
3520
|
+
if progress.Stopped ~= nil then
|
|
3521
|
+
ok, err = false, `the erase lease could not be renewed: {progress.Stopped}`
|
|
3522
|
+
elseif not lbOk and not progress.Lost then
|
|
3523
|
+
ok, err = false, `leaderboard entries remain: {tostring(lbErr)}`
|
|
3524
|
+
end
|
|
3525
|
+
end
|
|
3526
|
+
local finishedElsewhere = false
|
|
3527
|
+
if ok then
|
|
3528
|
+
ok, err = pcall(function()
|
|
3529
|
+
-- The sweep yields, so the key may have changed hands since the claim.
|
|
3530
|
+
local stored = store:GetAsync(self.KeyFor(userId))
|
|
3531
|
+
if stored == nil then
|
|
3532
|
+
finishedElsewhere = true
|
|
3533
|
+
return
|
|
3534
|
+
end
|
|
3535
|
+
if sessionLooksLive(stored) then
|
|
3536
|
+
error(LIVE_ELSEWHERE, 0)
|
|
3537
|
+
end
|
|
3538
|
+
local lost = markerLost(stored, leaseId)
|
|
3539
|
+
if lost ~= nil then
|
|
3540
|
+
error(lost, 0)
|
|
3541
|
+
end
|
|
3542
|
+
if not store:RemoveAsync(self.KeyFor(userId)) then
|
|
3543
|
+
error("RemoveAsync returned false")
|
|
3544
|
+
end
|
|
3545
|
+
end)
|
|
3546
|
+
end
|
|
3547
|
+
if not ok then
|
|
3548
|
+
-- Drop the lease so a retry need not wait for it; a refused drop leaves it to lapse.
|
|
3549
|
+
if marker ~= nil and not (progress ~= nil and progress.Lost) then
|
|
3550
|
+
renewErase(userId, marker, leaseId, true)
|
|
3551
|
+
end
|
|
3552
|
+
-- Same as Restore: without this a failed GDPR erase reaches no sink at all.
|
|
3553
|
+
Failure.Report({
|
|
3554
|
+
Subsystem = "Erase",
|
|
3555
|
+
Key = tostring(userId),
|
|
3556
|
+
Error = err,
|
|
3557
|
+
Category = "Persistence",
|
|
3558
|
+
Code = "PROFILE_ERASE_FAIL",
|
|
3559
|
+
Message = tostring(err),
|
|
3560
|
+
Context = { UserId = userId },
|
|
3561
|
+
})
|
|
3562
|
+
return false, tostring(err)
|
|
3563
|
+
end
|
|
3564
|
+
if finishedElsewhere then
|
|
3565
|
+
return true, nil
|
|
3566
|
+
end
|
|
3567
|
+
Log.Warn("Persistence", "PROFILE_ERASED", "profile erased (GDPR)", {
|
|
3568
|
+
UserId = userId,
|
|
3569
|
+
PlaceId = game.PlaceId,
|
|
3570
|
+
JobId = game.JobId,
|
|
3571
|
+
})
|
|
3572
|
+
return true, nil
|
|
3573
|
+
end
|
|
3574
|
+
|
|
3575
|
+
function self.Erase(userId: number): (boolean, string?)
|
|
3576
|
+
if mode == "NoSave" then
|
|
3577
|
+
return false, refuseNoSave("Erase")
|
|
3578
|
+
end
|
|
3579
|
+
if Health.GetStatus() == "Outage" then
|
|
3580
|
+
return false, "data services are experiencing an outage"
|
|
3581
|
+
end
|
|
3582
|
+
if self.EntryForUserId(userId) then
|
|
3583
|
+
return false, "user has an active session on this server"
|
|
3584
|
+
end
|
|
3585
|
+
erasing[userId] = (erasing[userId] or 0) + 1
|
|
3586
|
+
local callOk, ok, reason = pcall(eraseHeld, userId)
|
|
3587
|
+
erasing[userId] = if erasing[userId] > 1 then erasing[userId] - 1 else nil
|
|
3588
|
+
if not callOk then
|
|
3589
|
+
return false, tostring(ok)
|
|
3590
|
+
end
|
|
3591
|
+
return ok, reason
|
|
3592
|
+
end
|
|
3593
|
+
|
|
3594
|
+
local function jsonSafe(value: any): any
|
|
3595
|
+
local t = type(value)
|
|
3596
|
+
if t == "buffer" then
|
|
3597
|
+
return "base64:" .. Util.base64FromBuffer(value)
|
|
3598
|
+
elseif t == "table" then
|
|
3599
|
+
local out = {}
|
|
3600
|
+
for k, v in value do
|
|
3601
|
+
out[k] = jsonSafe(v)
|
|
3602
|
+
end
|
|
3603
|
+
return out
|
|
3604
|
+
end
|
|
3605
|
+
return value
|
|
3606
|
+
end
|
|
3607
|
+
|
|
3608
|
+
function self.Export(userId: number): string?
|
|
3609
|
+
local data = self.GetOffline(userId)
|
|
3610
|
+
if data == nil then
|
|
3611
|
+
return nil
|
|
3612
|
+
end
|
|
3613
|
+
local ok, encoded = pcall(function()
|
|
3614
|
+
return HttpService:JSONEncode(jsonSafe(data))
|
|
3615
|
+
end)
|
|
3616
|
+
if not ok then
|
|
3617
|
+
Log.Warn("Persistence", "EXPORT_ENCODE_FAIL", tostring(encoded), { UserId = userId })
|
|
3618
|
+
return nil
|
|
3619
|
+
end
|
|
3620
|
+
return encoded
|
|
3621
|
+
end
|
|
3622
|
+
|
|
3623
|
+
-- Shutdown -------------------------------------------------------------------
|
|
3624
|
+
|
|
3625
|
+
-- The BindToClose drain: stop accepting new sessions, flush the
|
|
3626
|
+
-- leaderboard queue, end every profile session, and wait within the
|
|
3627
|
+
-- budget. Exposed separately so the test harness can simulate shutdown.
|
|
3628
|
+
function self.Shutdown(budget: number?)
|
|
3629
|
+
self.Closing = true
|
|
3630
|
+
-- Before anything else: stop the store starting work this drain would then have to
|
|
3631
|
+
-- wait for. Its own BindToClose stands down (ExternalShutdown), so unless this
|
|
3632
|
+
-- runs, IsClosing never gets set and MessageAsync keeps retrying through the
|
|
3633
|
+
-- shutdown window.
|
|
3634
|
+
if type((ProfileStore :: any).BeginClose) == "function" then
|
|
3635
|
+
pcall((ProfileStore :: any).BeginClose)
|
|
3636
|
+
end
|
|
3637
|
+
local sessionCount = self.EntryCount
|
|
3638
|
+
Log.Info("Persistence", "SHUTDOWN_FLUSH", "shutting down; draining saves and leaderboards", {
|
|
3639
|
+
Sessions = sessionCount,
|
|
3640
|
+
})
|
|
3641
|
+
-- Single deadline shared by both phases so a slow leaderboard flush can never
|
|
3642
|
+
-- squeeze profile saves past Roblox's ~30s BindToClose window. Profile saves go
|
|
3643
|
+
-- first; the leaderboard queue is rebuildable and gets whatever time remains.
|
|
3644
|
+
local deadline = Clock.Monotonic() + (budget or 25)
|
|
3645
|
+
-- Tear down profile-less Loading entries up front: one parked in a
|
|
3646
|
+
-- LoadFailurePolicy = "Wait" backoff sleeps up to 60s before noticing Closing and
|
|
3647
|
+
-- would hold the drain loop below to its full budget. Their StartSessionAsync
|
|
3648
|
+
-- cannot succeed anymore (the Cancel hook sees Closing), and the loader no-ops on
|
|
3649
|
+
-- TornDown when it wakes.
|
|
3650
|
+
local strandedLoading: { Entry } = {}
|
|
3651
|
+
for _, entry in self.Entries :: { [Player]: Entry } do
|
|
3652
|
+
if entry.State == "Loading" and entry.Profile == nil then
|
|
3653
|
+
table.insert(strandedLoading, entry)
|
|
3654
|
+
end
|
|
3655
|
+
end
|
|
3656
|
+
for _, entry in strandedLoading do
|
|
3657
|
+
teardown(entry, "shutdown")
|
|
3658
|
+
end
|
|
3659
|
+
-- Snapshot before running anything, the same two-phase shape as the stranded pass
|
|
3660
|
+
-- above. A hook gets the real accessor and can kick a player or end another
|
|
3661
|
+
-- session, and clearing keys mid-traversal is legal while ADDING one is undefined.
|
|
3662
|
+
local draining: { Entry } = {}
|
|
3663
|
+
for _, entry in self.Entries :: { [Player]: Entry } do
|
|
3664
|
+
if entry.Profile and entry.Profile:IsActive() then
|
|
3665
|
+
table.insert(draining, entry)
|
|
3666
|
+
end
|
|
3667
|
+
end
|
|
3668
|
+
|
|
3669
|
+
-- Idempotent, because the cut-off below ends a parked hook's session on this
|
|
3670
|
+
-- thread and the hook then resumes into its own call to this.
|
|
3671
|
+
local function endSession(entry: Entry)
|
|
3672
|
+
if entry.LocalSessionEnd or not entry.Profile then
|
|
3673
|
+
return
|
|
3674
|
+
end
|
|
3675
|
+
entry.LocalSessionEnd = true
|
|
3676
|
+
entry.Profile:EndSession()
|
|
3677
|
+
end
|
|
3678
|
+
|
|
3679
|
+
local hookDeadline = Clock.Monotonic() + math.max(0, deadline - Clock.Monotonic()) * HOOK_PHASE_SHARE
|
|
3680
|
+
local pending = 0
|
|
3681
|
+
local hooksSkipped = 0
|
|
3682
|
+
for _, entry in draining do
|
|
3683
|
+
if not (entry.Profile and entry.Profile:IsActive()) then
|
|
3684
|
+
continue -- a hook that already ran ended this one for us
|
|
3685
|
+
end
|
|
3686
|
+
if Clock.Monotonic() >= hookDeadline then
|
|
3687
|
+
-- Out of hook time. The save is not optional; the hook is.
|
|
3688
|
+
hooksSkipped += 1
|
|
3689
|
+
endSession(entry)
|
|
3690
|
+
continue
|
|
3691
|
+
end
|
|
3692
|
+
pending += 1
|
|
3693
|
+
Sched.Spawn(function()
|
|
3694
|
+
-- Per entry, the hook still runs strictly before ITS OWN EndSession: that
|
|
3695
|
+
-- ordering is the documented guarantee, and losing it is silent because
|
|
3696
|
+
-- EndSession destroys the tree synchronously and every later hook write
|
|
3697
|
+
-- raises into the contained LISTENER_ERROR path. What is relaxed is the
|
|
3698
|
+
-- ordering BETWEEN entries; Spawn runs inline until the hook yields, so a
|
|
3699
|
+
-- non-yielding hook behaves exactly as it did as straight-line code.
|
|
3700
|
+
runLeavingHook(entry, "shutdown")
|
|
3701
|
+
pending -= 1
|
|
3702
|
+
endSession(entry)
|
|
3703
|
+
end)
|
|
3704
|
+
end
|
|
3705
|
+
while pending > 0 and Clock.Monotonic() < hookDeadline do
|
|
3706
|
+
Clock.Wait(HOOK_JOIN_POLL)
|
|
3707
|
+
end
|
|
3708
|
+
-- Whoever is still parked has run out of budget, and Luau cannot interrupt them.
|
|
3709
|
+
-- What they wrote before this point is already in the profile table and persists,
|
|
3710
|
+
-- so a timed-out hook is not an atomic one.
|
|
3711
|
+
local hooksTimedOut = 0
|
|
3712
|
+
for _, entry in draining do
|
|
3713
|
+
if entry.Profile and entry.Profile:IsActive() and not entry.LocalSessionEnd then
|
|
3714
|
+
hooksTimedOut += 1
|
|
3715
|
+
endSession(entry)
|
|
3716
|
+
end
|
|
3717
|
+
end
|
|
3718
|
+
if hooksSkipped > 0 then
|
|
3719
|
+
Metrics.Add("LeavingHooksSkipped", hooksSkipped)
|
|
3720
|
+
end
|
|
3721
|
+
if hooksTimedOut > 0 then
|
|
3722
|
+
Metrics.Add("LeavingHooksTimedOut", hooksTimedOut)
|
|
3723
|
+
end
|
|
3724
|
+
|
|
3725
|
+
local lbBudget = math.max(0, math.min(deadline - Clock.Monotonic(), 20))
|
|
3726
|
+
ctx.Leaderboards.FlushAllBlocking(lbBudget)
|
|
3727
|
+
while next(self.Entries) ~= nil and Clock.Monotonic() < deadline do
|
|
3728
|
+
Clock.Wait(0.2)
|
|
3729
|
+
end
|
|
3730
|
+
-- Everything Scribe owns is released by now, so this is the job-drain wait:
|
|
3731
|
+
-- it keeps the process alive until the saves those releases started have
|
|
3732
|
+
-- actually landed, and sweeps up any profile Scribe does not know about.
|
|
3733
|
+
if type((ProfileStore :: any).AwaitClose) == "function" then
|
|
3734
|
+
pcall((ProfileStore :: any).AwaitClose)
|
|
3735
|
+
end
|
|
3736
|
+
Log.Info("Persistence", "SHUTDOWN_DONE", "shutdown drain complete", {
|
|
3737
|
+
Remaining = self.EntryCount,
|
|
3738
|
+
HooksSkipped = hooksSkipped,
|
|
3739
|
+
HooksTimedOut = hooksTimedOut,
|
|
3740
|
+
})
|
|
3741
|
+
end
|
|
3742
|
+
|
|
3743
|
+
function self.BindShutdown()
|
|
3744
|
+
game:BindToClose(function()
|
|
3745
|
+
self.Shutdown(25)
|
|
3746
|
+
end)
|
|
3747
|
+
end
|
|
3748
|
+
|
|
3749
|
+
-- Release the process-global signal connections this bundle installed.
|
|
3750
|
+
function self.Stop()
|
|
3751
|
+
-- FIRST, before anything is disconnected: the loader reads it on its next turn,
|
|
3752
|
+
-- and a flag set after the yield-heavy teardown would let another round through.
|
|
3753
|
+
self.Stopped = true
|
|
3754
|
+
for _, entry in self.Entries do
|
|
3755
|
+
if entry.StopLoadWatchdog then
|
|
3756
|
+
entry.StopLoadWatchdog()
|
|
3757
|
+
entry.StopLoadWatchdog = nil
|
|
3758
|
+
end
|
|
3759
|
+
if entry.StopLeavingHookWatchdog then
|
|
3760
|
+
entry.StopLeavingHookWatchdog()
|
|
3761
|
+
entry.StopLeavingHookWatchdog = nil
|
|
3762
|
+
end
|
|
3763
|
+
end
|
|
3764
|
+
for _, connection in connections do
|
|
3765
|
+
pcall(function()
|
|
3766
|
+
connection:Disconnect()
|
|
3767
|
+
end)
|
|
3768
|
+
end
|
|
3769
|
+
table.clear(connections)
|
|
3770
|
+
end
|
|
3771
|
+
|
|
3772
|
+
return self
|
|
3773
|
+
end
|
|
3774
|
+
|
|
3775
|
+
return Persistence
|