@rbxts/scribe 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/LICENSE +21 -0
  2. package/NOTICE +21 -0
  3. package/README.md +100 -0
  4. package/Scribe-source-map.json +1 -0
  5. package/licenses/ProfileStore-Apache-2.0.txt +201 -0
  6. package/package.json +32 -0
  7. package/src/Client/DebugHook.luau +457 -0
  8. package/src/Client/init.luau +1607 -0
  9. package/src/Internal/Addons.luau +68 -0
  10. package/src/Internal/Big.luau +582 -0
  11. package/src/Internal/Budget.luau +377 -0
  12. package/src/Internal/Clock.luau +75 -0
  13. package/src/Internal/Config.luau +17 -0
  14. package/src/Internal/Conform.luau +400 -0
  15. package/src/Internal/Datatypes.luau +653 -0
  16. package/src/Internal/Defaults.luau +12 -0
  17. package/src/Internal/Derived.luau +428 -0
  18. package/src/Internal/Exchange.luau +556 -0
  19. package/src/Internal/ExchangeLeg.luau +1368 -0
  20. package/src/Internal/ExchangeVerdict.luau +179 -0
  21. package/src/Internal/Failure.luau +148 -0
  22. package/src/Internal/Health.luau +320 -0
  23. package/src/Internal/LeaderboardDiagnostics.luau +36 -0
  24. package/src/Internal/Log.luau +378 -0
  25. package/src/Internal/Marker.luau +76 -0
  26. package/src/Internal/Metrics.luau +152 -0
  27. package/src/Internal/Node.luau +3954 -0
  28. package/src/Internal/Profile.luau +69 -0
  29. package/src/Internal/Rng.luau +83 -0
  30. package/src/Internal/Sched.luau +56 -0
  31. package/src/Internal/Signal.luau +199 -0
  32. package/src/Internal/Store/ProfileStore.luau +3019 -0
  33. package/src/Internal/Store/init.luau +18 -0
  34. package/src/Internal/StudioCodec.luau +155 -0
  35. package/src/Internal/Template.luau +1848 -0
  36. package/src/Internal/Transport.luau +169 -0
  37. package/src/Internal/Util.luau +517 -0
  38. package/src/Internal/Version.luau +4 -0
  39. package/src/Internal/Wire.luau +1283 -0
  40. package/src/Server/Analytics.luau +223 -0
  41. package/src/Server/Commands.luau +728 -0
  42. package/src/Server/DebugHook.luau +1490 -0
  43. package/src/Server/Exchanges.luau +737 -0
  44. package/src/Server/Leaderboards.luau +2143 -0
  45. package/src/Server/Monetization.luau +3212 -0
  46. package/src/Server/Persistence.luau +3775 -0
  47. package/src/Server/Replication.luau +1253 -0
  48. package/src/Server/ServerStore.luau +81 -0
  49. package/src/Server/Timed.luau +579 -0
  50. package/src/Server/init.luau +672 -0
  51. package/src/Types.luau +480 -0
  52. package/src/init.luau +2231 -0
  53. package/types/api.d.ts +252 -0
  54. package/types/common.d.ts +355 -0
  55. package/types/index.d.ts +226 -0
  56. package/types/schema.d.ts +276 -0
@@ -0,0 +1,1253 @@
1
+ --!strict
2
+ -- Replication: per-player dirty op queues flushed once per frame
3
+ -- on PostSimulation, client-ready handshake, root visibility filtering, Shared
4
+ -- broadcast, and inbound frame dispatch (size caps + pcall'd decode).
5
+
6
+ local RunService = game:GetService("RunService")
7
+
8
+ local Internal = script.Parent.Parent.Internal
9
+ local Util = require(Internal.Util)
10
+ local Wire = require(Internal.Wire)
11
+ local Template = require(Internal.Template)
12
+ local Log = require(Internal.Log)
13
+ local Metrics = require(Internal.Metrics)
14
+ local Health = require(Internal.Health)
15
+ local Clock = require(Internal.Clock)
16
+ local Profile = require(Internal.Profile)
17
+
18
+ local Replication = {}
19
+
20
+ local HANDSHAKE_TIMEOUT = 10
21
+ local MAX_OPS_PER_FRAME = 60000
22
+ local REINIT_COOLDOWN = 5 -- seconds between full snapshots for repeated Hellos
23
+ local MALFORMED_LOG_INTERVAL = 1 -- seconds; throttle malformed-frame logs/anomalies per player
24
+ local MALFORMED_MAX = 32 -- after this many malformed frames, ignore all further inbound from the player
25
+ -- Its own budget rather than counting toward MALFORMED_MAX: a malformed frame
26
+ -- decoded and failed, an oversize frame was never read, and a shared counter would
27
+ -- let either mask the other.
28
+ local OVERSIZE_MAX = 32
29
+
30
+ local STATUS_TO_ID: { [string]: number } = { Healthy = 0, Degraded = 1, Outage = 2 }
31
+
32
+ -- Inverted once at load, so the oversize warning can name the kind of frame.
33
+ local FRAME_NAMES: { [number]: string } = {}
34
+ -- BytesOut by frame kind, keyed by the type byte.
35
+ local BYTES_OUT_BY_FRAME: { [number]: string } = {}
36
+ for name, id in Wire.Frames :: { [string]: number } do
37
+ FRAME_NAMES[id] = name
38
+ BYTES_OUT_BY_FRAME[id] = `BytesOut{name}`
39
+ end
40
+
41
+ function Replication.new(ctx: any)
42
+ local self = {} :: any
43
+ local compiled: Template.Compiled = ctx.Compiled
44
+ local transport = ctx.Transport
45
+ local options = ctx.Options
46
+
47
+ local maxInbound: number = options.MaxInboundBytes or 8192
48
+ -- Derived from the byte cap so the two stay in proportion: one inbound frame may
49
+ -- retain at most 16x its own bytes. 16 admits the densest legitimate shape (an
50
+ -- array of two-byte strings, 14.5x) while refusing the hostile ones (arrays of
51
+ -- booleans or of empty tables both cost 32x). See Wire.NewReader.
52
+ local maxInboundRetained: number = options.MaxInboundRetainedBytes or math.max(4096, maxInbound * 16)
53
+ local inboundFrameRate: number = ctx.InboundFrameRate
54
+
55
+ -- The largest buffer Scribe hands a transport in one call; anything above it is
56
+ -- fragmented (Wire.Fragment) and reassembled by the client. A ceiling is needed
57
+ -- because an Init snapshot is one op per root and MAX_OPS_PER_FRAME cannot divide
58
+ -- a single op. 65536 sits far below any platform payload limit and leaves ordinary
59
+ -- diffs unsplit. A smaller transport MaxFrameBytes wins: only the adapter knows
60
+ -- its own framing overhead.
61
+ local declaredMax: number? = (transport :: any).MaxFrameBytes
62
+ local maxOutbound: number = options.MaxOutboundBytes or 65536
63
+ if type(declaredMax) == "number" then
64
+ maxOutbound = math.min(maxOutbound, declaredMax)
65
+ end
66
+ -- At maxOutbound = 64 the u16 fragment count caps a logical frame at 3.7 MB,
67
+ -- which a real profile can reach. Refuse it at construction, where it is a typo.
68
+ if maxOutbound < 256 then
69
+ error(`Scribe: MaxOutboundBytes must be at least 256 (got {maxOutbound})`, 0)
70
+ end
71
+ -- Identifies one logical fragmented frame so the client can tell two apart. Wraps
72
+ -- at 65536; a collision needs 65535 intervening frames mid-reassembly.
73
+ local fragmentSeq = 0
74
+
75
+ -- One log per server, the first time a frame needs more fragments than this.
76
+ -- Measures the wire, unlike the profile-size warning: a Session root is absent
77
+ -- from LiveData but still rides in Init. Latched, being a template property.
78
+ local OVERSIZE_FRAGMENTS = 16
79
+ local warnedOversize = false
80
+
81
+ -- Same token-bucket shape as the command limiter, on its own per-entry clock:
82
+ -- refills at `inboundFrameRate` per second and caps there.
83
+ local function takeFrameToken(entry: any): boolean
84
+ local now = Clock.Monotonic()
85
+ local elapsed = now - entry.FrameLast
86
+ entry.FrameLast = now
87
+ entry.FrameTokens = math.min(inboundFrameRate, entry.FrameTokens + elapsed * inboundFrameRate)
88
+ if entry.FrameTokens >= 1 then
89
+ entry.FrameTokens -= 1
90
+ return true
91
+ end
92
+ return false
93
+ end
94
+
95
+ -- Op queueing ---------------------------------------------------------------
96
+
97
+ -- A whole-container Set/Insert can carry statically-declared ServerOnly
98
+ -- descendants inside its table value; strip them before the value reaches any
99
+ -- client queue (the Init snapshot does the same via CloneVisible).
100
+ local function stripServerOnly(op: Wire.Op, schema: Template.SchemaNode?)
101
+ if type(op.Value) ~= "table" or schema == nil then
102
+ return
103
+ end
104
+ local valueNode: Template.SchemaNode? = schema
105
+ if op.Kind == "Insert" then
106
+ valueNode = Template.ChildSchema(schema, 1)
107
+ end
108
+ if valueNode ~= nil then
109
+ op.Value = Template.CloneVisible(valueNode, op.Value)
110
+ end
111
+ end
112
+
113
+ -- Entries with something to flush or repair; never a departed one or a stopped bundle's.
114
+ local dirty: { [any]: boolean } = {}
115
+ local stopped = false
116
+ local function markDirty(entry: any)
117
+ if not stopped and not entry.TornDown then
118
+ dirty[entry] = true
119
+ end
120
+ end
121
+
122
+ function self.PushOp(entry: any, op: Wire.Op, schema: Template.SchemaNode?)
123
+ local visibility = if schema then schema.Visibility else Template.VisibilityOf(compiled, op.Path)
124
+ if visibility == "ServerOnly" then
125
+ return
126
+ end
127
+
128
+ stripServerOnly(op, schema)
129
+
130
+ Metrics.Add("OpsQueued")
131
+ -- Stamped on every op that may be queued, owner or Shared, so flush-time
132
+ -- compaction can read it in either queue.
133
+ local pathKey = Util.pathKey(op.Path);
134
+ (op :: any)._pk = pathKey
135
+ if entry.QueueOpen then
136
+ markDirty(entry)
137
+ local queue: { Wire.Op } = entry.Queue
138
+ if op.Kind == "Set" and op.Value ~= nil and type(op.Value) ~= "table" then
139
+ -- Scalar leaf Set: coalesce consecutive writes to the same path in place.
140
+ -- Supersession by whole-container writes is resolved at flush (compactOps).
141
+ local existing = entry.QueueIndex[pathKey]
142
+ if
143
+ existing
144
+ and queue[existing]
145
+ and queue[existing].Kind == "Set"
146
+ and (queue[existing] :: any)._pk == pathKey
147
+ then
148
+ queue[existing] = op
149
+ Metrics.Add("OpsCoalesced")
150
+ else
151
+ table.insert(queue, op)
152
+ entry.QueueIndex[pathKey] = #queue
153
+ end
154
+ else
155
+ -- Whole-container writes (table Set, nil Set, Clear) and array ops append,
156
+ -- and break scalar coalescing here and below: a later child Set would
157
+ -- otherwise collapse into a preceding slot and be dropped as superseded.
158
+ table.insert(queue, op)
159
+ entry.QueueIndex[pathKey] = nil
160
+ local descendantPrefix = pathKey .. "\31"
161
+ local prefixLen = #descendantPrefix
162
+ for k in entry.QueueIndex do
163
+ if string.sub(k, 1, prefixLen) == descendantPrefix then
164
+ entry.QueueIndex[k] = nil
165
+ end
166
+ end
167
+ end
168
+ end
169
+ -- Before the snapshot is taken there is nothing to diff against: the snapshot
170
+ -- about to be cloned carries this value, so queueing it would send it twice.
171
+
172
+ if visibility == "Shared" then
173
+ table.insert(entry.SharedQueue, op)
174
+ markDirty(entry)
175
+ end
176
+ end
177
+
178
+ -- Ranks a snapshot's roots by the bytes they contributed, for the oversize
179
+ -- warning. Scribe's own two roots are skipped: tens of bytes, never the answer.
180
+ -- "not persisted" means ProfileSize and the DataStore ceiling say nothing about
181
+ -- that root, not that it is unbounded.
182
+ local function describeRoots(ops: { Wire.Op }?, spans: { number }?): string
183
+ if ops == nil or spans == nil then
184
+ return ""
185
+ end
186
+ local ranked: { { Key: string, Bytes: number } } = {}
187
+ for i, op in ops do
188
+ local key = op.Path[1]
189
+ local span = spans[i]
190
+ if
191
+ type(key) == "string"
192
+ and span ~= nil
193
+ and key ~= Template.RESERVED_ROOT
194
+ and key ~= Template.SESSION_ROOT
195
+ then
196
+ table.insert(ranked, { Key = key, Bytes = span })
197
+ end
198
+ end
199
+ if #ranked == 0 then
200
+ return ""
201
+ end
202
+ table.sort(ranked, function(a: { Key: string, Bytes: number }, b: { Key: string, Bytes: number })
203
+ return a.Bytes > b.Bytes
204
+ end)
205
+ local rootChildren = compiled.Root.Children :: { [string]: Template.SchemaNode }
206
+ local out: { string } = {}
207
+ for i = 1, math.min(3, #ranked) do
208
+ local row = ranked[i]
209
+ local node = rootChildren[row.Key]
210
+ local suffix = if node ~= nil and node.Persist == false then " (not persisted)" else ""
211
+ table.insert(out, `{row.Key}={row.Bytes}B{suffix}`)
212
+ end
213
+ return table.concat(out, ", ")
214
+ end
215
+
216
+ -- One delivery to one client, with no accounting of its own: the callers count
217
+ -- once per frame. pcall takes the function and its arguments rather than a
218
+ -- closure because on the Shared fanout that is N^2 allocations a frame.
219
+ -- `bytes` names the kind, `accepted` is what the transport took, `repair` a re-sent snapshot.
220
+ local function countBytesOut(bytes: buffer, accepted: number, repair: boolean?)
221
+ Metrics.Add("BytesOut", accepted)
222
+ local kind = BYTES_OUT_BY_FRAME[buffer.readu8(bytes, 0)]
223
+ if kind ~= nil then
224
+ Metrics.Add(kind, accepted)
225
+ end
226
+ if repair then
227
+ Metrics.Add("BytesOutResync", accepted)
228
+ end
229
+ end
230
+
231
+ local function deliver(entry: any, bytes: buffer): boolean
232
+ local ok, err = pcall(transport.SendToClient, transport, entry.Player, bytes)
233
+ if not ok then
234
+ Log.Warn("Transport", "SEND_FAIL", tostring(err), { Player = entry.Player })
235
+ return false
236
+ end
237
+ return true
238
+ end
239
+
240
+ -- Defined below, but needed by markResync's driver in the flush loop above it.
241
+ local sendInit: (entry: any) -> ()
242
+
243
+ -- A frame did not reach the transport, so this client's mirror is now a value the
244
+ -- server never had: Insert, Remove and Clear are not idempotent, so every later
245
+ -- op lands on the wrong base, and a loaded client never Hellos again. Repair is a
246
+ -- fresh Init, not a replay: re-broadcasting a SharedDiff would double-apply an
247
+ -- Insert on the other recipients.
248
+ local function markResync(entry: any, what: string)
249
+ if entry.ResyncPending then
250
+ return
251
+ end
252
+ entry.ResyncPending = true
253
+ markDirty(entry)
254
+ -- Both flags, and the queue with them: HandshakeReady false stops broadcasts to
255
+ -- a client whose mirror is wrong, and clearing QueueOpen keeps out the
256
+ -- (ready, not open) state in which PushOp silently discards every owner op.
257
+ entry.HandshakeReady = false
258
+ entry.QueueOpen = false
259
+ entry.Queue = {}
260
+ entry.QueueIndex = {}
261
+ Metrics.Add("MirrorResyncs")
262
+ Log.Warn(
263
+ "Replication",
264
+ "MIRROR_RESYNC",
265
+ `a {what} frame could not be delivered, so this client's mirror is stale; re-sending the whole snapshot`,
266
+ { Player = entry.Player }
267
+ )
268
+ end
269
+
270
+ -- One frame, to one client. Returns whether the whole frame reached the transport,
271
+ -- which sendInit needs: a snapshot that did not land must not mark the client
272
+ -- ready. Fragmentation happens here rather than in `deliver` so a broadcast splits
273
+ -- once. `snapshotOps` and `snapshotSpans` are passed only by sendInit.
274
+ local function sendBuffer(
275
+ entry: any,
276
+ bytes: buffer,
277
+ snapshotOps: { Wire.Op }?,
278
+ snapshotSpans: { number }?,
279
+ repair: boolean?
280
+ ): boolean
281
+ fragmentSeq = (fragmentSeq + 1) % 65536
282
+ local parts = 0
283
+ -- BytesOut counts what the transport accepted.
284
+ local accepted = 0
285
+ local whole = Wire.Fragment(bytes, maxOutbound, fragmentSeq, function(part)
286
+ parts += 1
287
+ if not deliver(entry, part) then
288
+ return false
289
+ end
290
+ accepted += buffer.len(part)
291
+ return true
292
+ end)
293
+ if whole == nil then
294
+ if not deliver(entry, bytes) then
295
+ return false
296
+ end
297
+ Metrics.Add("FramesSent")
298
+ Metrics.Observe("BytesOutPerSend", buffer.len(bytes))
299
+ countBytesOut(bytes, buffer.len(bytes), repair)
300
+ return true
301
+ end
302
+ if not whole then
303
+ -- Stop at the first refusal rather than pushing the rest: a hole in a stream
304
+ -- the receiver believes is contiguous could splice two frames together. A
305
+ -- partial stays a prefix, which the next frame's index 0 discards cleanly.
306
+ Metrics.Add("FragmentedSendFailed")
307
+ countBytesOut(bytes, accepted, repair)
308
+ return false
309
+ end
310
+ if parts > OVERSIZE_FRAGMENTS and not warnedOversize then
311
+ warnedOversize = true
312
+ local frameName = FRAME_NAMES[buffer.readu8(bytes, 0)] or "unknown"
313
+ local roots = describeRoots(snapshotOps, snapshotSpans)
314
+ Log.Warn(
315
+ "Replication",
316
+ "OUTBOUND_OVERSIZE",
317
+ `a single {frameName} frame needed {parts} fragments to send; it is delivered, but the profile or `
318
+ .. "diff behind it has grown well past MaxOutboundBytes and every join pays for it"
319
+ .. (if roots ~= "" then `; largest roots: {roots}` else ""),
320
+ {
321
+ Player = entry.Player,
322
+ Bytes = buffer.len(bytes),
323
+ Fragments = parts,
324
+ Frame = frameName,
325
+ Roots = roots,
326
+ }
327
+ )
328
+ end
329
+ Metrics.Add("FramesSent")
330
+ Metrics.Add("FramesFragmented")
331
+ Metrics.Add("FragmentsSent", parts)
332
+ -- The logical frame's size, once. BytesOutPerSend means bytes per outbound
333
+ -- frame, so sampling the fragments would describe the budget, not the frames.
334
+ Metrics.Observe("BytesOutPerSend", buffer.len(bytes))
335
+ countBytesOut(bytes, accepted, repair)
336
+ return true
337
+ end
338
+
339
+ -- One frame, to every ready client except `exceptEntry`. Returns the entries the
340
+ -- frame did not reach, or nil when every recipient got it; the only place that
341
+ -- knows which clients were missed.
342
+ --
343
+ -- BytesOutPerSend is observed once here, not once per recipient: per-recipient
344
+ -- sampling cost O(N^2) reservoir writes a frame and skewed the distribution toward
345
+ -- the fanout. FramesSent still counts every hand-off to the transport.
346
+ local function broadcastBuffer(bytes: buffer, exceptEntry: any?, alsoExcept: any?): { any }?
347
+ -- Split once, before the recipient loop and before the fast-path branch: the
348
+ -- split depends only on the buffer, and the fast path never reaches `deliver`,
349
+ -- so a fragmenter placed there would let every broadcast out unsplit.
350
+ fragmentSeq = (fragmentSeq + 1) % 65536
351
+ local parts: { buffer } = {}
352
+ local split = Wire.Fragment(bytes, maxOutbound, fragmentSeq, function(part)
353
+ parts[#parts + 1] = part
354
+ return true
355
+ end) ~= nil
356
+
357
+ if transport.SendToAllClients and exceptEntry == nil and alsoExcept == nil then
358
+ -- No per-recipient attribution on this path: one raise means the frame
359
+ -- reached NOBODY, so every ready client is a missed recipient.
360
+ local function everyReady(): { any }
361
+ local all: { any } = {}
362
+ for _, entry in ctx.Persistence.Entries do
363
+ if entry.HandshakeReady then
364
+ all[#all + 1] = entry
365
+ end
366
+ end
367
+ return all
368
+ end
369
+ -- Once per session: the O(1) stand-in for the clients the engine delivers to.
370
+ local sessions = ctx.Persistence.EntryCount
371
+ local accepted = 0
372
+ if not split then
373
+ local ok, err = pcall(transport.SendToAllClients, transport, bytes)
374
+ if not ok then
375
+ Log.Warn("Transport", "SEND_FAIL", tostring(err), {})
376
+ return everyReady()
377
+ end
378
+ accepted = buffer.len(bytes)
379
+ else
380
+ for _, part in parts do
381
+ local ok, err = pcall(transport.SendToAllClients, transport, part)
382
+ if not ok then
383
+ Log.Warn("Transport", "SEND_FAIL", tostring(err), {})
384
+ Metrics.Add("FragmentedSendFailed")
385
+ countBytesOut(bytes, accepted * sessions)
386
+ return everyReady()
387
+ end
388
+ accepted += buffer.len(part)
389
+ end
390
+ Metrics.Add("FramesFragmented")
391
+ Metrics.Add("FragmentsSent", #parts)
392
+ end
393
+ Metrics.Add("FramesSent")
394
+ Metrics.Observe("BytesOutPerSend", buffer.len(bytes))
395
+ countBytesOut(bytes, accepted * sessions)
396
+ return nil
397
+ end
398
+
399
+ local sent = 0
400
+ local accepted = 0
401
+ local missed: { any }? = nil
402
+ for _, entry in ctx.Persistence.Entries do
403
+ if entry.HandshakeReady and entry ~= exceptEntry and entry ~= alsoExcept then
404
+ local complete = true
405
+ if not split then
406
+ complete = deliver(entry, bytes)
407
+ if complete then
408
+ accepted += buffer.len(bytes)
409
+ end
410
+ else
411
+ for _, part in parts do
412
+ if not deliver(entry, part) then
413
+ complete = false
414
+ break
415
+ end
416
+ accepted += buffer.len(part)
417
+ end
418
+ end
419
+ if complete then
420
+ sent += 1
421
+ else
422
+ -- Allocated only on the failing path, so the ordinary fanout costs
423
+ -- nothing extra.
424
+ local list: { any } = missed or {}
425
+ list[#list + 1] = entry
426
+ missed = list
427
+ end
428
+ end
429
+ end
430
+ if sent > 0 then
431
+ -- Accumulated and added once, never per recipient: a Metrics.Add inside
432
+ -- this loop costs O(N^2) hash writes a frame.
433
+ Metrics.Add("FramesSent", sent)
434
+ if split then
435
+ Metrics.Add("FramesFragmented")
436
+ Metrics.Add("FragmentsSent", #parts * sent)
437
+ end
438
+ Metrics.Observe("BytesOutPerSend", buffer.len(bytes))
439
+ end
440
+ if accepted > 0 then
441
+ countBytesOut(bytes, accepted)
442
+ end
443
+ return missed
444
+ end
445
+
446
+ local function compactOps(queue: { Wire.Op }): { Wire.Op }
447
+ local supersede: { [string]: boolean } = {} -- own path + descendants (Set table/nil)
448
+ local descendantSupersede: { [string]: boolean } = {} -- descendants only (Clear)
449
+ local reversed: { Wire.Op } = {}
450
+ for i = #queue, 1, -1 do
451
+ local op = queue[i]
452
+ local k = (op :: any)._pk :: string
453
+ local superseded = supersede[k] == true
454
+ if not superseded then
455
+ local pos = string.find(k, "\31", 1, true)
456
+ while pos do
457
+ local ancestor = string.sub(k, 1, pos - 1)
458
+ if supersede[ancestor] or descendantSupersede[ancestor] then
459
+ superseded = true
460
+ break
461
+ end
462
+ pos = string.find(k, "\31", pos + 1, true)
463
+ end
464
+ end
465
+ if not superseded then
466
+ reversed[#reversed + 1] = op
467
+ if op.Kind == "Clear" then
468
+ descendantSupersede[k] = true
469
+ elseif op.Kind == "Set" and (op.Value == nil or type(op.Value) == "table") then
470
+ supersede[k] = true
471
+ elseif op.Kind == "Insert" or op.Kind == "Remove" then
472
+ -- Index-shift barrier: an Insert/Remove rebinds which element every
473
+ -- numeric path under this container denotes, so a later table-Set at
474
+ -- arr[i] does not supersede an earlier op targeting the element that
475
+ -- occupied i before the shift. Drop the descendant marks under it.
476
+ local prefix = k .. "\31"
477
+ local prefixLen = #prefix
478
+ for marked in supersede do
479
+ if string.sub(marked, 1, prefixLen) == prefix then
480
+ supersede[marked] = nil
481
+ end
482
+ end
483
+ for marked in descendantSupersede do
484
+ if string.sub(marked, 1, prefixLen) == prefix then
485
+ descendantSupersede[marked] = nil
486
+ end
487
+ end
488
+ end
489
+ end
490
+ end
491
+ local out = table.create(#reversed) :: { Wire.Op }
492
+ for i = #reversed, 1, -1 do
493
+ out[#out + 1] = reversed[i]
494
+ end
495
+ return out
496
+ end
497
+
498
+ -- Flush an entry's queued Shared ops as SharedDiff frames to every other ready
499
+ -- client. Extracted so sendInit can drain the queue before a new client snapshots
500
+ -- that player; otherwise the op lands twice, double-applying Insert/Remove/Clear.
501
+ --- Takes the queued Shared ops, leaving a fresh queue for writes made during a send.
502
+ local function detachSharedQueue(entry: any): { Wire.Op }
503
+ local shared: { Wire.Op } = entry.SharedQueue
504
+ entry.SharedQueue = {}
505
+ return shared
506
+ end
507
+
508
+ local function sendSharedOps(entry: any, shared: { Wire.Op }, alsoExcept: any?)
509
+ if #shared == 0 then
510
+ return
511
+ end
512
+ shared = compactOps(shared)
513
+ local offset = 1
514
+ while offset <= #shared do
515
+ local chunk = table.move(shared, offset, math.min(offset + MAX_OPS_PER_FRAME - 1, #shared), 1, {})
516
+ local ok, bytesOrErr = pcall(Wire.EncodeOpsFrame, Wire.Frames.SharedDiff, compiled, chunk, function(w)
517
+ Wire.F64(w, entry.UserId)
518
+ end)
519
+ if ok then
520
+ -- A SharedDiff cannot be replayed to the client that missed it: the other
521
+ -- recipients already applied it. It gets a whole new snapshot instead.
522
+ local missed = broadcastBuffer(bytesOrErr, entry, alsoExcept)
523
+ if missed ~= nil then
524
+ for _, other in missed do
525
+ markResync(other, "SharedDiff")
526
+ end
527
+ end
528
+ else
529
+ Log.Error("Replication", "SERIALIZE_FAIL", tostring(bytesOrErr), { Player = entry.Player })
530
+ end
531
+ offset += MAX_OPS_PER_FRAME
532
+ end
533
+ end
534
+
535
+ local function flushSharedQueue(entry: any)
536
+ sendSharedOps(entry, detachSharedQueue(entry))
537
+ end
538
+
539
+ function self.FlushEntry(entry: any)
540
+ local queue: { Wire.Op } = entry.Queue
541
+ if #queue > 0 and entry.HandshakeReady then
542
+ entry.Queue = {}
543
+ entry.QueueIndex = {}
544
+ queue = compactOps(queue)
545
+ local offset = 1
546
+ while offset <= #queue do
547
+ local chunk = table.move(queue, offset, math.min(offset + MAX_OPS_PER_FRAME - 1, #queue), 1, {})
548
+ local ok, bytesOrErr = pcall(Wire.EncodeOpsFrame, Wire.Frames.Diff, compiled, chunk)
549
+ if ok then
550
+ if not sendBuffer(entry, bytesOrErr) then
551
+ -- The queue was replaced at the top of this function, so these ops
552
+ -- exist nowhere else and a discarded refusal is permanent divergence.
553
+ -- Later chunks depend on this one, so stop the loop.
554
+ markResync(entry, "Diff")
555
+ break
556
+ end
557
+ -- Counted only for a frame that actually left, so OpsSent agrees
558
+ -- with FramesSent.
559
+ Metrics.Add("DiffFlushes")
560
+ Metrics.Add("OpsSent", #chunk)
561
+ else
562
+ Log.Error("Replication", "SERIALIZE_FAIL", tostring(bytesOrErr), { Player = entry.Player })
563
+ end
564
+ offset += MAX_OPS_PER_FRAME
565
+ end
566
+ elseif #queue > 0 and not entry.QueueOpen then
567
+ -- No snapshot has been taken, so whatever is queued will be in the snapshot
568
+ -- when one is. Must NOT fire while a snapshot is in flight: those ops are
569
+ -- exactly the ones the snapshot cannot carry.
570
+ entry.Queue = {}
571
+ entry.QueueIndex = {}
572
+ end
573
+
574
+ flushSharedQueue(entry)
575
+ end
576
+
577
+ local connections: { any } = {}
578
+ -- Every entry with queued work flushes every frame, with deliberately no per-frame
579
+ -- budget: a budget needs a round-robin cursor over an ordered entry list, and
580
+ -- Entries is keyed by Player Instance, where next(Entries, lastPlayer) raises once
581
+ -- that player has left and a rehash on join silently skips entries. The
582
+ -- superlinear path is the Shared fanout, which a budget does not touch.
583
+ --- One frame's flush. Extracted so the profiler has something to name and so the
584
+ --- frame cost can be timed around it.
585
+ -- One queue for the whole server: a single writer, one frame to every ready client.
586
+ local storeQueue: { Wire.Op } = {}
587
+ -- Lets flushFrame skip the drain entirely. Read from the compiled, not from
588
+ -- ctx.ServerStore, which is assigned after this constructor returns.
589
+ local hasStore = compiled.Views.Store ~= nil
590
+
591
+ --- Queue a store op. Same visibility rule as a player op: ServerOnly never leaves.
592
+ function self.PushStoreOp(op: Wire.Op, schema: Template.SchemaNode?)
593
+ local visibility = if schema then schema.Visibility else Template.VisibilityOf(compiled, op.Path)
594
+ if visibility == "ServerOnly" then
595
+ return
596
+ end
597
+ stripServerOnly(op, schema)
598
+ Metrics.Add("OpsQueued")
599
+ -- compactOps reads _pk unconditionally. The semicolon is load-bearing: a line opening
600
+ -- with `(` after a call is ambiguous to Luau.
601
+ local pathKey = Util.pathKey(op.Path);
602
+ (op :: any)._pk = pathKey
603
+ storeQueue[#storeQueue + 1] = op
604
+ end
605
+
606
+ --- Encode whatever the store queued and broadcast it to every ready client.
607
+ --- `except` skips one entry: the joiner whose snapshot already holds these ops.
608
+ local function sendStoreOps(queue: { Wire.Op }, except: any?): number
609
+ local sentOps = 0
610
+ local offset = 1
611
+ while offset <= #queue do
612
+ local chunk = table.move(queue, offset, math.min(offset + MAX_OPS_PER_FRAME - 1, #queue), 1, {})
613
+ local ok, bytesOrErr = pcall(Wire.EncodeOpsFrame, Wire.Frames.Diff, compiled, chunk)
614
+ if ok then
615
+ -- An ordinary Diff against the full compiled: no new frame kind needed.
616
+ local missed = broadcastBuffer(bytesOrErr, except)
617
+ Metrics.Add("OpsSent", #chunk)
618
+ sentOps += #chunk
619
+ if missed ~= nil then
620
+ -- Nothing else repairs a missed store diff, so force a re-Init.
621
+ for _, entry in missed do
622
+ markResync(entry, "StoreDiff")
623
+ end
624
+ end
625
+ else
626
+ Log.Error("Replication", "SERIALIZE_FAIL", tostring(bytesOrErr), {})
627
+ Metrics.Add("SerializeFailed")
628
+ end
629
+ offset += MAX_OPS_PER_FRAME
630
+ end
631
+ return sentOps
632
+ end
633
+
634
+ function self.FlushStoreQueue(except: any?): number
635
+ if #storeQueue == 0 then
636
+ return 0
637
+ end
638
+ local queue = compactOps(storeQueue)
639
+ storeQueue = {}
640
+ return sendStoreOps(queue, except)
641
+ end
642
+
643
+ local function flushFrame(): (number, number, boolean)
644
+ local flushed = 0
645
+ local queued = 0
646
+ local resynced = 0
647
+ local batch = dirty
648
+ dirty = {}
649
+ for entry in batch do
650
+ if entry.TornDown then
651
+ continue
652
+ end
653
+ -- Repair before flushing: a resync re-snapshots live state, so any op that
654
+ -- would have flushed this frame is already in it. Throttled on the previous
655
+ -- attempt, not on LastInitAt, whose clock starts at the join: gating on that
656
+ -- would leave a client that lost a frame just after its handshake stale.
657
+ if entry.ResyncPending and entry.State == "Ready" and not entry.TornDown then
658
+ local now = Clock.Monotonic()
659
+ if now - (entry.LastResyncAt or -math.huge) >= REINIT_COOLDOWN then
660
+ entry.LastResyncAt = now
661
+ entry.ResyncPending = false
662
+ -- pcall for the reason onHello gives: a throw escaping here would take
663
+ -- down the flush loop for every other player.
664
+ local okInit, initErr = pcall(sendInit, entry)
665
+ resynced += 1
666
+ if not okInit then
667
+ Log.Error("Replication", "INIT_SEND_FAIL", tostring(initErr), { Player = entry.Player })
668
+ Metrics.Add("InitSendFailed")
669
+ entry.ResyncPending = true
670
+ markDirty(entry)
671
+ end
672
+ end
673
+ end
674
+ -- Counted before the flush, because FlushEntry replaces both queues. Ops
675
+ -- pending, not ops sent: compaction shrinks it on the way to the wire.
676
+ local pending = #entry.Queue + #entry.SharedQueue
677
+ if pending > 0 then
678
+ flushed += 1
679
+ queued += pending
680
+ self.FlushEntry(entry)
681
+ end
682
+ if entry.ResyncPending or #entry.Queue > 0 or #entry.SharedQueue > 0 then
683
+ markDirty(entry)
684
+ end
685
+ end
686
+ -- The store drains once per frame, not per entry: one queue, one broadcast.
687
+ local storeOps = 0
688
+ if hasStore then
689
+ storeOps = self.FlushStoreQueue()
690
+ end
691
+ return flushed, queued, flushed > 0 or storeOps > 0 or resynced > 0
692
+ end
693
+
694
+ --- Ask for a fresh Init on the next frame, as a lost frame would.
695
+ function self.RequestResync(entry: any, what: string)
696
+ markResync(entry, what)
697
+ end
698
+
699
+ function self.DirtyCount(): number
700
+ local count = 0
701
+ for _ in dirty do
702
+ count += 1
703
+ end
704
+ return count
705
+ end
706
+
707
+ connections[#connections + 1] = RunService.PostSimulation:Connect(function()
708
+ -- No memory category here: this body runs on a thread the engine may recycle,
709
+ -- so the tag would outlive the call and stain whatever ran next on it.
710
+ if next(dirty) == nil and (not hasStore or #storeQueue == 0) then
711
+ return
712
+ end
713
+ local startedAt = Clock.Monotonic()
714
+ -- SyncSpan, and flushFrame is synchronous: it encodes and fires, and waits on
715
+ -- nothing. That is the precondition, not an incidental property.
716
+ local flushed, queued, worked = Profile.SyncSpan("Scribe.Flush", flushFrame)
717
+ -- Nothing is recorded for an idle frame: a distribution padded with zeroes
718
+ -- would drag every percentile down and hide the busy frames.
719
+ if flushed > 0 then
720
+ Metrics.Observe("FlushEntriesPerFrame", flushed)
721
+ Metrics.Observe("FlushQueuedPerFrame", queued)
722
+ end
723
+ -- The frame cost, in seconds, whatever the work was.
724
+ if worked then
725
+ Metrics.Observe("FlushDuration", Clock.Monotonic() - startedAt)
726
+ end
727
+ end)
728
+
729
+ -- Snapshots -------------------------------------------------------------------
730
+
731
+ local function ownerSnapshotOps(entry: any): { Wire.Op }
732
+ local ops: { Wire.Op } = {}
733
+ local rootChildren = compiled.Root.Children :: { [string]: Template.SchemaNode }
734
+ -- Store roots come from the server's tree, not this player's. They ride the Init
735
+ -- because applyInit clears and refills from defaults. Nil when no store is declared.
736
+ local storeData = if ctx.ServerStore ~= nil then ctx.ServerStore.Data else nil
737
+ for _, rootKey in compiled.RootOrder do
738
+ local rootSchema = rootChildren[rootKey]
739
+ local store
740
+ if storeData ~= nil and storeData[rootKey] ~= nil then
741
+ store = storeData
742
+ elseif not rootSchema.Persist then
743
+ store = entry.SessionStore
744
+ else
745
+ store = entry.LiveData
746
+ end
747
+ local value = if store then store[rootKey] else nil
748
+ if value ~= nil then
749
+ -- CloneVisible strips ServerOnly subtrees and returns nil for roots with
750
+ -- nothing visible (mixed roots like _Scribe keep their Replicated subkeys).
751
+ local visible = Template.CloneVisible(rootSchema, value)
752
+ if visible ~= nil then
753
+ table.insert(ops, { Kind = "Set", Path = { rootKey }, Value = visible })
754
+ end
755
+ end
756
+ end
757
+ return ops
758
+ end
759
+
760
+ local function sharedSnapshotOps(entry: any): { Wire.Op }
761
+ local ops: { Wire.Op } = {}
762
+ local rootChildren = compiled.Root.Children :: { [string]: Template.SchemaNode }
763
+ for _, rootKey in compiled.RootOrder do
764
+ local rootSchema = rootChildren[rootKey]
765
+ -- Same store choice as ownerSnapshotOps: a Shared root that is also Session
766
+ -- lives in the session store, so reading LiveData broadcasts its default.
767
+ local store = if not rootSchema.Persist then entry.SessionStore else entry.LiveData
768
+ if rootSchema.Visibility == "Shared" and store then
769
+ local value = store[rootKey]
770
+ if value ~= nil then
771
+ -- CloneVisible strips ServerOnly subtrees nested under the Shared root;
772
+ -- a plain deepClone would broadcast them.
773
+ local visible = Template.CloneVisible(rootSchema, value)
774
+ if visible ~= nil then
775
+ table.insert(ops, { Kind = "Set", Path = { rootKey }, Value = visible })
776
+ end
777
+ end
778
+ end
779
+ end
780
+ return ops
781
+ end
782
+
783
+ --- Returns whether a unicast status reached the client; always true for the
784
+ --- broadcast form, which has no single recipient to answer for.
785
+ local function sendStatus(entry: any?): boolean
786
+ local w = Wire.NewWriter(4)
787
+ Wire.U8(w, Wire.Frames.Status)
788
+ Wire.U8(w, STATUS_TO_ID[Health.GetStatus()] or 0)
789
+ local bytes = Wire.Finish(w)
790
+ if entry then
791
+ return sendBuffer(entry, bytes)
792
+ end
793
+ -- A missed broadcast Status is not mirror state: the next Health transition
794
+ -- rebroadcasts, and a resync re-sends it as part of the handshake.
795
+ broadcastBuffer(bytes)
796
+ return true
797
+ end
798
+
799
+ -- Size of the most recently encoded Init frame, across all players.
800
+ local lastInitBytes: number? = nil
801
+
802
+ function sendInit(entry: any)
803
+ -- Sent after the snapshot is taken, so a write made inside Send lands past the boundary.
804
+ local pending: { Wire.Op }? = nil
805
+ if hasStore and #storeQueue > 0 then
806
+ pending = compactOps(storeQueue)
807
+ storeQueue = {}
808
+ end
809
+ -- Open the queue in the same breath as taking the snapshot, with no yield
810
+ -- between them, so every write from here on is either in this clone or in the
811
+ -- queue behind it. The clear drops what the snapshot supersedes.
812
+ entry.Queue = {}
813
+ entry.QueueIndex = {}
814
+ entry.QueueOpen = true
815
+ -- A repair is an Init for a client that already has a snapshot.
816
+ local repair = entry.InitDelivered == true
817
+ -- Stamped at the start of the snapshot: measuring from completion would let a
818
+ -- slow encode widen its own cooldown.
819
+ entry.LastInitAt = Clock.Monotonic()
820
+ local ops = ownerSnapshotOps(entry)
821
+ if pending ~= nil then
822
+ sendStoreOps(pending, entry)
823
+ end
824
+ -- Per-root byte spans, filled by the encode, so the oversize warning can name
825
+ -- what is large. `shippedOps` is what the delivered frame holds, not `ops`.
826
+ local spans: { number } = {}
827
+ local shippedOps: { Wire.Op } = ops
828
+ local ok, bytesOrErr =
829
+ pcall(Wire.EncodeOpsFrame, Wire.Frames.Init, compiled, ops, nil, spans, entry.InitSizeHint or lastInitBytes)
830
+ if not ok then
831
+ -- One un-encodable root must never brick the client, which would block every
832
+ -- client API on waitForLoaded. Drop it, surface it, and send the remainder.
833
+ Log.Error("Replication", "SERIALIZE_FAIL", tostring(bytesOrErr), { Player = entry.Player })
834
+ local safeOps = {}
835
+ for _, op in ops do
836
+ local encodable = pcall(Wire.EncodeOpsFrame, Wire.Frames.Init, compiled, { op })
837
+ if encodable then
838
+ table.insert(safeOps, op)
839
+ else
840
+ Metrics.Add("SnapshotRootDropped")
841
+ Log.Error("Replication", "SNAPSHOT_ROOT_DROPPED", "root omitted from Init (unserializable)", {
842
+ Player = entry.Player,
843
+ Root = op.Path[1],
844
+ })
845
+ if ctx.Signals and ctx.Signals.OnAnomaly then
846
+ ctx.Signals.OnAnomaly:Fire({
847
+ Player = entry.Player,
848
+ Path = op.Path,
849
+ Value = nil,
850
+ Reason = "UnserializableSnapshotRoot",
851
+ })
852
+ end
853
+ end
854
+ end
855
+ -- A fresh spans table, and shippedOps with it: attributing the sent frame
856
+ -- against the failed one's ops would name roots this client never received.
857
+ spans = {}
858
+ shippedOps = safeOps
859
+ local okSafe, safeBytes = pcall(Wire.EncodeOpsFrame, Wire.Frames.Init, compiled, safeOps, nil, spans)
860
+ if not okSafe then
861
+ -- Should be unreachable (each op individually encoded); fall back
862
+ -- to an empty Init rather than leaving the client unloaded.
863
+ spans = {}
864
+ shippedOps = {}
865
+ safeBytes = Wire.EncodeOpsFrame(Wire.Frames.Init, compiled, {})
866
+ end
867
+ bytesOrErr = safeBytes
868
+ end
869
+ -- What this handshake costs the client in bytes, summed across every frame it
870
+ -- takes, not just the Init: the SharedInit fan-out that follows can dwarf it.
871
+ local joinBytes = buffer.len(bytesOrErr :: buffer)
872
+ -- Seed the next Init's writer, this player's and the next joiner's. The last
873
+ -- size, not a running maximum: one whale profile would permanently seed every
874
+ -- later joiner's writer at the whale's size. Kept per entry and per context: the
875
+ -- entry's own last Init is the better estimate for a re-init, the context's the
876
+ -- only one a first join has.
877
+ entry.InitSizeHint = joinBytes
878
+ lastInitBytes = joinBytes
879
+ if not sendBuffer(entry, bytesOrErr, shippedOps, spans, repair) then
880
+ -- The snapshot never reached the transport, so this client has no mirror for
881
+ -- anything that follows. Leaving HandshakeReady false keeps that recoverable:
882
+ -- the client's Hello retry asks again, and the handshake alarm still fires.
883
+ Metrics.Add("InitSendFailed")
884
+ Log.Error("Replication", "INIT_SEND_FAIL", "the initial snapshot could not be delivered", {
885
+ Player = entry.Player,
886
+ Bytes = buffer.len(bytesOrErr :: buffer),
887
+ })
888
+ -- Un-take the snapshot. FlushEntry flushes when HandshakeReady and clears
889
+ -- when not QueueOpen, so QueueOpen true with HandshakeReady false matches
890
+ -- neither branch and the queue would grow unbounded.
891
+ entry.Queue = {}
892
+ entry.QueueIndex = {}
893
+ entry.QueueOpen = false
894
+ -- On a re-Init these matter as much as the clear above: HandshakeReady is still
895
+ -- true from the first snapshot, and (ready, not open) is the state in which
896
+ -- PushOp silently discards every owner op forever. ResyncPending asks again,
897
+ -- since a loaded client has no Hello left to send.
898
+ entry.HandshakeReady = false
899
+ entry.ResyncPending = true
900
+ markDirty(entry)
901
+ return
902
+ end
903
+ -- Only now: a client marked ready before its snapshot is on the wire can be
904
+ -- handed a diff it has no mirror to apply to. broadcastBuffer skips unready
905
+ -- entries, which is also what kept the Shared drain above safe.
906
+ entry.HandshakeReady = true
907
+ entry.InitDelivered = true
908
+ -- Any pending repair request is met by this snapshot.
909
+ entry.ResyncPending = false
910
+ -- The snapshot landed but the handshake is not finished: a Status or SharedInit
911
+ -- that does not arrive leaves this client correct about itself and blind to
912
+ -- everyone else. Ask for the whole handshake again.
913
+ if not sendStatus(entry) then
914
+ entry.ResyncPending = true
915
+ markDirty(entry)
916
+ end
917
+ if ctx.Leaderboards then
918
+ ctx.Leaderboards.OnClientReady(entry)
919
+ end
920
+
921
+ -- Existing players' Shared roots for the new client. Gated on HasShared:
922
+ -- without it this walks all N entries per join purely to send nothing.
923
+ if compiled.HasShared then
924
+ for _, other in ctx.Persistence.Entries do
925
+ if other ~= entry and other.State == "Ready" then
926
+ -- Snapshot and detach before any send: the snapshot carries what was queued,
927
+ -- the others get it as a diff, and a write made during either send waits for
928
+ -- the next flush.
929
+ local drained = detachSharedQueue(other)
930
+ local sharedOps = sharedSnapshotOps(other)
931
+ sendSharedOps(other, drained, entry)
932
+ if #sharedOps > 0 then
933
+ local okShared, sharedBytes = pcall(
934
+ Wire.EncodeOpsFrame,
935
+ Wire.Frames.SharedInit,
936
+ compiled,
937
+ sharedOps,
938
+ function(w)
939
+ Wire.F64(w, other.UserId)
940
+ end
941
+ )
942
+ if okShared then
943
+ if sendBuffer(entry, sharedBytes, nil, nil, repair) then
944
+ joinBytes += buffer.len(sharedBytes)
945
+ else
946
+ -- A joiner that misses another player's SharedInit has no
947
+ -- mirror of them for the rest of the session.
948
+ entry.ResyncPending = true
949
+ markDirty(entry)
950
+ end
951
+ end
952
+ end
953
+ end
954
+ end
955
+ end
956
+ Metrics.Add("Handshakes")
957
+ -- Past the early return above, so a snapshot that never reached the transport
958
+ -- is not sampled: JoinBytes means what a completed handshake cost.
959
+ Metrics.Observe("JoinBytes", joinBytes)
960
+ Log.Debug("Replication", "INIT_SENT", "initial snapshot sent", { Player = entry.Player })
961
+ end
962
+
963
+ -- Handshake ---------------------------------------------------------------
964
+
965
+ local function onHello(entry: any, reader: Wire.Reader)
966
+ local schemaHash = Wire.RU32(reader)
967
+ -- The client sends its wire PROTOCOL_VERSION after the hash. A mismatch means
968
+ -- the binary layout differs, not just the template, so decoding is unsafe: fail
969
+ -- closed, unlike the hash check below which the value-mode prefix keeps safe.
970
+ local protocolVersion = Wire.RU8(reader)
971
+ if protocolVersion ~= Wire.PROTOCOL_VERSION then
972
+ -- Metered as well as logged, so every refusal is countable.
973
+ Metrics.Add("ProtocolMismatch")
974
+ Log.Warn("Replication", "PROTOCOL_MISMATCH", "client wire protocol version differs from the server", {
975
+ Player = entry.Player,
976
+ })
977
+ return
978
+ end
979
+ if schemaHash ~= compiled.Hash % 4294967296 then
980
+ -- Fail closed. Paths decode from the receiver's own compiled.ById, so a field
981
+ -- id shared by two drifted templates maps to a different path on each side and
982
+ -- a server op lands on the wrong client key. The value-mode prefix keeps value
983
+ -- decoding safe, not path remapping.
984
+ Metrics.Add("SchemaMismatch")
985
+ Log.Warn(
986
+ "Replication",
987
+ "SCHEMA_MISMATCH",
988
+ "client derived a different schema than the server; refusing to replicate (template drift)",
989
+ { Player = entry.Player }
990
+ )
991
+ return
992
+ end
993
+ entry.HelloReceived = true
994
+ if entry.State == "Ready" then
995
+ -- A repeated Hello re-Inits (client-driven resync), but a full snapshot
996
+ -- encode is expensive, so throttle floods.
997
+ if entry.QueueOpen and Clock.Monotonic() - entry.LastInitAt < REINIT_COOLDOWN then
998
+ Metrics.Add("ReInitThrottled")
999
+ return
1000
+ end
1001
+ -- onHello runs inside the inbound pcall, whose failure branch charges the
1002
+ -- client's malformed budget and blocks its session after 32 strikes. Every
1003
+ -- throw sendInit can produce is a server fault, so an escaping one bans a
1004
+ -- player for a bug they did not cause.
1005
+ local okInit, initErr = pcall(sendInit, entry)
1006
+ if not okInit then
1007
+ Log.Error("Replication", "INIT_SEND_FAIL", tostring(initErr), { Player = entry.Player })
1008
+ Metrics.Add("InitSendFailed")
1009
+ end
1010
+ end
1011
+ end
1012
+
1013
+ function self.OnEntryReady(entry: any)
1014
+ -- QueueOpen, not HandshakeReady: the question is whether a snapshot has already
1015
+ -- been started, which is the same question the re-Init throttle asks. The two
1016
+ -- reading different flags is how the gap opened.
1017
+ if entry.HelloReceived and not entry.QueueOpen then
1018
+ local okInit, initErr = pcall(sendInit, entry)
1019
+ if not okInit then
1020
+ Log.Error("Replication", "INIT_SEND_FAIL", tostring(initErr), { Player = entry.Player })
1021
+ Metrics.Add("InitSendFailed")
1022
+ end
1023
+ end
1024
+ -- Broadcast this player's Shared snapshot. The queue is cleared before the send: what
1025
+ -- was queued is in the snapshot, and a write made during the send is the next flush's.
1026
+ local sharedOps = sharedSnapshotOps(entry)
1027
+ entry.SharedQueue = {}
1028
+ if #sharedOps > 0 then
1029
+ local ok, bytes = pcall(Wire.EncodeOpsFrame, Wire.Frames.SharedInit, compiled, sharedOps, function(w)
1030
+ Wire.F64(w, entry.UserId)
1031
+ end)
1032
+ if ok then
1033
+ local missed = broadcastBuffer(bytes, entry)
1034
+ if missed ~= nil then
1035
+ for _, other in missed do
1036
+ markResync(other, "SharedInit")
1037
+ end
1038
+ end
1039
+ end
1040
+ end
1041
+ Clock.Delay(HANDSHAKE_TIMEOUT, function()
1042
+ if not entry.TornDown and not entry.HandshakeReady then
1043
+ if entry.HelloReceived then
1044
+ -- A different failure with the same symptom: the client did ask and the
1045
+ -- answer could not be delivered. MaxOutboundBytes is named because a
1046
+ -- transport refusing a payload this size is the one innocent cause.
1047
+ Log.Warn(
1048
+ "Replication",
1049
+ "INIT_UNDELIVERED",
1050
+ "the client sent Hello but its snapshot never reached the transport; check MaxOutboundBytes against what this transport will carry",
1051
+ {
1052
+ Player = entry.Player,
1053
+ }
1054
+ )
1055
+ else
1056
+ Log.Warn(
1057
+ "Replication",
1058
+ "CLIENT_HANDSHAKE_TIMEOUT",
1059
+ "client never sent Hello, so replication never started; require the Scribe bundle on the client (a LocalScript), or check a custom client transport adapter",
1060
+ {
1061
+ Player = entry.Player,
1062
+ }
1063
+ )
1064
+ end
1065
+ end
1066
+ end)
1067
+ end
1068
+
1069
+ function self.OnEntryRemoved(entry: any)
1070
+ dirty[entry] = nil
1071
+ entry.HandshakeReady = false
1072
+ -- Symmetry rather than a fix: a removed entry's tree is already dead, so
1073
+ -- nothing can push to it.
1074
+ entry.QueueOpen = false
1075
+ -- Only meaningful when the template declares a Shared root: without one there is
1076
+ -- no mirror to drop. Sending it anyway costs one frame per remaining client on
1077
+ -- every leave, making a full server drain quadratic.
1078
+ if not compiled.HasShared then
1079
+ return
1080
+ end
1081
+ local w = Wire.NewWriter(16)
1082
+ Wire.U8(w, Wire.Frames.SharedGone)
1083
+ Wire.F64(w, entry.UserId)
1084
+ -- A missed SharedGone leaves a mirror of a player who has left, which no later
1085
+ -- frame ever clears.
1086
+ local missed = broadcastBuffer(Wire.Finish(w), entry)
1087
+ if missed ~= nil then
1088
+ for _, other in missed do
1089
+ markResync(other, "SharedGone")
1090
+ end
1091
+ end
1092
+ end
1093
+
1094
+ function self.BroadcastStatus()
1095
+ sendStatus(nil)
1096
+ end
1097
+
1098
+ --- Returns whether the frame reached the client. Leaderboards depends on it:
1099
+ --- stamping SentState for a refused board means broadcastBoard's dedup never
1100
+ --- resends it, leaving the client on a stale board.
1101
+ function self.SendTo(entry: any, bytes: buffer): boolean
1102
+ return sendBuffer(entry, bytes)
1103
+ end
1104
+
1105
+ function self.Broadcast(bytes: buffer)
1106
+ local missed = broadcastBuffer(bytes)
1107
+ if missed ~= nil then
1108
+ for _, other in missed do
1109
+ markResync(other, "Broadcast")
1110
+ end
1111
+ end
1112
+ end
1113
+
1114
+ connections[#connections + 1] = Health.OnStatusChanged:Connect(function()
1115
+ self.BroadcastStatus()
1116
+ end)
1117
+
1118
+ -- Inbound dispatch -------------------------------------------------
1119
+
1120
+ transport:ListenServer(function(player: Player, bytes: buffer)
1121
+ local entry = ctx.Persistence.Entries[player]
1122
+ if not entry then
1123
+ return
1124
+ end
1125
+ if entry.MalformedBlocked or entry.OversizeBlocked then
1126
+ -- Past the malformed or oversize limit; drop without parsing so a flood of
1127
+ -- crafted frames cannot keep burning CPU. Two counters because the logs need
1128
+ -- to say which budget tripped.
1129
+ return
1130
+ end
1131
+ -- Ahead of the size check on purpose: this is the one ceiling that sees every
1132
+ -- frame whatever its type, size or validity, and it is spent before the frame
1133
+ -- is looked at. Silent, because replying is the amplification this tier avoids.
1134
+ if not takeFrameToken(entry) then
1135
+ Metrics.Add("InboundRateLimited")
1136
+ local now = Clock.Monotonic()
1137
+ if now - (entry.LastInboundRateLog or -math.huge) >= MALFORMED_LOG_INTERVAL then
1138
+ entry.LastInboundRateLog = now
1139
+ Log.Warn("Transport", "INBOUND_RATE_LIMITED", `player exceeded {inboundFrameRate} frames/s`, {
1140
+ Player = player,
1141
+ })
1142
+ end
1143
+ return
1144
+ end
1145
+ if buffer.len(bytes) > maxInbound then
1146
+ entry.OversizeCount += 1
1147
+ Metrics.Add("InboundOversize")
1148
+ -- Throttle the log, on its own clock: this frame is attacker-controlled, so
1149
+ -- logging every one would evict the diagnostic ring. The metric counts all.
1150
+ local now = Clock.Monotonic()
1151
+ if now - (entry.LastOversizeLog or -math.huge) >= MALFORMED_LOG_INTERVAL then
1152
+ entry.LastOversizeLog = now
1153
+ Log.Warn(
1154
+ "Transport",
1155
+ "INBOUND_OVERSIZE",
1156
+ `dropped oversized inbound frame ({buffer.len(bytes)} bytes)`,
1157
+ { Player = player, Count = entry.OversizeCount }
1158
+ )
1159
+ end
1160
+ if entry.OversizeCount >= OVERSIZE_MAX and not entry.OversizeBlocked then
1161
+ entry.OversizeBlocked = true
1162
+ Log.Warn(
1163
+ "Transport",
1164
+ "INBOUND_OVERSIZE_LIMIT",
1165
+ `player exceeded {OVERSIZE_MAX} oversized frames; ignoring further inbound frames this session `
1166
+ .. `(if legitimate client traffic is this large, raise MaxInboundBytes)`,
1167
+ { Player = player, Count = entry.OversizeCount, Limit = maxInbound }
1168
+ )
1169
+ end
1170
+ return
1171
+ end
1172
+ -- Built outside the pcall so its overBudget flag survives the failure. The
1173
+ -- retained-bytes budget is what the byte cap above cannot express: `06 00` is
1174
+ -- two bytes and allocates, eight kilobytes of one string allocates once.
1175
+ local reader = Wire.NewReader(bytes, maxInboundRetained)
1176
+ local ok, err = pcall(function()
1177
+ local frameType = Wire.RU8(reader)
1178
+ if frameType == Wire.Frames.Hello then
1179
+ onHello(entry, reader)
1180
+ elseif frameType == Wire.Frames.Command then
1181
+ ctx.Commands.HandleFrame(entry, reader)
1182
+ else
1183
+ error(`unexpected inbound frame type {frameType}`)
1184
+ end
1185
+ end)
1186
+ if not ok and reader.overBudget then
1187
+ -- Its own counter, not the malformed budget, and deliberately with no blocking
1188
+ -- limit: a frame over this budget may be a legal payload from the game's own
1189
+ -- client, and a 32-strike session-permanent budget would drop that player for
1190
+ -- playing the game. Silent, like the rate limiter.
1191
+ entry.WorkLimitCount = (entry.WorkLimitCount or 0) + 1
1192
+ Metrics.Add("InboundWorkLimited")
1193
+ local now = Clock.Monotonic()
1194
+ if now - (entry.LastWorkLimitLog or -math.huge) >= MALFORMED_LOG_INTERVAL then
1195
+ entry.LastWorkLimitLog = now
1196
+ Log.Warn(
1197
+ "Transport",
1198
+ "INBOUND_WORK_LIMIT",
1199
+ `dropped an inbound frame that would retain more than {maxInboundRetained} bytes `
1200
+ .. `(if legitimate client traffic is this large, raise MaxInboundRetainedBytes)`,
1201
+ { Player = player, Count = entry.WorkLimitCount, Limit = maxInboundRetained }
1202
+ )
1203
+ end
1204
+ elseif not ok then
1205
+ entry.MalformedCount += 1
1206
+ Metrics.Add("MalformedFrames")
1207
+ -- Throttle the log + anomaly so a crafted-frame flood can't spam the log
1208
+ -- buffer or an OnAnomaly handler (the count still increments every time).
1209
+ local now = Clock.Monotonic()
1210
+ if now - (entry.LastMalformedLog or -math.huge) >= MALFORMED_LOG_INTERVAL then
1211
+ entry.LastMalformedLog = now
1212
+ Log.Warn("Transport", "MALFORMED_FRAME", tostring(err), {
1213
+ Player = player,
1214
+ Count = entry.MalformedCount,
1215
+ })
1216
+ ctx.Signals.OnAnomaly:Fire({
1217
+ Player = player,
1218
+ Path = {},
1219
+ Value = nil,
1220
+ Reason = "MalformedFrame",
1221
+ })
1222
+ end
1223
+ -- Past the limit, stop dispatching this player's inbound frames entirely
1224
+ -- for the rest of the session (a real client never sends malformed frames).
1225
+ if entry.MalformedCount >= MALFORMED_MAX and not entry.MalformedBlocked then
1226
+ entry.MalformedBlocked = true
1227
+ Log.Warn(
1228
+ "Transport",
1229
+ "MALFORMED_FRAME_LIMIT",
1230
+ `player exceeded {MALFORMED_MAX} malformed frames; ignoring further inbound frames this session`,
1231
+ { Player = player, Count = entry.MalformedCount }
1232
+ )
1233
+ end
1234
+ end
1235
+ end)
1236
+
1237
+ -- Release the per-frame flush handler and the status listener; without this a
1238
+ -- discarded bundle keeps flushing against a dead Persistence forever.
1239
+ function self.Stop()
1240
+ stopped = true
1241
+ for _, connection in connections do
1242
+ pcall(function()
1243
+ connection:Disconnect()
1244
+ end)
1245
+ end
1246
+ table.clear(connections)
1247
+ table.clear(dirty)
1248
+ end
1249
+
1250
+ return self
1251
+ end
1252
+
1253
+ return Replication