@rbxts/scribe 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +21 -0
- package/README.md +100 -0
- package/Scribe-source-map.json +1 -0
- package/licenses/ProfileStore-Apache-2.0.txt +201 -0
- package/package.json +32 -0
- package/src/Client/DebugHook.luau +457 -0
- package/src/Client/init.luau +1607 -0
- package/src/Internal/Addons.luau +68 -0
- package/src/Internal/Big.luau +582 -0
- package/src/Internal/Budget.luau +377 -0
- package/src/Internal/Clock.luau +75 -0
- package/src/Internal/Config.luau +17 -0
- package/src/Internal/Conform.luau +400 -0
- package/src/Internal/Datatypes.luau +653 -0
- package/src/Internal/Defaults.luau +12 -0
- package/src/Internal/Derived.luau +428 -0
- package/src/Internal/Exchange.luau +556 -0
- package/src/Internal/ExchangeLeg.luau +1368 -0
- package/src/Internal/ExchangeVerdict.luau +179 -0
- package/src/Internal/Failure.luau +148 -0
- package/src/Internal/Health.luau +320 -0
- package/src/Internal/LeaderboardDiagnostics.luau +36 -0
- package/src/Internal/Log.luau +378 -0
- package/src/Internal/Marker.luau +76 -0
- package/src/Internal/Metrics.luau +152 -0
- package/src/Internal/Node.luau +3954 -0
- package/src/Internal/Profile.luau +69 -0
- package/src/Internal/Rng.luau +83 -0
- package/src/Internal/Sched.luau +56 -0
- package/src/Internal/Signal.luau +199 -0
- package/src/Internal/Store/ProfileStore.luau +3019 -0
- package/src/Internal/Store/init.luau +18 -0
- package/src/Internal/StudioCodec.luau +155 -0
- package/src/Internal/Template.luau +1848 -0
- package/src/Internal/Transport.luau +169 -0
- package/src/Internal/Util.luau +517 -0
- package/src/Internal/Version.luau +4 -0
- package/src/Internal/Wire.luau +1283 -0
- package/src/Server/Analytics.luau +223 -0
- package/src/Server/Commands.luau +728 -0
- package/src/Server/DebugHook.luau +1490 -0
- package/src/Server/Exchanges.luau +737 -0
- package/src/Server/Leaderboards.luau +2143 -0
- package/src/Server/Monetization.luau +3212 -0
- package/src/Server/Persistence.luau +3775 -0
- package/src/Server/Replication.luau +1253 -0
- package/src/Server/ServerStore.luau +81 -0
- package/src/Server/Timed.luau +579 -0
- package/src/Server/init.luau +672 -0
- package/src/Types.luau +480 -0
- package/src/init.luau +2231 -0
- package/types/api.d.ts +252 -0
- package/types/common.d.ts +355 -0
- package/types/index.d.ts +226 -0
- package/types/schema.d.ts +276 -0
|
@@ -0,0 +1,1253 @@
|
|
|
1
|
+
--!strict
|
|
2
|
+
-- Replication: per-player dirty op queues flushed once per frame
|
|
3
|
+
-- on PostSimulation, client-ready handshake, root visibility filtering, Shared
|
|
4
|
+
-- broadcast, and inbound frame dispatch (size caps + pcall'd decode).
|
|
5
|
+
|
|
6
|
+
local RunService = game:GetService("RunService")
|
|
7
|
+
|
|
8
|
+
local Internal = script.Parent.Parent.Internal
|
|
9
|
+
local Util = require(Internal.Util)
|
|
10
|
+
local Wire = require(Internal.Wire)
|
|
11
|
+
local Template = require(Internal.Template)
|
|
12
|
+
local Log = require(Internal.Log)
|
|
13
|
+
local Metrics = require(Internal.Metrics)
|
|
14
|
+
local Health = require(Internal.Health)
|
|
15
|
+
local Clock = require(Internal.Clock)
|
|
16
|
+
local Profile = require(Internal.Profile)
|
|
17
|
+
|
|
18
|
+
local Replication = {}
|
|
19
|
+
|
|
20
|
+
local HANDSHAKE_TIMEOUT = 10
|
|
21
|
+
local MAX_OPS_PER_FRAME = 60000
|
|
22
|
+
local REINIT_COOLDOWN = 5 -- seconds between full snapshots for repeated Hellos
|
|
23
|
+
local MALFORMED_LOG_INTERVAL = 1 -- seconds; throttle malformed-frame logs/anomalies per player
|
|
24
|
+
local MALFORMED_MAX = 32 -- after this many malformed frames, ignore all further inbound from the player
|
|
25
|
+
-- Its own budget rather than counting toward MALFORMED_MAX: a malformed frame
|
|
26
|
+
-- decoded and failed, an oversize frame was never read, and a shared counter would
|
|
27
|
+
-- let either mask the other.
|
|
28
|
+
local OVERSIZE_MAX = 32
|
|
29
|
+
|
|
30
|
+
local STATUS_TO_ID: { [string]: number } = { Healthy = 0, Degraded = 1, Outage = 2 }
|
|
31
|
+
|
|
32
|
+
-- Inverted once at load, so the oversize warning can name the kind of frame.
|
|
33
|
+
local FRAME_NAMES: { [number]: string } = {}
|
|
34
|
+
-- BytesOut by frame kind, keyed by the type byte.
|
|
35
|
+
local BYTES_OUT_BY_FRAME: { [number]: string } = {}
|
|
36
|
+
for name, id in Wire.Frames :: { [string]: number } do
|
|
37
|
+
FRAME_NAMES[id] = name
|
|
38
|
+
BYTES_OUT_BY_FRAME[id] = `BytesOut{name}`
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
function Replication.new(ctx: any)
|
|
42
|
+
local self = {} :: any
|
|
43
|
+
local compiled: Template.Compiled = ctx.Compiled
|
|
44
|
+
local transport = ctx.Transport
|
|
45
|
+
local options = ctx.Options
|
|
46
|
+
|
|
47
|
+
local maxInbound: number = options.MaxInboundBytes or 8192
|
|
48
|
+
-- Derived from the byte cap so the two stay in proportion: one inbound frame may
|
|
49
|
+
-- retain at most 16x its own bytes. 16 admits the densest legitimate shape (an
|
|
50
|
+
-- array of two-byte strings, 14.5x) while refusing the hostile ones (arrays of
|
|
51
|
+
-- booleans or of empty tables both cost 32x). See Wire.NewReader.
|
|
52
|
+
local maxInboundRetained: number = options.MaxInboundRetainedBytes or math.max(4096, maxInbound * 16)
|
|
53
|
+
local inboundFrameRate: number = ctx.InboundFrameRate
|
|
54
|
+
|
|
55
|
+
-- The largest buffer Scribe hands a transport in one call; anything above it is
|
|
56
|
+
-- fragmented (Wire.Fragment) and reassembled by the client. A ceiling is needed
|
|
57
|
+
-- because an Init snapshot is one op per root and MAX_OPS_PER_FRAME cannot divide
|
|
58
|
+
-- a single op. 65536 sits far below any platform payload limit and leaves ordinary
|
|
59
|
+
-- diffs unsplit. A smaller transport MaxFrameBytes wins: only the adapter knows
|
|
60
|
+
-- its own framing overhead.
|
|
61
|
+
local declaredMax: number? = (transport :: any).MaxFrameBytes
|
|
62
|
+
local maxOutbound: number = options.MaxOutboundBytes or 65536
|
|
63
|
+
if type(declaredMax) == "number" then
|
|
64
|
+
maxOutbound = math.min(maxOutbound, declaredMax)
|
|
65
|
+
end
|
|
66
|
+
-- At maxOutbound = 64 the u16 fragment count caps a logical frame at 3.7 MB,
|
|
67
|
+
-- which a real profile can reach. Refuse it at construction, where it is a typo.
|
|
68
|
+
if maxOutbound < 256 then
|
|
69
|
+
error(`Scribe: MaxOutboundBytes must be at least 256 (got {maxOutbound})`, 0)
|
|
70
|
+
end
|
|
71
|
+
-- Identifies one logical fragmented frame so the client can tell two apart. Wraps
|
|
72
|
+
-- at 65536; a collision needs 65535 intervening frames mid-reassembly.
|
|
73
|
+
local fragmentSeq = 0
|
|
74
|
+
|
|
75
|
+
-- One log per server, the first time a frame needs more fragments than this.
|
|
76
|
+
-- Measures the wire, unlike the profile-size warning: a Session root is absent
|
|
77
|
+
-- from LiveData but still rides in Init. Latched, being a template property.
|
|
78
|
+
local OVERSIZE_FRAGMENTS = 16
|
|
79
|
+
local warnedOversize = false
|
|
80
|
+
|
|
81
|
+
-- Same token-bucket shape as the command limiter, on its own per-entry clock:
|
|
82
|
+
-- refills at `inboundFrameRate` per second and caps there.
|
|
83
|
+
local function takeFrameToken(entry: any): boolean
|
|
84
|
+
local now = Clock.Monotonic()
|
|
85
|
+
local elapsed = now - entry.FrameLast
|
|
86
|
+
entry.FrameLast = now
|
|
87
|
+
entry.FrameTokens = math.min(inboundFrameRate, entry.FrameTokens + elapsed * inboundFrameRate)
|
|
88
|
+
if entry.FrameTokens >= 1 then
|
|
89
|
+
entry.FrameTokens -= 1
|
|
90
|
+
return true
|
|
91
|
+
end
|
|
92
|
+
return false
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
-- Op queueing ---------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
-- A whole-container Set/Insert can carry statically-declared ServerOnly
|
|
98
|
+
-- descendants inside its table value; strip them before the value reaches any
|
|
99
|
+
-- client queue (the Init snapshot does the same via CloneVisible).
|
|
100
|
+
local function stripServerOnly(op: Wire.Op, schema: Template.SchemaNode?)
|
|
101
|
+
if type(op.Value) ~= "table" or schema == nil then
|
|
102
|
+
return
|
|
103
|
+
end
|
|
104
|
+
local valueNode: Template.SchemaNode? = schema
|
|
105
|
+
if op.Kind == "Insert" then
|
|
106
|
+
valueNode = Template.ChildSchema(schema, 1)
|
|
107
|
+
end
|
|
108
|
+
if valueNode ~= nil then
|
|
109
|
+
op.Value = Template.CloneVisible(valueNode, op.Value)
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
-- Entries with something to flush or repair; never a departed one or a stopped bundle's.
|
|
114
|
+
local dirty: { [any]: boolean } = {}
|
|
115
|
+
local stopped = false
|
|
116
|
+
local function markDirty(entry: any)
|
|
117
|
+
if not stopped and not entry.TornDown then
|
|
118
|
+
dirty[entry] = true
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
function self.PushOp(entry: any, op: Wire.Op, schema: Template.SchemaNode?)
|
|
123
|
+
local visibility = if schema then schema.Visibility else Template.VisibilityOf(compiled, op.Path)
|
|
124
|
+
if visibility == "ServerOnly" then
|
|
125
|
+
return
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
stripServerOnly(op, schema)
|
|
129
|
+
|
|
130
|
+
Metrics.Add("OpsQueued")
|
|
131
|
+
-- Stamped on every op that may be queued, owner or Shared, so flush-time
|
|
132
|
+
-- compaction can read it in either queue.
|
|
133
|
+
local pathKey = Util.pathKey(op.Path);
|
|
134
|
+
(op :: any)._pk = pathKey
|
|
135
|
+
if entry.QueueOpen then
|
|
136
|
+
markDirty(entry)
|
|
137
|
+
local queue: { Wire.Op } = entry.Queue
|
|
138
|
+
if op.Kind == "Set" and op.Value ~= nil and type(op.Value) ~= "table" then
|
|
139
|
+
-- Scalar leaf Set: coalesce consecutive writes to the same path in place.
|
|
140
|
+
-- Supersession by whole-container writes is resolved at flush (compactOps).
|
|
141
|
+
local existing = entry.QueueIndex[pathKey]
|
|
142
|
+
if
|
|
143
|
+
existing
|
|
144
|
+
and queue[existing]
|
|
145
|
+
and queue[existing].Kind == "Set"
|
|
146
|
+
and (queue[existing] :: any)._pk == pathKey
|
|
147
|
+
then
|
|
148
|
+
queue[existing] = op
|
|
149
|
+
Metrics.Add("OpsCoalesced")
|
|
150
|
+
else
|
|
151
|
+
table.insert(queue, op)
|
|
152
|
+
entry.QueueIndex[pathKey] = #queue
|
|
153
|
+
end
|
|
154
|
+
else
|
|
155
|
+
-- Whole-container writes (table Set, nil Set, Clear) and array ops append,
|
|
156
|
+
-- and break scalar coalescing here and below: a later child Set would
|
|
157
|
+
-- otherwise collapse into a preceding slot and be dropped as superseded.
|
|
158
|
+
table.insert(queue, op)
|
|
159
|
+
entry.QueueIndex[pathKey] = nil
|
|
160
|
+
local descendantPrefix = pathKey .. "\31"
|
|
161
|
+
local prefixLen = #descendantPrefix
|
|
162
|
+
for k in entry.QueueIndex do
|
|
163
|
+
if string.sub(k, 1, prefixLen) == descendantPrefix then
|
|
164
|
+
entry.QueueIndex[k] = nil
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
-- Before the snapshot is taken there is nothing to diff against: the snapshot
|
|
170
|
+
-- about to be cloned carries this value, so queueing it would send it twice.
|
|
171
|
+
|
|
172
|
+
if visibility == "Shared" then
|
|
173
|
+
table.insert(entry.SharedQueue, op)
|
|
174
|
+
markDirty(entry)
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
-- Ranks a snapshot's roots by the bytes they contributed, for the oversize
|
|
179
|
+
-- warning. Scribe's own two roots are skipped: tens of bytes, never the answer.
|
|
180
|
+
-- "not persisted" means ProfileSize and the DataStore ceiling say nothing about
|
|
181
|
+
-- that root, not that it is unbounded.
|
|
182
|
+
local function describeRoots(ops: { Wire.Op }?, spans: { number }?): string
|
|
183
|
+
if ops == nil or spans == nil then
|
|
184
|
+
return ""
|
|
185
|
+
end
|
|
186
|
+
local ranked: { { Key: string, Bytes: number } } = {}
|
|
187
|
+
for i, op in ops do
|
|
188
|
+
local key = op.Path[1]
|
|
189
|
+
local span = spans[i]
|
|
190
|
+
if
|
|
191
|
+
type(key) == "string"
|
|
192
|
+
and span ~= nil
|
|
193
|
+
and key ~= Template.RESERVED_ROOT
|
|
194
|
+
and key ~= Template.SESSION_ROOT
|
|
195
|
+
then
|
|
196
|
+
table.insert(ranked, { Key = key, Bytes = span })
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
if #ranked == 0 then
|
|
200
|
+
return ""
|
|
201
|
+
end
|
|
202
|
+
table.sort(ranked, function(a: { Key: string, Bytes: number }, b: { Key: string, Bytes: number })
|
|
203
|
+
return a.Bytes > b.Bytes
|
|
204
|
+
end)
|
|
205
|
+
local rootChildren = compiled.Root.Children :: { [string]: Template.SchemaNode }
|
|
206
|
+
local out: { string } = {}
|
|
207
|
+
for i = 1, math.min(3, #ranked) do
|
|
208
|
+
local row = ranked[i]
|
|
209
|
+
local node = rootChildren[row.Key]
|
|
210
|
+
local suffix = if node ~= nil and node.Persist == false then " (not persisted)" else ""
|
|
211
|
+
table.insert(out, `{row.Key}={row.Bytes}B{suffix}`)
|
|
212
|
+
end
|
|
213
|
+
return table.concat(out, ", ")
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
-- One delivery to one client, with no accounting of its own: the callers count
|
|
217
|
+
-- once per frame. pcall takes the function and its arguments rather than a
|
|
218
|
+
-- closure because on the Shared fanout that is N^2 allocations a frame.
|
|
219
|
+
-- `bytes` names the kind, `accepted` is what the transport took, `repair` a re-sent snapshot.
|
|
220
|
+
local function countBytesOut(bytes: buffer, accepted: number, repair: boolean?)
|
|
221
|
+
Metrics.Add("BytesOut", accepted)
|
|
222
|
+
local kind = BYTES_OUT_BY_FRAME[buffer.readu8(bytes, 0)]
|
|
223
|
+
if kind ~= nil then
|
|
224
|
+
Metrics.Add(kind, accepted)
|
|
225
|
+
end
|
|
226
|
+
if repair then
|
|
227
|
+
Metrics.Add("BytesOutResync", accepted)
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
local function deliver(entry: any, bytes: buffer): boolean
|
|
232
|
+
local ok, err = pcall(transport.SendToClient, transport, entry.Player, bytes)
|
|
233
|
+
if not ok then
|
|
234
|
+
Log.Warn("Transport", "SEND_FAIL", tostring(err), { Player = entry.Player })
|
|
235
|
+
return false
|
|
236
|
+
end
|
|
237
|
+
return true
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
-- Defined below, but needed by markResync's driver in the flush loop above it.
|
|
241
|
+
local sendInit: (entry: any) -> ()
|
|
242
|
+
|
|
243
|
+
-- A frame did not reach the transport, so this client's mirror is now a value the
|
|
244
|
+
-- server never had: Insert, Remove and Clear are not idempotent, so every later
|
|
245
|
+
-- op lands on the wrong base, and a loaded client never Hellos again. Repair is a
|
|
246
|
+
-- fresh Init, not a replay: re-broadcasting a SharedDiff would double-apply an
|
|
247
|
+
-- Insert on the other recipients.
|
|
248
|
+
local function markResync(entry: any, what: string)
|
|
249
|
+
if entry.ResyncPending then
|
|
250
|
+
return
|
|
251
|
+
end
|
|
252
|
+
entry.ResyncPending = true
|
|
253
|
+
markDirty(entry)
|
|
254
|
+
-- Both flags, and the queue with them: HandshakeReady false stops broadcasts to
|
|
255
|
+
-- a client whose mirror is wrong, and clearing QueueOpen keeps out the
|
|
256
|
+
-- (ready, not open) state in which PushOp silently discards every owner op.
|
|
257
|
+
entry.HandshakeReady = false
|
|
258
|
+
entry.QueueOpen = false
|
|
259
|
+
entry.Queue = {}
|
|
260
|
+
entry.QueueIndex = {}
|
|
261
|
+
Metrics.Add("MirrorResyncs")
|
|
262
|
+
Log.Warn(
|
|
263
|
+
"Replication",
|
|
264
|
+
"MIRROR_RESYNC",
|
|
265
|
+
`a {what} frame could not be delivered, so this client's mirror is stale; re-sending the whole snapshot`,
|
|
266
|
+
{ Player = entry.Player }
|
|
267
|
+
)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
-- One frame, to one client. Returns whether the whole frame reached the transport,
|
|
271
|
+
-- which sendInit needs: a snapshot that did not land must not mark the client
|
|
272
|
+
-- ready. Fragmentation happens here rather than in `deliver` so a broadcast splits
|
|
273
|
+
-- once. `snapshotOps` and `snapshotSpans` are passed only by sendInit.
|
|
274
|
+
local function sendBuffer(
|
|
275
|
+
entry: any,
|
|
276
|
+
bytes: buffer,
|
|
277
|
+
snapshotOps: { Wire.Op }?,
|
|
278
|
+
snapshotSpans: { number }?,
|
|
279
|
+
repair: boolean?
|
|
280
|
+
): boolean
|
|
281
|
+
fragmentSeq = (fragmentSeq + 1) % 65536
|
|
282
|
+
local parts = 0
|
|
283
|
+
-- BytesOut counts what the transport accepted.
|
|
284
|
+
local accepted = 0
|
|
285
|
+
local whole = Wire.Fragment(bytes, maxOutbound, fragmentSeq, function(part)
|
|
286
|
+
parts += 1
|
|
287
|
+
if not deliver(entry, part) then
|
|
288
|
+
return false
|
|
289
|
+
end
|
|
290
|
+
accepted += buffer.len(part)
|
|
291
|
+
return true
|
|
292
|
+
end)
|
|
293
|
+
if whole == nil then
|
|
294
|
+
if not deliver(entry, bytes) then
|
|
295
|
+
return false
|
|
296
|
+
end
|
|
297
|
+
Metrics.Add("FramesSent")
|
|
298
|
+
Metrics.Observe("BytesOutPerSend", buffer.len(bytes))
|
|
299
|
+
countBytesOut(bytes, buffer.len(bytes), repair)
|
|
300
|
+
return true
|
|
301
|
+
end
|
|
302
|
+
if not whole then
|
|
303
|
+
-- Stop at the first refusal rather than pushing the rest: a hole in a stream
|
|
304
|
+
-- the receiver believes is contiguous could splice two frames together. A
|
|
305
|
+
-- partial stays a prefix, which the next frame's index 0 discards cleanly.
|
|
306
|
+
Metrics.Add("FragmentedSendFailed")
|
|
307
|
+
countBytesOut(bytes, accepted, repair)
|
|
308
|
+
return false
|
|
309
|
+
end
|
|
310
|
+
if parts > OVERSIZE_FRAGMENTS and not warnedOversize then
|
|
311
|
+
warnedOversize = true
|
|
312
|
+
local frameName = FRAME_NAMES[buffer.readu8(bytes, 0)] or "unknown"
|
|
313
|
+
local roots = describeRoots(snapshotOps, snapshotSpans)
|
|
314
|
+
Log.Warn(
|
|
315
|
+
"Replication",
|
|
316
|
+
"OUTBOUND_OVERSIZE",
|
|
317
|
+
`a single {frameName} frame needed {parts} fragments to send; it is delivered, but the profile or `
|
|
318
|
+
.. "diff behind it has grown well past MaxOutboundBytes and every join pays for it"
|
|
319
|
+
.. (if roots ~= "" then `; largest roots: {roots}` else ""),
|
|
320
|
+
{
|
|
321
|
+
Player = entry.Player,
|
|
322
|
+
Bytes = buffer.len(bytes),
|
|
323
|
+
Fragments = parts,
|
|
324
|
+
Frame = frameName,
|
|
325
|
+
Roots = roots,
|
|
326
|
+
}
|
|
327
|
+
)
|
|
328
|
+
end
|
|
329
|
+
Metrics.Add("FramesSent")
|
|
330
|
+
Metrics.Add("FramesFragmented")
|
|
331
|
+
Metrics.Add("FragmentsSent", parts)
|
|
332
|
+
-- The logical frame's size, once. BytesOutPerSend means bytes per outbound
|
|
333
|
+
-- frame, so sampling the fragments would describe the budget, not the frames.
|
|
334
|
+
Metrics.Observe("BytesOutPerSend", buffer.len(bytes))
|
|
335
|
+
countBytesOut(bytes, accepted, repair)
|
|
336
|
+
return true
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
-- One frame, to every ready client except `exceptEntry`. Returns the entries the
|
|
340
|
+
-- frame did not reach, or nil when every recipient got it; the only place that
|
|
341
|
+
-- knows which clients were missed.
|
|
342
|
+
--
|
|
343
|
+
-- BytesOutPerSend is observed once here, not once per recipient: per-recipient
|
|
344
|
+
-- sampling cost O(N^2) reservoir writes a frame and skewed the distribution toward
|
|
345
|
+
-- the fanout. FramesSent still counts every hand-off to the transport.
|
|
346
|
+
local function broadcastBuffer(bytes: buffer, exceptEntry: any?, alsoExcept: any?): { any }?
|
|
347
|
+
-- Split once, before the recipient loop and before the fast-path branch: the
|
|
348
|
+
-- split depends only on the buffer, and the fast path never reaches `deliver`,
|
|
349
|
+
-- so a fragmenter placed there would let every broadcast out unsplit.
|
|
350
|
+
fragmentSeq = (fragmentSeq + 1) % 65536
|
|
351
|
+
local parts: { buffer } = {}
|
|
352
|
+
local split = Wire.Fragment(bytes, maxOutbound, fragmentSeq, function(part)
|
|
353
|
+
parts[#parts + 1] = part
|
|
354
|
+
return true
|
|
355
|
+
end) ~= nil
|
|
356
|
+
|
|
357
|
+
if transport.SendToAllClients and exceptEntry == nil and alsoExcept == nil then
|
|
358
|
+
-- No per-recipient attribution on this path: one raise means the frame
|
|
359
|
+
-- reached NOBODY, so every ready client is a missed recipient.
|
|
360
|
+
local function everyReady(): { any }
|
|
361
|
+
local all: { any } = {}
|
|
362
|
+
for _, entry in ctx.Persistence.Entries do
|
|
363
|
+
if entry.HandshakeReady then
|
|
364
|
+
all[#all + 1] = entry
|
|
365
|
+
end
|
|
366
|
+
end
|
|
367
|
+
return all
|
|
368
|
+
end
|
|
369
|
+
-- Once per session: the O(1) stand-in for the clients the engine delivers to.
|
|
370
|
+
local sessions = ctx.Persistence.EntryCount
|
|
371
|
+
local accepted = 0
|
|
372
|
+
if not split then
|
|
373
|
+
local ok, err = pcall(transport.SendToAllClients, transport, bytes)
|
|
374
|
+
if not ok then
|
|
375
|
+
Log.Warn("Transport", "SEND_FAIL", tostring(err), {})
|
|
376
|
+
return everyReady()
|
|
377
|
+
end
|
|
378
|
+
accepted = buffer.len(bytes)
|
|
379
|
+
else
|
|
380
|
+
for _, part in parts do
|
|
381
|
+
local ok, err = pcall(transport.SendToAllClients, transport, part)
|
|
382
|
+
if not ok then
|
|
383
|
+
Log.Warn("Transport", "SEND_FAIL", tostring(err), {})
|
|
384
|
+
Metrics.Add("FragmentedSendFailed")
|
|
385
|
+
countBytesOut(bytes, accepted * sessions)
|
|
386
|
+
return everyReady()
|
|
387
|
+
end
|
|
388
|
+
accepted += buffer.len(part)
|
|
389
|
+
end
|
|
390
|
+
Metrics.Add("FramesFragmented")
|
|
391
|
+
Metrics.Add("FragmentsSent", #parts)
|
|
392
|
+
end
|
|
393
|
+
Metrics.Add("FramesSent")
|
|
394
|
+
Metrics.Observe("BytesOutPerSend", buffer.len(bytes))
|
|
395
|
+
countBytesOut(bytes, accepted * sessions)
|
|
396
|
+
return nil
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
local sent = 0
|
|
400
|
+
local accepted = 0
|
|
401
|
+
local missed: { any }? = nil
|
|
402
|
+
for _, entry in ctx.Persistence.Entries do
|
|
403
|
+
if entry.HandshakeReady and entry ~= exceptEntry and entry ~= alsoExcept then
|
|
404
|
+
local complete = true
|
|
405
|
+
if not split then
|
|
406
|
+
complete = deliver(entry, bytes)
|
|
407
|
+
if complete then
|
|
408
|
+
accepted += buffer.len(bytes)
|
|
409
|
+
end
|
|
410
|
+
else
|
|
411
|
+
for _, part in parts do
|
|
412
|
+
if not deliver(entry, part) then
|
|
413
|
+
complete = false
|
|
414
|
+
break
|
|
415
|
+
end
|
|
416
|
+
accepted += buffer.len(part)
|
|
417
|
+
end
|
|
418
|
+
end
|
|
419
|
+
if complete then
|
|
420
|
+
sent += 1
|
|
421
|
+
else
|
|
422
|
+
-- Allocated only on the failing path, so the ordinary fanout costs
|
|
423
|
+
-- nothing extra.
|
|
424
|
+
local list: { any } = missed or {}
|
|
425
|
+
list[#list + 1] = entry
|
|
426
|
+
missed = list
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
end
|
|
430
|
+
if sent > 0 then
|
|
431
|
+
-- Accumulated and added once, never per recipient: a Metrics.Add inside
|
|
432
|
+
-- this loop costs O(N^2) hash writes a frame.
|
|
433
|
+
Metrics.Add("FramesSent", sent)
|
|
434
|
+
if split then
|
|
435
|
+
Metrics.Add("FramesFragmented")
|
|
436
|
+
Metrics.Add("FragmentsSent", #parts * sent)
|
|
437
|
+
end
|
|
438
|
+
Metrics.Observe("BytesOutPerSend", buffer.len(bytes))
|
|
439
|
+
end
|
|
440
|
+
if accepted > 0 then
|
|
441
|
+
countBytesOut(bytes, accepted)
|
|
442
|
+
end
|
|
443
|
+
return missed
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
local function compactOps(queue: { Wire.Op }): { Wire.Op }
|
|
447
|
+
local supersede: { [string]: boolean } = {} -- own path + descendants (Set table/nil)
|
|
448
|
+
local descendantSupersede: { [string]: boolean } = {} -- descendants only (Clear)
|
|
449
|
+
local reversed: { Wire.Op } = {}
|
|
450
|
+
for i = #queue, 1, -1 do
|
|
451
|
+
local op = queue[i]
|
|
452
|
+
local k = (op :: any)._pk :: string
|
|
453
|
+
local superseded = supersede[k] == true
|
|
454
|
+
if not superseded then
|
|
455
|
+
local pos = string.find(k, "\31", 1, true)
|
|
456
|
+
while pos do
|
|
457
|
+
local ancestor = string.sub(k, 1, pos - 1)
|
|
458
|
+
if supersede[ancestor] or descendantSupersede[ancestor] then
|
|
459
|
+
superseded = true
|
|
460
|
+
break
|
|
461
|
+
end
|
|
462
|
+
pos = string.find(k, "\31", pos + 1, true)
|
|
463
|
+
end
|
|
464
|
+
end
|
|
465
|
+
if not superseded then
|
|
466
|
+
reversed[#reversed + 1] = op
|
|
467
|
+
if op.Kind == "Clear" then
|
|
468
|
+
descendantSupersede[k] = true
|
|
469
|
+
elseif op.Kind == "Set" and (op.Value == nil or type(op.Value) == "table") then
|
|
470
|
+
supersede[k] = true
|
|
471
|
+
elseif op.Kind == "Insert" or op.Kind == "Remove" then
|
|
472
|
+
-- Index-shift barrier: an Insert/Remove rebinds which element every
|
|
473
|
+
-- numeric path under this container denotes, so a later table-Set at
|
|
474
|
+
-- arr[i] does not supersede an earlier op targeting the element that
|
|
475
|
+
-- occupied i before the shift. Drop the descendant marks under it.
|
|
476
|
+
local prefix = k .. "\31"
|
|
477
|
+
local prefixLen = #prefix
|
|
478
|
+
for marked in supersede do
|
|
479
|
+
if string.sub(marked, 1, prefixLen) == prefix then
|
|
480
|
+
supersede[marked] = nil
|
|
481
|
+
end
|
|
482
|
+
end
|
|
483
|
+
for marked in descendantSupersede do
|
|
484
|
+
if string.sub(marked, 1, prefixLen) == prefix then
|
|
485
|
+
descendantSupersede[marked] = nil
|
|
486
|
+
end
|
|
487
|
+
end
|
|
488
|
+
end
|
|
489
|
+
end
|
|
490
|
+
end
|
|
491
|
+
local out = table.create(#reversed) :: { Wire.Op }
|
|
492
|
+
for i = #reversed, 1, -1 do
|
|
493
|
+
out[#out + 1] = reversed[i]
|
|
494
|
+
end
|
|
495
|
+
return out
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
-- Flush an entry's queued Shared ops as SharedDiff frames to every other ready
|
|
499
|
+
-- client. Extracted so sendInit can drain the queue before a new client snapshots
|
|
500
|
+
-- that player; otherwise the op lands twice, double-applying Insert/Remove/Clear.
|
|
501
|
+
--- Takes the queued Shared ops, leaving a fresh queue for writes made during a send.
|
|
502
|
+
local function detachSharedQueue(entry: any): { Wire.Op }
|
|
503
|
+
local shared: { Wire.Op } = entry.SharedQueue
|
|
504
|
+
entry.SharedQueue = {}
|
|
505
|
+
return shared
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
local function sendSharedOps(entry: any, shared: { Wire.Op }, alsoExcept: any?)
|
|
509
|
+
if #shared == 0 then
|
|
510
|
+
return
|
|
511
|
+
end
|
|
512
|
+
shared = compactOps(shared)
|
|
513
|
+
local offset = 1
|
|
514
|
+
while offset <= #shared do
|
|
515
|
+
local chunk = table.move(shared, offset, math.min(offset + MAX_OPS_PER_FRAME - 1, #shared), 1, {})
|
|
516
|
+
local ok, bytesOrErr = pcall(Wire.EncodeOpsFrame, Wire.Frames.SharedDiff, compiled, chunk, function(w)
|
|
517
|
+
Wire.F64(w, entry.UserId)
|
|
518
|
+
end)
|
|
519
|
+
if ok then
|
|
520
|
+
-- A SharedDiff cannot be replayed to the client that missed it: the other
|
|
521
|
+
-- recipients already applied it. It gets a whole new snapshot instead.
|
|
522
|
+
local missed = broadcastBuffer(bytesOrErr, entry, alsoExcept)
|
|
523
|
+
if missed ~= nil then
|
|
524
|
+
for _, other in missed do
|
|
525
|
+
markResync(other, "SharedDiff")
|
|
526
|
+
end
|
|
527
|
+
end
|
|
528
|
+
else
|
|
529
|
+
Log.Error("Replication", "SERIALIZE_FAIL", tostring(bytesOrErr), { Player = entry.Player })
|
|
530
|
+
end
|
|
531
|
+
offset += MAX_OPS_PER_FRAME
|
|
532
|
+
end
|
|
533
|
+
end
|
|
534
|
+
|
|
535
|
+
local function flushSharedQueue(entry: any)
|
|
536
|
+
sendSharedOps(entry, detachSharedQueue(entry))
|
|
537
|
+
end
|
|
538
|
+
|
|
539
|
+
function self.FlushEntry(entry: any)
|
|
540
|
+
local queue: { Wire.Op } = entry.Queue
|
|
541
|
+
if #queue > 0 and entry.HandshakeReady then
|
|
542
|
+
entry.Queue = {}
|
|
543
|
+
entry.QueueIndex = {}
|
|
544
|
+
queue = compactOps(queue)
|
|
545
|
+
local offset = 1
|
|
546
|
+
while offset <= #queue do
|
|
547
|
+
local chunk = table.move(queue, offset, math.min(offset + MAX_OPS_PER_FRAME - 1, #queue), 1, {})
|
|
548
|
+
local ok, bytesOrErr = pcall(Wire.EncodeOpsFrame, Wire.Frames.Diff, compiled, chunk)
|
|
549
|
+
if ok then
|
|
550
|
+
if not sendBuffer(entry, bytesOrErr) then
|
|
551
|
+
-- The queue was replaced at the top of this function, so these ops
|
|
552
|
+
-- exist nowhere else and a discarded refusal is permanent divergence.
|
|
553
|
+
-- Later chunks depend on this one, so stop the loop.
|
|
554
|
+
markResync(entry, "Diff")
|
|
555
|
+
break
|
|
556
|
+
end
|
|
557
|
+
-- Counted only for a frame that actually left, so OpsSent agrees
|
|
558
|
+
-- with FramesSent.
|
|
559
|
+
Metrics.Add("DiffFlushes")
|
|
560
|
+
Metrics.Add("OpsSent", #chunk)
|
|
561
|
+
else
|
|
562
|
+
Log.Error("Replication", "SERIALIZE_FAIL", tostring(bytesOrErr), { Player = entry.Player })
|
|
563
|
+
end
|
|
564
|
+
offset += MAX_OPS_PER_FRAME
|
|
565
|
+
end
|
|
566
|
+
elseif #queue > 0 and not entry.QueueOpen then
|
|
567
|
+
-- No snapshot has been taken, so whatever is queued will be in the snapshot
|
|
568
|
+
-- when one is. Must NOT fire while a snapshot is in flight: those ops are
|
|
569
|
+
-- exactly the ones the snapshot cannot carry.
|
|
570
|
+
entry.Queue = {}
|
|
571
|
+
entry.QueueIndex = {}
|
|
572
|
+
end
|
|
573
|
+
|
|
574
|
+
flushSharedQueue(entry)
|
|
575
|
+
end
|
|
576
|
+
|
|
577
|
+
local connections: { any } = {}
|
|
578
|
+
-- Every entry with queued work flushes every frame, with deliberately no per-frame
|
|
579
|
+
-- budget: a budget needs a round-robin cursor over an ordered entry list, and
|
|
580
|
+
-- Entries is keyed by Player Instance, where next(Entries, lastPlayer) raises once
|
|
581
|
+
-- that player has left and a rehash on join silently skips entries. The
|
|
582
|
+
-- superlinear path is the Shared fanout, which a budget does not touch.
|
|
583
|
+
--- One frame's flush. Extracted so the profiler has something to name and so the
|
|
584
|
+
--- frame cost can be timed around it.
|
|
585
|
+
-- One queue for the whole server: a single writer, one frame to every ready client.
|
|
586
|
+
local storeQueue: { Wire.Op } = {}
|
|
587
|
+
-- Lets flushFrame skip the drain entirely. Read from the compiled, not from
|
|
588
|
+
-- ctx.ServerStore, which is assigned after this constructor returns.
|
|
589
|
+
local hasStore = compiled.Views.Store ~= nil
|
|
590
|
+
|
|
591
|
+
--- Queue a store op. Same visibility rule as a player op: ServerOnly never leaves.
|
|
592
|
+
function self.PushStoreOp(op: Wire.Op, schema: Template.SchemaNode?)
|
|
593
|
+
local visibility = if schema then schema.Visibility else Template.VisibilityOf(compiled, op.Path)
|
|
594
|
+
if visibility == "ServerOnly" then
|
|
595
|
+
return
|
|
596
|
+
end
|
|
597
|
+
stripServerOnly(op, schema)
|
|
598
|
+
Metrics.Add("OpsQueued")
|
|
599
|
+
-- compactOps reads _pk unconditionally. The semicolon is load-bearing: a line opening
|
|
600
|
+
-- with `(` after a call is ambiguous to Luau.
|
|
601
|
+
local pathKey = Util.pathKey(op.Path);
|
|
602
|
+
(op :: any)._pk = pathKey
|
|
603
|
+
storeQueue[#storeQueue + 1] = op
|
|
604
|
+
end
|
|
605
|
+
|
|
606
|
+
--- Encode whatever the store queued and broadcast it to every ready client.
|
|
607
|
+
--- `except` skips one entry: the joiner whose snapshot already holds these ops.
|
|
608
|
+
local function sendStoreOps(queue: { Wire.Op }, except: any?): number
|
|
609
|
+
local sentOps = 0
|
|
610
|
+
local offset = 1
|
|
611
|
+
while offset <= #queue do
|
|
612
|
+
local chunk = table.move(queue, offset, math.min(offset + MAX_OPS_PER_FRAME - 1, #queue), 1, {})
|
|
613
|
+
local ok, bytesOrErr = pcall(Wire.EncodeOpsFrame, Wire.Frames.Diff, compiled, chunk)
|
|
614
|
+
if ok then
|
|
615
|
+
-- An ordinary Diff against the full compiled: no new frame kind needed.
|
|
616
|
+
local missed = broadcastBuffer(bytesOrErr, except)
|
|
617
|
+
Metrics.Add("OpsSent", #chunk)
|
|
618
|
+
sentOps += #chunk
|
|
619
|
+
if missed ~= nil then
|
|
620
|
+
-- Nothing else repairs a missed store diff, so force a re-Init.
|
|
621
|
+
for _, entry in missed do
|
|
622
|
+
markResync(entry, "StoreDiff")
|
|
623
|
+
end
|
|
624
|
+
end
|
|
625
|
+
else
|
|
626
|
+
Log.Error("Replication", "SERIALIZE_FAIL", tostring(bytesOrErr), {})
|
|
627
|
+
Metrics.Add("SerializeFailed")
|
|
628
|
+
end
|
|
629
|
+
offset += MAX_OPS_PER_FRAME
|
|
630
|
+
end
|
|
631
|
+
return sentOps
|
|
632
|
+
end
|
|
633
|
+
|
|
634
|
+
function self.FlushStoreQueue(except: any?): number
|
|
635
|
+
if #storeQueue == 0 then
|
|
636
|
+
return 0
|
|
637
|
+
end
|
|
638
|
+
local queue = compactOps(storeQueue)
|
|
639
|
+
storeQueue = {}
|
|
640
|
+
return sendStoreOps(queue, except)
|
|
641
|
+
end
|
|
642
|
+
|
|
643
|
+
local function flushFrame(): (number, number, boolean)
|
|
644
|
+
local flushed = 0
|
|
645
|
+
local queued = 0
|
|
646
|
+
local resynced = 0
|
|
647
|
+
local batch = dirty
|
|
648
|
+
dirty = {}
|
|
649
|
+
for entry in batch do
|
|
650
|
+
if entry.TornDown then
|
|
651
|
+
continue
|
|
652
|
+
end
|
|
653
|
+
-- Repair before flushing: a resync re-snapshots live state, so any op that
|
|
654
|
+
-- would have flushed this frame is already in it. Throttled on the previous
|
|
655
|
+
-- attempt, not on LastInitAt, whose clock starts at the join: gating on that
|
|
656
|
+
-- would leave a client that lost a frame just after its handshake stale.
|
|
657
|
+
if entry.ResyncPending and entry.State == "Ready" and not entry.TornDown then
|
|
658
|
+
local now = Clock.Monotonic()
|
|
659
|
+
if now - (entry.LastResyncAt or -math.huge) >= REINIT_COOLDOWN then
|
|
660
|
+
entry.LastResyncAt = now
|
|
661
|
+
entry.ResyncPending = false
|
|
662
|
+
-- pcall for the reason onHello gives: a throw escaping here would take
|
|
663
|
+
-- down the flush loop for every other player.
|
|
664
|
+
local okInit, initErr = pcall(sendInit, entry)
|
|
665
|
+
resynced += 1
|
|
666
|
+
if not okInit then
|
|
667
|
+
Log.Error("Replication", "INIT_SEND_FAIL", tostring(initErr), { Player = entry.Player })
|
|
668
|
+
Metrics.Add("InitSendFailed")
|
|
669
|
+
entry.ResyncPending = true
|
|
670
|
+
markDirty(entry)
|
|
671
|
+
end
|
|
672
|
+
end
|
|
673
|
+
end
|
|
674
|
+
-- Counted before the flush, because FlushEntry replaces both queues. Ops
|
|
675
|
+
-- pending, not ops sent: compaction shrinks it on the way to the wire.
|
|
676
|
+
local pending = #entry.Queue + #entry.SharedQueue
|
|
677
|
+
if pending > 0 then
|
|
678
|
+
flushed += 1
|
|
679
|
+
queued += pending
|
|
680
|
+
self.FlushEntry(entry)
|
|
681
|
+
end
|
|
682
|
+
if entry.ResyncPending or #entry.Queue > 0 or #entry.SharedQueue > 0 then
|
|
683
|
+
markDirty(entry)
|
|
684
|
+
end
|
|
685
|
+
end
|
|
686
|
+
-- The store drains once per frame, not per entry: one queue, one broadcast.
|
|
687
|
+
local storeOps = 0
|
|
688
|
+
if hasStore then
|
|
689
|
+
storeOps = self.FlushStoreQueue()
|
|
690
|
+
end
|
|
691
|
+
return flushed, queued, flushed > 0 or storeOps > 0 or resynced > 0
|
|
692
|
+
end
|
|
693
|
+
|
|
694
|
+
--- Ask for a fresh Init on the next frame, as a lost frame would.
|
|
695
|
+
function self.RequestResync(entry: any, what: string)
|
|
696
|
+
markResync(entry, what)
|
|
697
|
+
end
|
|
698
|
+
|
|
699
|
+
function self.DirtyCount(): number
|
|
700
|
+
local count = 0
|
|
701
|
+
for _ in dirty do
|
|
702
|
+
count += 1
|
|
703
|
+
end
|
|
704
|
+
return count
|
|
705
|
+
end
|
|
706
|
+
|
|
707
|
+
connections[#connections + 1] = RunService.PostSimulation:Connect(function()
|
|
708
|
+
-- No memory category here: this body runs on a thread the engine may recycle,
|
|
709
|
+
-- so the tag would outlive the call and stain whatever ran next on it.
|
|
710
|
+
if next(dirty) == nil and (not hasStore or #storeQueue == 0) then
|
|
711
|
+
return
|
|
712
|
+
end
|
|
713
|
+
local startedAt = Clock.Monotonic()
|
|
714
|
+
-- SyncSpan, and flushFrame is synchronous: it encodes and fires, and waits on
|
|
715
|
+
-- nothing. That is the precondition, not an incidental property.
|
|
716
|
+
local flushed, queued, worked = Profile.SyncSpan("Scribe.Flush", flushFrame)
|
|
717
|
+
-- Nothing is recorded for an idle frame: a distribution padded with zeroes
|
|
718
|
+
-- would drag every percentile down and hide the busy frames.
|
|
719
|
+
if flushed > 0 then
|
|
720
|
+
Metrics.Observe("FlushEntriesPerFrame", flushed)
|
|
721
|
+
Metrics.Observe("FlushQueuedPerFrame", queued)
|
|
722
|
+
end
|
|
723
|
+
-- The frame cost, in seconds, whatever the work was.
|
|
724
|
+
if worked then
|
|
725
|
+
Metrics.Observe("FlushDuration", Clock.Monotonic() - startedAt)
|
|
726
|
+
end
|
|
727
|
+
end)
|
|
728
|
+
|
|
729
|
+
-- Snapshots -------------------------------------------------------------------
|
|
730
|
+
|
|
731
|
+
local function ownerSnapshotOps(entry: any): { Wire.Op }
|
|
732
|
+
local ops: { Wire.Op } = {}
|
|
733
|
+
local rootChildren = compiled.Root.Children :: { [string]: Template.SchemaNode }
|
|
734
|
+
-- Store roots come from the server's tree, not this player's. They ride the Init
|
|
735
|
+
-- because applyInit clears and refills from defaults. Nil when no store is declared.
|
|
736
|
+
local storeData = if ctx.ServerStore ~= nil then ctx.ServerStore.Data else nil
|
|
737
|
+
for _, rootKey in compiled.RootOrder do
|
|
738
|
+
local rootSchema = rootChildren[rootKey]
|
|
739
|
+
local store
|
|
740
|
+
if storeData ~= nil and storeData[rootKey] ~= nil then
|
|
741
|
+
store = storeData
|
|
742
|
+
elseif not rootSchema.Persist then
|
|
743
|
+
store = entry.SessionStore
|
|
744
|
+
else
|
|
745
|
+
store = entry.LiveData
|
|
746
|
+
end
|
|
747
|
+
local value = if store then store[rootKey] else nil
|
|
748
|
+
if value ~= nil then
|
|
749
|
+
-- CloneVisible strips ServerOnly subtrees and returns nil for roots with
|
|
750
|
+
-- nothing visible (mixed roots like _Scribe keep their Replicated subkeys).
|
|
751
|
+
local visible = Template.CloneVisible(rootSchema, value)
|
|
752
|
+
if visible ~= nil then
|
|
753
|
+
table.insert(ops, { Kind = "Set", Path = { rootKey }, Value = visible })
|
|
754
|
+
end
|
|
755
|
+
end
|
|
756
|
+
end
|
|
757
|
+
return ops
|
|
758
|
+
end
|
|
759
|
+
|
|
760
|
+
local function sharedSnapshotOps(entry: any): { Wire.Op }
|
|
761
|
+
local ops: { Wire.Op } = {}
|
|
762
|
+
local rootChildren = compiled.Root.Children :: { [string]: Template.SchemaNode }
|
|
763
|
+
for _, rootKey in compiled.RootOrder do
|
|
764
|
+
local rootSchema = rootChildren[rootKey]
|
|
765
|
+
-- Same store choice as ownerSnapshotOps: a Shared root that is also Session
|
|
766
|
+
-- lives in the session store, so reading LiveData broadcasts its default.
|
|
767
|
+
local store = if not rootSchema.Persist then entry.SessionStore else entry.LiveData
|
|
768
|
+
if rootSchema.Visibility == "Shared" and store then
|
|
769
|
+
local value = store[rootKey]
|
|
770
|
+
if value ~= nil then
|
|
771
|
+
-- CloneVisible strips ServerOnly subtrees nested under the Shared root;
|
|
772
|
+
-- a plain deepClone would broadcast them.
|
|
773
|
+
local visible = Template.CloneVisible(rootSchema, value)
|
|
774
|
+
if visible ~= nil then
|
|
775
|
+
table.insert(ops, { Kind = "Set", Path = { rootKey }, Value = visible })
|
|
776
|
+
end
|
|
777
|
+
end
|
|
778
|
+
end
|
|
779
|
+
end
|
|
780
|
+
return ops
|
|
781
|
+
end
|
|
782
|
+
|
|
783
|
+
--- Returns whether a unicast status reached the client; always true for the
|
|
784
|
+
--- broadcast form, which has no single recipient to answer for.
|
|
785
|
+
local function sendStatus(entry: any?): boolean
|
|
786
|
+
local w = Wire.NewWriter(4)
|
|
787
|
+
Wire.U8(w, Wire.Frames.Status)
|
|
788
|
+
Wire.U8(w, STATUS_TO_ID[Health.GetStatus()] or 0)
|
|
789
|
+
local bytes = Wire.Finish(w)
|
|
790
|
+
if entry then
|
|
791
|
+
return sendBuffer(entry, bytes)
|
|
792
|
+
end
|
|
793
|
+
-- A missed broadcast Status is not mirror state: the next Health transition
|
|
794
|
+
-- rebroadcasts, and a resync re-sends it as part of the handshake.
|
|
795
|
+
broadcastBuffer(bytes)
|
|
796
|
+
return true
|
|
797
|
+
end
|
|
798
|
+
|
|
799
|
+
-- Size of the most recently encoded Init frame, across all players.
|
|
800
|
+
local lastInitBytes: number? = nil
|
|
801
|
+
|
|
802
|
+
function sendInit(entry: any)
|
|
803
|
+
-- Sent after the snapshot is taken, so a write made inside Send lands past the boundary.
|
|
804
|
+
local pending: { Wire.Op }? = nil
|
|
805
|
+
if hasStore and #storeQueue > 0 then
|
|
806
|
+
pending = compactOps(storeQueue)
|
|
807
|
+
storeQueue = {}
|
|
808
|
+
end
|
|
809
|
+
-- Open the queue in the same breath as taking the snapshot, with no yield
|
|
810
|
+
-- between them, so every write from here on is either in this clone or in the
|
|
811
|
+
-- queue behind it. The clear drops what the snapshot supersedes.
|
|
812
|
+
entry.Queue = {}
|
|
813
|
+
entry.QueueIndex = {}
|
|
814
|
+
entry.QueueOpen = true
|
|
815
|
+
-- A repair is an Init for a client that already has a snapshot.
|
|
816
|
+
local repair = entry.InitDelivered == true
|
|
817
|
+
-- Stamped at the start of the snapshot: measuring from completion would let a
|
|
818
|
+
-- slow encode widen its own cooldown.
|
|
819
|
+
entry.LastInitAt = Clock.Monotonic()
|
|
820
|
+
local ops = ownerSnapshotOps(entry)
|
|
821
|
+
if pending ~= nil then
|
|
822
|
+
sendStoreOps(pending, entry)
|
|
823
|
+
end
|
|
824
|
+
-- Per-root byte spans, filled by the encode, so the oversize warning can name
|
|
825
|
+
-- what is large. `shippedOps` is what the delivered frame holds, not `ops`.
|
|
826
|
+
local spans: { number } = {}
|
|
827
|
+
local shippedOps: { Wire.Op } = ops
|
|
828
|
+
local ok, bytesOrErr =
|
|
829
|
+
pcall(Wire.EncodeOpsFrame, Wire.Frames.Init, compiled, ops, nil, spans, entry.InitSizeHint or lastInitBytes)
|
|
830
|
+
if not ok then
|
|
831
|
+
-- One un-encodable root must never brick the client, which would block every
|
|
832
|
+
-- client API on waitForLoaded. Drop it, surface it, and send the remainder.
|
|
833
|
+
Log.Error("Replication", "SERIALIZE_FAIL", tostring(bytesOrErr), { Player = entry.Player })
|
|
834
|
+
local safeOps = {}
|
|
835
|
+
for _, op in ops do
|
|
836
|
+
local encodable = pcall(Wire.EncodeOpsFrame, Wire.Frames.Init, compiled, { op })
|
|
837
|
+
if encodable then
|
|
838
|
+
table.insert(safeOps, op)
|
|
839
|
+
else
|
|
840
|
+
Metrics.Add("SnapshotRootDropped")
|
|
841
|
+
Log.Error("Replication", "SNAPSHOT_ROOT_DROPPED", "root omitted from Init (unserializable)", {
|
|
842
|
+
Player = entry.Player,
|
|
843
|
+
Root = op.Path[1],
|
|
844
|
+
})
|
|
845
|
+
if ctx.Signals and ctx.Signals.OnAnomaly then
|
|
846
|
+
ctx.Signals.OnAnomaly:Fire({
|
|
847
|
+
Player = entry.Player,
|
|
848
|
+
Path = op.Path,
|
|
849
|
+
Value = nil,
|
|
850
|
+
Reason = "UnserializableSnapshotRoot",
|
|
851
|
+
})
|
|
852
|
+
end
|
|
853
|
+
end
|
|
854
|
+
end
|
|
855
|
+
-- A fresh spans table, and shippedOps with it: attributing the sent frame
|
|
856
|
+
-- against the failed one's ops would name roots this client never received.
|
|
857
|
+
spans = {}
|
|
858
|
+
shippedOps = safeOps
|
|
859
|
+
local okSafe, safeBytes = pcall(Wire.EncodeOpsFrame, Wire.Frames.Init, compiled, safeOps, nil, spans)
|
|
860
|
+
if not okSafe then
|
|
861
|
+
-- Should be unreachable (each op individually encoded); fall back
|
|
862
|
+
-- to an empty Init rather than leaving the client unloaded.
|
|
863
|
+
spans = {}
|
|
864
|
+
shippedOps = {}
|
|
865
|
+
safeBytes = Wire.EncodeOpsFrame(Wire.Frames.Init, compiled, {})
|
|
866
|
+
end
|
|
867
|
+
bytesOrErr = safeBytes
|
|
868
|
+
end
|
|
869
|
+
-- What this handshake costs the client in bytes, summed across every frame it
|
|
870
|
+
-- takes, not just the Init: the SharedInit fan-out that follows can dwarf it.
|
|
871
|
+
local joinBytes = buffer.len(bytesOrErr :: buffer)
|
|
872
|
+
-- Seed the next Init's writer, this player's and the next joiner's. The last
|
|
873
|
+
-- size, not a running maximum: one whale profile would permanently seed every
|
|
874
|
+
-- later joiner's writer at the whale's size. Kept per entry and per context: the
|
|
875
|
+
-- entry's own last Init is the better estimate for a re-init, the context's the
|
|
876
|
+
-- only one a first join has.
|
|
877
|
+
entry.InitSizeHint = joinBytes
|
|
878
|
+
lastInitBytes = joinBytes
|
|
879
|
+
if not sendBuffer(entry, bytesOrErr, shippedOps, spans, repair) then
|
|
880
|
+
-- The snapshot never reached the transport, so this client has no mirror for
|
|
881
|
+
-- anything that follows. Leaving HandshakeReady false keeps that recoverable:
|
|
882
|
+
-- the client's Hello retry asks again, and the handshake alarm still fires.
|
|
883
|
+
Metrics.Add("InitSendFailed")
|
|
884
|
+
Log.Error("Replication", "INIT_SEND_FAIL", "the initial snapshot could not be delivered", {
|
|
885
|
+
Player = entry.Player,
|
|
886
|
+
Bytes = buffer.len(bytesOrErr :: buffer),
|
|
887
|
+
})
|
|
888
|
+
-- Un-take the snapshot. FlushEntry flushes when HandshakeReady and clears
|
|
889
|
+
-- when not QueueOpen, so QueueOpen true with HandshakeReady false matches
|
|
890
|
+
-- neither branch and the queue would grow unbounded.
|
|
891
|
+
entry.Queue = {}
|
|
892
|
+
entry.QueueIndex = {}
|
|
893
|
+
entry.QueueOpen = false
|
|
894
|
+
-- On a re-Init these matter as much as the clear above: HandshakeReady is still
|
|
895
|
+
-- true from the first snapshot, and (ready, not open) is the state in which
|
|
896
|
+
-- PushOp silently discards every owner op forever. ResyncPending asks again,
|
|
897
|
+
-- since a loaded client has no Hello left to send.
|
|
898
|
+
entry.HandshakeReady = false
|
|
899
|
+
entry.ResyncPending = true
|
|
900
|
+
markDirty(entry)
|
|
901
|
+
return
|
|
902
|
+
end
|
|
903
|
+
-- Only now: a client marked ready before its snapshot is on the wire can be
|
|
904
|
+
-- handed a diff it has no mirror to apply to. broadcastBuffer skips unready
|
|
905
|
+
-- entries, which is also what kept the Shared drain above safe.
|
|
906
|
+
entry.HandshakeReady = true
|
|
907
|
+
entry.InitDelivered = true
|
|
908
|
+
-- Any pending repair request is met by this snapshot.
|
|
909
|
+
entry.ResyncPending = false
|
|
910
|
+
-- The snapshot landed but the handshake is not finished: a Status or SharedInit
|
|
911
|
+
-- that does not arrive leaves this client correct about itself and blind to
|
|
912
|
+
-- everyone else. Ask for the whole handshake again.
|
|
913
|
+
if not sendStatus(entry) then
|
|
914
|
+
entry.ResyncPending = true
|
|
915
|
+
markDirty(entry)
|
|
916
|
+
end
|
|
917
|
+
if ctx.Leaderboards then
|
|
918
|
+
ctx.Leaderboards.OnClientReady(entry)
|
|
919
|
+
end
|
|
920
|
+
|
|
921
|
+
-- Existing players' Shared roots for the new client. Gated on HasShared:
|
|
922
|
+
-- without it this walks all N entries per join purely to send nothing.
|
|
923
|
+
if compiled.HasShared then
|
|
924
|
+
for _, other in ctx.Persistence.Entries do
|
|
925
|
+
if other ~= entry and other.State == "Ready" then
|
|
926
|
+
-- Snapshot and detach before any send: the snapshot carries what was queued,
|
|
927
|
+
-- the others get it as a diff, and a write made during either send waits for
|
|
928
|
+
-- the next flush.
|
|
929
|
+
local drained = detachSharedQueue(other)
|
|
930
|
+
local sharedOps = sharedSnapshotOps(other)
|
|
931
|
+
sendSharedOps(other, drained, entry)
|
|
932
|
+
if #sharedOps > 0 then
|
|
933
|
+
local okShared, sharedBytes = pcall(
|
|
934
|
+
Wire.EncodeOpsFrame,
|
|
935
|
+
Wire.Frames.SharedInit,
|
|
936
|
+
compiled,
|
|
937
|
+
sharedOps,
|
|
938
|
+
function(w)
|
|
939
|
+
Wire.F64(w, other.UserId)
|
|
940
|
+
end
|
|
941
|
+
)
|
|
942
|
+
if okShared then
|
|
943
|
+
if sendBuffer(entry, sharedBytes, nil, nil, repair) then
|
|
944
|
+
joinBytes += buffer.len(sharedBytes)
|
|
945
|
+
else
|
|
946
|
+
-- A joiner that misses another player's SharedInit has no
|
|
947
|
+
-- mirror of them for the rest of the session.
|
|
948
|
+
entry.ResyncPending = true
|
|
949
|
+
markDirty(entry)
|
|
950
|
+
end
|
|
951
|
+
end
|
|
952
|
+
end
|
|
953
|
+
end
|
|
954
|
+
end
|
|
955
|
+
end
|
|
956
|
+
Metrics.Add("Handshakes")
|
|
957
|
+
-- Past the early return above, so a snapshot that never reached the transport
|
|
958
|
+
-- is not sampled: JoinBytes means what a completed handshake cost.
|
|
959
|
+
Metrics.Observe("JoinBytes", joinBytes)
|
|
960
|
+
Log.Debug("Replication", "INIT_SENT", "initial snapshot sent", { Player = entry.Player })
|
|
961
|
+
end
|
|
962
|
+
|
|
963
|
+
-- Handshake ---------------------------------------------------------------
|
|
964
|
+
|
|
965
|
+
local function onHello(entry: any, reader: Wire.Reader)
|
|
966
|
+
local schemaHash = Wire.RU32(reader)
|
|
967
|
+
-- The client sends its wire PROTOCOL_VERSION after the hash. A mismatch means
|
|
968
|
+
-- the binary layout differs, not just the template, so decoding is unsafe: fail
|
|
969
|
+
-- closed, unlike the hash check below which the value-mode prefix keeps safe.
|
|
970
|
+
local protocolVersion = Wire.RU8(reader)
|
|
971
|
+
if protocolVersion ~= Wire.PROTOCOL_VERSION then
|
|
972
|
+
-- Metered as well as logged, so every refusal is countable.
|
|
973
|
+
Metrics.Add("ProtocolMismatch")
|
|
974
|
+
Log.Warn("Replication", "PROTOCOL_MISMATCH", "client wire protocol version differs from the server", {
|
|
975
|
+
Player = entry.Player,
|
|
976
|
+
})
|
|
977
|
+
return
|
|
978
|
+
end
|
|
979
|
+
if schemaHash ~= compiled.Hash % 4294967296 then
|
|
980
|
+
-- Fail closed. Paths decode from the receiver's own compiled.ById, so a field
|
|
981
|
+
-- id shared by two drifted templates maps to a different path on each side and
|
|
982
|
+
-- a server op lands on the wrong client key. The value-mode prefix keeps value
|
|
983
|
+
-- decoding safe, not path remapping.
|
|
984
|
+
Metrics.Add("SchemaMismatch")
|
|
985
|
+
Log.Warn(
|
|
986
|
+
"Replication",
|
|
987
|
+
"SCHEMA_MISMATCH",
|
|
988
|
+
"client derived a different schema than the server; refusing to replicate (template drift)",
|
|
989
|
+
{ Player = entry.Player }
|
|
990
|
+
)
|
|
991
|
+
return
|
|
992
|
+
end
|
|
993
|
+
entry.HelloReceived = true
|
|
994
|
+
if entry.State == "Ready" then
|
|
995
|
+
-- A repeated Hello re-Inits (client-driven resync), but a full snapshot
|
|
996
|
+
-- encode is expensive, so throttle floods.
|
|
997
|
+
if entry.QueueOpen and Clock.Monotonic() - entry.LastInitAt < REINIT_COOLDOWN then
|
|
998
|
+
Metrics.Add("ReInitThrottled")
|
|
999
|
+
return
|
|
1000
|
+
end
|
|
1001
|
+
-- onHello runs inside the inbound pcall, whose failure branch charges the
|
|
1002
|
+
-- client's malformed budget and blocks its session after 32 strikes. Every
|
|
1003
|
+
-- throw sendInit can produce is a server fault, so an escaping one bans a
|
|
1004
|
+
-- player for a bug they did not cause.
|
|
1005
|
+
local okInit, initErr = pcall(sendInit, entry)
|
|
1006
|
+
if not okInit then
|
|
1007
|
+
Log.Error("Replication", "INIT_SEND_FAIL", tostring(initErr), { Player = entry.Player })
|
|
1008
|
+
Metrics.Add("InitSendFailed")
|
|
1009
|
+
end
|
|
1010
|
+
end
|
|
1011
|
+
end
|
|
1012
|
+
|
|
1013
|
+
function self.OnEntryReady(entry: any)
|
|
1014
|
+
-- QueueOpen, not HandshakeReady: the question is whether a snapshot has already
|
|
1015
|
+
-- been started, which is the same question the re-Init throttle asks. The two
|
|
1016
|
+
-- reading different flags is how the gap opened.
|
|
1017
|
+
if entry.HelloReceived and not entry.QueueOpen then
|
|
1018
|
+
local okInit, initErr = pcall(sendInit, entry)
|
|
1019
|
+
if not okInit then
|
|
1020
|
+
Log.Error("Replication", "INIT_SEND_FAIL", tostring(initErr), { Player = entry.Player })
|
|
1021
|
+
Metrics.Add("InitSendFailed")
|
|
1022
|
+
end
|
|
1023
|
+
end
|
|
1024
|
+
-- Broadcast this player's Shared snapshot. The queue is cleared before the send: what
|
|
1025
|
+
-- was queued is in the snapshot, and a write made during the send is the next flush's.
|
|
1026
|
+
local sharedOps = sharedSnapshotOps(entry)
|
|
1027
|
+
entry.SharedQueue = {}
|
|
1028
|
+
if #sharedOps > 0 then
|
|
1029
|
+
local ok, bytes = pcall(Wire.EncodeOpsFrame, Wire.Frames.SharedInit, compiled, sharedOps, function(w)
|
|
1030
|
+
Wire.F64(w, entry.UserId)
|
|
1031
|
+
end)
|
|
1032
|
+
if ok then
|
|
1033
|
+
local missed = broadcastBuffer(bytes, entry)
|
|
1034
|
+
if missed ~= nil then
|
|
1035
|
+
for _, other in missed do
|
|
1036
|
+
markResync(other, "SharedInit")
|
|
1037
|
+
end
|
|
1038
|
+
end
|
|
1039
|
+
end
|
|
1040
|
+
end
|
|
1041
|
+
Clock.Delay(HANDSHAKE_TIMEOUT, function()
|
|
1042
|
+
if not entry.TornDown and not entry.HandshakeReady then
|
|
1043
|
+
if entry.HelloReceived then
|
|
1044
|
+
-- A different failure with the same symptom: the client did ask and the
|
|
1045
|
+
-- answer could not be delivered. MaxOutboundBytes is named because a
|
|
1046
|
+
-- transport refusing a payload this size is the one innocent cause.
|
|
1047
|
+
Log.Warn(
|
|
1048
|
+
"Replication",
|
|
1049
|
+
"INIT_UNDELIVERED",
|
|
1050
|
+
"the client sent Hello but its snapshot never reached the transport; check MaxOutboundBytes against what this transport will carry",
|
|
1051
|
+
{
|
|
1052
|
+
Player = entry.Player,
|
|
1053
|
+
}
|
|
1054
|
+
)
|
|
1055
|
+
else
|
|
1056
|
+
Log.Warn(
|
|
1057
|
+
"Replication",
|
|
1058
|
+
"CLIENT_HANDSHAKE_TIMEOUT",
|
|
1059
|
+
"client never sent Hello, so replication never started; require the Scribe bundle on the client (a LocalScript), or check a custom client transport adapter",
|
|
1060
|
+
{
|
|
1061
|
+
Player = entry.Player,
|
|
1062
|
+
}
|
|
1063
|
+
)
|
|
1064
|
+
end
|
|
1065
|
+
end
|
|
1066
|
+
end)
|
|
1067
|
+
end
|
|
1068
|
+
|
|
1069
|
+
function self.OnEntryRemoved(entry: any)
|
|
1070
|
+
dirty[entry] = nil
|
|
1071
|
+
entry.HandshakeReady = false
|
|
1072
|
+
-- Symmetry rather than a fix: a removed entry's tree is already dead, so
|
|
1073
|
+
-- nothing can push to it.
|
|
1074
|
+
entry.QueueOpen = false
|
|
1075
|
+
-- Only meaningful when the template declares a Shared root: without one there is
|
|
1076
|
+
-- no mirror to drop. Sending it anyway costs one frame per remaining client on
|
|
1077
|
+
-- every leave, making a full server drain quadratic.
|
|
1078
|
+
if not compiled.HasShared then
|
|
1079
|
+
return
|
|
1080
|
+
end
|
|
1081
|
+
local w = Wire.NewWriter(16)
|
|
1082
|
+
Wire.U8(w, Wire.Frames.SharedGone)
|
|
1083
|
+
Wire.F64(w, entry.UserId)
|
|
1084
|
+
-- A missed SharedGone leaves a mirror of a player who has left, which no later
|
|
1085
|
+
-- frame ever clears.
|
|
1086
|
+
local missed = broadcastBuffer(Wire.Finish(w), entry)
|
|
1087
|
+
if missed ~= nil then
|
|
1088
|
+
for _, other in missed do
|
|
1089
|
+
markResync(other, "SharedGone")
|
|
1090
|
+
end
|
|
1091
|
+
end
|
|
1092
|
+
end
|
|
1093
|
+
|
|
1094
|
+
function self.BroadcastStatus()
|
|
1095
|
+
sendStatus(nil)
|
|
1096
|
+
end
|
|
1097
|
+
|
|
1098
|
+
--- Returns whether the frame reached the client. Leaderboards depends on it:
|
|
1099
|
+
--- stamping SentState for a refused board means broadcastBoard's dedup never
|
|
1100
|
+
--- resends it, leaving the client on a stale board.
|
|
1101
|
+
function self.SendTo(entry: any, bytes: buffer): boolean
|
|
1102
|
+
return sendBuffer(entry, bytes)
|
|
1103
|
+
end
|
|
1104
|
+
|
|
1105
|
+
function self.Broadcast(bytes: buffer)
|
|
1106
|
+
local missed = broadcastBuffer(bytes)
|
|
1107
|
+
if missed ~= nil then
|
|
1108
|
+
for _, other in missed do
|
|
1109
|
+
markResync(other, "Broadcast")
|
|
1110
|
+
end
|
|
1111
|
+
end
|
|
1112
|
+
end
|
|
1113
|
+
|
|
1114
|
+
connections[#connections + 1] = Health.OnStatusChanged:Connect(function()
|
|
1115
|
+
self.BroadcastStatus()
|
|
1116
|
+
end)
|
|
1117
|
+
|
|
1118
|
+
-- Inbound dispatch -------------------------------------------------
|
|
1119
|
+
|
|
1120
|
+
transport:ListenServer(function(player: Player, bytes: buffer)
|
|
1121
|
+
local entry = ctx.Persistence.Entries[player]
|
|
1122
|
+
if not entry then
|
|
1123
|
+
return
|
|
1124
|
+
end
|
|
1125
|
+
if entry.MalformedBlocked or entry.OversizeBlocked then
|
|
1126
|
+
-- Past the malformed or oversize limit; drop without parsing so a flood of
|
|
1127
|
+
-- crafted frames cannot keep burning CPU. Two counters because the logs need
|
|
1128
|
+
-- to say which budget tripped.
|
|
1129
|
+
return
|
|
1130
|
+
end
|
|
1131
|
+
-- Ahead of the size check on purpose: this is the one ceiling that sees every
|
|
1132
|
+
-- frame whatever its type, size or validity, and it is spent before the frame
|
|
1133
|
+
-- is looked at. Silent, because replying is the amplification this tier avoids.
|
|
1134
|
+
if not takeFrameToken(entry) then
|
|
1135
|
+
Metrics.Add("InboundRateLimited")
|
|
1136
|
+
local now = Clock.Monotonic()
|
|
1137
|
+
if now - (entry.LastInboundRateLog or -math.huge) >= MALFORMED_LOG_INTERVAL then
|
|
1138
|
+
entry.LastInboundRateLog = now
|
|
1139
|
+
Log.Warn("Transport", "INBOUND_RATE_LIMITED", `player exceeded {inboundFrameRate} frames/s`, {
|
|
1140
|
+
Player = player,
|
|
1141
|
+
})
|
|
1142
|
+
end
|
|
1143
|
+
return
|
|
1144
|
+
end
|
|
1145
|
+
if buffer.len(bytes) > maxInbound then
|
|
1146
|
+
entry.OversizeCount += 1
|
|
1147
|
+
Metrics.Add("InboundOversize")
|
|
1148
|
+
-- Throttle the log, on its own clock: this frame is attacker-controlled, so
|
|
1149
|
+
-- logging every one would evict the diagnostic ring. The metric counts all.
|
|
1150
|
+
local now = Clock.Monotonic()
|
|
1151
|
+
if now - (entry.LastOversizeLog or -math.huge) >= MALFORMED_LOG_INTERVAL then
|
|
1152
|
+
entry.LastOversizeLog = now
|
|
1153
|
+
Log.Warn(
|
|
1154
|
+
"Transport",
|
|
1155
|
+
"INBOUND_OVERSIZE",
|
|
1156
|
+
`dropped oversized inbound frame ({buffer.len(bytes)} bytes)`,
|
|
1157
|
+
{ Player = player, Count = entry.OversizeCount }
|
|
1158
|
+
)
|
|
1159
|
+
end
|
|
1160
|
+
if entry.OversizeCount >= OVERSIZE_MAX and not entry.OversizeBlocked then
|
|
1161
|
+
entry.OversizeBlocked = true
|
|
1162
|
+
Log.Warn(
|
|
1163
|
+
"Transport",
|
|
1164
|
+
"INBOUND_OVERSIZE_LIMIT",
|
|
1165
|
+
`player exceeded {OVERSIZE_MAX} oversized frames; ignoring further inbound frames this session `
|
|
1166
|
+
.. `(if legitimate client traffic is this large, raise MaxInboundBytes)`,
|
|
1167
|
+
{ Player = player, Count = entry.OversizeCount, Limit = maxInbound }
|
|
1168
|
+
)
|
|
1169
|
+
end
|
|
1170
|
+
return
|
|
1171
|
+
end
|
|
1172
|
+
-- Built outside the pcall so its overBudget flag survives the failure. The
|
|
1173
|
+
-- retained-bytes budget is what the byte cap above cannot express: `06 00` is
|
|
1174
|
+
-- two bytes and allocates, eight kilobytes of one string allocates once.
|
|
1175
|
+
local reader = Wire.NewReader(bytes, maxInboundRetained)
|
|
1176
|
+
local ok, err = pcall(function()
|
|
1177
|
+
local frameType = Wire.RU8(reader)
|
|
1178
|
+
if frameType == Wire.Frames.Hello then
|
|
1179
|
+
onHello(entry, reader)
|
|
1180
|
+
elseif frameType == Wire.Frames.Command then
|
|
1181
|
+
ctx.Commands.HandleFrame(entry, reader)
|
|
1182
|
+
else
|
|
1183
|
+
error(`unexpected inbound frame type {frameType}`)
|
|
1184
|
+
end
|
|
1185
|
+
end)
|
|
1186
|
+
if not ok and reader.overBudget then
|
|
1187
|
+
-- Its own counter, not the malformed budget, and deliberately with no blocking
|
|
1188
|
+
-- limit: a frame over this budget may be a legal payload from the game's own
|
|
1189
|
+
-- client, and a 32-strike session-permanent budget would drop that player for
|
|
1190
|
+
-- playing the game. Silent, like the rate limiter.
|
|
1191
|
+
entry.WorkLimitCount = (entry.WorkLimitCount or 0) + 1
|
|
1192
|
+
Metrics.Add("InboundWorkLimited")
|
|
1193
|
+
local now = Clock.Monotonic()
|
|
1194
|
+
if now - (entry.LastWorkLimitLog or -math.huge) >= MALFORMED_LOG_INTERVAL then
|
|
1195
|
+
entry.LastWorkLimitLog = now
|
|
1196
|
+
Log.Warn(
|
|
1197
|
+
"Transport",
|
|
1198
|
+
"INBOUND_WORK_LIMIT",
|
|
1199
|
+
`dropped an inbound frame that would retain more than {maxInboundRetained} bytes `
|
|
1200
|
+
.. `(if legitimate client traffic is this large, raise MaxInboundRetainedBytes)`,
|
|
1201
|
+
{ Player = player, Count = entry.WorkLimitCount, Limit = maxInboundRetained }
|
|
1202
|
+
)
|
|
1203
|
+
end
|
|
1204
|
+
elseif not ok then
|
|
1205
|
+
entry.MalformedCount += 1
|
|
1206
|
+
Metrics.Add("MalformedFrames")
|
|
1207
|
+
-- Throttle the log + anomaly so a crafted-frame flood can't spam the log
|
|
1208
|
+
-- buffer or an OnAnomaly handler (the count still increments every time).
|
|
1209
|
+
local now = Clock.Monotonic()
|
|
1210
|
+
if now - (entry.LastMalformedLog or -math.huge) >= MALFORMED_LOG_INTERVAL then
|
|
1211
|
+
entry.LastMalformedLog = now
|
|
1212
|
+
Log.Warn("Transport", "MALFORMED_FRAME", tostring(err), {
|
|
1213
|
+
Player = player,
|
|
1214
|
+
Count = entry.MalformedCount,
|
|
1215
|
+
})
|
|
1216
|
+
ctx.Signals.OnAnomaly:Fire({
|
|
1217
|
+
Player = player,
|
|
1218
|
+
Path = {},
|
|
1219
|
+
Value = nil,
|
|
1220
|
+
Reason = "MalformedFrame",
|
|
1221
|
+
})
|
|
1222
|
+
end
|
|
1223
|
+
-- Past the limit, stop dispatching this player's inbound frames entirely
|
|
1224
|
+
-- for the rest of the session (a real client never sends malformed frames).
|
|
1225
|
+
if entry.MalformedCount >= MALFORMED_MAX and not entry.MalformedBlocked then
|
|
1226
|
+
entry.MalformedBlocked = true
|
|
1227
|
+
Log.Warn(
|
|
1228
|
+
"Transport",
|
|
1229
|
+
"MALFORMED_FRAME_LIMIT",
|
|
1230
|
+
`player exceeded {MALFORMED_MAX} malformed frames; ignoring further inbound frames this session`,
|
|
1231
|
+
{ Player = player, Count = entry.MalformedCount }
|
|
1232
|
+
)
|
|
1233
|
+
end
|
|
1234
|
+
end
|
|
1235
|
+
end)
|
|
1236
|
+
|
|
1237
|
+
-- Release the per-frame flush handler and the status listener; without this a
|
|
1238
|
+
-- discarded bundle keeps flushing against a dead Persistence forever.
|
|
1239
|
+
function self.Stop()
|
|
1240
|
+
stopped = true
|
|
1241
|
+
for _, connection in connections do
|
|
1242
|
+
pcall(function()
|
|
1243
|
+
connection:Disconnect()
|
|
1244
|
+
end)
|
|
1245
|
+
end
|
|
1246
|
+
table.clear(connections)
|
|
1247
|
+
table.clear(dirty)
|
|
1248
|
+
end
|
|
1249
|
+
|
|
1250
|
+
return self
|
|
1251
|
+
end
|
|
1252
|
+
|
|
1253
|
+
return Replication
|