@syncular/server 0.59.0 → 0.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +4 -100
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +4 -107
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/src/handler.ts
CHANGED
|
@@ -11,7 +11,9 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import {
|
|
13
13
|
DecodeError,
|
|
14
|
-
|
|
14
|
+
decodeRequest,
|
|
15
|
+
echoDigest,
|
|
16
|
+
type ProgressFrame,
|
|
15
17
|
type PullHeaderFrame,
|
|
16
18
|
type PushCommitFrame,
|
|
17
19
|
type PushResultDetailsFrame,
|
|
@@ -21,7 +23,10 @@ import {
|
|
|
21
23
|
type ResponseFrame,
|
|
22
24
|
type RoundFailureFrame,
|
|
23
25
|
type ScopeMap,
|
|
24
|
-
type
|
|
26
|
+
type UnitFrame,
|
|
27
|
+
type UnitScopes,
|
|
28
|
+
registrationDigest,
|
|
29
|
+
unitId,
|
|
25
30
|
} from '@syncular/core';
|
|
26
31
|
import type { SyncRequestContext } from './context';
|
|
27
32
|
import {
|
|
@@ -34,7 +39,7 @@ import {
|
|
|
34
39
|
import { reportError, SyncError, syncError } from './errors';
|
|
35
40
|
import {
|
|
36
41
|
emitEvent,
|
|
37
|
-
type
|
|
42
|
+
type PullUnitSummary,
|
|
38
43
|
type SyncularServerEvents,
|
|
39
44
|
} from './events';
|
|
40
45
|
import {
|
|
@@ -46,26 +51,48 @@ import type { LeaseRecord } from './lease-store';
|
|
|
46
51
|
import {
|
|
47
52
|
ACCEPT_EXTERNAL_ROWS,
|
|
48
53
|
ACCEPT_INLINE_ROWS,
|
|
54
|
+
bootstrapSegments,
|
|
49
55
|
clampPullLimits,
|
|
56
|
+
commitFrame,
|
|
57
|
+
coveredTables,
|
|
58
|
+
type PullLimits,
|
|
50
59
|
type PullSectionTrace,
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
type
|
|
60
|
+
planBootstrap,
|
|
61
|
+
planUnit,
|
|
62
|
+
type UnitPlan,
|
|
54
63
|
} from './pull';
|
|
55
64
|
import { type ProcessedPushCommit, processPushCommitWithTrace } from './push';
|
|
56
65
|
import { serveNotReadyError } from './readiness';
|
|
57
|
-
import type { CompiledSchema } from './schema';
|
|
66
|
+
import type { CompiledSchema, CompiledTable } from './schema';
|
|
58
67
|
import { compileSchema } from './schema';
|
|
59
68
|
import { schemaWindowOf, projectRowPayload } from './schema-window';
|
|
60
|
-
import {
|
|
69
|
+
import type { ResolvedScopes } from './scopes';
|
|
61
70
|
import { StorageQueryError } from './storage-errors';
|
|
62
71
|
import {
|
|
63
|
-
type
|
|
72
|
+
type CatchingUpUnit,
|
|
73
|
+
type ClientRecord,
|
|
74
|
+
type ClientRegistration,
|
|
75
|
+
type LiveWindow,
|
|
64
76
|
type PartitionRegistryEntry,
|
|
65
77
|
serveGateRefusal,
|
|
66
78
|
serveGateTokenChanged,
|
|
67
79
|
} from './storage';
|
|
68
80
|
|
|
81
|
+
interface PlannedUnit {
|
|
82
|
+
readonly plan: UnitPlan;
|
|
83
|
+
/** Absent for a live unit. */
|
|
84
|
+
readonly progress: ProgressFrame | undefined;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
interface PullPlan {
|
|
88
|
+
readonly header: PullHeaderFrame;
|
|
89
|
+
readonly limits: PullLimits;
|
|
90
|
+
/** The traveling list (§4.1); absent when the request referred to it. */
|
|
91
|
+
readonly registration: ClientRegistration | undefined;
|
|
92
|
+
/** The effective list; absent when the answer is `unregistered`. */
|
|
93
|
+
readonly units: readonly PlannedUnit[] | undefined;
|
|
94
|
+
}
|
|
95
|
+
|
|
69
96
|
interface RequestPlan {
|
|
70
97
|
readonly wireVersion: number;
|
|
71
98
|
readonly logEpoch: string;
|
|
@@ -74,8 +101,9 @@ interface RequestPlan {
|
|
|
74
101
|
/** §1.5: the code this request reports for its last unseen round failure. */
|
|
75
102
|
readonly roundFailure: RoundFailureFrame | undefined;
|
|
76
103
|
readonly pushes: readonly PushCommitFrame[];
|
|
77
|
-
readonly pull:
|
|
78
|
-
|
|
104
|
+
readonly pull: PullPlan | undefined;
|
|
105
|
+
/** The client record read for the actor binding (§1.5, §4.5). */
|
|
106
|
+
readonly record: ClientRecord | undefined;
|
|
79
107
|
readonly resolved: ResolvedScopes;
|
|
80
108
|
readonly schemaFloor: boolean;
|
|
81
109
|
/**
|
|
@@ -100,7 +128,11 @@ function validateResolvedKeys(allowed: ScopeMap, schema: CompiledSchema): void {
|
|
|
100
128
|
// bug — fail the request, do not guess.
|
|
101
129
|
throw syncError(
|
|
102
130
|
'sync.invalid_subscription',
|
|
103
|
-
|
|
131
|
+
'resolveScopes returned an undeclared scope variable (§3.2)',
|
|
132
|
+
JSON.stringify({
|
|
133
|
+
reason: 'undeclared_resolved_variable',
|
|
134
|
+
variable: key,
|
|
135
|
+
}),
|
|
104
136
|
);
|
|
105
137
|
}
|
|
106
138
|
}
|
|
@@ -163,8 +195,8 @@ async function resolveOnce(
|
|
|
163
195
|
});
|
|
164
196
|
} catch (error) {
|
|
165
197
|
if (error instanceof SyncError) throw error;
|
|
166
|
-
// §3.2 rule 5 / §3.4 step 4: fail loud, never leak —
|
|
167
|
-
//
|
|
198
|
+
// §3.2 rule 5 / §3.4 step 4: fail loud, never leak — units revoke and
|
|
199
|
+
// writes reject with sync.forbidden. A throw is NOT an
|
|
168
200
|
// outage (§7.3.3): it never reaches the lease.
|
|
169
201
|
const events = ctx.events;
|
|
170
202
|
if (events !== undefined) {
|
|
@@ -216,13 +248,15 @@ async function planRequest(
|
|
|
216
248
|
}
|
|
217
249
|
const pushes: PushCommitFrame[] = [];
|
|
218
250
|
let roundFailure: RoundFailureFrame | undefined;
|
|
219
|
-
let
|
|
220
|
-
const
|
|
251
|
+
let pullHeader: PullHeaderFrame | undefined;
|
|
252
|
+
const unitFrames: UnitFrame[] = [];
|
|
253
|
+
const progressFrames: ProgressFrame[] = [];
|
|
221
254
|
for (const frame of request.frames) {
|
|
222
255
|
if (frame.type === 'ROUND_FAILURE') roundFailure = frame;
|
|
223
256
|
else if (frame.type === 'PUSH_COMMIT') pushes.push(frame);
|
|
224
|
-
else if (frame.type === 'PULL_HEADER')
|
|
225
|
-
else if (frame.type === '
|
|
257
|
+
else if (frame.type === 'PULL_HEADER') pullHeader = frame;
|
|
258
|
+
else if (frame.type === 'UNIT') unitFrames.push(frame);
|
|
259
|
+
else if (frame.type === 'PROGRESS') progressFrames.push(frame);
|
|
226
260
|
}
|
|
227
261
|
|
|
228
262
|
if (header.logEpoch === undefined && pushes.length > 0) {
|
|
@@ -232,40 +266,25 @@ async function planRequest(
|
|
|
232
266
|
);
|
|
233
267
|
}
|
|
234
268
|
const epochReset = header.logEpoch !== registry.logEpoch;
|
|
269
|
+
const unserved = {
|
|
270
|
+
wireVersion: request.wireVersion,
|
|
271
|
+
logEpoch: registry.logEpoch,
|
|
272
|
+
epochReset,
|
|
273
|
+
header,
|
|
274
|
+
roundFailure,
|
|
275
|
+
pushes: [],
|
|
276
|
+
pull: undefined,
|
|
277
|
+
record: undefined,
|
|
278
|
+
resolved: { ok: false },
|
|
279
|
+
leaseToEmit: undefined,
|
|
280
|
+
} as const;
|
|
235
281
|
|
|
236
282
|
if (header.schemaVersion !== schema.version) {
|
|
237
283
|
// §2.4: no degraded encoding — answer with the schema floor (§1.6).
|
|
238
284
|
// No lease is issued on a floor round (§7.3.3).
|
|
239
|
-
return {
|
|
240
|
-
wireVersion: request.wireVersion,
|
|
241
|
-
logEpoch: registry.logEpoch,
|
|
242
|
-
epochReset,
|
|
243
|
-
header,
|
|
244
|
-
roundFailure,
|
|
245
|
-
pushes,
|
|
246
|
-
pull,
|
|
247
|
-
subscriptions: [],
|
|
248
|
-
resolved: { ok: false },
|
|
249
|
-
schemaFloor: true,
|
|
250
|
-
leaseToEmit: undefined,
|
|
251
|
-
};
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
if (epochReset) {
|
|
255
|
-
return {
|
|
256
|
-
wireVersion: request.wireVersion,
|
|
257
|
-
logEpoch: registry.logEpoch,
|
|
258
|
-
epochReset: true,
|
|
259
|
-
header,
|
|
260
|
-
roundFailure,
|
|
261
|
-
pushes: [],
|
|
262
|
-
pull: undefined,
|
|
263
|
-
subscriptions: [],
|
|
264
|
-
resolved: { ok: false },
|
|
265
|
-
schemaFloor: false,
|
|
266
|
-
leaseToEmit: undefined,
|
|
267
|
-
};
|
|
285
|
+
return { ...unserved, schemaFloor: true };
|
|
268
286
|
}
|
|
287
|
+
if (epochReset) return { ...unserved, schemaFloor: false };
|
|
269
288
|
|
|
270
289
|
if (header.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
|
|
271
290
|
throw syncError(
|
|
@@ -292,83 +311,111 @@ async function planRequest(
|
|
|
292
311
|
if (totalOperations > limits.maxOperationsPerRequest) {
|
|
293
312
|
throw syncError(
|
|
294
313
|
'sync.too_many_operations',
|
|
295
|
-
|
|
314
|
+
'request carries more operations than the server accepts (§6.1)',
|
|
315
|
+
JSON.stringify({
|
|
316
|
+
max: limits.maxOperationsPerRequest,
|
|
317
|
+
actual: totalOperations,
|
|
318
|
+
}),
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
// §6.1 sequence rules: with an outbox every clientSeq is ≥ 1 and strictly
|
|
322
|
+
// increasing in frame order; without one every clientSeq is 0.
|
|
323
|
+
if (
|
|
324
|
+
pushes.some((push, i) =>
|
|
325
|
+
header.outboxId === undefined
|
|
326
|
+
? push.clientSeq !== 0
|
|
327
|
+
: push.clientSeq < 1 ||
|
|
328
|
+
(i > 0 && push.clientSeq <= pushes[i - 1]!.clientSeq),
|
|
329
|
+
)
|
|
330
|
+
) {
|
|
331
|
+
throw syncError(
|
|
332
|
+
'sync.invalid_request',
|
|
333
|
+
'push commits break the clientSeq rules (§6.1)',
|
|
334
|
+
JSON.stringify({
|
|
335
|
+
outbox: header.outboxId !== undefined,
|
|
336
|
+
clientSeqs: pushes.map((push) => push.clientSeq),
|
|
337
|
+
}),
|
|
296
338
|
);
|
|
297
339
|
}
|
|
298
|
-
// §1.7:
|
|
299
|
-
if (
|
|
340
|
+
// §1.7: the UNIT cap, checked before any unit is planned.
|
|
341
|
+
if (unitFrames.length > limits.maxSubscriptionsPerRequest) {
|
|
300
342
|
throw syncError(
|
|
301
343
|
'sync.limit_exceeded',
|
|
302
|
-
'request carries more
|
|
344
|
+
'request carries more units than the server accepts (§1.7)',
|
|
303
345
|
JSON.stringify({
|
|
304
346
|
limit: 'maxSubscriptionsPerRequest',
|
|
305
347
|
max: limits.maxSubscriptionsPerRequest,
|
|
306
|
-
actual:
|
|
348
|
+
actual: unitFrames.length,
|
|
307
349
|
}),
|
|
308
350
|
);
|
|
309
351
|
}
|
|
310
|
-
for (const frame of subFrames) {
|
|
311
|
-
let values = 0;
|
|
312
|
-
for (const list of Object.values(frame.scopes)) values += list.length;
|
|
313
|
-
if (values > limits.maxScopeValuesPerSubscription) {
|
|
314
|
-
throw syncError(
|
|
315
|
-
'sync.limit_exceeded',
|
|
316
|
-
'subscription requests more scope values than the server accepts (§1.7)',
|
|
317
|
-
JSON.stringify({
|
|
318
|
-
limit: 'maxScopeValuesPerSubscription',
|
|
319
|
-
max: limits.maxScopeValuesPerSubscription,
|
|
320
|
-
actual: values,
|
|
321
|
-
subscriptionId: frame.id,
|
|
322
|
-
}),
|
|
323
|
-
);
|
|
324
|
-
}
|
|
325
|
-
}
|
|
326
352
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
353
|
+
// §4.1: the effective list is the traveling list, or the stored list the
|
|
354
|
+
// digest names; a digest naming neither answers `unregistered`.
|
|
355
|
+
let list: { scopes: UnitScopes; tables: CompiledTable[] }[] | undefined;
|
|
356
|
+
let registration: ClientRegistration | undefined;
|
|
357
|
+
if (pullHeader !== undefined) {
|
|
358
|
+
// §4.2: rows support is mandatory (bits 0 and 1).
|
|
359
|
+
if (
|
|
360
|
+
(pullHeader.accept & (ACCEPT_INLINE_ROWS | ACCEPT_EXTERNAL_ROWS)) ===
|
|
361
|
+
0
|
|
362
|
+
) {
|
|
331
363
|
throw syncError(
|
|
332
364
|
'sync.invalid_request',
|
|
333
365
|
'accept must include rows-segment support (bits 0 and 1, §4.2)',
|
|
334
366
|
);
|
|
335
367
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
if (seenIds.has(frame.id)) {
|
|
344
|
-
throw syncError(
|
|
345
|
-
'sync.invalid_subscription',
|
|
346
|
-
`duplicate subscription id ${JSON.stringify(frame.id)} (§4.1)`,
|
|
347
|
-
);
|
|
348
|
-
}
|
|
349
|
-
seenIds.add(frame.id);
|
|
350
|
-
const table = schema.tables.get(frame.table);
|
|
351
|
-
if (table === undefined) {
|
|
352
|
-
throw syncError(
|
|
353
|
-
'sync.unknown_table',
|
|
354
|
-
`subscription names unknown table ${JSON.stringify(frame.table)} (§4.3)`,
|
|
355
|
-
);
|
|
356
|
-
}
|
|
357
|
-
for (const [key, values] of Object.entries(frame.scopes)) {
|
|
358
|
-
if (!table.declaredVariables.has(key)) {
|
|
368
|
+
const traveling =
|
|
369
|
+
unitFrames.length > 0 ||
|
|
370
|
+
sameBytes(pullHeader.registrationDigest, registrationDigest([]));
|
|
371
|
+
if (traveling) {
|
|
372
|
+
const units = unitFrames.map((frame) => frame.scopes);
|
|
373
|
+
const digest = registrationDigest(units);
|
|
374
|
+
if (!sameBytes(digest, pullHeader.registrationDigest)) {
|
|
359
375
|
throw syncError(
|
|
360
376
|
'sync.invalid_subscription',
|
|
361
|
-
|
|
377
|
+
'the registration digest differs from the traveling list (§4.1)',
|
|
378
|
+
JSON.stringify({ reason: 'registration_digest' }),
|
|
362
379
|
);
|
|
363
380
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
381
|
+
registration = { units, digest };
|
|
382
|
+
}
|
|
383
|
+
const units =
|
|
384
|
+
registration?.units ??
|
|
385
|
+
(record?.registration !== undefined &&
|
|
386
|
+
sameBytes(record.registration.digest, pullHeader.registrationDigest)
|
|
387
|
+
? record.registration.units
|
|
388
|
+
: undefined);
|
|
389
|
+
if (units !== undefined) {
|
|
390
|
+
const ids = new Set<string>();
|
|
391
|
+
list = units.map((scopes) => {
|
|
392
|
+
const tables = coveredTables(schema, scopes);
|
|
393
|
+
const id = unitId(scopes);
|
|
394
|
+
if (ids.has(id)) {
|
|
395
|
+
throw syncError(
|
|
396
|
+
'sync.invalid_subscription',
|
|
397
|
+
'the registration list names a scope map twice (§4.1)',
|
|
398
|
+
JSON.stringify({ reason: 'duplicate_unit', scopes }),
|
|
399
|
+
);
|
|
400
|
+
}
|
|
401
|
+
ids.add(id);
|
|
402
|
+
return { scopes, tables };
|
|
403
|
+
});
|
|
404
|
+
const progressed = new Set<string>();
|
|
405
|
+
for (const frame of progressFrames) {
|
|
406
|
+
if (!ids.has(frame.id) || progressed.has(frame.id)) {
|
|
407
|
+
throw syncError(
|
|
408
|
+
'sync.invalid_subscription',
|
|
409
|
+
'a PROGRESS frame names no registered unit or repeats one (§4.1)',
|
|
410
|
+
JSON.stringify({
|
|
411
|
+
reason: ids.has(frame.id) ? 'duplicate_progress' : 'unknown_unit',
|
|
412
|
+
id: frame.id,
|
|
413
|
+
}),
|
|
414
|
+
);
|
|
415
|
+
}
|
|
416
|
+
progressed.add(frame.id);
|
|
369
417
|
}
|
|
370
418
|
}
|
|
371
|
-
validated.push({ frame, table: frame.table });
|
|
372
419
|
}
|
|
373
420
|
|
|
374
421
|
const { resolved, leaseToEmit } = await resolveOnce(
|
|
@@ -376,21 +423,7 @@ async function planRequest(
|
|
|
376
423
|
header.clientId,
|
|
377
424
|
compileSchema(ctx.schema),
|
|
378
425
|
);
|
|
379
|
-
|
|
380
|
-
const subscriptions: SubscriptionPlan[] = validated.map(({ frame }) => {
|
|
381
|
-
const table = schema.tables.get(frame.table);
|
|
382
|
-
if (table === undefined) throw new Error('unreachable: validated table');
|
|
383
|
-
const outcome = computeEffective(
|
|
384
|
-
table.declaredVariables,
|
|
385
|
-
frame.scopes,
|
|
386
|
-
resolved,
|
|
387
|
-
);
|
|
388
|
-
if (outcome.status === 'active') {
|
|
389
|
-
return { frame, table, status: 'active', effective: outcome.effective };
|
|
390
|
-
}
|
|
391
|
-
return { frame, table, status: 'revoked', effective: {} };
|
|
392
|
-
});
|
|
393
|
-
|
|
426
|
+
const progress = new Map(progressFrames.map((frame) => [frame.id, frame]));
|
|
394
427
|
return {
|
|
395
428
|
wireVersion: request.wireVersion,
|
|
396
429
|
logEpoch: registry.logEpoch,
|
|
@@ -398,14 +431,29 @@ async function planRequest(
|
|
|
398
431
|
header,
|
|
399
432
|
roundFailure,
|
|
400
433
|
pushes,
|
|
401
|
-
pull
|
|
402
|
-
|
|
434
|
+
pull:
|
|
435
|
+
pullHeader === undefined
|
|
436
|
+
? undefined
|
|
437
|
+
: {
|
|
438
|
+
header: pullHeader,
|
|
439
|
+
limits: clampPullLimits(pullHeader),
|
|
440
|
+
registration,
|
|
441
|
+
units: list?.map(({ scopes, tables }) => {
|
|
442
|
+
const plan = planUnit(scopes, tables, resolved);
|
|
443
|
+
return { plan, progress: progress.get(plan.id) };
|
|
444
|
+
}),
|
|
445
|
+
},
|
|
446
|
+
record,
|
|
403
447
|
resolved,
|
|
404
448
|
schemaFloor: false,
|
|
405
449
|
leaseToEmit,
|
|
406
450
|
};
|
|
407
451
|
}
|
|
408
452
|
|
|
453
|
+
function sameBytes(a: Uint8Array, b: Uint8Array): boolean {
|
|
454
|
+
return a.length === b.length && a.every((byte, i) => byte === b[i]);
|
|
455
|
+
}
|
|
456
|
+
|
|
409
457
|
/** Mutable outcome box shared with the instrumented stream wrapper. */
|
|
410
458
|
interface RequestReport {
|
|
411
459
|
outcome: 'ok' | 'schema_floor' | 'epoch_acquired' | 'reset' | 'error';
|
|
@@ -561,19 +609,41 @@ async function* streamResponse(
|
|
|
561
609
|
});
|
|
562
610
|
}
|
|
563
611
|
}
|
|
612
|
+
const { clientId, outboxId } = plan.header;
|
|
564
613
|
try {
|
|
565
614
|
// Push half (§6): one PUSH_RESULT per PUSH_COMMIT, in request order.
|
|
566
615
|
const currentSchema = compileSchema(ctx.schema);
|
|
567
616
|
let acceptedThrough = 0;
|
|
568
|
-
|
|
617
|
+
// §6.3.2: the request's first commit settles the outbox and reads `W`.
|
|
618
|
+
let watermark = 0;
|
|
619
|
+
for (const [index, push] of plan.pushes.entries()) {
|
|
569
620
|
const processed = await processPushCommitWithTrace(
|
|
570
621
|
ctx,
|
|
571
622
|
currentSchema,
|
|
572
623
|
plan.resolved,
|
|
573
|
-
|
|
624
|
+
clientId,
|
|
574
625
|
push,
|
|
575
626
|
schema,
|
|
627
|
+
outboxId === undefined
|
|
628
|
+
? undefined
|
|
629
|
+
: {
|
|
630
|
+
outboxId,
|
|
631
|
+
clientSeq: push.clientSeq,
|
|
632
|
+
watermark,
|
|
633
|
+
...(index === 0
|
|
634
|
+
? {
|
|
635
|
+
settlement: {
|
|
636
|
+
outboxId,
|
|
637
|
+
commits: plan.pushes.map((commit) => ({
|
|
638
|
+
clientCommitId: commit.clientCommitId,
|
|
639
|
+
clientSeq: commit.clientSeq,
|
|
640
|
+
})),
|
|
641
|
+
},
|
|
642
|
+
}
|
|
643
|
+
: {}),
|
|
644
|
+
},
|
|
576
645
|
);
|
|
646
|
+
watermark = processed.watermark ?? watermark;
|
|
577
647
|
acceptedThrough = Math.max(
|
|
578
648
|
acceptedThrough,
|
|
579
649
|
processed.frame.commitSeq ?? 0,
|
|
@@ -618,7 +688,7 @@ async function* streamResponse(
|
|
|
618
688
|
}),
|
|
619
689
|
};
|
|
620
690
|
if (events !== undefined) {
|
|
621
|
-
emitPushEvent(events, ctx,
|
|
691
|
+
emitPushEvent(events, ctx, clientId, push, processed);
|
|
622
692
|
}
|
|
623
693
|
yield encodeResponseFrame(frame, plan.wireVersion);
|
|
624
694
|
const details = pushResultDetailsFrame(frame);
|
|
@@ -627,12 +697,34 @@ async function* streamResponse(
|
|
|
627
697
|
}
|
|
628
698
|
}
|
|
629
699
|
|
|
630
|
-
|
|
631
|
-
const
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
700
|
+
const previous = plan.record;
|
|
701
|
+
const base = {
|
|
702
|
+
clientId,
|
|
703
|
+
actorId: ctx.actorId,
|
|
704
|
+
wireVersion: plan.wireVersion,
|
|
705
|
+
schemaVersion: schema.version,
|
|
706
|
+
};
|
|
707
|
+
const pull = plan.pull;
|
|
708
|
+
if (pull === undefined) {
|
|
709
|
+
// A push-only round keeps the registration, its states and the cursor;
|
|
710
|
+
// its record binds the actor (§1.5) and anchors the watermark rows
|
|
711
|
+
// (§6.3.2).
|
|
712
|
+
report.persist = () =>
|
|
713
|
+
ctx.storage.putClientRecord(
|
|
714
|
+
ctx.partition,
|
|
715
|
+
{
|
|
716
|
+
...base,
|
|
717
|
+
cursor: previous?.cursor ?? -1,
|
|
718
|
+
updatedAtMs: clockOf(ctx)(),
|
|
719
|
+
catchingUp: previous?.catchingUp ?? [],
|
|
720
|
+
},
|
|
721
|
+
plan.roundFailure !== undefined
|
|
722
|
+
? { keepRoundOutcome: true }
|
|
723
|
+
: undefined,
|
|
724
|
+
);
|
|
725
|
+
} else {
|
|
726
|
+
// Pull half (§4): unit sections in registration order, then one live
|
|
727
|
+
// section.
|
|
636
728
|
const maxSeq = await ctx.storage.getMaxCommitSeq(ctx.partition);
|
|
637
729
|
if (maxSeq < acceptedThrough)
|
|
638
730
|
throw syncError(
|
|
@@ -640,166 +732,423 @@ async function* streamResponse(
|
|
|
640
732
|
'storage returned a commit sequence older than the accepted push; disable query caching for sync storage',
|
|
641
733
|
JSON.stringify({ acceptedThrough, storageMaxCommitSeq: maxSeq }),
|
|
642
734
|
);
|
|
643
|
-
const
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
)
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
735
|
+
const { limits } = pull;
|
|
736
|
+
const units = pull.units;
|
|
737
|
+
if (units === undefined) {
|
|
738
|
+
// §4.1: no unit section, no COMMIT; PROGRESS frames are ignored.
|
|
739
|
+
yield encodeResponseFrame(
|
|
740
|
+
{ type: 'LIVE_START', status: 'unregistered', reasonCode: '' },
|
|
741
|
+
plan.wireVersion,
|
|
742
|
+
);
|
|
743
|
+
yield encodeResponseFrame(
|
|
744
|
+
{ type: 'LIVE_END', throughSeq: -1, echoDigest: new Uint8Array() },
|
|
745
|
+
plan.wireVersion,
|
|
746
|
+
);
|
|
747
|
+
if (events !== undefined) {
|
|
748
|
+
emitEvent(events, {
|
|
749
|
+
type: 'pull.served',
|
|
750
|
+
atMs: clockOf(ctx)(),
|
|
751
|
+
partition: ctx.partition,
|
|
752
|
+
actorId: ctx.actorId,
|
|
753
|
+
clientId,
|
|
754
|
+
acceptedThrough,
|
|
755
|
+
storageMaxCommitSeq: maxSeq,
|
|
756
|
+
registrationTraveled: false,
|
|
757
|
+
units: [],
|
|
758
|
+
live: {
|
|
759
|
+
status: 'unregistered',
|
|
760
|
+
fromCursor: pull.header.cursor,
|
|
761
|
+
throughSeq: -1,
|
|
762
|
+
commits: 0,
|
|
763
|
+
changes: 0,
|
|
764
|
+
},
|
|
765
|
+
});
|
|
766
|
+
}
|
|
767
|
+
// §4.5: an `unregistered` answer writes only the round outcome; the
|
|
768
|
+
// stored record keeps its cursor, timestamp and states.
|
|
769
|
+
if (previous !== undefined) {
|
|
770
|
+
report.persist = () =>
|
|
771
|
+
ctx.storage.putClientRecord(
|
|
772
|
+
ctx.partition,
|
|
773
|
+
{
|
|
774
|
+
clientId,
|
|
775
|
+
actorId: previous.actorId,
|
|
776
|
+
wireVersion: previous.wireVersion,
|
|
777
|
+
cursor: previous.cursor,
|
|
778
|
+
updatedAtMs: previous.updatedAtMs,
|
|
779
|
+
lastRoundAtMs: clockOf(ctx)(),
|
|
780
|
+
...(previous.schemaVersion !== undefined
|
|
781
|
+
? { schemaVersion: previous.schemaVersion }
|
|
782
|
+
: {}),
|
|
783
|
+
catchingUp: previous.catchingUp,
|
|
784
|
+
},
|
|
785
|
+
plan.roundFailure !== undefined
|
|
786
|
+
? { keepRoundOutcome: true }
|
|
787
|
+
: undefined,
|
|
788
|
+
);
|
|
789
|
+
}
|
|
790
|
+
} else {
|
|
791
|
+
const horizonSeq = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
792
|
+
const R = pull.header.cursor;
|
|
793
|
+
const echo = echoDigest(
|
|
794
|
+
units.map(({ plan: unit }) => ({
|
|
795
|
+
id: unit.id,
|
|
796
|
+
status: unit.status,
|
|
797
|
+
fenced: unit.fenced,
|
|
798
|
+
})),
|
|
799
|
+
);
|
|
800
|
+
// §4.4 rule 1: every unit gets a section when the list traveled or
|
|
801
|
+
// the client holds another echo.
|
|
802
|
+
const echoSections =
|
|
803
|
+
pull.registration !== undefined ||
|
|
804
|
+
!sameBytes(pull.header.echoDigest, echo);
|
|
805
|
+
const liveUnits = units.some(
|
|
806
|
+
(unit) =>
|
|
807
|
+
unit.progress === undefined && unit.plan.status === 'active',
|
|
666
808
|
);
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
809
|
+
// §4.6 replica reset: a live unit with R outside [horizon, max].
|
|
810
|
+
let liveReset = liveUnits && (R < horizonSeq || R > maxSeq);
|
|
811
|
+
// Per unit: its section kind and its scan-set bound (§4.5).
|
|
812
|
+
type UnitState =
|
|
813
|
+
| { kind: 'revoked' }
|
|
814
|
+
| { kind: 'reset' }
|
|
815
|
+
| { kind: 'live' }
|
|
816
|
+
| { kind: 'post'; cursor: number }
|
|
817
|
+
| {
|
|
818
|
+
kind: 'bootstrap';
|
|
819
|
+
pin: number;
|
|
820
|
+
frames: ResponseFrame[];
|
|
821
|
+
token: string | undefined;
|
|
822
|
+
};
|
|
823
|
+
const states: UnitState[] = units.map(({ plan: unit, progress }) => {
|
|
824
|
+
if (unit.status === 'revoked') return { kind: 'revoked' };
|
|
825
|
+
if (progress === undefined) return { kind: 'live' };
|
|
826
|
+
// §4.7: a token, cursor -1 or a cursor from the future bootstraps.
|
|
827
|
+
if (
|
|
828
|
+
progress.bootstrapState === undefined &&
|
|
829
|
+
progress.cursor >= 0 &&
|
|
830
|
+
progress.cursor <= maxSeq
|
|
831
|
+
) {
|
|
832
|
+
return progress.cursor < horizonSeq
|
|
833
|
+
? { kind: 'reset' }
|
|
834
|
+
: { kind: 'post', cursor: progress.cursor };
|
|
673
835
|
}
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
836
|
+
// Replaced by the section the bootstrap below produces.
|
|
837
|
+
return { kind: 'bootstrap', pin: -1, frames: [], token: '' };
|
|
838
|
+
});
|
|
839
|
+
const traces = units.map((): PullSectionTrace | undefined =>
|
|
840
|
+
events !== undefined ? { segments: [] } : undefined,
|
|
841
|
+
);
|
|
842
|
+
// Bootstrap sections run before the live read: a bootstrap that
|
|
843
|
+
// completes in this response enters the scan set at its pin (§4.5).
|
|
844
|
+
// A window of units runs its pre-frame reads together, so a
|
|
845
|
+
// batching storage answers them in one statement per table. A unit
|
|
846
|
+
// reads ahead the first page of at most `maxSnapshotPages` tables,
|
|
847
|
+
// and a window holds at most 100,000 first-page rows, so read memory
|
|
848
|
+
// follows the configured limits, never the unit count.
|
|
849
|
+
const bootstrapping = states.flatMap((state, index) =>
|
|
850
|
+
state.kind === 'bootstrap' ? [index] : [],
|
|
851
|
+
);
|
|
852
|
+
const firstPages = Math.max(
|
|
853
|
+
1,
|
|
854
|
+
Math.floor(100_000 / (limits.limitSnapshotRows + 1)),
|
|
855
|
+
);
|
|
856
|
+
for (let next = 0; next < bootstrapping.length;) {
|
|
857
|
+
const window: number[] = [];
|
|
858
|
+
let pages = 0;
|
|
859
|
+
while (next < bootstrapping.length) {
|
|
860
|
+
const index = bootstrapping[next]!;
|
|
861
|
+
const reads = Math.min(
|
|
862
|
+
limits.maxSnapshotPages,
|
|
863
|
+
units[index]!.plan.pairs.size,
|
|
864
|
+
);
|
|
865
|
+
if (window.length > 0 && pages + reads > firstPages) break;
|
|
866
|
+
window.push(index);
|
|
867
|
+
pages += reads;
|
|
868
|
+
next += 1;
|
|
869
|
+
}
|
|
870
|
+
const batch = window.map((index) => ({
|
|
871
|
+
index,
|
|
872
|
+
plan: planBootstrap(
|
|
873
|
+
ctx,
|
|
874
|
+
schema,
|
|
875
|
+
limits,
|
|
876
|
+
units[index]!.plan,
|
|
877
|
+
units[index]!.progress?.bootstrapState,
|
|
878
|
+
maxSeq,
|
|
879
|
+
horizonSeq,
|
|
880
|
+
plan.logEpoch,
|
|
881
|
+
),
|
|
882
|
+
}));
|
|
883
|
+
for (const { index, plan: bootstrap } of batch) {
|
|
884
|
+
const frames: ResponseFrame[] = [];
|
|
885
|
+
const segments = bootstrapSegments(
|
|
886
|
+
ctx,
|
|
887
|
+
schema,
|
|
888
|
+
limits,
|
|
889
|
+
bootstrap,
|
|
890
|
+
traces[index],
|
|
891
|
+
plan.logEpoch,
|
|
892
|
+
);
|
|
893
|
+
let step = await segments.next();
|
|
894
|
+
while (!step.done) {
|
|
895
|
+
frames.push(step.value);
|
|
896
|
+
step = await segments.next();
|
|
897
|
+
}
|
|
898
|
+
states[index] = {
|
|
899
|
+
kind: 'bootstrap',
|
|
900
|
+
pin: bootstrap.pin,
|
|
901
|
+
frames,
|
|
902
|
+
token: step.value,
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
const bound = (state: UnitState): number | undefined =>
|
|
907
|
+
state.kind === 'live'
|
|
908
|
+
? R
|
|
909
|
+
: state.kind === 'post'
|
|
910
|
+
? state.cursor
|
|
911
|
+
: state.kind === 'bootstrap' && state.token === undefined
|
|
912
|
+
? state.pin
|
|
913
|
+
: undefined;
|
|
914
|
+
// §4.5 order of work: read and buffer the live section, then recheck
|
|
915
|
+
// the horizon. A bound below it leaves the scan set (a live unit's
|
|
916
|
+
// resets the live section), and the section is read again.
|
|
917
|
+
let live: LiveWindow | undefined;
|
|
918
|
+
for (;;) {
|
|
919
|
+
const scan = units.flatMap(({ plan: unit }, index) => {
|
|
920
|
+
const afterSeq = bound(states[index]!);
|
|
921
|
+
return afterSeq === undefined
|
|
922
|
+
? []
|
|
923
|
+
: [{ scopes: unit.scopes, afterSeq, tables: unit.pairs }];
|
|
924
|
+
});
|
|
925
|
+
if (liveReset || scan.length === 0) break;
|
|
926
|
+
live = await ctx.storage.readLiveWindow(ctx.partition, {
|
|
927
|
+
units: scan,
|
|
928
|
+
throughSeq: maxSeq,
|
|
929
|
+
limitChanges: limits.limitCommits,
|
|
930
|
+
});
|
|
931
|
+
const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
932
|
+
let expired = false;
|
|
933
|
+
for (const [index, state] of states.entries()) {
|
|
934
|
+
const afterSeq = bound(state);
|
|
935
|
+
if (afterSeq === undefined || afterSeq >= recheck) continue;
|
|
936
|
+
expired = true;
|
|
937
|
+
if (state.kind === 'live') liveReset = true;
|
|
938
|
+
else states[index] = { kind: 'reset' };
|
|
939
|
+
}
|
|
940
|
+
if (!expired) break;
|
|
941
|
+
live = undefined;
|
|
942
|
+
}
|
|
943
|
+
const T = liveReset ? -1 : (live?.throughSeq ?? maxSeq);
|
|
944
|
+
|
|
945
|
+
const summaries: PullUnitSummary[] = [];
|
|
946
|
+
for (const [index, { plan: unit, progress }] of units.entries()) {
|
|
947
|
+
const state = states[index]!;
|
|
948
|
+
const requestCursor = progress?.cursor ?? R;
|
|
949
|
+
const section =
|
|
950
|
+
state.kind === 'revoked' ||
|
|
951
|
+
state.kind === 'reset' ||
|
|
952
|
+
state.kind === 'bootstrap' ||
|
|
953
|
+
echoSections;
|
|
954
|
+
if (section) {
|
|
955
|
+
yield encodeResponseFrame(
|
|
956
|
+
state.kind === 'revoked'
|
|
957
|
+
? {
|
|
958
|
+
type: 'SUB_START',
|
|
959
|
+
id: unit.id,
|
|
960
|
+
status: 'revoked',
|
|
961
|
+
reasonCode: 'sync.scope_revoked',
|
|
962
|
+
bootstrap: false,
|
|
963
|
+
fenced: [],
|
|
964
|
+
}
|
|
965
|
+
: state.kind === 'reset'
|
|
966
|
+
? {
|
|
967
|
+
type: 'SUB_START',
|
|
968
|
+
id: unit.id,
|
|
969
|
+
status: 'reset',
|
|
970
|
+
reasonCode: 'sync.cursor_expired',
|
|
971
|
+
bootstrap: false,
|
|
972
|
+
fenced: [],
|
|
973
|
+
}
|
|
974
|
+
: {
|
|
975
|
+
type: 'SUB_START',
|
|
976
|
+
id: unit.id,
|
|
977
|
+
status: 'active',
|
|
978
|
+
reasonCode: '',
|
|
979
|
+
bootstrap: state.kind === 'bootstrap',
|
|
980
|
+
fenced: [...unit.fenced],
|
|
981
|
+
},
|
|
982
|
+
plan.wireVersion,
|
|
983
|
+
);
|
|
984
|
+
if (state.kind === 'bootstrap') {
|
|
985
|
+
for (const frame of state.frames) {
|
|
986
|
+
yield encodeResponseFrame(frame, plan.wireVersion);
|
|
704
987
|
}
|
|
705
988
|
}
|
|
706
|
-
yield encodeResponseFrame(
|
|
707
|
-
|
|
989
|
+
yield encodeResponseFrame(
|
|
990
|
+
state.kind === 'bootstrap'
|
|
991
|
+
? {
|
|
992
|
+
type: 'SUB_END',
|
|
993
|
+
nextCursor: state.pin,
|
|
994
|
+
...(state.token !== undefined
|
|
995
|
+
? { bootstrapState: state.token }
|
|
996
|
+
: {}),
|
|
997
|
+
}
|
|
998
|
+
: { type: 'SUB_END', nextCursor: requestCursor },
|
|
999
|
+
plan.wireVersion,
|
|
1000
|
+
);
|
|
708
1001
|
}
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
1002
|
+
summaries.push({
|
|
1003
|
+
id: unit.id,
|
|
1004
|
+
scopes: unit.scopes,
|
|
1005
|
+
status:
|
|
1006
|
+
state.kind === 'revoked' || state.kind === 'reset'
|
|
1007
|
+
? state.kind
|
|
1008
|
+
: 'active',
|
|
1009
|
+
mode:
|
|
1010
|
+
state.kind === 'revoked' || state.kind === 'reset'
|
|
1011
|
+
? 'none'
|
|
1012
|
+
: state.kind === 'post'
|
|
1013
|
+
? 'catching-up'
|
|
1014
|
+
: state.kind,
|
|
1015
|
+
section,
|
|
1016
|
+
fenced: unit.fenced,
|
|
1017
|
+
fromCursor: requestCursor,
|
|
1018
|
+
segments: traces[index]?.segments ?? [],
|
|
1019
|
+
});
|
|
1020
|
+
}
|
|
1021
|
+
yield encodeResponseFrame(
|
|
1022
|
+
liveReset
|
|
1023
|
+
? {
|
|
1024
|
+
type: 'LIVE_START',
|
|
1025
|
+
status: 'reset',
|
|
1026
|
+
reasonCode: 'sync.cursor_expired',
|
|
1027
|
+
}
|
|
1028
|
+
: { type: 'LIVE_START', status: 'active', reasonCode: '' },
|
|
1029
|
+
plan.wireVersion,
|
|
1030
|
+
);
|
|
1031
|
+
let changes = 0;
|
|
1032
|
+
for (const commit of live?.commits ?? []) {
|
|
1033
|
+
changes += commit.changes.length;
|
|
1034
|
+
yield encodeResponseFrame(
|
|
1035
|
+
commitFrame(commit, currentSchema, schema),
|
|
1036
|
+
plan.wireVersion,
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
yield encodeResponseFrame(
|
|
1040
|
+
{ type: 'LIVE_END', throughSeq: T, echoDigest: echo },
|
|
1041
|
+
plan.wireVersion,
|
|
1042
|
+
);
|
|
1043
|
+
if (events !== undefined) {
|
|
1044
|
+
const commits = live?.commits ?? [];
|
|
1045
|
+
emitEvent(events, {
|
|
1046
|
+
type: 'pull.served',
|
|
1047
|
+
atMs: clockOf(ctx)(),
|
|
1048
|
+
partition: ctx.partition,
|
|
1049
|
+
actorId: ctx.actorId,
|
|
1050
|
+
clientId,
|
|
1051
|
+
acceptedThrough,
|
|
1052
|
+
storageMaxCommitSeq: maxSeq,
|
|
1053
|
+
registrationTraveled: pull.registration !== undefined,
|
|
1054
|
+
units: summaries,
|
|
1055
|
+
live: {
|
|
1056
|
+
status: liveReset ? 'reset' : 'active',
|
|
1057
|
+
fromCursor: R,
|
|
1058
|
+
throughSeq: T,
|
|
1059
|
+
commits: commits.length,
|
|
729
1060
|
changes,
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
1061
|
+
...(commits.length > 0
|
|
1062
|
+
? {
|
|
1063
|
+
firstCommitSeq: commits[0]!.commitSeq,
|
|
1064
|
+
lastCommitSeq: commits.at(-1)!.commitSeq,
|
|
1065
|
+
}
|
|
1066
|
+
: {}),
|
|
1067
|
+
},
|
|
1068
|
+
});
|
|
733
1069
|
}
|
|
734
|
-
}
|
|
735
|
-
if (events !== undefined && summaries !== undefined) {
|
|
736
|
-
emitEvent(events, {
|
|
737
|
-
type: 'pull.served',
|
|
738
|
-
atMs: clockOf(ctx)(),
|
|
739
|
-
partition: ctx.partition,
|
|
740
|
-
actorId: ctx.actorId,
|
|
741
|
-
clientId: plan.header.clientId,
|
|
742
|
-
acceptedThrough,
|
|
743
|
-
storageMaxCommitSeq: maxSeq,
|
|
744
|
-
subscriptions: summaries,
|
|
745
|
-
});
|
|
746
|
-
}
|
|
747
|
-
}
|
|
748
1070
|
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
1071
|
+
// §4.5 join rule J and the record the round writes: the live units,
|
|
1072
|
+
// the catching-up ids with their cursors, and the record cursor.
|
|
1073
|
+
const joined = Math.max(R, T);
|
|
1074
|
+
let anyLive = false;
|
|
1075
|
+
const catchingUp: CatchingUpUnit[] = [];
|
|
1076
|
+
for (const [index, { plan: unit }] of units.entries()) {
|
|
1077
|
+
const state = states[index]!;
|
|
1078
|
+
const catching = (cursor: number): void => {
|
|
1079
|
+
catchingUp.push({ id: unit.id, cursor });
|
|
1080
|
+
};
|
|
1081
|
+
if (state.kind === 'live') {
|
|
1082
|
+
if (liveReset) catching(-1);
|
|
1083
|
+
else anyLive = true;
|
|
1084
|
+
} else if (state.kind === 'revoked' || state.kind === 'reset') {
|
|
1085
|
+
catching(-1);
|
|
1086
|
+
} else if (state.kind === 'bootstrap' && state.token !== undefined) {
|
|
1087
|
+
catching(state.pin);
|
|
1088
|
+
} else {
|
|
1089
|
+
// Post-bootstrap, or a bootstrap completed here at its pin.
|
|
1090
|
+
const cursor = state.kind === 'post' ? state.cursor : state.pin;
|
|
1091
|
+
if (liveReset) catching(cursor);
|
|
1092
|
+
else if (Math.max(cursor, T) >= joined) anyLive = true;
|
|
1093
|
+
else catching(Math.max(cursor, T));
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
const held = [
|
|
1097
|
+
...(anyLive ? [joined] : []),
|
|
1098
|
+
...catchingUp.map((unit) => unit.cursor).filter((c) => c >= 0),
|
|
1099
|
+
];
|
|
1100
|
+
const liveIds = (
|
|
1101
|
+
listed: readonly UnitScopes[],
|
|
1102
|
+
catching: readonly CatchingUpUnit[],
|
|
1103
|
+
): string =>
|
|
1104
|
+
JSON.stringify(
|
|
1105
|
+
listed
|
|
1106
|
+
.map((scopes) => unitId(scopes))
|
|
1107
|
+
.filter((id) => !catching.some((unit) => unit.id === id)),
|
|
1108
|
+
);
|
|
1109
|
+
const next: ClientRecord = {
|
|
1110
|
+
...base,
|
|
1111
|
+
cursor: held.length === 0 ? -1 : Math.min(...held),
|
|
775
1112
|
updatedAtMs: clockOf(ctx)(),
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
1113
|
+
...(pull.registration !== undefined
|
|
1114
|
+
? { registration: pull.registration }
|
|
1115
|
+
: {}),
|
|
1116
|
+
catchingUp,
|
|
1117
|
+
};
|
|
1118
|
+
report.persist = async () => {
|
|
1119
|
+
await ctx.storage.putClientRecord(
|
|
1120
|
+
ctx.partition,
|
|
1121
|
+
next,
|
|
1122
|
+
// §4.5: a round that carried a `ROUND_FAILURE` report completes
|
|
1123
|
+
// without clearing the reported code and streak.
|
|
1124
|
+
plan.roundFailure !== undefined
|
|
1125
|
+
? { keepRoundOutcome: true }
|
|
1126
|
+
: undefined,
|
|
1127
|
+
);
|
|
1128
|
+
// §8.1: a traveled list or a changed live set re-registers the
|
|
1129
|
+
// client's sockets on this hub.
|
|
1130
|
+
if (
|
|
1131
|
+
ctx.realtime?.notifySubscriptions !== undefined &&
|
|
1132
|
+
(pull.registration !== undefined ||
|
|
1133
|
+
liveIds(
|
|
1134
|
+
previous?.registration?.units ?? [],
|
|
1135
|
+
previous?.catchingUp ?? [],
|
|
1136
|
+
) !==
|
|
1137
|
+
liveIds(
|
|
1138
|
+
units.map(({ plan: unit }) => unit.scopes),
|
|
1139
|
+
catchingUp,
|
|
1140
|
+
))
|
|
1141
|
+
) {
|
|
1142
|
+
await ctx.realtime.notifySubscriptions(ctx.partition, clientId);
|
|
1143
|
+
}
|
|
1144
|
+
};
|
|
796
1145
|
}
|
|
797
|
-
}
|
|
1146
|
+
}
|
|
798
1147
|
} catch (error) {
|
|
799
1148
|
if (error instanceof SyncError) {
|
|
800
1149
|
report.outcome = 'error';
|
|
801
1150
|
report.errorCode = error.code;
|
|
802
|
-
await recordRoundFailure(ctx,
|
|
1151
|
+
await recordRoundFailure(ctx, clientId, error);
|
|
803
1152
|
// §1.6: in-band ERROR, then END, nothing else.
|
|
804
1153
|
yield encodeResponseFrame(
|
|
805
1154
|
{
|
|
@@ -867,7 +1216,7 @@ export async function createSyncResponseStream(
|
|
|
867
1216
|
errorCode: error instanceof SyncError ? error.code : 'internal',
|
|
868
1217
|
pushCommits: 0,
|
|
869
1218
|
pulled: false,
|
|
870
|
-
|
|
1219
|
+
units: 0,
|
|
871
1220
|
});
|
|
872
1221
|
await recordRoundFailure(ctx, clientId, error);
|
|
873
1222
|
throw error;
|
|
@@ -877,11 +1226,8 @@ export async function createSyncResponseStream(
|
|
|
877
1226
|
/** The `REQ_HEADER` clientId of a request that failed before planning. */
|
|
878
1227
|
function requestClientId(bytes: Uint8Array): string | undefined {
|
|
879
1228
|
try {
|
|
880
|
-
const
|
|
881
|
-
|
|
882
|
-
return message.msgKind === 'request' && header?.type === 'REQ_HEADER'
|
|
883
|
-
? header.clientId
|
|
884
|
-
: undefined;
|
|
1229
|
+
const header = decodeRequest(bytes).frames[0];
|
|
1230
|
+
return header?.type === 'REQ_HEADER' ? header.clientId : undefined;
|
|
885
1231
|
} catch {
|
|
886
1232
|
return undefined;
|
|
887
1233
|
}
|
|
@@ -947,11 +1293,7 @@ async function createStreamCore(
|
|
|
947
1293
|
): Promise<AsyncIterable<Uint8Array>> {
|
|
948
1294
|
let request: RequestMessage;
|
|
949
1295
|
try {
|
|
950
|
-
|
|
951
|
-
if (message.msgKind !== 'request') {
|
|
952
|
-
throw syncError('sync.invalid_request', 'expected a request message');
|
|
953
|
-
}
|
|
954
|
-
request = message;
|
|
1296
|
+
request = decodeRequest(bytes);
|
|
955
1297
|
} catch (error) {
|
|
956
1298
|
if (error instanceof DecodeError) {
|
|
957
1299
|
throw syncError(error.code, error.message);
|
|
@@ -1083,7 +1425,7 @@ async function* instrumentedStream(
|
|
|
1083
1425
|
: {}),
|
|
1084
1426
|
pushCommits: plan.pushes.length,
|
|
1085
1427
|
pulled: plan.pull !== undefined,
|
|
1086
|
-
|
|
1428
|
+
units: plan.pull?.units?.length ?? 0,
|
|
1087
1429
|
});
|
|
1088
1430
|
};
|
|
1089
1431
|
try {
|