@terreno/api 57.14.0 → 57.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/sync/routes.d.ts +1 -0
- package/dist/sync/routes.js +204 -102
- package/dist/sync/syncApp.d.ts +3 -2
- package/dist/sync/syncApp.js +3 -2
- package/dist/sync/types.d.ts +18 -0
- package/package.json +3 -3
- package/src/sync/routes.ts +251 -108
- package/src/sync/syncApp.ts +3 -2
- package/src/sync/syncRoutes.test.ts +108 -0
- package/src/sync/types.ts +21 -0
package/dist/sync/routes.d.ts
CHANGED
|
@@ -69,6 +69,7 @@ export declare const MAX_ENTITY_FETCH = 100;
|
|
|
69
69
|
* Mount the SyncDB HTTP routes:
|
|
70
70
|
* - GET /sync/streams — the authoritative set of streams the caller belongs to (C2)
|
|
71
71
|
* - GET /sync/snapshot — per-stream bootstrap/catch-up with server-enforced scoping
|
|
72
|
+
* - POST /sync/snapshot/batch — one page for each of many streams in one round trip
|
|
72
73
|
* - POST /sync/mutate — HTTP fallback mutation channel over applySyncMutation
|
|
73
74
|
* - GET /sync/key — per-user key material for the default encryption KeyProvider
|
|
74
75
|
*/
|
package/dist/sync/routes.js
CHANGED
|
@@ -31,6 +31,13 @@ var syncBatch_2 = require("./syncBatch");
|
|
|
31
31
|
Object.defineProperty(exports, "MAX_SYNC_HTTP_MUTATIONS_PER_SECOND", { enumerable: true, get: function () { return syncBatch_2.MAX_SYNC_MUTATIONS_PER_SECOND; } });
|
|
32
32
|
const MAX_SNAPSHOT_LIMIT = 100;
|
|
33
33
|
const DEFAULT_SNAPSHOT_LIMIT = 100;
|
|
34
|
+
/** Entity budget of one `POST /sync/snapshot/batch` response, shared by its streams. */
|
|
35
|
+
const DEFAULT_SNAPSHOT_BATCH_LIMIT = 5_000;
|
|
36
|
+
const MAX_SNAPSHOT_BATCH_LIMIT = 10_000;
|
|
37
|
+
/** Most streams one batch request may name. */
|
|
38
|
+
const MAX_SNAPSHOT_BATCH_STREAMS = 500;
|
|
39
|
+
/** Streams paged at once per batch request, so one request cannot drain the Mongo pool. */
|
|
40
|
+
const SNAPSHOT_BATCH_CONCURRENCY = 8;
|
|
34
41
|
/**
|
|
35
42
|
* Look up a registered model by name as a `SyncedModel`. `mongoose.model(name)` has no
|
|
36
43
|
* schema to infer from, so its static type carries none of the synced-document fields.
|
|
@@ -331,10 +338,174 @@ const castEntityIds = ({ model, ids, }) => {
|
|
|
331
338
|
}
|
|
332
339
|
return ids.map((id) => new mongoose_1.default.Types.ObjectId(id));
|
|
333
340
|
};
|
|
341
|
+
/**
|
|
342
|
+
* Serve one snapshot page for one stream: membership/permission checks, frontier and
|
|
343
|
+
* retention signals, then the legacy or seq stratum. Shared by `GET /sync/snapshot` and
|
|
344
|
+
* `POST /sync/snapshot/batch`.
|
|
345
|
+
*/
|
|
346
|
+
const snapshotStreamPage = async ({ req, user, streamKey, cursor, limit, legacyCursorIn, options, }) => {
|
|
347
|
+
if (!streamKey) {
|
|
348
|
+
throw new errors_1.APIError({ status: 400, title: "stream query parameter is required" });
|
|
349
|
+
}
|
|
350
|
+
const parsed = (0, streams_1.parseStreamKey)(streamKey);
|
|
351
|
+
if (!parsed) {
|
|
352
|
+
throw new errors_1.APIError({ status: 400, title: `Invalid stream key: ${streamKey}` });
|
|
353
|
+
}
|
|
354
|
+
const entry = (0, registry_1.findSyncEntryByCollectionTag)(parsed.collectionTag);
|
|
355
|
+
if (!entry) {
|
|
356
|
+
throw new errors_1.APIError({
|
|
357
|
+
status: 404,
|
|
358
|
+
title: `Unknown sync collection: ${parsed.collectionTag}`,
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
if (!(await (0, permissions_1.checkPermissions)("list", entry.options.permissions.list, user))) {
|
|
362
|
+
throw new errors_1.APIError({
|
|
363
|
+
status: 403,
|
|
364
|
+
title: `Access to sync snapshot for ${parsed.collectionTag} denied for ${user.id}`,
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
// C2: a client must not snapshot a stream it does not belong to.
|
|
368
|
+
const memberStreams = await (0, streams_1.resolveUserStreamsForEntry)({
|
|
369
|
+
entry,
|
|
370
|
+
getUserScopes: options.getUserScopes,
|
|
371
|
+
user,
|
|
372
|
+
});
|
|
373
|
+
if (!memberStreams.includes(streamKey)) {
|
|
374
|
+
throw new errors_1.APIError({
|
|
375
|
+
status: 403,
|
|
376
|
+
title: `User ${user.id} does not belong to stream ${streamKey}`,
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
const snapshotFilterResult = entry.config.snapshotFilter
|
|
380
|
+
? await entry.config.snapshotFilter({ id: String(user.id) })
|
|
381
|
+
: undefined;
|
|
382
|
+
const queryFilterResult = await resolveSyncQueryFilter({ entry, user });
|
|
383
|
+
const scopeFilter = composeSyncReadFilter({
|
|
384
|
+
queryFilter: queryFilterResult.filter,
|
|
385
|
+
scopeFilter: (0, exports.buildSnapshotScopeFilter)({
|
|
386
|
+
entry,
|
|
387
|
+
scopeValue: parsed.scopeValue,
|
|
388
|
+
snapshotFilterResult,
|
|
389
|
+
}),
|
|
390
|
+
});
|
|
391
|
+
const model = getSyncedModel(entry.modelName);
|
|
392
|
+
const frontierSeq = await (0, models_1.computeStableFrontier)({ stream: streamKey });
|
|
393
|
+
// C7 (Task 9.15): the retention signal is the durable compaction watermark, not
|
|
394
|
+
// min(retained seq). The old computation was pinned low forever by any early doc that
|
|
395
|
+
// was never deleted, so a stale cursor sitting above a compacted tombstone's seq
|
|
396
|
+
// passed the check and silently never learned about that deletion.
|
|
397
|
+
const oldestRetainedSeq = await (0, models_1.getCompactedThroughSeq)({ stream: streamKey });
|
|
398
|
+
// Task 9.19: a `queryFilter` that denies this caller means "no visible rows", exactly
|
|
399
|
+
// as REST's list endpoint answers. Serve a terminal empty page (cursor at the
|
|
400
|
+
// frontier, hasMore false) so the client settles instead of paging forever.
|
|
401
|
+
if (queryFilterResult.denied) {
|
|
402
|
+
const deniedResponse = {
|
|
403
|
+
cursor: frontierSeq,
|
|
404
|
+
entities: [],
|
|
405
|
+
frontierSeq,
|
|
406
|
+
hasMore: false,
|
|
407
|
+
oldestRetainedSeq,
|
|
408
|
+
stream: streamKey,
|
|
409
|
+
};
|
|
410
|
+
return deniedResponse;
|
|
411
|
+
}
|
|
412
|
+
// C3: legacy (seq-0) stratum, paged by _id. Drained fully before seq paging begins.
|
|
413
|
+
if (cursor === 0) {
|
|
414
|
+
const legacyResult = await pageLegacyStratum({
|
|
415
|
+
entry,
|
|
416
|
+
legacyCursorIn,
|
|
417
|
+
limit,
|
|
418
|
+
model,
|
|
419
|
+
req,
|
|
420
|
+
scopeFilter,
|
|
421
|
+
});
|
|
422
|
+
if (legacyResult) {
|
|
423
|
+
const response = {
|
|
424
|
+
cursor: 0,
|
|
425
|
+
entities: legacyResult.entities,
|
|
426
|
+
frontierSeq,
|
|
427
|
+
hasMore: true,
|
|
428
|
+
legacyCursor: legacyResult.legacyCursor,
|
|
429
|
+
oldestRetainedSeq,
|
|
430
|
+
stream: streamKey,
|
|
431
|
+
};
|
|
432
|
+
return response;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
const seqResult = await pageSeqStratum({
|
|
436
|
+
cursor,
|
|
437
|
+
entry,
|
|
438
|
+
frontierSeq,
|
|
439
|
+
limit,
|
|
440
|
+
model,
|
|
441
|
+
req,
|
|
442
|
+
scopeFilter,
|
|
443
|
+
streamKey,
|
|
444
|
+
});
|
|
445
|
+
const response = {
|
|
446
|
+
cursor: seqResult.cursor,
|
|
447
|
+
entities: seqResult.entities,
|
|
448
|
+
frontierSeq,
|
|
449
|
+
hasMore: seqResult.hasMore,
|
|
450
|
+
oldestRetainedSeq,
|
|
451
|
+
stream: streamKey,
|
|
452
|
+
};
|
|
453
|
+
return response;
|
|
454
|
+
};
|
|
455
|
+
/** Validate the `streams` list of a snapshot batch body: non-empty, capped, no repeats. */
|
|
456
|
+
const parseSnapshotBatchStreams = (raw) => {
|
|
457
|
+
if (!Array.isArray(raw) || raw.length === 0) {
|
|
458
|
+
throw new errors_1.APIError({ status: 400, title: "streams must be a non-empty array" });
|
|
459
|
+
}
|
|
460
|
+
if (raw.length > MAX_SNAPSHOT_BATCH_STREAMS) {
|
|
461
|
+
throw new errors_1.APIError({
|
|
462
|
+
detail: `Requested ${raw.length} streams; the limit is ${MAX_SNAPSHOT_BATCH_STREAMS}`,
|
|
463
|
+
status: 400,
|
|
464
|
+
title: "Too many streams in snapshot batch",
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
const seen = new Set();
|
|
468
|
+
return raw.map((item) => {
|
|
469
|
+
const candidate = item;
|
|
470
|
+
if (!candidate || typeof candidate.stream !== "string" || !candidate.stream) {
|
|
471
|
+
throw new errors_1.APIError({ status: 400, title: "Each snapshot batch entry needs a stream" });
|
|
472
|
+
}
|
|
473
|
+
if (candidate.legacyCursor !== undefined && typeof candidate.legacyCursor !== "string") {
|
|
474
|
+
throw new errors_1.APIError({ status: 400, title: `Invalid legacyCursor for ${candidate.stream}` });
|
|
475
|
+
}
|
|
476
|
+
if (seen.has(candidate.stream)) {
|
|
477
|
+
throw new errors_1.APIError({
|
|
478
|
+
status: 400,
|
|
479
|
+
title: `Duplicate stream in snapshot batch: ${candidate.stream}`,
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
seen.add(candidate.stream);
|
|
483
|
+
return {
|
|
484
|
+
cursor: candidate.cursor,
|
|
485
|
+
legacyCursor: candidate.legacyCursor,
|
|
486
|
+
stream: candidate.stream,
|
|
487
|
+
};
|
|
488
|
+
});
|
|
489
|
+
};
|
|
490
|
+
/** `items.map(map)` with at most `concurrency` calls in flight; results keep item order. */
|
|
491
|
+
const mapWithConcurrency = async ({ items, concurrency, map, }) => {
|
|
492
|
+
const results = new Array(items.length);
|
|
493
|
+
let next = 0;
|
|
494
|
+
const worker = async () => {
|
|
495
|
+
while (next < items.length) {
|
|
496
|
+
const index = next;
|
|
497
|
+
next += 1;
|
|
498
|
+
results[index] = await map(items[index]);
|
|
499
|
+
}
|
|
500
|
+
};
|
|
501
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
|
|
502
|
+
return results;
|
|
503
|
+
};
|
|
334
504
|
/**
|
|
335
505
|
* Mount the SyncDB HTTP routes:
|
|
336
506
|
* - GET /sync/streams — the authoritative set of streams the caller belongs to (C2)
|
|
337
507
|
* - GET /sync/snapshot — per-stream bootstrap/catch-up with server-enforced scoping
|
|
508
|
+
* - POST /sync/snapshot/batch — one page for each of many streams in one round trip
|
|
338
509
|
* - POST /sync/mutate — HTTP fallback mutation channel over applySyncMutation
|
|
339
510
|
* - GET /sync/key — per-user key material for the default encryption KeyProvider
|
|
340
511
|
*/
|
|
@@ -377,119 +548,50 @@ const addSyncRoutes = (app, options = {}) => {
|
|
|
377
548
|
if (!user) {
|
|
378
549
|
throw new errors_1.APIError({ status: 401, title: "Authentication required" });
|
|
379
550
|
}
|
|
380
|
-
const streamKey = String(req.query.stream ?? "");
|
|
381
|
-
if (!streamKey) {
|
|
382
|
-
throw new errors_1.APIError({ status: 400, title: "stream query parameter is required" });
|
|
383
|
-
}
|
|
384
|
-
const parsed = (0, streams_1.parseStreamKey)(streamKey);
|
|
385
|
-
if (!parsed) {
|
|
386
|
-
throw new errors_1.APIError({ status: 400, title: `Invalid stream key: ${streamKey}` });
|
|
387
|
-
}
|
|
388
|
-
const entry = (0, registry_1.findSyncEntryByCollectionTag)(parsed.collectionTag);
|
|
389
|
-
if (!entry) {
|
|
390
|
-
throw new errors_1.APIError({
|
|
391
|
-
status: 404,
|
|
392
|
-
title: `Unknown sync collection: ${parsed.collectionTag}`,
|
|
393
|
-
});
|
|
394
|
-
}
|
|
395
|
-
if (!(await (0, permissions_1.checkPermissions)("list", entry.options.permissions.list, user))) {
|
|
396
|
-
throw new errors_1.APIError({
|
|
397
|
-
status: 403,
|
|
398
|
-
title: `Access to sync snapshot for ${parsed.collectionTag} denied for ${user.id}`,
|
|
399
|
-
});
|
|
400
|
-
}
|
|
401
|
-
// C2: a client must not snapshot a stream it does not belong to.
|
|
402
|
-
const memberStreams = await (0, streams_1.resolveUserStreamsForEntry)({
|
|
403
|
-
entry,
|
|
404
|
-
getUserScopes: options.getUserScopes,
|
|
405
|
-
user,
|
|
406
|
-
});
|
|
407
|
-
if (!memberStreams.includes(streamKey)) {
|
|
408
|
-
throw new errors_1.APIError({
|
|
409
|
-
status: 403,
|
|
410
|
-
title: `User ${user.id} does not belong to stream ${streamKey}`,
|
|
411
|
-
});
|
|
412
|
-
}
|
|
413
551
|
const cursor = parseNonNegativeInt(req.query.cursor, "cursor", 0);
|
|
414
552
|
const requestedLimit = parseNonNegativeInt(req.query.limit, "limit", options.defaultSnapshotLimit ?? DEFAULT_SNAPSHOT_LIMIT);
|
|
415
553
|
const limit = Math.min(Math.max(requestedLimit, 1), MAX_SNAPSHOT_LIMIT);
|
|
416
554
|
const legacyCursorIn = typeof req.query.legacyCursor === "string" && req.query.legacyCursor.length > 0
|
|
417
555
|
? req.query.legacyCursor
|
|
418
556
|
: undefined;
|
|
419
|
-
const
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
scopeValue: parsed.scopeValue,
|
|
428
|
-
snapshotFilterResult,
|
|
429
|
-
}),
|
|
557
|
+
const response = await snapshotStreamPage({
|
|
558
|
+
cursor,
|
|
559
|
+
legacyCursorIn,
|
|
560
|
+
limit,
|
|
561
|
+
options,
|
|
562
|
+
req,
|
|
563
|
+
streamKey: String(req.query.stream ?? ""),
|
|
564
|
+
user,
|
|
430
565
|
});
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
if (queryFilterResult.denied) {
|
|
442
|
-
const deniedResponse = {
|
|
443
|
-
cursor: frontierSeq,
|
|
444
|
-
entities: [],
|
|
445
|
-
frontierSeq,
|
|
446
|
-
hasMore: false,
|
|
447
|
-
oldestRetainedSeq,
|
|
448
|
-
stream: streamKey,
|
|
449
|
-
};
|
|
450
|
-
return res.json(deniedResponse);
|
|
566
|
+
return res.json(response);
|
|
567
|
+
}));
|
|
568
|
+
// Page many streams in one round trip: one page per requested stream, in request order.
|
|
569
|
+
// `limit` is the entity budget for the whole response, split evenly across the streams,
|
|
570
|
+
// so a request stays bounded however many streams it names; streams with more pages
|
|
571
|
+
// are asked for again (with a larger share) in the client's next round.
|
|
572
|
+
router.post("/sync/snapshot/batch", (0, auth_1.authenticateMiddleware)(), (0, api_1.asyncHandler)(async (req, res) => {
|
|
573
|
+
const user = req.user;
|
|
574
|
+
if (!user) {
|
|
575
|
+
throw new errors_1.APIError({ status: 401, title: "Authentication required" });
|
|
451
576
|
}
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
577
|
+
const body = req.body;
|
|
578
|
+
const items = parseSnapshotBatchStreams(body?.streams);
|
|
579
|
+
const budget = Math.min(Math.max(parseNonNegativeInt(body?.limit, "limit", DEFAULT_SNAPSHOT_BATCH_LIMIT), items.length), MAX_SNAPSHOT_BATCH_LIMIT);
|
|
580
|
+
const limit = Math.max(1, Math.floor(budget / items.length));
|
|
581
|
+
const pages = await mapWithConcurrency({
|
|
582
|
+
concurrency: SNAPSHOT_BATCH_CONCURRENCY,
|
|
583
|
+
items,
|
|
584
|
+
map: (item) => snapshotStreamPage({
|
|
585
|
+
cursor: parseNonNegativeInt(item.cursor, "cursor", 0),
|
|
586
|
+
legacyCursorIn: item.legacyCursor || undefined,
|
|
457
587
|
limit,
|
|
458
|
-
|
|
588
|
+
options,
|
|
459
589
|
req,
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
const response = {
|
|
464
|
-
cursor: 0,
|
|
465
|
-
entities: legacyResult.entities,
|
|
466
|
-
frontierSeq,
|
|
467
|
-
hasMore: true,
|
|
468
|
-
legacyCursor: legacyResult.legacyCursor,
|
|
469
|
-
oldestRetainedSeq,
|
|
470
|
-
stream: streamKey,
|
|
471
|
-
};
|
|
472
|
-
return res.json(response);
|
|
473
|
-
}
|
|
474
|
-
}
|
|
475
|
-
const seqResult = await pageSeqStratum({
|
|
476
|
-
cursor,
|
|
477
|
-
entry,
|
|
478
|
-
frontierSeq,
|
|
479
|
-
limit,
|
|
480
|
-
model,
|
|
481
|
-
req,
|
|
482
|
-
scopeFilter,
|
|
483
|
-
streamKey,
|
|
590
|
+
streamKey: item.stream,
|
|
591
|
+
user,
|
|
592
|
+
}),
|
|
484
593
|
});
|
|
485
|
-
const response = {
|
|
486
|
-
cursor: seqResult.cursor,
|
|
487
|
-
entities: seqResult.entities,
|
|
488
|
-
frontierSeq,
|
|
489
|
-
hasMore: seqResult.hasMore,
|
|
490
|
-
oldestRetainedSeq,
|
|
491
|
-
stream: streamKey,
|
|
492
|
-
};
|
|
594
|
+
const response = { pages };
|
|
493
595
|
return res.json(response);
|
|
494
596
|
}));
|
|
495
597
|
router.get("/sync/entities", (0, auth_1.authenticateMiddleware)(), (0, api_1.asyncHandler)(async (req, res) => {
|
package/dist/sync/syncApp.d.ts
CHANGED
|
@@ -5,8 +5,9 @@ import { type SyncAppOptions } from "./routes";
|
|
|
5
5
|
export declare const getSyncAppOptions: (app: express.Application) => SyncAppOptions | undefined;
|
|
6
6
|
/**
|
|
7
7
|
* TerrenoPlugin mounting the SyncDB local-first sync HTTP routes
|
|
8
|
-
* (`GET /sync/snapshot`, `POST /sync/mutate`, `GET /sync/key`).
|
|
9
|
-
* modelRouter's `sync` option; this plugin serves the registered
|
|
8
|
+
* (`GET /sync/snapshot`, `POST /sync/snapshot/batch`, `POST /sync/mutate`, `GET /sync/key`).
|
|
9
|
+
* Models opt in via modelRouter's `sync` option; this plugin serves the registered
|
|
10
|
+
* collections.
|
|
10
11
|
*
|
|
11
12
|
* Registration attaches the plugin's options (notably `getUserScopes`) to this Express
|
|
12
13
|
* application so RealtimeApp's connection handler can install the socket
|
package/dist/sync/syncApp.js
CHANGED
|
@@ -10,8 +10,9 @@ const getSyncAppOptions = (app) => app.locals[SYNC_APP_OPTIONS_LOCAL_KEY];
|
|
|
10
10
|
exports.getSyncAppOptions = getSyncAppOptions;
|
|
11
11
|
/**
|
|
12
12
|
* TerrenoPlugin mounting the SyncDB local-first sync HTTP routes
|
|
13
|
-
* (`GET /sync/snapshot`, `POST /sync/mutate`, `GET /sync/key`).
|
|
14
|
-
* modelRouter's `sync` option; this plugin serves the registered
|
|
13
|
+
* (`GET /sync/snapshot`, `POST /sync/snapshot/batch`, `POST /sync/mutate`, `GET /sync/key`).
|
|
14
|
+
* Models opt in via modelRouter's `sync` option; this plugin serves the registered
|
|
15
|
+
* collections.
|
|
15
16
|
*
|
|
16
17
|
* Registration attaches the plugin's options (notably `getUserScopes`) to this Express
|
|
17
18
|
* application so RealtimeApp's connection handler can install the socket
|
package/dist/sync/types.d.ts
CHANGED
|
@@ -108,6 +108,24 @@ export interface SyncSnapshotResponse {
|
|
|
108
108
|
*/
|
|
109
109
|
legacyCursor?: string;
|
|
110
110
|
}
|
|
111
|
+
/** One stream's page request inside `POST /sync/snapshot/batch`. */
|
|
112
|
+
export interface SyncSnapshotBatchStreamRequest {
|
|
113
|
+
stream: string;
|
|
114
|
+
/** Resume cursor (highest seq already applied); 0 = from the beginning. */
|
|
115
|
+
cursor?: number;
|
|
116
|
+
/** C3: legacy-stratum token echoed back from this stream's previous page. */
|
|
117
|
+
legacyCursor?: string;
|
|
118
|
+
}
|
|
119
|
+
/** Body of `POST /sync/snapshot/batch`. */
|
|
120
|
+
export interface SyncSnapshotBatchRequest {
|
|
121
|
+
streams: SyncSnapshotBatchStreamRequest[];
|
|
122
|
+
/** Entity budget for the whole response, split evenly across `streams`. */
|
|
123
|
+
limit?: number;
|
|
124
|
+
}
|
|
125
|
+
/** Response of `POST /sync/snapshot/batch`: one page per requested stream, in request order. */
|
|
126
|
+
export interface SyncSnapshotBatchResponse {
|
|
127
|
+
pages: SyncSnapshotResponse[];
|
|
128
|
+
}
|
|
111
129
|
/** Response shape for `GET /sync/entities` (point lookup for repair). */
|
|
112
130
|
export interface SyncEntitiesResponse {
|
|
113
131
|
entities: SyncEntityPayload[];
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"@modelcontextprotocol/node": "^2.0.0",
|
|
12
12
|
"@modelcontextprotocol/server": "^2.0.0",
|
|
13
13
|
"@sentry/bun": "^10.39.0",
|
|
14
|
-
"@terreno/test": "57.
|
|
14
|
+
"@terreno/test": "57.16.1",
|
|
15
15
|
"@thream/socketio-jwt": "^3.1.4",
|
|
16
16
|
"@types/qs": "^6.14.0",
|
|
17
17
|
"ajv": "8.20.0",
|
|
@@ -48,7 +48,7 @@
|
|
|
48
48
|
"devDependencies": {
|
|
49
49
|
"@biomejs/biome": "^2.3.6",
|
|
50
50
|
"@modelcontextprotocol/client": "^2.0.0",
|
|
51
|
-
"@terreno/syncdb": "57.
|
|
51
|
+
"@terreno/syncdb": "57.16.1",
|
|
52
52
|
"@types/bun": "^1.4.0",
|
|
53
53
|
"@types/cors": "^2.8.17",
|
|
54
54
|
"@types/express": "^5.0.6",
|
|
@@ -132,5 +132,5 @@
|
|
|
132
132
|
"updateSnapshot": "bun test --update-snapshots"
|
|
133
133
|
},
|
|
134
134
|
"types": "dist/index.d.ts",
|
|
135
|
-
"version": "57.
|
|
135
|
+
"version": "57.16.1"
|
|
136
136
|
}
|
package/src/sync/routes.ts
CHANGED
|
@@ -36,6 +36,9 @@ import type {
|
|
|
36
36
|
SyncMutateBatchRequest,
|
|
37
37
|
SyncMutateRequest,
|
|
38
38
|
SyncNackCode,
|
|
39
|
+
SyncSnapshotBatchRequest,
|
|
40
|
+
SyncSnapshotBatchResponse,
|
|
41
|
+
SyncSnapshotBatchStreamRequest,
|
|
39
42
|
SyncSnapshotResponse,
|
|
40
43
|
SyncStreamInfo,
|
|
41
44
|
SyncStreamsResponse,
|
|
@@ -77,6 +80,13 @@ export interface SyncAppOptions {
|
|
|
77
80
|
|
|
78
81
|
const MAX_SNAPSHOT_LIMIT = 100;
|
|
79
82
|
const DEFAULT_SNAPSHOT_LIMIT = 100;
|
|
83
|
+
/** Entity budget of one `POST /sync/snapshot/batch` response, shared by its streams. */
|
|
84
|
+
const DEFAULT_SNAPSHOT_BATCH_LIMIT = 5_000;
|
|
85
|
+
const MAX_SNAPSHOT_BATCH_LIMIT = 10_000;
|
|
86
|
+
/** Most streams one batch request may name. */
|
|
87
|
+
const MAX_SNAPSHOT_BATCH_STREAMS = 500;
|
|
88
|
+
/** Streams paged at once per batch request, so one request cannot drain the Mongo pool. */
|
|
89
|
+
const SNAPSHOT_BATCH_CONCURRENCY = 8;
|
|
80
90
|
|
|
81
91
|
/** Minimal shape of a synced document as read by the snapshot/entities routes. */
|
|
82
92
|
interface SyncedDoc {
|
|
@@ -486,10 +496,205 @@ const castEntityIds = ({
|
|
|
486
496
|
return ids.map((id) => new mongoose.Types.ObjectId(id));
|
|
487
497
|
};
|
|
488
498
|
|
|
499
|
+
/**
|
|
500
|
+
* Serve one snapshot page for one stream: membership/permission checks, frontier and
|
|
501
|
+
* retention signals, then the legacy or seq stratum. Shared by `GET /sync/snapshot` and
|
|
502
|
+
* `POST /sync/snapshot/batch`.
|
|
503
|
+
*/
|
|
504
|
+
const snapshotStreamPage = async ({
|
|
505
|
+
req,
|
|
506
|
+
user,
|
|
507
|
+
streamKey,
|
|
508
|
+
cursor,
|
|
509
|
+
limit,
|
|
510
|
+
legacyCursorIn,
|
|
511
|
+
options,
|
|
512
|
+
}: {
|
|
513
|
+
req: express.Request;
|
|
514
|
+
user: User;
|
|
515
|
+
streamKey: string;
|
|
516
|
+
cursor: number;
|
|
517
|
+
limit: number;
|
|
518
|
+
legacyCursorIn?: string;
|
|
519
|
+
options: SyncAppOptions;
|
|
520
|
+
}): Promise<SyncSnapshotResponse> => {
|
|
521
|
+
if (!streamKey) {
|
|
522
|
+
throw new APIError({status: 400, title: "stream query parameter is required"});
|
|
523
|
+
}
|
|
524
|
+
const parsed = parseStreamKey(streamKey);
|
|
525
|
+
if (!parsed) {
|
|
526
|
+
throw new APIError({status: 400, title: `Invalid stream key: ${streamKey}`});
|
|
527
|
+
}
|
|
528
|
+
const entry = findSyncEntryByCollectionTag(parsed.collectionTag);
|
|
529
|
+
if (!entry) {
|
|
530
|
+
throw new APIError({
|
|
531
|
+
status: 404,
|
|
532
|
+
title: `Unknown sync collection: ${parsed.collectionTag}`,
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
if (!(await checkPermissions("list", entry.options.permissions.list, user))) {
|
|
536
|
+
throw new APIError({
|
|
537
|
+
status: 403,
|
|
538
|
+
title: `Access to sync snapshot for ${parsed.collectionTag} denied for ${user.id}`,
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
// C2: a client must not snapshot a stream it does not belong to.
|
|
542
|
+
const memberStreams = await resolveUserStreamsForEntry({
|
|
543
|
+
entry,
|
|
544
|
+
getUserScopes: options.getUserScopes,
|
|
545
|
+
user,
|
|
546
|
+
});
|
|
547
|
+
if (!memberStreams.includes(streamKey)) {
|
|
548
|
+
throw new APIError({
|
|
549
|
+
status: 403,
|
|
550
|
+
title: `User ${user.id} does not belong to stream ${streamKey}`,
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
const snapshotFilterResult = entry.config.snapshotFilter
|
|
555
|
+
? await entry.config.snapshotFilter({id: String(user.id)})
|
|
556
|
+
: undefined;
|
|
557
|
+
const queryFilterResult = await resolveSyncQueryFilter({entry, user});
|
|
558
|
+
const scopeFilter = composeSyncReadFilter({
|
|
559
|
+
queryFilter: queryFilterResult.filter,
|
|
560
|
+
scopeFilter: buildSnapshotScopeFilter({
|
|
561
|
+
entry,
|
|
562
|
+
scopeValue: parsed.scopeValue,
|
|
563
|
+
snapshotFilterResult,
|
|
564
|
+
}),
|
|
565
|
+
});
|
|
566
|
+
const model = getSyncedModel(entry.modelName);
|
|
567
|
+
const frontierSeq = await computeStableFrontier({stream: streamKey});
|
|
568
|
+
// C7 (Task 9.15): the retention signal is the durable compaction watermark, not
|
|
569
|
+
// min(retained seq). The old computation was pinned low forever by any early doc that
|
|
570
|
+
// was never deleted, so a stale cursor sitting above a compacted tombstone's seq
|
|
571
|
+
// passed the check and silently never learned about that deletion.
|
|
572
|
+
const oldestRetainedSeq = await getCompactedThroughSeq({stream: streamKey});
|
|
573
|
+
|
|
574
|
+
// Task 9.19: a `queryFilter` that denies this caller means "no visible rows", exactly
|
|
575
|
+
// as REST's list endpoint answers. Serve a terminal empty page (cursor at the
|
|
576
|
+
// frontier, hasMore false) so the client settles instead of paging forever.
|
|
577
|
+
if (queryFilterResult.denied) {
|
|
578
|
+
const deniedResponse: SyncSnapshotResponse = {
|
|
579
|
+
cursor: frontierSeq,
|
|
580
|
+
entities: [],
|
|
581
|
+
frontierSeq,
|
|
582
|
+
hasMore: false,
|
|
583
|
+
oldestRetainedSeq,
|
|
584
|
+
stream: streamKey,
|
|
585
|
+
};
|
|
586
|
+
return deniedResponse;
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
// C3: legacy (seq-0) stratum, paged by _id. Drained fully before seq paging begins.
|
|
590
|
+
if (cursor === 0) {
|
|
591
|
+
const legacyResult = await pageLegacyStratum({
|
|
592
|
+
entry,
|
|
593
|
+
legacyCursorIn,
|
|
594
|
+
limit,
|
|
595
|
+
model,
|
|
596
|
+
req,
|
|
597
|
+
scopeFilter,
|
|
598
|
+
});
|
|
599
|
+
if (legacyResult) {
|
|
600
|
+
const response: SyncSnapshotResponse = {
|
|
601
|
+
cursor: 0,
|
|
602
|
+
entities: legacyResult.entities,
|
|
603
|
+
frontierSeq,
|
|
604
|
+
hasMore: true,
|
|
605
|
+
legacyCursor: legacyResult.legacyCursor,
|
|
606
|
+
oldestRetainedSeq,
|
|
607
|
+
stream: streamKey,
|
|
608
|
+
};
|
|
609
|
+
return response;
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
const seqResult = await pageSeqStratum({
|
|
614
|
+
cursor,
|
|
615
|
+
entry,
|
|
616
|
+
frontierSeq,
|
|
617
|
+
limit,
|
|
618
|
+
model,
|
|
619
|
+
req,
|
|
620
|
+
scopeFilter,
|
|
621
|
+
streamKey,
|
|
622
|
+
});
|
|
623
|
+
const response: SyncSnapshotResponse = {
|
|
624
|
+
cursor: seqResult.cursor,
|
|
625
|
+
entities: seqResult.entities,
|
|
626
|
+
frontierSeq,
|
|
627
|
+
hasMore: seqResult.hasMore,
|
|
628
|
+
oldestRetainedSeq,
|
|
629
|
+
stream: streamKey,
|
|
630
|
+
};
|
|
631
|
+
return response;
|
|
632
|
+
};
|
|
633
|
+
|
|
634
|
+
/** Validate the `streams` list of a snapshot batch body: non-empty, capped, no repeats. */
|
|
635
|
+
const parseSnapshotBatchStreams = (raw: unknown): SyncSnapshotBatchStreamRequest[] => {
|
|
636
|
+
if (!Array.isArray(raw) || raw.length === 0) {
|
|
637
|
+
throw new APIError({status: 400, title: "streams must be a non-empty array"});
|
|
638
|
+
}
|
|
639
|
+
if (raw.length > MAX_SNAPSHOT_BATCH_STREAMS) {
|
|
640
|
+
throw new APIError({
|
|
641
|
+
detail: `Requested ${raw.length} streams; the limit is ${MAX_SNAPSHOT_BATCH_STREAMS}`,
|
|
642
|
+
status: 400,
|
|
643
|
+
title: "Too many streams in snapshot batch",
|
|
644
|
+
});
|
|
645
|
+
}
|
|
646
|
+
const seen = new Set<string>();
|
|
647
|
+
return raw.map((item: unknown) => {
|
|
648
|
+
const candidate = item as Partial<SyncSnapshotBatchStreamRequest> | null;
|
|
649
|
+
if (!candidate || typeof candidate.stream !== "string" || !candidate.stream) {
|
|
650
|
+
throw new APIError({status: 400, title: "Each snapshot batch entry needs a stream"});
|
|
651
|
+
}
|
|
652
|
+
if (candidate.legacyCursor !== undefined && typeof candidate.legacyCursor !== "string") {
|
|
653
|
+
throw new APIError({status: 400, title: `Invalid legacyCursor for ${candidate.stream}`});
|
|
654
|
+
}
|
|
655
|
+
if (seen.has(candidate.stream)) {
|
|
656
|
+
throw new APIError({
|
|
657
|
+
status: 400,
|
|
658
|
+
title: `Duplicate stream in snapshot batch: ${candidate.stream}`,
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
seen.add(candidate.stream);
|
|
662
|
+
return {
|
|
663
|
+
cursor: candidate.cursor,
|
|
664
|
+
legacyCursor: candidate.legacyCursor,
|
|
665
|
+
stream: candidate.stream,
|
|
666
|
+
};
|
|
667
|
+
});
|
|
668
|
+
};
|
|
669
|
+
|
|
670
|
+
/** `items.map(map)` with at most `concurrency` calls in flight; results keep item order. */
|
|
671
|
+
const mapWithConcurrency = async <T, R>({
|
|
672
|
+
items,
|
|
673
|
+
concurrency,
|
|
674
|
+
map,
|
|
675
|
+
}: {
|
|
676
|
+
items: T[];
|
|
677
|
+
concurrency: number;
|
|
678
|
+
map: (item: T) => Promise<R>;
|
|
679
|
+
}): Promise<R[]> => {
|
|
680
|
+
const results = new Array<R>(items.length);
|
|
681
|
+
let next = 0;
|
|
682
|
+
const worker = async (): Promise<void> => {
|
|
683
|
+
while (next < items.length) {
|
|
684
|
+
const index = next;
|
|
685
|
+
next += 1;
|
|
686
|
+
results[index] = await map(items[index] as T);
|
|
687
|
+
}
|
|
688
|
+
};
|
|
689
|
+
await Promise.all(Array.from({length: Math.min(concurrency, items.length)}, worker));
|
|
690
|
+
return results;
|
|
691
|
+
};
|
|
692
|
+
|
|
489
693
|
/**
|
|
490
694
|
* Mount the SyncDB HTTP routes:
|
|
491
695
|
* - GET /sync/streams — the authoritative set of streams the caller belongs to (C2)
|
|
492
696
|
* - GET /sync/snapshot — per-stream bootstrap/catch-up with server-enforced scoping
|
|
697
|
+
* - POST /sync/snapshot/batch — one page for each of many streams in one round trip
|
|
493
698
|
* - POST /sync/mutate — HTTP fallback mutation channel over applySyncMutation
|
|
494
699
|
* - GET /sync/key — per-user key material for the default encryption KeyProvider
|
|
495
700
|
*/
|
|
@@ -541,40 +746,6 @@ export const addSyncRoutes = (app: express.Application, options: SyncAppOptions
|
|
|
541
746
|
if (!user) {
|
|
542
747
|
throw new APIError({status: 401, title: "Authentication required"});
|
|
543
748
|
}
|
|
544
|
-
const streamKey = String(req.query.stream ?? "");
|
|
545
|
-
if (!streamKey) {
|
|
546
|
-
throw new APIError({status: 400, title: "stream query parameter is required"});
|
|
547
|
-
}
|
|
548
|
-
const parsed = parseStreamKey(streamKey);
|
|
549
|
-
if (!parsed) {
|
|
550
|
-
throw new APIError({status: 400, title: `Invalid stream key: ${streamKey}`});
|
|
551
|
-
}
|
|
552
|
-
const entry = findSyncEntryByCollectionTag(parsed.collectionTag);
|
|
553
|
-
if (!entry) {
|
|
554
|
-
throw new APIError({
|
|
555
|
-
status: 404,
|
|
556
|
-
title: `Unknown sync collection: ${parsed.collectionTag}`,
|
|
557
|
-
});
|
|
558
|
-
}
|
|
559
|
-
if (!(await checkPermissions("list", entry.options.permissions.list, user))) {
|
|
560
|
-
throw new APIError({
|
|
561
|
-
status: 403,
|
|
562
|
-
title: `Access to sync snapshot for ${parsed.collectionTag} denied for ${user.id}`,
|
|
563
|
-
});
|
|
564
|
-
}
|
|
565
|
-
// C2: a client must not snapshot a stream it does not belong to.
|
|
566
|
-
const memberStreams = await resolveUserStreamsForEntry({
|
|
567
|
-
entry,
|
|
568
|
-
getUserScopes: options.getUserScopes,
|
|
569
|
-
user,
|
|
570
|
-
});
|
|
571
|
-
if (!memberStreams.includes(streamKey)) {
|
|
572
|
-
throw new APIError({
|
|
573
|
-
status: 403,
|
|
574
|
-
title: `User ${user.id} does not belong to stream ${streamKey}`,
|
|
575
|
-
});
|
|
576
|
-
}
|
|
577
|
-
|
|
578
749
|
const cursor = parseNonNegativeInt(req.query.cursor, "cursor", 0);
|
|
579
750
|
const requestedLimit = parseNonNegativeInt(
|
|
580
751
|
req.query.limit,
|
|
@@ -586,84 +757,56 @@ export const addSyncRoutes = (app: express.Application, options: SyncAppOptions
|
|
|
586
757
|
typeof req.query.legacyCursor === "string" && req.query.legacyCursor.length > 0
|
|
587
758
|
? req.query.legacyCursor
|
|
588
759
|
: undefined;
|
|
589
|
-
|
|
590
|
-
const snapshotFilterResult = entry.config.snapshotFilter
|
|
591
|
-
? await entry.config.snapshotFilter({id: String(user.id)})
|
|
592
|
-
: undefined;
|
|
593
|
-
const queryFilterResult = await resolveSyncQueryFilter({entry, user});
|
|
594
|
-
const scopeFilter = composeSyncReadFilter({
|
|
595
|
-
queryFilter: queryFilterResult.filter,
|
|
596
|
-
scopeFilter: buildSnapshotScopeFilter({
|
|
597
|
-
entry,
|
|
598
|
-
scopeValue: parsed.scopeValue,
|
|
599
|
-
snapshotFilterResult,
|
|
600
|
-
}),
|
|
601
|
-
});
|
|
602
|
-
const model = getSyncedModel(entry.modelName);
|
|
603
|
-
const frontierSeq = await computeStableFrontier({stream: streamKey});
|
|
604
|
-
// C7 (Task 9.15): the retention signal is the durable compaction watermark, not
|
|
605
|
-
// min(retained seq). The old computation was pinned low forever by any early doc that
|
|
606
|
-
// was never deleted, so a stale cursor sitting above a compacted tombstone's seq
|
|
607
|
-
// passed the check and silently never learned about that deletion.
|
|
608
|
-
const oldestRetainedSeq = await getCompactedThroughSeq({stream: streamKey});
|
|
609
|
-
|
|
610
|
-
// Task 9.19: a `queryFilter` that denies this caller means "no visible rows", exactly
|
|
611
|
-
// as REST's list endpoint answers. Serve a terminal empty page (cursor at the
|
|
612
|
-
// frontier, hasMore false) so the client settles instead of paging forever.
|
|
613
|
-
if (queryFilterResult.denied) {
|
|
614
|
-
const deniedResponse: SyncSnapshotResponse = {
|
|
615
|
-
cursor: frontierSeq,
|
|
616
|
-
entities: [],
|
|
617
|
-
frontierSeq,
|
|
618
|
-
hasMore: false,
|
|
619
|
-
oldestRetainedSeq,
|
|
620
|
-
stream: streamKey,
|
|
621
|
-
};
|
|
622
|
-
return res.json(deniedResponse);
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
// C3: legacy (seq-0) stratum, paged by _id. Drained fully before seq paging begins.
|
|
626
|
-
if (cursor === 0) {
|
|
627
|
-
const legacyResult = await pageLegacyStratum({
|
|
628
|
-
entry,
|
|
629
|
-
legacyCursorIn,
|
|
630
|
-
limit,
|
|
631
|
-
model,
|
|
632
|
-
req,
|
|
633
|
-
scopeFilter,
|
|
634
|
-
});
|
|
635
|
-
if (legacyResult) {
|
|
636
|
-
const response: SyncSnapshotResponse = {
|
|
637
|
-
cursor: 0,
|
|
638
|
-
entities: legacyResult.entities,
|
|
639
|
-
frontierSeq,
|
|
640
|
-
hasMore: true,
|
|
641
|
-
legacyCursor: legacyResult.legacyCursor,
|
|
642
|
-
oldestRetainedSeq,
|
|
643
|
-
stream: streamKey,
|
|
644
|
-
};
|
|
645
|
-
return res.json(response);
|
|
646
|
-
}
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
const seqResult = await pageSeqStratum({
|
|
760
|
+
const response = await snapshotStreamPage({
|
|
650
761
|
cursor,
|
|
651
|
-
|
|
652
|
-
frontierSeq,
|
|
762
|
+
legacyCursorIn,
|
|
653
763
|
limit,
|
|
654
|
-
|
|
764
|
+
options,
|
|
655
765
|
req,
|
|
656
|
-
|
|
657
|
-
|
|
766
|
+
streamKey: String(req.query.stream ?? ""),
|
|
767
|
+
user,
|
|
658
768
|
});
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
769
|
+
return res.json(response);
|
|
770
|
+
})
|
|
771
|
+
);
|
|
772
|
+
|
|
773
|
+
// Page many streams in one round trip: one page per requested stream, in request order.
|
|
774
|
+
// `limit` is the entity budget for the whole response, split evenly across the streams,
|
|
775
|
+
// so a request stays bounded however many streams it names; streams with more pages
|
|
776
|
+
// are asked for again (with a larger share) in the client's next round.
|
|
777
|
+
router.post(
|
|
778
|
+
"/sync/snapshot/batch",
|
|
779
|
+
authenticateMiddleware(),
|
|
780
|
+
asyncHandler(async (req, res) => {
|
|
781
|
+
const user = req.user as User | undefined;
|
|
782
|
+
if (!user) {
|
|
783
|
+
throw new APIError({status: 401, title: "Authentication required"});
|
|
784
|
+
}
|
|
785
|
+
const body = req.body as Partial<SyncSnapshotBatchRequest> | undefined;
|
|
786
|
+
const items = parseSnapshotBatchStreams(body?.streams);
|
|
787
|
+
const budget = Math.min(
|
|
788
|
+
Math.max(
|
|
789
|
+
parseNonNegativeInt(body?.limit, "limit", DEFAULT_SNAPSHOT_BATCH_LIMIT),
|
|
790
|
+
items.length
|
|
791
|
+
),
|
|
792
|
+
MAX_SNAPSHOT_BATCH_LIMIT
|
|
793
|
+
);
|
|
794
|
+
const limit = Math.max(1, Math.floor(budget / items.length));
|
|
795
|
+
const pages = await mapWithConcurrency({
|
|
796
|
+
concurrency: SNAPSHOT_BATCH_CONCURRENCY,
|
|
797
|
+
items,
|
|
798
|
+
map: (item) =>
|
|
799
|
+
snapshotStreamPage({
|
|
800
|
+
cursor: parseNonNegativeInt(item.cursor, "cursor", 0),
|
|
801
|
+
legacyCursorIn: item.legacyCursor || undefined,
|
|
802
|
+
limit,
|
|
803
|
+
options,
|
|
804
|
+
req,
|
|
805
|
+
streamKey: item.stream,
|
|
806
|
+
user,
|
|
807
|
+
}),
|
|
808
|
+
});
|
|
809
|
+
const response: SyncSnapshotBatchResponse = {pages};
|
|
667
810
|
return res.json(response);
|
|
668
811
|
})
|
|
669
812
|
);
|
package/src/sync/syncApp.ts
CHANGED
|
@@ -12,8 +12,9 @@ export const getSyncAppOptions = (app: express.Application): SyncAppOptions | un
|
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* TerrenoPlugin mounting the SyncDB local-first sync HTTP routes
|
|
15
|
-
* (`GET /sync/snapshot`, `POST /sync/mutate`, `GET /sync/key`).
|
|
16
|
-
* modelRouter's `sync` option; this plugin serves the registered
|
|
15
|
+
* (`GET /sync/snapshot`, `POST /sync/snapshot/batch`, `POST /sync/mutate`, `GET /sync/key`).
|
|
16
|
+
* Models opt in via modelRouter's `sync` option; this plugin serves the registered
|
|
17
|
+
* collections.
|
|
17
18
|
*
|
|
18
19
|
* Registration attaches the plugin's options (notably `getUserScopes`) to this Express
|
|
19
20
|
* application so RealtimeApp's connection handler can install the socket
|
|
@@ -403,6 +403,114 @@ describe("sync routes", () => {
|
|
|
403
403
|
});
|
|
404
404
|
});
|
|
405
405
|
|
|
406
|
+
describe("POST /sync/snapshot/batch", () => {
|
|
407
|
+
const ownerStream = (): string => `routeStuff|owner:${notAdminId}`;
|
|
408
|
+
const tenantStream = "routeProjects|tenant:org1";
|
|
409
|
+
|
|
410
|
+
it("requires authentication", async () => {
|
|
411
|
+
await server
|
|
412
|
+
.post("/sync/snapshot/batch")
|
|
413
|
+
.send({streams: [{stream: "routeStuff|owner:x"}]})
|
|
414
|
+
.expect(401);
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
it("returns one scoped page per stream, in request order", async () => {
|
|
418
|
+
await RouteStuffModel.create({name: "mine", ownerId: notAdminId});
|
|
419
|
+
await RouteStuffModel.create({name: "theirs", ownerId: "someoneElse"});
|
|
420
|
+
await RouteProjectModel.create({orgId: "org1", title: "visible"});
|
|
421
|
+
await RouteProjectModel.create({orgId: "org2", title: "hidden"});
|
|
422
|
+
|
|
423
|
+
const res = await agent
|
|
424
|
+
.post("/sync/snapshot/batch")
|
|
425
|
+
.send({streams: [{stream: tenantStream}, {cursor: 0, stream: ownerStream()}]})
|
|
426
|
+
.expect(200);
|
|
427
|
+
|
|
428
|
+
expect(res.body.pages.map((page: {stream: string}) => page.stream)).toEqual([
|
|
429
|
+
tenantStream,
|
|
430
|
+
ownerStream(),
|
|
431
|
+
]);
|
|
432
|
+
expect(res.body.pages[0].entities.map((e: SnapshotEntity) => e.data.title)).toEqual([
|
|
433
|
+
"visible",
|
|
434
|
+
]);
|
|
435
|
+
expect(res.body.pages[1].entities.map((e: SnapshotEntity) => e.data.name)).toEqual(["mine"]);
|
|
436
|
+
expect(res.body.pages[1].hasMore).toBe(false);
|
|
437
|
+
expect(res.body.pages[1].cursor).toBe(res.body.pages[1].frontierSeq);
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
it("splits the entity budget across streams and resumes from each stream's cursor", async () => {
|
|
441
|
+
for (let i = 1; i <= 5; i++) {
|
|
442
|
+
await RouteStuffModel.create({name: `item ${i}`, ownerId: notAdminId});
|
|
443
|
+
}
|
|
444
|
+
await RouteProjectModel.create({orgId: "org1", title: "project"});
|
|
445
|
+
|
|
446
|
+
const first = await agent
|
|
447
|
+
.post("/sync/snapshot/batch")
|
|
448
|
+
.send({limit: 4, streams: [{stream: ownerStream()}, {stream: tenantStream}]})
|
|
449
|
+
.expect(200);
|
|
450
|
+
const [ownerPage, tenantPage] = first.body.pages;
|
|
451
|
+
expect(ownerPage.entities).toHaveLength(2);
|
|
452
|
+
expect(ownerPage.hasMore).toBe(true);
|
|
453
|
+
expect(tenantPage.entities).toHaveLength(1);
|
|
454
|
+
expect(tenantPage.hasMore).toBe(false);
|
|
455
|
+
|
|
456
|
+
const second = await agent
|
|
457
|
+
.post("/sync/snapshot/batch")
|
|
458
|
+
.send({limit: 4, streams: [{cursor: ownerPage.cursor, stream: ownerStream()}]})
|
|
459
|
+
.expect(200);
|
|
460
|
+
expect(second.body.pages[0].entities.map((e: SnapshotEntity) => e.data.name)).toEqual([
|
|
461
|
+
"item 3",
|
|
462
|
+
"item 4",
|
|
463
|
+
"item 5",
|
|
464
|
+
]);
|
|
465
|
+
expect(second.body.pages[0].hasMore).toBe(false);
|
|
466
|
+
});
|
|
467
|
+
|
|
468
|
+
it("serves the same page as GET /sync/snapshot for the same stream and cursor", async () => {
|
|
469
|
+
for (let i = 1; i <= 3; i++) {
|
|
470
|
+
await RouteStuffModel.create({name: `item ${i}`, ownerId: notAdminId});
|
|
471
|
+
}
|
|
472
|
+
const single = await agent
|
|
473
|
+
.get(`/sync/snapshot?stream=${encodeURIComponent(ownerStream())}&limit=2&cursor=1`)
|
|
474
|
+
.expect(200);
|
|
475
|
+
const batch = await agent
|
|
476
|
+
.post("/sync/snapshot/batch")
|
|
477
|
+
.send({limit: 2, streams: [{cursor: 1, stream: ownerStream()}]})
|
|
478
|
+
.expect(200);
|
|
479
|
+
expect(batch.body.pages[0]).toEqual(single.body);
|
|
480
|
+
});
|
|
481
|
+
|
|
482
|
+
it("rejects the whole request when any stream is not the caller's", async () => {
|
|
483
|
+
await agent
|
|
484
|
+
.post("/sync/snapshot/batch")
|
|
485
|
+
.send({streams: [{stream: ownerStream()}, {stream: "routeStuff|owner:someoneElse"}]})
|
|
486
|
+
.expect(403);
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
it("rejects malformed bodies", async () => {
|
|
490
|
+
const tooMany = Array.from({length: 501}, (_unused, index) => ({
|
|
491
|
+
stream: `routeStuff|owner:u${index}`,
|
|
492
|
+
}));
|
|
493
|
+
const bodies: unknown[] = [
|
|
494
|
+
{},
|
|
495
|
+
{streams: []},
|
|
496
|
+
{streams: "routeStuff"},
|
|
497
|
+
{streams: tooMany},
|
|
498
|
+
{streams: [{cursor: 0}]},
|
|
499
|
+
{streams: [{stream: ownerStream()}, {stream: ownerStream()}]},
|
|
500
|
+
{streams: [{cursor: -1, stream: ownerStream()}]},
|
|
501
|
+
{streams: [{legacyCursor: 5, stream: ownerStream()}]},
|
|
502
|
+
{limit: "lots", streams: [{stream: ownerStream()}]},
|
|
503
|
+
{streams: [{stream: "notAStreamKey"}]},
|
|
504
|
+
];
|
|
505
|
+
for (const body of bodies) {
|
|
506
|
+
await agent
|
|
507
|
+
.post("/sync/snapshot/batch")
|
|
508
|
+
.send(body as object)
|
|
509
|
+
.expect(400);
|
|
510
|
+
}
|
|
511
|
+
});
|
|
512
|
+
});
|
|
513
|
+
|
|
406
514
|
describe("buildSnapshotScopeFilter", () => {
|
|
407
515
|
it("500s for a custom scope entry that has no snapshotFilter result", () => {
|
|
408
516
|
// Registration rejects this shape up front, so build the entry directly.
|
package/src/sync/types.ts
CHANGED
|
@@ -120,6 +120,27 @@ export interface SyncSnapshotResponse {
|
|
|
120
120
|
legacyCursor?: string;
|
|
121
121
|
}
|
|
122
122
|
|
|
123
|
+
/** One stream's page request inside `POST /sync/snapshot/batch`. */
|
|
124
|
+
export interface SyncSnapshotBatchStreamRequest {
|
|
125
|
+
stream: string;
|
|
126
|
+
/** Resume cursor (highest seq already applied); 0 = from the beginning. */
|
|
127
|
+
cursor?: number;
|
|
128
|
+
/** C3: legacy-stratum token echoed back from this stream's previous page. */
|
|
129
|
+
legacyCursor?: string;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Body of `POST /sync/snapshot/batch`. */
|
|
133
|
+
export interface SyncSnapshotBatchRequest {
|
|
134
|
+
streams: SyncSnapshotBatchStreamRequest[];
|
|
135
|
+
/** Entity budget for the whole response, split evenly across `streams`. */
|
|
136
|
+
limit?: number;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Response of `POST /sync/snapshot/batch`: one page per requested stream, in request order. */
|
|
140
|
+
export interface SyncSnapshotBatchResponse {
|
|
141
|
+
pages: SyncSnapshotResponse[];
|
|
142
|
+
}
|
|
143
|
+
|
|
123
144
|
/** Response shape for `GET /sync/entities` (point lookup for repair). */
|
|
124
145
|
export interface SyncEntitiesResponse {
|
|
125
146
|
entities: SyncEntityPayload[];
|