@terreno/api 57.14.0 → 57.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -69,6 +69,7 @@ export declare const MAX_ENTITY_FETCH = 100;
69
69
  * Mount the SyncDB HTTP routes:
70
70
  * - GET /sync/streams — the authoritative set of streams the caller belongs to (C2)
71
71
  * - GET /sync/snapshot — per-stream bootstrap/catch-up with server-enforced scoping
72
+ * - POST /sync/snapshot/batch — one page for each of many streams in one round trip
72
73
  * - POST /sync/mutate — HTTP fallback mutation channel over applySyncMutation
73
74
  * - GET /sync/key — per-user key material for the default encryption KeyProvider
74
75
  */
@@ -31,6 +31,13 @@ var syncBatch_2 = require("./syncBatch");
31
31
  Object.defineProperty(exports, "MAX_SYNC_HTTP_MUTATIONS_PER_SECOND", { enumerable: true, get: function () { return syncBatch_2.MAX_SYNC_MUTATIONS_PER_SECOND; } });
32
32
  const MAX_SNAPSHOT_LIMIT = 100;
33
33
  const DEFAULT_SNAPSHOT_LIMIT = 100;
34
+ /** Entity budget of one `POST /sync/snapshot/batch` response, shared by its streams. */
35
+ const DEFAULT_SNAPSHOT_BATCH_LIMIT = 5_000;
36
+ const MAX_SNAPSHOT_BATCH_LIMIT = 10_000;
37
+ /** Most streams one batch request may name. */
38
+ const MAX_SNAPSHOT_BATCH_STREAMS = 500;
39
+ /** Streams paged at once per batch request, so one request cannot drain the Mongo pool. */
40
+ const SNAPSHOT_BATCH_CONCURRENCY = 8;
34
41
  /**
35
42
  * Look up a registered model by name as a `SyncedModel`. `mongoose.model(name)` has no
36
43
  * schema to infer from, so its static type carries none of the synced-document fields.
@@ -331,10 +338,174 @@ const castEntityIds = ({ model, ids, }) => {
331
338
  }
332
339
  return ids.map((id) => new mongoose_1.default.Types.ObjectId(id));
333
340
  };
341
+ /**
342
+ * Serve one snapshot page for one stream: membership/permission checks, frontier and
343
+ * retention signals, then the legacy or seq stratum. Shared by `GET /sync/snapshot` and
344
+ * `POST /sync/snapshot/batch`.
345
+ */
346
+ const snapshotStreamPage = async ({ req, user, streamKey, cursor, limit, legacyCursorIn, options, }) => {
347
+ if (!streamKey) {
348
+ throw new errors_1.APIError({ status: 400, title: "stream query parameter is required" });
349
+ }
350
+ const parsed = (0, streams_1.parseStreamKey)(streamKey);
351
+ if (!parsed) {
352
+ throw new errors_1.APIError({ status: 400, title: `Invalid stream key: ${streamKey}` });
353
+ }
354
+ const entry = (0, registry_1.findSyncEntryByCollectionTag)(parsed.collectionTag);
355
+ if (!entry) {
356
+ throw new errors_1.APIError({
357
+ status: 404,
358
+ title: `Unknown sync collection: ${parsed.collectionTag}`,
359
+ });
360
+ }
361
+ if (!(await (0, permissions_1.checkPermissions)("list", entry.options.permissions.list, user))) {
362
+ throw new errors_1.APIError({
363
+ status: 403,
364
+ title: `Access to sync snapshot for ${parsed.collectionTag} denied for ${user.id}`,
365
+ });
366
+ }
367
+ // C2: a client must not snapshot a stream it does not belong to.
368
+ const memberStreams = await (0, streams_1.resolveUserStreamsForEntry)({
369
+ entry,
370
+ getUserScopes: options.getUserScopes,
371
+ user,
372
+ });
373
+ if (!memberStreams.includes(streamKey)) {
374
+ throw new errors_1.APIError({
375
+ status: 403,
376
+ title: `User ${user.id} does not belong to stream ${streamKey}`,
377
+ });
378
+ }
379
+ const snapshotFilterResult = entry.config.snapshotFilter
380
+ ? await entry.config.snapshotFilter({ id: String(user.id) })
381
+ : undefined;
382
+ const queryFilterResult = await resolveSyncQueryFilter({ entry, user });
383
+ const scopeFilter = composeSyncReadFilter({
384
+ queryFilter: queryFilterResult.filter,
385
+ scopeFilter: (0, exports.buildSnapshotScopeFilter)({
386
+ entry,
387
+ scopeValue: parsed.scopeValue,
388
+ snapshotFilterResult,
389
+ }),
390
+ });
391
+ const model = getSyncedModel(entry.modelName);
392
+ const frontierSeq = await (0, models_1.computeStableFrontier)({ stream: streamKey });
393
+ // C7 (Task 9.15): the retention signal is the durable compaction watermark, not
394
+ // min(retained seq). The old computation was pinned low forever by any early doc that
395
+ // was never deleted, so a stale cursor sitting above a compacted tombstone's seq
396
+ // passed the check and silently never learned about that deletion.
397
+ const oldestRetainedSeq = await (0, models_1.getCompactedThroughSeq)({ stream: streamKey });
398
+ // Task 9.19: a `queryFilter` that denies this caller means "no visible rows", exactly
399
+ // as REST's list endpoint answers. Serve a terminal empty page (cursor at the
400
+ // frontier, hasMore false) so the client settles instead of paging forever.
401
+ if (queryFilterResult.denied) {
402
+ const deniedResponse = {
403
+ cursor: frontierSeq,
404
+ entities: [],
405
+ frontierSeq,
406
+ hasMore: false,
407
+ oldestRetainedSeq,
408
+ stream: streamKey,
409
+ };
410
+ return deniedResponse;
411
+ }
412
+ // C3: legacy (seq-0) stratum, paged by _id. Drained fully before seq paging begins.
413
+ if (cursor === 0) {
414
+ const legacyResult = await pageLegacyStratum({
415
+ entry,
416
+ legacyCursorIn,
417
+ limit,
418
+ model,
419
+ req,
420
+ scopeFilter,
421
+ });
422
+ if (legacyResult) {
423
+ const response = {
424
+ cursor: 0,
425
+ entities: legacyResult.entities,
426
+ frontierSeq,
427
+ hasMore: true,
428
+ legacyCursor: legacyResult.legacyCursor,
429
+ oldestRetainedSeq,
430
+ stream: streamKey,
431
+ };
432
+ return response;
433
+ }
434
+ }
435
+ const seqResult = await pageSeqStratum({
436
+ cursor,
437
+ entry,
438
+ frontierSeq,
439
+ limit,
440
+ model,
441
+ req,
442
+ scopeFilter,
443
+ streamKey,
444
+ });
445
+ const response = {
446
+ cursor: seqResult.cursor,
447
+ entities: seqResult.entities,
448
+ frontierSeq,
449
+ hasMore: seqResult.hasMore,
450
+ oldestRetainedSeq,
451
+ stream: streamKey,
452
+ };
453
+ return response;
454
+ };
455
+ /** Validate the `streams` list of a snapshot batch body: non-empty, capped, no repeats. */
456
+ const parseSnapshotBatchStreams = (raw) => {
457
+ if (!Array.isArray(raw) || raw.length === 0) {
458
+ throw new errors_1.APIError({ status: 400, title: "streams must be a non-empty array" });
459
+ }
460
+ if (raw.length > MAX_SNAPSHOT_BATCH_STREAMS) {
461
+ throw new errors_1.APIError({
462
+ detail: `Requested ${raw.length} streams; the limit is ${MAX_SNAPSHOT_BATCH_STREAMS}`,
463
+ status: 400,
464
+ title: "Too many streams in snapshot batch",
465
+ });
466
+ }
467
+ const seen = new Set();
468
+ return raw.map((item) => {
469
+ const candidate = item;
470
+ if (!candidate || typeof candidate.stream !== "string" || !candidate.stream) {
471
+ throw new errors_1.APIError({ status: 400, title: "Each snapshot batch entry needs a stream" });
472
+ }
473
+ if (candidate.legacyCursor !== undefined && typeof candidate.legacyCursor !== "string") {
474
+ throw new errors_1.APIError({ status: 400, title: `Invalid legacyCursor for ${candidate.stream}` });
475
+ }
476
+ if (seen.has(candidate.stream)) {
477
+ throw new errors_1.APIError({
478
+ status: 400,
479
+ title: `Duplicate stream in snapshot batch: ${candidate.stream}`,
480
+ });
481
+ }
482
+ seen.add(candidate.stream);
483
+ return {
484
+ cursor: candidate.cursor,
485
+ legacyCursor: candidate.legacyCursor,
486
+ stream: candidate.stream,
487
+ };
488
+ });
489
+ };
490
+ /** `items.map(map)` with at most `concurrency` calls in flight; results keep item order. */
491
+ const mapWithConcurrency = async ({ items, concurrency, map, }) => {
492
+ const results = new Array(items.length);
493
+ let next = 0;
494
+ const worker = async () => {
495
+ while (next < items.length) {
496
+ const index = next;
497
+ next += 1;
498
+ results[index] = await map(items[index]);
499
+ }
500
+ };
501
+ await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
502
+ return results;
503
+ };
334
504
  /**
335
505
  * Mount the SyncDB HTTP routes:
336
506
  * - GET /sync/streams — the authoritative set of streams the caller belongs to (C2)
337
507
  * - GET /sync/snapshot — per-stream bootstrap/catch-up with server-enforced scoping
508
+ * - POST /sync/snapshot/batch — one page for each of many streams in one round trip
338
509
  * - POST /sync/mutate — HTTP fallback mutation channel over applySyncMutation
339
510
  * - GET /sync/key — per-user key material for the default encryption KeyProvider
340
511
  */
@@ -377,119 +548,50 @@ const addSyncRoutes = (app, options = {}) => {
377
548
  if (!user) {
378
549
  throw new errors_1.APIError({ status: 401, title: "Authentication required" });
379
550
  }
380
- const streamKey = String(req.query.stream ?? "");
381
- if (!streamKey) {
382
- throw new errors_1.APIError({ status: 400, title: "stream query parameter is required" });
383
- }
384
- const parsed = (0, streams_1.parseStreamKey)(streamKey);
385
- if (!parsed) {
386
- throw new errors_1.APIError({ status: 400, title: `Invalid stream key: ${streamKey}` });
387
- }
388
- const entry = (0, registry_1.findSyncEntryByCollectionTag)(parsed.collectionTag);
389
- if (!entry) {
390
- throw new errors_1.APIError({
391
- status: 404,
392
- title: `Unknown sync collection: ${parsed.collectionTag}`,
393
- });
394
- }
395
- if (!(await (0, permissions_1.checkPermissions)("list", entry.options.permissions.list, user))) {
396
- throw new errors_1.APIError({
397
- status: 403,
398
- title: `Access to sync snapshot for ${parsed.collectionTag} denied for ${user.id}`,
399
- });
400
- }
401
- // C2: a client must not snapshot a stream it does not belong to.
402
- const memberStreams = await (0, streams_1.resolveUserStreamsForEntry)({
403
- entry,
404
- getUserScopes: options.getUserScopes,
405
- user,
406
- });
407
- if (!memberStreams.includes(streamKey)) {
408
- throw new errors_1.APIError({
409
- status: 403,
410
- title: `User ${user.id} does not belong to stream ${streamKey}`,
411
- });
412
- }
413
551
  const cursor = parseNonNegativeInt(req.query.cursor, "cursor", 0);
414
552
  const requestedLimit = parseNonNegativeInt(req.query.limit, "limit", options.defaultSnapshotLimit ?? DEFAULT_SNAPSHOT_LIMIT);
415
553
  const limit = Math.min(Math.max(requestedLimit, 1), MAX_SNAPSHOT_LIMIT);
416
554
  const legacyCursorIn = typeof req.query.legacyCursor === "string" && req.query.legacyCursor.length > 0
417
555
  ? req.query.legacyCursor
418
556
  : undefined;
419
- const snapshotFilterResult = entry.config.snapshotFilter
420
- ? await entry.config.snapshotFilter({ id: String(user.id) })
421
- : undefined;
422
- const queryFilterResult = await resolveSyncQueryFilter({ entry, user });
423
- const scopeFilter = composeSyncReadFilter({
424
- queryFilter: queryFilterResult.filter,
425
- scopeFilter: (0, exports.buildSnapshotScopeFilter)({
426
- entry,
427
- scopeValue: parsed.scopeValue,
428
- snapshotFilterResult,
429
- }),
557
+ const response = await snapshotStreamPage({
558
+ cursor,
559
+ legacyCursorIn,
560
+ limit,
561
+ options,
562
+ req,
563
+ streamKey: String(req.query.stream ?? ""),
564
+ user,
430
565
  });
431
- const model = getSyncedModel(entry.modelName);
432
- const frontierSeq = await (0, models_1.computeStableFrontier)({ stream: streamKey });
433
- // C7 (Task 9.15): the retention signal is the durable compaction watermark, not
434
- // min(retained seq). The old computation was pinned low forever by any early doc that
435
- // was never deleted, so a stale cursor sitting above a compacted tombstone's seq
436
- // passed the check and silently never learned about that deletion.
437
- const oldestRetainedSeq = await (0, models_1.getCompactedThroughSeq)({ stream: streamKey });
438
- // Task 9.19: a `queryFilter` that denies this caller means "no visible rows", exactly
439
- // as REST's list endpoint answers. Serve a terminal empty page (cursor at the
440
- // frontier, hasMore false) so the client settles instead of paging forever.
441
- if (queryFilterResult.denied) {
442
- const deniedResponse = {
443
- cursor: frontierSeq,
444
- entities: [],
445
- frontierSeq,
446
- hasMore: false,
447
- oldestRetainedSeq,
448
- stream: streamKey,
449
- };
450
- return res.json(deniedResponse);
566
+ return res.json(response);
567
+ }));
568
+ // Page many streams in one round trip: one page per requested stream, in request order.
569
+ // `limit` is the entity budget for the whole response, split evenly across the streams,
570
+ // so a request stays bounded however many streams it names; streams with more pages
571
+ // are asked for again (with a larger share) in the client's next round.
572
+ router.post("/sync/snapshot/batch", (0, auth_1.authenticateMiddleware)(), (0, api_1.asyncHandler)(async (req, res) => {
573
+ const user = req.user;
574
+ if (!user) {
575
+ throw new errors_1.APIError({ status: 401, title: "Authentication required" });
451
576
  }
452
- // C3: legacy (seq-0) stratum, paged by _id. Drained fully before seq paging begins.
453
- if (cursor === 0) {
454
- const legacyResult = await pageLegacyStratum({
455
- entry,
456
- legacyCursorIn,
577
+ const body = req.body;
578
+ const items = parseSnapshotBatchStreams(body?.streams);
579
+ const budget = Math.min(Math.max(parseNonNegativeInt(body?.limit, "limit", DEFAULT_SNAPSHOT_BATCH_LIMIT), items.length), MAX_SNAPSHOT_BATCH_LIMIT);
580
+ const limit = Math.max(1, Math.floor(budget / items.length));
581
+ const pages = await mapWithConcurrency({
582
+ concurrency: SNAPSHOT_BATCH_CONCURRENCY,
583
+ items,
584
+ map: (item) => snapshotStreamPage({
585
+ cursor: parseNonNegativeInt(item.cursor, "cursor", 0),
586
+ legacyCursorIn: item.legacyCursor || undefined,
457
587
  limit,
458
- model,
588
+ options,
459
589
  req,
460
- scopeFilter,
461
- });
462
- if (legacyResult) {
463
- const response = {
464
- cursor: 0,
465
- entities: legacyResult.entities,
466
- frontierSeq,
467
- hasMore: true,
468
- legacyCursor: legacyResult.legacyCursor,
469
- oldestRetainedSeq,
470
- stream: streamKey,
471
- };
472
- return res.json(response);
473
- }
474
- }
475
- const seqResult = await pageSeqStratum({
476
- cursor,
477
- entry,
478
- frontierSeq,
479
- limit,
480
- model,
481
- req,
482
- scopeFilter,
483
- streamKey,
590
+ streamKey: item.stream,
591
+ user,
592
+ }),
484
593
  });
485
- const response = {
486
- cursor: seqResult.cursor,
487
- entities: seqResult.entities,
488
- frontierSeq,
489
- hasMore: seqResult.hasMore,
490
- oldestRetainedSeq,
491
- stream: streamKey,
492
- };
594
+ const response = { pages };
493
595
  return res.json(response);
494
596
  }));
495
597
  router.get("/sync/entities", (0, auth_1.authenticateMiddleware)(), (0, api_1.asyncHandler)(async (req, res) => {
@@ -5,8 +5,9 @@ import { type SyncAppOptions } from "./routes";
5
5
  export declare const getSyncAppOptions: (app: express.Application) => SyncAppOptions | undefined;
6
6
  /**
7
7
  * TerrenoPlugin mounting the SyncDB local-first sync HTTP routes
8
- * (`GET /sync/snapshot`, `POST /sync/mutate`, `GET /sync/key`). Models opt in via
9
- * modelRouter's `sync` option; this plugin serves the registered collections.
8
+ * (`GET /sync/snapshot`, `POST /sync/snapshot/batch`, `POST /sync/mutate`, `GET /sync/key`).
9
+ * Models opt in via modelRouter's `sync` option; this plugin serves the registered
10
+ * collections.
10
11
  *
11
12
  * Registration attaches the plugin's options (notably `getUserScopes`) to this Express
12
13
  * application so RealtimeApp's connection handler can install the socket
@@ -10,8 +10,9 @@ const getSyncAppOptions = (app) => app.locals[SYNC_APP_OPTIONS_LOCAL_KEY];
10
10
  exports.getSyncAppOptions = getSyncAppOptions;
11
11
  /**
12
12
  * TerrenoPlugin mounting the SyncDB local-first sync HTTP routes
13
- * (`GET /sync/snapshot`, `POST /sync/mutate`, `GET /sync/key`). Models opt in via
14
- * modelRouter's `sync` option; this plugin serves the registered collections.
13
+ * (`GET /sync/snapshot`, `POST /sync/snapshot/batch`, `POST /sync/mutate`, `GET /sync/key`).
14
+ * Models opt in via modelRouter's `sync` option; this plugin serves the registered
15
+ * collections.
15
16
  *
16
17
  * Registration attaches the plugin's options (notably `getUserScopes`) to this Express
17
18
  * application so RealtimeApp's connection handler can install the socket
@@ -108,6 +108,24 @@ export interface SyncSnapshotResponse {
108
108
  */
109
109
  legacyCursor?: string;
110
110
  }
111
+ /** One stream's page request inside `POST /sync/snapshot/batch`. */
112
+ export interface SyncSnapshotBatchStreamRequest {
113
+ stream: string;
114
+ /** Resume cursor (highest seq already applied); 0 = from the beginning. */
115
+ cursor?: number;
116
+ /** C3: legacy-stratum token echoed back from this stream's previous page. */
117
+ legacyCursor?: string;
118
+ }
119
+ /** Body of `POST /sync/snapshot/batch`. */
120
+ export interface SyncSnapshotBatchRequest {
121
+ streams: SyncSnapshotBatchStreamRequest[];
122
+ /** Entity budget for the whole response, split evenly across `streams`. */
123
+ limit?: number;
124
+ }
125
+ /** Response of `POST /sync/snapshot/batch`: one page per requested stream, in request order. */
126
+ export interface SyncSnapshotBatchResponse {
127
+ pages: SyncSnapshotResponse[];
128
+ }
111
129
  /** Response shape for `GET /sync/entities` (point lookup for repair). */
112
130
  export interface SyncEntitiesResponse {
113
131
  entities: SyncEntityPayload[];
package/package.json CHANGED
@@ -11,7 +11,7 @@
11
11
  "@modelcontextprotocol/node": "^2.0.0",
12
12
  "@modelcontextprotocol/server": "^2.0.0",
13
13
  "@sentry/bun": "^10.39.0",
14
- "@terreno/test": "57.14.0",
14
+ "@terreno/test": "57.16.1",
15
15
  "@thream/socketio-jwt": "^3.1.4",
16
16
  "@types/qs": "^6.14.0",
17
17
  "ajv": "8.20.0",
@@ -48,7 +48,7 @@
48
48
  "devDependencies": {
49
49
  "@biomejs/biome": "^2.3.6",
50
50
  "@modelcontextprotocol/client": "^2.0.0",
51
- "@terreno/syncdb": "57.14.0",
51
+ "@terreno/syncdb": "57.16.1",
52
52
  "@types/bun": "^1.4.0",
53
53
  "@types/cors": "^2.8.17",
54
54
  "@types/express": "^5.0.6",
@@ -132,5 +132,5 @@
132
132
  "updateSnapshot": "bun test --update-snapshots"
133
133
  },
134
134
  "types": "dist/index.d.ts",
135
- "version": "57.14.0"
135
+ "version": "57.16.1"
136
136
  }
@@ -36,6 +36,9 @@ import type {
36
36
  SyncMutateBatchRequest,
37
37
  SyncMutateRequest,
38
38
  SyncNackCode,
39
+ SyncSnapshotBatchRequest,
40
+ SyncSnapshotBatchResponse,
41
+ SyncSnapshotBatchStreamRequest,
39
42
  SyncSnapshotResponse,
40
43
  SyncStreamInfo,
41
44
  SyncStreamsResponse,
@@ -77,6 +80,13 @@ export interface SyncAppOptions {
77
80
 
78
81
  const MAX_SNAPSHOT_LIMIT = 100;
79
82
  const DEFAULT_SNAPSHOT_LIMIT = 100;
83
+ /** Entity budget of one `POST /sync/snapshot/batch` response, shared by its streams. */
84
+ const DEFAULT_SNAPSHOT_BATCH_LIMIT = 5_000;
85
+ const MAX_SNAPSHOT_BATCH_LIMIT = 10_000;
86
+ /** Most streams one batch request may name. */
87
+ const MAX_SNAPSHOT_BATCH_STREAMS = 500;
88
+ /** Streams paged at once per batch request, so one request cannot drain the Mongo pool. */
89
+ const SNAPSHOT_BATCH_CONCURRENCY = 8;
80
90
 
81
91
  /** Minimal shape of a synced document as read by the snapshot/entities routes. */
82
92
  interface SyncedDoc {
@@ -486,10 +496,205 @@ const castEntityIds = ({
486
496
  return ids.map((id) => new mongoose.Types.ObjectId(id));
487
497
  };
488
498
 
499
+ /**
500
+ * Serve one snapshot page for one stream: membership/permission checks, frontier and
501
+ * retention signals, then the legacy or seq stratum. Shared by `GET /sync/snapshot` and
502
+ * `POST /sync/snapshot/batch`.
503
+ */
504
+ const snapshotStreamPage = async ({
505
+ req,
506
+ user,
507
+ streamKey,
508
+ cursor,
509
+ limit,
510
+ legacyCursorIn,
511
+ options,
512
+ }: {
513
+ req: express.Request;
514
+ user: User;
515
+ streamKey: string;
516
+ cursor: number;
517
+ limit: number;
518
+ legacyCursorIn?: string;
519
+ options: SyncAppOptions;
520
+ }): Promise<SyncSnapshotResponse> => {
521
+ if (!streamKey) {
522
+ throw new APIError({status: 400, title: "stream query parameter is required"});
523
+ }
524
+ const parsed = parseStreamKey(streamKey);
525
+ if (!parsed) {
526
+ throw new APIError({status: 400, title: `Invalid stream key: ${streamKey}`});
527
+ }
528
+ const entry = findSyncEntryByCollectionTag(parsed.collectionTag);
529
+ if (!entry) {
530
+ throw new APIError({
531
+ status: 404,
532
+ title: `Unknown sync collection: ${parsed.collectionTag}`,
533
+ });
534
+ }
535
+ if (!(await checkPermissions("list", entry.options.permissions.list, user))) {
536
+ throw new APIError({
537
+ status: 403,
538
+ title: `Access to sync snapshot for ${parsed.collectionTag} denied for ${user.id}`,
539
+ });
540
+ }
541
+ // C2: a client must not snapshot a stream it does not belong to.
542
+ const memberStreams = await resolveUserStreamsForEntry({
543
+ entry,
544
+ getUserScopes: options.getUserScopes,
545
+ user,
546
+ });
547
+ if (!memberStreams.includes(streamKey)) {
548
+ throw new APIError({
549
+ status: 403,
550
+ title: `User ${user.id} does not belong to stream ${streamKey}`,
551
+ });
552
+ }
553
+
554
+ const snapshotFilterResult = entry.config.snapshotFilter
555
+ ? await entry.config.snapshotFilter({id: String(user.id)})
556
+ : undefined;
557
+ const queryFilterResult = await resolveSyncQueryFilter({entry, user});
558
+ const scopeFilter = composeSyncReadFilter({
559
+ queryFilter: queryFilterResult.filter,
560
+ scopeFilter: buildSnapshotScopeFilter({
561
+ entry,
562
+ scopeValue: parsed.scopeValue,
563
+ snapshotFilterResult,
564
+ }),
565
+ });
566
+ const model = getSyncedModel(entry.modelName);
567
+ const frontierSeq = await computeStableFrontier({stream: streamKey});
568
+ // C7 (Task 9.15): the retention signal is the durable compaction watermark, not
569
+ // min(retained seq). The old computation was pinned low forever by any early doc that
570
+ // was never deleted, so a stale cursor sitting above a compacted tombstone's seq
571
+ // passed the check and silently never learned about that deletion.
572
+ const oldestRetainedSeq = await getCompactedThroughSeq({stream: streamKey});
573
+
574
+ // Task 9.19: a `queryFilter` that denies this caller means "no visible rows", exactly
575
+ // as REST's list endpoint answers. Serve a terminal empty page (cursor at the
576
+ // frontier, hasMore false) so the client settles instead of paging forever.
577
+ if (queryFilterResult.denied) {
578
+ const deniedResponse: SyncSnapshotResponse = {
579
+ cursor: frontierSeq,
580
+ entities: [],
581
+ frontierSeq,
582
+ hasMore: false,
583
+ oldestRetainedSeq,
584
+ stream: streamKey,
585
+ };
586
+ return deniedResponse;
587
+ }
588
+
589
+ // C3: legacy (seq-0) stratum, paged by _id. Drained fully before seq paging begins.
590
+ if (cursor === 0) {
591
+ const legacyResult = await pageLegacyStratum({
592
+ entry,
593
+ legacyCursorIn,
594
+ limit,
595
+ model,
596
+ req,
597
+ scopeFilter,
598
+ });
599
+ if (legacyResult) {
600
+ const response: SyncSnapshotResponse = {
601
+ cursor: 0,
602
+ entities: legacyResult.entities,
603
+ frontierSeq,
604
+ hasMore: true,
605
+ legacyCursor: legacyResult.legacyCursor,
606
+ oldestRetainedSeq,
607
+ stream: streamKey,
608
+ };
609
+ return response;
610
+ }
611
+ }
612
+
613
+ const seqResult = await pageSeqStratum({
614
+ cursor,
615
+ entry,
616
+ frontierSeq,
617
+ limit,
618
+ model,
619
+ req,
620
+ scopeFilter,
621
+ streamKey,
622
+ });
623
+ const response: SyncSnapshotResponse = {
624
+ cursor: seqResult.cursor,
625
+ entities: seqResult.entities,
626
+ frontierSeq,
627
+ hasMore: seqResult.hasMore,
628
+ oldestRetainedSeq,
629
+ stream: streamKey,
630
+ };
631
+ return response;
632
+ };
633
+
634
+ /** Validate the `streams` list of a snapshot batch body: non-empty, capped, no repeats. */
635
+ const parseSnapshotBatchStreams = (raw: unknown): SyncSnapshotBatchStreamRequest[] => {
636
+ if (!Array.isArray(raw) || raw.length === 0) {
637
+ throw new APIError({status: 400, title: "streams must be a non-empty array"});
638
+ }
639
+ if (raw.length > MAX_SNAPSHOT_BATCH_STREAMS) {
640
+ throw new APIError({
641
+ detail: `Requested ${raw.length} streams; the limit is ${MAX_SNAPSHOT_BATCH_STREAMS}`,
642
+ status: 400,
643
+ title: "Too many streams in snapshot batch",
644
+ });
645
+ }
646
+ const seen = new Set<string>();
647
+ return raw.map((item: unknown) => {
648
+ const candidate = item as Partial<SyncSnapshotBatchStreamRequest> | null;
649
+ if (!candidate || typeof candidate.stream !== "string" || !candidate.stream) {
650
+ throw new APIError({status: 400, title: "Each snapshot batch entry needs a stream"});
651
+ }
652
+ if (candidate.legacyCursor !== undefined && typeof candidate.legacyCursor !== "string") {
653
+ throw new APIError({status: 400, title: `Invalid legacyCursor for ${candidate.stream}`});
654
+ }
655
+ if (seen.has(candidate.stream)) {
656
+ throw new APIError({
657
+ status: 400,
658
+ title: `Duplicate stream in snapshot batch: ${candidate.stream}`,
659
+ });
660
+ }
661
+ seen.add(candidate.stream);
662
+ return {
663
+ cursor: candidate.cursor,
664
+ legacyCursor: candidate.legacyCursor,
665
+ stream: candidate.stream,
666
+ };
667
+ });
668
+ };
669
+
670
+ /** `items.map(map)` with at most `concurrency` calls in flight; results keep item order. */
671
+ const mapWithConcurrency = async <T, R>({
672
+ items,
673
+ concurrency,
674
+ map,
675
+ }: {
676
+ items: T[];
677
+ concurrency: number;
678
+ map: (item: T) => Promise<R>;
679
+ }): Promise<R[]> => {
680
+ const results = new Array<R>(items.length);
681
+ let next = 0;
682
+ const worker = async (): Promise<void> => {
683
+ while (next < items.length) {
684
+ const index = next;
685
+ next += 1;
686
+ results[index] = await map(items[index] as T);
687
+ }
688
+ };
689
+ await Promise.all(Array.from({length: Math.min(concurrency, items.length)}, worker));
690
+ return results;
691
+ };
692
+
489
693
  /**
490
694
  * Mount the SyncDB HTTP routes:
491
695
  * - GET /sync/streams — the authoritative set of streams the caller belongs to (C2)
492
696
  * - GET /sync/snapshot — per-stream bootstrap/catch-up with server-enforced scoping
697
+ * - POST /sync/snapshot/batch — one page for each of many streams in one round trip
493
698
  * - POST /sync/mutate — HTTP fallback mutation channel over applySyncMutation
494
699
  * - GET /sync/key — per-user key material for the default encryption KeyProvider
495
700
  */
@@ -541,40 +746,6 @@ export const addSyncRoutes = (app: express.Application, options: SyncAppOptions
541
746
  if (!user) {
542
747
  throw new APIError({status: 401, title: "Authentication required"});
543
748
  }
544
- const streamKey = String(req.query.stream ?? "");
545
- if (!streamKey) {
546
- throw new APIError({status: 400, title: "stream query parameter is required"});
547
- }
548
- const parsed = parseStreamKey(streamKey);
549
- if (!parsed) {
550
- throw new APIError({status: 400, title: `Invalid stream key: ${streamKey}`});
551
- }
552
- const entry = findSyncEntryByCollectionTag(parsed.collectionTag);
553
- if (!entry) {
554
- throw new APIError({
555
- status: 404,
556
- title: `Unknown sync collection: ${parsed.collectionTag}`,
557
- });
558
- }
559
- if (!(await checkPermissions("list", entry.options.permissions.list, user))) {
560
- throw new APIError({
561
- status: 403,
562
- title: `Access to sync snapshot for ${parsed.collectionTag} denied for ${user.id}`,
563
- });
564
- }
565
- // C2: a client must not snapshot a stream it does not belong to.
566
- const memberStreams = await resolveUserStreamsForEntry({
567
- entry,
568
- getUserScopes: options.getUserScopes,
569
- user,
570
- });
571
- if (!memberStreams.includes(streamKey)) {
572
- throw new APIError({
573
- status: 403,
574
- title: `User ${user.id} does not belong to stream ${streamKey}`,
575
- });
576
- }
577
-
578
749
  const cursor = parseNonNegativeInt(req.query.cursor, "cursor", 0);
579
750
  const requestedLimit = parseNonNegativeInt(
580
751
  req.query.limit,
@@ -586,84 +757,56 @@ export const addSyncRoutes = (app: express.Application, options: SyncAppOptions
586
757
  typeof req.query.legacyCursor === "string" && req.query.legacyCursor.length > 0
587
758
  ? req.query.legacyCursor
588
759
  : undefined;
589
-
590
- const snapshotFilterResult = entry.config.snapshotFilter
591
- ? await entry.config.snapshotFilter({id: String(user.id)})
592
- : undefined;
593
- const queryFilterResult = await resolveSyncQueryFilter({entry, user});
594
- const scopeFilter = composeSyncReadFilter({
595
- queryFilter: queryFilterResult.filter,
596
- scopeFilter: buildSnapshotScopeFilter({
597
- entry,
598
- scopeValue: parsed.scopeValue,
599
- snapshotFilterResult,
600
- }),
601
- });
602
- const model = getSyncedModel(entry.modelName);
603
- const frontierSeq = await computeStableFrontier({stream: streamKey});
604
- // C7 (Task 9.15): the retention signal is the durable compaction watermark, not
605
- // min(retained seq). The old computation was pinned low forever by any early doc that
606
- // was never deleted, so a stale cursor sitting above a compacted tombstone's seq
607
- // passed the check and silently never learned about that deletion.
608
- const oldestRetainedSeq = await getCompactedThroughSeq({stream: streamKey});
609
-
610
- // Task 9.19: a `queryFilter` that denies this caller means "no visible rows", exactly
611
- // as REST's list endpoint answers. Serve a terminal empty page (cursor at the
612
- // frontier, hasMore false) so the client settles instead of paging forever.
613
- if (queryFilterResult.denied) {
614
- const deniedResponse: SyncSnapshotResponse = {
615
- cursor: frontierSeq,
616
- entities: [],
617
- frontierSeq,
618
- hasMore: false,
619
- oldestRetainedSeq,
620
- stream: streamKey,
621
- };
622
- return res.json(deniedResponse);
623
- }
624
-
625
- // C3: legacy (seq-0) stratum, paged by _id. Drained fully before seq paging begins.
626
- if (cursor === 0) {
627
- const legacyResult = await pageLegacyStratum({
628
- entry,
629
- legacyCursorIn,
630
- limit,
631
- model,
632
- req,
633
- scopeFilter,
634
- });
635
- if (legacyResult) {
636
- const response: SyncSnapshotResponse = {
637
- cursor: 0,
638
- entities: legacyResult.entities,
639
- frontierSeq,
640
- hasMore: true,
641
- legacyCursor: legacyResult.legacyCursor,
642
- oldestRetainedSeq,
643
- stream: streamKey,
644
- };
645
- return res.json(response);
646
- }
647
- }
648
-
649
- const seqResult = await pageSeqStratum({
760
+ const response = await snapshotStreamPage({
650
761
  cursor,
651
- entry,
652
- frontierSeq,
762
+ legacyCursorIn,
653
763
  limit,
654
- model,
764
+ options,
655
765
  req,
656
- scopeFilter,
657
- streamKey,
766
+ streamKey: String(req.query.stream ?? ""),
767
+ user,
658
768
  });
659
- const response: SyncSnapshotResponse = {
660
- cursor: seqResult.cursor,
661
- entities: seqResult.entities,
662
- frontierSeq,
663
- hasMore: seqResult.hasMore,
664
- oldestRetainedSeq,
665
- stream: streamKey,
666
- };
769
+ return res.json(response);
770
+ })
771
+ );
772
+
773
+ // Page many streams in one round trip: one page per requested stream, in request order.
774
+ // `limit` is the entity budget for the whole response, split evenly across the streams,
775
+ // so a request stays bounded however many streams it names; streams with more pages
776
+ // are asked for again (with a larger share) in the client's next round.
777
+ router.post(
778
+ "/sync/snapshot/batch",
779
+ authenticateMiddleware(),
780
+ asyncHandler(async (req, res) => {
781
+ const user = req.user as User | undefined;
782
+ if (!user) {
783
+ throw new APIError({status: 401, title: "Authentication required"});
784
+ }
785
+ const body = req.body as Partial<SyncSnapshotBatchRequest> | undefined;
786
+ const items = parseSnapshotBatchStreams(body?.streams);
787
+ const budget = Math.min(
788
+ Math.max(
789
+ parseNonNegativeInt(body?.limit, "limit", DEFAULT_SNAPSHOT_BATCH_LIMIT),
790
+ items.length
791
+ ),
792
+ MAX_SNAPSHOT_BATCH_LIMIT
793
+ );
794
+ const limit = Math.max(1, Math.floor(budget / items.length));
795
+ const pages = await mapWithConcurrency({
796
+ concurrency: SNAPSHOT_BATCH_CONCURRENCY,
797
+ items,
798
+ map: (item) =>
799
+ snapshotStreamPage({
800
+ cursor: parseNonNegativeInt(item.cursor, "cursor", 0),
801
+ legacyCursorIn: item.legacyCursor || undefined,
802
+ limit,
803
+ options,
804
+ req,
805
+ streamKey: item.stream,
806
+ user,
807
+ }),
808
+ });
809
+ const response: SyncSnapshotBatchResponse = {pages};
667
810
  return res.json(response);
668
811
  })
669
812
  );
@@ -12,8 +12,9 @@ export const getSyncAppOptions = (app: express.Application): SyncAppOptions | un
12
12
 
13
13
  /**
14
14
  * TerrenoPlugin mounting the SyncDB local-first sync HTTP routes
15
- * (`GET /sync/snapshot`, `POST /sync/mutate`, `GET /sync/key`). Models opt in via
16
- * modelRouter's `sync` option; this plugin serves the registered collections.
15
+ * (`GET /sync/snapshot`, `POST /sync/snapshot/batch`, `POST /sync/mutate`, `GET /sync/key`).
16
+ * Models opt in via modelRouter's `sync` option; this plugin serves the registered
17
+ * collections.
17
18
  *
18
19
  * Registration attaches the plugin's options (notably `getUserScopes`) to this Express
19
20
  * application so RealtimeApp's connection handler can install the socket
@@ -403,6 +403,114 @@ describe("sync routes", () => {
403
403
  });
404
404
  });
405
405
 
406
+ describe("POST /sync/snapshot/batch", () => {
407
+ const ownerStream = (): string => `routeStuff|owner:${notAdminId}`;
408
+ const tenantStream = "routeProjects|tenant:org1";
409
+
410
+ it("requires authentication", async () => {
411
+ await server
412
+ .post("/sync/snapshot/batch")
413
+ .send({streams: [{stream: "routeStuff|owner:x"}]})
414
+ .expect(401);
415
+ });
416
+
417
+ it("returns one scoped page per stream, in request order", async () => {
418
+ await RouteStuffModel.create({name: "mine", ownerId: notAdminId});
419
+ await RouteStuffModel.create({name: "theirs", ownerId: "someoneElse"});
420
+ await RouteProjectModel.create({orgId: "org1", title: "visible"});
421
+ await RouteProjectModel.create({orgId: "org2", title: "hidden"});
422
+
423
+ const res = await agent
424
+ .post("/sync/snapshot/batch")
425
+ .send({streams: [{stream: tenantStream}, {cursor: 0, stream: ownerStream()}]})
426
+ .expect(200);
427
+
428
+ expect(res.body.pages.map((page: {stream: string}) => page.stream)).toEqual([
429
+ tenantStream,
430
+ ownerStream(),
431
+ ]);
432
+ expect(res.body.pages[0].entities.map((e: SnapshotEntity) => e.data.title)).toEqual([
433
+ "visible",
434
+ ]);
435
+ expect(res.body.pages[1].entities.map((e: SnapshotEntity) => e.data.name)).toEqual(["mine"]);
436
+ expect(res.body.pages[1].hasMore).toBe(false);
437
+ expect(res.body.pages[1].cursor).toBe(res.body.pages[1].frontierSeq);
438
+ });
439
+
440
+ it("splits the entity budget across streams and resumes from each stream's cursor", async () => {
441
+ for (let i = 1; i <= 5; i++) {
442
+ await RouteStuffModel.create({name: `item ${i}`, ownerId: notAdminId});
443
+ }
444
+ await RouteProjectModel.create({orgId: "org1", title: "project"});
445
+
446
+ const first = await agent
447
+ .post("/sync/snapshot/batch")
448
+ .send({limit: 4, streams: [{stream: ownerStream()}, {stream: tenantStream}]})
449
+ .expect(200);
450
+ const [ownerPage, tenantPage] = first.body.pages;
451
+ expect(ownerPage.entities).toHaveLength(2);
452
+ expect(ownerPage.hasMore).toBe(true);
453
+ expect(tenantPage.entities).toHaveLength(1);
454
+ expect(tenantPage.hasMore).toBe(false);
455
+
456
+ const second = await agent
457
+ .post("/sync/snapshot/batch")
458
+ .send({limit: 4, streams: [{cursor: ownerPage.cursor, stream: ownerStream()}]})
459
+ .expect(200);
460
+ expect(second.body.pages[0].entities.map((e: SnapshotEntity) => e.data.name)).toEqual([
461
+ "item 3",
462
+ "item 4",
463
+ "item 5",
464
+ ]);
465
+ expect(second.body.pages[0].hasMore).toBe(false);
466
+ });
467
+
468
+ it("serves the same page as GET /sync/snapshot for the same stream and cursor", async () => {
469
+ for (let i = 1; i <= 3; i++) {
470
+ await RouteStuffModel.create({name: `item ${i}`, ownerId: notAdminId});
471
+ }
472
+ const single = await agent
473
+ .get(`/sync/snapshot?stream=${encodeURIComponent(ownerStream())}&limit=2&cursor=1`)
474
+ .expect(200);
475
+ const batch = await agent
476
+ .post("/sync/snapshot/batch")
477
+ .send({limit: 2, streams: [{cursor: 1, stream: ownerStream()}]})
478
+ .expect(200);
479
+ expect(batch.body.pages[0]).toEqual(single.body);
480
+ });
481
+
482
+ it("rejects the whole request when any stream is not the caller's", async () => {
483
+ await agent
484
+ .post("/sync/snapshot/batch")
485
+ .send({streams: [{stream: ownerStream()}, {stream: "routeStuff|owner:someoneElse"}]})
486
+ .expect(403);
487
+ });
488
+
489
+ it("rejects malformed bodies", async () => {
490
+ const tooMany = Array.from({length: 501}, (_unused, index) => ({
491
+ stream: `routeStuff|owner:u${index}`,
492
+ }));
493
+ const bodies: unknown[] = [
494
+ {},
495
+ {streams: []},
496
+ {streams: "routeStuff"},
497
+ {streams: tooMany},
498
+ {streams: [{cursor: 0}]},
499
+ {streams: [{stream: ownerStream()}, {stream: ownerStream()}]},
500
+ {streams: [{cursor: -1, stream: ownerStream()}]},
501
+ {streams: [{legacyCursor: 5, stream: ownerStream()}]},
502
+ {limit: "lots", streams: [{stream: ownerStream()}]},
503
+ {streams: [{stream: "notAStreamKey"}]},
504
+ ];
505
+ for (const body of bodies) {
506
+ await agent
507
+ .post("/sync/snapshot/batch")
508
+ .send(body as object)
509
+ .expect(400);
510
+ }
511
+ });
512
+ });
513
+
406
514
  describe("buildSnapshotScopeFilter", () => {
407
515
  it("500s for a custom scope entry that has no snapshotFilter result", () => {
408
516
  // Registration rejects this shape up front, so build the entry directly.
package/src/sync/types.ts CHANGED
@@ -120,6 +120,27 @@ export interface SyncSnapshotResponse {
120
120
  legacyCursor?: string;
121
121
  }
122
122
 
123
+ /** One stream's page request inside `POST /sync/snapshot/batch`. */
124
+ export interface SyncSnapshotBatchStreamRequest {
125
+ stream: string;
126
+ /** Resume cursor (highest seq already applied); 0 = from the beginning. */
127
+ cursor?: number;
128
+ /** C3: legacy-stratum token echoed back from this stream's previous page. */
129
+ legacyCursor?: string;
130
+ }
131
+
132
+ /** Body of `POST /sync/snapshot/batch`. */
133
+ export interface SyncSnapshotBatchRequest {
134
+ streams: SyncSnapshotBatchStreamRequest[];
135
+ /** Entity budget for the whole response, split evenly across `streams`. */
136
+ limit?: number;
137
+ }
138
+
139
+ /** Response of `POST /sync/snapshot/batch`: one page per requested stream, in request order. */
140
+ export interface SyncSnapshotBatchResponse {
141
+ pages: SyncSnapshotResponse[];
142
+ }
143
+
123
144
  /** Response shape for `GET /sync/entities` (point lookup for repair). */
124
145
  export interface SyncEntitiesResponse {
125
146
  entities: SyncEntityPayload[];