@terreno/api 0.30.0 → 56.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +10 -0
- package/dist/api.js +110 -361
- package/dist/auth.d.ts +26 -0
- package/dist/auth.js +100 -1
- package/dist/betterAuth.d.ts +5 -0
- package/dist/betterAuthSetup.js +23 -4
- package/dist/docLoader.js +12 -9
- package/dist/errors.d.ts +13 -0
- package/dist/errors.js +29 -1
- package/dist/index.d.ts +17 -1
- package/dist/index.js +29 -1
- package/dist/middleware.d.ts +7 -0
- package/dist/middleware.js +16 -1
- package/dist/openApiCompat.d.ts +10 -1
- package/dist/openApiCompat.js +115 -47
- package/dist/permissions.d.ts +27 -0
- package/dist/permissions.js +42 -1
- package/dist/plugins.d.ts +13 -0
- package/dist/plugins.js +35 -1
- package/dist/realtime/changeStreamWatcher.d.ts +100 -3
- package/dist/realtime/changeStreamWatcher.js +542 -137
- package/dist/realtime/realtimeApp.d.ts +2 -0
- package/dist/realtime/realtimeApp.js +73 -21
- package/dist/realtime/sessionRevalidation.d.ts +90 -0
- package/dist/realtime/sessionRevalidation.js +438 -0
- package/dist/realtime/socketAuth.d.ts +97 -0
- package/dist/realtime/socketAuth.js +272 -0
- package/dist/realtime/socketUser.d.ts +54 -0
- package/dist/realtime/socketUser.js +75 -5
- package/dist/realtime/types.d.ts +40 -0
- package/dist/secretProviders.d.ts +23 -0
- package/dist/secretProviders.js +9 -3
- package/dist/sync/executors.d.ts +198 -0
- package/dist/sync/executors.js +670 -0
- package/dist/sync/loadHarness.js +807 -0
- package/dist/sync/models.d.ts +216 -0
- package/dist/sync/models.js +644 -0
- package/dist/sync/mutationHandler.d.ts +97 -0
- package/dist/sync/mutationHandler.js +886 -0
- package/dist/sync/registry.d.ts +73 -0
- package/dist/sync/registry.js +223 -0
- package/dist/sync/routes.d.ts +62 -0
- package/dist/sync/routes.js +936 -0
- package/dist/sync/scripts/compactTombstones.d.ts +47 -0
- package/dist/sync/scripts/compactTombstones.js +270 -0
- package/dist/sync/serialize.d.ts +20 -0
- package/dist/{tests/bunSetup.js → sync/serialize.js} +35 -68
- package/dist/sync/socketHandlers.d.ts +54 -0
- package/dist/sync/socketHandlers.js +470 -0
- package/dist/sync/streams.d.ts +85 -0
- package/dist/sync/streams.js +221 -0
- package/dist/sync/syncApp.d.ts +29 -0
- package/dist/sync/syncApp.js +44 -0
- package/dist/sync/syncBatch.d.ts +71 -0
- package/dist/sync/syncBatch.js +252 -0
- package/dist/sync/syncSeqPlugin.d.ts +8 -0
- package/dist/sync/syncSeqPlugin.js +903 -0
- package/dist/sync/types.d.ts +254 -0
- package/dist/sync/types.js +9 -0
- package/dist/terrenoApp.js +90 -27
- package/dist/vendor/wesleytodd-openapi/index.d.ts +1 -1
- package/dist/vendor/wesleytodd-openapi/lib/ui.d.ts +2 -1
- package/package.json +7 -6
- package/src/__tests__/openApiCompat.test.ts +20 -0
- package/src/actions.openApi.test.ts +7 -10
- package/src/api.test.ts +12 -0
- package/src/api.ts +87 -295
- package/src/auth.test.ts +128 -176
- package/src/auth.ts +109 -0
- package/src/betterAuth.ts +6 -0
- package/src/betterAuthSetup.test.ts +34 -0
- package/src/betterAuthSetup.ts +20 -1
- package/src/docLoader.ts +3 -1
- package/src/errors.ts +33 -0
- package/src/expressServer.test.ts +7 -1
- package/src/index.ts +42 -0
- package/src/middleware.test.ts +40 -1
- package/src/middleware.ts +15 -0
- package/src/openApiCompat.ts +99 -45
- package/src/permissions.test.ts +75 -1
- package/src/permissions.ts +49 -0
- package/src/plugins.test.ts +54 -0
- package/src/plugins.ts +43 -0
- package/src/realtime/changeStreamWatcher.ts +552 -114
- package/src/realtime/realtime.test.ts +157 -0
- package/src/realtime/realtimeApp.ts +75 -11
- package/src/realtime/sessionRevalidation.test.ts +423 -0
- package/src/realtime/sessionRevalidation.ts +273 -0
- package/src/realtime/socketAuth.test.ts +355 -0
- package/src/realtime/socketAuth.ts +228 -0
- package/src/realtime/socketUser.ts +69 -0
- package/src/realtime/types.ts +40 -0
- package/src/secretProviders.test.ts +7 -1
- package/src/secretProviders.ts +15 -3
- package/src/sync/ensureSyncIndexes.test.ts +144 -0
- package/src/sync/executors.test.ts +882 -0
- package/src/sync/executors.ts +703 -0
- package/src/sync/integration.test.ts +1160 -0
- package/src/sync/loadHarness.ts +618 -0
- package/src/sync/models.ts +640 -0
- package/src/sync/mutationHandler.test.ts +1252 -0
- package/src/sync/mutationHandler.ts +850 -0
- package/src/sync/registry.ts +232 -0
- package/src/sync/routes.ts +812 -0
- package/src/sync/scripts/compactTombstones.test.ts +249 -0
- package/src/sync/scripts/compactTombstones.ts +145 -0
- package/src/sync/serialize.ts +48 -0
- package/src/sync/sessionRevalidationIntegration.test.ts +281 -0
- package/src/sync/socketHandlers.ts +320 -0
- package/src/sync/streams.ts +188 -0
- package/src/sync/sync.test.ts +582 -0
- package/src/sync/syncApp.ts +44 -0
- package/src/sync/syncBatch.ts +219 -0
- package/src/sync/syncFrontier.test.ts +494 -0
- package/src/sync/syncPhaseC.test.ts +718 -0
- package/src/sync/syncRoutes.test.ts +865 -0
- package/src/sync/syncSeqPlugin.ts +738 -0
- package/src/sync/syncSocket.test.ts +1440 -0
- package/src/sync/types.ts +270 -0
- package/src/terrenoApp.ts +37 -17
- package/src/tests/bunSetup.ts +8 -0
- package/tsconfig.json +4 -4
- package/tsconfig.typedoc.json +7 -1
- package/dist/__tests__/openApiCompat.test.js +0 -317
- package/dist/__tests__/versionCheckPlugin.test.d.ts +0 -1
- package/dist/__tests__/versionCheckPlugin.test.js +0 -597
- package/dist/actions.openApi.test.d.ts +0 -1
- package/dist/actions.openApi.test.js +0 -255
- package/dist/actions.test.d.ts +0 -1
- package/dist/actions.test.js +0 -1089
- package/dist/api.arrayOperations.test.d.ts +0 -1
- package/dist/api.arrayOperations.test.js +0 -878
- package/dist/api.asyncHandler.test.d.ts +0 -1
- package/dist/api.asyncHandler.test.js +0 -236
- package/dist/api.coverage.test.d.ts +0 -1
- package/dist/api.coverage.test.js +0 -194
- package/dist/api.errorPaths.test.d.ts +0 -1
- package/dist/api.errorPaths.test.js +0 -801
- package/dist/api.errors.test.d.ts +0 -1
- package/dist/api.errors.test.js +0 -185
- package/dist/api.hooks.test.d.ts +0 -1
- package/dist/api.hooks.test.js +0 -950
- package/dist/api.query.test.d.ts +0 -1
- package/dist/api.query.test.js +0 -948
- package/dist/api.test.d.ts +0 -1
- package/dist/api.test.js +0 -2820
- package/dist/auth.compile.test.d.ts +0 -1
- package/dist/auth.compile.test.js +0 -201
- package/dist/auth.coverage.test.d.ts +0 -1
- package/dist/auth.coverage.test.js +0 -202
- package/dist/auth.test.d.ts +0 -1
- package/dist/auth.test.js +0 -2286
- package/dist/betterAuth.test.d.ts +0 -1
- package/dist/betterAuth.test.js +0 -181
- package/dist/betterAuthApp.test.d.ts +0 -1
- package/dist/betterAuthApp.test.js +0 -242
- package/dist/betterAuthSetup.test.d.ts +0 -1
- package/dist/betterAuthSetup.test.js +0 -762
- package/dist/config.test.d.ts +0 -1
- package/dist/config.test.js +0 -353
- package/dist/configuration.test.d.ts +0 -1
- package/dist/configuration.test.js +0 -980
- package/dist/configurationPlugin.test.d.ts +0 -1
- package/dist/configurationPlugin.test.js +0 -827
- package/dist/consentApp.test.d.ts +0 -1
- package/dist/consentApp.test.js +0 -1415
- package/dist/docLoader.test.d.ts +0 -1
- package/dist/docLoader.test.js +0 -137
- package/dist/envConfigurationPlugin.test.d.ts +0 -1
- package/dist/envConfigurationPlugin.test.js +0 -369
- package/dist/errors.test.d.ts +0 -1
- package/dist/errors.test.js +0 -607
- package/dist/expressServer.test.d.ts +0 -1
- package/dist/expressServer.test.js +0 -1249
- package/dist/githubAuth.test.d.ts +0 -1
- package/dist/githubAuth.test.js +0 -1142
- package/dist/httpClient.test.d.ts +0 -1
- package/dist/httpClient.test.js +0 -1471
- package/dist/logger.test.d.ts +0 -1
- package/dist/logger.test.js +0 -685
- package/dist/middleware.test.d.ts +0 -1
- package/dist/middleware.test.js +0 -241
- package/dist/notifiers/googleChatNotifier.test.d.ts +0 -1
- package/dist/notifiers/googleChatNotifier.test.js +0 -298
- package/dist/notifiers/slackNotifier.test.d.ts +0 -1
- package/dist/notifiers/slackNotifier.test.js +0 -297
- package/dist/notifiers/zoomNotifier.test.d.ts +0 -1
- package/dist/notifiers/zoomNotifier.test.js +0 -373
- package/dist/openApi.test.d.ts +0 -1
- package/dist/openApi.test.js +0 -620
- package/dist/openApiBuilder.test.d.ts +0 -1
- package/dist/openApiBuilder.test.js +0 -601
- package/dist/openApiEtag.test.d.ts +0 -1
- package/dist/openApiEtag.test.js +0 -86
- package/dist/openApiValidator.test.d.ts +0 -1
- package/dist/openApiValidator.test.js +0 -761
- package/dist/permissions.middleware.test.d.ts +0 -1
- package/dist/permissions.middleware.test.js +0 -308
- package/dist/permissions.test.d.ts +0 -1
- package/dist/permissions.test.js +0 -439
- package/dist/plugins.test.d.ts +0 -1
- package/dist/plugins.test.js +0 -890
- package/dist/populate.test.d.ts +0 -1
- package/dist/populate.test.js +0 -373
- package/dist/realtime/realtime.test.d.ts +0 -10
- package/dist/realtime/realtime.test.js +0 -3982
- package/dist/requestContext.test.d.ts +0 -1
- package/dist/requestContext.test.js +0 -412
- package/dist/scriptRunner.test.d.ts +0 -1
- package/dist/scriptRunner.test.js +0 -386
- package/dist/secretProviders.test.d.ts +0 -1
- package/dist/secretProviders.test.js +0 -726
- package/dist/syncConsents.test.d.ts +0 -1
- package/dist/syncConsents.test.js +0 -522
- package/dist/terrenoApp.test.d.ts +0 -1
- package/dist/terrenoApp.test.js +0 -489
- package/dist/tests/bunSetup.d.ts +0 -1
- package/dist/tests/fixtures/compileAuthEntry.d.ts +0 -1
- package/dist/tests/fixtures/compileAuthEntry.js +0 -37
- package/dist/tests/testEnv.d.ts +0 -0
- package/dist/tests/testEnv.js +0 -6
- package/dist/transformers.test.d.ts +0 -1
- package/dist/transformers.test.js +0 -423
- package/dist/utils.test.d.ts +0 -1
- package/dist/utils.test.js +0 -242
- /package/dist/{__tests__/openApiCompat.test.d.ts → sync/loadHarness.d.ts} +0 -0
|
@@ -0,0 +1,812 @@
|
|
|
1
|
+
// noExplicitAny: sync routes operate generically across registered models
|
|
2
|
+
// biome-ignore-all lint/suspicious/noExplicitAny: sync routes operate generically across registered models
|
|
3
|
+
import express from "express";
|
|
4
|
+
import mongoose from "mongoose";
|
|
5
|
+
import {asyncHandler} from "../api";
|
|
6
|
+
import {authenticateMiddleware, type User} from "../auth";
|
|
7
|
+
import {APIError, apiErrorMiddleware, apiUnauthorizedMiddleware} from "../errors";
|
|
8
|
+
import {logger} from "../logger";
|
|
9
|
+
import {checkPermissions} from "../permissions";
|
|
10
|
+
import {
|
|
11
|
+
computeStableFrontier,
|
|
12
|
+
getCompactedThroughSeq,
|
|
13
|
+
getOrCreateSyncKeyMaterial,
|
|
14
|
+
SyncCounter,
|
|
15
|
+
SyncScopeMove,
|
|
16
|
+
type SyncScopeMoveDocument,
|
|
17
|
+
} from "./models";
|
|
18
|
+
import {findSyncEntryByCollectionTag, getSyncRegistry, type SyncRegistryEntry} from "./registry";
|
|
19
|
+
import {serializeSyncPayload} from "./serialize";
|
|
20
|
+
import {
|
|
21
|
+
getScopeField,
|
|
22
|
+
parseStreamKey,
|
|
23
|
+
resolveStreamForDoc,
|
|
24
|
+
resolveUserStreamsForEntry,
|
|
25
|
+
} from "./streams";
|
|
26
|
+
import {runSyncBatch, runSyncMutation} from "./syncBatch";
|
|
27
|
+
import type {
|
|
28
|
+
SyncEntitiesResponse,
|
|
29
|
+
SyncEntityPayload,
|
|
30
|
+
SyncMutateBatchRequest,
|
|
31
|
+
SyncMutateRequest,
|
|
32
|
+
SyncNackCode,
|
|
33
|
+
SyncSnapshotResponse,
|
|
34
|
+
SyncStreamInfo,
|
|
35
|
+
SyncStreamsResponse,
|
|
36
|
+
} from "./types";
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Maximum mutations per user per second on the HTTP mutate routes.
|
|
40
|
+
*
|
|
41
|
+
* Task 9.20: an alias for the shared budget — HTTP and socket mutations now draw from ONE
|
|
42
|
+
* per-user window (see `syncBatch.ts`), so there is no longer a separate HTTP allowance.
|
|
43
|
+
* Re-exported (rather than copied) so the two names can never drift; a live re-export also
|
|
44
|
+
* keeps this module's evaluation independent of `syncBatch`, which imports back into it.
|
|
45
|
+
*/
|
|
46
|
+
export {MAX_SYNC_MUTATIONS_PER_SECOND as MAX_SYNC_HTTP_MUTATIONS_PER_SECOND} from "./syncBatch";
|
|
47
|
+
|
|
48
|
+
/** Options for the SyncApp plugin's HTTP routes. */
|
|
49
|
+
export interface SyncAppOptions {
|
|
50
|
+
/**
|
|
51
|
+
* Resolve the scope values a user belongs to for tenant-scoped models
|
|
52
|
+
* (e.g. the user's organization ids). Required when any registered model uses a
|
|
53
|
+
* tenant scope.
|
|
54
|
+
*/
|
|
55
|
+
getUserScopes?: (user: User, entry: SyncRegistryEntry) => Promise<string[]> | string[];
|
|
56
|
+
/** Default page size for snapshots (default 100, max 100). */
|
|
57
|
+
defaultSnapshotLimit?: number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const MAX_SNAPSHOT_LIMIT = 100;
|
|
61
|
+
const DEFAULT_SNAPSHOT_LIMIT = 100;
|
|
62
|
+
|
|
63
|
+
/** HTTP status for each nack code returned by `POST /sync/mutate`. */
|
|
64
|
+
const NACK_HTTP_STATUS: Record<SyncNackCode, number> = {
|
|
65
|
+
conflict: 409,
|
|
66
|
+
error: 500,
|
|
67
|
+
rate_limited: 429,
|
|
68
|
+
unauthorized: 403,
|
|
69
|
+
validation: 422,
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Serialize a document for a sync payload through the fallback chain:
|
|
74
|
+
* sync responseHandler > modelRouter responseHandler > toJSON.
|
|
75
|
+
* Delegates to the shared `serializeSyncPayload` (also used for `sync:delta` emission).
|
|
76
|
+
*/
|
|
77
|
+
export const serializeSyncDoc = async ({
|
|
78
|
+
entry,
|
|
79
|
+
doc,
|
|
80
|
+
req,
|
|
81
|
+
}: {
|
|
82
|
+
entry: SyncRegistryEntry;
|
|
83
|
+
doc: mongoose.Document;
|
|
84
|
+
req: express.Request;
|
|
85
|
+
}): Promise<unknown> =>
|
|
86
|
+
serializeSyncPayload({doc: doc as unknown as Record<string, unknown>, entry, req});
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* C2: build the server-enforced scope filter for a SINGLE stream. The stream's scope
|
|
90
|
+
* value has already been verified against the user's membership set by the caller, so
|
|
91
|
+
* this filters to exactly that one value (`{field: value}`), never an `$in`.
|
|
92
|
+
*
|
|
93
|
+
* Custom-resolver scopes cannot be inverted into a query field, so they still route
|
|
94
|
+
* through the required `snapshotFilter` (parameterized by the user, as before).
|
|
95
|
+
*
|
|
96
|
+
* Task 9.19: for owner/tenant/broadcast scopes a consumer-supplied `snapshotFilter` used
|
|
97
|
+
* to be computed and then discarded — silently widening the snapshot past what the
|
|
98
|
+
* consumer asked for. It is now composed with the scope clause via `$and` (never
|
|
99
|
+
* spread-merged, which would let one clobber the other).
|
|
100
|
+
*/
|
|
101
|
+
export const buildSnapshotScopeFilter = ({
|
|
102
|
+
entry,
|
|
103
|
+
scopeValue,
|
|
104
|
+
snapshotFilterResult,
|
|
105
|
+
}: {
|
|
106
|
+
entry: SyncRegistryEntry;
|
|
107
|
+
scopeValue: string | null;
|
|
108
|
+
snapshotFilterResult?: Record<string, unknown>;
|
|
109
|
+
}): Record<string, unknown> => {
|
|
110
|
+
const {scope} = entry.config;
|
|
111
|
+
if (typeof scope === "function") {
|
|
112
|
+
// Custom scope: snapshotFilter is required at registration.
|
|
113
|
+
if (!snapshotFilterResult) {
|
|
114
|
+
throw new APIError({
|
|
115
|
+
status: 500,
|
|
116
|
+
title: `Sync collection ${entry.collectionTag} has a custom scope without a snapshotFilter`,
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
return snapshotFilterResult;
|
|
120
|
+
}
|
|
121
|
+
const scopeClause: Record<string, unknown> =
|
|
122
|
+
scope.type === "broadcast" ? {} : {[getScopeField(scope) as string]: scopeValue};
|
|
123
|
+
if (!snapshotFilterResult) {
|
|
124
|
+
return scopeClause;
|
|
125
|
+
}
|
|
126
|
+
return {$and: [scopeClause, snapshotFilterResult]};
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Task 9.19: resolve the modelRouter `queryFilter` for a sync read.
|
|
131
|
+
*
|
|
132
|
+
* The most common REST shape is `read: [IsAuthenticated]` plus a `queryFilter` (e.g.
|
|
133
|
+
* `OwnerQueryFilter`) doing the actual row-level scoping. Sync built only scope + seq
|
|
134
|
+
* clauses, so for a `broadcast`/`custom`-scoped collection every document in the stream
|
|
135
|
+
* leaked through the snapshot and `GET /sync/entities` even though the REST list endpoint
|
|
136
|
+
* hid it. Both read paths now `$and` the resolved filter into their query, giving sync and
|
|
137
|
+
* REST the same visible row set.
|
|
138
|
+
*
|
|
139
|
+
* `queryFilter` is called with an empty query object: a sync read carries no user-supplied
|
|
140
|
+
* query params, so there is nothing to validate — only the filter clause to apply.
|
|
141
|
+
* Returning `null` means "this caller may see nothing" (REST answers with an empty list),
|
|
142
|
+
* which sync mirrors by serving an empty page rather than erroring. A throwing filter is
|
|
143
|
+
* denied the same way the realtime path denies it: fail closed, never fall through open.
|
|
144
|
+
*/
|
|
145
|
+
const resolveSyncQueryFilter = async ({
|
|
146
|
+
entry,
|
|
147
|
+
user,
|
|
148
|
+
}: {
|
|
149
|
+
entry: SyncRegistryEntry;
|
|
150
|
+
user: User;
|
|
151
|
+
}): Promise<{denied: boolean; filter?: Record<string, unknown>}> => {
|
|
152
|
+
if (!entry.options.queryFilter) {
|
|
153
|
+
return {denied: false};
|
|
154
|
+
}
|
|
155
|
+
let resolved: Record<string, unknown> | null;
|
|
156
|
+
try {
|
|
157
|
+
resolved = await entry.options.queryFilter(user, {});
|
|
158
|
+
} catch (error: unknown) {
|
|
159
|
+
logger.error("[sync] queryFilter threw for a sync read; denying the read", {
|
|
160
|
+
collection: entry.collectionTag,
|
|
161
|
+
error: String(error),
|
|
162
|
+
userId: String(user.id),
|
|
163
|
+
});
|
|
164
|
+
return {denied: true};
|
|
165
|
+
}
|
|
166
|
+
if (resolved === null) {
|
|
167
|
+
return {denied: true};
|
|
168
|
+
}
|
|
169
|
+
return {denied: false, filter: Object.keys(resolved).length > 0 ? resolved : undefined};
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* `$and` the resolved `queryFilter` onto a sync read's scope filter. Spread-merging would
|
|
174
|
+
* let either side clobber a same-named key (e.g. both constraining `ownerId`), so both
|
|
175
|
+
* clauses are kept and both must match.
|
|
176
|
+
*/
|
|
177
|
+
const composeSyncReadFilter = ({
|
|
178
|
+
queryFilter,
|
|
179
|
+
scopeFilter,
|
|
180
|
+
}: {
|
|
181
|
+
queryFilter?: Record<string, unknown>;
|
|
182
|
+
scopeFilter: Record<string, unknown>;
|
|
183
|
+
}): Record<string, unknown> => {
|
|
184
|
+
if (!queryFilter) {
|
|
185
|
+
return scopeFilter;
|
|
186
|
+
}
|
|
187
|
+
if (Object.keys(scopeFilter).length === 0) {
|
|
188
|
+
return queryFilter;
|
|
189
|
+
}
|
|
190
|
+
return {$and: [scopeFilter, queryFilter]};
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Task 9.27: parse strictly. `Number.parseInt` stops at the first non-digit, so `"12abc"`
|
|
195
|
+
* and `"1e9"` used to silently become 12 and 1 — a client with a malformed cursor got a
|
|
196
|
+
* plausible-looking page instead of an error, and resumed paging from the wrong seq.
|
|
197
|
+
*/
|
|
198
|
+
const parseNonNegativeInt = (raw: unknown, name: string, fallback: number): number => {
|
|
199
|
+
if (raw === undefined) {
|
|
200
|
+
return fallback;
|
|
201
|
+
}
|
|
202
|
+
const text = String(raw).trim();
|
|
203
|
+
if (!/^\d+$/.test(text)) {
|
|
204
|
+
throw new APIError({status: 400, title: `Invalid ${name}: ${String(raw)}`});
|
|
205
|
+
}
|
|
206
|
+
const value = Number.parseInt(text, 10);
|
|
207
|
+
if (!Number.isSafeInteger(value)) {
|
|
208
|
+
throw new APIError({status: 400, title: `Invalid ${name}: ${String(raw)}`});
|
|
209
|
+
}
|
|
210
|
+
return value;
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Task 9.14: the legacy cursor is a raw `_id` string, so it must be compared in the
|
|
215
|
+
* model's OWN `_id` BSON type. Synced models are designed around client-minted string
|
|
216
|
+
* `_id`s: casting those to `ObjectId` either throws (non-hex ids) or — worse — silently
|
|
217
|
+
* compares across BSON types and matches nothing, so the legacy stratum reported itself
|
|
218
|
+
* exhausted after one page and the remaining documents were never bootstrapped.
|
|
219
|
+
* Only ObjectId-keyed models get the cast; a malformed cursor for one is a 400, not a 500.
|
|
220
|
+
*/
|
|
221
|
+
const legacyCursorClause = ({
|
|
222
|
+
model,
|
|
223
|
+
legacyCursorIn,
|
|
224
|
+
}: {
|
|
225
|
+
model: any;
|
|
226
|
+
legacyCursorIn: string;
|
|
227
|
+
}): Record<string, unknown> => {
|
|
228
|
+
const idInstance = (model.schema?.path("_id") as {instance?: string} | undefined)?.instance;
|
|
229
|
+
if (idInstance !== "ObjectId") {
|
|
230
|
+
return {_id: {$gt: legacyCursorIn}};
|
|
231
|
+
}
|
|
232
|
+
if (!mongoose.isValidObjectId(legacyCursorIn)) {
|
|
233
|
+
throw new APIError({status: 400, title: `Invalid legacyCursor: ${legacyCursorIn}`});
|
|
234
|
+
}
|
|
235
|
+
return {_id: {$gt: new mongoose.Types.ObjectId(legacyCursorIn)}};
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* C3: page the legacy (seq-0) stratum by `_id`. Legacy documents predate `syncPlugin`
|
|
240
|
+
* and carry no `_syncSeq` (or a literal 0). Returns a page + a forward `legacyCursor`
|
|
241
|
+
* while the stratum has more; returns `undefined` once the stratum is exhausted, at
|
|
242
|
+
* which point the caller switches to normal seq paging. Runs the same per-doc read
|
|
243
|
+
* check as the seq page (C6/M2).
|
|
244
|
+
*/
|
|
245
|
+
const pageLegacyStratum = async ({
|
|
246
|
+
model,
|
|
247
|
+
scopeFilter,
|
|
248
|
+
legacyCursorIn,
|
|
249
|
+
limit,
|
|
250
|
+
entry,
|
|
251
|
+
req,
|
|
252
|
+
}: {
|
|
253
|
+
model: any;
|
|
254
|
+
scopeFilter: Record<string, unknown>;
|
|
255
|
+
legacyCursorIn?: string;
|
|
256
|
+
limit: number;
|
|
257
|
+
entry: SyncRegistryEntry;
|
|
258
|
+
req: express.Request;
|
|
259
|
+
}): Promise<{entities: SyncEntityPayload[]; legacyCursor: string} | undefined> => {
|
|
260
|
+
const user = req.user as User | undefined;
|
|
261
|
+
// `deleted` MUST stay a TOP-LEVEL key so isDeletedPlugin does not re-inject its
|
|
262
|
+
// {deleted: {$ne: true}} exclusion (which only fires when the top-level filter has no
|
|
263
|
+
// `deleted` key) and hide legacy tombstones. See the seq-page query for the full note.
|
|
264
|
+
const legacyFilter: Record<string, unknown> = {
|
|
265
|
+
$and: [
|
|
266
|
+
scopeFilter,
|
|
267
|
+
{_syncSeq: {$in: [null, 0]}},
|
|
268
|
+
...(legacyCursorIn ? [legacyCursorClause({legacyCursorIn, model})] : []),
|
|
269
|
+
],
|
|
270
|
+
deleted: {$in: [true, false]},
|
|
271
|
+
};
|
|
272
|
+
const docs = await model
|
|
273
|
+
.find(legacyFilter)
|
|
274
|
+
.sort({_id: 1})
|
|
275
|
+
.limit(limit + 1);
|
|
276
|
+
if (docs.length === 0) {
|
|
277
|
+
// Stratum exhausted (or never had legacy docs) — caller proceeds by seq.
|
|
278
|
+
return undefined;
|
|
279
|
+
}
|
|
280
|
+
const page = docs.slice(0, limit);
|
|
281
|
+
const entities: SyncEntityPayload[] = [];
|
|
282
|
+
for (const doc of page) {
|
|
283
|
+
if (!(await checkPermissions("read", entry.options.permissions.read, user, doc))) {
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
entities.push({
|
|
287
|
+
data: await serializeSyncDoc({doc, entry, req}),
|
|
288
|
+
deleted: Boolean(doc.deleted),
|
|
289
|
+
id: String(doc._id),
|
|
290
|
+
seq: 0,
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
const lastId = String(page[page.length - 1]._id);
|
|
294
|
+
return {entities, legacyCursor: lastId};
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* C1: true when the stream's head (highest claimed seq) exceeds the stable frontier —
|
|
299
|
+
* i.e. committed seqs are still coming once the in-flight writes below the frontier land.
|
|
300
|
+
*/
|
|
301
|
+
const frontierBelowStreamHead = async (
|
|
302
|
+
streamKey: string,
|
|
303
|
+
frontierSeq: number
|
|
304
|
+
): Promise<boolean> => {
|
|
305
|
+
const counter = await SyncCounter.findOne({stream: streamKey}).select({seq: 1}).lean();
|
|
306
|
+
const head = counter ? ((counter as {seq?: number}).seq ?? 0) : 0;
|
|
307
|
+
return head > frontierSeq;
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Page the normal seq stratum: documents with `_syncSeq` in `(cursor, frontierSeq]`, unioned
|
|
312
|
+
* with the `SyncScopeMove` tombstones for the same seq range (C4). Mirrors
|
|
313
|
+
* `pageLegacyStratum` as the other half of the snapshot's paging: the caller drains the
|
|
314
|
+
* legacy stratum first, then hands every subsequent page to this helper.
|
|
315
|
+
*
|
|
316
|
+
* Returns the page's entities plus the cursor to resume from and whether more remains.
|
|
317
|
+
*/
|
|
318
|
+
const pageSeqStratum = async ({
|
|
319
|
+
model,
|
|
320
|
+
scopeFilter,
|
|
321
|
+
cursor,
|
|
322
|
+
limit,
|
|
323
|
+
frontierSeq,
|
|
324
|
+
streamKey,
|
|
325
|
+
entry,
|
|
326
|
+
req,
|
|
327
|
+
}: {
|
|
328
|
+
model: any;
|
|
329
|
+
scopeFilter: Record<string, unknown>;
|
|
330
|
+
cursor: number;
|
|
331
|
+
limit: number;
|
|
332
|
+
frontierSeq: number;
|
|
333
|
+
streamKey: string;
|
|
334
|
+
entry: SyncRegistryEntry;
|
|
335
|
+
req: express.Request;
|
|
336
|
+
}): Promise<{cursor: number; entities: SyncEntityPayload[]; hasMore: boolean}> => {
|
|
337
|
+
const user = req.user as User | undefined;
|
|
338
|
+
// C1: never page past the stable frontier — a cursor must not cross an uncommitted hole.
|
|
339
|
+
const seqFilter = {_syncSeq: {$gt: cursor, $lte: frontierSeq}};
|
|
340
|
+
// M1: compose the scope + seq clauses with $and (never spread-merge, which lets a
|
|
341
|
+
// scopeFilter $or clobber the seq clause). `deleted` MUST stay a TOP-LEVEL key:
|
|
342
|
+
// isDeletedPlugin injects {deleted: {$ne: true}} only when the top-level filter has
|
|
343
|
+
// no `deleted` key — burying it inside $and would let the plugin re-inject its
|
|
344
|
+
// exclusion and hide the tombstones catch-up depends on.
|
|
345
|
+
const query = {$and: [scopeFilter, seqFilter], deleted: {$in: [true, false]}};
|
|
346
|
+
const docs = await model
|
|
347
|
+
.find(query)
|
|
348
|
+
.sort({_syncSeq: 1})
|
|
349
|
+
.limit(limit + 1);
|
|
350
|
+
|
|
351
|
+
// C4: merge SyncScopeMove markers for THIS (old) stream into the page as tombstones,
|
|
352
|
+
// so an offline old-stream client learns the doc left its stream.
|
|
353
|
+
const markers = await SyncScopeMove.find({
|
|
354
|
+
fromStream: streamKey,
|
|
355
|
+
seq: {$gt: cursor, $lte: frontierSeq},
|
|
356
|
+
})
|
|
357
|
+
.sort({seq: 1})
|
|
358
|
+
.limit(limit + 1)
|
|
359
|
+
.lean();
|
|
360
|
+
|
|
361
|
+
const page = docs.slice(0, limit);
|
|
362
|
+
const markerPage = markers.slice(0, limit);
|
|
363
|
+
const docsHaveMore = docs.length > limit;
|
|
364
|
+
const markersHaveMore = markers.length > limit;
|
|
365
|
+
|
|
366
|
+
// The highest seq this page has fully examined. Docs and markers are fetched with
|
|
367
|
+
// independent limits, so coverage stops at the lower of the two: past that seq, the
|
|
368
|
+
// truncated side may still hold rows this page never saw. When a side was not
|
|
369
|
+
// truncated, everything up to the frontier is covered.
|
|
370
|
+
const lastDocSeq = page.length > 0 ? ((page[page.length - 1]._syncSeq as number) ?? 0) : 0;
|
|
371
|
+
const lastMarkerSeq = markerPage.length > 0 ? markerPage[markerPage.length - 1].seq : 0;
|
|
372
|
+
const coveredSeq = Math.min(
|
|
373
|
+
docsHaveMore ? lastDocSeq : frontierSeq,
|
|
374
|
+
markersHaveMore ? lastMarkerSeq : frontierSeq
|
|
375
|
+
);
|
|
376
|
+
|
|
377
|
+
// C6 (M2): run the same per-doc read permission the delta path uses; drop denied
|
|
378
|
+
// docs but still advance the cursor past them (parity with delta behavior).
|
|
379
|
+
const docEntities: SyncEntityPayload[] = [];
|
|
380
|
+
for (const doc of page as any[]) {
|
|
381
|
+
const allowed = await checkPermissions("read", entry.options.permissions.read, user, doc);
|
|
382
|
+
if (!allowed) {
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
385
|
+
const isTombstone = Boolean(doc.deleted);
|
|
386
|
+
docEntities.push({
|
|
387
|
+
// C7: tombstones carry no data (privacy + payload growth) — only id/seq/deleted.
|
|
388
|
+
data: isTombstone ? null : await serializeSyncDoc({doc, entry, req}),
|
|
389
|
+
deleted: isTombstone,
|
|
390
|
+
id: String(doc._id),
|
|
391
|
+
seq: doc._syncSeq ?? 0,
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
const markerEntities: SyncEntityPayload[] = markerPage.map(
|
|
395
|
+
(m: SyncScopeMoveDocument): SyncEntityPayload => ({
|
|
396
|
+
data: null,
|
|
397
|
+
deleted: true,
|
|
398
|
+
id: m.entityId,
|
|
399
|
+
seq: m.seq,
|
|
400
|
+
})
|
|
401
|
+
);
|
|
402
|
+
// Union doc page + marker tombstones, sort by seq, and drop anything beyond the
|
|
403
|
+
// covered range (it is re-fetched on the next page, in order).
|
|
404
|
+
const merged = [...docEntities, ...markerEntities]
|
|
405
|
+
.sort((a, b) => a.seq - b.seq)
|
|
406
|
+
.filter((entity) => entity.seq <= coveredSeq);
|
|
407
|
+
const entities = merged.slice(0, limit);
|
|
408
|
+
const entitiesTruncated = merged.length > entities.length;
|
|
409
|
+
|
|
410
|
+
// hasMore when: a full doc page was returned, extra markers remain, included
|
|
411
|
+
// entities were truncated by the limit, or the frontier sits below the head (more
|
|
412
|
+
// committed seqs are coming once in-flight writes land).
|
|
413
|
+
const frontierBelowHead = await frontierBelowStreamHead(streamKey, frontierSeq);
|
|
414
|
+
const hasMore = docsHaveMore || markersHaveMore || entitiesTruncated || frontierBelowHead;
|
|
415
|
+
|
|
416
|
+
// Advance the cursor over every seq this page CONSUMED, not just the entities it
|
|
417
|
+
// returned: a page whose docs were all read-denied returns nothing, and stalling the
|
|
418
|
+
// cursor there loops the client forever. Truncation by `limit` still pins the cursor
|
|
419
|
+
// to the last delivered entity so nothing undelivered is skipped, and the cursor
|
|
420
|
+
// never crosses the stable frontier (C1) nor moves backwards.
|
|
421
|
+
const advanceTo = entitiesTruncated ? entities[entities.length - 1].seq : coveredSeq;
|
|
422
|
+
return {cursor: Math.max(cursor, Math.min(advanceTo, frontierSeq)), entities, hasMore};
|
|
423
|
+
};
|
|
424
|
+
|
|
425
|
+
/** Hard cap on ids per `GET /sync/entities` request. */
|
|
426
|
+
export const MAX_ENTITY_FETCH = 100;
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Task 9.27: `_id: {$in: ids}` leaves the cast to Mongoose, so one non-hex id against an
|
|
430
|
+
* ObjectId-keyed model threw a CastError that surfaced as a 500 for what is purely bad
|
|
431
|
+
* caller input. Validate up front and name the offending ids in a 400. Models with
|
|
432
|
+
* client-minted string `_id`s (the syncdb default, see `legacyCursorClause`) take any shape.
|
|
433
|
+
*/
|
|
434
|
+
const castEntityIds = ({
|
|
435
|
+
model,
|
|
436
|
+
ids,
|
|
437
|
+
}: {
|
|
438
|
+
ids: string[];
|
|
439
|
+
model: any;
|
|
440
|
+
}): (string | mongoose.Types.ObjectId)[] => {
|
|
441
|
+
const idInstance = (model.schema?.path("_id") as {instance?: string} | undefined)?.instance;
|
|
442
|
+
if (idInstance !== "ObjectId") {
|
|
443
|
+
return ids;
|
|
444
|
+
}
|
|
445
|
+
const invalid = ids.filter((id) => !mongoose.isValidObjectId(id));
|
|
446
|
+
if (invalid.length > 0) {
|
|
447
|
+
throw new APIError({
|
|
448
|
+
status: 400,
|
|
449
|
+
title: `Invalid ids for ${String(model.modelName)}: ${invalid.join(",")}`,
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
return ids.map((id) => new mongoose.Types.ObjectId(id));
|
|
453
|
+
};
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* Mount the SyncDB HTTP routes:
|
|
457
|
+
* - GET /sync/streams — the authoritative set of streams the caller belongs to (C2)
|
|
458
|
+
* - GET /sync/snapshot — per-stream bootstrap/catch-up with server-enforced scoping
|
|
459
|
+
* - POST /sync/mutate — HTTP fallback mutation channel over applySyncMutation
|
|
460
|
+
* - GET /sync/key — per-user key material for the default encryption KeyProvider
|
|
461
|
+
*/
|
|
462
|
+
|
|
463
|
+
export const addSyncRoutes = (app: express.Application, options: SyncAppOptions = {}): void => {
|
|
464
|
+
const router = express.Router();
|
|
465
|
+
|
|
466
|
+
// C2: authoritative membership discovery. Runs against the full req.user (D2) so
|
|
467
|
+
// tenant memberships resolve from current organizationIds.
|
|
468
|
+
router.get(
|
|
469
|
+
"/sync/streams",
|
|
470
|
+
authenticateMiddleware(),
|
|
471
|
+
asyncHandler(async (req, res) => {
|
|
472
|
+
const user = req.user as User | undefined;
|
|
473
|
+
if (!user) {
|
|
474
|
+
throw new APIError({status: 401, title: "Authentication required"});
|
|
475
|
+
}
|
|
476
|
+
const streams: SyncStreamInfo[] = [];
|
|
477
|
+
for (const entry of getSyncRegistry()) {
|
|
478
|
+
if (!(await checkPermissions("list", entry.options.permissions.list, user))) {
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
try {
|
|
482
|
+
const entryStreams = await resolveUserStreamsForEntry({
|
|
483
|
+
entry,
|
|
484
|
+
getUserScopes: options.getUserScopes,
|
|
485
|
+
user,
|
|
486
|
+
});
|
|
487
|
+
for (const stream of entryStreams) {
|
|
488
|
+
streams.push({collection: entry.collectionTag, stream});
|
|
489
|
+
}
|
|
490
|
+
} catch (error: unknown) {
|
|
491
|
+
throw new APIError({
|
|
492
|
+
status: 500,
|
|
493
|
+
title: `Failed to resolve streams for ${entry.collectionTag}: ${String(error)}`,
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
const response: SyncStreamsResponse = {streams};
|
|
498
|
+
return res.json(response);
|
|
499
|
+
})
|
|
500
|
+
);
|
|
501
|
+
|
|
502
|
+
router.get(
|
|
503
|
+
"/sync/snapshot",
|
|
504
|
+
authenticateMiddleware(),
|
|
505
|
+
asyncHandler(async (req, res) => {
|
|
506
|
+
const user = req.user as User | undefined;
|
|
507
|
+
if (!user) {
|
|
508
|
+
throw new APIError({status: 401, title: "Authentication required"});
|
|
509
|
+
}
|
|
510
|
+
const streamKey = String(req.query.stream ?? "");
|
|
511
|
+
if (!streamKey) {
|
|
512
|
+
throw new APIError({status: 400, title: "stream query parameter is required"});
|
|
513
|
+
}
|
|
514
|
+
const parsed = parseStreamKey(streamKey);
|
|
515
|
+
if (!parsed) {
|
|
516
|
+
throw new APIError({status: 400, title: `Invalid stream key: ${streamKey}`});
|
|
517
|
+
}
|
|
518
|
+
const entry = findSyncEntryByCollectionTag(parsed.collectionTag);
|
|
519
|
+
if (!entry) {
|
|
520
|
+
throw new APIError({
|
|
521
|
+
status: 404,
|
|
522
|
+
title: `Unknown sync collection: ${parsed.collectionTag}`,
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
if (!(await checkPermissions("list", entry.options.permissions.list, user))) {
|
|
526
|
+
throw new APIError({
|
|
527
|
+
status: 403,
|
|
528
|
+
title: `Access to sync snapshot for ${parsed.collectionTag} denied for ${user.id}`,
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
// C2: a client must not snapshot a stream it does not belong to.
|
|
532
|
+
const memberStreams = await resolveUserStreamsForEntry({
|
|
533
|
+
entry,
|
|
534
|
+
getUserScopes: options.getUserScopes,
|
|
535
|
+
user,
|
|
536
|
+
});
|
|
537
|
+
if (!memberStreams.includes(streamKey)) {
|
|
538
|
+
throw new APIError({
|
|
539
|
+
status: 403,
|
|
540
|
+
title: `User ${user.id} does not belong to stream ${streamKey}`,
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
const cursor = parseNonNegativeInt(req.query.cursor, "cursor", 0);
|
|
545
|
+
const requestedLimit = parseNonNegativeInt(
|
|
546
|
+
req.query.limit,
|
|
547
|
+
"limit",
|
|
548
|
+
options.defaultSnapshotLimit ?? DEFAULT_SNAPSHOT_LIMIT
|
|
549
|
+
);
|
|
550
|
+
const limit = Math.min(Math.max(requestedLimit, 1), MAX_SNAPSHOT_LIMIT);
|
|
551
|
+
const legacyCursorIn =
|
|
552
|
+
typeof req.query.legacyCursor === "string" && req.query.legacyCursor.length > 0
|
|
553
|
+
? req.query.legacyCursor
|
|
554
|
+
: undefined;
|
|
555
|
+
|
|
556
|
+
const snapshotFilterResult = entry.config.snapshotFilter
|
|
557
|
+
? await entry.config.snapshotFilter({id: String(user.id)})
|
|
558
|
+
: undefined;
|
|
559
|
+
const queryFilterResult = await resolveSyncQueryFilter({entry, user});
|
|
560
|
+
const scopeFilter = composeSyncReadFilter({
|
|
561
|
+
queryFilter: queryFilterResult.filter,
|
|
562
|
+
scopeFilter: buildSnapshotScopeFilter({
|
|
563
|
+
entry,
|
|
564
|
+
scopeValue: parsed.scopeValue,
|
|
565
|
+
snapshotFilterResult,
|
|
566
|
+
}),
|
|
567
|
+
});
|
|
568
|
+
const model = mongoose.model(entry.modelName);
|
|
569
|
+
const frontierSeq = await computeStableFrontier({stream: streamKey});
|
|
570
|
+
// C7 (Task 9.15): the retention signal is the durable compaction watermark, not
|
|
571
|
+
// min(retained seq). The old computation was pinned low forever by any early doc that
|
|
572
|
+
// was never deleted, so a stale cursor sitting above a compacted tombstone's seq
|
|
573
|
+
// passed the check and silently never learned about that deletion.
|
|
574
|
+
const oldestRetainedSeq = await getCompactedThroughSeq({stream: streamKey});
|
|
575
|
+
|
|
576
|
+
// Task 9.19: a `queryFilter` that denies this caller means "no visible rows", exactly
|
|
577
|
+
// as REST's list endpoint answers. Serve a terminal empty page (cursor at the
|
|
578
|
+
// frontier, hasMore false) so the client settles instead of paging forever.
|
|
579
|
+
if (queryFilterResult.denied) {
|
|
580
|
+
const deniedResponse: SyncSnapshotResponse = {
|
|
581
|
+
cursor: frontierSeq,
|
|
582
|
+
entities: [],
|
|
583
|
+
frontierSeq,
|
|
584
|
+
hasMore: false,
|
|
585
|
+
oldestRetainedSeq,
|
|
586
|
+
stream: streamKey,
|
|
587
|
+
};
|
|
588
|
+
return res.json(deniedResponse);
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
// C3: legacy (seq-0) stratum, paged by _id. Drained fully before seq paging begins.
|
|
592
|
+
if (cursor === 0) {
|
|
593
|
+
const legacyResult = await pageLegacyStratum({
|
|
594
|
+
entry,
|
|
595
|
+
legacyCursorIn,
|
|
596
|
+
limit,
|
|
597
|
+
model,
|
|
598
|
+
req,
|
|
599
|
+
scopeFilter,
|
|
600
|
+
});
|
|
601
|
+
if (legacyResult) {
|
|
602
|
+
const response: SyncSnapshotResponse = {
|
|
603
|
+
cursor: 0,
|
|
604
|
+
entities: legacyResult.entities,
|
|
605
|
+
frontierSeq,
|
|
606
|
+
hasMore: true,
|
|
607
|
+
legacyCursor: legacyResult.legacyCursor,
|
|
608
|
+
oldestRetainedSeq,
|
|
609
|
+
stream: streamKey,
|
|
610
|
+
};
|
|
611
|
+
return res.json(response);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
const seqResult = await pageSeqStratum({
|
|
616
|
+
cursor,
|
|
617
|
+
entry,
|
|
618
|
+
frontierSeq,
|
|
619
|
+
limit,
|
|
620
|
+
model,
|
|
621
|
+
req,
|
|
622
|
+
scopeFilter,
|
|
623
|
+
streamKey,
|
|
624
|
+
});
|
|
625
|
+
const response: SyncSnapshotResponse = {
|
|
626
|
+
cursor: seqResult.cursor,
|
|
627
|
+
entities: seqResult.entities,
|
|
628
|
+
frontierSeq,
|
|
629
|
+
hasMore: seqResult.hasMore,
|
|
630
|
+
oldestRetainedSeq,
|
|
631
|
+
stream: streamKey,
|
|
632
|
+
};
|
|
633
|
+
return res.json(response);
|
|
634
|
+
})
|
|
635
|
+
);
|
|
636
|
+
|
|
637
|
+
router.get(
|
|
638
|
+
"/sync/entities",
|
|
639
|
+
authenticateMiddleware(),
|
|
640
|
+
asyncHandler(async (req, res) => {
|
|
641
|
+
const user = req.user as User | undefined;
|
|
642
|
+
if (!user) {
|
|
643
|
+
throw new APIError({status: 401, title: "Authentication required"});
|
|
644
|
+
}
|
|
645
|
+
const collection = String(req.query.collection ?? "").trim();
|
|
646
|
+
if (!collection) {
|
|
647
|
+
throw new APIError({status: 400, title: "collection query parameter is required"});
|
|
648
|
+
}
|
|
649
|
+
const idsParam = req.query.ids;
|
|
650
|
+
const ids =
|
|
651
|
+
typeof idsParam === "string"
|
|
652
|
+
? idsParam
|
|
653
|
+
.split(",")
|
|
654
|
+
.map((id) => id.trim())
|
|
655
|
+
.filter((id) => id.length > 0)
|
|
656
|
+
: [];
|
|
657
|
+
if (ids.length === 0) {
|
|
658
|
+
throw new APIError({status: 400, title: "ids query parameter is required"});
|
|
659
|
+
}
|
|
660
|
+
// Task 9.27: this used to `.slice(0, MAX_ENTITY_FETCH)`, so an over-cap repair fetch
|
|
661
|
+
// came back 200 with a partial body and the client treated the entities it never
|
|
662
|
+
// received as absent. Reject instead, so the caller re-requests in chunks.
|
|
663
|
+
if (ids.length > MAX_ENTITY_FETCH) {
|
|
664
|
+
throw new APIError({
|
|
665
|
+
status: 400,
|
|
666
|
+
title: `Too many ids: ${ids.length} requested, limit is ${MAX_ENTITY_FETCH}`,
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
const entry = findSyncEntryByCollectionTag(collection);
|
|
671
|
+
if (!entry) {
|
|
672
|
+
throw new APIError({
|
|
673
|
+
status: 404,
|
|
674
|
+
title: `Unknown sync collection: ${collection}`,
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
if (!(await checkPermissions("list", entry.options.permissions.list, user))) {
|
|
678
|
+
throw new APIError({
|
|
679
|
+
status: 403,
|
|
680
|
+
title: `Access to sync entities for ${collection} denied for ${user.id}`,
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
const memberStreams = await resolveUserStreamsForEntry({
|
|
685
|
+
entry,
|
|
686
|
+
getUserScopes: options.getUserScopes,
|
|
687
|
+
user,
|
|
688
|
+
});
|
|
689
|
+
const memberSet = new Set(memberStreams);
|
|
690
|
+
|
|
691
|
+
// Task 9.19: mirror the REST list endpoint's row-level scoping. Without this, a
|
|
692
|
+
// collection that relies on `queryFilter` (rather than a per-doc read permission)
|
|
693
|
+
// served every requested id here, whatever the caller was allowed to list.
|
|
694
|
+
const queryFilterResult = await resolveSyncQueryFilter({entry, user});
|
|
695
|
+
if (queryFilterResult.denied) {
|
|
696
|
+
const deniedResponse: SyncEntitiesResponse = {entities: []};
|
|
697
|
+
return res.json(deniedResponse);
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
const model = mongoose.model(entry.modelName);
|
|
701
|
+
const castIds = castEntityIds({ids, model});
|
|
702
|
+
const docs = await model.find({
|
|
703
|
+
// `deleted` MUST stay a TOP-LEVEL key so isDeletedPlugin does not re-inject its
|
|
704
|
+
// {deleted: {$ne: true}} exclusion and hide the tombstones a re-fetching client needs.
|
|
705
|
+
...composeSyncReadFilter({
|
|
706
|
+
queryFilter: queryFilterResult.filter,
|
|
707
|
+
scopeFilter: {_id: {$in: castIds}},
|
|
708
|
+
}),
|
|
709
|
+
deleted: {$in: [true, false]},
|
|
710
|
+
});
|
|
711
|
+
|
|
712
|
+
const entities: SyncEntityPayload[] = [];
|
|
713
|
+
for (const doc of docs as mongoose.Document[]) {
|
|
714
|
+
const allowed = await checkPermissions("read", entry.options.permissions.read, user, doc);
|
|
715
|
+
if (!allowed) {
|
|
716
|
+
continue;
|
|
717
|
+
}
|
|
718
|
+
const docObj = doc.toObject() as Record<string, unknown>;
|
|
719
|
+
const stream = resolveStreamForDoc({
|
|
720
|
+
collectionTag: entry.collectionTag,
|
|
721
|
+
doc: docObj,
|
|
722
|
+
scope: entry.config.scope,
|
|
723
|
+
});
|
|
724
|
+
if (!memberSet.has(stream)) {
|
|
725
|
+
continue;
|
|
726
|
+
}
|
|
727
|
+
const isTombstone = Boolean(docObj.deleted);
|
|
728
|
+
entities.push({
|
|
729
|
+
data: isTombstone ? null : await serializeSyncDoc({doc, entry, req}),
|
|
730
|
+
deleted: isTombstone,
|
|
731
|
+
id: String(doc._id),
|
|
732
|
+
seq: (docObj._syncSeq as number | undefined) ?? 0,
|
|
733
|
+
});
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
const response: SyncEntitiesResponse = {entities};
|
|
737
|
+
return res.json(response);
|
|
738
|
+
})
|
|
739
|
+
);
|
|
740
|
+
|
|
741
|
+
router.post(
|
|
742
|
+
"/sync/mutate",
|
|
743
|
+
authenticateMiddleware(),
|
|
744
|
+
asyncHandler(async (req, res) => {
|
|
745
|
+
const user = req.user as User | undefined;
|
|
746
|
+
if (!user) {
|
|
747
|
+
throw new APIError({status: 401, title: "Authentication required"});
|
|
748
|
+
}
|
|
749
|
+
// Task 9.20: rate limiting + apply live in the shared runner, so this route and
|
|
750
|
+
// `sync:mutate` behave identically under the same inputs.
|
|
751
|
+
const {outcome} = await runSyncMutation({
|
|
752
|
+
mutation: req.body as SyncMutateRequest,
|
|
753
|
+
req,
|
|
754
|
+
scopeResolver: options.getUserScopes,
|
|
755
|
+
user,
|
|
756
|
+
});
|
|
757
|
+
if (outcome.type === "ack") {
|
|
758
|
+
return res.json({ack: outcome.ack});
|
|
759
|
+
}
|
|
760
|
+
// Duplicate deliveries reading a recorded outcome map to the same statuses.
|
|
761
|
+
return res.status(NACK_HTTP_STATUS[outcome.nack.code]).json({nack: outcome.nack});
|
|
762
|
+
})
|
|
763
|
+
);
|
|
764
|
+
|
|
765
|
+
router.post(
|
|
766
|
+
"/sync/mutate/batch",
|
|
767
|
+
authenticateMiddleware(),
|
|
768
|
+
asyncHandler(async (req, res) => {
|
|
769
|
+
const user = req.user as User | undefined;
|
|
770
|
+
if (!user) {
|
|
771
|
+
throw new APIError({status: 401, title: "Authentication required"});
|
|
772
|
+
}
|
|
773
|
+
const body = req.body as SyncMutateBatchRequest | undefined;
|
|
774
|
+
const mutations = Array.isArray(body?.mutations) ? body.mutations : [];
|
|
775
|
+
|
|
776
|
+
// Task 9.20: validate -> rate limit -> apply, shared with `sync:mutateBatch`. An
|
|
777
|
+
// invalid batch (oversized, duplicate ids) is rejected before any budget is charged.
|
|
778
|
+
const {response, stage} = await runSyncBatch({
|
|
779
|
+
mutations,
|
|
780
|
+
req,
|
|
781
|
+
scopeResolver: options.getUserScopes,
|
|
782
|
+
user,
|
|
783
|
+
});
|
|
784
|
+
if (stage === "validation") {
|
|
785
|
+
return res.status(422).json(response);
|
|
786
|
+
}
|
|
787
|
+
if (stage === "rate_limited") {
|
|
788
|
+
return res.status(NACK_HTTP_STATUS.rate_limited).json(response);
|
|
789
|
+
}
|
|
790
|
+
return res.json(response);
|
|
791
|
+
})
|
|
792
|
+
);
|
|
793
|
+
|
|
794
|
+
router.get(
|
|
795
|
+
"/sync/key",
|
|
796
|
+
authenticateMiddleware(),
|
|
797
|
+
asyncHandler(async (req, res) => {
|
|
798
|
+
const user = req.user as User | undefined;
|
|
799
|
+
if (!user) {
|
|
800
|
+
throw new APIError({status: 401, title: "Authentication required"});
|
|
801
|
+
}
|
|
802
|
+
const keyMaterial = await getOrCreateSyncKeyMaterial({userId: String(user.id)});
|
|
803
|
+
return res.json({keyMaterial});
|
|
804
|
+
})
|
|
805
|
+
);
|
|
806
|
+
|
|
807
|
+
// Authenticate middleware rejects with Error("Unauthorized"); convert that to
|
|
808
|
+
// 401 before the generic error handler (and before Express's default 500).
|
|
809
|
+
router.use(apiUnauthorizedMiddleware);
|
|
810
|
+
router.use(apiErrorMiddleware);
|
|
811
|
+
app.use(router);
|
|
812
|
+
};
|