@terreno/api 57.0.0 → 57.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ai/skills/mongoose-schema-safety/SKILL.md +8 -0
- package/dist/actions.d.ts +7 -2
- package/dist/actions.js +60 -25
- package/dist/adminModelRouter.d.ts +14 -0
- package/dist/adminModelRouter.js +199 -0
- package/dist/adminTypes.d.ts +182 -0
- package/dist/adminTypes.js +5 -0
- package/dist/api.d.ts +44 -3
- package/dist/api.js +74 -26
- package/dist/auth.d.ts +18 -1
- package/dist/auth.js +122 -20
- package/dist/consentApp.d.ts +2 -0
- package/dist/consentApp.js +98 -0
- package/dist/index.d.ts +22 -1
- package/dist/index.js +51 -7
- package/dist/mcp/auth.d.ts +11 -0
- package/dist/mcp/auth.js +133 -0
- package/dist/mcp/createMCPRequest.d.ts +16 -0
- package/dist/mcp/createMCPRequest.js +26 -0
- package/dist/mcp/handlers.d.ts +8 -0
- package/dist/mcp/handlers.js +793 -0
- package/dist/mcp/query.d.ts +32 -0
- package/dist/mcp/query.js +232 -0
- package/dist/mcp/registry.d.ts +21 -0
- package/dist/mcp/registry.js +54 -0
- package/dist/mcp/schemaGenerator.d.ts +14 -0
- package/dist/mcp/schemaGenerator.js +379 -0
- package/dist/mcp/server.d.ts +8 -0
- package/dist/mcp/server.js +133 -0
- package/dist/mcp/toolGenerator.d.ts +14 -0
- package/dist/mcp/toolGenerator.js +246 -0
- package/dist/mcp/types.d.ts +85 -0
- package/dist/mcp/types.js +2 -0
- package/dist/openApi.d.ts +1 -1
- package/dist/openApi.js +26 -8
- package/dist/permissions.js +15 -15
- package/dist/rbac/access.d.ts +3 -0
- package/dist/rbac/access.js +294 -0
- package/dist/rbac/auditModel.d.ts +31 -0
- package/dist/rbac/auditModel.js +112 -0
- package/dist/rbac/backfillAdmins.d.ts +18 -0
- package/dist/rbac/backfillAdmins.js +139 -0
- package/dist/rbac/fieldViews.d.ts +3 -0
- package/dist/rbac/fieldViews.js +201 -0
- package/dist/rbac/middleware.d.ts +21 -0
- package/dist/rbac/middleware.js +108 -0
- package/dist/rbac/modelRouterAccess.d.ts +42 -0
- package/dist/rbac/modelRouterAccess.js +388 -0
- package/dist/rbac/permissionUtils.d.ts +8 -0
- package/dist/rbac/permissionUtils.js +209 -0
- package/dist/rbac/realtimeAccess.d.ts +18 -0
- package/dist/rbac/realtimeAccess.js +173 -0
- package/dist/rbac/resolve.d.ts +28 -0
- package/dist/rbac/resolve.js +387 -0
- package/dist/rbac/roleManager.d.ts +20 -0
- package/dist/rbac/roleManager.js +741 -0
- package/dist/rbac/roleModel.d.ts +42 -0
- package/dist/rbac/roleModel.js +267 -0
- package/dist/rbac/routes.d.ts +9 -0
- package/dist/rbac/routes.js +211 -0
- package/dist/rbac/scopes.d.ts +15 -0
- package/dist/rbac/scopes.js +155 -0
- package/dist/rbac/statements.d.ts +30 -0
- package/dist/rbac/statements.js +126 -0
- package/dist/rbac/types.d.ts +202 -0
- package/dist/rbac/types.js +2 -0
- package/dist/rbac/userPlugin.d.ts +12 -0
- package/dist/rbac/userPlugin.js +41 -0
- package/dist/realtime/changeStreamWatcher.d.ts +1 -1
- package/dist/realtime/changeStreamWatcher.js +48 -39
- package/dist/realtime/realtimeApp.js +4 -8
- package/dist/realtime/registry.d.ts +5 -0
- package/dist/realtime/registry.js +13 -1
- package/dist/scrubAdminFields.d.ts +17 -0
- package/dist/scrubAdminFields.js +192 -0
- package/dist/sync/executors.d.ts +1 -1
- package/dist/sync/executors.js +63 -40
- package/dist/sync/loadHarness.js +4 -5
- package/dist/sync/registry.d.ts +5 -0
- package/dist/sync/registry.js +13 -1
- package/dist/sync/routes.js +41 -36
- package/dist/sync/serialize.js +5 -6
- package/dist/terrenoApp.d.ts +18 -0
- package/dist/terrenoApp.js +46 -3
- package/dist/terrenoPlugin.d.ts +8 -1
- package/dist/tests/createTestData.js +8 -0
- package/dist/tests/models.d.ts +1 -0
- package/dist/tests/models.js +5 -0
- package/dist/tests/testHelper.js +1 -2
- package/dist/transformers.js +16 -1
- package/package.json +17 -13
- package/src/__snapshots__/openApi.test.ts.snap +192 -0
- package/src/actions.test.ts +194 -1
- package/src/actions.ts +65 -8
- package/src/adminModelRouter.ts +154 -0
- package/src/adminTypes.ts +206 -0
- package/src/api.test.ts +66 -1
- package/src/api.ts +131 -12
- package/src/auth.test.ts +135 -1
- package/src/auth.ts +81 -3
- package/src/consentApp.test.ts +24 -0
- package/src/consentApp.ts +100 -0
- package/src/index.ts +56 -0
- package/src/mcp/auth.test.ts +197 -0
- package/src/mcp/auth.ts +100 -0
- package/src/mcp/createMCPRequest.test.ts +47 -0
- package/src/mcp/createMCPRequest.ts +32 -0
- package/src/mcp/customTools.test.ts +54 -0
- package/src/mcp/handlers.ts +639 -0
- package/src/mcp/integration.test.ts +1493 -0
- package/src/mcp/query.test.ts +172 -0
- package/src/mcp/query.ts +179 -0
- package/src/mcp/registry.ts +70 -0
- package/src/mcp/schemaGenerator.test.ts +182 -0
- package/src/mcp/schemaGenerator.ts +354 -0
- package/src/mcp/server.test.ts +226 -0
- package/src/mcp/server.ts +86 -0
- package/src/mcp/toolGenerator.test.ts +284 -0
- package/src/mcp/toolGenerator.ts +199 -0
- package/src/mcp/types.ts +95 -0
- package/src/openApi.test.ts +107 -0
- package/src/openApi.ts +32 -1
- package/src/permissions.ts +3 -2
- package/src/rbac/access.test.ts +357 -0
- package/src/rbac/access.ts +201 -0
- package/src/rbac/auditModel.ts +103 -0
- package/src/rbac/backfillAdmins.ts +54 -0
- package/src/rbac/fieldViews.test.ts +39 -0
- package/src/rbac/fieldViews.ts +122 -0
- package/src/rbac/middleware.ts +55 -0
- package/src/rbac/modelRouterAccess.test.ts +336 -0
- package/src/rbac/modelRouterAccess.ts +331 -0
- package/src/rbac/permissionUtils.test.ts +112 -0
- package/src/rbac/permissionUtils.ts +83 -0
- package/src/rbac/realtimeAccess.test.ts +242 -0
- package/src/rbac/realtimeAccess.ts +138 -0
- package/src/rbac/resolve.test.ts +358 -0
- package/src/rbac/resolve.ts +285 -0
- package/src/rbac/roleManager.test.ts +915 -0
- package/src/rbac/roleManager.ts +440 -0
- package/src/rbac/roleModel.test.ts +114 -0
- package/src/rbac/roleModel.ts +196 -0
- package/src/rbac/routes.ts +170 -0
- package/src/rbac/scopes.test.ts +105 -0
- package/src/rbac/scopes.ts +81 -0
- package/src/rbac/statements.test.ts +67 -0
- package/src/rbac/statements.ts +73 -0
- package/src/rbac/types.ts +198 -0
- package/src/rbac/userPlugin.test.ts +55 -0
- package/src/rbac/userPlugin.ts +25 -0
- package/src/realtime/changeStreamWatcher.ts +20 -15
- package/src/realtime/realtime.test.ts +25 -1
- package/src/realtime/realtimeApp.ts +2 -5
- package/src/realtime/registry.ts +15 -0
- package/src/scrubAdminFields.test.ts +113 -0
- package/src/scrubAdminFields.ts +158 -0
- package/src/sync/executors.ts +31 -1
- package/src/sync/loadHarness.ts +8 -7
- package/src/sync/models.test.ts +262 -0
- package/src/sync/registry.ts +15 -0
- package/src/sync/routes.ts +25 -11
- package/src/sync/scripts/compactTombstones.test.ts +4 -6
- package/src/sync/serialize.ts +17 -9
- package/src/sync/sync.test.ts +52 -0
- package/src/sync/syncRoutes.test.ts +5 -2
- package/src/sync/syncSocket.test.ts +28 -0
- package/src/terrenoApp.test.ts +100 -0
- package/src/terrenoApp.ts +65 -3
- package/src/terrenoPlugin.ts +10 -1
- package/src/tests/bunSetup.ts +2 -0
- package/src/tests/createTestData.ts +5 -1
- package/src/tests/models.ts +6 -0
- package/src/tests/testHelper.ts +1 -3
- package/src/transformers.test.ts +12 -0
- package/src/transformers.ts +7 -1
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import type {Schema, SchemaType} from "mongoose";
|
|
2
|
+
|
|
3
|
+
import type {AdminConfig, AdminModelAdminMap} from "./adminTypes";
|
|
4
|
+
|
|
5
|
+
const toPlainObject = (value: unknown): Record<string, unknown> => {
|
|
6
|
+
if (
|
|
7
|
+
value &&
|
|
8
|
+
typeof value === "object" &&
|
|
9
|
+
"toObject" in value &&
|
|
10
|
+
typeof (value as {toObject?: unknown}).toObject === "function"
|
|
11
|
+
) {
|
|
12
|
+
return (value as {toObject: () => Record<string, unknown>}).toObject();
|
|
13
|
+
}
|
|
14
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
15
|
+
return value as Record<string, unknown>;
|
|
16
|
+
}
|
|
17
|
+
return {};
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const adminFieldsToStrip = (admin: AdminConfig): Set<string> => {
|
|
21
|
+
return new Set([...(admin.excludeFields ?? []), ...(admin.hiddenFields ?? [])]);
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
const getRefModelName = (schemaPath: SchemaType | undefined): string | undefined => {
|
|
25
|
+
if (!schemaPath) {
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
const options = schemaPath.options as {ref?: string} | undefined;
|
|
29
|
+
if (typeof options?.ref === "string" && options.ref.length > 0) {
|
|
30
|
+
return options.ref;
|
|
31
|
+
}
|
|
32
|
+
return undefined;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const isPopulatedDocument = (value: unknown): value is Record<string, unknown> => {
|
|
36
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
const plain = value as Record<string, unknown>;
|
|
40
|
+
return plain._id != null && Object.keys(plain).length > 1;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const scrubPlainObject = (
|
|
44
|
+
plain: Record<string, unknown>,
|
|
45
|
+
admin: AdminConfig,
|
|
46
|
+
schema: Schema | undefined,
|
|
47
|
+
allModelAdmins: AdminModelAdminMap | undefined
|
|
48
|
+
): Record<string, unknown> => {
|
|
49
|
+
const strip = adminFieldsToStrip(admin);
|
|
50
|
+
const next: Record<string, unknown> = {};
|
|
51
|
+
|
|
52
|
+
for (const [key, fieldValue] of Object.entries(plain)) {
|
|
53
|
+
if (strip.has(key)) {
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (fieldValue == null) {
|
|
58
|
+
next[key] = fieldValue;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (Array.isArray(fieldValue)) {
|
|
63
|
+
next[key] = fieldValue.map((item) =>
|
|
64
|
+
scrubAdminFields(item, {admin, allModelAdmins, fieldName: key, schema})
|
|
65
|
+
);
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (isPopulatedDocument(fieldValue) && schema) {
|
|
70
|
+
const refModelName = getRefModelName(schema.path(key));
|
|
71
|
+
const refAdmin = refModelName ? allModelAdmins?.[refModelName] : undefined;
|
|
72
|
+
if (refAdmin) {
|
|
73
|
+
next[key] = scrubAdminFields(fieldValue, {
|
|
74
|
+
admin: refAdmin,
|
|
75
|
+
allModelAdmins,
|
|
76
|
+
schema: undefined,
|
|
77
|
+
});
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
next[key] = fieldValue;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return next;
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
export interface ScrubAdminFieldsParams {
|
|
89
|
+
admin: AdminConfig;
|
|
90
|
+
allModelAdmins?: AdminModelAdminMap;
|
|
91
|
+
fieldName?: string;
|
|
92
|
+
schema?: Schema;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Removes `excludeFields` and `hiddenFields` from a document (or nested value).
|
|
97
|
+
* Populated refs are scrubbed using the referenced model's admin config when present
|
|
98
|
+
* in `allModelAdmins` (keyed by Mongoose model name).
|
|
99
|
+
*/
|
|
100
|
+
export const scrubAdminFields = (value: unknown, params: ScrubAdminFieldsParams): unknown => {
|
|
101
|
+
if (value == null) {
|
|
102
|
+
return value;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (Array.isArray(value)) {
|
|
106
|
+
return value.map((item) => scrubAdminFields(item, params));
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (typeof value !== "object") {
|
|
110
|
+
return value;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const plain = toPlainObject(value);
|
|
114
|
+
return scrubPlainObject(plain, params.admin, params.schema, params.allModelAdmins);
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
/** Keys stripped from POST/PATCH bodies when `admin` is set on modelRouter. */
|
|
118
|
+
export const adminBodyFieldsToStrip = (admin: AdminConfig): Set<string> => {
|
|
119
|
+
return new Set([
|
|
120
|
+
...(admin.readonlyFields ?? []),
|
|
121
|
+
...(admin.excludeFields ?? []),
|
|
122
|
+
...(admin.hiddenFields ?? []),
|
|
123
|
+
]);
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
export const stripAdminBodyFields = <T extends Record<string, unknown>>(
|
|
127
|
+
body: T | T[] | null | undefined,
|
|
128
|
+
admin: AdminConfig
|
|
129
|
+
): T | T[] | null | undefined => {
|
|
130
|
+
if (body == null) {
|
|
131
|
+
return body;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const strip = adminBodyFieldsToStrip(admin);
|
|
135
|
+
|
|
136
|
+
const stripObject = (obj: T): T => {
|
|
137
|
+
const next = {...obj};
|
|
138
|
+
for (const key of strip) {
|
|
139
|
+
delete next[key];
|
|
140
|
+
}
|
|
141
|
+
return next;
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
if (Array.isArray(body)) {
|
|
145
|
+
return body.map((item) => {
|
|
146
|
+
if (!item || typeof item !== "object") {
|
|
147
|
+
return item;
|
|
148
|
+
}
|
|
149
|
+
return stripObject(item);
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (typeof body !== "object") {
|
|
154
|
+
return body;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return stripObject(body);
|
|
158
|
+
};
|
package/src/sync/executors.ts
CHANGED
|
@@ -20,7 +20,7 @@ import {DateTime} from "luxon";
|
|
|
20
20
|
import type {Document, Model} from "mongoose";
|
|
21
21
|
|
|
22
22
|
import {addPopulateToQuery, type ModelRouterOptions} from "../api";
|
|
23
|
-
import type
|
|
23
|
+
import {omitUserRolesFromWriteBody, type User} from "../auth";
|
|
24
24
|
import {loadDocOr404} from "../docLoader";
|
|
25
25
|
import {
|
|
26
26
|
APIError,
|
|
@@ -33,6 +33,7 @@ import {
|
|
|
33
33
|
passthroughOrWrapWrite,
|
|
34
34
|
} from "../errors";
|
|
35
35
|
import {checkPermissions} from "../permissions";
|
|
36
|
+
import {validateAccessWritePayload} from "../rbac/modelRouterAccess";
|
|
36
37
|
import {transform} from "../transformers";
|
|
37
38
|
|
|
38
39
|
/** A hydrated mongoose document joined with the router's base type. */
|
|
@@ -243,6 +244,13 @@ export const executeCreate = async <T>({
|
|
|
243
244
|
});
|
|
244
245
|
}
|
|
245
246
|
}
|
|
247
|
+
cleanedBody = omitUserRolesFromWriteBody(
|
|
248
|
+
model.modelName,
|
|
249
|
+
options.accessControl,
|
|
250
|
+
cleanedBody,
|
|
251
|
+
(request as express.Request & {terrenoAllowUserAdminWrite?: boolean})
|
|
252
|
+
.terrenoAllowUserAdminWrite === true
|
|
253
|
+
) as typeof cleanedBody;
|
|
246
254
|
if (cleanedBody === undefined) {
|
|
247
255
|
throw new BadRequestError({
|
|
248
256
|
code: "invalid-request-body",
|
|
@@ -250,6 +258,12 @@ export const executeCreate = async <T>({
|
|
|
250
258
|
title: "Invalid request body",
|
|
251
259
|
});
|
|
252
260
|
}
|
|
261
|
+
await validateAccessWritePayload({
|
|
262
|
+
body: cleanedBody,
|
|
263
|
+
options,
|
|
264
|
+
phase: "create",
|
|
265
|
+
user,
|
|
266
|
+
});
|
|
253
267
|
let data: ExecutorDoc<T>;
|
|
254
268
|
try {
|
|
255
269
|
data = (await model.create(cleanedBody as T)) as ExecutorDoc<T>;
|
|
@@ -413,6 +427,22 @@ export const executeUpdate = async <T>({
|
|
|
413
427
|
}
|
|
414
428
|
}
|
|
415
429
|
|
|
430
|
+
cleanedBody = omitUserRolesFromWriteBody(
|
|
431
|
+
model.modelName,
|
|
432
|
+
options.accessControl,
|
|
433
|
+
cleanedBody,
|
|
434
|
+
(request as express.Request & {terrenoAllowUserAdminWrite?: boolean})
|
|
435
|
+
.terrenoAllowUserAdminWrite === true
|
|
436
|
+
) as typeof cleanedBody;
|
|
437
|
+
|
|
438
|
+
await validateAccessWritePayload({
|
|
439
|
+
body: cleanedBody,
|
|
440
|
+
doc,
|
|
441
|
+
options,
|
|
442
|
+
phase: "write",
|
|
443
|
+
user,
|
|
444
|
+
});
|
|
445
|
+
|
|
416
446
|
// Conflict detection runs after preUpdate so that unauthorized mutations
|
|
417
447
|
// are rejected before we leak document data in a conflict response.
|
|
418
448
|
if (concurrencyCheck?.type === "timestamp") {
|
package/src/sync/loadHarness.ts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// noExplicitAny: the harness bridges generic model/router/client types, mirroring integration.test.ts
|
|
2
|
-
// biome-ignore-all lint/suspicious/noExplicitAny: the harness bridges generic model/router/client types, mirroring integration.test.ts
|
|
3
1
|
/**
|
|
4
2
|
* This is a manual load-generation script — NOT run by `bun test`. Invoke via
|
|
5
3
|
* `bun run api:load` (see root package.json) or directly:
|
|
@@ -42,7 +40,7 @@ import mongoose, {model, Schema} from "mongoose";
|
|
|
42
40
|
import passportLocalMongoose from "passport-local-mongoose";
|
|
43
41
|
|
|
44
42
|
import {modelRouter} from "../api";
|
|
45
|
-
import {addAuthRoutes, generateTokens, setupAuth} from "../auth";
|
|
43
|
+
import {type UserModel as AuthUserModel, addAuthRoutes, generateTokens, setupAuth} from "../auth";
|
|
46
44
|
import {logger} from "../logger";
|
|
47
45
|
import {Permissions} from "../permissions";
|
|
48
46
|
import {createdUpdatedPlugin, type IsDeleted, isDeletedPlugin} from "../plugins";
|
|
@@ -254,13 +252,16 @@ const buildRig = async (
|
|
|
254
252
|
|
|
255
253
|
const app = express();
|
|
256
254
|
app.use(express.json());
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
255
|
+
// passport-local-mongoose adds createStrategy/serializeUser at runtime, so the compiled
|
|
256
|
+
// model's static type does not carry them — bridge to the auth package's UserModel shape.
|
|
257
|
+
const authUserModel = UserModel as unknown as AuthUserModel;
|
|
258
|
+
setupAuth(app, authUserModel);
|
|
259
|
+
addAuthRoutes(app, authUserModel);
|
|
260
|
+
new SyncApp({}).register(app);
|
|
260
261
|
app.use(registration.path, registration.router);
|
|
261
262
|
|
|
262
263
|
const realtimeApp = new RealtimeApp({});
|
|
263
|
-
realtimeApp.register(app
|
|
264
|
+
realtimeApp.register(app);
|
|
264
265
|
|
|
265
266
|
const httpServer = createServer(app);
|
|
266
267
|
await new Promise<void>((resolve) => {
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import {beforeAll, beforeEach, describe, it} from "bun:test";
|
|
2
|
+
import {assert} from "chai";
|
|
3
|
+
import mongoose from "mongoose";
|
|
4
|
+
import {setupDb} from "../tests";
|
|
5
|
+
import {
|
|
6
|
+
claimSyncSeqs,
|
|
7
|
+
computeStableFrontier,
|
|
8
|
+
confirmSyncSeqs,
|
|
9
|
+
ensureSyncModelIndexes,
|
|
10
|
+
getOrCreateSyncKeyMaterial,
|
|
11
|
+
PENDING_CLAIM_LEASE_MS,
|
|
12
|
+
recordCompactedThroughSeq,
|
|
13
|
+
releaseSyncSeqs,
|
|
14
|
+
SyncCounter,
|
|
15
|
+
SyncScopeMove,
|
|
16
|
+
} from "./models";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Coverage for the sync bookkeeping models' branches that the protocol suites never
|
|
20
|
+
* reach: the session-backed claim fast path, the upsert-race retry, the empty-seq
|
|
21
|
+
* short-circuits in confirm/release, the `seq <= 0` watermark guard, and the legacy
|
|
22
|
+
* TTL-index drop (both the found and the failing branch).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
type LooseFindOneAndUpdate = (
|
|
26
|
+
filter: Record<string, unknown>,
|
|
27
|
+
update: unknown,
|
|
28
|
+
options?: Record<string, unknown>
|
|
29
|
+
) => Promise<unknown>;
|
|
30
|
+
|
|
31
|
+
type LooseUpdateOne = (
|
|
32
|
+
filter: Record<string, unknown>,
|
|
33
|
+
update: unknown,
|
|
34
|
+
options?: Record<string, unknown>
|
|
35
|
+
) => Promise<unknown>;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Replace `SyncCounter.findOneAndUpdate` with `stub`. Returns a restore function and the
|
|
39
|
+
* original, bound implementation so a stub can delegate to the real claim.
|
|
40
|
+
*/
|
|
41
|
+
const stubFindOneAndUpdate = (
|
|
42
|
+
build: (original: LooseFindOneAndUpdate) => LooseFindOneAndUpdate
|
|
43
|
+
): (() => void) => {
|
|
44
|
+
const original = SyncCounter.findOneAndUpdate.bind(
|
|
45
|
+
SyncCounter
|
|
46
|
+
) as unknown as LooseFindOneAndUpdate;
|
|
47
|
+
Object.assign(SyncCounter, {
|
|
48
|
+
findOneAndUpdate: build(original) as unknown as typeof SyncCounter.findOneAndUpdate,
|
|
49
|
+
});
|
|
50
|
+
return () => {
|
|
51
|
+
Object.assign(SyncCounter, {findOneAndUpdate: original});
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
describe("claimSyncSeqs", () => {
|
|
56
|
+
beforeAll(async () => {
|
|
57
|
+
await setupDb();
|
|
58
|
+
await SyncCounter.ensureIndexes();
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
beforeEach(async () => {
|
|
62
|
+
await SyncCounter.deleteMany({});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it("skips the pending registry for a session-backed claim", async () => {
|
|
66
|
+
const stream = "models-session|owner:1";
|
|
67
|
+
const session = await mongoose.startSession();
|
|
68
|
+
try {
|
|
69
|
+
const claim = await claimSyncSeqs({count: 3, session, stream});
|
|
70
|
+
assert.strictEqual(claim.lastSeq, 3);
|
|
71
|
+
assert.deepEqual(claim.seqs, [1, 2, 3]);
|
|
72
|
+
// The $inc commits with the caller's write, so there is no uncommitted window to
|
|
73
|
+
// fence and nothing for confirmSyncSeqs to clear.
|
|
74
|
+
assert.isFalse(claim.registered);
|
|
75
|
+
const counter = await SyncCounter.findOne({stream});
|
|
76
|
+
assert.strictEqual(counter?.seq, 3);
|
|
77
|
+
assert.lengthOf(counter?.pending ?? [], 0);
|
|
78
|
+
} finally {
|
|
79
|
+
await session.endSession();
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it("retries the claim once when two concurrent upserts race (E11000)", async () => {
|
|
84
|
+
const stream = "models-race|owner:1";
|
|
85
|
+
let threw = false;
|
|
86
|
+
const restore = stubFindOneAndUpdate((original) => async (filter, update, options) => {
|
|
87
|
+
if (!threw) {
|
|
88
|
+
threw = true;
|
|
89
|
+
throw Object.assign(new Error("duplicate key"), {code: 11000});
|
|
90
|
+
}
|
|
91
|
+
return original(filter, update, options);
|
|
92
|
+
});
|
|
93
|
+
try {
|
|
94
|
+
const claim = await claimSyncSeqs({stream});
|
|
95
|
+
assert.isTrue(threw);
|
|
96
|
+
assert.strictEqual(claim.lastSeq, 1);
|
|
97
|
+
assert.isTrue(claim.registered);
|
|
98
|
+
} finally {
|
|
99
|
+
restore();
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it("rethrows a claim failure that is not the upsert race", async () => {
|
|
104
|
+
const restore = stubFindOneAndUpdate(() => async () => {
|
|
105
|
+
throw Object.assign(new Error("unrelated write failure"), {code: 66});
|
|
106
|
+
});
|
|
107
|
+
try {
|
|
108
|
+
let caught: unknown;
|
|
109
|
+
try {
|
|
110
|
+
await claimSyncSeqs({stream: "models-error|owner:1"});
|
|
111
|
+
} catch (error: unknown) {
|
|
112
|
+
caught = error;
|
|
113
|
+
}
|
|
114
|
+
assert.instanceOf(caught, Error);
|
|
115
|
+
assert.strictEqual((caught as Error).message, "unrelated write failure");
|
|
116
|
+
} finally {
|
|
117
|
+
restore();
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
describe("confirmSyncSeqs / releaseSyncSeqs", () => {
|
|
123
|
+
beforeAll(async () => {
|
|
124
|
+
await setupDb();
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
/** Counts `$pull` updates issued against the counter while `run` executes. */
|
|
128
|
+
const countPulls = async (run: () => Promise<void>): Promise<number> => {
|
|
129
|
+
let pulls = 0;
|
|
130
|
+
const original = SyncCounter.updateOne.bind(SyncCounter) as unknown as LooseUpdateOne;
|
|
131
|
+
const stub = (async (
|
|
132
|
+
filter: Record<string, unknown>,
|
|
133
|
+
update: Record<string, unknown>,
|
|
134
|
+
options?: Record<string, unknown>
|
|
135
|
+
) => {
|
|
136
|
+
if (update && Object.hasOwn(update, "$pull")) {
|
|
137
|
+
pulls += 1;
|
|
138
|
+
}
|
|
139
|
+
return original(filter, update, options);
|
|
140
|
+
}) as unknown as typeof SyncCounter.updateOne;
|
|
141
|
+
Object.assign(SyncCounter, {updateOne: stub});
|
|
142
|
+
try {
|
|
143
|
+
await run();
|
|
144
|
+
} finally {
|
|
145
|
+
Object.assign(SyncCounter, {updateOne: original});
|
|
146
|
+
}
|
|
147
|
+
return pulls;
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
it("treats an empty confirm as already cleared without touching the counter", async () => {
|
|
151
|
+
let result: {cleared: boolean} | undefined;
|
|
152
|
+
const pulls = await countPulls(async () => {
|
|
153
|
+
result = await confirmSyncSeqs({seqs: [], stream: "models-empty|owner:1"});
|
|
154
|
+
});
|
|
155
|
+
assert.deepEqual(result, {cleared: true});
|
|
156
|
+
assert.strictEqual(pulls, 0);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("no-ops an empty release without touching the counter", async () => {
|
|
160
|
+
const pulls = await countPulls(async () => {
|
|
161
|
+
await releaseSyncSeqs({seqs: [], stream: "models-empty|owner:1"});
|
|
162
|
+
});
|
|
163
|
+
assert.strictEqual(pulls, 0);
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
describe("computeStableFrontier", () => {
|
|
168
|
+
beforeAll(async () => {
|
|
169
|
+
await setupDb();
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("still reports the frontier when the stale-claim cleanup fails", async () => {
|
|
173
|
+
const stream = "models-stale|owner:1";
|
|
174
|
+
await SyncCounter.deleteMany({stream});
|
|
175
|
+
await SyncCounter.create({
|
|
176
|
+
pending: [{claimedAt: new Date(Date.now() - PENDING_CLAIM_LEASE_MS - 1_000), seq: 4}],
|
|
177
|
+
seq: 4,
|
|
178
|
+
stream,
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
const original = SyncCounter.updateOne.bind(SyncCounter) as unknown as LooseUpdateOne;
|
|
182
|
+
Object.assign(SyncCounter, {
|
|
183
|
+
updateOne: async () => {
|
|
184
|
+
throw new Error("simulated stale-claim $pull failure");
|
|
185
|
+
},
|
|
186
|
+
});
|
|
187
|
+
try {
|
|
188
|
+
// The opportunistic cleanup is best-effort: a failed $pull must not stall the
|
|
189
|
+
// frontier, which is already the head because every claim aged out.
|
|
190
|
+
assert.strictEqual(await computeStableFrontier({stream}), 4);
|
|
191
|
+
} finally {
|
|
192
|
+
Object.assign(SyncCounter, {updateOne: original});
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
describe("recordCompactedThroughSeq", () => {
|
|
198
|
+
beforeAll(async () => {
|
|
199
|
+
await setupDb();
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it("ignores a non-positive watermark", async () => {
|
|
203
|
+
const stream = "models-watermark|owner:1";
|
|
204
|
+
await SyncCounter.deleteMany({stream});
|
|
205
|
+
await SyncCounter.create({compactedThroughSeq: 5, seq: 5, stream});
|
|
206
|
+
await recordCompactedThroughSeq({seq: 0, stream});
|
|
207
|
+
const counter = await SyncCounter.findOne({stream});
|
|
208
|
+
assert.strictEqual(counter?.compactedThroughSeq, 5);
|
|
209
|
+
});
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
describe("ensureSyncModelIndexes", () => {
|
|
213
|
+
beforeAll(async () => {
|
|
214
|
+
await setupDb();
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it("drops the legacy SyncScopeMove TTL index", async () => {
|
|
218
|
+
await SyncScopeMove.createCollection();
|
|
219
|
+
const existing = await SyncScopeMove.collection.indexes();
|
|
220
|
+
if (!existing.some((index) => index.name === "created_1")) {
|
|
221
|
+
await SyncScopeMove.collection.createIndex({created: 1}, {expireAfterSeconds: 90 * 86_400});
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
await ensureSyncModelIndexes();
|
|
225
|
+
|
|
226
|
+
const indexes = await SyncScopeMove.collection.indexes();
|
|
227
|
+
const legacy = indexes.find(
|
|
228
|
+
(index) => index.name === "created_1" && index.expireAfterSeconds !== undefined
|
|
229
|
+
);
|
|
230
|
+
assert.isUndefined(legacy);
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it("logs and continues when the legacy index lookup fails", async () => {
|
|
234
|
+
const collection = SyncScopeMove.collection;
|
|
235
|
+
const original = collection.indexes.bind(collection);
|
|
236
|
+
Object.assign(collection, {
|
|
237
|
+
indexes: async () => {
|
|
238
|
+
throw new Error("simulated index listing failure");
|
|
239
|
+
},
|
|
240
|
+
});
|
|
241
|
+
try {
|
|
242
|
+
// A leftover TTL index degrades retention accounting but must never block startup.
|
|
243
|
+
await ensureSyncModelIndexes();
|
|
244
|
+
} finally {
|
|
245
|
+
Object.assign(collection, {indexes: original});
|
|
246
|
+
}
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
describe("getOrCreateSyncKeyMaterial", () => {
|
|
251
|
+
beforeAll(async () => {
|
|
252
|
+
await setupDb();
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
it("returns the persisted material on every later call", async () => {
|
|
256
|
+
const userId = new mongoose.Types.ObjectId().toString();
|
|
257
|
+
const first = await getOrCreateSyncKeyMaterial({userId});
|
|
258
|
+
const second = await getOrCreateSyncKeyMaterial({userId});
|
|
259
|
+
assert.strictEqual(second, first);
|
|
260
|
+
assert.strictEqual(Buffer.from(first, "base64").length, 32);
|
|
261
|
+
});
|
|
262
|
+
});
|
package/src/sync/registry.ts
CHANGED
|
@@ -150,6 +150,21 @@ export const registerSync = <T>({
|
|
|
150
150
|
indexPromise.catch(() => {});
|
|
151
151
|
};
|
|
152
152
|
|
|
153
|
+
/**
|
|
154
|
+
* Replace options on an existing sync registry entry after TerrenoApp injects
|
|
155
|
+
* accessControl and modelRouter resolves RBAC permissions.
|
|
156
|
+
*/
|
|
157
|
+
export const updateSyncRegistryOptions = (
|
|
158
|
+
routePath: string,
|
|
159
|
+
options: ModelRouterOptions<unknown>
|
|
160
|
+
): void => {
|
|
161
|
+
const existing = syncRegistry.find((entry) => entry.routePath === routePath);
|
|
162
|
+
if (!existing) {
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
existing.options = options;
|
|
166
|
+
};
|
|
167
|
+
|
|
153
168
|
/**
|
|
154
169
|
* C8: await every enqueued sync index creation — per-model snapshot indexes from
|
|
155
170
|
* `registerSync` plus the bookkeeping-model indexes from `SyncApp.register` — throwing on
|
package/src/sync/routes.ts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// noExplicitAny: sync routes operate generically across registered models
|
|
2
|
-
// biome-ignore-all lint/suspicious/noExplicitAny: sync routes operate generically across registered models
|
|
3
1
|
import express from "express";
|
|
4
2
|
import mongoose from "mongoose";
|
|
5
3
|
import {asyncHandler} from "../api";
|
|
@@ -60,6 +58,22 @@ export interface SyncAppOptions {
|
|
|
60
58
|
const MAX_SNAPSHOT_LIMIT = 100;
|
|
61
59
|
const DEFAULT_SNAPSHOT_LIMIT = 100;
|
|
62
60
|
|
|
61
|
+
/** Minimal shape of a synced document as read by the snapshot/entities routes. */
|
|
62
|
+
interface SyncedDoc {
|
|
63
|
+
_syncSeq?: number;
|
|
64
|
+
deleted?: boolean;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** A registered sync model, viewed through the fields these routes read. */
|
|
68
|
+
type SyncedModel = mongoose.Model<SyncedDoc>;
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Look up a registered model by name as a `SyncedModel`. `mongoose.model(name)` has no
|
|
72
|
+
* schema to infer from, so its static type carries none of the synced-document fields.
|
|
73
|
+
*/
|
|
74
|
+
const getSyncedModel = (modelName: string): SyncedModel =>
|
|
75
|
+
mongoose.model(modelName) as unknown as SyncedModel;
|
|
76
|
+
|
|
63
77
|
/** HTTP status for each nack code returned by `POST /sync/mutate`. */
|
|
64
78
|
const NACK_HTTP_STATUS: Record<SyncNackCode, number> = {
|
|
65
79
|
conflict: 409,
|
|
@@ -222,7 +236,7 @@ const legacyCursorClause = ({
|
|
|
222
236
|
model,
|
|
223
237
|
legacyCursorIn,
|
|
224
238
|
}: {
|
|
225
|
-
model:
|
|
239
|
+
model: SyncedModel;
|
|
226
240
|
legacyCursorIn: string;
|
|
227
241
|
}): Record<string, unknown> => {
|
|
228
242
|
const idInstance = (model.schema?.path("_id") as {instance?: string} | undefined)?.instance;
|
|
@@ -250,7 +264,7 @@ const pageLegacyStratum = async ({
|
|
|
250
264
|
entry,
|
|
251
265
|
req,
|
|
252
266
|
}: {
|
|
253
|
-
model:
|
|
267
|
+
model: SyncedModel;
|
|
254
268
|
scopeFilter: Record<string, unknown>;
|
|
255
269
|
legacyCursorIn?: string;
|
|
256
270
|
limit: number;
|
|
@@ -325,7 +339,7 @@ const pageSeqStratum = async ({
|
|
|
325
339
|
entry,
|
|
326
340
|
req,
|
|
327
341
|
}: {
|
|
328
|
-
model:
|
|
342
|
+
model: SyncedModel;
|
|
329
343
|
scopeFilter: Record<string, unknown>;
|
|
330
344
|
cursor: number;
|
|
331
345
|
limit: number;
|
|
@@ -367,7 +381,7 @@ const pageSeqStratum = async ({
|
|
|
367
381
|
// independent limits, so coverage stops at the lower of the two: past that seq, the
|
|
368
382
|
// truncated side may still hold rows this page never saw. When a side was not
|
|
369
383
|
// truncated, everything up to the frontier is covered.
|
|
370
|
-
const lastDocSeq = page.length > 0 ? (
|
|
384
|
+
const lastDocSeq = page.length > 0 ? (page[page.length - 1]._syncSeq ?? 0) : 0;
|
|
371
385
|
const lastMarkerSeq = markerPage.length > 0 ? markerPage[markerPage.length - 1].seq : 0;
|
|
372
386
|
const coveredSeq = Math.min(
|
|
373
387
|
docsHaveMore ? lastDocSeq : frontierSeq,
|
|
@@ -377,7 +391,7 @@ const pageSeqStratum = async ({
|
|
|
377
391
|
// C6 (M2): run the same per-doc read permission the delta path uses; drop denied
|
|
378
392
|
// docs but still advance the cursor past them (parity with delta behavior).
|
|
379
393
|
const docEntities: SyncEntityPayload[] = [];
|
|
380
|
-
for (const doc of page
|
|
394
|
+
for (const doc of page) {
|
|
381
395
|
const allowed = await checkPermissions("read", entry.options.permissions.read, user, doc);
|
|
382
396
|
if (!allowed) {
|
|
383
397
|
continue;
|
|
@@ -436,7 +450,7 @@ const castEntityIds = ({
|
|
|
436
450
|
ids,
|
|
437
451
|
}: {
|
|
438
452
|
ids: string[];
|
|
439
|
-
model:
|
|
453
|
+
model: SyncedModel;
|
|
440
454
|
}): (string | mongoose.Types.ObjectId)[] => {
|
|
441
455
|
const idInstance = (model.schema?.path("_id") as {instance?: string} | undefined)?.instance;
|
|
442
456
|
if (idInstance !== "ObjectId") {
|
|
@@ -446,7 +460,7 @@ const castEntityIds = ({
|
|
|
446
460
|
if (invalid.length > 0) {
|
|
447
461
|
throw new APIError({
|
|
448
462
|
status: 400,
|
|
449
|
-
title: `Invalid ids for ${
|
|
463
|
+
title: `Invalid ids for ${model.modelName}: ${invalid.join(",")}`,
|
|
450
464
|
});
|
|
451
465
|
}
|
|
452
466
|
return ids.map((id) => new mongoose.Types.ObjectId(id));
|
|
@@ -565,7 +579,7 @@ export const addSyncRoutes = (app: express.Application, options: SyncAppOptions
|
|
|
565
579
|
snapshotFilterResult,
|
|
566
580
|
}),
|
|
567
581
|
});
|
|
568
|
-
const model =
|
|
582
|
+
const model = getSyncedModel(entry.modelName);
|
|
569
583
|
const frontierSeq = await computeStableFrontier({stream: streamKey});
|
|
570
584
|
// C7 (Task 9.15): the retention signal is the durable compaction watermark, not
|
|
571
585
|
// min(retained seq). The old computation was pinned low forever by any early doc that
|
|
@@ -697,7 +711,7 @@ export const addSyncRoutes = (app: express.Application, options: SyncAppOptions
|
|
|
697
711
|
return res.json(deniedResponse);
|
|
698
712
|
}
|
|
699
713
|
|
|
700
|
-
const model =
|
|
714
|
+
const model = getSyncedModel(entry.modelName);
|
|
701
715
|
const castIds = castEntityIds({ids, model});
|
|
702
716
|
const docs = await model.find({
|
|
703
717
|
// `deleted` MUST stay a TOP-LEVEL key so isDeletedPlugin does not re-inject its
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// noExplicitAny: test model typing
|
|
2
|
-
// biome-ignore-all lint/suspicious/noExplicitAny: test model typing
|
|
3
1
|
import {beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
4
2
|
import {DateTime} from "luxon";
|
|
5
3
|
import {model, Schema} from "mongoose";
|
|
@@ -42,7 +40,7 @@ const options = {
|
|
|
42
40
|
read: [Permissions.IsAuthenticated],
|
|
43
41
|
update: [Permissions.IsAuthenticated],
|
|
44
42
|
},
|
|
45
|
-
} as unknown as ModelRouterOptions<
|
|
43
|
+
} as unknown as ModelRouterOptions<CompactStuff>;
|
|
46
44
|
|
|
47
45
|
beforeAll(async () => {
|
|
48
46
|
await setupDb();
|
|
@@ -61,7 +59,7 @@ describe("compactTombstones (C7 retention)", () => {
|
|
|
61
59
|
const registerWithRetention = (retentionDays?: number): void => {
|
|
62
60
|
registerSync({
|
|
63
61
|
config: {scope: {type: "owner"}, ...(retentionDays !== undefined ? {retentionDays} : {})},
|
|
64
|
-
model: CompactStuffModel
|
|
62
|
+
model: CompactStuffModel,
|
|
65
63
|
options,
|
|
66
64
|
routePath: "/compactStuff",
|
|
67
65
|
});
|
|
@@ -91,7 +89,7 @@ describe("compactTombstones (C7 retention)", () => {
|
|
|
91
89
|
expect(result.byCollection.compactStuff.tombstones).toBe(1);
|
|
92
90
|
|
|
93
91
|
const remaining = await CompactStuffModel.collection.find({}).toArray();
|
|
94
|
-
const names = remaining.map((d
|
|
92
|
+
const names = remaining.map((d) => d.name).sort();
|
|
95
93
|
// The old tombstone is gone; the recent tombstone and the live doc remain.
|
|
96
94
|
expect(names).toEqual(["live", "recent tombstone"]);
|
|
97
95
|
});
|
|
@@ -234,7 +232,7 @@ const clearSyncRegistryAndReRegister = (retentionDays: number) => {
|
|
|
234
232
|
clearSyncRegistry();
|
|
235
233
|
registerSync({
|
|
236
234
|
config: {retentionDays, scope: {type: "owner"}},
|
|
237
|
-
model: CompactStuffModel
|
|
235
|
+
model: CompactStuffModel,
|
|
238
236
|
options,
|
|
239
237
|
routePath: "/compactStuff",
|
|
240
238
|
});
|