@terreno/api 0.31.0 → 56.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +10 -0
- package/dist/api.js +110 -361
- package/dist/auth.d.ts +26 -0
- package/dist/auth.js +121 -26
- package/dist/betterAuth.d.ts +5 -0
- package/dist/betterAuthSetup.js +23 -4
- package/dist/docLoader.js +12 -9
- package/dist/errors.d.ts +13 -0
- package/dist/errors.js +29 -1
- package/dist/index.d.ts +17 -1
- package/dist/index.js +29 -1
- package/dist/middleware.d.ts +7 -0
- package/dist/middleware.js +16 -1
- package/dist/openApiCompat.d.ts +10 -1
- package/dist/openApiCompat.js +115 -47
- package/dist/permissions.d.ts +27 -0
- package/dist/permissions.js +42 -1
- package/dist/plugins.d.ts +13 -0
- package/dist/plugins.js +47 -1
- package/dist/realtime/changeStreamWatcher.d.ts +100 -3
- package/dist/realtime/changeStreamWatcher.js +542 -137
- package/dist/realtime/realtimeApp.d.ts +2 -0
- package/dist/realtime/realtimeApp.js +83 -27
- package/dist/realtime/sessionRevalidation.d.ts +90 -0
- package/dist/realtime/sessionRevalidation.js +438 -0
- package/dist/realtime/socketAuth.d.ts +97 -0
- package/dist/realtime/socketAuth.js +272 -0
- package/dist/realtime/socketUser.d.ts +54 -0
- package/dist/realtime/socketUser.js +75 -5
- package/dist/realtime/types.d.ts +54 -0
- package/dist/secretProviders.d.ts +23 -0
- package/dist/secretProviders.js +9 -3
- package/dist/sync/executors.d.ts +198 -0
- package/dist/sync/executors.js +670 -0
- package/dist/sync/loadHarness.js +807 -0
- package/dist/sync/models.d.ts +216 -0
- package/dist/sync/models.js +644 -0
- package/dist/sync/mutationHandler.d.ts +97 -0
- package/dist/sync/mutationHandler.js +886 -0
- package/dist/sync/registry.d.ts +73 -0
- package/dist/sync/registry.js +223 -0
- package/dist/sync/routes.d.ts +62 -0
- package/dist/sync/routes.js +936 -0
- package/dist/sync/scripts/compactTombstones.d.ts +47 -0
- package/dist/sync/scripts/compactTombstones.js +270 -0
- package/dist/sync/serialize.d.ts +20 -0
- package/dist/{tests/bunSetup.js → sync/serialize.js} +35 -68
- package/dist/sync/socketHandlers.d.ts +54 -0
- package/dist/sync/socketHandlers.js +470 -0
- package/dist/sync/streams.d.ts +85 -0
- package/dist/sync/streams.js +221 -0
- package/dist/sync/syncApp.d.ts +29 -0
- package/dist/sync/syncApp.js +44 -0
- package/dist/sync/syncBatch.d.ts +71 -0
- package/dist/sync/syncBatch.js +252 -0
- package/dist/sync/syncSeqPlugin.d.ts +8 -0
- package/dist/sync/syncSeqPlugin.js +899 -0
- package/dist/sync/types.d.ts +254 -0
- package/dist/sync/types.js +9 -0
- package/dist/terrenoApp.js +90 -27
- package/dist/vendor/wesleytodd-openapi/index.d.ts +1 -1
- package/dist/vendor/wesleytodd-openapi/lib/ui.d.ts +2 -1
- package/package.json +7 -6
- package/src/__tests__/openApiCompat.test.ts +20 -0
- package/src/api.errors.test.ts +32 -23
- package/src/api.test.ts +12 -0
- package/src/api.ts +87 -296
- package/src/auth.test.ts +202 -0
- package/src/auth.ts +134 -23
- package/src/betterAuth.ts +6 -0
- package/src/betterAuthSetup.test.ts +34 -0
- package/src/betterAuthSetup.ts +20 -1
- package/src/configuration.test.ts +59 -38
- package/src/configurationPlugin.test.ts +72 -42
- package/src/docLoader.ts +3 -1
- package/src/errors.ts +33 -0
- package/src/expressServer.test.ts +7 -1
- package/src/index.ts +42 -0
- package/src/middleware.test.ts +56 -1
- package/src/middleware.ts +15 -0
- package/src/openApiCompat.ts +99 -45
- package/src/permissions.test.ts +101 -17
- package/src/permissions.ts +49 -0
- package/src/plugins.test.ts +64 -0
- package/src/plugins.ts +56 -0
- package/src/realtime/changeStreamWatcher.ts +552 -114
- package/src/realtime/realtime.test.ts +157 -0
- package/src/realtime/realtimeApp.ts +82 -14
- package/src/realtime/realtimeAppSetup.test.ts +25 -27
- package/src/realtime/sessionRevalidation.test.ts +423 -0
- package/src/realtime/sessionRevalidation.ts +273 -0
- package/src/realtime/socketAuth.test.ts +355 -0
- package/src/realtime/socketAuth.ts +228 -0
- package/src/realtime/socketUser.ts +69 -0
- package/src/realtime/types.ts +54 -0
- package/src/secretProviders.test.ts +7 -1
- package/src/secretProviders.ts +15 -3
- package/src/secretProvidersGcpClient.test.ts +66 -0
- package/src/sync/ensureSyncIndexes.test.ts +144 -0
- package/src/sync/executors.test.ts +882 -0
- package/src/sync/executors.ts +705 -0
- package/src/sync/integration.test.ts +1160 -0
- package/src/sync/loadHarness.ts +618 -0
- package/src/sync/models.ts +641 -0
- package/src/sync/mutationHandler.test.ts +1252 -0
- package/src/sync/mutationHandler.ts +850 -0
- package/src/sync/registry.ts +232 -0
- package/src/sync/routes.ts +812 -0
- package/src/sync/scripts/compactTombstones.test.ts +249 -0
- package/src/sync/scripts/compactTombstones.ts +145 -0
- package/src/sync/serialize.ts +48 -0
- package/src/sync/sessionRevalidationIntegration.test.ts +281 -0
- package/src/sync/socketHandlers.ts +320 -0
- package/src/sync/streams.ts +188 -0
- package/src/sync/sync.test.ts +582 -0
- package/src/sync/syncApp.ts +44 -0
- package/src/sync/syncBatch.ts +219 -0
- package/src/sync/syncFrontier.test.ts +494 -0
- package/src/sync/syncPhaseC.test.ts +718 -0
- package/src/sync/syncRoutes.test.ts +865 -0
- package/src/sync/syncSeqPlugin.ts +730 -0
- package/src/sync/syncSocket.test.ts +1440 -0
- package/src/sync/types.ts +270 -0
- package/src/terrenoApp.ts +37 -17
- package/src/tests/bunSetup.ts +8 -0
- package/tsconfig.json +4 -4
- package/tsconfig.typedoc.json +7 -1
- package/dist/__tests__/openApiCompat.test.js +0 -317
- package/dist/__tests__/versionCheckPlugin.test.d.ts +0 -1
- package/dist/__tests__/versionCheckPlugin.test.js +0 -597
- package/dist/actions.openApi.test.d.ts +0 -1
- package/dist/actions.openApi.test.js +0 -253
- package/dist/actions.test.d.ts +0 -1
- package/dist/actions.test.js +0 -1087
- package/dist/api.arrayOperations.test.d.ts +0 -1
- package/dist/api.arrayOperations.test.js +0 -878
- package/dist/api.asyncHandler.test.d.ts +0 -1
- package/dist/api.asyncHandler.test.js +0 -236
- package/dist/api.coverage.test.d.ts +0 -1
- package/dist/api.coverage.test.js +0 -192
- package/dist/api.errorPaths.test.d.ts +0 -1
- package/dist/api.errorPaths.test.js +0 -887
- package/dist/api.errors.test.d.ts +0 -1
- package/dist/api.errors.test.js +0 -185
- package/dist/api.hooks.test.d.ts +0 -1
- package/dist/api.hooks.test.js +0 -950
- package/dist/api.query.test.d.ts +0 -1
- package/dist/api.query.test.js +0 -948
- package/dist/api.test.d.ts +0 -1
- package/dist/api.test.js +0 -2820
- package/dist/auth.compile.test.d.ts +0 -1
- package/dist/auth.compile.test.js +0 -201
- package/dist/auth.coverage.test.d.ts +0 -1
- package/dist/auth.coverage.test.js +0 -202
- package/dist/auth.test.d.ts +0 -1
- package/dist/auth.test.js +0 -2286
- package/dist/betterAuth.test.d.ts +0 -1
- package/dist/betterAuth.test.js +0 -181
- package/dist/betterAuthApp.test.d.ts +0 -1
- package/dist/betterAuthApp.test.js +0 -242
- package/dist/betterAuthSetup.test.d.ts +0 -1
- package/dist/betterAuthSetup.test.js +0 -762
- package/dist/config.test.d.ts +0 -1
- package/dist/config.test.js +0 -353
- package/dist/configuration.test.d.ts +0 -1
- package/dist/configuration.test.js +0 -980
- package/dist/configurationPlugin.test.d.ts +0 -1
- package/dist/configurationPlugin.test.js +0 -827
- package/dist/consentApp.test.d.ts +0 -1
- package/dist/consentApp.test.js +0 -1411
- package/dist/docLoader.test.d.ts +0 -1
- package/dist/docLoader.test.js +0 -137
- package/dist/envConfigurationPlugin.test.d.ts +0 -1
- package/dist/envConfigurationPlugin.test.js +0 -369
- package/dist/errors.test.d.ts +0 -1
- package/dist/errors.test.js +0 -607
- package/dist/expressServer.test.d.ts +0 -1
- package/dist/expressServer.test.js +0 -1249
- package/dist/githubAuth.test.d.ts +0 -1
- package/dist/githubAuth.test.js +0 -1142
- package/dist/httpClient.test.d.ts +0 -1
- package/dist/httpClient.test.js +0 -1471
- package/dist/logger.test.d.ts +0 -1
- package/dist/logger.test.js +0 -685
- package/dist/middleware.test.d.ts +0 -1
- package/dist/middleware.test.js +0 -241
- package/dist/notifiers/googleChatNotifier.test.d.ts +0 -1
- package/dist/notifiers/googleChatNotifier.test.js +0 -298
- package/dist/notifiers/slackNotifier.test.d.ts +0 -1
- package/dist/notifiers/slackNotifier.test.js +0 -297
- package/dist/notifiers/zoomNotifier.test.d.ts +0 -1
- package/dist/notifiers/zoomNotifier.test.js +0 -373
- package/dist/openApi.test.d.ts +0 -1
- package/dist/openApi.test.js +0 -620
- package/dist/openApiBuilder.test.d.ts +0 -1
- package/dist/openApiBuilder.test.js +0 -599
- package/dist/openApiEtag.test.d.ts +0 -1
- package/dist/openApiEtag.test.js +0 -86
- package/dist/openApiValidator.test.d.ts +0 -1
- package/dist/openApiValidator.test.js +0 -795
- package/dist/permissions.middleware.test.d.ts +0 -1
- package/dist/permissions.middleware.test.js +0 -308
- package/dist/permissions.test.d.ts +0 -1
- package/dist/permissions.test.js +0 -439
- package/dist/plugins.test.d.ts +0 -1
- package/dist/plugins.test.js +0 -902
- package/dist/populate.test.d.ts +0 -1
- package/dist/populate.test.js +0 -370
- package/dist/realtime/realtime.test.d.ts +0 -10
- package/dist/realtime/realtime.test.js +0 -4213
- package/dist/realtime/realtimeAppSetup.test.d.ts +0 -1
- package/dist/realtime/realtimeAppSetup.test.js +0 -213
- package/dist/requestContext.test.d.ts +0 -1
- package/dist/requestContext.test.js +0 -412
- package/dist/scriptRunner.test.d.ts +0 -1
- package/dist/scriptRunner.test.js +0 -386
- package/dist/secretProviders.test.d.ts +0 -1
- package/dist/secretProviders.test.js +0 -726
- package/dist/syncConsents.test.d.ts +0 -1
- package/dist/syncConsents.test.js +0 -522
- package/dist/terrenoApp.test.d.ts +0 -1
- package/dist/terrenoApp.test.js +0 -489
- package/dist/tests/bunSetup.d.ts +0 -1
- package/dist/tests/fixtures/compileAuthEntry.d.ts +0 -1
- package/dist/tests/fixtures/compileAuthEntry.js +0 -37
- package/dist/tests/testEnv.d.ts +0 -0
- package/dist/tests/testEnv.js +0 -6
- package/dist/transformers.test.d.ts +0 -1
- package/dist/transformers.test.js +0 -423
- package/dist/utils.test.d.ts +0 -1
- package/dist/utils.test.js +0 -242
- /package/dist/{__tests__/openApiCompat.test.d.ts → sync/loadHarness.d.ts} +0 -0
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// noExplicitAny: test model typing
|
|
2
|
+
// biome-ignore-all lint/suspicious/noExplicitAny: test model typing
|
|
3
|
+
import {beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
4
|
+
import {DateTime} from "luxon";
|
|
5
|
+
import {model, Schema} from "mongoose";
|
|
6
|
+
import type {ModelRouterOptions} from "../../api";
|
|
7
|
+
import {Permissions} from "../../permissions";
|
|
8
|
+
import {createdUpdatedPlugin, type IsDeleted, isDeletedPlugin} from "../../plugins";
|
|
9
|
+
import {setupDb} from "../../tests";
|
|
10
|
+
import {getCompactedThroughSeq, SyncCounter, SyncScopeMove} from "../models";
|
|
11
|
+
import {clearSyncRegistry, registerSync} from "../registry";
|
|
12
|
+
import {syncPlugin} from "../syncSeqPlugin";
|
|
13
|
+
import {
|
|
14
|
+
compactEntryTombstones,
|
|
15
|
+
compactTombstones,
|
|
16
|
+
DEFAULT_TOMBSTONE_RETENTION_DAYS,
|
|
17
|
+
} from "./compactTombstones";
|
|
18
|
+
|
|
19
|
+
interface CompactStuff extends IsDeleted {
|
|
20
|
+
_id: string;
|
|
21
|
+
name: string;
|
|
22
|
+
ownerId: string;
|
|
23
|
+
created: Date;
|
|
24
|
+
updated?: Date;
|
|
25
|
+
_syncSeq?: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const compactStuffSchema = new Schema<CompactStuff>({
|
|
29
|
+
name: {description: "The name of the item", required: true, type: String},
|
|
30
|
+
ownerId: {description: "The owner", type: String},
|
|
31
|
+
});
|
|
32
|
+
compactStuffSchema.plugin(isDeletedPlugin);
|
|
33
|
+
compactStuffSchema.plugin(createdUpdatedPlugin);
|
|
34
|
+
compactStuffSchema.plugin(syncPlugin);
|
|
35
|
+
const CompactStuffModel = model<CompactStuff>("SyncCompactStuff", compactStuffSchema);
|
|
36
|
+
|
|
37
|
+
const options = {
|
|
38
|
+
permissions: {
|
|
39
|
+
create: [Permissions.IsAuthenticated],
|
|
40
|
+
delete: [Permissions.IsAuthenticated],
|
|
41
|
+
list: [Permissions.IsAuthenticated],
|
|
42
|
+
read: [Permissions.IsAuthenticated],
|
|
43
|
+
update: [Permissions.IsAuthenticated],
|
|
44
|
+
},
|
|
45
|
+
} as unknown as ModelRouterOptions<any>;
|
|
46
|
+
|
|
47
|
+
beforeAll(async () => {
|
|
48
|
+
await setupDb();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
describe("compactTombstones (C7 retention)", () => {
|
|
52
|
+
beforeEach(async () => {
|
|
53
|
+
clearSyncRegistry();
|
|
54
|
+
await Promise.all([
|
|
55
|
+
CompactStuffModel.collection.deleteMany({}),
|
|
56
|
+
SyncCounter.deleteMany({}),
|
|
57
|
+
SyncScopeMove.deleteMany({}),
|
|
58
|
+
]);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
const registerWithRetention = (retentionDays?: number): void => {
|
|
62
|
+
registerSync({
|
|
63
|
+
config: {scope: {type: "owner"}, ...(retentionDays !== undefined ? {retentionDays} : {})},
|
|
64
|
+
model: CompactStuffModel as any,
|
|
65
|
+
options,
|
|
66
|
+
routePath: "/compactStuff",
|
|
67
|
+
});
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
it("hard-deletes tombstones older than the retention window and keeps recent ones", async () => {
|
|
71
|
+
registerWithRetention();
|
|
72
|
+
const oldCutoff = DateTime.now()
|
|
73
|
+
.minus({days: DEFAULT_TOMBSTONE_RETENTION_DAYS + 5})
|
|
74
|
+
.toJSDate();
|
|
75
|
+
|
|
76
|
+
// Old tombstone (should be compacted): bypass mongoose so we can backdate `updated`.
|
|
77
|
+
await CompactStuffModel.collection.insertMany([
|
|
78
|
+
{created: oldCutoff, deleted: true, name: "old tombstone", ownerId: "u1", updated: oldCutoff},
|
|
79
|
+
{
|
|
80
|
+
created: new Date(),
|
|
81
|
+
deleted: true,
|
|
82
|
+
name: "recent tombstone",
|
|
83
|
+
ownerId: "u1",
|
|
84
|
+
updated: new Date(),
|
|
85
|
+
},
|
|
86
|
+
{created: new Date(), deleted: false, name: "live", ownerId: "u1", updated: new Date()},
|
|
87
|
+
]);
|
|
88
|
+
|
|
89
|
+
const result = await compactTombstones();
|
|
90
|
+
expect(result.totalTombstones).toBe(1);
|
|
91
|
+
expect(result.byCollection.compactStuff.tombstones).toBe(1);
|
|
92
|
+
|
|
93
|
+
const remaining = await CompactStuffModel.collection.find({}).toArray();
|
|
94
|
+
const names = remaining.map((d: any) => d.name).sort();
|
|
95
|
+
// The old tombstone is gone; the recent tombstone and the live doc remain.
|
|
96
|
+
expect(names).toEqual(["live", "recent tombstone"]);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it("hard-deletes scope-move markers older than the retention window", async () => {
|
|
100
|
+
registerWithRetention();
|
|
101
|
+
const old = DateTime.now()
|
|
102
|
+
.minus({days: DEFAULT_TOMBSTONE_RETENTION_DAYS + 1})
|
|
103
|
+
.toJSDate();
|
|
104
|
+
await SyncScopeMove.collection.insertMany([
|
|
105
|
+
{
|
|
106
|
+
collectionTag: "compactStuff",
|
|
107
|
+
created: old,
|
|
108
|
+
entityId: "e-old",
|
|
109
|
+
fromStream: "compactStuff|owner:u1",
|
|
110
|
+
seq: 5,
|
|
111
|
+
toStream: "compactStuff|owner:u2",
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
collectionTag: "compactStuff",
|
|
115
|
+
created: new Date(),
|
|
116
|
+
entityId: "e-new",
|
|
117
|
+
fromStream: "compactStuff|owner:u1",
|
|
118
|
+
seq: 6,
|
|
119
|
+
toStream: "compactStuff|owner:u2",
|
|
120
|
+
},
|
|
121
|
+
]);
|
|
122
|
+
|
|
123
|
+
const result = await compactTombstones();
|
|
124
|
+
expect(result.totalMarkers).toBe(1);
|
|
125
|
+
const remaining = await SyncScopeMove.find({});
|
|
126
|
+
expect(remaining.map((m) => m.entityId)).toEqual(["e-new"]);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it("honors a per-model retentionDays override", async () => {
|
|
130
|
+
// 1-day retention: a 2-day-old tombstone is compacted even though the default (90) would keep it.
|
|
131
|
+
registerWithRetention(1);
|
|
132
|
+
const twoDaysAgo = DateTime.now().minus({days: 2}).toJSDate();
|
|
133
|
+
await CompactStuffModel.collection.insertMany([
|
|
134
|
+
{
|
|
135
|
+
created: twoDaysAgo,
|
|
136
|
+
deleted: true,
|
|
137
|
+
name: "two days old",
|
|
138
|
+
ownerId: "u1",
|
|
139
|
+
updated: twoDaysAgo,
|
|
140
|
+
},
|
|
141
|
+
]);
|
|
142
|
+
const entry = clearSyncRegistryAndReRegister(1);
|
|
143
|
+
const counts = await compactEntryTombstones(entry);
|
|
144
|
+
expect(counts.tombstones).toBe(1);
|
|
145
|
+
expect(await CompactStuffModel.collection.countDocuments({})).toBe(0);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it("leaves everything intact when nothing is past retention", async () => {
|
|
149
|
+
registerWithRetention();
|
|
150
|
+
await CompactStuffModel.collection.insertMany([
|
|
151
|
+
{created: new Date(), deleted: true, name: "recent", ownerId: "u1", updated: new Date()},
|
|
152
|
+
]);
|
|
153
|
+
const result = await compactTombstones();
|
|
154
|
+
expect(result.totalTombstones).toBe(0);
|
|
155
|
+
expect(await CompactStuffModel.collection.countDocuments({})).toBe(1);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
// ── Task 9.15: the retention watermark ────────────────────────────────────────
|
|
159
|
+
it("Task 9.15: records compactedThroughSeq per stream from the highest reaped tombstone seq", async () => {
|
|
160
|
+
registerWithRetention();
|
|
161
|
+
const old = DateTime.now()
|
|
162
|
+
.minus({days: DEFAULT_TOMBSTONE_RETENTION_DAYS + 5})
|
|
163
|
+
.toJSDate();
|
|
164
|
+
await SyncCounter.create([
|
|
165
|
+
{seq: 30, stream: "compactStuff|owner:u1"},
|
|
166
|
+
{seq: 30, stream: "compactStuff|owner:u2"},
|
|
167
|
+
]);
|
|
168
|
+
await CompactStuffModel.collection.insertMany([
|
|
169
|
+
{_syncSeq: 4, created: old, deleted: true, name: "u1 low", ownerId: "u1", updated: old},
|
|
170
|
+
{_syncSeq: 11, created: old, deleted: true, name: "u1 high", ownerId: "u1", updated: old},
|
|
171
|
+
{_syncSeq: 7, created: old, deleted: true, name: "u2 only", ownerId: "u2", updated: old},
|
|
172
|
+
// A recent tombstone is not reaped, so it must not raise the watermark.
|
|
173
|
+
{
|
|
174
|
+
_syncSeq: 29,
|
|
175
|
+
created: new Date(),
|
|
176
|
+
deleted: true,
|
|
177
|
+
name: "u1 recent",
|
|
178
|
+
ownerId: "u1",
|
|
179
|
+
updated: new Date(),
|
|
180
|
+
},
|
|
181
|
+
]);
|
|
182
|
+
|
|
183
|
+
await compactTombstones();
|
|
184
|
+
|
|
185
|
+
expect(await getCompactedThroughSeq({stream: "compactStuff|owner:u1"})).toBe(11);
|
|
186
|
+
expect(await getCompactedThroughSeq({stream: "compactStuff|owner:u2"})).toBe(7);
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it("Task 9.15: a reaped scope-move marker raises its OLD stream's watermark", async () => {
|
|
190
|
+
registerWithRetention();
|
|
191
|
+
const old = DateTime.now()
|
|
192
|
+
.minus({days: DEFAULT_TOMBSTONE_RETENTION_DAYS + 1})
|
|
193
|
+
.toJSDate();
|
|
194
|
+
await SyncCounter.create([{seq: 20, stream: "compactStuff|owner:u1"}]);
|
|
195
|
+
await SyncScopeMove.collection.insertMany([
|
|
196
|
+
{
|
|
197
|
+
collectionTag: "compactStuff",
|
|
198
|
+
created: old,
|
|
199
|
+
entityId: "e-old",
|
|
200
|
+
fromStream: "compactStuff|owner:u1",
|
|
201
|
+
seq: 9,
|
|
202
|
+
toStream: "compactStuff|owner:u2",
|
|
203
|
+
},
|
|
204
|
+
]);
|
|
205
|
+
|
|
206
|
+
await compactTombstones();
|
|
207
|
+
|
|
208
|
+
expect(await getCompactedThroughSeq({stream: "compactStuff|owner:u1"})).toBe(9);
|
|
209
|
+
// The destination stream lost nothing, so it has no retention gap.
|
|
210
|
+
expect(await getCompactedThroughSeq({stream: "compactStuff|owner:u2"})).toBe(0);
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
it("Task 9.15: the watermark never moves backwards across compaction passes", async () => {
|
|
214
|
+
registerWithRetention(1);
|
|
215
|
+
const old = DateTime.now().minus({days: 5}).toJSDate();
|
|
216
|
+
await SyncCounter.create([{seq: 40, stream: "compactStuff|owner:u1"}]);
|
|
217
|
+
await CompactStuffModel.collection.insertMany([
|
|
218
|
+
{_syncSeq: 20, created: old, deleted: true, name: "high", ownerId: "u1", updated: old},
|
|
219
|
+
]);
|
|
220
|
+
await compactTombstones();
|
|
221
|
+
expect(await getCompactedThroughSeq({stream: "compactStuff|owner:u1"})).toBe(20);
|
|
222
|
+
|
|
223
|
+
// A later pass reaping only a LOWER seq must not lower the watermark.
|
|
224
|
+
await CompactStuffModel.collection.insertMany([
|
|
225
|
+
{_syncSeq: 3, created: old, deleted: true, name: "low", ownerId: "u1", updated: old},
|
|
226
|
+
]);
|
|
227
|
+
await compactTombstones();
|
|
228
|
+
expect(await getCompactedThroughSeq({stream: "compactStuff|owner:u1"})).toBe(20);
|
|
229
|
+
});
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
// Helper: re-register and return the entry (retentionDays override) for compactEntryTombstones.
|
|
233
|
+
const clearSyncRegistryAndReRegister = (retentionDays: number) => {
|
|
234
|
+
clearSyncRegistry();
|
|
235
|
+
registerSync({
|
|
236
|
+
config: {retentionDays, scope: {type: "owner"}},
|
|
237
|
+
model: CompactStuffModel as any,
|
|
238
|
+
options,
|
|
239
|
+
routePath: "/compactStuff",
|
|
240
|
+
});
|
|
241
|
+
return {
|
|
242
|
+
collectionName: CompactStuffModel.collection.collectionName,
|
|
243
|
+
collectionTag: "compactStuff",
|
|
244
|
+
config: {retentionDays, scope: {type: "owner"} as const},
|
|
245
|
+
modelName: CompactStuffModel.modelName,
|
|
246
|
+
options,
|
|
247
|
+
routePath: "/compactStuff",
|
|
248
|
+
};
|
|
249
|
+
};
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// noExplicitAny: operates generically across registered sync models
|
|
2
|
+
// biome-ignore-all lint/suspicious/noExplicitAny: operates generically across registered sync models
|
|
3
|
+
import {DateTime} from "luxon";
|
|
4
|
+
import mongoose from "mongoose";
|
|
5
|
+
import {logger} from "../../logger";
|
|
6
|
+
import {recordCompactedThroughSeq, SyncScopeMove} from "../models";
|
|
7
|
+
import {getSyncRegistry, type SyncRegistryEntry} from "../registry";
|
|
8
|
+
import {getScopeField, resolveStreamForDoc} from "../streams";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* C7 tombstone retention maintenance.
|
|
12
|
+
*
|
|
13
|
+
* Tombstones (soft-deleted documents kept so offline clients learn of deletions) and
|
|
14
|
+
* `SyncScopeMove` markers accumulate forever otherwise. This script hard-deletes both
|
|
15
|
+
* once they are older than the model's `retentionDays` (default 90). It is paired with
|
|
16
|
+
* the client-side rule that a cursor older than the snapshot response's
|
|
17
|
+
* `oldestRetainedSeq` triggers a full re-bootstrap of that stream — so compacting
|
|
18
|
+
* tombstones a client has already seen is safe, and a client that missed them recovers
|
|
19
|
+
* by re-bootstrapping rather than silently keeping stale data.
|
|
20
|
+
*
|
|
21
|
+
* Run as a periodic maintenance job (cron), NOT on the request path. Requires an active
|
|
22
|
+
* Mongoose connection.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** Default retention window when a model does not override `sync.retentionDays`. */
|
|
26
|
+
export const DEFAULT_TOMBSTONE_RETENTION_DAYS = 90;
|
|
27
|
+
|
|
28
|
+
export interface CompactTombstonesResult {
|
|
29
|
+
/** Per-collection counts of hard-deleted tombstones and scope-move markers. */
|
|
30
|
+
byCollection: Record<string, {tombstones: number; markers: number}>;
|
|
31
|
+
/** Total tombstones hard-deleted. */
|
|
32
|
+
totalTombstones: number;
|
|
33
|
+
/** Total scope-move markers hard-deleted. */
|
|
34
|
+
totalMarkers: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const retentionCutoff = (entry: SyncRegistryEntry): Date => {
|
|
38
|
+
const days = entry.config.retentionDays ?? DEFAULT_TOMBSTONE_RETENTION_DAYS;
|
|
39
|
+
return DateTime.now().minus({days}).toJSDate();
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Compact tombstones and scope-move markers for one registry entry: hard-delete
|
|
44
|
+
* soft-deleted documents whose `updated` (falling back to `created`) predates the
|
|
45
|
+
* retention window, plus scope-move markers older than the window.
|
|
46
|
+
*
|
|
47
|
+
* Task 9.15: the rows are READ before they are deleted so the highest reaped seq per
|
|
48
|
+
* stream can be recorded as that stream's `compactedThroughSeq` watermark. That watermark
|
|
49
|
+
* is the snapshot's `oldestRetainedSeq`, and it is the only signal a client has that its
|
|
50
|
+
* cursor sits below a deletion it can no longer be told about. Deleting without raising it
|
|
51
|
+
* silently strands the client on stale data. This is also why `SyncScopeMove` no longer
|
|
52
|
+
* carries a TTL index: an expiry that fires outside this function reaps rows with no
|
|
53
|
+
* watermark update and ignores the model's `retentionDays`.
|
|
54
|
+
*/
|
|
55
|
+
export const compactEntryTombstones = async (
|
|
56
|
+
entry: SyncRegistryEntry
|
|
57
|
+
): Promise<{tombstones: number; markers: number}> => {
|
|
58
|
+
const cutoff = retentionCutoff(entry);
|
|
59
|
+
const model = mongoose.model(entry.modelName);
|
|
60
|
+
const scopeField = getScopeField(entry.config.scope);
|
|
61
|
+
const tombstoneFilter = {
|
|
62
|
+
$or: [{updated: {$lt: cutoff}}, {created: {$lt: cutoff}, updated: {$exists: false}}],
|
|
63
|
+
deleted: true,
|
|
64
|
+
};
|
|
65
|
+
const markerFilter = {collectionTag: entry.collectionTag, created: {$lt: cutoff}};
|
|
66
|
+
|
|
67
|
+
/** stream key -> highest seq being reaped on that stream. */
|
|
68
|
+
const watermarks = new Map<string, number>();
|
|
69
|
+
const raiseWatermark = (stream: string, seq: unknown): void => {
|
|
70
|
+
if (typeof seq !== "number" || seq <= 0) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
if (seq > (watermarks.get(stream) ?? 0)) {
|
|
74
|
+
watermarks.set(stream, seq);
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
// A custom scope resolver may read any field, so it needs whole documents; broadcast and
|
|
79
|
+
// field-based scopes only need the seq (plus the scope field) to resolve the stream.
|
|
80
|
+
const isCustomScope = typeof entry.config.scope === "function";
|
|
81
|
+
const projection = scopeField
|
|
82
|
+
? {_id: 1, _syncSeq: 1, [scopeField]: 1}
|
|
83
|
+
: isCustomScope
|
|
84
|
+
? undefined
|
|
85
|
+
: {_id: 1, _syncSeq: 1};
|
|
86
|
+
const doomedTombstones = await model.collection
|
|
87
|
+
.find(tombstoneFilter, projection ? {projection} : {})
|
|
88
|
+
.toArray();
|
|
89
|
+
for (const row of doomedTombstones) {
|
|
90
|
+
raiseWatermark(
|
|
91
|
+
resolveStreamForDoc({
|
|
92
|
+
collectionTag: entry.collectionTag,
|
|
93
|
+
doc: row as Record<string, unknown>,
|
|
94
|
+
scope: entry.config.scope,
|
|
95
|
+
}),
|
|
96
|
+
(row as {_syncSeq?: unknown})._syncSeq
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const doomedMarkers = await SyncScopeMove.collection
|
|
101
|
+
.find(markerFilter, {projection: {_id: 1, fromStream: 1, seq: 1}})
|
|
102
|
+
.toArray();
|
|
103
|
+
for (const marker of doomedMarkers) {
|
|
104
|
+
raiseWatermark(
|
|
105
|
+
String((marker as {fromStream?: unknown}).fromStream),
|
|
106
|
+
(marker as {seq?: unknown}).seq
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Hard delete soft-deleted docs older than the cutoff. deleteMany on the raw collection
|
|
111
|
+
// bypasses the syncPlugin guard (which blocks deleteMany on the model) — this is a
|
|
112
|
+
// deliberate maintenance-only escape hatch.
|
|
113
|
+
const tombstoneResult = await model.collection.deleteMany(tombstoneFilter);
|
|
114
|
+
const markerResult = await SyncScopeMove.collection.deleteMany(markerFilter);
|
|
115
|
+
|
|
116
|
+
// Raise the watermarks AFTER the deletes: a watermark that runs ahead of the actual
|
|
117
|
+
// deletion would force re-bootstraps for data still perfectly readable.
|
|
118
|
+
for (const [stream, seq] of watermarks) {
|
|
119
|
+
await recordCompactedThroughSeq({seq, stream});
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
markers: markerResult.deletedCount ?? 0,
|
|
124
|
+
tombstones: tombstoneResult.deletedCount ?? 0,
|
|
125
|
+
};
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
/** Compact tombstones across every registered sync model. */
|
|
129
|
+
export const compactTombstones = async (): Promise<CompactTombstonesResult> => {
|
|
130
|
+
const byCollection: CompactTombstonesResult["byCollection"] = {};
|
|
131
|
+
let totalTombstones = 0;
|
|
132
|
+
let totalMarkers = 0;
|
|
133
|
+
for (const entry of getSyncRegistry()) {
|
|
134
|
+
const counts = await compactEntryTombstones(entry);
|
|
135
|
+
byCollection[entry.collectionTag] = counts;
|
|
136
|
+
totalTombstones += counts.tombstones;
|
|
137
|
+
totalMarkers += counts.markers;
|
|
138
|
+
logger.info("[sync] Compacted tombstones", {
|
|
139
|
+
collection: entry.collectionTag,
|
|
140
|
+
markers: counts.markers,
|
|
141
|
+
tombstones: counts.tombstones,
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
return {byCollection, totalMarkers, totalTombstones};
|
|
145
|
+
};
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// noExplicitAny: serialization operates generically across registered models
|
|
2
|
+
// biome-ignore-all lint/suspicious/noExplicitAny: serialization operates generically across registered models
|
|
3
|
+
import type express from "express";
|
|
4
|
+
|
|
5
|
+
import type {SyncRegistryEntry} from "./registry";
|
|
6
|
+
import type {SyncMutationOperation} from "./types";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Shared serializer for sync payloads (snapshot entities, conflict server docs, and
|
|
10
|
+
* change-stream deltas), applying the fallback chain:
|
|
11
|
+
*
|
|
12
|
+
* sync `responseHandler` > modelRouter `responseHandler` > `toJSON()` > raw object.
|
|
13
|
+
*
|
|
14
|
+
* Lives in its own module so both the HTTP routes/mutation handler and the realtime
|
|
15
|
+
* change-stream watcher can import it without deepening the routes -> mutationHandler
|
|
16
|
+
* import cycle. Accepts hydrated Mongoose documents and the raw BSON objects change
|
|
17
|
+
* streams deliver (which have neither `toObject` nor `toJSON`).
|
|
18
|
+
*/
|
|
19
|
+
export const serializeSyncPayload = async ({
|
|
20
|
+
entry,
|
|
21
|
+
doc,
|
|
22
|
+
method = "update",
|
|
23
|
+
req,
|
|
24
|
+
}: {
|
|
25
|
+
entry: SyncRegistryEntry;
|
|
26
|
+
doc: Record<string, unknown>;
|
|
27
|
+
method?: SyncMutationOperation;
|
|
28
|
+
req: express.Request;
|
|
29
|
+
}): Promise<unknown> => {
|
|
30
|
+
const plain =
|
|
31
|
+
typeof (doc as any).toObject === "function"
|
|
32
|
+
? ((doc as any).toObject() as Record<string, unknown>)
|
|
33
|
+
: doc;
|
|
34
|
+
if (entry.config.responseHandler) {
|
|
35
|
+
return entry.config.responseHandler(plain, method);
|
|
36
|
+
}
|
|
37
|
+
if (entry.options.responseHandler) {
|
|
38
|
+
// C8: sync serializes a SINGLE entity (snapshot row, conflict doc, or delta), so the
|
|
39
|
+
// modelRouter responseHandler must run with single-entity `"read"` semantics — not
|
|
40
|
+
// the hardcoded `"list"`, which can trigger list-only shaping (field trimming, array
|
|
41
|
+
// wrapping) that corrupts a single-doc payload.
|
|
42
|
+
return entry.options.responseHandler(doc as any, "read", req, entry.options);
|
|
43
|
+
}
|
|
44
|
+
if (typeof (doc as any).toJSON === "function") {
|
|
45
|
+
return (doc as any).toJSON();
|
|
46
|
+
}
|
|
47
|
+
return plain;
|
|
48
|
+
};
|