@terreno/api 0.31.0 → 56.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +10 -0
- package/dist/api.js +110 -361
- package/dist/auth.d.ts +26 -0
- package/dist/auth.js +121 -26
- package/dist/betterAuth.d.ts +5 -0
- package/dist/betterAuthSetup.js +23 -4
- package/dist/docLoader.js +12 -9
- package/dist/errors.d.ts +13 -0
- package/dist/errors.js +29 -1
- package/dist/index.d.ts +17 -1
- package/dist/index.js +29 -1
- package/dist/middleware.d.ts +7 -0
- package/dist/middleware.js +16 -1
- package/dist/openApiCompat.d.ts +10 -1
- package/dist/openApiCompat.js +115 -47
- package/dist/permissions.d.ts +27 -0
- package/dist/permissions.js +42 -1
- package/dist/plugins.d.ts +13 -0
- package/dist/plugins.js +47 -1
- package/dist/realtime/changeStreamWatcher.d.ts +100 -3
- package/dist/realtime/changeStreamWatcher.js +542 -137
- package/dist/realtime/realtimeApp.d.ts +2 -0
- package/dist/realtime/realtimeApp.js +83 -27
- package/dist/realtime/sessionRevalidation.d.ts +90 -0
- package/dist/realtime/sessionRevalidation.js +438 -0
- package/dist/realtime/socketAuth.d.ts +97 -0
- package/dist/realtime/socketAuth.js +272 -0
- package/dist/realtime/socketUser.d.ts +54 -0
- package/dist/realtime/socketUser.js +75 -5
- package/dist/realtime/types.d.ts +54 -0
- package/dist/secretProviders.d.ts +23 -0
- package/dist/secretProviders.js +9 -3
- package/dist/sync/executors.d.ts +198 -0
- package/dist/sync/executors.js +670 -0
- package/dist/sync/loadHarness.js +807 -0
- package/dist/sync/models.d.ts +216 -0
- package/dist/sync/models.js +644 -0
- package/dist/sync/mutationHandler.d.ts +97 -0
- package/dist/sync/mutationHandler.js +886 -0
- package/dist/sync/registry.d.ts +73 -0
- package/dist/sync/registry.js +223 -0
- package/dist/sync/routes.d.ts +62 -0
- package/dist/sync/routes.js +936 -0
- package/dist/sync/scripts/compactTombstones.d.ts +47 -0
- package/dist/sync/scripts/compactTombstones.js +270 -0
- package/dist/sync/serialize.d.ts +20 -0
- package/dist/{tests/bunSetup.js → sync/serialize.js} +35 -68
- package/dist/sync/socketHandlers.d.ts +54 -0
- package/dist/sync/socketHandlers.js +470 -0
- package/dist/sync/streams.d.ts +85 -0
- package/dist/sync/streams.js +221 -0
- package/dist/sync/syncApp.d.ts +29 -0
- package/dist/sync/syncApp.js +44 -0
- package/dist/sync/syncBatch.d.ts +71 -0
- package/dist/sync/syncBatch.js +252 -0
- package/dist/sync/syncSeqPlugin.d.ts +8 -0
- package/dist/sync/syncSeqPlugin.js +899 -0
- package/dist/sync/types.d.ts +254 -0
- package/dist/sync/types.js +9 -0
- package/dist/terrenoApp.js +90 -27
- package/dist/vendor/wesleytodd-openapi/index.d.ts +1 -1
- package/dist/vendor/wesleytodd-openapi/lib/ui.d.ts +2 -1
- package/package.json +7 -6
- package/src/__tests__/openApiCompat.test.ts +20 -0
- package/src/api.errors.test.ts +32 -23
- package/src/api.test.ts +12 -0
- package/src/api.ts +87 -296
- package/src/auth.test.ts +202 -0
- package/src/auth.ts +134 -23
- package/src/betterAuth.ts +6 -0
- package/src/betterAuthSetup.test.ts +34 -0
- package/src/betterAuthSetup.ts +20 -1
- package/src/configuration.test.ts +59 -38
- package/src/configurationPlugin.test.ts +72 -42
- package/src/docLoader.ts +3 -1
- package/src/errors.ts +33 -0
- package/src/expressServer.test.ts +7 -1
- package/src/index.ts +42 -0
- package/src/middleware.test.ts +56 -1
- package/src/middleware.ts +15 -0
- package/src/openApiCompat.ts +99 -45
- package/src/permissions.test.ts +101 -17
- package/src/permissions.ts +49 -0
- package/src/plugins.test.ts +64 -0
- package/src/plugins.ts +56 -0
- package/src/realtime/changeStreamWatcher.ts +552 -114
- package/src/realtime/realtime.test.ts +157 -0
- package/src/realtime/realtimeApp.ts +82 -14
- package/src/realtime/realtimeAppSetup.test.ts +25 -27
- package/src/realtime/sessionRevalidation.test.ts +423 -0
- package/src/realtime/sessionRevalidation.ts +273 -0
- package/src/realtime/socketAuth.test.ts +355 -0
- package/src/realtime/socketAuth.ts +228 -0
- package/src/realtime/socketUser.ts +69 -0
- package/src/realtime/types.ts +54 -0
- package/src/secretProviders.test.ts +7 -1
- package/src/secretProviders.ts +15 -3
- package/src/secretProvidersGcpClient.test.ts +66 -0
- package/src/sync/ensureSyncIndexes.test.ts +144 -0
- package/src/sync/executors.test.ts +882 -0
- package/src/sync/executors.ts +705 -0
- package/src/sync/integration.test.ts +1160 -0
- package/src/sync/loadHarness.ts +618 -0
- package/src/sync/models.ts +641 -0
- package/src/sync/mutationHandler.test.ts +1252 -0
- package/src/sync/mutationHandler.ts +850 -0
- package/src/sync/registry.ts +232 -0
- package/src/sync/routes.ts +812 -0
- package/src/sync/scripts/compactTombstones.test.ts +249 -0
- package/src/sync/scripts/compactTombstones.ts +145 -0
- package/src/sync/serialize.ts +48 -0
- package/src/sync/sessionRevalidationIntegration.test.ts +281 -0
- package/src/sync/socketHandlers.ts +320 -0
- package/src/sync/streams.ts +188 -0
- package/src/sync/sync.test.ts +582 -0
- package/src/sync/syncApp.ts +44 -0
- package/src/sync/syncBatch.ts +219 -0
- package/src/sync/syncFrontier.test.ts +494 -0
- package/src/sync/syncPhaseC.test.ts +718 -0
- package/src/sync/syncRoutes.test.ts +865 -0
- package/src/sync/syncSeqPlugin.ts +730 -0
- package/src/sync/syncSocket.test.ts +1440 -0
- package/src/sync/types.ts +270 -0
- package/src/terrenoApp.ts +37 -17
- package/src/tests/bunSetup.ts +8 -0
- package/tsconfig.json +4 -4
- package/tsconfig.typedoc.json +7 -1
- package/dist/__tests__/openApiCompat.test.js +0 -317
- package/dist/__tests__/versionCheckPlugin.test.d.ts +0 -1
- package/dist/__tests__/versionCheckPlugin.test.js +0 -597
- package/dist/actions.openApi.test.d.ts +0 -1
- package/dist/actions.openApi.test.js +0 -253
- package/dist/actions.test.d.ts +0 -1
- package/dist/actions.test.js +0 -1087
- package/dist/api.arrayOperations.test.d.ts +0 -1
- package/dist/api.arrayOperations.test.js +0 -878
- package/dist/api.asyncHandler.test.d.ts +0 -1
- package/dist/api.asyncHandler.test.js +0 -236
- package/dist/api.coverage.test.d.ts +0 -1
- package/dist/api.coverage.test.js +0 -192
- package/dist/api.errorPaths.test.d.ts +0 -1
- package/dist/api.errorPaths.test.js +0 -887
- package/dist/api.errors.test.d.ts +0 -1
- package/dist/api.errors.test.js +0 -185
- package/dist/api.hooks.test.d.ts +0 -1
- package/dist/api.hooks.test.js +0 -950
- package/dist/api.query.test.d.ts +0 -1
- package/dist/api.query.test.js +0 -948
- package/dist/api.test.d.ts +0 -1
- package/dist/api.test.js +0 -2820
- package/dist/auth.compile.test.d.ts +0 -1
- package/dist/auth.compile.test.js +0 -201
- package/dist/auth.coverage.test.d.ts +0 -1
- package/dist/auth.coverage.test.js +0 -202
- package/dist/auth.test.d.ts +0 -1
- package/dist/auth.test.js +0 -2286
- package/dist/betterAuth.test.d.ts +0 -1
- package/dist/betterAuth.test.js +0 -181
- package/dist/betterAuthApp.test.d.ts +0 -1
- package/dist/betterAuthApp.test.js +0 -242
- package/dist/betterAuthSetup.test.d.ts +0 -1
- package/dist/betterAuthSetup.test.js +0 -762
- package/dist/config.test.d.ts +0 -1
- package/dist/config.test.js +0 -353
- package/dist/configuration.test.d.ts +0 -1
- package/dist/configuration.test.js +0 -980
- package/dist/configurationPlugin.test.d.ts +0 -1
- package/dist/configurationPlugin.test.js +0 -827
- package/dist/consentApp.test.d.ts +0 -1
- package/dist/consentApp.test.js +0 -1411
- package/dist/docLoader.test.d.ts +0 -1
- package/dist/docLoader.test.js +0 -137
- package/dist/envConfigurationPlugin.test.d.ts +0 -1
- package/dist/envConfigurationPlugin.test.js +0 -369
- package/dist/errors.test.d.ts +0 -1
- package/dist/errors.test.js +0 -607
- package/dist/expressServer.test.d.ts +0 -1
- package/dist/expressServer.test.js +0 -1249
- package/dist/githubAuth.test.d.ts +0 -1
- package/dist/githubAuth.test.js +0 -1142
- package/dist/httpClient.test.d.ts +0 -1
- package/dist/httpClient.test.js +0 -1471
- package/dist/logger.test.d.ts +0 -1
- package/dist/logger.test.js +0 -685
- package/dist/middleware.test.d.ts +0 -1
- package/dist/middleware.test.js +0 -241
- package/dist/notifiers/googleChatNotifier.test.d.ts +0 -1
- package/dist/notifiers/googleChatNotifier.test.js +0 -298
- package/dist/notifiers/slackNotifier.test.d.ts +0 -1
- package/dist/notifiers/slackNotifier.test.js +0 -297
- package/dist/notifiers/zoomNotifier.test.d.ts +0 -1
- package/dist/notifiers/zoomNotifier.test.js +0 -373
- package/dist/openApi.test.d.ts +0 -1
- package/dist/openApi.test.js +0 -620
- package/dist/openApiBuilder.test.d.ts +0 -1
- package/dist/openApiBuilder.test.js +0 -599
- package/dist/openApiEtag.test.d.ts +0 -1
- package/dist/openApiEtag.test.js +0 -86
- package/dist/openApiValidator.test.d.ts +0 -1
- package/dist/openApiValidator.test.js +0 -795
- package/dist/permissions.middleware.test.d.ts +0 -1
- package/dist/permissions.middleware.test.js +0 -308
- package/dist/permissions.test.d.ts +0 -1
- package/dist/permissions.test.js +0 -439
- package/dist/plugins.test.d.ts +0 -1
- package/dist/plugins.test.js +0 -902
- package/dist/populate.test.d.ts +0 -1
- package/dist/populate.test.js +0 -370
- package/dist/realtime/realtime.test.d.ts +0 -10
- package/dist/realtime/realtime.test.js +0 -4213
- package/dist/realtime/realtimeAppSetup.test.d.ts +0 -1
- package/dist/realtime/realtimeAppSetup.test.js +0 -213
- package/dist/requestContext.test.d.ts +0 -1
- package/dist/requestContext.test.js +0 -412
- package/dist/scriptRunner.test.d.ts +0 -1
- package/dist/scriptRunner.test.js +0 -386
- package/dist/secretProviders.test.d.ts +0 -1
- package/dist/secretProviders.test.js +0 -726
- package/dist/syncConsents.test.d.ts +0 -1
- package/dist/syncConsents.test.js +0 -522
- package/dist/terrenoApp.test.d.ts +0 -1
- package/dist/terrenoApp.test.js +0 -489
- package/dist/tests/bunSetup.d.ts +0 -1
- package/dist/tests/fixtures/compileAuthEntry.d.ts +0 -1
- package/dist/tests/fixtures/compileAuthEntry.js +0 -37
- package/dist/tests/testEnv.d.ts +0 -0
- package/dist/tests/testEnv.js +0 -6
- package/dist/transformers.test.d.ts +0 -1
- package/dist/transformers.test.js +0 -423
- package/dist/utils.test.d.ts +0 -1
- package/dist/utils.test.js +0 -242
- /package/dist/{__tests__/openApiCompat.test.d.ts → sync/loadHarness.d.ts} +0 -0
|
@@ -1,12 +1,28 @@
|
|
|
1
|
+
// noExplicitAny: change stream and socket handlers use dynamic document shapes
|
|
2
|
+
// biome-ignore-all lint/suspicious/noExplicitAny: change stream and socket handlers use dynamic document shapes
|
|
3
|
+
|
|
1
4
|
import * as Sentry from "@sentry/bun";
|
|
2
5
|
import type express from "express";
|
|
3
6
|
import {DateTime} from "luxon";
|
|
7
|
+
import type {ChangeStream, ChangeStreamDocument, ChangeStreamOptions} from "mongodb";
|
|
4
8
|
import mongoose from "mongoose";
|
|
5
9
|
import type {Server, Socket} from "socket.io";
|
|
6
10
|
|
|
7
|
-
type
|
|
8
|
-
|
|
9
|
-
|
|
11
|
+
import type {User} from "../auth";
|
|
12
|
+
import {APIError} from "../errors";
|
|
13
|
+
import {logger} from "../logger";
|
|
14
|
+
import {checkPermissions, type PermissionMethod} from "../permissions";
|
|
15
|
+
import {computeStableFrontier, SyncScopeMove} from "../sync/models";
|
|
16
|
+
import {findSyncEntryByCollectionName, type SyncRegistryEntry} from "../sync/registry";
|
|
17
|
+
import {serializeSyncPayload} from "../sync/serialize";
|
|
18
|
+
import {syncRoomForStream} from "../sync/socketHandlers";
|
|
19
|
+
import {resolveStreamForDoc} from "../sync/streams";
|
|
20
|
+
import type {SyncDelta, SyncMutationOperation, SyncResyncHint} from "../sync/types";
|
|
21
|
+
import {matchesQuery} from "./queryMatcher";
|
|
22
|
+
import {getQuerySubscriptionsForCollection} from "./queryStore";
|
|
23
|
+
import {findRegistryEntryByCollection, type RealtimeRegistryEntry} from "./registry";
|
|
24
|
+
import {getSocketUser, type SocketWithDecodedToken} from "./socketUser";
|
|
25
|
+
import type {ChangeStreamConfig, RealtimeEvent} from "./types";
|
|
10
26
|
|
|
11
27
|
/**
|
|
12
28
|
* The subset of ChangeStreamDocument variants this watcher actually processes.
|
|
@@ -18,19 +34,98 @@ type WatchedChange = Extract<
|
|
|
18
34
|
{operationType: "insert" | "update" | "replace" | "delete"}
|
|
19
35
|
>;
|
|
20
36
|
|
|
21
|
-
import type {User} from "../auth";
|
|
22
|
-
import {APIError} from "../errors";
|
|
23
|
-
import {logger} from "../logger";
|
|
24
|
-
import {checkPermissions} from "../permissions";
|
|
25
|
-
import {matchesQuery} from "./queryMatcher";
|
|
26
|
-
import {getQuerySubscriptionsForCollection} from "./queryStore";
|
|
27
|
-
import {findRegistryEntryByCollection, type RealtimeRegistryEntry} from "./registry";
|
|
28
|
-
import {getSocketUser, type SocketWithDecodedToken} from "./socketUser";
|
|
29
|
-
import type {ChangeStreamConfig, RealtimeEvent} from "./types";
|
|
30
|
-
|
|
31
37
|
let changeWatcher: ChangeStream | null = null;
|
|
32
38
|
|
|
33
|
-
|
|
39
|
+
/**
|
|
40
|
+
* Task 9.16: restart state for the single process-wide watcher.
|
|
41
|
+
*
|
|
42
|
+
* The watcher used to only LOG on `error`/`close`: a replica-set election, a network blip,
|
|
43
|
+
* or a dropped cursor permanently killed every realtime event and every `sync:delta` until
|
|
44
|
+
* the process was restarted — online clients silently stopped receiving data while looking
|
|
45
|
+
* perfectly connected. The stream is now re-created with backoff, resuming from the last
|
|
46
|
+
* observed resume token so no events in the gap are missed.
|
|
47
|
+
*/
|
|
48
|
+
/** Resume token of the last change seen, replayed via `resumeAfter` on restart. */
|
|
49
|
+
let lastResumeToken: unknown;
|
|
50
|
+
|
|
51
|
+
/** Bumped on every open/stop so callbacks from a replaced stream are ignored. */
|
|
52
|
+
let watcherGeneration = 0;
|
|
53
|
+
|
|
54
|
+
/** True between `stopChangeStreamWatcher()` and the next `startChangeStreamWatcher()`. */
|
|
55
|
+
let watcherStopped = true;
|
|
56
|
+
|
|
57
|
+
/** Consecutive failed opens, driving the backoff delay. Reset on a successful change. */
|
|
58
|
+
let restartAttempts = 0;
|
|
59
|
+
|
|
60
|
+
let restartTimer: ReturnType<typeof setTimeout> | null = null;
|
|
61
|
+
|
|
62
|
+
/** First restart delay; doubles per consecutive attempt up to the max. */
|
|
63
|
+
export const CHANGE_STREAM_RESTART_BASE_DELAY_MS = 250;
|
|
64
|
+
|
|
65
|
+
/** Ceiling for the restart backoff — a long outage retries once every 30s, forever. */
|
|
66
|
+
export const CHANGE_STREAM_RESTART_MAX_DELAY_MS = 30_000;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Socket event telling clients their cached cursors may have gaps and they must
|
|
70
|
+
* re-bootstrap. Emitted when the change stream's oplog history is gone
|
|
71
|
+
* (`ChangeStreamHistoryLost`), which is the one failure a resume token cannot repair.
|
|
72
|
+
* Payload: {@link SyncResyncHint}.
|
|
73
|
+
*/
|
|
74
|
+
export const SYNC_RESYNC_EVENT = "sync:resync-required";
|
|
75
|
+
|
|
76
|
+
/** MongoDB error code / codeName for an unresumable change stream (oplog rolled past). */
|
|
77
|
+
const CHANGE_STREAM_HISTORY_LOST_CODE = 286;
|
|
78
|
+
|
|
79
|
+
/** True when the error means the resume token can no longer be used. */
|
|
80
|
+
const isHistoryLostError = (error: unknown): boolean => {
|
|
81
|
+
const {code, codeName, message} = (error ?? {}) as {
|
|
82
|
+
code?: unknown;
|
|
83
|
+
codeName?: string;
|
|
84
|
+
message?: string;
|
|
85
|
+
};
|
|
86
|
+
return (
|
|
87
|
+
code === CHANGE_STREAM_HISTORY_LOST_CODE ||
|
|
88
|
+
codeName === "ChangeStreamHistoryLost" ||
|
|
89
|
+
Boolean(message?.includes("ChangeStreamHistoryLost")) ||
|
|
90
|
+
Boolean(message?.includes("resume point may no longer be in the oplog"))
|
|
91
|
+
);
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* C8: per-entity serialized dispatch. The change-stream `"change"` handler is async and
|
|
96
|
+
* multiple events can be in flight at once; without ordering, two events for the SAME
|
|
97
|
+
* document could emit their deltas out of order (a client would then see LWW-by-seq
|
|
98
|
+
* violated within an entity). This keeps a promise chain PER entity key
|
|
99
|
+
* (`{collection}:{docId}`) so same-entity deltas dispatch strictly in change-stream
|
|
100
|
+
* order, while different entities still run concurrently. Chains self-clean when idle.
|
|
101
|
+
*/
|
|
102
|
+
const entityDispatchChains = new Map<string, Promise<void>>();
|
|
103
|
+
|
|
104
|
+
const serializePerEntity = (key: string, task: () => Promise<void>): void => {
|
|
105
|
+
const prior = entityDispatchChains.get(key) ?? Promise.resolve();
|
|
106
|
+
const next = prior.then(task, task);
|
|
107
|
+
entityDispatchChains.set(key, next);
|
|
108
|
+
void next.finally(() => {
|
|
109
|
+
// Drop the chain once this was the last queued task, so the map does not grow.
|
|
110
|
+
if (entityDispatchChains.get(key) === next) {
|
|
111
|
+
entityDispatchChains.delete(key);
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* C7: the sync bookkeeping collections must never drive fan-out — their own change
|
|
118
|
+
* events are internal (counters/ledger/markers/keys), and processing them would emit
|
|
119
|
+
* spurious deltas or reprocess scope-move markers. Exported for testing.
|
|
120
|
+
*/
|
|
121
|
+
export const DEFAULT_IGNORED_COLLECTIONS = [
|
|
122
|
+
"socketio",
|
|
123
|
+
"sessions",
|
|
124
|
+
"synccounters",
|
|
125
|
+
"syncmutations",
|
|
126
|
+
"syncscopemoves",
|
|
127
|
+
"synckeys",
|
|
128
|
+
];
|
|
34
129
|
|
|
35
130
|
/**
|
|
36
131
|
* Map MongoDB change stream operation types to our method names.
|
|
@@ -94,8 +189,17 @@ const getSocketsInRoom = (io: Server, room: string): RealtimeSocketWithAuth[] =>
|
|
|
94
189
|
return sockets;
|
|
95
190
|
};
|
|
96
191
|
|
|
192
|
+
/**
|
|
193
|
+
* The minimal registry-entry shape the authorized-room emitters need. Both
|
|
194
|
+
* `RealtimeRegistryEntry` and `SyncRegistryEntry` satisfy it structurally.
|
|
195
|
+
*/
|
|
196
|
+
export interface AuthorizedEmitEntry {
|
|
197
|
+
modelName: string;
|
|
198
|
+
options: {permissions: {read: PermissionMethod<any>[]}};
|
|
199
|
+
}
|
|
200
|
+
|
|
97
201
|
const canReadDocument = async (
|
|
98
|
-
entry:
|
|
202
|
+
entry: AuthorizedEmitEntry,
|
|
99
203
|
user?: User,
|
|
100
204
|
doc?: Record<string, unknown>
|
|
101
205
|
): Promise<boolean> => {
|
|
@@ -218,14 +322,32 @@ export const serializeDoc = async (
|
|
|
218
322
|
);
|
|
219
323
|
};
|
|
220
324
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
325
|
+
/**
|
|
326
|
+
* Generalized authorized-room emitter: iterates the sockets in `room`, runs the per-socket
|
|
327
|
+
* read-permission check against `fullDocument`, builds the payload per socket (so
|
|
328
|
+
* responseHandlers can tailor output to the receiving user), and emits `eventName`.
|
|
329
|
+
*
|
|
330
|
+
* `emitToAuthorizedRoom` (legacy "sync" events) and the sync layer's `sync:delta`
|
|
331
|
+
* emission both delegate here.
|
|
332
|
+
*/
|
|
333
|
+
export const emitPayloadToAuthorizedRoom = async ({
|
|
334
|
+
io,
|
|
335
|
+
room,
|
|
336
|
+
eventName,
|
|
337
|
+
entry,
|
|
338
|
+
fullDocument,
|
|
339
|
+
buildPayload,
|
|
340
|
+
logDebug,
|
|
341
|
+
}: {
|
|
342
|
+
io: Server;
|
|
343
|
+
room: string;
|
|
344
|
+
eventName: string;
|
|
345
|
+
entry: AuthorizedEmitEntry;
|
|
346
|
+
fullDocument: Record<string, unknown> | undefined;
|
|
347
|
+
/** Build the per-socket payload; a throw drops the emission for that socket only. */
|
|
348
|
+
buildPayload: (user: User | undefined) => Promise<unknown> | unknown;
|
|
349
|
+
logDebug: (msg: string) => void;
|
|
350
|
+
}): Promise<void> => {
|
|
229
351
|
const sockets = getSocketsInRoom(io, room);
|
|
230
352
|
for (const socket of sockets) {
|
|
231
353
|
const user = getSocketUser(socket);
|
|
@@ -239,22 +361,40 @@ export const emitToAuthorizedRoom = async (
|
|
|
239
361
|
continue;
|
|
240
362
|
}
|
|
241
363
|
|
|
242
|
-
|
|
243
|
-
socket.emit("sync", event);
|
|
244
|
-
continue;
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
const data = await serializeDoc(entry, fullDocument, event.method, user);
|
|
248
|
-
socket.emit("sync", {...event, data});
|
|
364
|
+
socket.emit(eventName, await buildPayload(user));
|
|
249
365
|
} catch (error) {
|
|
250
366
|
logger.error(
|
|
251
|
-
`[realtime] Failed to emit ${entry.modelName}
|
|
367
|
+
`[realtime] Failed to emit ${eventName} for ${entry.modelName} to socket ${socket.id}: ${error}`
|
|
252
368
|
);
|
|
253
369
|
Sentry.captureException(error);
|
|
254
370
|
}
|
|
255
371
|
}
|
|
256
372
|
};
|
|
257
373
|
|
|
374
|
+
export const emitToAuthorizedRoom = async (
|
|
375
|
+
io: Server,
|
|
376
|
+
room: string,
|
|
377
|
+
event: RealtimeEvent,
|
|
378
|
+
entry: RealtimeRegistryEntry,
|
|
379
|
+
fullDocument: Record<string, unknown> | undefined,
|
|
380
|
+
logDebug: (msg: string) => void
|
|
381
|
+
): Promise<void> =>
|
|
382
|
+
emitPayloadToAuthorizedRoom({
|
|
383
|
+
buildPayload: async (user) => {
|
|
384
|
+
if (!fullDocument) {
|
|
385
|
+
return event;
|
|
386
|
+
}
|
|
387
|
+
const data = await serializeDoc(entry, fullDocument, event.method, user);
|
|
388
|
+
return {...event, data};
|
|
389
|
+
},
|
|
390
|
+
entry,
|
|
391
|
+
eventName: "sync",
|
|
392
|
+
fullDocument,
|
|
393
|
+
io,
|
|
394
|
+
logDebug,
|
|
395
|
+
room,
|
|
396
|
+
});
|
|
397
|
+
|
|
258
398
|
/**
|
|
259
399
|
* Emit a sync event to document-specific and query rooms.
|
|
260
400
|
*
|
|
@@ -367,14 +507,324 @@ export const emitToDocumentAndQueryRooms = async (
|
|
|
367
507
|
}
|
|
368
508
|
};
|
|
369
509
|
|
|
510
|
+
/**
|
|
511
|
+
* Emit legacy realtime "sync" events for a change on a realtime-registered collection.
|
|
512
|
+
* Extracted verbatim from the change handler so sync-delta emission (which is keyed off
|
|
513
|
+
* the independent sync registry) runs even when the realtime path skips the change.
|
|
514
|
+
*/
|
|
515
|
+
const processRealtimeChange = async ({
|
|
516
|
+
io,
|
|
517
|
+
entry,
|
|
518
|
+
change,
|
|
519
|
+
docId,
|
|
520
|
+
logDebug,
|
|
521
|
+
logInfo,
|
|
522
|
+
}: {
|
|
523
|
+
io: Server;
|
|
524
|
+
entry: RealtimeRegistryEntry;
|
|
525
|
+
change: WatchedChange;
|
|
526
|
+
docId: string;
|
|
527
|
+
logDebug: (msg: string) => void;
|
|
528
|
+
logInfo: (msg: string) => void;
|
|
529
|
+
}): Promise<void> => {
|
|
530
|
+
// Map to our method type. Pass enabledMethods so soft deletes only
|
|
531
|
+
// remap to "delete" when the model actually subscribes to deletes —
|
|
532
|
+
// otherwise the change is kept as "update" so update subscribers
|
|
533
|
+
// still receive it.
|
|
534
|
+
const method = mapOperationType(change.operationType, change, entry.config.methods);
|
|
535
|
+
if (!method) {
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// Check if this method is enabled for this model
|
|
540
|
+
if (!entry.config.methods.includes(method)) {
|
|
541
|
+
logDebug(`[realtime] Method ${method} not enabled for ${entry.modelName}`);
|
|
542
|
+
return;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// fullDocument is present on insert/update/replace; absent on delete.
|
|
546
|
+
const fullDocument = change.operationType === "delete" ? undefined : change.fullDocument;
|
|
547
|
+
|
|
548
|
+
// For hard deletes, we don't have the full document
|
|
549
|
+
const isHardDelete = method === "delete" && !fullDocument;
|
|
550
|
+
|
|
551
|
+
// Determine target rooms
|
|
552
|
+
let rooms: string[];
|
|
553
|
+
if (isHardDelete) {
|
|
554
|
+
// Hard delete: no fullDocument, so we can't resolve owner/custom rooms.
|
|
555
|
+
// For owner strategy we cannot safely fan out to a model room without
|
|
556
|
+
// leaking deletes across users — admins still receive the event via the
|
|
557
|
+
// admin room and any document-specific subscribers via document rooms.
|
|
558
|
+
if (entry.config.roomStrategy === "owner") {
|
|
559
|
+
rooms = ["admin"];
|
|
560
|
+
} else if (entry.config.roomStrategy === "broadcast") {
|
|
561
|
+
rooms = ["authenticated"];
|
|
562
|
+
} else {
|
|
563
|
+
const collectionTag = getCollectionTag(entry.routePath);
|
|
564
|
+
rooms = [`model:${collectionTag}`];
|
|
565
|
+
}
|
|
566
|
+
} else {
|
|
567
|
+
rooms = resolveRooms(entry, fullDocument ?? {}, method);
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
const collection = getCollectionTag(entry.routePath);
|
|
571
|
+
|
|
572
|
+
const event: RealtimeEvent = {
|
|
573
|
+
collection,
|
|
574
|
+
id: docId,
|
|
575
|
+
method,
|
|
576
|
+
model: entry.modelName,
|
|
577
|
+
timestamp: DateTime.now().toMillis(),
|
|
578
|
+
...(change.operationType === "update" && change.updateDescription?.updatedFields
|
|
579
|
+
? {updatedFields: Object.keys(change.updateDescription.updatedFields)}
|
|
580
|
+
: {}),
|
|
581
|
+
};
|
|
582
|
+
|
|
583
|
+
// Emit to strategy-based rooms (model/owner/broadcast)
|
|
584
|
+
for (const room of rooms) {
|
|
585
|
+
await emitToAuthorizedRoom(io, room, event, entry, fullDocument, logDebug);
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
// Emit to document-specific and query rooms
|
|
589
|
+
await emitToDocumentAndQueryRooms(io, collection, event, fullDocument, logDebug, entry);
|
|
590
|
+
|
|
591
|
+
logDebug(
|
|
592
|
+
`[realtime] Emitted ${method} for ${entry.modelName}/${docId} to rooms: ${rooms.join(", ")}`
|
|
593
|
+
);
|
|
594
|
+
// Log only metadata — never the document payload, which may contain sensitive fields.
|
|
595
|
+
const metadata: Record<string, unknown> = {
|
|
596
|
+
collection: event.collection,
|
|
597
|
+
id: event.id,
|
|
598
|
+
method: event.method,
|
|
599
|
+
model: event.model,
|
|
600
|
+
timestamp: event.timestamp,
|
|
601
|
+
};
|
|
602
|
+
if (event.updatedFields) {
|
|
603
|
+
metadata.updatedFields = event.updatedFields;
|
|
604
|
+
}
|
|
605
|
+
logInfo(`[realtime] sync event: ${JSON.stringify(metadata)}`);
|
|
606
|
+
};
|
|
607
|
+
|
|
608
|
+
/**
|
|
609
|
+
* Emit `sync:delta` events for a change on a sync-registered collection.
|
|
610
|
+
*
|
|
611
|
+
* Deltas fan out to the dedicated `sync:{stream}` rooms joined via `sync:subscribe`
|
|
612
|
+
* (independent of the legacy realtime rooms), with the same per-socket read-permission
|
|
613
|
+
* checks as legacy events. `seq` and `stream` come from the post-image: the `syncPlugin`
|
|
614
|
+
* stamps `_syncSeq` on every synced write and the watcher runs `updateLookup`.
|
|
615
|
+
*
|
|
616
|
+
* Soft deletes (an update setting `deleted: true`, reclassified by `mapOperationType`)
|
|
617
|
+
* produce a `method: "delete"` delta with `deleted: true` and the tombstone data intact.
|
|
618
|
+
*
|
|
619
|
+
* Scope moves (C4): the old-stream tombstone is derived from durable `SyncScopeMove`
|
|
620
|
+
* markers (written by `syncPlugin` in the same op-scope as the move), NOT from the racy
|
|
621
|
+
* `_syncPrevStream` post-image — a second write racing this event's processing can reset
|
|
622
|
+
* `_syncPrevStream`, but it cannot erase the durable marker. For each marker on this
|
|
623
|
+
* document, a data-less tombstone is emitted to the marker's `fromStream` (carrying the
|
|
624
|
+
* marker's old-stream seq), and the new-stream delta is reported as `method: "create"`.
|
|
625
|
+
* Client tombstone application is idempotent by seq, so re-emitting a marker on a later
|
|
626
|
+
* change of the same doc is harmless.
|
|
627
|
+
*
|
|
628
|
+
* Every emitted delta carries `frontierSeq` (the emitting stream's stable frontier at
|
|
629
|
+
* emit time, C1); the client advances its cursor to `min(delta.seq, delta.frontierSeq)`
|
|
630
|
+
* so a delta observed out of commit order never advances a cursor past an uncommitted hole.
|
|
631
|
+
*
|
|
632
|
+
* A model with both `realtime` and `sync` configs emits both the legacy "sync" event and
|
|
633
|
+
* "sync:delta" — distinct event names, no interference (documented as transitional).
|
|
634
|
+
*
|
|
635
|
+
* Exported for testing.
|
|
636
|
+
*/
|
|
637
|
+
export const emitSyncDeltaForChange = async ({
|
|
638
|
+
io,
|
|
639
|
+
entry,
|
|
640
|
+
change,
|
|
641
|
+
docId,
|
|
642
|
+
logDebug,
|
|
643
|
+
}: {
|
|
644
|
+
io: Server;
|
|
645
|
+
entry: SyncRegistryEntry;
|
|
646
|
+
change: WatchedChange;
|
|
647
|
+
docId: string;
|
|
648
|
+
logDebug: (msg: string) => void;
|
|
649
|
+
}): Promise<void> => {
|
|
650
|
+
if (change.operationType === "delete") {
|
|
651
|
+
// Hard deletes are blocked on synced models by syncPlugin; a direct collection-level
|
|
652
|
+
// delete has no post-image, so neither stream nor seq can be resolved. Clients
|
|
653
|
+
// reconcile via snapshot instead.
|
|
654
|
+
logDebug(
|
|
655
|
+
`[sync] Skipping hard delete for synced collection ${entry.collectionTag}/${docId}: ` +
|
|
656
|
+
"no post-image to resolve stream/seq"
|
|
657
|
+
);
|
|
658
|
+
return;
|
|
659
|
+
}
|
|
660
|
+
const fullDocument = change.fullDocument as Record<string, unknown> | undefined;
|
|
661
|
+
if (!fullDocument) {
|
|
662
|
+
logDebug(`[sync] Skipping ${entry.collectionTag}/${docId}: change has no fullDocument`);
|
|
663
|
+
return;
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
const mapped = mapOperationType(change.operationType, change);
|
|
667
|
+
if (!mapped) {
|
|
668
|
+
return;
|
|
669
|
+
}
|
|
670
|
+
let method: SyncMutationOperation = mapped;
|
|
671
|
+
const deleted = fullDocument.deleted === true;
|
|
672
|
+
const seq = typeof fullDocument._syncSeq === "number" ? fullDocument._syncSeq : 0;
|
|
673
|
+
const stream = resolveStreamForDoc({
|
|
674
|
+
collectionTag: entry.collectionTag,
|
|
675
|
+
doc: fullDocument,
|
|
676
|
+
scope: entry.config.scope,
|
|
677
|
+
});
|
|
678
|
+
|
|
679
|
+
// C4: emit old-stream tombstones from durable markers, not the racy _syncPrevStream.
|
|
680
|
+
const markers = await SyncScopeMove.find({
|
|
681
|
+
collectionTag: entry.collectionTag,
|
|
682
|
+
entityId: docId,
|
|
683
|
+
}).lean();
|
|
684
|
+
let movedAway = false;
|
|
685
|
+
for (const marker of markers) {
|
|
686
|
+
if (marker.fromStream === stream) {
|
|
687
|
+
continue;
|
|
688
|
+
}
|
|
689
|
+
movedAway = true;
|
|
690
|
+
const markerFrontier = await computeStableFrontier({stream: marker.fromStream});
|
|
691
|
+
const tombstone: SyncDelta = {
|
|
692
|
+
collection: entry.collectionTag,
|
|
693
|
+
deleted: true,
|
|
694
|
+
frontierSeq: markerFrontier,
|
|
695
|
+
id: docId,
|
|
696
|
+
method: "delete",
|
|
697
|
+
seq: marker.seq,
|
|
698
|
+
stream: marker.fromStream,
|
|
699
|
+
};
|
|
700
|
+
await emitPayloadToAuthorizedRoom({
|
|
701
|
+
buildPayload: () => tombstone,
|
|
702
|
+
entry,
|
|
703
|
+
eventName: "sync:delta",
|
|
704
|
+
fullDocument,
|
|
705
|
+
io,
|
|
706
|
+
logDebug,
|
|
707
|
+
room: syncRoomForStream(marker.fromStream),
|
|
708
|
+
});
|
|
709
|
+
logDebug(
|
|
710
|
+
`[sync] Emitted scope-move tombstone for ${entry.collectionTag}/${docId} to ${marker.fromStream} seq=${marker.seq}`
|
|
711
|
+
);
|
|
712
|
+
}
|
|
713
|
+
// The new stream sees a moved-away doc for the first time: report it as a create
|
|
714
|
+
// (unless the same write also soft-deleted the doc, in which case the tombstone wins).
|
|
715
|
+
if (movedAway && method !== "delete") {
|
|
716
|
+
method = "create";
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
const frontierSeq = await computeStableFrontier({stream});
|
|
720
|
+
const delta: SyncDelta = {
|
|
721
|
+
collection: entry.collectionTag,
|
|
722
|
+
frontierSeq,
|
|
723
|
+
id: docId,
|
|
724
|
+
method,
|
|
725
|
+
seq,
|
|
726
|
+
stream,
|
|
727
|
+
...(deleted ? {deleted: true} : {}),
|
|
728
|
+
};
|
|
729
|
+
await emitPayloadToAuthorizedRoom({
|
|
730
|
+
// C7: tombstone deltas carry no data (only id/seq/deleted); live deltas serialize.
|
|
731
|
+
buildPayload: deleted
|
|
732
|
+
? () => delta
|
|
733
|
+
: async (user) => {
|
|
734
|
+
const syntheticReq = {params: {}, query: {}, user} as unknown as express.Request;
|
|
735
|
+
const data = ensureApiId(
|
|
736
|
+
await serializeSyncPayload({doc: fullDocument, entry, method, req: syntheticReq})
|
|
737
|
+
);
|
|
738
|
+
return {...delta, data};
|
|
739
|
+
},
|
|
740
|
+
entry,
|
|
741
|
+
eventName: "sync:delta",
|
|
742
|
+
fullDocument,
|
|
743
|
+
io,
|
|
744
|
+
logDebug,
|
|
745
|
+
room: syncRoomForStream(stream),
|
|
746
|
+
});
|
|
747
|
+
logDebug(
|
|
748
|
+
`[sync] Emitted sync:delta ${method} for ${entry.collectionTag}/${docId} seq=${seq} stream=${stream} frontier=${frontierSeq}`
|
|
749
|
+
);
|
|
750
|
+
};
|
|
751
|
+
|
|
370
752
|
/**
|
|
371
753
|
* Start watching MongoDB change streams and emitting real-time events.
|
|
754
|
+
*
|
|
755
|
+
* Task 9.16: the stream is supervised — an `error`/`close`/`end` schedules a re-open with
|
|
756
|
+
* exponential backoff, resuming from the last seen token. Call
|
|
757
|
+
* {@link stopChangeStreamWatcher} to end supervision.
|
|
372
758
|
*/
|
|
373
759
|
export const startChangeStreamWatcher = (
|
|
374
760
|
io: Server,
|
|
375
761
|
config: ChangeStreamConfig = {},
|
|
376
762
|
debug = false
|
|
377
763
|
): void => {
|
|
764
|
+
watcherStopped = false;
|
|
765
|
+
restartAttempts = 0;
|
|
766
|
+
lastResumeToken = undefined;
|
|
767
|
+
openChangeStream({config, debug, io});
|
|
768
|
+
};
|
|
769
|
+
|
|
770
|
+
/**
|
|
771
|
+
* Task 9.16: schedule a re-open after a stream failure. Silently returns when the watcher
|
|
772
|
+
* was stopped, when a newer stream is already running (a failing stream can fire several
|
|
773
|
+
* terminal events), or when a restart is already pending.
|
|
774
|
+
*/
|
|
775
|
+
const scheduleChangeStreamRestart = ({
|
|
776
|
+
config,
|
|
777
|
+
debug,
|
|
778
|
+
generation,
|
|
779
|
+
io,
|
|
780
|
+
reason,
|
|
781
|
+
}: {
|
|
782
|
+
config: ChangeStreamConfig;
|
|
783
|
+
debug: boolean;
|
|
784
|
+
generation: number;
|
|
785
|
+
io: Server;
|
|
786
|
+
reason: string;
|
|
787
|
+
}): void => {
|
|
788
|
+
if (watcherStopped || generation !== watcherGeneration || restartTimer) {
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
restartAttempts += 1;
|
|
792
|
+
const delayMs = Math.min(
|
|
793
|
+
CHANGE_STREAM_RESTART_BASE_DELAY_MS * 2 ** (restartAttempts - 1),
|
|
794
|
+
CHANGE_STREAM_RESTART_MAX_DELAY_MS
|
|
795
|
+
);
|
|
796
|
+
logger.warn(
|
|
797
|
+
`[realtime] Change stream ${reason}; reopening in ${delayMs}ms (attempt ${restartAttempts})`,
|
|
798
|
+
{resumable: lastResumeToken !== undefined}
|
|
799
|
+
);
|
|
800
|
+
restartTimer = setTimeout(() => {
|
|
801
|
+
restartTimer = null;
|
|
802
|
+
if (watcherStopped) {
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
openChangeStream({config, debug, io, isReopen: true});
|
|
806
|
+
}, delayMs);
|
|
807
|
+
// Never hold the process open just to retry a watcher.
|
|
808
|
+
restartTimer.unref?.();
|
|
809
|
+
};
|
|
810
|
+
|
|
811
|
+
/**
|
|
812
|
+
* Open (or re-open) the change stream and wire its listeners. Throws only for a
|
|
813
|
+
* configuration-level failure on the FIRST open (no connection, `watch` returned nothing),
|
|
814
|
+
* which must fail startup loudly; a failure while reopening is retried with backoff.
|
|
815
|
+
*/
|
|
816
|
+
const openChangeStream = ({
|
|
817
|
+
config,
|
|
818
|
+
debug,
|
|
819
|
+
io,
|
|
820
|
+
isReopen = false,
|
|
821
|
+
}: {
|
|
822
|
+
config: ChangeStreamConfig;
|
|
823
|
+
debug: boolean;
|
|
824
|
+
io: Server;
|
|
825
|
+
/** True for a supervised reopen: failures retry with backoff instead of throwing. */
|
|
826
|
+
isReopen?: boolean;
|
|
827
|
+
}): void => {
|
|
378
828
|
const logInfo = (message: string): void => {
|
|
379
829
|
if (debug) {
|
|
380
830
|
logger.info(message);
|
|
@@ -387,6 +837,8 @@ export const startChangeStreamWatcher = (
|
|
|
387
837
|
}
|
|
388
838
|
};
|
|
389
839
|
|
|
840
|
+
const generation = ++watcherGeneration;
|
|
841
|
+
|
|
390
842
|
try {
|
|
391
843
|
logInfo("[realtime] Initializing change stream watcher...");
|
|
392
844
|
|
|
@@ -428,16 +880,30 @@ export const startChangeStreamWatcher = (
|
|
|
428
880
|
// How long the cursor waits for new events before yielding control.
|
|
429
881
|
// Lower values give more responsive updates at the cost of more frequent driver round-trips.
|
|
430
882
|
maxAwaitTimeMS: 1000,
|
|
883
|
+
// Task 9.16: resume where the previous stream died so events during the outage are
|
|
884
|
+
// delivered rather than lost. Absent on a first open and after history loss.
|
|
885
|
+
...(lastResumeToken !== undefined ? {resumeAfter: lastResumeToken} : {}),
|
|
431
886
|
};
|
|
432
887
|
|
|
433
|
-
|
|
888
|
+
const stream = nativeDb.watch(pipeline, options);
|
|
434
889
|
|
|
435
|
-
if (!
|
|
890
|
+
if (!stream) {
|
|
436
891
|
throw new APIError({status: 500, title: "Failed to create change stream watcher"});
|
|
437
892
|
}
|
|
893
|
+
changeWatcher = stream;
|
|
438
894
|
|
|
439
|
-
|
|
895
|
+
stream.on("change", async (rawChange: ChangeStreamDocument) => {
|
|
440
896
|
try {
|
|
897
|
+
// Task 9.16: record the resume token FIRST, for every change — including ones this
|
|
898
|
+
// watcher ignores. A token only from processed events would rewind the stream to an
|
|
899
|
+
// old position after a restart and re-deliver everything since.
|
|
900
|
+
const token = (rawChange as {_id?: unknown})._id;
|
|
901
|
+
if (token !== undefined) {
|
|
902
|
+
lastResumeToken = token;
|
|
903
|
+
}
|
|
904
|
+
// A stream that is delivering events has recovered; drop the accumulated backoff so
|
|
905
|
+
// the next unrelated failure retries promptly instead of at the previous ceiling.
|
|
906
|
+
restartAttempts = 0;
|
|
441
907
|
// The pipeline restricts operationType to a subset that always has ns/documentKey;
|
|
442
908
|
// narrow once here so downstream code doesn't need repeated casts.
|
|
443
909
|
if (
|
|
@@ -461,122 +927,94 @@ export const startChangeStreamWatcher = (
|
|
|
461
927
|
return;
|
|
462
928
|
}
|
|
463
929
|
|
|
464
|
-
//
|
|
930
|
+
// Realtime and sync registries are independent: a model may be in either or both.
|
|
931
|
+
// A model with both configs emits both the legacy "sync" event and "sync:delta".
|
|
465
932
|
const entry = findRegistryEntryByCollection(collectionName);
|
|
933
|
+
const syncEntry = findSyncEntryByCollectionName(collectionName);
|
|
466
934
|
|
|
467
|
-
if (!entry) {
|
|
468
|
-
// Not a registered realtime model — skip
|
|
935
|
+
if (!entry && !syncEntry) {
|
|
936
|
+
// Not a registered realtime or sync model — skip
|
|
469
937
|
logDebug(`[realtime] No registry entry for collection: ${collectionName}`);
|
|
470
938
|
return;
|
|
471
939
|
}
|
|
472
940
|
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
// otherwise the change is kept as "update" so update subscribers
|
|
476
|
-
// still receive it.
|
|
477
|
-
const method = mapOperationType(change.operationType, change, entry.config.methods);
|
|
478
|
-
if (!method) {
|
|
479
|
-
return;
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
// Check if this method is enabled for this model
|
|
483
|
-
if (!entry.config.methods.includes(method)) {
|
|
484
|
-
logDebug(`[realtime] Method ${method} not enabled for ${entry.modelName}`);
|
|
485
|
-
return;
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
// fullDocument is present on insert/update/replace; absent on delete.
|
|
489
|
-
const fullDocument = change.operationType === "delete" ? undefined : change.fullDocument;
|
|
490
|
-
|
|
491
|
-
// For hard deletes, we don't have the full document
|
|
492
|
-
const isHardDelete = method === "delete" && !fullDocument;
|
|
493
|
-
|
|
494
|
-
// Determine target rooms
|
|
495
|
-
let rooms: string[];
|
|
496
|
-
if (isHardDelete) {
|
|
497
|
-
// Hard delete: no fullDocument, so we can't resolve owner/custom rooms.
|
|
498
|
-
// For owner strategy we cannot safely fan out to a model room without
|
|
499
|
-
// leaking deletes across users — admins still receive the event via the
|
|
500
|
-
// admin room and any document-specific subscribers via document rooms.
|
|
501
|
-
if (entry.config.roomStrategy === "owner") {
|
|
502
|
-
rooms = ["admin"];
|
|
503
|
-
} else if (entry.config.roomStrategy === "broadcast") {
|
|
504
|
-
rooms = ["authenticated"];
|
|
505
|
-
} else {
|
|
506
|
-
const collectionTag = getCollectionTag(entry.routePath);
|
|
507
|
-
rooms = [`model:${collectionTag}`];
|
|
508
|
-
}
|
|
509
|
-
} else {
|
|
510
|
-
rooms = resolveRooms(entry, fullDocument ?? {}, method);
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
const collection = getCollectionTag(entry.routePath);
|
|
514
|
-
|
|
515
|
-
const event: RealtimeEvent = {
|
|
516
|
-
collection,
|
|
517
|
-
id: docId,
|
|
518
|
-
method,
|
|
519
|
-
model: entry.modelName,
|
|
520
|
-
timestamp: DateTime.now().toMillis(),
|
|
521
|
-
...(change.operationType === "update" && change.updateDescription?.updatedFields
|
|
522
|
-
? {updatedFields: Object.keys(change.updateDescription.updatedFields)}
|
|
523
|
-
: {}),
|
|
524
|
-
};
|
|
525
|
-
|
|
526
|
-
// Emit to strategy-based rooms (model/owner/broadcast)
|
|
527
|
-
for (const room of rooms) {
|
|
528
|
-
await emitToAuthorizedRoom(io, room, event, entry, fullDocument, logDebug);
|
|
941
|
+
if (entry) {
|
|
942
|
+
await processRealtimeChange({change, docId, entry, io, logDebug, logInfo});
|
|
529
943
|
}
|
|
530
944
|
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
timestamp: event.timestamp,
|
|
544
|
-
};
|
|
545
|
-
if (event.updatedFields) {
|
|
546
|
-
metadata.updatedFields = event.updatedFields;
|
|
945
|
+
if (syncEntry) {
|
|
946
|
+
// C8: serialize per entity so same-doc deltas emit in change-stream order;
|
|
947
|
+
// different docs still dispatch concurrently. See serializePerEntity + the
|
|
948
|
+
// per-entity LWW-by-seq contract documented in sync/types.ts (SyncDelta).
|
|
949
|
+
serializePerEntity(`${collectionName}:${docId}`, async () => {
|
|
950
|
+
try {
|
|
951
|
+
await emitSyncDeltaForChange({change, docId, entry: syncEntry, io, logDebug});
|
|
952
|
+
} catch (error) {
|
|
953
|
+
logger.error(`[sync] Error emitting sync delta: ${error}`);
|
|
954
|
+
Sentry.captureException(error);
|
|
955
|
+
}
|
|
956
|
+
});
|
|
547
957
|
}
|
|
548
|
-
logInfo(`[realtime] sync event: ${JSON.stringify(metadata)}`);
|
|
549
958
|
} catch (error) {
|
|
550
959
|
logger.error(`[realtime] Error processing change event: ${error}`);
|
|
551
960
|
Sentry.captureException(error);
|
|
552
961
|
}
|
|
553
962
|
});
|
|
554
963
|
|
|
555
|
-
|
|
964
|
+
stream.on("error", (err: Error) => {
|
|
556
965
|
Sentry.captureException(err);
|
|
557
966
|
logger.error(`[realtime] Change stream error: ${err?.message || err}`);
|
|
967
|
+
if (isHistoryLostError(err)) {
|
|
968
|
+
// The oplog no longer contains the resume point, so the gap can never be replayed.
|
|
969
|
+
// Reopen from NOW and tell clients to re-bootstrap: their cursors may sit below
|
|
970
|
+
// changes (including deletions) they will otherwise never hear about.
|
|
971
|
+
lastResumeToken = undefined;
|
|
972
|
+
logger.error(
|
|
973
|
+
"[sync] Change stream history lost; reopening without a resume token and asking " +
|
|
974
|
+
"clients to resync"
|
|
975
|
+
);
|
|
976
|
+
const hint: SyncResyncHint = {reason: "history_lost"};
|
|
977
|
+
io.emit(SYNC_RESYNC_EVENT, hint);
|
|
978
|
+
}
|
|
979
|
+
scheduleChangeStreamRestart({config, debug, generation, io, reason: "errored"});
|
|
558
980
|
});
|
|
559
981
|
|
|
560
|
-
|
|
982
|
+
stream.on("close", () => {
|
|
561
983
|
logger.warn("[realtime] Change stream closed");
|
|
984
|
+
scheduleChangeStreamRestart({config, debug, generation, io, reason: "closed"});
|
|
562
985
|
});
|
|
563
986
|
|
|
564
|
-
|
|
987
|
+
stream.on("end", () => {
|
|
565
988
|
logger.warn("[realtime] Change stream ended");
|
|
989
|
+
scheduleChangeStreamRestart({config, debug, generation, io, reason: "ended"});
|
|
566
990
|
});
|
|
567
991
|
|
|
568
992
|
logInfo("[realtime] Change stream watcher initialized successfully");
|
|
569
993
|
} catch (error) {
|
|
570
994
|
logger.error(`[realtime] Failed to initialize change stream watcher: ${error}`);
|
|
571
995
|
Sentry.captureException(error);
|
|
996
|
+
// A first open must fail startup loudly; a failed REOPEN (the DB is down mid-flight)
|
|
997
|
+
// has no caller to throw to, so keep retrying with backoff instead.
|
|
998
|
+
if (isReopen && !watcherStopped) {
|
|
999
|
+
scheduleChangeStreamRestart({config, debug, generation, io, reason: "failed to open"});
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
572
1002
|
throw error;
|
|
573
1003
|
}
|
|
574
1004
|
};
|
|
575
1005
|
|
|
576
1006
|
/**
|
|
577
|
-
* Stop the change stream watcher.
|
|
1007
|
+
* Stop the change stream watcher and end Task 9.16's restart supervision (a pending
|
|
1008
|
+
* reopen is cancelled, and terminal events from the closing stream are ignored).
|
|
578
1009
|
*/
|
|
579
1010
|
export const stopChangeStreamWatcher = async (): Promise<void> => {
|
|
1011
|
+
watcherStopped = true;
|
|
1012
|
+
if (restartTimer) {
|
|
1013
|
+
clearTimeout(restartTimer);
|
|
1014
|
+
restartTimer = null;
|
|
1015
|
+
}
|
|
1016
|
+
restartAttempts = 0;
|
|
1017
|
+
lastResumeToken = undefined;
|
|
580
1018
|
if (changeWatcher) {
|
|
581
1019
|
await changeWatcher.close();
|
|
582
1020
|
changeWatcher = null;
|