@terreno/api 0.31.0 → 56.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (250) hide show
  1. package/CHANGELOG.md +122 -2
  2. package/LICENSE +202 -21
  3. package/dist/api.d.ts +12 -3
  4. package/dist/api.js +110 -361
  5. package/dist/auth.d.ts +26 -0
  6. package/dist/auth.js +100 -1
  7. package/dist/betterAuth.d.ts +5 -0
  8. package/dist/betterAuthSetup.d.ts +1 -15
  9. package/dist/betterAuthSetup.js +29 -8
  10. package/dist/configurationPlugin.js +2 -2
  11. package/dist/docLoader.js +12 -9
  12. package/dist/errors.d.ts +13 -0
  13. package/dist/errors.js +29 -1
  14. package/dist/expressServer.js +5 -11
  15. package/dist/githubAuth.js +4 -1
  16. package/dist/index.d.ts +17 -1
  17. package/dist/index.js +29 -1
  18. package/dist/middleware.d.ts +7 -0
  19. package/dist/middleware.js +16 -1
  20. package/dist/openApiCompat.d.ts +10 -1
  21. package/dist/openApiCompat.js +115 -47
  22. package/dist/permissions.d.ts +27 -0
  23. package/dist/permissions.js +42 -1
  24. package/dist/plugins.d.ts +15 -3
  25. package/dist/plugins.js +36 -6
  26. package/dist/realtime/changeStreamWatcher.d.ts +100 -3
  27. package/dist/realtime/changeStreamWatcher.js +542 -137
  28. package/dist/realtime/realtimeApp.d.ts +3 -1
  29. package/dist/realtime/realtimeApp.js +75 -27
  30. package/dist/realtime/sessionRevalidation.d.ts +90 -0
  31. package/dist/realtime/sessionRevalidation.js +438 -0
  32. package/dist/realtime/socketAuth.d.ts +97 -0
  33. package/dist/realtime/socketAuth.js +272 -0
  34. package/dist/realtime/socketUser.d.ts +54 -0
  35. package/dist/realtime/socketUser.js +75 -5
  36. package/dist/realtime/types.d.ts +40 -0
  37. package/dist/secretProviders.d.ts +23 -0
  38. package/dist/secretProviders.js +9 -3
  39. package/dist/sync/executors.d.ts +198 -0
  40. package/dist/sync/executors.js +670 -0
  41. package/dist/sync/loadHarness.js +807 -0
  42. package/dist/sync/models.d.ts +216 -0
  43. package/dist/sync/models.js +644 -0
  44. package/dist/sync/mutationHandler.d.ts +97 -0
  45. package/dist/sync/mutationHandler.js +886 -0
  46. package/dist/sync/registry.d.ts +73 -0
  47. package/dist/sync/registry.js +223 -0
  48. package/dist/sync/routes.d.ts +62 -0
  49. package/dist/sync/routes.js +936 -0
  50. package/dist/sync/scripts/compactTombstones.d.ts +47 -0
  51. package/dist/sync/scripts/compactTombstones.js +270 -0
  52. package/dist/sync/serialize.d.ts +20 -0
  53. package/dist/{tests/bunSetup.js → sync/serialize.js} +35 -68
  54. package/dist/sync/socketHandlers.d.ts +54 -0
  55. package/dist/sync/socketHandlers.js +470 -0
  56. package/dist/sync/streams.d.ts +85 -0
  57. package/dist/sync/streams.js +221 -0
  58. package/dist/sync/syncApp.d.ts +29 -0
  59. package/dist/sync/syncApp.js +44 -0
  60. package/dist/sync/syncBatch.d.ts +71 -0
  61. package/dist/sync/syncBatch.js +252 -0
  62. package/dist/sync/syncSeqPlugin.d.ts +8 -0
  63. package/dist/sync/syncSeqPlugin.js +903 -0
  64. package/dist/sync/types.d.ts +254 -0
  65. package/dist/sync/types.js +9 -0
  66. package/dist/terrenoApp.js +90 -27
  67. package/dist/tests/models.d.ts +11 -27
  68. package/dist/tests/models.js +3 -5
  69. package/dist/tests/testHelper.d.ts +3 -9
  70. package/dist/vendor/wesleytodd-openapi/index.d.ts +1 -1
  71. package/dist/vendor/wesleytodd-openapi/lib/ui.d.ts +2 -1
  72. package/package.json +10 -8
  73. package/src/__tests__/openApiCompat.test.ts +20 -0
  74. package/src/actions.test.ts +12 -19
  75. package/src/actions.ts +2 -2
  76. package/src/api.coverage.test.ts +9 -7
  77. package/src/api.errorPaths.test.ts +1 -47
  78. package/src/api.test.ts +12 -0
  79. package/src/api.ts +98 -317
  80. package/src/auth.test.ts +128 -176
  81. package/src/auth.ts +109 -0
  82. package/src/betterAuth.ts +6 -0
  83. package/src/betterAuthSetup.test.ts +34 -0
  84. package/src/betterAuthSetup.ts +27 -8
  85. package/src/configurationPlugin.ts +2 -2
  86. package/src/consentApp.test.ts +12 -12
  87. package/src/docLoader.ts +3 -1
  88. package/src/errors.ts +33 -0
  89. package/src/expressServer.test.ts +7 -1
  90. package/src/expressServer.ts +5 -11
  91. package/src/githubAuth.ts +3 -1
  92. package/src/index.ts +42 -0
  93. package/src/middleware.test.ts +40 -1
  94. package/src/middleware.ts +15 -0
  95. package/src/models/versionConfig.ts +1 -1
  96. package/src/openApiBuilder.test.ts +9 -24
  97. package/src/openApiCompat.ts +99 -45
  98. package/src/openApiValidator.test.ts +0 -41
  99. package/src/permissions.test.ts +75 -1
  100. package/src/permissions.ts +49 -0
  101. package/src/plugins.test.ts +54 -14
  102. package/src/plugins.ts +47 -9
  103. package/src/populate.test.ts +16 -25
  104. package/src/realtime/changeStreamWatcher.ts +552 -114
  105. package/src/realtime/realtime.test.ts +157 -239
  106. package/src/realtime/realtimeApp.ts +80 -19
  107. package/src/realtime/sessionRevalidation.test.ts +423 -0
  108. package/src/realtime/sessionRevalidation.ts +273 -0
  109. package/src/realtime/socketAuth.test.ts +355 -0
  110. package/src/realtime/socketAuth.ts +228 -0
  111. package/src/realtime/socketUser.ts +69 -0
  112. package/src/realtime/types.ts +40 -0
  113. package/src/secretProviders.test.ts +7 -1
  114. package/src/secretProviders.ts +15 -3
  115. package/src/sync/ensureSyncIndexes.test.ts +144 -0
  116. package/src/sync/executors.test.ts +882 -0
  117. package/src/sync/executors.ts +703 -0
  118. package/src/sync/integration.test.ts +1160 -0
  119. package/src/sync/loadHarness.ts +618 -0
  120. package/src/sync/models.ts +640 -0
  121. package/src/sync/mutationHandler.test.ts +1252 -0
  122. package/src/sync/mutationHandler.ts +850 -0
  123. package/src/sync/registry.ts +232 -0
  124. package/src/sync/routes.ts +812 -0
  125. package/src/sync/scripts/compactTombstones.test.ts +249 -0
  126. package/src/sync/scripts/compactTombstones.ts +145 -0
  127. package/src/sync/serialize.ts +48 -0
  128. package/src/sync/sessionRevalidationIntegration.test.ts +281 -0
  129. package/src/sync/socketHandlers.ts +320 -0
  130. package/src/sync/streams.ts +188 -0
  131. package/src/sync/sync.test.ts +582 -0
  132. package/src/sync/syncApp.ts +44 -0
  133. package/src/sync/syncBatch.ts +219 -0
  134. package/src/sync/syncFrontier.test.ts +494 -0
  135. package/src/sync/syncPhaseC.test.ts +718 -0
  136. package/src/sync/syncRoutes.test.ts +865 -0
  137. package/src/sync/syncSeqPlugin.ts +738 -0
  138. package/src/sync/syncSocket.test.ts +1440 -0
  139. package/src/sync/types.ts +270 -0
  140. package/src/terrenoApp.ts +37 -17
  141. package/src/tests/bunSetup.ts +8 -0
  142. package/src/tests/models.ts +6 -8
  143. package/tsconfig.json +4 -4
  144. package/tsconfig.typedoc.json +7 -1
  145. package/dist/__tests__/openApiCompat.test.js +0 -317
  146. package/dist/__tests__/versionCheckPlugin.test.d.ts +0 -1
  147. package/dist/__tests__/versionCheckPlugin.test.js +0 -597
  148. package/dist/actions.openApi.test.d.ts +0 -1
  149. package/dist/actions.openApi.test.js +0 -253
  150. package/dist/actions.test.d.ts +0 -1
  151. package/dist/actions.test.js +0 -1087
  152. package/dist/api.arrayOperations.test.d.ts +0 -1
  153. package/dist/api.arrayOperations.test.js +0 -878
  154. package/dist/api.asyncHandler.test.d.ts +0 -1
  155. package/dist/api.asyncHandler.test.js +0 -236
  156. package/dist/api.coverage.test.d.ts +0 -1
  157. package/dist/api.coverage.test.js +0 -192
  158. package/dist/api.errorPaths.test.d.ts +0 -1
  159. package/dist/api.errorPaths.test.js +0 -887
  160. package/dist/api.errors.test.d.ts +0 -1
  161. package/dist/api.errors.test.js +0 -185
  162. package/dist/api.hooks.test.d.ts +0 -1
  163. package/dist/api.hooks.test.js +0 -950
  164. package/dist/api.query.test.d.ts +0 -1
  165. package/dist/api.query.test.js +0 -948
  166. package/dist/api.test.d.ts +0 -1
  167. package/dist/api.test.js +0 -2820
  168. package/dist/auth.compile.test.d.ts +0 -1
  169. package/dist/auth.compile.test.js +0 -201
  170. package/dist/auth.coverage.test.d.ts +0 -1
  171. package/dist/auth.coverage.test.js +0 -202
  172. package/dist/auth.test.d.ts +0 -1
  173. package/dist/auth.test.js +0 -2286
  174. package/dist/betterAuth.test.d.ts +0 -1
  175. package/dist/betterAuth.test.js +0 -181
  176. package/dist/betterAuthApp.test.d.ts +0 -1
  177. package/dist/betterAuthApp.test.js +0 -242
  178. package/dist/betterAuthSetup.test.d.ts +0 -1
  179. package/dist/betterAuthSetup.test.js +0 -762
  180. package/dist/config.test.d.ts +0 -1
  181. package/dist/config.test.js +0 -353
  182. package/dist/configuration.test.d.ts +0 -1
  183. package/dist/configuration.test.js +0 -980
  184. package/dist/configurationPlugin.test.d.ts +0 -1
  185. package/dist/configurationPlugin.test.js +0 -827
  186. package/dist/consentApp.test.d.ts +0 -1
  187. package/dist/consentApp.test.js +0 -1411
  188. package/dist/docLoader.test.d.ts +0 -1
  189. package/dist/docLoader.test.js +0 -137
  190. package/dist/envConfigurationPlugin.test.d.ts +0 -1
  191. package/dist/envConfigurationPlugin.test.js +0 -369
  192. package/dist/errors.test.d.ts +0 -1
  193. package/dist/errors.test.js +0 -607
  194. package/dist/expressServer.test.d.ts +0 -1
  195. package/dist/expressServer.test.js +0 -1249
  196. package/dist/githubAuth.test.d.ts +0 -1
  197. package/dist/githubAuth.test.js +0 -1142
  198. package/dist/httpClient.test.d.ts +0 -1
  199. package/dist/httpClient.test.js +0 -1471
  200. package/dist/logger.test.d.ts +0 -1
  201. package/dist/logger.test.js +0 -685
  202. package/dist/middleware.test.d.ts +0 -1
  203. package/dist/middleware.test.js +0 -241
  204. package/dist/notifiers/googleChatNotifier.test.d.ts +0 -1
  205. package/dist/notifiers/googleChatNotifier.test.js +0 -298
  206. package/dist/notifiers/slackNotifier.test.d.ts +0 -1
  207. package/dist/notifiers/slackNotifier.test.js +0 -297
  208. package/dist/notifiers/zoomNotifier.test.d.ts +0 -1
  209. package/dist/notifiers/zoomNotifier.test.js +0 -373
  210. package/dist/openApi.test.d.ts +0 -1
  211. package/dist/openApi.test.js +0 -620
  212. package/dist/openApiBuilder.test.d.ts +0 -1
  213. package/dist/openApiBuilder.test.js +0 -599
  214. package/dist/openApiEtag.test.d.ts +0 -1
  215. package/dist/openApiEtag.test.js +0 -86
  216. package/dist/openApiValidator.test.d.ts +0 -1
  217. package/dist/openApiValidator.test.js +0 -795
  218. package/dist/permissions.middleware.test.d.ts +0 -1
  219. package/dist/permissions.middleware.test.js +0 -308
  220. package/dist/permissions.test.d.ts +0 -1
  221. package/dist/permissions.test.js +0 -439
  222. package/dist/plugins.test.d.ts +0 -1
  223. package/dist/plugins.test.js +0 -902
  224. package/dist/populate.test.d.ts +0 -1
  225. package/dist/populate.test.js +0 -370
  226. package/dist/realtime/realtime.test.d.ts +0 -10
  227. package/dist/realtime/realtime.test.js +0 -4213
  228. package/dist/realtime/realtimeAppSetup.test.d.ts +0 -1
  229. package/dist/realtime/realtimeAppSetup.test.js +0 -213
  230. package/dist/requestContext.test.d.ts +0 -1
  231. package/dist/requestContext.test.js +0 -412
  232. package/dist/scriptRunner.test.d.ts +0 -1
  233. package/dist/scriptRunner.test.js +0 -386
  234. package/dist/secretProviders.test.d.ts +0 -1
  235. package/dist/secretProviders.test.js +0 -726
  236. package/dist/syncConsents.test.d.ts +0 -1
  237. package/dist/syncConsents.test.js +0 -522
  238. package/dist/terrenoApp.test.d.ts +0 -1
  239. package/dist/terrenoApp.test.js +0 -489
  240. package/dist/tests/bunSetup.d.ts +0 -1
  241. package/dist/tests/fixtures/compileAuthEntry.d.ts +0 -1
  242. package/dist/tests/fixtures/compileAuthEntry.js +0 -37
  243. package/dist/tests/testEnv.d.ts +0 -0
  244. package/dist/tests/testEnv.js +0 -6
  245. package/dist/transformers.test.d.ts +0 -1
  246. package/dist/transformers.test.js +0 -423
  247. package/dist/utils.test.d.ts +0 -1
  248. package/dist/utils.test.js +0 -242
  249. package/src/realtime/realtimeAppSetup.test.ts +0 -161
  250. /package/dist/{__tests__/openApiCompat.test.d.ts → sync/loadHarness.d.ts} +0 -0
@@ -1,12 +1,28 @@
1
+ // noExplicitAny: change stream and socket handlers use dynamic document shapes
2
+ // biome-ignore-all lint/suspicious/noExplicitAny: change stream and socket handlers use dynamic document shapes
3
+
1
4
  import * as Sentry from "@sentry/bun";
2
5
  import type express from "express";
3
6
  import {DateTime} from "luxon";
7
+ import type {ChangeStream, ChangeStreamDocument, ChangeStreamOptions} from "mongodb";
4
8
  import mongoose from "mongoose";
5
9
  import type {Server, Socket} from "socket.io";
6
10
 
7
- type ChangeStream = mongoose.mongo.ChangeStream;
8
- type ChangeStreamDocument = mongoose.mongo.ChangeStreamDocument;
9
- type ChangeStreamOptions = mongoose.mongo.ChangeStreamOptions;
11
+ import type {User} from "../auth";
12
+ import {APIError} from "../errors";
13
+ import {logger} from "../logger";
14
+ import {checkPermissions, type PermissionMethod} from "../permissions";
15
+ import {computeStableFrontier, SyncScopeMove} from "../sync/models";
16
+ import {findSyncEntryByCollectionName, type SyncRegistryEntry} from "../sync/registry";
17
+ import {serializeSyncPayload} from "../sync/serialize";
18
+ import {syncRoomForStream} from "../sync/socketHandlers";
19
+ import {resolveStreamForDoc} from "../sync/streams";
20
+ import type {SyncDelta, SyncMutationOperation, SyncResyncHint} from "../sync/types";
21
+ import {matchesQuery} from "./queryMatcher";
22
+ import {getQuerySubscriptionsForCollection} from "./queryStore";
23
+ import {findRegistryEntryByCollection, type RealtimeRegistryEntry} from "./registry";
24
+ import {getSocketUser, type SocketWithDecodedToken} from "./socketUser";
25
+ import type {ChangeStreamConfig, RealtimeEvent} from "./types";
10
26
 
11
27
  /**
12
28
  * The subset of ChangeStreamDocument variants this watcher actually processes.
@@ -18,19 +34,98 @@ type WatchedChange = Extract<
18
34
  {operationType: "insert" | "update" | "replace" | "delete"}
19
35
  >;
20
36
 
21
- import type {User} from "../auth";
22
- import {APIError} from "../errors";
23
- import {logger} from "../logger";
24
- import {checkPermissions} from "../permissions";
25
- import {matchesQuery} from "./queryMatcher";
26
- import {getQuerySubscriptionsForCollection} from "./queryStore";
27
- import {findRegistryEntryByCollection, type RealtimeRegistryEntry} from "./registry";
28
- import {getSocketUser, type SocketWithDecodedToken} from "./socketUser";
29
- import type {ChangeStreamConfig, RealtimeEvent} from "./types";
30
-
31
37
  let changeWatcher: ChangeStream | null = null;
32
38
 
33
- const DEFAULT_IGNORED_COLLECTIONS = ["socketio", "sessions"];
39
+ /**
40
+ * Task 9.16: restart state for the single process-wide watcher.
41
+ *
42
+ * The watcher used to only LOG on `error`/`close`: a replica-set election, a network blip,
43
+ * or a dropped cursor permanently killed every realtime event and every `sync:delta` until
44
+ * the process was restarted — online clients silently stopped receiving data while looking
45
+ * perfectly connected. The stream is now re-created with backoff, resuming from the last
46
+ * observed resume token so no events in the gap are missed.
47
+ */
48
+ /** Resume token of the last change seen, replayed via `resumeAfter` on restart. */
49
+ let lastResumeToken: unknown;
50
+
51
+ /** Bumped on every open/stop so callbacks from a replaced stream are ignored. */
52
+ let watcherGeneration = 0;
53
+
54
+ /** True between `stopChangeStreamWatcher()` and the next `startChangeStreamWatcher()`. */
55
+ let watcherStopped = true;
56
+
57
+ /** Consecutive failed opens, driving the backoff delay. Reset on a successful change. */
58
+ let restartAttempts = 0;
59
+
60
+ let restartTimer: ReturnType<typeof setTimeout> | null = null;
61
+
62
+ /** First restart delay; doubles per consecutive attempt up to the max. */
63
+ export const CHANGE_STREAM_RESTART_BASE_DELAY_MS = 250;
64
+
65
+ /** Ceiling for the restart backoff — a long outage retries once every 30s, forever. */
66
+ export const CHANGE_STREAM_RESTART_MAX_DELAY_MS = 30_000;
67
+
68
+ /**
69
+ * Socket event telling clients their cached cursors may have gaps and they must
70
+ * re-bootstrap. Emitted when the change stream's oplog history is gone
71
+ * (`ChangeStreamHistoryLost`), which is the one failure a resume token cannot repair.
72
+ * Payload: {@link SyncResyncHint}.
73
+ */
74
+ export const SYNC_RESYNC_EVENT = "sync:resync-required";
75
+
76
+ /** MongoDB error code / codeName for an unresumable change stream (oplog rolled past). */
77
+ const CHANGE_STREAM_HISTORY_LOST_CODE = 286;
78
+
79
+ /** True when the error means the resume token can no longer be used. */
80
+ const isHistoryLostError = (error: unknown): boolean => {
81
+ const {code, codeName, message} = (error ?? {}) as {
82
+ code?: unknown;
83
+ codeName?: string;
84
+ message?: string;
85
+ };
86
+ return (
87
+ code === CHANGE_STREAM_HISTORY_LOST_CODE ||
88
+ codeName === "ChangeStreamHistoryLost" ||
89
+ Boolean(message?.includes("ChangeStreamHistoryLost")) ||
90
+ Boolean(message?.includes("resume point may no longer be in the oplog"))
91
+ );
92
+ };
93
+
94
+ /**
95
+ * C8: per-entity serialized dispatch. The change-stream `"change"` handler is async and
96
+ * multiple events can be in flight at once; without ordering, two events for the SAME
97
+ * document could emit their deltas out of order (a client would then see LWW-by-seq
98
+ * violated within an entity). This keeps a promise chain PER entity key
99
+ * (`{collection}:{docId}`) so same-entity deltas dispatch strictly in change-stream
100
+ * order, while different entities still run concurrently. Chains self-clean when idle.
101
+ */
102
+ const entityDispatchChains = new Map<string, Promise<void>>();
103
+
104
+ const serializePerEntity = (key: string, task: () => Promise<void>): void => {
105
+ const prior = entityDispatchChains.get(key) ?? Promise.resolve();
106
+ const next = prior.then(task, task);
107
+ entityDispatchChains.set(key, next);
108
+ void next.finally(() => {
109
+ // Drop the chain once this was the last queued task, so the map does not grow.
110
+ if (entityDispatchChains.get(key) === next) {
111
+ entityDispatchChains.delete(key);
112
+ }
113
+ });
114
+ };
115
+
116
+ /**
117
+ * C7: the sync bookkeeping collections must never drive fan-out — their own change
118
+ * events are internal (counters/ledger/markers/keys), and processing them would emit
119
+ * spurious deltas or reprocess scope-move markers. Exported for testing.
120
+ */
121
+ export const DEFAULT_IGNORED_COLLECTIONS = [
122
+ "socketio",
123
+ "sessions",
124
+ "synccounters",
125
+ "syncmutations",
126
+ "syncscopemoves",
127
+ "synckeys",
128
+ ];
34
129
 
35
130
  /**
36
131
  * Map MongoDB change stream operation types to our method names.
@@ -94,8 +189,17 @@ const getSocketsInRoom = (io: Server, room: string): RealtimeSocketWithAuth[] =>
94
189
  return sockets;
95
190
  };
96
191
 
192
+ /**
193
+ * The minimal registry-entry shape the authorized-room emitters need. Both
194
+ * `RealtimeRegistryEntry` and `SyncRegistryEntry` satisfy it structurally.
195
+ */
196
+ export interface AuthorizedEmitEntry {
197
+ modelName: string;
198
+ options: {permissions: {read: PermissionMethod<any>[]}};
199
+ }
200
+
97
201
  const canReadDocument = async (
98
- entry: RealtimeRegistryEntry,
202
+ entry: AuthorizedEmitEntry,
99
203
  user?: User,
100
204
  doc?: Record<string, unknown>
101
205
  ): Promise<boolean> => {
@@ -218,14 +322,32 @@ export const serializeDoc = async (
218
322
  );
219
323
  };
220
324
 
221
- export const emitToAuthorizedRoom = async (
222
- io: Server,
223
- room: string,
224
- event: RealtimeEvent,
225
- entry: RealtimeRegistryEntry,
226
- fullDocument: Record<string, unknown> | undefined,
227
- logDebug: (msg: string) => void
228
- ): Promise<void> => {
325
+ /**
326
+ * Generalized authorized-room emitter: iterates the sockets in `room`, runs the per-socket
327
+ * read-permission check against `fullDocument`, builds the payload per socket (so
328
+ * responseHandlers can tailor output to the receiving user), and emits `eventName`.
329
+ *
330
+ * `emitToAuthorizedRoom` (legacy "sync" events) and the sync layer's `sync:delta`
331
+ * emission both delegate here.
332
+ */
333
+ export const emitPayloadToAuthorizedRoom = async ({
334
+ io,
335
+ room,
336
+ eventName,
337
+ entry,
338
+ fullDocument,
339
+ buildPayload,
340
+ logDebug,
341
+ }: {
342
+ io: Server;
343
+ room: string;
344
+ eventName: string;
345
+ entry: AuthorizedEmitEntry;
346
+ fullDocument: Record<string, unknown> | undefined;
347
+ /** Build the per-socket payload; a throw drops the emission for that socket only. */
348
+ buildPayload: (user: User | undefined) => Promise<unknown> | unknown;
349
+ logDebug: (msg: string) => void;
350
+ }): Promise<void> => {
229
351
  const sockets = getSocketsInRoom(io, room);
230
352
  for (const socket of sockets) {
231
353
  const user = getSocketUser(socket);
@@ -239,22 +361,40 @@ export const emitToAuthorizedRoom = async (
239
361
  continue;
240
362
  }
241
363
 
242
- if (!fullDocument) {
243
- socket.emit("sync", event);
244
- continue;
245
- }
246
-
247
- const data = await serializeDoc(entry, fullDocument, event.method, user);
248
- socket.emit("sync", {...event, data});
364
+ socket.emit(eventName, await buildPayload(user));
249
365
  } catch (error) {
250
366
  logger.error(
251
- `[realtime] Failed to emit ${entry.modelName}/${event.method} to socket ${socket.id}: ${error}`
367
+ `[realtime] Failed to emit ${eventName} for ${entry.modelName} to socket ${socket.id}: ${error}`
252
368
  );
253
369
  Sentry.captureException(error);
254
370
  }
255
371
  }
256
372
  };
257
373
 
374
+ export const emitToAuthorizedRoom = async (
375
+ io: Server,
376
+ room: string,
377
+ event: RealtimeEvent,
378
+ entry: RealtimeRegistryEntry,
379
+ fullDocument: Record<string, unknown> | undefined,
380
+ logDebug: (msg: string) => void
381
+ ): Promise<void> =>
382
+ emitPayloadToAuthorizedRoom({
383
+ buildPayload: async (user) => {
384
+ if (!fullDocument) {
385
+ return event;
386
+ }
387
+ const data = await serializeDoc(entry, fullDocument, event.method, user);
388
+ return {...event, data};
389
+ },
390
+ entry,
391
+ eventName: "sync",
392
+ fullDocument,
393
+ io,
394
+ logDebug,
395
+ room,
396
+ });
397
+
258
398
  /**
259
399
  * Emit a sync event to document-specific and query rooms.
260
400
  *
@@ -367,14 +507,324 @@ export const emitToDocumentAndQueryRooms = async (
367
507
  }
368
508
  };
369
509
 
510
+ /**
511
+ * Emit legacy realtime "sync" events for a change on a realtime-registered collection.
512
+ * Extracted verbatim from the change handler so sync-delta emission (which is keyed off
513
+ * the independent sync registry) runs even when the realtime path skips the change.
514
+ */
515
+ const processRealtimeChange = async ({
516
+ io,
517
+ entry,
518
+ change,
519
+ docId,
520
+ logDebug,
521
+ logInfo,
522
+ }: {
523
+ io: Server;
524
+ entry: RealtimeRegistryEntry;
525
+ change: WatchedChange;
526
+ docId: string;
527
+ logDebug: (msg: string) => void;
528
+ logInfo: (msg: string) => void;
529
+ }): Promise<void> => {
530
+ // Map to our method type. Pass enabledMethods so soft deletes only
531
+ // remap to "delete" when the model actually subscribes to deletes —
532
+ // otherwise the change is kept as "update" so update subscribers
533
+ // still receive it.
534
+ const method = mapOperationType(change.operationType, change, entry.config.methods);
535
+ if (!method) {
536
+ return;
537
+ }
538
+
539
+ // Check if this method is enabled for this model
540
+ if (!entry.config.methods.includes(method)) {
541
+ logDebug(`[realtime] Method ${method} not enabled for ${entry.modelName}`);
542
+ return;
543
+ }
544
+
545
+ // fullDocument is present on insert/update/replace; absent on delete.
546
+ const fullDocument = change.operationType === "delete" ? undefined : change.fullDocument;
547
+
548
+ // For hard deletes, we don't have the full document
549
+ const isHardDelete = method === "delete" && !fullDocument;
550
+
551
+ // Determine target rooms
552
+ let rooms: string[];
553
+ if (isHardDelete) {
554
+ // Hard delete: no fullDocument, so we can't resolve owner/custom rooms.
555
+ // For owner strategy we cannot safely fan out to a model room without
556
+ // leaking deletes across users — admins still receive the event via the
557
+ // admin room and any document-specific subscribers via document rooms.
558
+ if (entry.config.roomStrategy === "owner") {
559
+ rooms = ["admin"];
560
+ } else if (entry.config.roomStrategy === "broadcast") {
561
+ rooms = ["authenticated"];
562
+ } else {
563
+ const collectionTag = getCollectionTag(entry.routePath);
564
+ rooms = [`model:${collectionTag}`];
565
+ }
566
+ } else {
567
+ rooms = resolveRooms(entry, fullDocument ?? {}, method);
568
+ }
569
+
570
+ const collection = getCollectionTag(entry.routePath);
571
+
572
+ const event: RealtimeEvent = {
573
+ collection,
574
+ id: docId,
575
+ method,
576
+ model: entry.modelName,
577
+ timestamp: DateTime.now().toMillis(),
578
+ ...(change.operationType === "update" && change.updateDescription?.updatedFields
579
+ ? {updatedFields: Object.keys(change.updateDescription.updatedFields)}
580
+ : {}),
581
+ };
582
+
583
+ // Emit to strategy-based rooms (model/owner/broadcast)
584
+ for (const room of rooms) {
585
+ await emitToAuthorizedRoom(io, room, event, entry, fullDocument, logDebug);
586
+ }
587
+
588
+ // Emit to document-specific and query rooms
589
+ await emitToDocumentAndQueryRooms(io, collection, event, fullDocument, logDebug, entry);
590
+
591
+ logDebug(
592
+ `[realtime] Emitted ${method} for ${entry.modelName}/${docId} to rooms: ${rooms.join(", ")}`
593
+ );
594
+ // Log only metadata — never the document payload, which may contain sensitive fields.
595
+ const metadata: Record<string, unknown> = {
596
+ collection: event.collection,
597
+ id: event.id,
598
+ method: event.method,
599
+ model: event.model,
600
+ timestamp: event.timestamp,
601
+ };
602
+ if (event.updatedFields) {
603
+ metadata.updatedFields = event.updatedFields;
604
+ }
605
+ logInfo(`[realtime] sync event: ${JSON.stringify(metadata)}`);
606
+ };
607
+
608
+ /**
609
+ * Emit `sync:delta` events for a change on a sync-registered collection.
610
+ *
611
+ * Deltas fan out to the dedicated `sync:{stream}` rooms joined via `sync:subscribe`
612
+ * (independent of the legacy realtime rooms), with the same per-socket read-permission
613
+ * checks as legacy events. `seq` and `stream` come from the post-image: the `syncPlugin`
614
+ * stamps `_syncSeq` on every synced write and the watcher runs `updateLookup`.
615
+ *
616
+ * Soft deletes (an update setting `deleted: true`, reclassified by `mapOperationType`)
617
+ * produce a `method: "delete"` delta with `deleted: true` and the tombstone data intact.
618
+ *
619
+ * Scope moves (C4): the old-stream tombstone is derived from durable `SyncScopeMove`
620
+ * markers (written by `syncPlugin` in the same op-scope as the move), NOT from the racy
621
+ * `_syncPrevStream` post-image — a second write racing this event's processing can reset
622
+ * `_syncPrevStream`, but it cannot erase the durable marker. For each marker on this
623
+ * document, a data-less tombstone is emitted to the marker's `fromStream` (carrying the
624
+ * marker's old-stream seq), and the new-stream delta is reported as `method: "create"`.
625
+ * Client tombstone application is idempotent by seq, so re-emitting a marker on a later
626
+ * change of the same doc is harmless.
627
+ *
628
+ * Every emitted delta carries `frontierSeq` (the emitting stream's stable frontier at
629
+ * emit time, C1); the client advances its cursor to `min(delta.seq, delta.frontierSeq)`
630
+ * so a delta observed out of commit order never advances a cursor past an uncommitted hole.
631
+ *
632
+ * A model with both `realtime` and `sync` configs emits both the legacy "sync" event and
633
+ * "sync:delta" — distinct event names, no interference (documented as transitional).
634
+ *
635
+ * Exported for testing.
636
+ */
637
+ export const emitSyncDeltaForChange = async ({
638
+ io,
639
+ entry,
640
+ change,
641
+ docId,
642
+ logDebug,
643
+ }: {
644
+ io: Server;
645
+ entry: SyncRegistryEntry;
646
+ change: WatchedChange;
647
+ docId: string;
648
+ logDebug: (msg: string) => void;
649
+ }): Promise<void> => {
650
+ if (change.operationType === "delete") {
651
+ // Hard deletes are blocked on synced models by syncPlugin; a direct collection-level
652
+ // delete has no post-image, so neither stream nor seq can be resolved. Clients
653
+ // reconcile via snapshot instead.
654
+ logDebug(
655
+ `[sync] Skipping hard delete for synced collection ${entry.collectionTag}/${docId}: ` +
656
+ "no post-image to resolve stream/seq"
657
+ );
658
+ return;
659
+ }
660
+ const fullDocument = change.fullDocument as Record<string, unknown> | undefined;
661
+ if (!fullDocument) {
662
+ logDebug(`[sync] Skipping ${entry.collectionTag}/${docId}: change has no fullDocument`);
663
+ return;
664
+ }
665
+
666
+ const mapped = mapOperationType(change.operationType, change);
667
+ if (!mapped) {
668
+ return;
669
+ }
670
+ let method: SyncMutationOperation = mapped;
671
+ const deleted = fullDocument.deleted === true;
672
+ const seq = typeof fullDocument._syncSeq === "number" ? fullDocument._syncSeq : 0;
673
+ const stream = resolveStreamForDoc({
674
+ collectionTag: entry.collectionTag,
675
+ doc: fullDocument,
676
+ scope: entry.config.scope,
677
+ });
678
+
679
+ // C4: emit old-stream tombstones from durable markers, not the racy _syncPrevStream.
680
+ const markers = await SyncScopeMove.find({
681
+ collectionTag: entry.collectionTag,
682
+ entityId: docId,
683
+ }).lean();
684
+ let movedAway = false;
685
+ for (const marker of markers) {
686
+ if (marker.fromStream === stream) {
687
+ continue;
688
+ }
689
+ movedAway = true;
690
+ const markerFrontier = await computeStableFrontier({stream: marker.fromStream});
691
+ const tombstone: SyncDelta = {
692
+ collection: entry.collectionTag,
693
+ deleted: true,
694
+ frontierSeq: markerFrontier,
695
+ id: docId,
696
+ method: "delete",
697
+ seq: marker.seq,
698
+ stream: marker.fromStream,
699
+ };
700
+ await emitPayloadToAuthorizedRoom({
701
+ buildPayload: () => tombstone,
702
+ entry,
703
+ eventName: "sync:delta",
704
+ fullDocument,
705
+ io,
706
+ logDebug,
707
+ room: syncRoomForStream(marker.fromStream),
708
+ });
709
+ logDebug(
710
+ `[sync] Emitted scope-move tombstone for ${entry.collectionTag}/${docId} to ${marker.fromStream} seq=${marker.seq}`
711
+ );
712
+ }
713
+ // The new stream sees a moved-away doc for the first time: report it as a create
714
+ // (unless the same write also soft-deleted the doc, in which case the tombstone wins).
715
+ if (movedAway && method !== "delete") {
716
+ method = "create";
717
+ }
718
+
719
+ const frontierSeq = await computeStableFrontier({stream});
720
+ const delta: SyncDelta = {
721
+ collection: entry.collectionTag,
722
+ frontierSeq,
723
+ id: docId,
724
+ method,
725
+ seq,
726
+ stream,
727
+ ...(deleted ? {deleted: true} : {}),
728
+ };
729
+ await emitPayloadToAuthorizedRoom({
730
+ // C7: tombstone deltas carry no data (only id/seq/deleted); live deltas serialize.
731
+ buildPayload: deleted
732
+ ? () => delta
733
+ : async (user) => {
734
+ const syntheticReq = {params: {}, query: {}, user} as unknown as express.Request;
735
+ const data = ensureApiId(
736
+ await serializeSyncPayload({doc: fullDocument, entry, method, req: syntheticReq})
737
+ );
738
+ return {...delta, data};
739
+ },
740
+ entry,
741
+ eventName: "sync:delta",
742
+ fullDocument,
743
+ io,
744
+ logDebug,
745
+ room: syncRoomForStream(stream),
746
+ });
747
+ logDebug(
748
+ `[sync] Emitted sync:delta ${method} for ${entry.collectionTag}/${docId} seq=${seq} stream=${stream} frontier=${frontierSeq}`
749
+ );
750
+ };
751
+
370
752
  /**
371
753
  * Start watching MongoDB change streams and emitting real-time events.
754
+ *
755
+ * Task 9.16: the stream is supervised — an `error`/`close`/`end` schedules a re-open with
756
+ * exponential backoff, resuming from the last seen token. Call
757
+ * {@link stopChangeStreamWatcher} to end supervision.
372
758
  */
373
759
  export const startChangeStreamWatcher = (
374
760
  io: Server,
375
761
  config: ChangeStreamConfig = {},
376
762
  debug = false
377
763
  ): void => {
764
+ watcherStopped = false;
765
+ restartAttempts = 0;
766
+ lastResumeToken = undefined;
767
+ openChangeStream({config, debug, io});
768
+ };
769
+
770
+ /**
771
+ * Task 9.16: schedule a re-open after a stream failure. Silently returns when the watcher
772
+ * was stopped, when a newer stream is already running (a failing stream can fire several
773
+ * terminal events), or when a restart is already pending.
774
+ */
775
+ const scheduleChangeStreamRestart = ({
776
+ config,
777
+ debug,
778
+ generation,
779
+ io,
780
+ reason,
781
+ }: {
782
+ config: ChangeStreamConfig;
783
+ debug: boolean;
784
+ generation: number;
785
+ io: Server;
786
+ reason: string;
787
+ }): void => {
788
+ if (watcherStopped || generation !== watcherGeneration || restartTimer) {
789
+ return;
790
+ }
791
+ restartAttempts += 1;
792
+ const delayMs = Math.min(
793
+ CHANGE_STREAM_RESTART_BASE_DELAY_MS * 2 ** (restartAttempts - 1),
794
+ CHANGE_STREAM_RESTART_MAX_DELAY_MS
795
+ );
796
+ logger.warn(
797
+ `[realtime] Change stream ${reason}; reopening in ${delayMs}ms (attempt ${restartAttempts})`,
798
+ {resumable: lastResumeToken !== undefined}
799
+ );
800
+ restartTimer = setTimeout(() => {
801
+ restartTimer = null;
802
+ if (watcherStopped) {
803
+ return;
804
+ }
805
+ openChangeStream({config, debug, io, isReopen: true});
806
+ }, delayMs);
807
+ // Never hold the process open just to retry a watcher.
808
+ restartTimer.unref?.();
809
+ };
810
+
811
+ /**
812
+ * Open (or re-open) the change stream and wire its listeners. Throws only for a
813
+ * configuration-level failure on the FIRST open (no connection, `watch` returned nothing),
814
+ * which must fail startup loudly; a failure while reopening is retried with backoff.
815
+ */
816
+ const openChangeStream = ({
817
+ config,
818
+ debug,
819
+ io,
820
+ isReopen = false,
821
+ }: {
822
+ config: ChangeStreamConfig;
823
+ debug: boolean;
824
+ io: Server;
825
+ /** True for a supervised reopen: failures retry with backoff instead of throwing. */
826
+ isReopen?: boolean;
827
+ }): void => {
378
828
  const logInfo = (message: string): void => {
379
829
  if (debug) {
380
830
  logger.info(message);
@@ -387,6 +837,8 @@ export const startChangeStreamWatcher = (
387
837
  }
388
838
  };
389
839
 
840
+ const generation = ++watcherGeneration;
841
+
390
842
  try {
391
843
  logInfo("[realtime] Initializing change stream watcher...");
392
844
 
@@ -428,16 +880,30 @@ export const startChangeStreamWatcher = (
428
880
  // How long the cursor waits for new events before yielding control.
429
881
  // Lower values give more responsive updates at the cost of more frequent driver round-trips.
430
882
  maxAwaitTimeMS: 1000,
883
+ // Task 9.16: resume where the previous stream died so events during the outage are
884
+ // delivered rather than lost. Absent on a first open and after history loss.
885
+ ...(lastResumeToken !== undefined ? {resumeAfter: lastResumeToken} : {}),
431
886
  };
432
887
 
433
- changeWatcher = nativeDb.watch(pipeline, options);
888
+ const stream = nativeDb.watch(pipeline, options);
434
889
 
435
- if (!changeWatcher) {
890
+ if (!stream) {
436
891
  throw new APIError({status: 500, title: "Failed to create change stream watcher"});
437
892
  }
893
+ changeWatcher = stream;
438
894
 
439
- changeWatcher.on("change", async (rawChange: ChangeStreamDocument) => {
895
+ stream.on("change", async (rawChange: ChangeStreamDocument) => {
440
896
  try {
897
+ // Task 9.16: record the resume token FIRST, for every change — including ones this
898
+ // watcher ignores. A token only from processed events would rewind the stream to an
899
+ // old position after a restart and re-deliver everything since.
900
+ const token = (rawChange as {_id?: unknown})._id;
901
+ if (token !== undefined) {
902
+ lastResumeToken = token;
903
+ }
904
+ // A stream that is delivering events has recovered; drop the accumulated backoff so
905
+ // the next unrelated failure retries promptly instead of at the previous ceiling.
906
+ restartAttempts = 0;
441
907
  // The pipeline restricts operationType to a subset that always has ns/documentKey;
442
908
  // narrow once here so downstream code doesn't need repeated casts.
443
909
  if (
@@ -461,122 +927,94 @@ export const startChangeStreamWatcher = (
461
927
  return;
462
928
  }
463
929
 
464
- // Find the registry entry for this collection
930
+ // Realtime and sync registries are independent: a model may be in either or both.
931
+ // A model with both configs emits both the legacy "sync" event and "sync:delta".
465
932
  const entry = findRegistryEntryByCollection(collectionName);
933
+ const syncEntry = findSyncEntryByCollectionName(collectionName);
466
934
 
467
- if (!entry) {
468
- // Not a registered realtime model — skip
935
+ if (!entry && !syncEntry) {
936
+ // Not a registered realtime or sync model — skip
469
937
  logDebug(`[realtime] No registry entry for collection: ${collectionName}`);
470
938
  return;
471
939
  }
472
940
 
473
- // Map to our method type. Pass enabledMethods so soft deletes only
474
- // remap to "delete" when the model actually subscribes to deletes —
475
- // otherwise the change is kept as "update" so update subscribers
476
- // still receive it.
477
- const method = mapOperationType(change.operationType, change, entry.config.methods);
478
- if (!method) {
479
- return;
480
- }
481
-
482
- // Check if this method is enabled for this model
483
- if (!entry.config.methods.includes(method)) {
484
- logDebug(`[realtime] Method ${method} not enabled for ${entry.modelName}`);
485
- return;
486
- }
487
-
488
- // fullDocument is present on insert/update/replace; absent on delete.
489
- const fullDocument = change.operationType === "delete" ? undefined : change.fullDocument;
490
-
491
- // For hard deletes, we don't have the full document
492
- const isHardDelete = method === "delete" && !fullDocument;
493
-
494
- // Determine target rooms
495
- let rooms: string[];
496
- if (isHardDelete) {
497
- // Hard delete: no fullDocument, so we can't resolve owner/custom rooms.
498
- // For owner strategy we cannot safely fan out to a model room without
499
- // leaking deletes across users — admins still receive the event via the
500
- // admin room and any document-specific subscribers via document rooms.
501
- if (entry.config.roomStrategy === "owner") {
502
- rooms = ["admin"];
503
- } else if (entry.config.roomStrategy === "broadcast") {
504
- rooms = ["authenticated"];
505
- } else {
506
- const collectionTag = getCollectionTag(entry.routePath);
507
- rooms = [`model:${collectionTag}`];
508
- }
509
- } else {
510
- rooms = resolveRooms(entry, fullDocument ?? {}, method);
511
- }
512
-
513
- const collection = getCollectionTag(entry.routePath);
514
-
515
- const event: RealtimeEvent = {
516
- collection,
517
- id: docId,
518
- method,
519
- model: entry.modelName,
520
- timestamp: DateTime.now().toMillis(),
521
- ...(change.operationType === "update" && change.updateDescription?.updatedFields
522
- ? {updatedFields: Object.keys(change.updateDescription.updatedFields)}
523
- : {}),
524
- };
525
-
526
- // Emit to strategy-based rooms (model/owner/broadcast)
527
- for (const room of rooms) {
528
- await emitToAuthorizedRoom(io, room, event, entry, fullDocument, logDebug);
941
+ if (entry) {
942
+ await processRealtimeChange({change, docId, entry, io, logDebug, logInfo});
529
943
  }
530
944
 
531
- // Emit to document-specific and query rooms
532
- await emitToDocumentAndQueryRooms(io, collection, event, fullDocument, logDebug, entry);
533
-
534
- logDebug(
535
- `[realtime] Emitted ${method} for ${entry.modelName}/${docId} to rooms: ${rooms.join(", ")}`
536
- );
537
- // Log only metadata — never the document payload, which may contain sensitive fields.
538
- const metadata: Record<string, unknown> = {
539
- collection: event.collection,
540
- id: event.id,
541
- method: event.method,
542
- model: event.model,
543
- timestamp: event.timestamp,
544
- };
545
- if (event.updatedFields) {
546
- metadata.updatedFields = event.updatedFields;
945
+ if (syncEntry) {
946
+ // C8: serialize per entity so same-doc deltas emit in change-stream order;
947
+ // different docs still dispatch concurrently. See serializePerEntity + the
948
+ // per-entity LWW-by-seq contract documented in sync/types.ts (SyncDelta).
949
+ serializePerEntity(`${collectionName}:${docId}`, async () => {
950
+ try {
951
+ await emitSyncDeltaForChange({change, docId, entry: syncEntry, io, logDebug});
952
+ } catch (error) {
953
+ logger.error(`[sync] Error emitting sync delta: ${error}`);
954
+ Sentry.captureException(error);
955
+ }
956
+ });
547
957
  }
548
- logInfo(`[realtime] sync event: ${JSON.stringify(metadata)}`);
549
958
  } catch (error) {
550
959
  logger.error(`[realtime] Error processing change event: ${error}`);
551
960
  Sentry.captureException(error);
552
961
  }
553
962
  });
554
963
 
555
- changeWatcher.on("error", (err: Error) => {
964
+ stream.on("error", (err: Error) => {
556
965
  Sentry.captureException(err);
557
966
  logger.error(`[realtime] Change stream error: ${err?.message || err}`);
967
+ if (isHistoryLostError(err)) {
968
+ // The oplog no longer contains the resume point, so the gap can never be replayed.
969
+ // Reopen from NOW and tell clients to re-bootstrap: their cursors may sit below
970
+ // changes (including deletions) they will otherwise never hear about.
971
+ lastResumeToken = undefined;
972
+ logger.error(
973
+ "[sync] Change stream history lost; reopening without a resume token and asking " +
974
+ "clients to resync"
975
+ );
976
+ const hint: SyncResyncHint = {reason: "history_lost"};
977
+ io.emit(SYNC_RESYNC_EVENT, hint);
978
+ }
979
+ scheduleChangeStreamRestart({config, debug, generation, io, reason: "errored"});
558
980
  });
559
981
 
560
- changeWatcher.on("close", () => {
982
+ stream.on("close", () => {
561
983
  logger.warn("[realtime] Change stream closed");
984
+ scheduleChangeStreamRestart({config, debug, generation, io, reason: "closed"});
562
985
  });
563
986
 
564
- changeWatcher.on("end", () => {
987
+ stream.on("end", () => {
565
988
  logger.warn("[realtime] Change stream ended");
989
+ scheduleChangeStreamRestart({config, debug, generation, io, reason: "ended"});
566
990
  });
567
991
 
568
992
  logInfo("[realtime] Change stream watcher initialized successfully");
569
993
  } catch (error) {
570
994
  logger.error(`[realtime] Failed to initialize change stream watcher: ${error}`);
571
995
  Sentry.captureException(error);
996
+ // A first open must fail startup loudly; a failed REOPEN (the DB is down mid-flight)
997
+ // has no caller to throw to, so keep retrying with backoff instead.
998
+ if (isReopen && !watcherStopped) {
999
+ scheduleChangeStreamRestart({config, debug, generation, io, reason: "failed to open"});
1000
+ return;
1001
+ }
572
1002
  throw error;
573
1003
  }
574
1004
  };
575
1005
 
576
1006
  /**
577
- * Stop the change stream watcher.
1007
+ * Stop the change stream watcher and end Task 9.16's restart supervision (a pending
1008
+ * reopen is cancelled, and terminal events from the closing stream are ignored).
578
1009
  */
579
1010
  export const stopChangeStreamWatcher = async (): Promise<void> => {
1011
+ watcherStopped = true;
1012
+ if (restartTimer) {
1013
+ clearTimeout(restartTimer);
1014
+ restartTimer = null;
1015
+ }
1016
+ restartAttempts = 0;
1017
+ lastResumeToken = undefined;
580
1018
  if (changeWatcher) {
581
1019
  await changeWatcher.close();
582
1020
  changeWatcher = null;