@terreno/api 0.31.0 → 56.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/dist/api.d.ts +10 -0
  2. package/dist/api.js +110 -361
  3. package/dist/auth.d.ts +26 -0
  4. package/dist/auth.js +121 -26
  5. package/dist/betterAuth.d.ts +5 -0
  6. package/dist/betterAuthSetup.js +23 -4
  7. package/dist/docLoader.js +12 -9
  8. package/dist/errors.d.ts +13 -0
  9. package/dist/errors.js +29 -1
  10. package/dist/index.d.ts +17 -1
  11. package/dist/index.js +29 -1
  12. package/dist/middleware.d.ts +7 -0
  13. package/dist/middleware.js +16 -1
  14. package/dist/openApiCompat.d.ts +10 -1
  15. package/dist/openApiCompat.js +115 -47
  16. package/dist/permissions.d.ts +27 -0
  17. package/dist/permissions.js +42 -1
  18. package/dist/plugins.d.ts +13 -0
  19. package/dist/plugins.js +47 -1
  20. package/dist/realtime/changeStreamWatcher.d.ts +100 -3
  21. package/dist/realtime/changeStreamWatcher.js +542 -137
  22. package/dist/realtime/realtimeApp.d.ts +2 -0
  23. package/dist/realtime/realtimeApp.js +83 -27
  24. package/dist/realtime/sessionRevalidation.d.ts +90 -0
  25. package/dist/realtime/sessionRevalidation.js +438 -0
  26. package/dist/realtime/socketAuth.d.ts +97 -0
  27. package/dist/realtime/socketAuth.js +272 -0
  28. package/dist/realtime/socketUser.d.ts +54 -0
  29. package/dist/realtime/socketUser.js +75 -5
  30. package/dist/realtime/types.d.ts +54 -0
  31. package/dist/secretProviders.d.ts +23 -0
  32. package/dist/secretProviders.js +9 -3
  33. package/dist/sync/executors.d.ts +198 -0
  34. package/dist/sync/executors.js +670 -0
  35. package/dist/sync/loadHarness.js +807 -0
  36. package/dist/sync/models.d.ts +216 -0
  37. package/dist/sync/models.js +644 -0
  38. package/dist/sync/mutationHandler.d.ts +97 -0
  39. package/dist/sync/mutationHandler.js +886 -0
  40. package/dist/sync/registry.d.ts +73 -0
  41. package/dist/sync/registry.js +223 -0
  42. package/dist/sync/routes.d.ts +62 -0
  43. package/dist/sync/routes.js +936 -0
  44. package/dist/sync/scripts/compactTombstones.d.ts +47 -0
  45. package/dist/sync/scripts/compactTombstones.js +270 -0
  46. package/dist/sync/serialize.d.ts +20 -0
  47. package/dist/{tests/bunSetup.js → sync/serialize.js} +35 -68
  48. package/dist/sync/socketHandlers.d.ts +54 -0
  49. package/dist/sync/socketHandlers.js +470 -0
  50. package/dist/sync/streams.d.ts +85 -0
  51. package/dist/sync/streams.js +221 -0
  52. package/dist/sync/syncApp.d.ts +29 -0
  53. package/dist/sync/syncApp.js +44 -0
  54. package/dist/sync/syncBatch.d.ts +71 -0
  55. package/dist/sync/syncBatch.js +252 -0
  56. package/dist/sync/syncSeqPlugin.d.ts +8 -0
  57. package/dist/sync/syncSeqPlugin.js +899 -0
  58. package/dist/sync/types.d.ts +254 -0
  59. package/dist/sync/types.js +9 -0
  60. package/dist/terrenoApp.js +90 -27
  61. package/dist/vendor/wesleytodd-openapi/index.d.ts +1 -1
  62. package/dist/vendor/wesleytodd-openapi/lib/ui.d.ts +2 -1
  63. package/package.json +7 -6
  64. package/src/__tests__/openApiCompat.test.ts +20 -0
  65. package/src/api.errors.test.ts +32 -23
  66. package/src/api.test.ts +12 -0
  67. package/src/api.ts +87 -296
  68. package/src/auth.test.ts +202 -0
  69. package/src/auth.ts +134 -23
  70. package/src/betterAuth.ts +6 -0
  71. package/src/betterAuthSetup.test.ts +34 -0
  72. package/src/betterAuthSetup.ts +20 -1
  73. package/src/configuration.test.ts +59 -38
  74. package/src/configurationPlugin.test.ts +72 -42
  75. package/src/docLoader.ts +3 -1
  76. package/src/errors.ts +33 -0
  77. package/src/expressServer.test.ts +7 -1
  78. package/src/index.ts +42 -0
  79. package/src/middleware.test.ts +56 -1
  80. package/src/middleware.ts +15 -0
  81. package/src/openApiCompat.ts +99 -45
  82. package/src/permissions.test.ts +101 -17
  83. package/src/permissions.ts +49 -0
  84. package/src/plugins.test.ts +64 -0
  85. package/src/plugins.ts +56 -0
  86. package/src/realtime/changeStreamWatcher.ts +552 -114
  87. package/src/realtime/realtime.test.ts +157 -0
  88. package/src/realtime/realtimeApp.ts +82 -14
  89. package/src/realtime/realtimeAppSetup.test.ts +25 -27
  90. package/src/realtime/sessionRevalidation.test.ts +423 -0
  91. package/src/realtime/sessionRevalidation.ts +273 -0
  92. package/src/realtime/socketAuth.test.ts +355 -0
  93. package/src/realtime/socketAuth.ts +228 -0
  94. package/src/realtime/socketUser.ts +69 -0
  95. package/src/realtime/types.ts +54 -0
  96. package/src/secretProviders.test.ts +7 -1
  97. package/src/secretProviders.ts +15 -3
  98. package/src/secretProvidersGcpClient.test.ts +66 -0
  99. package/src/sync/ensureSyncIndexes.test.ts +144 -0
  100. package/src/sync/executors.test.ts +882 -0
  101. package/src/sync/executors.ts +705 -0
  102. package/src/sync/integration.test.ts +1160 -0
  103. package/src/sync/loadHarness.ts +618 -0
  104. package/src/sync/models.ts +641 -0
  105. package/src/sync/mutationHandler.test.ts +1252 -0
  106. package/src/sync/mutationHandler.ts +850 -0
  107. package/src/sync/registry.ts +232 -0
  108. package/src/sync/routes.ts +812 -0
  109. package/src/sync/scripts/compactTombstones.test.ts +249 -0
  110. package/src/sync/scripts/compactTombstones.ts +145 -0
  111. package/src/sync/serialize.ts +48 -0
  112. package/src/sync/sessionRevalidationIntegration.test.ts +281 -0
  113. package/src/sync/socketHandlers.ts +320 -0
  114. package/src/sync/streams.ts +188 -0
  115. package/src/sync/sync.test.ts +582 -0
  116. package/src/sync/syncApp.ts +44 -0
  117. package/src/sync/syncBatch.ts +219 -0
  118. package/src/sync/syncFrontier.test.ts +494 -0
  119. package/src/sync/syncPhaseC.test.ts +718 -0
  120. package/src/sync/syncRoutes.test.ts +865 -0
  121. package/src/sync/syncSeqPlugin.ts +730 -0
  122. package/src/sync/syncSocket.test.ts +1440 -0
  123. package/src/sync/types.ts +270 -0
  124. package/src/terrenoApp.ts +37 -17
  125. package/src/tests/bunSetup.ts +8 -0
  126. package/tsconfig.json +4 -4
  127. package/tsconfig.typedoc.json +7 -1
  128. package/dist/__tests__/openApiCompat.test.js +0 -317
  129. package/dist/__tests__/versionCheckPlugin.test.d.ts +0 -1
  130. package/dist/__tests__/versionCheckPlugin.test.js +0 -597
  131. package/dist/actions.openApi.test.d.ts +0 -1
  132. package/dist/actions.openApi.test.js +0 -253
  133. package/dist/actions.test.d.ts +0 -1
  134. package/dist/actions.test.js +0 -1087
  135. package/dist/api.arrayOperations.test.d.ts +0 -1
  136. package/dist/api.arrayOperations.test.js +0 -878
  137. package/dist/api.asyncHandler.test.d.ts +0 -1
  138. package/dist/api.asyncHandler.test.js +0 -236
  139. package/dist/api.coverage.test.d.ts +0 -1
  140. package/dist/api.coverage.test.js +0 -192
  141. package/dist/api.errorPaths.test.d.ts +0 -1
  142. package/dist/api.errorPaths.test.js +0 -887
  143. package/dist/api.errors.test.d.ts +0 -1
  144. package/dist/api.errors.test.js +0 -185
  145. package/dist/api.hooks.test.d.ts +0 -1
  146. package/dist/api.hooks.test.js +0 -950
  147. package/dist/api.query.test.d.ts +0 -1
  148. package/dist/api.query.test.js +0 -948
  149. package/dist/api.test.d.ts +0 -1
  150. package/dist/api.test.js +0 -2820
  151. package/dist/auth.compile.test.d.ts +0 -1
  152. package/dist/auth.compile.test.js +0 -201
  153. package/dist/auth.coverage.test.d.ts +0 -1
  154. package/dist/auth.coverage.test.js +0 -202
  155. package/dist/auth.test.d.ts +0 -1
  156. package/dist/auth.test.js +0 -2286
  157. package/dist/betterAuth.test.d.ts +0 -1
  158. package/dist/betterAuth.test.js +0 -181
  159. package/dist/betterAuthApp.test.d.ts +0 -1
  160. package/dist/betterAuthApp.test.js +0 -242
  161. package/dist/betterAuthSetup.test.d.ts +0 -1
  162. package/dist/betterAuthSetup.test.js +0 -762
  163. package/dist/config.test.d.ts +0 -1
  164. package/dist/config.test.js +0 -353
  165. package/dist/configuration.test.d.ts +0 -1
  166. package/dist/configuration.test.js +0 -980
  167. package/dist/configurationPlugin.test.d.ts +0 -1
  168. package/dist/configurationPlugin.test.js +0 -827
  169. package/dist/consentApp.test.d.ts +0 -1
  170. package/dist/consentApp.test.js +0 -1411
  171. package/dist/docLoader.test.d.ts +0 -1
  172. package/dist/docLoader.test.js +0 -137
  173. package/dist/envConfigurationPlugin.test.d.ts +0 -1
  174. package/dist/envConfigurationPlugin.test.js +0 -369
  175. package/dist/errors.test.d.ts +0 -1
  176. package/dist/errors.test.js +0 -607
  177. package/dist/expressServer.test.d.ts +0 -1
  178. package/dist/expressServer.test.js +0 -1249
  179. package/dist/githubAuth.test.d.ts +0 -1
  180. package/dist/githubAuth.test.js +0 -1142
  181. package/dist/httpClient.test.d.ts +0 -1
  182. package/dist/httpClient.test.js +0 -1471
  183. package/dist/logger.test.d.ts +0 -1
  184. package/dist/logger.test.js +0 -685
  185. package/dist/middleware.test.d.ts +0 -1
  186. package/dist/middleware.test.js +0 -241
  187. package/dist/notifiers/googleChatNotifier.test.d.ts +0 -1
  188. package/dist/notifiers/googleChatNotifier.test.js +0 -298
  189. package/dist/notifiers/slackNotifier.test.d.ts +0 -1
  190. package/dist/notifiers/slackNotifier.test.js +0 -297
  191. package/dist/notifiers/zoomNotifier.test.d.ts +0 -1
  192. package/dist/notifiers/zoomNotifier.test.js +0 -373
  193. package/dist/openApi.test.d.ts +0 -1
  194. package/dist/openApi.test.js +0 -620
  195. package/dist/openApiBuilder.test.d.ts +0 -1
  196. package/dist/openApiBuilder.test.js +0 -599
  197. package/dist/openApiEtag.test.d.ts +0 -1
  198. package/dist/openApiEtag.test.js +0 -86
  199. package/dist/openApiValidator.test.d.ts +0 -1
  200. package/dist/openApiValidator.test.js +0 -795
  201. package/dist/permissions.middleware.test.d.ts +0 -1
  202. package/dist/permissions.middleware.test.js +0 -308
  203. package/dist/permissions.test.d.ts +0 -1
  204. package/dist/permissions.test.js +0 -439
  205. package/dist/plugins.test.d.ts +0 -1
  206. package/dist/plugins.test.js +0 -902
  207. package/dist/populate.test.d.ts +0 -1
  208. package/dist/populate.test.js +0 -370
  209. package/dist/realtime/realtime.test.d.ts +0 -10
  210. package/dist/realtime/realtime.test.js +0 -4213
  211. package/dist/realtime/realtimeAppSetup.test.d.ts +0 -1
  212. package/dist/realtime/realtimeAppSetup.test.js +0 -213
  213. package/dist/requestContext.test.d.ts +0 -1
  214. package/dist/requestContext.test.js +0 -412
  215. package/dist/scriptRunner.test.d.ts +0 -1
  216. package/dist/scriptRunner.test.js +0 -386
  217. package/dist/secretProviders.test.d.ts +0 -1
  218. package/dist/secretProviders.test.js +0 -726
  219. package/dist/syncConsents.test.d.ts +0 -1
  220. package/dist/syncConsents.test.js +0 -522
  221. package/dist/terrenoApp.test.d.ts +0 -1
  222. package/dist/terrenoApp.test.js +0 -489
  223. package/dist/tests/bunSetup.d.ts +0 -1
  224. package/dist/tests/fixtures/compileAuthEntry.d.ts +0 -1
  225. package/dist/tests/fixtures/compileAuthEntry.js +0 -37
  226. package/dist/tests/testEnv.d.ts +0 -0
  227. package/dist/tests/testEnv.js +0 -6
  228. package/dist/transformers.test.d.ts +0 -1
  229. package/dist/transformers.test.js +0 -423
  230. package/dist/utils.test.d.ts +0 -1
  231. package/dist/utils.test.js +0 -242
  232. /package/dist/{__tests__/openApiCompat.test.d.ts → sync/loadHarness.d.ts} +0 -0
@@ -0,0 +1,641 @@
1
+ import crypto from "node:crypto";
2
+ import mongoose, {type ClientSession, type Model, Schema} from "mongoose";
3
+ import {logger} from "../logger";
4
+ import {findOneOrNoneFor} from "../plugins";
5
+
6
+ /**
7
+ * Mongoose models backing the SyncDB protocol: the per-stream monotonic counter
8
+ * and the per-user encryption key material.
9
+ */
10
+
11
+ /** An uncommitted seq claim recorded on the counter's in-flight registry (C1). */
12
+ export interface SyncPendingClaim {
13
+ /** The claimed seq (or, for a batch, one entry per claimed seq). */
14
+ seq: number;
15
+ /** When the claim was registered; a claim older than the lease is reclaimable. */
16
+ claimedAt: Date;
17
+ }
18
+
19
+ export interface SyncCounterDocument {
20
+ _id: mongoose.Types.ObjectId;
21
+ stream: string;
22
+ seq: number;
23
+ /**
24
+ * C1: seqs claimed but not yet confirmed committed. The stable frontier is
25
+ * `min(pending.seq) - 1`, or `seq` (the head) when empty, so a cursor never
26
+ * advances past a seq whose owning write has not yet landed.
27
+ */
28
+ pending: SyncPendingClaim[];
29
+ /**
30
+ * C7 retention watermark: the highest seq on this stream whose row (tombstone or
31
+ * scope-move marker) has been hard-deleted by `compactTombstones`. A client whose
32
+ * cursor is below this may have missed a compacted deletion and must re-bootstrap;
33
+ * a client at or above it has already seen everything that was reaped. 0 until the
34
+ * first compaction pass touches the stream.
35
+ */
36
+ compactedThroughSeq: number;
37
+ }
38
+
39
+ /**
40
+ * C1: a `pending` claim older than this is treated as abandoned (the writer
41
+ * crashed between claiming and confirming) and excluded from the frontier — a
42
+ * crashed writer must never freeze the frontier forever. Default 60s.
43
+ */
44
+ export const PENDING_CLAIM_LEASE_MS = 60 * 1_000;
45
+
46
+ const syncPendingClaimSchema = new Schema<SyncPendingClaim>(
47
+ {
48
+ claimedAt: {
49
+ default: () => new Date(),
50
+ description: "When this seq was claimed; a claim older than PENDING_CLAIM_LEASE_MS is stale",
51
+ type: Date,
52
+ },
53
+ seq: {
54
+ description: "A claimed-but-uncommitted sequence number",
55
+ required: true,
56
+ type: Number,
57
+ },
58
+ },
59
+ {_id: false, strict: "throw"}
60
+ );
61
+
62
+ const syncCounterSchema = new Schema<SyncCounterDocument>(
63
+ {
64
+ compactedThroughSeq: {
65
+ default: 0,
66
+ description:
67
+ "C7 retention watermark: the highest seq whose tombstone/scope-move row has been " +
68
+ "hard-deleted by compactTombstones; served as the snapshot's oldestRetainedSeq",
69
+ type: Number,
70
+ },
71
+ pending: {
72
+ default: [],
73
+ description:
74
+ "C1 in-flight registry: seqs claimed but not yet confirmed committed; the stable " +
75
+ "frontier is min(pending.seq) - 1 (or the head when empty)",
76
+ type: [syncPendingClaimSchema],
77
+ },
78
+ seq: {
79
+ default: 0,
80
+ description: "The last sequence number claimed for this stream",
81
+ type: Number,
82
+ },
83
+ stream: {
84
+ description: "Stream key this counter belongs to (e.g. 'todos|owner:123')",
85
+ required: true,
86
+ type: String,
87
+ unique: true,
88
+ },
89
+ },
90
+ // Consumer apps run checkModelsStrict() at startup; framework models must comply.
91
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
92
+ );
93
+
94
+ export const SyncCounter: Model<SyncCounterDocument> =
95
+ (mongoose.models.SyncCounter as Model<SyncCounterDocument>) ??
96
+ mongoose.model<SyncCounterDocument>("SyncCounter", syncCounterSchema);
97
+
98
+ /**
99
+ * Result of a seq claim: the last seq claimed (range [lastSeq - count + 1, lastSeq])
100
+ * and whether the claim registered a pending entry that a later `confirmSyncSeqs`
101
+ * must clear. A session-backed claim skips the pending registry entirely (the write
102
+ * and the `$inc` commit atomically), so `registered` is false and no confirm is due.
103
+ */
104
+ export interface SyncSeqClaim {
105
+ /** The last (highest) seq claimed; the range is [lastSeq - count + 1, lastSeq]. */
106
+ lastSeq: number;
107
+ /** Every seq claimed, in ascending order (the range materialized). */
108
+ seqs: number[];
109
+ /** True when a pending registry entry was recorded and `confirmSyncSeqs` must clear it. */
110
+ registered: boolean;
111
+ }
112
+
113
+ /**
114
+ * C1: atomically claim `count` sequence numbers for a stream AND register them on
115
+ * the in-flight registry so the stable frontier can exclude them until committed.
116
+ *
117
+ * Two modes:
118
+ * - **No caller session (the hot path):** one `findOneAndUpdate` does `$inc: {seq}`
119
+ * plus `$push` of a `pending` entry per claimed seq. The write commits separately,
120
+ * so `confirmSyncSeqs` must `$pull` the entries once it lands. Until then the
121
+ * frontier holds below the lowest pending seq.
122
+ * - **Caller session present:** the `$inc` and the document write commit atomically
123
+ * in the caller's transaction, so there is no window where a claimed seq is
124
+ * uncommitted — the pending registry is skipped (`registered: false`, nothing to
125
+ * confirm). Frontier logic treats a session-backed claim as already committed.
126
+ *
127
+ * Retries once on the upsert race (two concurrent first claims for a new stream).
128
+ */
129
+ export const claimSyncSeqs = async ({
130
+ stream,
131
+ count = 1,
132
+ session,
133
+ }: {
134
+ stream: string;
135
+ count?: number;
136
+ session?: ClientSession | null;
137
+ }): Promise<SyncSeqClaim> => {
138
+ const materialize = (lastSeq: number): number[] => {
139
+ const seqs: number[] = [];
140
+ for (let s = lastSeq - count + 1; s <= lastSeq; s++) {
141
+ seqs.push(s);
142
+ }
143
+ return seqs;
144
+ };
145
+
146
+ // Session-backed fast path: the write and the $inc are already atomic, so we skip
147
+ // the pending registry — there is never an uncommitted-but-claimed window to fence.
148
+ if (session) {
149
+ const doc = await SyncCounter.findOneAndUpdate(
150
+ {stream},
151
+ {$inc: {seq: count}},
152
+ {new: true, session, upsert: true}
153
+ );
154
+ return {lastSeq: doc.seq, registered: false, seqs: materialize(doc.seq)};
155
+ }
156
+
157
+ const claim = async (): Promise<SyncSeqClaim> => {
158
+ const now = new Date();
159
+ // Claim (`$inc`) AND register the pending entries in ONE atomic aggregation-pipeline
160
+ // update, so there is never a window where the incremented head is visible without
161
+ // its pending claims (which would let `computeStableFrontier` transiently report a
162
+ // frontier above an about-to-be-pending seq). `$range(oldSeq+1, newSeq+1)` materializes
163
+ // the claimed range from the just-incremented value; `$map` turns it into pending docs.
164
+ // Mongoose 9 disallows update pipelines unless `updatePipeline: true` is set.
165
+ const doc = await SyncCounter.findOneAndUpdate(
166
+ {stream},
167
+ [
168
+ {$set: {_syncOldSeq: {$ifNull: ["$seq", 0]}}},
169
+ {$set: {seq: {$add: ["$_syncOldSeq", count]}}},
170
+ {
171
+ $set: {
172
+ pending: {
173
+ $concatArrays: [
174
+ {$ifNull: ["$pending", []]},
175
+ {
176
+ $map: {
177
+ as: "s",
178
+ in: {claimedAt: now, seq: "$$s"},
179
+ input: {$range: [{$add: ["$_syncOldSeq", 1]}, {$add: ["$seq", 1]}]},
180
+ },
181
+ },
182
+ ],
183
+ },
184
+ },
185
+ },
186
+ {$unset: "_syncOldSeq"},
187
+ ],
188
+ {returnDocument: "after", updatePipeline: true, upsert: true}
189
+ );
190
+ return {lastSeq: doc.seq, registered: true, seqs: materialize(doc.seq)};
191
+ };
192
+ try {
193
+ return await claim();
194
+ } catch (error: unknown) {
195
+ // E11000: two concurrent upserts raced to create the counter; the loser retries
196
+ // against the now-existing document.
197
+ if ((error as {code?: number}).code === 11000) {
198
+ return claim();
199
+ }
200
+ throw error;
201
+ }
202
+ };
203
+
204
+ /** Outcome of clearing pending claims from a stream's in-flight registry. */
205
+ export interface SyncSeqConfirmResult {
206
+ /**
207
+ * True when at least one pending entry was actually removed. False means the claim
208
+ * was already gone — for a `registered` claim that is the reaped-lease signature
209
+ * (Task 9.17): the writer stalled past {@link PENDING_CLAIM_LEASE_MS},
210
+ * `computeStableFrontier` reclaimed its seq, and the frontier has already advanced
211
+ * past a seq whose write only just landed. Callers must re-stamp the document (see
212
+ * `restampReapedSeq`) or log loudly rather than silently stranding it.
213
+ */
214
+ cleared: boolean;
215
+ }
216
+
217
+ /** Remove the given pending claims from a stream's in-flight registry. */
218
+ const pullPendingClaims = async ({
219
+ stream,
220
+ seqs,
221
+ session,
222
+ }: {
223
+ stream: string;
224
+ seqs: number[];
225
+ session?: ClientSession | null;
226
+ }): Promise<SyncSeqConfirmResult> => {
227
+ const result = await SyncCounter.updateOne(
228
+ {stream},
229
+ {$pull: {pending: {seq: {$in: seqs}}}},
230
+ session ? {session} : {}
231
+ );
232
+ return {cleared: (result.modifiedCount ?? 0) > 0};
233
+ };
234
+
235
+ /**
236
+ * C1: confirm that the writes owning `seqs` on `stream` have committed, clearing
237
+ * their pending registry entries so the stable frontier can advance past them.
238
+ * A no-op when `seqs` is empty (a session-backed claim registered nothing). Runs
239
+ * after the document write commits (`post("save")` / query-write post hook); a
240
+ * `$pull` failure is logged by the caller and left to age out via the lease.
241
+ *
242
+ * Returns whether anything was actually cleared so callers can detect a reaped
243
+ * lease — see {@link SyncSeqConfirmResult}.
244
+ */
245
+ export const confirmSyncSeqs = async ({
246
+ stream,
247
+ seqs,
248
+ session,
249
+ }: {
250
+ stream: string;
251
+ seqs: number[];
252
+ session?: ClientSession | null;
253
+ }): Promise<SyncSeqConfirmResult> => {
254
+ if (seqs.length === 0) {
255
+ return {cleared: true};
256
+ }
257
+ return pullPendingClaims({seqs, session, stream});
258
+ };
259
+
260
+ /**
261
+ * Cancel claimed-but-never-used seqs (Task 9.13): the owning write failed, so the seqs
262
+ * will never be stamped on any document and holding the frontier below them for the
263
+ * full lease would stall every catch-up cursor on the stream for no reason. Unlike
264
+ * `confirmSyncSeqs` this is only safe when the write is known NOT to have committed
265
+ * (e.g. a Mongoose `VersionError`, where the conditional update matched no document).
266
+ */
267
+ export const releaseSyncSeqs = async ({
268
+ stream,
269
+ seqs,
270
+ session,
271
+ }: {
272
+ stream: string;
273
+ seqs: number[];
274
+ session?: ClientSession | null;
275
+ }): Promise<void> => {
276
+ if (seqs.length === 0) {
277
+ return;
278
+ }
279
+ await pullPendingClaims({seqs, session, stream});
280
+ };
281
+
282
+ /**
283
+ * C1: the stable frontier for a stream — the highest seq below which no claim is
284
+ * uncommitted. A cursor (snapshot or delta) may advance to seq N only when N is at
285
+ * or below this value, so no committed document is ever permanently skipped.
286
+ *
287
+ * frontier = `min(live pending.seq) - 1`, or the head `seq` when no live pending
288
+ * entries remain. A pending entry older than {@link PENDING_CLAIM_LEASE_MS} is
289
+ * considered abandoned (crashed writer): it is excluded from the min AND `$pull`ed
290
+ * opportunistically so a crash cannot freeze the frontier forever.
291
+ */
292
+ export const computeStableFrontier = async ({stream}: {stream: string}): Promise<number> => {
293
+ const counter = await SyncCounter.findOne({stream});
294
+ if (!counter) {
295
+ return 0;
296
+ }
297
+ const pending = counter.pending ?? [];
298
+ if (pending.length === 0) {
299
+ return counter.seq;
300
+ }
301
+ const cutoff = Date.now() - PENDING_CLAIM_LEASE_MS;
302
+ const staleSeqs: number[] = [];
303
+ let minLiveSeq = Number.POSITIVE_INFINITY;
304
+ for (const claim of pending) {
305
+ const claimedAtMs = new Date(claim.claimedAt).getTime();
306
+ if (claimedAtMs < cutoff) {
307
+ staleSeqs.push(claim.seq);
308
+ continue;
309
+ }
310
+ if (claim.seq < minLiveSeq) {
311
+ minLiveSeq = claim.seq;
312
+ }
313
+ }
314
+ if (staleSeqs.length > 0) {
315
+ // Opportunistic cleanup: a crashed writer's claim must not freeze the frontier.
316
+ await SyncCounter.updateOne({stream}, {$pull: {pending: {seq: {$in: staleSeqs}}}}).catch(
317
+ () => {}
318
+ );
319
+ }
320
+ // All pending entries were stale → frontier is the head; otherwise one below the
321
+ // lowest live (uncommitted) claim.
322
+ return minLiveSeq === Number.POSITIVE_INFINITY ? counter.seq : minLiveSeq - 1;
323
+ };
324
+
325
+ /**
326
+ * C7: raise a stream's retention watermark to `seq` (never lowers it — `$max`), called by
327
+ * `compactTombstones` after it hard-deletes rows. Served to clients as the snapshot's
328
+ * `oldestRetainedSeq`, which is the only signal that a stale cursor must re-bootstrap.
329
+ * Does not create the counter: a stream with compacted rows always already has one.
330
+ */
331
+ export const recordCompactedThroughSeq = async ({
332
+ stream,
333
+ seq,
334
+ }: {
335
+ stream: string;
336
+ seq: number;
337
+ }): Promise<void> => {
338
+ if (seq <= 0) {
339
+ return;
340
+ }
341
+ await SyncCounter.updateOne({stream}, {$max: {compactedThroughSeq: seq}});
342
+ };
343
+
344
+ /**
345
+ * C7: the stream's retention watermark — the highest seq whose tombstone/marker has been
346
+ * compacted away. 0 when nothing has been compacted (no retention gap to enforce).
347
+ */
348
+ export const getCompactedThroughSeq = async ({stream}: {stream: string}): Promise<number> => {
349
+ const counter = await findOneOrNoneFor(SyncCounter, {stream});
350
+ return counter?.compactedThroughSeq ?? 0;
351
+ };
352
+
353
+ export interface SyncScopeMoveDocument {
354
+ _id: mongoose.Types.ObjectId;
355
+ collectionTag: string;
356
+ entityId: string;
357
+ fromStream: string;
358
+ toStream: string;
359
+ seq: number;
360
+ created: Date;
361
+ }
362
+
363
+ const syncScopeMoveSchema = new Schema<SyncScopeMoveDocument>(
364
+ {
365
+ collectionTag: {
366
+ description: "The collection tag of the moved document",
367
+ required: true,
368
+ type: String,
369
+ },
370
+ created: {
371
+ default: () => new Date(),
372
+ description:
373
+ "When the move was recorded; reaped by compactTombstones once older than the " +
374
+ "owning model's retentionDays (deliberately NOT TTL-indexed — see the model doc)",
375
+ type: Date,
376
+ },
377
+ entityId: {
378
+ description: "The _id of the document that moved between streams",
379
+ required: true,
380
+ type: String,
381
+ },
382
+ fromStream: {
383
+ description: "The stream the document left (the one to tombstone)",
384
+ required: true,
385
+ type: String,
386
+ },
387
+ seq: {
388
+ description: "Seq claimed from the OLD stream's counter; orders the tombstone in that stream",
389
+ required: true,
390
+ type: Number,
391
+ },
392
+ toStream: {
393
+ description: "The stream the document moved into",
394
+ required: true,
395
+ type: String,
396
+ },
397
+ },
398
+ // Consumer apps run checkModelsStrict() at startup; framework models must comply.
399
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
400
+ );
401
+ // Old-stream snapshot catch-up pages markers by {fromStream, seq}.
402
+ syncScopeMoveSchema.index({fromStream: 1, seq: 1});
403
+ syncScopeMoveSchema.index({collectionTag: 1, entityId: 1});
404
+ // `compactTombstones` reaps markers by {collectionTag, created}.
405
+ syncScopeMoveSchema.index({collectionTag: 1, created: 1});
406
+
407
+ /**
408
+ * Name of the TTL index earlier versions built on `created`. Task 9.15 removed it:
409
+ * a fixed 90-day TTL ignored per-model `retentionDays`, so markers for a
410
+ * longer-retention model vanished with no watermark update and old-stream clients
411
+ * silently never learned about the move. `compactTombstones` is now the sole reaper
412
+ * (it raises `compactedThroughSeq` as it deletes), so the stale index is dropped at
413
+ * startup to stop it from reaping behind the watermark's back.
414
+ */
415
+ const LEGACY_SCOPE_MOVE_TTL_INDEX_NAME = "created_1";
416
+
417
+ /**
418
+ * C4: durable marker written in the same op-scope as a scope move, replacing the
419
+ * racy `_syncPrevStream` post-image read. The old stream tombstones the document
420
+ * from this marker (change-stream fan-out + snapshot catch-up), so a racing second
421
+ * write that overwrites `_syncPrevStream` can no longer erase the tombstone.
422
+ *
423
+ * Retention: reaped only by `compactTombstones`, which honors the owning model's
424
+ * `retentionDays` and records a `compactedThroughSeq` watermark as it deletes.
425
+ */
426
+ export const SyncScopeMove: Model<SyncScopeMoveDocument> =
427
+ (mongoose.models.SyncScopeMove as Model<SyncScopeMoveDocument>) ??
428
+ mongoose.model<SyncScopeMoveDocument>("SyncScopeMove", syncScopeMoveSchema);
429
+
430
+ /** Lifecycle status of a sync mutation ledger row. */
431
+ export type SyncMutationStatus = "pending" | "applied" | "conflicted" | "failed";
432
+
433
+ export interface SyncMutationDocument {
434
+ _id: mongoose.Types.ObjectId;
435
+ mutationId: string;
436
+ userId: string;
437
+ status: SyncMutationStatus;
438
+ nackCode?: string;
439
+ resultId?: string;
440
+ resultSeq?: number;
441
+ error?: string;
442
+ created: Date;
443
+ claimedAt: Date;
444
+ }
445
+
446
+ /** Ledger rows are only needed while a client could still replay a mutation. */
447
+ const SYNC_MUTATION_TTL_SECONDS = 30 * 24 * 60 * 60;
448
+
449
+ /**
450
+ * C5 (FIX 6): a `pending` row older than this may be taken over by a fresh
451
+ * delivery — the original claimant is assumed to have crashed between the
452
+ * claim and finalizing the ledger row.
453
+ */
454
+ export const SYNC_MUTATION_LEASE_MS = 60 * 1_000;
455
+
456
+ const syncMutationSchema = new Schema<SyncMutationDocument>(
457
+ {
458
+ claimedAt: {
459
+ default: () => new Date(),
460
+ description:
461
+ "When this delivery claimed the mutation (lease); a pending row older than " +
462
+ "SYNC_MUTATION_LEASE_MS may be taken over by a fresh delivery via findOneAndUpdate",
463
+ type: Date,
464
+ },
465
+ created: {
466
+ default: () => new Date(),
467
+ description: "When the mutation was first claimed; TTL-indexed so rows expire after 30 days",
468
+ type: Date,
469
+ },
470
+ error: {
471
+ description: "Error message recorded for rejected mutations",
472
+ type: String,
473
+ },
474
+ mutationId: {
475
+ description:
476
+ "Client-generated idempotency key; unique so concurrent deliveries claim exactly once",
477
+ required: true,
478
+ type: String,
479
+ unique: true,
480
+ },
481
+ nackCode: {
482
+ description:
483
+ "Nack code recorded for rejected mutations (conflict/unauthorized/validation/error)",
484
+ type: String,
485
+ },
486
+ resultId: {
487
+ description:
488
+ "The affected document id, recorded when the mutation is applied or conflicts; a " +
489
+ "duplicate delivery of a conflict re-serializes the live document from this id",
490
+ type: String,
491
+ },
492
+ resultSeq: {
493
+ description: "The document's _syncSeq after apply (applied) or at conflict time (conflicted)",
494
+ type: Number,
495
+ },
496
+ status: {
497
+ description: "Lifecycle status: pending while applying, then applied, conflicted, or failed",
498
+ enum: ["pending", "applied", "conflicted", "failed"],
499
+ required: true,
500
+ type: String,
501
+ },
502
+ userId: {
503
+ description: "The user who submitted the mutation; replays from other users are rejected",
504
+ required: true,
505
+ type: String,
506
+ },
507
+ },
508
+ // Consumer apps run checkModelsStrict() at startup; framework models must comply.
509
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
510
+ );
511
+ syncMutationSchema.index({created: 1}, {expireAfterSeconds: SYNC_MUTATION_TTL_SECONDS});
512
+
513
+ /**
514
+ * Idempotency ledger for the sync mutation channel. A row is inserted with status
515
+ * `pending` before a mutation is applied (the atomic claim on the unique mutationId);
516
+ * the outcome is recorded on the same row so duplicate deliveries — socket retries or
517
+ * the HTTP fallback racing a socket send — read back the recorded outcome instead of
518
+ * re-applying.
519
+ *
520
+ * Task 9.21: a row NEVER stores document data. A conflict records only
521
+ * `{resultId, resultSeq}`; a duplicate delivery re-serializes the live document from
522
+ * `resultId` through the collection's sync `responseHandler`, so PHI is not retained in
523
+ * `syncmutations` for the 30-day TTL and the client also sees current (not stale) state.
524
+ */
525
+ export const SyncMutation: Model<SyncMutationDocument> =
526
+ (mongoose.models.SyncMutation as Model<SyncMutationDocument>) ??
527
+ mongoose.model<SyncMutationDocument>("SyncMutation", syncMutationSchema);
528
+
529
+ export interface SyncKeyDocument {
530
+ _id: mongoose.Types.ObjectId;
531
+ userId: string;
532
+ keyMaterial: string;
533
+ created: Date;
534
+ }
535
+
536
+ const syncKeySchema = new Schema<SyncKeyDocument>(
537
+ {
538
+ created: {
539
+ default: () => new Date(),
540
+ description: "When this key material was generated",
541
+ type: Date,
542
+ },
543
+ keyMaterial: {
544
+ description: "Server-generated random key material (32 bytes, base64) for HKDF derivation",
545
+ required: true,
546
+ type: String,
547
+ },
548
+ userId: {
549
+ description: "The user this key material belongs to",
550
+ required: true,
551
+ type: String,
552
+ unique: true,
553
+ },
554
+ },
555
+ // Consumer apps run checkModelsStrict() at startup; framework models must comply.
556
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
557
+ );
558
+
559
+ export const SyncKey: Model<SyncKeyDocument> =
560
+ (mongoose.models.SyncKey as Model<SyncKeyDocument>) ??
561
+ mongoose.model<SyncKeyDocument>("SyncKey", syncKeySchema);
562
+
563
+ /**
564
+ * Every model backing the SyncDB protocol. Their indexes are correctness-critical, not
565
+ * just performance: without the unique `SyncMutation.mutationId` index a duplicate
566
+ * delivery double-applies a mutation (INV-3), and without the unique `SyncCounter.stream`
567
+ * index the counter upsert race mints duplicate seqs. Schema-level `unique: true` only
568
+ * builds them when Mongoose `autoIndex` is on — commonly disabled in production — so
569
+ * startup builds them explicitly.
570
+ */
571
+ // noExplicitAny: a heterogeneous list of models is only used for ensureIndexes
572
+ // biome-ignore lint/suspicious/noExplicitAny: a heterogeneous list of models is only used for ensureIndexes
573
+ const SYNC_BOOKKEEPING_MODELS: Model<any>[] = [SyncCounter, SyncMutation, SyncScopeMove, SyncKey];
574
+
575
+ /**
576
+ * Best-effort removal of the pre-Task-9.15 `SyncScopeMove` TTL index. Failure is only
577
+ * logged: a leftover TTL index degrades retention accounting but must never block
578
+ * startup, and the index is absent on fresh databases (the common case).
579
+ */
580
+ const dropLegacyScopeMoveTtlIndex = async (): Promise<void> => {
581
+ try {
582
+ const indexes = await SyncScopeMove.collection.indexes();
583
+ const legacy = indexes.find(
584
+ (index) =>
585
+ index.name === LEGACY_SCOPE_MOVE_TTL_INDEX_NAME && index.expireAfterSeconds !== undefined
586
+ );
587
+ if (!legacy) {
588
+ return;
589
+ }
590
+ await SyncScopeMove.collection.dropIndex(LEGACY_SCOPE_MOVE_TTL_INDEX_NAME);
591
+ logger.info(
592
+ "[sync] Dropped the legacy SyncScopeMove TTL index; compactTombstones is now the " +
593
+ "sole reaper of scope-move markers (it honors per-model retentionDays)."
594
+ );
595
+ } catch (error: unknown) {
596
+ logger.warn("[sync] Could not drop the legacy SyncScopeMove TTL index", {
597
+ error: String(error),
598
+ });
599
+ }
600
+ };
601
+
602
+ /**
603
+ * Build the indexes for every sync bookkeeping model, throwing on the first failure so
604
+ * server startup fails loudly. Kicked off by `SyncApp.register` and awaited through
605
+ * `ensureSyncIndexes()`; idempotent, so calling it repeatedly is safe.
606
+ */
607
+ export const ensureSyncModelIndexes = async (): Promise<void> => {
608
+ await dropLegacyScopeMoveTtlIndex();
609
+ await Promise.all(
610
+ SYNC_BOOKKEEPING_MODELS.map(async (model) => {
611
+ try {
612
+ await model.ensureIndexes();
613
+ } catch (error: unknown) {
614
+ logger.error(`[sync] Failed to ensure sync indexes for ${model.modelName}`, {
615
+ error: String(error),
616
+ });
617
+ throw new Error(
618
+ `Failed to ensure sync indexes for ${model.modelName}: ${String(error)}. ` +
619
+ "The sync protocol depends on these indexes (unique mutationId for mutation " +
620
+ "idempotency, unique stream for seq allocation); fix the DB and restart."
621
+ );
622
+ }
623
+ })
624
+ );
625
+ };
626
+
627
+ /**
628
+ * Return the user's key material, generating it on first call. Race-safe: concurrent
629
+ * first calls converge on the single persisted value via `$setOnInsert` upsert — a
630
+ * caller must never receive bytes that were not persisted, or its encrypted store
631
+ * would be undecryptable by any other session.
632
+ */
633
+ export const getOrCreateSyncKeyMaterial = async ({userId}: {userId: string}): Promise<string> => {
634
+ const candidate = crypto.randomBytes(32).toString("base64");
635
+ const doc = await SyncKey.findOneAndUpdate(
636
+ {userId},
637
+ {$setOnInsert: {created: new Date(), keyMaterial: candidate, userId}},
638
+ {new: true, upsert: true}
639
+ );
640
+ return doc.keyMaterial;
641
+ };