@terreno/api 0.30.0 → 56.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. package/dist/api.d.ts +10 -0
  2. package/dist/api.js +110 -361
  3. package/dist/auth.d.ts +26 -0
  4. package/dist/auth.js +100 -1
  5. package/dist/betterAuth.d.ts +5 -0
  6. package/dist/betterAuthSetup.js +23 -4
  7. package/dist/docLoader.js +12 -9
  8. package/dist/errors.d.ts +13 -0
  9. package/dist/errors.js +29 -1
  10. package/dist/index.d.ts +17 -1
  11. package/dist/index.js +29 -1
  12. package/dist/middleware.d.ts +7 -0
  13. package/dist/middleware.js +16 -1
  14. package/dist/openApiCompat.d.ts +10 -1
  15. package/dist/openApiCompat.js +115 -47
  16. package/dist/permissions.d.ts +27 -0
  17. package/dist/permissions.js +42 -1
  18. package/dist/plugins.d.ts +13 -0
  19. package/dist/plugins.js +35 -1
  20. package/dist/realtime/changeStreamWatcher.d.ts +100 -3
  21. package/dist/realtime/changeStreamWatcher.js +542 -137
  22. package/dist/realtime/realtimeApp.d.ts +2 -0
  23. package/dist/realtime/realtimeApp.js +73 -21
  24. package/dist/realtime/sessionRevalidation.d.ts +90 -0
  25. package/dist/realtime/sessionRevalidation.js +438 -0
  26. package/dist/realtime/socketAuth.d.ts +97 -0
  27. package/dist/realtime/socketAuth.js +272 -0
  28. package/dist/realtime/socketUser.d.ts +54 -0
  29. package/dist/realtime/socketUser.js +75 -5
  30. package/dist/realtime/types.d.ts +40 -0
  31. package/dist/secretProviders.d.ts +23 -0
  32. package/dist/secretProviders.js +9 -3
  33. package/dist/sync/executors.d.ts +198 -0
  34. package/dist/sync/executors.js +670 -0
  35. package/dist/sync/loadHarness.js +807 -0
  36. package/dist/sync/models.d.ts +216 -0
  37. package/dist/sync/models.js +644 -0
  38. package/dist/sync/mutationHandler.d.ts +97 -0
  39. package/dist/sync/mutationHandler.js +886 -0
  40. package/dist/sync/registry.d.ts +73 -0
  41. package/dist/sync/registry.js +223 -0
  42. package/dist/sync/routes.d.ts +62 -0
  43. package/dist/sync/routes.js +936 -0
  44. package/dist/sync/scripts/compactTombstones.d.ts +47 -0
  45. package/dist/sync/scripts/compactTombstones.js +270 -0
  46. package/dist/sync/serialize.d.ts +20 -0
  47. package/dist/{tests/bunSetup.js → sync/serialize.js} +35 -68
  48. package/dist/sync/socketHandlers.d.ts +54 -0
  49. package/dist/sync/socketHandlers.js +470 -0
  50. package/dist/sync/streams.d.ts +85 -0
  51. package/dist/sync/streams.js +221 -0
  52. package/dist/sync/syncApp.d.ts +29 -0
  53. package/dist/sync/syncApp.js +44 -0
  54. package/dist/sync/syncBatch.d.ts +71 -0
  55. package/dist/sync/syncBatch.js +252 -0
  56. package/dist/sync/syncSeqPlugin.d.ts +8 -0
  57. package/dist/sync/syncSeqPlugin.js +903 -0
  58. package/dist/sync/types.d.ts +254 -0
  59. package/dist/sync/types.js +9 -0
  60. package/dist/terrenoApp.js +90 -27
  61. package/dist/vendor/wesleytodd-openapi/index.d.ts +1 -1
  62. package/dist/vendor/wesleytodd-openapi/lib/ui.d.ts +2 -1
  63. package/package.json +7 -6
  64. package/src/__tests__/openApiCompat.test.ts +20 -0
  65. package/src/actions.openApi.test.ts +7 -10
  66. package/src/api.test.ts +12 -0
  67. package/src/api.ts +87 -295
  68. package/src/auth.test.ts +128 -176
  69. package/src/auth.ts +109 -0
  70. package/src/betterAuth.ts +6 -0
  71. package/src/betterAuthSetup.test.ts +34 -0
  72. package/src/betterAuthSetup.ts +20 -1
  73. package/src/docLoader.ts +3 -1
  74. package/src/errors.ts +33 -0
  75. package/src/expressServer.test.ts +7 -1
  76. package/src/index.ts +42 -0
  77. package/src/middleware.test.ts +40 -1
  78. package/src/middleware.ts +15 -0
  79. package/src/openApiCompat.ts +99 -45
  80. package/src/permissions.test.ts +75 -1
  81. package/src/permissions.ts +49 -0
  82. package/src/plugins.test.ts +54 -0
  83. package/src/plugins.ts +43 -0
  84. package/src/realtime/changeStreamWatcher.ts +552 -114
  85. package/src/realtime/realtime.test.ts +157 -0
  86. package/src/realtime/realtimeApp.ts +75 -11
  87. package/src/realtime/sessionRevalidation.test.ts +423 -0
  88. package/src/realtime/sessionRevalidation.ts +273 -0
  89. package/src/realtime/socketAuth.test.ts +355 -0
  90. package/src/realtime/socketAuth.ts +228 -0
  91. package/src/realtime/socketUser.ts +69 -0
  92. package/src/realtime/types.ts +40 -0
  93. package/src/secretProviders.test.ts +7 -1
  94. package/src/secretProviders.ts +15 -3
  95. package/src/sync/ensureSyncIndexes.test.ts +144 -0
  96. package/src/sync/executors.test.ts +882 -0
  97. package/src/sync/executors.ts +703 -0
  98. package/src/sync/integration.test.ts +1160 -0
  99. package/src/sync/loadHarness.ts +618 -0
  100. package/src/sync/models.ts +640 -0
  101. package/src/sync/mutationHandler.test.ts +1252 -0
  102. package/src/sync/mutationHandler.ts +850 -0
  103. package/src/sync/registry.ts +232 -0
  104. package/src/sync/routes.ts +812 -0
  105. package/src/sync/scripts/compactTombstones.test.ts +249 -0
  106. package/src/sync/scripts/compactTombstones.ts +145 -0
  107. package/src/sync/serialize.ts +48 -0
  108. package/src/sync/sessionRevalidationIntegration.test.ts +281 -0
  109. package/src/sync/socketHandlers.ts +320 -0
  110. package/src/sync/streams.ts +188 -0
  111. package/src/sync/sync.test.ts +582 -0
  112. package/src/sync/syncApp.ts +44 -0
  113. package/src/sync/syncBatch.ts +219 -0
  114. package/src/sync/syncFrontier.test.ts +494 -0
  115. package/src/sync/syncPhaseC.test.ts +718 -0
  116. package/src/sync/syncRoutes.test.ts +865 -0
  117. package/src/sync/syncSeqPlugin.ts +738 -0
  118. package/src/sync/syncSocket.test.ts +1440 -0
  119. package/src/sync/types.ts +270 -0
  120. package/src/terrenoApp.ts +37 -17
  121. package/src/tests/bunSetup.ts +8 -0
  122. package/tsconfig.json +4 -4
  123. package/tsconfig.typedoc.json +7 -1
  124. package/dist/__tests__/openApiCompat.test.js +0 -317
  125. package/dist/__tests__/versionCheckPlugin.test.d.ts +0 -1
  126. package/dist/__tests__/versionCheckPlugin.test.js +0 -597
  127. package/dist/actions.openApi.test.d.ts +0 -1
  128. package/dist/actions.openApi.test.js +0 -255
  129. package/dist/actions.test.d.ts +0 -1
  130. package/dist/actions.test.js +0 -1089
  131. package/dist/api.arrayOperations.test.d.ts +0 -1
  132. package/dist/api.arrayOperations.test.js +0 -878
  133. package/dist/api.asyncHandler.test.d.ts +0 -1
  134. package/dist/api.asyncHandler.test.js +0 -236
  135. package/dist/api.coverage.test.d.ts +0 -1
  136. package/dist/api.coverage.test.js +0 -194
  137. package/dist/api.errorPaths.test.d.ts +0 -1
  138. package/dist/api.errorPaths.test.js +0 -801
  139. package/dist/api.errors.test.d.ts +0 -1
  140. package/dist/api.errors.test.js +0 -185
  141. package/dist/api.hooks.test.d.ts +0 -1
  142. package/dist/api.hooks.test.js +0 -950
  143. package/dist/api.query.test.d.ts +0 -1
  144. package/dist/api.query.test.js +0 -948
  145. package/dist/api.test.d.ts +0 -1
  146. package/dist/api.test.js +0 -2820
  147. package/dist/auth.compile.test.d.ts +0 -1
  148. package/dist/auth.compile.test.js +0 -201
  149. package/dist/auth.coverage.test.d.ts +0 -1
  150. package/dist/auth.coverage.test.js +0 -202
  151. package/dist/auth.test.d.ts +0 -1
  152. package/dist/auth.test.js +0 -2286
  153. package/dist/betterAuth.test.d.ts +0 -1
  154. package/dist/betterAuth.test.js +0 -181
  155. package/dist/betterAuthApp.test.d.ts +0 -1
  156. package/dist/betterAuthApp.test.js +0 -242
  157. package/dist/betterAuthSetup.test.d.ts +0 -1
  158. package/dist/betterAuthSetup.test.js +0 -762
  159. package/dist/config.test.d.ts +0 -1
  160. package/dist/config.test.js +0 -353
  161. package/dist/configuration.test.d.ts +0 -1
  162. package/dist/configuration.test.js +0 -980
  163. package/dist/configurationPlugin.test.d.ts +0 -1
  164. package/dist/configurationPlugin.test.js +0 -827
  165. package/dist/consentApp.test.d.ts +0 -1
  166. package/dist/consentApp.test.js +0 -1415
  167. package/dist/docLoader.test.d.ts +0 -1
  168. package/dist/docLoader.test.js +0 -137
  169. package/dist/envConfigurationPlugin.test.d.ts +0 -1
  170. package/dist/envConfigurationPlugin.test.js +0 -369
  171. package/dist/errors.test.d.ts +0 -1
  172. package/dist/errors.test.js +0 -607
  173. package/dist/expressServer.test.d.ts +0 -1
  174. package/dist/expressServer.test.js +0 -1249
  175. package/dist/githubAuth.test.d.ts +0 -1
  176. package/dist/githubAuth.test.js +0 -1142
  177. package/dist/httpClient.test.d.ts +0 -1
  178. package/dist/httpClient.test.js +0 -1471
  179. package/dist/logger.test.d.ts +0 -1
  180. package/dist/logger.test.js +0 -685
  181. package/dist/middleware.test.d.ts +0 -1
  182. package/dist/middleware.test.js +0 -241
  183. package/dist/notifiers/googleChatNotifier.test.d.ts +0 -1
  184. package/dist/notifiers/googleChatNotifier.test.js +0 -298
  185. package/dist/notifiers/slackNotifier.test.d.ts +0 -1
  186. package/dist/notifiers/slackNotifier.test.js +0 -297
  187. package/dist/notifiers/zoomNotifier.test.d.ts +0 -1
  188. package/dist/notifiers/zoomNotifier.test.js +0 -373
  189. package/dist/openApi.test.d.ts +0 -1
  190. package/dist/openApi.test.js +0 -620
  191. package/dist/openApiBuilder.test.d.ts +0 -1
  192. package/dist/openApiBuilder.test.js +0 -601
  193. package/dist/openApiEtag.test.d.ts +0 -1
  194. package/dist/openApiEtag.test.js +0 -86
  195. package/dist/openApiValidator.test.d.ts +0 -1
  196. package/dist/openApiValidator.test.js +0 -761
  197. package/dist/permissions.middleware.test.d.ts +0 -1
  198. package/dist/permissions.middleware.test.js +0 -308
  199. package/dist/permissions.test.d.ts +0 -1
  200. package/dist/permissions.test.js +0 -439
  201. package/dist/plugins.test.d.ts +0 -1
  202. package/dist/plugins.test.js +0 -890
  203. package/dist/populate.test.d.ts +0 -1
  204. package/dist/populate.test.js +0 -373
  205. package/dist/realtime/realtime.test.d.ts +0 -10
  206. package/dist/realtime/realtime.test.js +0 -3982
  207. package/dist/requestContext.test.d.ts +0 -1
  208. package/dist/requestContext.test.js +0 -412
  209. package/dist/scriptRunner.test.d.ts +0 -1
  210. package/dist/scriptRunner.test.js +0 -386
  211. package/dist/secretProviders.test.d.ts +0 -1
  212. package/dist/secretProviders.test.js +0 -726
  213. package/dist/syncConsents.test.d.ts +0 -1
  214. package/dist/syncConsents.test.js +0 -522
  215. package/dist/terrenoApp.test.d.ts +0 -1
  216. package/dist/terrenoApp.test.js +0 -489
  217. package/dist/tests/bunSetup.d.ts +0 -1
  218. package/dist/tests/fixtures/compileAuthEntry.d.ts +0 -1
  219. package/dist/tests/fixtures/compileAuthEntry.js +0 -37
  220. package/dist/tests/testEnv.d.ts +0 -0
  221. package/dist/tests/testEnv.js +0 -6
  222. package/dist/transformers.test.d.ts +0 -1
  223. package/dist/transformers.test.js +0 -423
  224. package/dist/utils.test.d.ts +0 -1
  225. package/dist/utils.test.js +0 -242
  226. /package/dist/{__tests__/openApiCompat.test.d.ts → sync/loadHarness.d.ts} +0 -0
@@ -0,0 +1,640 @@
1
+ import crypto from "node:crypto";
2
+ import mongoose, {type ClientSession, type Model, Schema} from "mongoose";
3
+ import {logger} from "../logger";
4
+ import {findOneOrNoneFor} from "../plugins";
5
+
6
+ /**
7
+ * Mongoose models backing the SyncDB protocol: the per-stream monotonic counter
8
+ * and the per-user encryption key material.
9
+ */
10
+
11
+ /** An uncommitted seq claim recorded on the counter's in-flight registry (C1). */
12
+ export interface SyncPendingClaim {
13
+ /** The claimed seq (or, for a batch, one entry per claimed seq). */
14
+ seq: number;
15
+ /** When the claim was registered; a claim older than the lease is reclaimable. */
16
+ claimedAt: Date;
17
+ }
18
+
19
+ export interface SyncCounterDocument {
20
+ _id: mongoose.Types.ObjectId;
21
+ stream: string;
22
+ seq: number;
23
+ /**
24
+ * C1: seqs claimed but not yet confirmed committed. The stable frontier is
25
+ * `min(pending.seq) - 1`, or `seq` (the head) when empty, so a cursor never
26
+ * advances past a seq whose owning write has not yet landed.
27
+ */
28
+ pending: SyncPendingClaim[];
29
+ /**
30
+ * C7 retention watermark: the highest seq on this stream whose row (tombstone or
31
+ * scope-move marker) has been hard-deleted by `compactTombstones`. A client whose
32
+ * cursor is below this may have missed a compacted deletion and must re-bootstrap;
33
+ * a client at or above it has already seen everything that was reaped. 0 until the
34
+ * first compaction pass touches the stream.
35
+ */
36
+ compactedThroughSeq: number;
37
+ }
38
+
39
+ /**
40
+ * C1: a `pending` claim older than this is treated as abandoned (the writer
41
+ * crashed between claiming and confirming) and excluded from the frontier — a
42
+ * crashed writer must never freeze the frontier forever. Default 60s.
43
+ */
44
+ export const PENDING_CLAIM_LEASE_MS = 60 * 1_000;
45
+
46
+ const syncPendingClaimSchema = new Schema<SyncPendingClaim>(
47
+ {
48
+ claimedAt: {
49
+ default: () => new Date(),
50
+ description: "When this seq was claimed; a claim older than PENDING_CLAIM_LEASE_MS is stale",
51
+ type: Date,
52
+ },
53
+ seq: {
54
+ description: "A claimed-but-uncommitted sequence number",
55
+ required: true,
56
+ type: Number,
57
+ },
58
+ },
59
+ {_id: false, strict: "throw"}
60
+ );
61
+
62
+ const syncCounterSchema = new Schema<SyncCounterDocument>(
63
+ {
64
+ compactedThroughSeq: {
65
+ default: 0,
66
+ description:
67
+ "C7 retention watermark: the highest seq whose tombstone/scope-move row has been " +
68
+ "hard-deleted by compactTombstones; served as the snapshot's oldestRetainedSeq",
69
+ type: Number,
70
+ },
71
+ pending: {
72
+ default: [],
73
+ description:
74
+ "C1 in-flight registry: seqs claimed but not yet confirmed committed; the stable " +
75
+ "frontier is min(pending.seq) - 1 (or the head when empty)",
76
+ type: [syncPendingClaimSchema],
77
+ },
78
+ seq: {
79
+ default: 0,
80
+ description: "The last sequence number claimed for this stream",
81
+ type: Number,
82
+ },
83
+ stream: {
84
+ description: "Stream key this counter belongs to (e.g. 'todos|owner:123')",
85
+ required: true,
86
+ type: String,
87
+ unique: true,
88
+ },
89
+ },
90
+ // Consumer apps run checkModelsStrict() at startup; framework models must comply.
91
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
92
+ );
93
+
94
+ export const SyncCounter: Model<SyncCounterDocument> =
95
+ (mongoose.models.SyncCounter as Model<SyncCounterDocument>) ??
96
+ mongoose.model<SyncCounterDocument>("SyncCounter", syncCounterSchema);
97
+
98
+ /**
99
+ * Result of a seq claim: the last seq claimed (range [lastSeq - count + 1, lastSeq])
100
+ * and whether the claim registered a pending entry that a later `confirmSyncSeqs`
101
+ * must clear. A session-backed claim skips the pending registry entirely (the write
102
+ * and the `$inc` commit atomically), so `registered` is false and no confirm is due.
103
+ */
104
+ export interface SyncSeqClaim {
105
+ /** The last (highest) seq claimed; the range is [lastSeq - count + 1, lastSeq]. */
106
+ lastSeq: number;
107
+ /** Every seq claimed, in ascending order (the range materialized). */
108
+ seqs: number[];
109
+ /** True when a pending registry entry was recorded and `confirmSyncSeqs` must clear it. */
110
+ registered: boolean;
111
+ }
112
+
113
+ /**
114
+ * C1: atomically claim `count` sequence numbers for a stream AND register them on
115
+ * the in-flight registry so the stable frontier can exclude them until committed.
116
+ *
117
+ * Two modes:
118
+ * - **No caller session (the hot path):** one `findOneAndUpdate` does `$inc: {seq}`
119
+ * plus `$push` of a `pending` entry per claimed seq. The write commits separately,
120
+ * so `confirmSyncSeqs` must `$pull` the entries once it lands. Until then the
121
+ * frontier holds below the lowest pending seq.
122
+ * - **Caller session present:** the `$inc` and the document write commit atomically
123
+ * in the caller's transaction, so there is no window where a claimed seq is
124
+ * uncommitted — the pending registry is skipped (`registered: false`, nothing to
125
+ * confirm). Frontier logic treats a session-backed claim as already committed.
126
+ *
127
+ * Retries once on the upsert race (two concurrent first claims for a new stream).
128
+ */
129
+ export const claimSyncSeqs = async ({
130
+ stream,
131
+ count = 1,
132
+ session,
133
+ }: {
134
+ stream: string;
135
+ count?: number;
136
+ session?: ClientSession | null;
137
+ }): Promise<SyncSeqClaim> => {
138
+ const materialize = (lastSeq: number): number[] => {
139
+ const seqs: number[] = [];
140
+ for (let s = lastSeq - count + 1; s <= lastSeq; s++) {
141
+ seqs.push(s);
142
+ }
143
+ return seqs;
144
+ };
145
+
146
+ // Session-backed fast path: the write and the $inc are already atomic, so we skip
147
+ // the pending registry — there is never an uncommitted-but-claimed window to fence.
148
+ if (session) {
149
+ const doc = await SyncCounter.findOneAndUpdate(
150
+ {stream},
151
+ {$inc: {seq: count}},
152
+ {new: true, session, upsert: true}
153
+ );
154
+ return {lastSeq: doc.seq, registered: false, seqs: materialize(doc.seq)};
155
+ }
156
+
157
+ const claim = async (): Promise<SyncSeqClaim> => {
158
+ const now = new Date();
159
+ // Claim (`$inc`) AND register the pending entries in ONE atomic aggregation-pipeline
160
+ // update, so there is never a window where the incremented head is visible without
161
+ // its pending claims (which would let `computeStableFrontier` transiently report a
162
+ // frontier above an about-to-be-pending seq). `$range(oldSeq+1, newSeq+1)` materializes
163
+ // the claimed range from the just-incremented value; `$map` turns it into pending docs.
164
+ const doc = await SyncCounter.findOneAndUpdate(
165
+ {stream},
166
+ [
167
+ {$set: {_syncOldSeq: {$ifNull: ["$seq", 0]}}},
168
+ {$set: {seq: {$add: ["$_syncOldSeq", count]}}},
169
+ {
170
+ $set: {
171
+ pending: {
172
+ $concatArrays: [
173
+ {$ifNull: ["$pending", []]},
174
+ {
175
+ $map: {
176
+ as: "s",
177
+ in: {claimedAt: now, seq: "$$s"},
178
+ input: {$range: [{$add: ["$_syncOldSeq", 1]}, {$add: ["$seq", 1]}]},
179
+ },
180
+ },
181
+ ],
182
+ },
183
+ },
184
+ },
185
+ {$unset: "_syncOldSeq"},
186
+ ],
187
+ {new: true, upsert: true}
188
+ );
189
+ return {lastSeq: doc.seq, registered: true, seqs: materialize(doc.seq)};
190
+ };
191
+ try {
192
+ return await claim();
193
+ } catch (error: unknown) {
194
+ // E11000: two concurrent upserts raced to create the counter; the loser retries
195
+ // against the now-existing document.
196
+ if ((error as {code?: number}).code === 11000) {
197
+ return claim();
198
+ }
199
+ throw error;
200
+ }
201
+ };
202
+
203
+ /** Outcome of clearing pending claims from a stream's in-flight registry. */
204
+ export interface SyncSeqConfirmResult {
205
+ /**
206
+ * True when at least one pending entry was actually removed. False means the claim
207
+ * was already gone — for a `registered` claim that is the reaped-lease signature
208
+ * (Task 9.17): the writer stalled past {@link PENDING_CLAIM_LEASE_MS},
209
+ * `computeStableFrontier` reclaimed its seq, and the frontier has already advanced
210
+ * past a seq whose write only just landed. Callers must re-stamp the document (see
211
+ * `restampReapedSeq`) or log loudly rather than silently stranding it.
212
+ */
213
+ cleared: boolean;
214
+ }
215
+
216
+ /** Remove the given pending claims from a stream's in-flight registry. */
217
+ const pullPendingClaims = async ({
218
+ stream,
219
+ seqs,
220
+ session,
221
+ }: {
222
+ stream: string;
223
+ seqs: number[];
224
+ session?: ClientSession | null;
225
+ }): Promise<SyncSeqConfirmResult> => {
226
+ const result = await SyncCounter.updateOne(
227
+ {stream},
228
+ {$pull: {pending: {seq: {$in: seqs}}}},
229
+ session ? {session} : {}
230
+ );
231
+ return {cleared: (result.modifiedCount ?? 0) > 0};
232
+ };
233
+
234
+ /**
235
+ * C1: confirm that the writes owning `seqs` on `stream` have committed, clearing
236
+ * their pending registry entries so the stable frontier can advance past them.
237
+ * A no-op when `seqs` is empty (a session-backed claim registered nothing). Runs
238
+ * after the document write commits (`post("save")` / query-write post hook); a
239
+ * `$pull` failure is logged by the caller and left to age out via the lease.
240
+ *
241
+ * Returns whether anything was actually cleared so callers can detect a reaped
242
+ * lease — see {@link SyncSeqConfirmResult}.
243
+ */
244
+ export const confirmSyncSeqs = async ({
245
+ stream,
246
+ seqs,
247
+ session,
248
+ }: {
249
+ stream: string;
250
+ seqs: number[];
251
+ session?: ClientSession | null;
252
+ }): Promise<SyncSeqConfirmResult> => {
253
+ if (seqs.length === 0) {
254
+ return {cleared: true};
255
+ }
256
+ return pullPendingClaims({seqs, session, stream});
257
+ };
258
+
259
+ /**
260
+ * Cancel claimed-but-never-used seqs (Task 9.13): the owning write failed, so the seqs
261
+ * will never be stamped on any document and holding the frontier below them for the
262
+ * full lease would stall every catch-up cursor on the stream for no reason. Unlike
263
+ * `confirmSyncSeqs` this is only safe when the write is known NOT to have committed
264
+ * (e.g. a Mongoose `VersionError`, where the conditional update matched no document).
265
+ */
266
+ export const releaseSyncSeqs = async ({
267
+ stream,
268
+ seqs,
269
+ session,
270
+ }: {
271
+ stream: string;
272
+ seqs: number[];
273
+ session?: ClientSession | null;
274
+ }): Promise<void> => {
275
+ if (seqs.length === 0) {
276
+ return;
277
+ }
278
+ await pullPendingClaims({seqs, session, stream});
279
+ };
280
+
281
+ /**
282
+ * C1: the stable frontier for a stream — the highest seq below which no claim is
283
+ * uncommitted. A cursor (snapshot or delta) may advance to seq N only when N is at
284
+ * or below this value, so no committed document is ever permanently skipped.
285
+ *
286
+ * frontier = `min(live pending.seq) - 1`, or the head `seq` when no live pending
287
+ * entries remain. A pending entry older than {@link PENDING_CLAIM_LEASE_MS} is
288
+ * considered abandoned (crashed writer): it is excluded from the min AND `$pull`ed
289
+ * opportunistically so a crash cannot freeze the frontier forever.
290
+ */
291
+ export const computeStableFrontier = async ({stream}: {stream: string}): Promise<number> => {
292
+ const counter = await SyncCounter.findOne({stream});
293
+ if (!counter) {
294
+ return 0;
295
+ }
296
+ const pending = counter.pending ?? [];
297
+ if (pending.length === 0) {
298
+ return counter.seq;
299
+ }
300
+ const cutoff = Date.now() - PENDING_CLAIM_LEASE_MS;
301
+ const staleSeqs: number[] = [];
302
+ let minLiveSeq = Number.POSITIVE_INFINITY;
303
+ for (const claim of pending) {
304
+ const claimedAtMs = new Date(claim.claimedAt).getTime();
305
+ if (claimedAtMs < cutoff) {
306
+ staleSeqs.push(claim.seq);
307
+ continue;
308
+ }
309
+ if (claim.seq < minLiveSeq) {
310
+ minLiveSeq = claim.seq;
311
+ }
312
+ }
313
+ if (staleSeqs.length > 0) {
314
+ // Opportunistic cleanup: a crashed writer's claim must not freeze the frontier.
315
+ await SyncCounter.updateOne({stream}, {$pull: {pending: {seq: {$in: staleSeqs}}}}).catch(
316
+ () => {}
317
+ );
318
+ }
319
+ // All pending entries were stale → frontier is the head; otherwise one below the
320
+ // lowest live (uncommitted) claim.
321
+ return minLiveSeq === Number.POSITIVE_INFINITY ? counter.seq : minLiveSeq - 1;
322
+ };
323
+
324
+ /**
325
+ * C7: raise a stream's retention watermark to `seq` (never lowers it — `$max`), called by
326
+ * `compactTombstones` after it hard-deletes rows. Served to clients as the snapshot's
327
+ * `oldestRetainedSeq`, which is the only signal that a stale cursor must re-bootstrap.
328
+ * Does not create the counter: a stream with compacted rows always already has one.
329
+ */
330
+ export const recordCompactedThroughSeq = async ({
331
+ stream,
332
+ seq,
333
+ }: {
334
+ stream: string;
335
+ seq: number;
336
+ }): Promise<void> => {
337
+ if (seq <= 0) {
338
+ return;
339
+ }
340
+ await SyncCounter.updateOne({stream}, {$max: {compactedThroughSeq: seq}});
341
+ };
342
+
343
+ /**
344
+ * C7: the stream's retention watermark — the highest seq whose tombstone/marker has been
345
+ * compacted away. 0 when nothing has been compacted (no retention gap to enforce).
346
+ */
347
+ export const getCompactedThroughSeq = async ({stream}: {stream: string}): Promise<number> => {
348
+ const counter = await findOneOrNoneFor(SyncCounter, {stream});
349
+ return counter?.compactedThroughSeq ?? 0;
350
+ };
351
+
352
+ export interface SyncScopeMoveDocument {
353
+ _id: mongoose.Types.ObjectId;
354
+ collectionTag: string;
355
+ entityId: string;
356
+ fromStream: string;
357
+ toStream: string;
358
+ seq: number;
359
+ created: Date;
360
+ }
361
+
362
+ const syncScopeMoveSchema = new Schema<SyncScopeMoveDocument>(
363
+ {
364
+ collectionTag: {
365
+ description: "The collection tag of the moved document",
366
+ required: true,
367
+ type: String,
368
+ },
369
+ created: {
370
+ default: () => new Date(),
371
+ description:
372
+ "When the move was recorded; reaped by compactTombstones once older than the " +
373
+ "owning model's retentionDays (deliberately NOT TTL-indexed — see the model doc)",
374
+ type: Date,
375
+ },
376
+ entityId: {
377
+ description: "The _id of the document that moved between streams",
378
+ required: true,
379
+ type: String,
380
+ },
381
+ fromStream: {
382
+ description: "The stream the document left (the one to tombstone)",
383
+ required: true,
384
+ type: String,
385
+ },
386
+ seq: {
387
+ description: "Seq claimed from the OLD stream's counter; orders the tombstone in that stream",
388
+ required: true,
389
+ type: Number,
390
+ },
391
+ toStream: {
392
+ description: "The stream the document moved into",
393
+ required: true,
394
+ type: String,
395
+ },
396
+ },
397
+ // Consumer apps run checkModelsStrict() at startup; framework models must comply.
398
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
399
+ );
400
+ // Old-stream snapshot catch-up pages markers by {fromStream, seq}.
401
+ syncScopeMoveSchema.index({fromStream: 1, seq: 1});
402
+ syncScopeMoveSchema.index({collectionTag: 1, entityId: 1});
403
+ // `compactTombstones` reaps markers by {collectionTag, created}.
404
+ syncScopeMoveSchema.index({collectionTag: 1, created: 1});
405
+
406
+ /**
407
+ * Name of the TTL index earlier versions built on `created`. Task 9.15 removed it:
408
+ * a fixed 90-day TTL ignored per-model `retentionDays`, so markers for a
409
+ * longer-retention model vanished with no watermark update and old-stream clients
410
+ * silently never learned about the move. `compactTombstones` is now the sole reaper
411
+ * (it raises `compactedThroughSeq` as it deletes), so the stale index is dropped at
412
+ * startup to stop it from reaping behind the watermark's back.
413
+ */
414
+ const LEGACY_SCOPE_MOVE_TTL_INDEX_NAME = "created_1";
415
+
416
+ /**
417
+ * C4: durable marker written in the same op-scope as a scope move, replacing the
418
+ * racy `_syncPrevStream` post-image read. The old stream tombstones the document
419
+ * from this marker (change-stream fan-out + snapshot catch-up), so a racing second
420
+ * write that overwrites `_syncPrevStream` can no longer erase the tombstone.
421
+ *
422
+ * Retention: reaped only by `compactTombstones`, which honors the owning model's
423
+ * `retentionDays` and records a `compactedThroughSeq` watermark as it deletes.
424
+ */
425
+ export const SyncScopeMove: Model<SyncScopeMoveDocument> =
426
+ (mongoose.models.SyncScopeMove as Model<SyncScopeMoveDocument>) ??
427
+ mongoose.model<SyncScopeMoveDocument>("SyncScopeMove", syncScopeMoveSchema);
428
+
429
+ /** Lifecycle status of a sync mutation ledger row. */
430
+ export type SyncMutationStatus = "pending" | "applied" | "conflicted" | "failed";
431
+
432
+ export interface SyncMutationDocument {
433
+ _id: mongoose.Types.ObjectId;
434
+ mutationId: string;
435
+ userId: string;
436
+ status: SyncMutationStatus;
437
+ nackCode?: string;
438
+ resultId?: string;
439
+ resultSeq?: number;
440
+ error?: string;
441
+ created: Date;
442
+ claimedAt: Date;
443
+ }
444
+
445
+ /** Ledger rows are only needed while a client could still replay a mutation. */
446
+ const SYNC_MUTATION_TTL_SECONDS = 30 * 24 * 60 * 60;
447
+
448
+ /**
449
+ * C5 (FIX 6): a `pending` row older than this may be taken over by a fresh
450
+ * delivery — the original claimant is assumed to have crashed between the
451
+ * claim and finalizing the ledger row.
452
+ */
453
+ export const SYNC_MUTATION_LEASE_MS = 60 * 1_000;
454
+
455
+ const syncMutationSchema = new Schema<SyncMutationDocument>(
456
+ {
457
+ claimedAt: {
458
+ default: () => new Date(),
459
+ description:
460
+ "When this delivery claimed the mutation (lease); a pending row older than " +
461
+ "SYNC_MUTATION_LEASE_MS may be taken over by a fresh delivery via findOneAndUpdate",
462
+ type: Date,
463
+ },
464
+ created: {
465
+ default: () => new Date(),
466
+ description: "When the mutation was first claimed; TTL-indexed so rows expire after 30 days",
467
+ type: Date,
468
+ },
469
+ error: {
470
+ description: "Error message recorded for rejected mutations",
471
+ type: String,
472
+ },
473
+ mutationId: {
474
+ description:
475
+ "Client-generated idempotency key; unique so concurrent deliveries claim exactly once",
476
+ required: true,
477
+ type: String,
478
+ unique: true,
479
+ },
480
+ nackCode: {
481
+ description:
482
+ "Nack code recorded for rejected mutations (conflict/unauthorized/validation/error)",
483
+ type: String,
484
+ },
485
+ resultId: {
486
+ description:
487
+ "The affected document id, recorded when the mutation is applied or conflicts; a " +
488
+ "duplicate delivery of a conflict re-serializes the live document from this id",
489
+ type: String,
490
+ },
491
+ resultSeq: {
492
+ description: "The document's _syncSeq after apply (applied) or at conflict time (conflicted)",
493
+ type: Number,
494
+ },
495
+ status: {
496
+ description: "Lifecycle status: pending while applying, then applied, conflicted, or failed",
497
+ enum: ["pending", "applied", "conflicted", "failed"],
498
+ required: true,
499
+ type: String,
500
+ },
501
+ userId: {
502
+ description: "The user who submitted the mutation; replays from other users are rejected",
503
+ required: true,
504
+ type: String,
505
+ },
506
+ },
507
+ // Consumer apps run checkModelsStrict() at startup; framework models must comply.
508
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
509
+ );
510
+ syncMutationSchema.index({created: 1}, {expireAfterSeconds: SYNC_MUTATION_TTL_SECONDS});
511
+
512
+ /**
513
+ * Idempotency ledger for the sync mutation channel. A row is inserted with status
514
+ * `pending` before a mutation is applied (the atomic claim on the unique mutationId);
515
+ * the outcome is recorded on the same row so duplicate deliveries — socket retries or
516
+ * the HTTP fallback racing a socket send — read back the recorded outcome instead of
517
+ * re-applying.
518
+ *
519
+ * Task 9.21: a row NEVER stores document data. A conflict records only
520
+ * `{resultId, resultSeq}`; a duplicate delivery re-serializes the live document from
521
+ * `resultId` through the collection's sync `responseHandler`, so PHI is not retained in
522
+ * `syncmutations` for the 30-day TTL and the client also sees current (not stale) state.
523
+ */
524
+ export const SyncMutation: Model<SyncMutationDocument> =
525
+ (mongoose.models.SyncMutation as Model<SyncMutationDocument>) ??
526
+ mongoose.model<SyncMutationDocument>("SyncMutation", syncMutationSchema);
527
+
528
+ export interface SyncKeyDocument {
529
+ _id: mongoose.Types.ObjectId;
530
+ userId: string;
531
+ keyMaterial: string;
532
+ created: Date;
533
+ }
534
+
535
+ const syncKeySchema = new Schema<SyncKeyDocument>(
536
+ {
537
+ created: {
538
+ default: () => new Date(),
539
+ description: "When this key material was generated",
540
+ type: Date,
541
+ },
542
+ keyMaterial: {
543
+ description: "Server-generated random key material (32 bytes, base64) for HKDF derivation",
544
+ required: true,
545
+ type: String,
546
+ },
547
+ userId: {
548
+ description: "The user this key material belongs to",
549
+ required: true,
550
+ type: String,
551
+ unique: true,
552
+ },
553
+ },
554
+ // Consumer apps run checkModelsStrict() at startup; framework models must comply.
555
+ {strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
556
+ );
557
+
558
+ export const SyncKey: Model<SyncKeyDocument> =
559
+ (mongoose.models.SyncKey as Model<SyncKeyDocument>) ??
560
+ mongoose.model<SyncKeyDocument>("SyncKey", syncKeySchema);
561
+
562
+ /**
563
+ * Every model backing the SyncDB protocol. Their indexes are correctness-critical, not
564
+ * just performance: without the unique `SyncMutation.mutationId` index a duplicate
565
+ * delivery double-applies a mutation (INV-3), and without the unique `SyncCounter.stream`
566
+ * index the counter upsert race mints duplicate seqs. Schema-level `unique: true` only
567
+ * builds them when Mongoose `autoIndex` is on — commonly disabled in production — so
568
+ * startup builds them explicitly.
569
+ */
570
+ // noExplicitAny: a heterogeneous list of models is only used for ensureIndexes
571
+ // biome-ignore lint/suspicious/noExplicitAny: a heterogeneous list of models is only used for ensureIndexes
572
+ const SYNC_BOOKKEEPING_MODELS: Model<any>[] = [SyncCounter, SyncMutation, SyncScopeMove, SyncKey];
573
+
574
+ /**
575
+ * Best-effort removal of the pre-Task-9.15 `SyncScopeMove` TTL index. Failure is only
576
+ * logged: a leftover TTL index degrades retention accounting but must never block
577
+ * startup, and the index is absent on fresh databases (the common case).
578
+ */
579
+ const dropLegacyScopeMoveTtlIndex = async (): Promise<void> => {
580
+ try {
581
+ const indexes = await SyncScopeMove.collection.indexes();
582
+ const legacy = indexes.find(
583
+ (index) =>
584
+ index.name === LEGACY_SCOPE_MOVE_TTL_INDEX_NAME && index.expireAfterSeconds !== undefined
585
+ );
586
+ if (!legacy) {
587
+ return;
588
+ }
589
+ await SyncScopeMove.collection.dropIndex(LEGACY_SCOPE_MOVE_TTL_INDEX_NAME);
590
+ logger.info(
591
+ "[sync] Dropped the legacy SyncScopeMove TTL index; compactTombstones is now the " +
592
+ "sole reaper of scope-move markers (it honors per-model retentionDays)."
593
+ );
594
+ } catch (error: unknown) {
595
+ logger.warn("[sync] Could not drop the legacy SyncScopeMove TTL index", {
596
+ error: String(error),
597
+ });
598
+ }
599
+ };
600
+
601
+ /**
602
+ * Build the indexes for every sync bookkeeping model, throwing on the first failure so
603
+ * server startup fails loudly. Kicked off by `SyncApp.register` and awaited through
604
+ * `ensureSyncIndexes()`; idempotent, so calling it repeatedly is safe.
605
+ */
606
+ export const ensureSyncModelIndexes = async (): Promise<void> => {
607
+ await dropLegacyScopeMoveTtlIndex();
608
+ await Promise.all(
609
+ SYNC_BOOKKEEPING_MODELS.map(async (model) => {
610
+ try {
611
+ await model.ensureIndexes();
612
+ } catch (error: unknown) {
613
+ logger.error(`[sync] Failed to ensure sync indexes for ${model.modelName}`, {
614
+ error: String(error),
615
+ });
616
+ throw new Error(
617
+ `Failed to ensure sync indexes for ${model.modelName}: ${String(error)}. ` +
618
+ "The sync protocol depends on these indexes (unique mutationId for mutation " +
619
+ "idempotency, unique stream for seq allocation); fix the DB and restart."
620
+ );
621
+ }
622
+ })
623
+ );
624
+ };
625
+
626
+ /**
627
+ * Return the user's key material, generating it on first call. Race-safe: concurrent
628
+ * first calls converge on the single persisted value via `$setOnInsert` upsert — a
629
+ * caller must never receive bytes that were not persisted, or its encrypted store
630
+ * would be undecryptable by any other session.
631
+ */
632
+ export const getOrCreateSyncKeyMaterial = async ({userId}: {userId: string}): Promise<string> => {
633
+ const candidate = crypto.randomBytes(32).toString("base64");
634
+ const doc = await SyncKey.findOneAndUpdate(
635
+ {userId},
636
+ {$setOnInsert: {created: new Date(), keyMaterial: candidate, userId}},
637
+ {new: true, upsert: true}
638
+ );
639
+ return doc.keyMaterial;
640
+ };