@classytic/mongokit 3.43.1 → 3.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -3
- package/dist/{PaginationEngine-DZnkbSrr.mjs → PaginationEngine-BRlLTQQz.mjs} +13 -27
- package/dist/{PaginationEngine-CRHyuUtw.d.mts → PaginationEngine-BWb-c47a.d.mts} +1 -3
- package/dist/{Repository-DnL4xk5w.d.mts → Repository-DJuRmBNx.d.mts} +44 -13
- package/dist/{Repository-Dj6xqPxJ.mjs → Repository-DheA7oW-.mjs} +330 -112
- package/dist/actions/index.d.mts +1 -1
- package/dist/actions/index.mjs +2 -2
- package/dist/{actions-ChYfkPYP.mjs → actions-DKQx9ToJ.mjs} +3 -4
- package/dist/adapter/index.d.mts +6 -10
- package/dist/adapter/index.mjs +1 -1
- package/dist/{adapter-CkMoMF83.mjs → adapter-Qzgc3ZD_.mjs} +4 -6
- package/dist/ai/index.d.mts +21 -8
- package/dist/ai/index.mjs +15 -9
- package/dist/better-auth/index.d.mts +21 -20
- package/dist/better-auth/index.mjs +12 -23
- package/dist/{coercion-BjdPX3YF.d.mts → coercion-COp9DRNw.d.mts} +0 -2
- package/dist/{create-repository-UjwPWR1z.mjs → create-repository-C9DHHcl7.mjs} +4 -5
- package/dist/{create-repository-CJtVdwig.d.mts → create-repository-DG-gwOf0.d.mts} +5 -7
- package/dist/{derive-DRBiegRH.mjs → derive-Dxsc31fV.mjs} +3 -6
- package/dist/{error-BuByZwKU.mjs → error-DjdmPLWO.mjs} +1 -2
- package/dist/{field-selection-FSQFLpsU.mjs → field-selection-DPFaFw9Y.mjs} +1 -2
- package/dist/{id-chunks-BmAwsqNk.mjs → id-chunks-DH-RAZWG.mjs} +13 -8
- package/dist/{id-resolution-BqDrk0S2.mjs → id-resolution-Cqh49d9J.mjs} +1 -2
- package/dist/idempotency/index.d.mts +4 -7
- package/dist/idempotency/index.mjs +1 -2
- package/dist/{index-SYF8N28D.d.mts → index-5mhQ63_B.d.mts} +3 -4
- package/dist/{index-C0AjVqX3.d.mts → index-ehGZLbrd.d.mts} +2 -8
- package/dist/index.d.mts +70 -102
- package/dist/index.mjs +36 -52
- package/dist/indexes/verify-indexes.d.mts +8 -11
- package/dist/indexes/verify-indexes.mjs +1 -2
- package/dist/{indexes-DIjMpYES.d.mts → indexes-FfyOuhDK.d.mts} +0 -2
- package/dist/{join-scope-DSAznE8E.mjs → join-scope-C1gOtZ59.mjs} +2 -3
- package/dist/kernel-conformance/index.d.mts +39 -13
- package/dist/kernel-conformance/index.mjs +4 -7
- package/dist/lock/index.d.mts +134 -7
- package/dist/lock/index.mjs +349 -14
- package/dist/{logger-K7dPmu2U.mjs → logger-hbWOlj_X.mjs} +1 -2
- package/dist/{method-registry.plugin-C63pGn1d.mjs → method-registry.plugin-BeJfQBJV.mjs} +5 -7
- package/dist/{model-registry-ClKtWiHa.mjs → model-registry-DrVfJ3Nt.mjs} +1 -2
- package/dist/{mongooseToJsonSchema-C3k4ide0.mjs → mongooseToJsonSchema-DMDotz4_.mjs} +2 -3
- package/dist/{multi-tenant.plugin-B_q9NUHc.d.mts → multi-tenant.plugin-BTPtCLNK.d.mts} +143 -11
- package/dist/{operations-XiFgG5US.d.mts → operations-D6XsHx4e.d.mts} +19 -11
- package/dist/outbox/index.d.mts +24 -9
- package/dist/outbox/index.mjs +17 -6
- package/dist/pagination/PaginationEngine.d.mts +1 -1
- package/dist/pagination/PaginationEngine.mjs +1 -1
- package/dist/pagination/utils/limits.d.mts +6 -9
- package/dist/pagination/utils/limits.mjs +2 -3
- package/dist/{pagination-D6psses0.d.mts → pagination-BAM9cA4N.d.mts} +6 -16
- package/dist/{plugin-options-Cirmb4Kp.d.mts → plugin-options-DrNGWV5b.d.mts} +2 -4
- package/dist/plugins/index.d.mts +6 -7
- package/dist/plugins/index.mjs +6 -7
- package/dist/query/primitives/coercion.d.mts +1 -1
- package/dist/query/primitives/coercion.mjs +2 -3
- package/dist/query/primitives/geo.d.mts +11 -14
- package/dist/query/primitives/geo.mjs +1 -2
- package/dist/query/primitives/indexes.d.mts +1 -1
- package/dist/query/primitives/indexes.mjs +1 -2
- package/dist/{query-defaults-BxUcSOoF.mjs → query-defaults-DwvWUjZU.mjs} +18 -54
- package/dist/read-model/index.d.mts +18 -21
- package/dist/read-model/index.mjs +3 -4
- package/dist/{repository-MqAP409e.d.mts → repository-B0aFVgP2.d.mts} +25 -29
- package/dist/retention-purge-CCd_VXUf.d.mts +38 -0
- package/dist/{rolldown-runtime-D7D4PA-g.mjs → rolldown-runtime-D8jQpDG7.mjs} +1 -2
- package/dist/search/index.d.mts +1 -1
- package/dist/search/index.mjs +1 -1
- package/dist/sync/index.d.mts +113 -17
- package/dist/sync/index.mjs +421 -65
- package/dist/tenant/index.d.mts +8 -11
- package/dist/tenant/index.mjs +1 -2
- package/dist/testkit/index.d.mts +179 -23
- package/dist/testkit/index.mjs +421 -23
- package/dist/{timestamp.plugin-DpOWavVI.mjs → timestamp.plugin-B-IJrlYb.mjs} +192 -49
- package/dist/{to-plain-vksRrLtc.mjs → to-plain-CEKv_ylg.mjs} +2 -4
- package/dist/{to-plain-o5tUY2nV.d.mts → to-plain-Do-8jTKC.d.mts} +2 -16
- package/dist/{transaction-DmiB4k2P.mjs → transaction-C6Oj941P.mjs} +18 -19
- package/dist/{type-utils-CLzRFtIA.d.mts → type-utils-CKvuewAL.d.mts} +0 -2
- package/dist/{update-WgeAUJ_I.mjs → update-CYaJMu6j.mjs} +13 -23
- package/dist/usage/index.d.mts +2 -5
- package/dist/usage/index.mjs +8 -6
- package/dist/utils/index.d.mts +8 -10
- package/dist/utils/index.mjs +7 -7
- package/dist/{validation-chain.plugin-BAeC5eVT.d.mts → validation-chain.plugin-COYrUdd8.d.mts} +150 -48
- package/dist/{validation-chain.plugin-DRApTmkN.mjs → validation-chain.plugin-DJplVJ12.mjs} +210 -55
- package/llms.txt +33 -0
- package/package.json +29 -25
- package/skills/mongokit/SKILL.md +112 -0
- package/skills/mongokit/references/pagination-search.md +87 -0
- package/skills/mongokit/references/plugins-tenancy.md +103 -0
- package/skills/mongokit/references/repository-api.md +83 -0
- package/skills/mongokit/references/transactions-outbox.md +103 -0
package/dist/sync/index.mjs
CHANGED
|
@@ -1,8 +1,27 @@
|
|
|
1
|
-
import { o as withTransaction } from "../transaction-
|
|
1
|
+
import { o as withTransaction } from "../transaction-C6Oj941P.mjs";
|
|
2
2
|
import { Schema, Types } from "mongoose";
|
|
3
|
-
|
|
3
|
+
import { CursorExpiredError } from "@classytic/repo-core/sync";
|
|
4
|
+
// src/sync/index.ts
|
|
5
|
+
/**
|
|
6
|
+
* The mongo stores behind `@classytic/repo-core/sync` (replica set): the change feed
|
|
7
|
+
* (`createChangeLogStore`, filled by `changeLogPlugin`) and the command streams
|
|
8
|
+
* (`createCommandStreamStore`) a sync engine such as `@classytic/arc-sync` records verdicts in.
|
|
9
|
+
* Proven by `runChangeLogStoreConformance` and `runCommandStreamStoreConformance`.
|
|
10
|
+
*
|
|
11
|
+
* ```ts
|
|
12
|
+
* const changes = createChangeLogStore(createChangeLogModels(connection));
|
|
13
|
+
* const streams = createCommandStreamStore(connection, createCommandStreamModels(connection), changes);
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* Feed cursors are a gap-free sequence taken from ONE counter document inside the caller's
|
|
17
|
+
* transaction, so sequence order IS commit order: no reader can checkpoint past an entry that has
|
|
18
|
+
* not committed yet. The single counter caps append throughput — the price of never losing an entry.
|
|
19
|
+
*/
|
|
20
|
+
/** Default deferred-sequencer lease: a crashed holder delays numbering at most this long. */
|
|
21
|
+
const DEFAULT_SEQUENCER_LEASE_MS = 5e3;
|
|
4
22
|
function createChangeLogModels(connection, options = {}) {
|
|
5
|
-
const { collection = "sync_changes", modelName = "SyncChange", sequencing = "in-transaction" } = options;
|
|
23
|
+
const { collection = "sync_changes", modelName = "SyncChange", sequencing = "in-transaction", sequencerLeaseMs = DEFAULT_SEQUENCER_LEASE_MS } = options;
|
|
24
|
+
assertLeaseMs(sequencerLeaseMs);
|
|
6
25
|
const entriesName = modelName;
|
|
7
26
|
const counterName = `${modelName}Counter`;
|
|
8
27
|
const entries = connection.models[entriesName] ?? connection.model(entriesName, (() => {
|
|
@@ -60,6 +79,7 @@ function createChangeLogModels(connection, options = {}) {
|
|
|
60
79
|
docId: 1,
|
|
61
80
|
seq: 1
|
|
62
81
|
}, { name: "doc_versions" });
|
|
82
|
+
schema.index({ at: 1 }, { name: "feed_age" });
|
|
63
83
|
return schema;
|
|
64
84
|
})());
|
|
65
85
|
const counter = connection.models[counterName] ?? connection.model(counterName, new Schema({
|
|
@@ -73,11 +93,42 @@ function createChangeLogModels(connection, options = {}) {
|
|
|
73
93
|
},
|
|
74
94
|
/** The deferred sequencer's lease. */
|
|
75
95
|
lockUntil: { type: Date },
|
|
76
|
-
lockBy: { type: String }
|
|
96
|
+
lockBy: { type: String },
|
|
97
|
+
/** `prune`'s feed-wide horizon: the highest seq it deleted. An unfiltered cursor below it is expired. */
|
|
98
|
+
horizon: { type: Number }
|
|
77
99
|
}, {
|
|
78
100
|
collection: `${collection}_counter`,
|
|
79
101
|
versionKey: false
|
|
80
102
|
}));
|
|
103
|
+
const horizonName = `${modelName}Horizon`;
|
|
104
|
+
const horizons = connection.models[horizonName] ?? connection.model(horizonName, (() => {
|
|
105
|
+
const schema = new Schema({
|
|
106
|
+
tenantId: {
|
|
107
|
+
type: String,
|
|
108
|
+
default: null
|
|
109
|
+
},
|
|
110
|
+
scope: {
|
|
111
|
+
type: String,
|
|
112
|
+
required: true
|
|
113
|
+
},
|
|
114
|
+
horizon: {
|
|
115
|
+
type: Number,
|
|
116
|
+
required: true
|
|
117
|
+
}
|
|
118
|
+
}, {
|
|
119
|
+
collection: `${collection}_horizon`,
|
|
120
|
+
versionKey: false
|
|
121
|
+
});
|
|
122
|
+
schema.index({
|
|
123
|
+
tenantId: 1,
|
|
124
|
+
scope: 1
|
|
125
|
+
}, {
|
|
126
|
+
unique: true,
|
|
127
|
+
name: "tenant_scope_horizon"
|
|
128
|
+
});
|
|
129
|
+
schema.index({ scope: 1 }, { name: "scope_horizon" });
|
|
130
|
+
return schema;
|
|
131
|
+
})());
|
|
81
132
|
const pendingName = `${modelName}Pending`;
|
|
82
133
|
const pending = sequencing === "deferred" ? connection.models[pendingName] ?? connection.model(pendingName, new Schema({
|
|
83
134
|
scope: {
|
|
@@ -111,11 +162,16 @@ function createChangeLogModels(connection, options = {}) {
|
|
|
111
162
|
return {
|
|
112
163
|
entries,
|
|
113
164
|
counter,
|
|
165
|
+
horizons,
|
|
114
166
|
...pending ? { pending } : {},
|
|
115
167
|
sequencing,
|
|
168
|
+
sequencerLeaseMs,
|
|
116
169
|
feed: collection
|
|
117
170
|
};
|
|
118
171
|
}
|
|
172
|
+
function assertLeaseMs(ms) {
|
|
173
|
+
if (typeof ms !== "number" || !Number.isInteger(ms) || ms < 100) throw new Error("[mongokit:sync] sequencerLeaseMs must be an integer of at least 100");
|
|
174
|
+
}
|
|
119
175
|
/** A change as staged under deferred sequencing — the feed row without its `seq`. */
|
|
120
176
|
function stagedRow(entry, at) {
|
|
121
177
|
return {
|
|
@@ -137,36 +193,81 @@ const fromCursor = (cursor) => {
|
|
|
137
193
|
return seq;
|
|
138
194
|
};
|
|
139
195
|
const DEFAULT_LIMIT = 500;
|
|
140
|
-
/** How long one sequencer holds the lease — far above one batch, so a live holder never loses it mid-batch. */
|
|
141
|
-
const SEQUENCER_LEASE_MS = 3e4;
|
|
142
196
|
const SEQUENCER_BATCH = 500;
|
|
143
|
-
|
|
144
|
-
|
|
197
|
+
/** Thrown inside a batch transaction whose lease was taken over: the transaction aborts, nothing it wrote lands. */
|
|
198
|
+
var SequencerLeaseLost = class extends Error {};
|
|
199
|
+
function createChangeLogStore(models, storeOptions = {}) {
|
|
200
|
+
const { entries, counter, feed, pending, horizons } = models;
|
|
145
201
|
const deferred = models.sequencing === "deferred" && pending !== void 0;
|
|
202
|
+
const leaseMs = models.sequencerLeaseMs ?? 5e3;
|
|
203
|
+
assertLeaseMs(leaseMs);
|
|
204
|
+
const now = storeOptions.now ?? (() => /* @__PURE__ */ new Date());
|
|
205
|
+
const onSequenced = storeOptions.onSequenced;
|
|
206
|
+
const counterId = feed;
|
|
207
|
+
const report = (event) => {
|
|
208
|
+
try {
|
|
209
|
+
onSequenced?.(event);
|
|
210
|
+
} catch {}
|
|
211
|
+
};
|
|
212
|
+
/** The feed's numbered top (the counter row's `seq`; 0 before the first entry). */
|
|
213
|
+
const counterTop = async () => {
|
|
214
|
+
const row = await counter.collection.findOne({ _id: counterId }, { projection: { seq: 1 } });
|
|
215
|
+
return typeof row?.seq === "number" ? row.seq : 0;
|
|
216
|
+
};
|
|
217
|
+
/**
|
|
218
|
+
* After a pass found the lease busy (and a hook is set): retry in the background until the holder is done, then
|
|
219
|
+
* report what got numbered meanwhile. One follow-up per store at a time; bounded by three leases.
|
|
220
|
+
*/
|
|
221
|
+
let followUp = false;
|
|
222
|
+
const followBusyLease = async () => {
|
|
223
|
+
if (!onSequenced || followUp) return;
|
|
224
|
+
followUp = true;
|
|
225
|
+
try {
|
|
226
|
+
const seen = await counterTop();
|
|
227
|
+
const deadline = Date.now() + leaseMs * 3;
|
|
228
|
+
let delay = Math.max(10, Math.min(50, Math.floor(leaseMs / 10)));
|
|
229
|
+
for (;;) {
|
|
230
|
+
await new Promise((r) => {
|
|
231
|
+
setTimeout(r, delay).unref?.();
|
|
232
|
+
});
|
|
233
|
+
const result = await runPass();
|
|
234
|
+
if (result !== null) {
|
|
235
|
+
if (result === 0) {
|
|
236
|
+
const top = await counterTop();
|
|
237
|
+
if (top > seen) report({
|
|
238
|
+
cursor: toCursor(top),
|
|
239
|
+
numbered: top - seen,
|
|
240
|
+
by: "other"
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
if (Date.now() > deadline) return;
|
|
246
|
+
delay = Math.min(delay * 2, Math.max(50, Math.floor(leaseMs / 2)));
|
|
247
|
+
}
|
|
248
|
+
} catch {} finally {
|
|
249
|
+
followUp = false;
|
|
250
|
+
}
|
|
251
|
+
};
|
|
146
252
|
/**
|
|
147
253
|
* One pass under the lease: committed staged rows → the feed, in staging order, numbered above everything.
|
|
148
|
-
* Nothing staged costs ONE read.
|
|
149
|
-
*
|
|
254
|
+
* Nothing staged costs ONE read. The lease lives on the counter row and is judged on the database clock; each
|
|
255
|
+
* batch commits in ONE transaction (insert, counter + lease renewal gated on this pass's token, staged-row delete),
|
|
256
|
+
* so a holder that lost the lease to a takeover writes nothing, and a crash leaves no half batch. Without
|
|
257
|
+
* transactions (standalone) the batch runs unfenced, as before. A pre-existing half batch is still converged:
|
|
258
|
+
* an entry already in the feed is recognised by its unique `pendingId`, a stale counter by the unique `seq` index.
|
|
150
259
|
*/
|
|
151
|
-
const
|
|
260
|
+
const runPass = async () => {
|
|
152
261
|
if (!deferred || !pending) return 0;
|
|
153
262
|
if (!await pending.exists({}).exec()) return 0;
|
|
154
263
|
const token = new Types.ObjectId().toString();
|
|
155
|
-
const
|
|
156
|
-
_id:
|
|
157
|
-
$
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
};
|
|
163
|
-
const takeLease = () => counter.findOneAndUpdate(unlocked, { $set: {
|
|
164
|
-
lockUntil: new Date(Date.now() + SEQUENCER_LEASE_MS),
|
|
165
|
-
lockBy: token
|
|
166
|
-
} }, {
|
|
167
|
-
returnDocument: "after",
|
|
168
|
-
lean: true
|
|
169
|
-
}).exec();
|
|
264
|
+
const takeLease = async () => await counter.collection.findOneAndUpdate({
|
|
265
|
+
_id: counterId,
|
|
266
|
+
$expr: { $lt: ["$lockUntil", "$$NOW"] }
|
|
267
|
+
}, [{ $set: {
|
|
268
|
+
lockUntil: { $add: ["$$NOW", leaseMs] },
|
|
269
|
+
lockBy: { $literal: token }
|
|
270
|
+
} }], { returnDocument: "after" });
|
|
170
271
|
let lease = await takeLease();
|
|
171
272
|
if (!lease) {
|
|
172
273
|
if (await counter.exists({ _id: feed }).exec()) return null;
|
|
@@ -181,39 +282,53 @@ function createChangeLogStore(models) {
|
|
|
181
282
|
}
|
|
182
283
|
let seq = lease.seq ?? 0;
|
|
183
284
|
let numbered = 0;
|
|
285
|
+
const build = (from, list) => list.map((r, i) => {
|
|
286
|
+
const { _id, ...rest } = r;
|
|
287
|
+
return {
|
|
288
|
+
...rest,
|
|
289
|
+
seq: from + i + 1,
|
|
290
|
+
pendingId: String(_id)
|
|
291
|
+
};
|
|
292
|
+
});
|
|
293
|
+
/** Commit one batch atomically under the lease; `fresh` are the rows not yet in the feed. */
|
|
294
|
+
const commit = (rows, fresh) => withTransaction(entries.db, async (session) => {
|
|
295
|
+
if (fresh.length > 0) await entries.insertMany(build(seq, fresh), {
|
|
296
|
+
ordered: true,
|
|
297
|
+
session
|
|
298
|
+
});
|
|
299
|
+
if ((await counter.collection.updateOne({
|
|
300
|
+
_id: counterId,
|
|
301
|
+
lockBy: token
|
|
302
|
+
}, [{ $set: {
|
|
303
|
+
seq: { $max: ["$seq", seq + fresh.length] },
|
|
304
|
+
lockUntil: { $add: ["$$NOW", leaseMs] }
|
|
305
|
+
} }], { session })).matchedCount === 0) throw new SequencerLeaseLost();
|
|
306
|
+
await pending.deleteMany({ _id: { $in: rows.map((r) => r._id) } }, { session }).exec();
|
|
307
|
+
}, { allowFallback: true });
|
|
184
308
|
try {
|
|
185
309
|
for (;;) {
|
|
186
310
|
const rows = await pending.find({}).sort({ _id: 1 }).limit(SEQUENCER_BATCH).lean().exec();
|
|
187
311
|
if (rows.length === 0) break;
|
|
188
|
-
const build = (from, list) => list.map((r, i) => {
|
|
189
|
-
const { _id, ...rest } = r;
|
|
190
|
-
return {
|
|
191
|
-
...rest,
|
|
192
|
-
seq: from + i + 1,
|
|
193
|
-
pendingId: String(_id)
|
|
194
|
-
};
|
|
195
|
-
});
|
|
196
312
|
try {
|
|
197
|
-
await
|
|
313
|
+
await commit(rows, rows);
|
|
198
314
|
seq += rows.length;
|
|
199
315
|
numbered += rows.length;
|
|
200
316
|
} catch (err) {
|
|
317
|
+
if (err instanceof SequencerLeaseLost) break;
|
|
201
318
|
if (err.code !== 11e3) throw err;
|
|
202
319
|
const placed = new Set((await entries.find({ pendingId: { $in: rows.map((r) => String(r._id)) } }).select({ pendingId: 1 }).lean().exec()).map((e) => e.pendingId));
|
|
203
320
|
const fresh = rows.filter((r) => !placed.has(String(r._id)));
|
|
204
321
|
const top = await entries.findOne({}).sort({ seq: -1 }).select({ seq: 1 }).lean().exec();
|
|
205
322
|
seq = Math.max(seq, top?.seq ?? 0);
|
|
206
|
-
|
|
207
|
-
await
|
|
208
|
-
|
|
209
|
-
|
|
323
|
+
try {
|
|
324
|
+
await commit(rows, fresh);
|
|
325
|
+
} catch (again) {
|
|
326
|
+
if (again instanceof SequencerLeaseLost) break;
|
|
327
|
+
throw again;
|
|
210
328
|
}
|
|
329
|
+
seq += fresh.length;
|
|
330
|
+
numbered += fresh.length;
|
|
211
331
|
}
|
|
212
|
-
await counter.updateOne({
|
|
213
|
-
_id: feed,
|
|
214
|
-
lockBy: token
|
|
215
|
-
}, { $max: { seq } }).exec();
|
|
216
|
-
await pending.deleteMany({ _id: { $in: rows.map((r) => r._id) } }).exec();
|
|
217
332
|
if (rows.length < SEQUENCER_BATCH) break;
|
|
218
333
|
}
|
|
219
334
|
} finally {
|
|
@@ -225,18 +340,99 @@ function createChangeLogStore(models) {
|
|
|
225
340
|
lockBy: null
|
|
226
341
|
} }).exec();
|
|
227
342
|
}
|
|
343
|
+
if (numbered > 0) report({
|
|
344
|
+
cursor: toCursor(seq),
|
|
345
|
+
numbered,
|
|
346
|
+
by: "self"
|
|
347
|
+
});
|
|
228
348
|
return numbered;
|
|
229
349
|
};
|
|
350
|
+
/** One pass; a busy lease starts the background follow-up (only with `onSequenced`). */
|
|
351
|
+
const sequencePending = async () => {
|
|
352
|
+
const result = await runPass();
|
|
353
|
+
if (result === null) followBusyLease();
|
|
354
|
+
return result;
|
|
355
|
+
};
|
|
230
356
|
const sessionOf = (options) => options?.session ? { session: options.session } : {};
|
|
231
|
-
|
|
357
|
+
/** What a reader sees: its tenant (plus shared scopes), narrowed by `scopes`. Matches entries and horizon rows. */
|
|
358
|
+
const viewOf = (options = {}) => {
|
|
232
359
|
const { tenantId, scopes, sharedScopes } = options;
|
|
233
|
-
const tenant = tenantId === void 0 ? {} : sharedScopes?.length ? { $or: [{ tenantId }, { scope: { $in: [...sharedScopes] } }] } : { tenantId };
|
|
234
360
|
return {
|
|
235
|
-
|
|
236
|
-
...tenant,
|
|
361
|
+
...tenantId === void 0 ? {} : sharedScopes?.length ? { $or: [{ tenantId }, { scope: { $in: [...sharedScopes] } }] } : { tenantId },
|
|
237
362
|
...scopes ? { scope: { $in: [...scopes] } } : {}
|
|
238
363
|
};
|
|
239
364
|
};
|
|
365
|
+
const filterOf = (after, options = {}) => ({
|
|
366
|
+
seq: { $gt: after },
|
|
367
|
+
...viewOf(options)
|
|
368
|
+
});
|
|
369
|
+
/** The recorded feed-wide compaction horizon (0: never pruned). Raw driver: the counter row is schemaless here. */
|
|
370
|
+
const horizonSeq = async () => {
|
|
371
|
+
const row = await counter.collection.findOne({ _id: feed }, { projection: { horizon: 1 } });
|
|
372
|
+
return typeof row?.horizon === "number" ? row.horizon : 0;
|
|
373
|
+
};
|
|
374
|
+
/** The horizon a reader is held to: the highest one among the `(tenantId, scope)` rows it can see. */
|
|
375
|
+
const horizonFor = async (options = {}) => {
|
|
376
|
+
if (!horizons || options.tenantId === void 0 && !options.scopes) return horizonSeq();
|
|
377
|
+
const row = await horizons.collection.findOne(viewOf(options), {
|
|
378
|
+
sort: { horizon: -1 },
|
|
379
|
+
projection: { horizon: 1 }
|
|
380
|
+
});
|
|
381
|
+
return typeof row?.horizon === "number" ? row.horizon : 0;
|
|
382
|
+
};
|
|
383
|
+
/**
|
|
384
|
+
* One document's identity: tenants sharing a docId in one scope are different documents. A missing
|
|
385
|
+
* tenantId groups (and, as `null`, matches) as untenanted.
|
|
386
|
+
*/
|
|
387
|
+
const docKey = {
|
|
388
|
+
tenantId: { $ifNull: ["$tenantId", null] },
|
|
389
|
+
scope: "$scope",
|
|
390
|
+
docId: "$docId"
|
|
391
|
+
};
|
|
392
|
+
const docOf = (id) => ({
|
|
393
|
+
tenantId: id.tenantId ?? null,
|
|
394
|
+
scope: id.scope,
|
|
395
|
+
docId: id.docId
|
|
396
|
+
});
|
|
397
|
+
/**
|
|
398
|
+
* The entry a reader of one document ends on, which retention keeps. A tombstone applies
|
|
399
|
+
* unconditionally (`changeLogPlugin` writes version 0); an upsert only above the version held. So:
|
|
400
|
+
* the last tombstone, unless upserts follow it, then the highest of those (the first on a tie).
|
|
401
|
+
* Not simply the last cursor: branch stock records a lower version after a higher one.
|
|
402
|
+
*/
|
|
403
|
+
const winnerOf = (rows) => {
|
|
404
|
+
let held = null;
|
|
405
|
+
for (const row of [...rows].sort((a, b) => a.seq - b.seq)) if (row.op === "delete") held = {
|
|
406
|
+
seq: row.seq,
|
|
407
|
+
version: Number.NEGATIVE_INFINITY
|
|
408
|
+
};
|
|
409
|
+
else if (!held || row.version > held.version) held = {
|
|
410
|
+
seq: row.seq,
|
|
411
|
+
version: row.version
|
|
412
|
+
};
|
|
413
|
+
return held?.seq ?? 0;
|
|
414
|
+
};
|
|
415
|
+
const winnerSeq = async (filter) => {
|
|
416
|
+
const tomb = await entries.findOne({
|
|
417
|
+
...filter,
|
|
418
|
+
op: "delete"
|
|
419
|
+
}).sort({ seq: -1 }).select({ seq: 1 }).lean().exec();
|
|
420
|
+
const bound = filter.seq ?? {};
|
|
421
|
+
return (await entries.findOne(tomb?.seq === void 0 ? {
|
|
422
|
+
...filter,
|
|
423
|
+
op: "upsert"
|
|
424
|
+
} : {
|
|
425
|
+
...filter,
|
|
426
|
+
op: "upsert",
|
|
427
|
+
seq: {
|
|
428
|
+
...bound,
|
|
429
|
+
$gt: tomb.seq
|
|
430
|
+
}
|
|
431
|
+
}).sort({
|
|
432
|
+
version: -1,
|
|
433
|
+
seq: 1
|
|
434
|
+
}).select({ seq: 1 }).lean().exec())?.seq ?? tomb?.seq ?? 0;
|
|
435
|
+
};
|
|
240
436
|
const toEntry = (row) => ({
|
|
241
437
|
scope: row.scope,
|
|
242
438
|
docId: row.docId,
|
|
@@ -254,12 +450,12 @@ function createChangeLogStore(models) {
|
|
|
254
450
|
await this.append(entry, options);
|
|
255
451
|
return;
|
|
256
452
|
}
|
|
257
|
-
await pending.create([stagedRow(entry,
|
|
453
|
+
await pending.create([stagedRow(entry, now())], sessionOf(options));
|
|
258
454
|
},
|
|
259
455
|
async append(entry, options) {
|
|
260
456
|
const session = sessionOf(options);
|
|
261
457
|
if (deferred && pending) {
|
|
262
|
-
const at =
|
|
458
|
+
const at = now();
|
|
263
459
|
const [staged] = await pending.create([stagedRow(entry, at)], session);
|
|
264
460
|
if (options?.session) return {
|
|
265
461
|
...entry,
|
|
@@ -267,7 +463,8 @@ function createChangeLogStore(models) {
|
|
|
267
463
|
cursor: ""
|
|
268
464
|
};
|
|
269
465
|
const stagedId = String(staged?._id);
|
|
270
|
-
|
|
466
|
+
const deadline = Date.now() + Math.max(1e3, leaseMs + 1e3);
|
|
467
|
+
for (let attempt = 0; attempt < 50 || Date.now() < deadline; attempt++) {
|
|
271
468
|
await sequencePending();
|
|
272
469
|
const at2 = await entries.findOne({ pendingId: stagedId }).select({ seq: 1 }).lean().exec();
|
|
273
470
|
if (typeof at2?.seq === "number") return {
|
|
@@ -285,7 +482,7 @@ function createChangeLogStore(models) {
|
|
|
285
482
|
lean: true,
|
|
286
483
|
...session
|
|
287
484
|
}).exec()).seq;
|
|
288
|
-
const at =
|
|
485
|
+
const at = now();
|
|
289
486
|
await entries.create([{
|
|
290
487
|
seq,
|
|
291
488
|
scope: entry.scope,
|
|
@@ -305,7 +502,10 @@ function createChangeLogStore(models) {
|
|
|
305
502
|
async since(cursor, options = {}) {
|
|
306
503
|
if (deferred) await sequencePending();
|
|
307
504
|
const limit = options.limit ?? DEFAULT_LIMIT;
|
|
308
|
-
const
|
|
505
|
+
const after = fromCursor(cursor);
|
|
506
|
+
const rows = await entries.find(filterOf(after, options)).sort({ seq: 1 }).limit(limit + 1).lean().exec();
|
|
507
|
+
const horizon = await horizonFor(options);
|
|
508
|
+
if (horizon > 0 && after < horizon) throw new CursorExpiredError(cursor, toCursor(horizon));
|
|
309
509
|
const page = rows.slice(0, limit).map(toEntry);
|
|
310
510
|
const last = page[page.length - 1];
|
|
311
511
|
return {
|
|
@@ -314,24 +514,42 @@ function createChangeLogStore(models) {
|
|
|
314
514
|
hasMore: rows.length > limit
|
|
315
515
|
};
|
|
316
516
|
},
|
|
517
|
+
async latestOf(scope, docId, options = {}) {
|
|
518
|
+
const filter = {
|
|
519
|
+
scope,
|
|
520
|
+
docId,
|
|
521
|
+
tenantId: options.tenantId ?? null
|
|
522
|
+
};
|
|
523
|
+
const session = options.session ? { session: options.session } : {};
|
|
524
|
+
if (deferred && pending) {
|
|
525
|
+
const staged = await pending.findOne(filter, null, session).sort({ _id: -1 }).lean().exec();
|
|
526
|
+
if (staged) return {
|
|
527
|
+
...toEntry({
|
|
528
|
+
...staged,
|
|
529
|
+
seq: 0
|
|
530
|
+
}),
|
|
531
|
+
cursor: ""
|
|
532
|
+
};
|
|
533
|
+
}
|
|
534
|
+
const row = await entries.findOne(filter, null, session).sort({ seq: -1 }).lean().exec();
|
|
535
|
+
return row ? toEntry(row) : null;
|
|
536
|
+
},
|
|
317
537
|
async latestCursor(options = {}) {
|
|
318
538
|
if (deferred) await sequencePending();
|
|
319
539
|
const row = await entries.findOne(filterOf(0, options)).sort({ seq: -1 }).select({ seq: 1 }).lean().exec();
|
|
320
|
-
|
|
540
|
+
const seq = Math.max(row?.seq ?? 0, await horizonFor(options));
|
|
541
|
+
return seq > 0 ? toCursor(seq) : "";
|
|
321
542
|
},
|
|
322
543
|
async compactSuperseded(options = {}) {
|
|
323
544
|
const batchSize = options.batchSize ?? 1e3;
|
|
324
545
|
const groups = entries.aggregate([
|
|
325
546
|
{ $group: {
|
|
326
|
-
_id:
|
|
327
|
-
|
|
328
|
-
docId: "$docId"
|
|
329
|
-
},
|
|
330
|
-
latest: { $max: "$seq" },
|
|
547
|
+
_id: docKey,
|
|
548
|
+
top: { $max: "$seq" },
|
|
331
549
|
n: { $sum: 1 }
|
|
332
550
|
} },
|
|
333
551
|
{ $match: { n: { $gt: 1 } } },
|
|
334
|
-
{ $project: {
|
|
552
|
+
{ $project: { top: 1 } }
|
|
335
553
|
]).allowDiskUse(true).cursor({ batchSize });
|
|
336
554
|
let removed = 0;
|
|
337
555
|
let ops = [];
|
|
@@ -341,15 +559,154 @@ function createChangeLogStore(models) {
|
|
|
341
559
|
ops = [];
|
|
342
560
|
};
|
|
343
561
|
for await (const g of groups) {
|
|
562
|
+
const doc = docOf(g._id);
|
|
563
|
+
const keep = await winnerSeq({
|
|
564
|
+
...doc,
|
|
565
|
+
seq: { $lte: g.top }
|
|
566
|
+
});
|
|
344
567
|
ops.push({ deleteMany: { filter: {
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
568
|
+
...doc,
|
|
569
|
+
seq: {
|
|
570
|
+
$lte: g.top,
|
|
571
|
+
$ne: keep
|
|
572
|
+
}
|
|
348
573
|
} } });
|
|
349
574
|
if (ops.length >= batchSize) await flush();
|
|
350
575
|
}
|
|
351
576
|
await flush();
|
|
352
577
|
return { removed };
|
|
578
|
+
},
|
|
579
|
+
async horizon(options) {
|
|
580
|
+
const seq = await horizonFor(options);
|
|
581
|
+
return seq > 0 ? toCursor(seq) : "";
|
|
582
|
+
},
|
|
583
|
+
async snapshot(cursor, options = {}) {
|
|
584
|
+
const { upTo: bound, ...view } = options;
|
|
585
|
+
const limit = view.limit ?? DEFAULT_LIMIT;
|
|
586
|
+
const upTo = bound === void 0 ? await horizonFor(view) : fromCursor(bound);
|
|
587
|
+
const after = fromCursor(cursor);
|
|
588
|
+
const upToCursor = upTo > 0 ? toCursor(upTo) : "";
|
|
589
|
+
if (after >= upTo) return {
|
|
590
|
+
changes: [],
|
|
591
|
+
cursor,
|
|
592
|
+
hasMore: false,
|
|
593
|
+
upTo: upToCursor
|
|
594
|
+
};
|
|
595
|
+
const rows = await entries.find({
|
|
596
|
+
seq: {
|
|
597
|
+
$gt: after,
|
|
598
|
+
$lte: upTo
|
|
599
|
+
},
|
|
600
|
+
...viewOf(view)
|
|
601
|
+
}).sort({ seq: 1 }).limit(limit + 1).lean().exec();
|
|
602
|
+
const page = rows.slice(0, limit);
|
|
603
|
+
const latest = /* @__PURE__ */ new Map();
|
|
604
|
+
if (page.length > 0) {
|
|
605
|
+
const keyOf = (r) => JSON.stringify([
|
|
606
|
+
typeof r.tenantId === "string" ? r.tenantId : null,
|
|
607
|
+
r.scope,
|
|
608
|
+
r.docId
|
|
609
|
+
]);
|
|
610
|
+
const docs = new Map(page.map((r) => [keyOf(r), r]));
|
|
611
|
+
const groups = await entries.aggregate([{ $match: {
|
|
612
|
+
seq: { $lte: upTo },
|
|
613
|
+
$or: [...docs.values()].map((r) => docOf({
|
|
614
|
+
tenantId: typeof r.tenantId === "string" ? r.tenantId : null,
|
|
615
|
+
scope: r.scope,
|
|
616
|
+
docId: r.docId
|
|
617
|
+
}))
|
|
618
|
+
} }, { $group: {
|
|
619
|
+
_id: docKey,
|
|
620
|
+
rows: { $push: {
|
|
621
|
+
seq: "$seq",
|
|
622
|
+
version: "$version",
|
|
623
|
+
op: "$op"
|
|
624
|
+
} }
|
|
625
|
+
} }]).allowDiskUse(true).exec();
|
|
626
|
+
for (const g of groups) latest.set(keyOf(g._id), winnerOf(g.rows));
|
|
627
|
+
const changes = page.filter((r) => latest.get(keyOf(r)) === r.seq && r.op === "upsert").map(toEntry);
|
|
628
|
+
const last = page[page.length - 1];
|
|
629
|
+
return {
|
|
630
|
+
changes,
|
|
631
|
+
cursor: toCursor(last.seq),
|
|
632
|
+
hasMore: rows.length > limit,
|
|
633
|
+
upTo: upToCursor
|
|
634
|
+
};
|
|
635
|
+
}
|
|
636
|
+
return {
|
|
637
|
+
changes: [],
|
|
638
|
+
cursor,
|
|
639
|
+
hasMore: false,
|
|
640
|
+
upTo: upToCursor
|
|
641
|
+
};
|
|
642
|
+
},
|
|
643
|
+
async prune(before, options = {}) {
|
|
644
|
+
if (!(before instanceof Date) || Number.isNaN(before.getTime())) throw new Error("[mongokit:sync] prune(before) needs a valid Date");
|
|
645
|
+
const batchSize = options.batchSize ?? 1e3;
|
|
646
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) throw new Error("[mongokit:sync] prune batchSize must be a positive integer");
|
|
647
|
+
if (deferred) await sequencePending();
|
|
648
|
+
const old = { $lt: before };
|
|
649
|
+
const groups = entries.aggregate([{ $match: { at: old } }, { $group: {
|
|
650
|
+
_id: docKey,
|
|
651
|
+
oldMax: { $max: "$seq" },
|
|
652
|
+
n: { $sum: 1 }
|
|
653
|
+
} }]).allowDiskUse(true).cursor({ batchSize });
|
|
654
|
+
const topSeq = async (filter) => {
|
|
655
|
+
return (await entries.findOne(filter).sort({ seq: -1 }).select({ seq: 1 }).lean().exec())?.seq ?? 0;
|
|
656
|
+
};
|
|
657
|
+
let ops = [];
|
|
658
|
+
let batchMax = 0;
|
|
659
|
+
/** Highest seq this batch deletes per `(tenantId, scope)` — the rows a filtered reader is held to. */
|
|
660
|
+
let viewMax = /* @__PURE__ */ new Map();
|
|
661
|
+
const flush = async () => {
|
|
662
|
+
if (ops.length === 0) return;
|
|
663
|
+
if (horizons) await horizons.collection.bulkWrite([...viewMax.values()].map(({ tenantId, scope, seq }) => ({ updateOne: {
|
|
664
|
+
filter: {
|
|
665
|
+
tenantId,
|
|
666
|
+
scope
|
|
667
|
+
},
|
|
668
|
+
update: { $max: { horizon: seq } },
|
|
669
|
+
upsert: true
|
|
670
|
+
} })), { ordered: false });
|
|
671
|
+
await counter.collection.updateOne({ _id: feed }, { $max: { horizon: batchMax } }, { upsert: true });
|
|
672
|
+
await entries.bulkWrite(ops, { ordered: false });
|
|
673
|
+
ops = [];
|
|
674
|
+
batchMax = 0;
|
|
675
|
+
viewMax = /* @__PURE__ */ new Map();
|
|
676
|
+
};
|
|
677
|
+
for await (const g of groups) {
|
|
678
|
+
const doc = docOf(g._id);
|
|
679
|
+
const keep = await winnerSeq(doc);
|
|
680
|
+
const deletedMax = await topSeq({
|
|
681
|
+
...doc,
|
|
682
|
+
at: old,
|
|
683
|
+
seq: {
|
|
684
|
+
$lte: g.oldMax,
|
|
685
|
+
$ne: keep
|
|
686
|
+
}
|
|
687
|
+
});
|
|
688
|
+
if (deletedMax <= 0) continue;
|
|
689
|
+
ops.push({ deleteMany: { filter: {
|
|
690
|
+
...doc,
|
|
691
|
+
at: old,
|
|
692
|
+
seq: {
|
|
693
|
+
$lte: deletedMax,
|
|
694
|
+
$ne: keep
|
|
695
|
+
}
|
|
696
|
+
} } });
|
|
697
|
+
batchMax = Math.max(batchMax, deletedMax);
|
|
698
|
+
const key = JSON.stringify([doc.tenantId, doc.scope]);
|
|
699
|
+
const seen = viewMax.get(key);
|
|
700
|
+
if (!seen || seen.seq < deletedMax) viewMax.set(key, {
|
|
701
|
+
tenantId: doc.tenantId,
|
|
702
|
+
scope: doc.scope,
|
|
703
|
+
seq: deletedMax
|
|
704
|
+
});
|
|
705
|
+
if (ops.length >= batchSize) await flush();
|
|
706
|
+
}
|
|
707
|
+
await flush();
|
|
708
|
+
const horizon = await horizonSeq();
|
|
709
|
+
return horizon > 0 ? toCursor(horizon) : "";
|
|
353
710
|
}
|
|
354
711
|
};
|
|
355
712
|
}
|
|
@@ -542,5 +899,4 @@ function createCommandStreamStore(connection, models, changes) {
|
|
|
542
899
|
}
|
|
543
900
|
};
|
|
544
901
|
}
|
|
545
|
-
|
|
546
|
-
export { createChangeLogModels, createChangeLogStore, createCommandStreamModels, createCommandStreamStore };
|
|
902
|
+
export { DEFAULT_SEQUENCER_LEASE_MS, createChangeLogModels, createChangeLogStore, createCommandStreamModels, createCommandStreamStore };
|