@classytic/mongokit 3.41.0 → 3.42.0

package/dist/plugins/index.mjs

@@ -7,6 +7,8 @@ const PLUGIN_NAME = "changeLog";
 const CAPTURED = Symbol.for("mongokit.changeLog.capture");
 /** Where `before:updateMany` / `before:deleteMany` leave the ids they matched for the after hook. */
 const MATCHED_IDS = Symbol("changeLog.matchedIds");
+/** Ids per `$in` when a many-document write is read back. */
+const READ_BACK_CHUNK = 5e3;
 function isSkipped(context) {
 	const list = context.skipPlugins;
 	return Array.isArray(list) && list.includes(PLUGIN_NAME);
@@ -29,29 +31,34 @@ function changeLogPlugin(options) {
 		return raw === void 0 || raw === null ? void 0 : String(raw);
 	};
 	const appendOptions = (context) => context.session !== void 0 ? { session: context.session } : void 0;
+	/** `stage` when the store offers it (no cursor wait under deferred sequencing); the contract's `append` otherwise. */
+	const recordChange = (entry, context) => {
+		const staging = store.stage;
+		return staging ? staging.call(store, entry, appendOptions(context)) : store.append(entry, appendOptions(context));
+	};
 	const upsert = async (doc, context) => {
 		const record = toRecord(doc);
 		if (!record || record._id === void 0) return;
 		const tenantId = tenantOf(record, context);
-		await store.append({
+		await recordChange({
 			scope,
 			docId: String(record._id),
 			op: "upsert",
 			version: versionOf(record),
 			doc: project ? project(record) : record,
 			...tenantId !== void 0 ? { tenantId } : {}
-		}, appendOptions(context));
+		}, context);
 	};
 	const tombstone = async (docId, context) => {
 		if (docId === void 0 || docId === null) return;
 		const tenantId = tenantOf(null, context);
-		await store.append({
+		await recordChange({
 			scope,
 			docId: String(docId),
 			op: "delete",
 			version: 0,
 			...tenantId !== void 0 ? { tenantId } : {}
-		}, appendOptions(context));
+		}, context);
 	};
 	return {
 		name: PLUGIN_NAME,
@@ -72,6 +79,22 @@ function changeLogPlugin(options) {
 				if (context.session !== void 0) query.session(context.session);
 				return query.exec();
 			};
+			/** A write's result is the feed row only when it is the whole post-image, as a plain object. */
+			const isWholePostImage = (result, context) => {
+				if (!result || typeof result !== "object") return false;
+				if (typeof result.toObject === "function") return false;
+				const c = context;
+				return !c.select && !c.projection && !c.populate && c.returnDocument !== "before";
+			};
+			/** `upsert` from the result the write returned; a narrowed result is re-read by id (one read, that verb only). */
+			const captureResult = async (result, context) => {
+				const id = toRecord(result)?._id;
+				if (id === void 0) return;
+				if (isWholePostImage(result, context)) return upsert(result, context);
+				const doc = await reread(id, context);
+				if (doc) await upsert(doc, context);
+				else await tombstone(id, context);
+			};
 			const matchingIds = async (context, filter) => {
 				const query = repo.Model.find(filter ?? {}).select({ _id: 1 }).lean();
 				if (context.session !== void 0) query.session(context.session);
@@ -85,29 +108,37 @@ function changeLogPlugin(options) {
 				for (const doc of result) await upsert(doc, context);
 			});
 			repo.on("after:update", async ({ context, result }) => {
-				if (!isSkipped(context) && result) await upsert(result, context);
+				if (isSkipped(context) || !result) return;
+				const c = context;
+				if (c.select || c.projection || c.populate) await captureResult(result, context);
+				else await upsert(result, context);
+			});
+			repo.on("after:getOrCreate", async ({ context, result }) => {
+				const doc = result?.doc;
+				if (!isSkipped(context) && doc) await upsert(doc, context);
 			});
 			for (const op of [
 				"claim",
 				"claimVersion",
 				"findOneAndUpdate",
-				"getOrCreate",
 				"restore"
 			]) repo.on(`after:${op}`, async ({ context, result }) => {
-				if (isSkipped(context)) return;
-				const id = toRecord(result)?._id;
-				if (id === void 0) return;
-				const doc = await reread(id, context);
-				if (doc) await upsert(doc, context);
-				else await tombstone(id, context);
+				if (!isSkipped(context)) await captureResult(result, context);
 			});
 			for (const op of ["updateMany", "deleteMany"]) repo.on(`before:${op}`, async (context) => {
 				if (!isSkipped(context)) context[MATCHED_IDS] = await matchingIds(context, context.query);
 			});
-			/** Each re-read document as an upsert, or a tombstone where it is gone. */
+			/** Each target as an upsert, or a tombstone where it is gone — read back in batches, not one query per id. */
 			const capture = async (ids, context) => {
-				for (const id of ids) {
-					const doc = await reread(id, context);
+				const wanted = ids.filter((id) => id !== void 0 && id !== null);
+				const found = /* @__PURE__ */ new Map();
+				for (let i = 0; i < wanted.length; i += READ_BACK_CHUNK) {
+					const query = repo.Model.find({ _id: { $in: wanted.slice(i, i + READ_BACK_CHUNK) } }).lean();
+					if (context.session !== void 0) query.session(context.session);
+					for (const doc of await query.exec()) found.set(String(doc._id), doc);
+				}
+				for (const id of wanted) {
+					const doc = found.get(String(id));
 					if (doc) await upsert(doc, context);
 					else await tombstone(id, context);
 				}

package/dist/sync/index.d.mts

@@ -1,15 +1,26 @@
 import { ClientSession, Connection, Model } from "mongoose";
-import { ChangeLogStore, CommandStreamStore, CommandStreamTx } from "@classytic/repo-core/sync";
+import { ChangeLogAppendOptions, ChangeLogStore, CommandStreamStore, CommandStreamTx } from "@classytic/repo-core/sync";
 //#region src/sync/index.d.ts
 interface ChangeLogModelOptions {
   /** Entries collection. Default `sync_changes`; its counter lives in `<collection>_counter`. */
   collection?: string;
   /** Mongoose model name prefix. Default `SyncChange`. Must be unique per connection. */
   modelName?: string;
+  /**
+   * When an entry gets its cursor. `'in-transaction'` (default): from the counter, inside the writer's transaction —
+   * every writer serialises on that one document and overlapping transactions abort and retry. `'deferred'`: the
+   * writer stages the entry (`<collection>_pending`, no counter); a single leased sequencer numbers COMMITTED
+   * entries before every read. A cursor is only ever given to a committed entry and is above every cursor given
+   * before, so no reader can checkpoint past an entry that appears later.
+   */
+  sequencing?: 'in-transaction' | 'deferred';
 }
 interface ChangeLogModels {
   entries: Model<Record<string, unknown>>;
   counter: Model<Record<string, unknown>>;
+  /** Staged, not-yet-numbered entries — present only under `sequencing: 'deferred'`. */
+  pending?: Model<Record<string, unknown>>;
+  sequencing: 'in-transaction' | 'deferred';
   /** The counter document's id (one feed per entries collection). */
   feed: string;
 }
@@ -26,6 +37,16 @@ interface MongoChangeLogStore extends ChangeLogStore {
   }): Promise<{
     removed: number;
   }>;
+  /**
+   * Number every staged entry (deferred sequencing; a no-op otherwise). Reads call it first; a scheduler may too.
+   * Returns how many it numbered, or `null` when another sequencer holds the lease.
+   */
+  sequencePending(): Promise<number | null>;
+  /**
+   * Record a change WITHOUT waiting for its cursor (deferred sequencing): the next read numbers it. For writers that
+   * never use the cursor (`changeLogPlugin`). Under in-transaction sequencing it is `append`.
+   */
+  stage(entry: Parameters<ChangeLogStore['append']>[0], options?: ChangeLogAppendOptions): Promise<void>;
 }
 declare function createChangeLogStore(models: ChangeLogModels): MongoChangeLogStore;
 interface CommandStreamModelOptions {

package/dist/sync/index.mjs

@@ -1,77 +1,133 @@
 import { o as withTransaction } from "../transaction-DmiB4k2P.mjs";
-import { Schema } from "mongoose";
+import { Schema, Types } from "mongoose";
 //#region src/sync/index.ts
 function createChangeLogModels(connection, options = {}) {
-	const { collection = "sync_changes", modelName = "SyncChange" } = options;
+	const { collection = "sync_changes", modelName = "SyncChange", sequencing = "in-transaction" } = options;
 	const entriesName = modelName;
 	const counterName = `${modelName}Counter`;
-	return {
-		entries: connection.models[entriesName] ?? connection.model(entriesName, (() => {
-			const schema = new Schema({
-				seq: {
-					type: Number,
-					required: true
-				},
-				scope: {
-					type: String,
-					required: true
-				},
-				docId: {
-					type: String,
-					required: true
-				},
-				op: {
-					type: String,
-					enum: ["upsert", "delete"],
-					required: true
-				},
-				version: {
-					type: Number,
-					required: true
-				},
-				doc: { type: Schema.Types.Mixed },
-				tenantId: { type: String },
-				at: {
-					type: Date,
-					required: true
-				}
-			}, {
-				collection,
-				versionKey: false,
-				minimize: false
-			});
-			schema.index({ seq: 1 }, {
-				unique: true,
-				name: "feed_order"
-			});
-			schema.index({
-				tenantId: 1,
-				scope: 1,
-				seq: 1
-			}, { name: "tenant_scope_feed" });
-			schema.index({
-				scope: 1,
-				docId: 1,
-				seq: 1
-			}, { name: "doc_versions" });
-			return schema;
-		})()),
-		counter: connection.models[counterName] ?? connection.model(counterName, new Schema({
-			_id: {
+	const entries = connection.models[entriesName] ?? connection.model(entriesName, (() => {
+		const schema = new Schema({
+			seq: {
+				type: Number,
+				required: true
+			},
+			scope: {
 				type: String,
 				required: true
 			},
-			seq: {
+			docId: {
+				type: String,
+				required: true
+			},
+			op: {
+				type: String,
+				enum: ["upsert", "delete"],
+				required: true
+			},
+			version: {
 				type: Number,
 				required: true
-			}
+			},
+			doc: { type: Schema.Types.Mixed },
+			tenantId: { type: String },
+			at: {
+				type: Date,
+				required: true
+			},
+			/** The staged row this entry was numbered from (deferred sequencing) — the sequencer's idempotency key. */
+			pendingId: { type: String }
 		}, {
-			collection: `${collection}_counter`,
-			versionKey: false
-		})),
+			collection,
+			versionKey: false,
+			minimize: false
+		});
+		schema.index({ seq: 1 }, {
+			unique: true,
+			name: "feed_order"
+		});
+		schema.index({ pendingId: 1 }, {
+			unique: true,
+			name: "sequenced_from",
+			partialFilterExpression: { pendingId: { $gte: "" } }
+		});
+		schema.index({
+			tenantId: 1,
+			scope: 1,
+			seq: 1
+		}, { name: "tenant_scope_feed" });
+		schema.index({
+			scope: 1,
+			docId: 1,
+			seq: 1
+		}, { name: "doc_versions" });
+		return schema;
+	})());
+	const counter = connection.models[counterName] ?? connection.model(counterName, new Schema({
+		_id: {
+			type: String,
+			required: true
+		},
+		seq: {
+			type: Number,
+			required: true
+		},
+		/** The deferred sequencer's lease. */
+		lockUntil: { type: Date },
+		lockBy: { type: String }
+	}, {
+		collection: `${collection}_counter`,
+		versionKey: false
+	}));
+	const pendingName = `${modelName}Pending`;
+	const pending = sequencing === "deferred" ? connection.models[pendingName] ?? connection.model(pendingName, new Schema({
+		scope: {
+			type: String,
+			required: true
+		},
+		docId: {
+			type: String,
+			required: true
+		},
+		op: {
+			type: String,
+			enum: ["upsert", "delete"],
+			required: true
+		},
+		version: {
+			type: Number,
+			required: true
+		},
+		doc: { type: Schema.Types.Mixed },
+		tenantId: { type: String },
+		at: {
+			type: Date,
+			required: true
+		}
+	}, {
+		collection: `${collection}_pending`,
+		versionKey: false,
+		minimize: false
+	})) : void 0;
+	return {
+		entries,
+		counter,
+		...pending ? { pending } : {},
+		sequencing,
 		feed: collection
 	};
 }
+/** A change as staged under deferred sequencing — the feed row without its `seq`. */
+function stagedRow(entry, at) {
+	return {
+		scope: entry.scope,
+		docId: entry.docId,
+		op: entry.op,
+		version: entry.version,
+		...entry.op === "upsert" && entry.doc !== void 0 ? { doc: entry.doc } : {},
+		...entry.tenantId !== void 0 ? { tenantId: entry.tenantId } : {},
+		at
+	};
+}
 /** Fixed-width, so the opaque cursor also sorts as text. */
 const toCursor = (seq) => String(seq).padStart(16, "0");
 const fromCursor = (cursor) => {
@@ -81,8 +137,96 @@ const fromCursor = (cursor) => {
 	return seq;
 };
 const DEFAULT_LIMIT = 500;
+/** How long one sequencer holds the lease — far above one batch, so a live holder never loses it mid-batch. */
+const SEQUENCER_LEASE_MS = 3e4;
+const SEQUENCER_BATCH = 500;
 function createChangeLogStore(models) {
-	const { entries, counter, feed } = models;
+	const { entries, counter, feed, pending } = models;
+	const deferred = models.sequencing === "deferred" && pending !== void 0;
+	/**
+	* One pass under the lease: committed staged rows → the feed, in staging order, numbered above everything.
+	* Nothing staged costs ONE read. A crash mid-pass is converged by the next one: an entry already in the feed is
+	* recognised by its unique `pendingId`, and a stale counter is caught by the unique `seq` index.
+	*/
+	const sequencePending = async () => {
+		if (!deferred || !pending) return 0;
+		if (!await pending.exists({}).exec()) return 0;
+		const token = new Types.ObjectId().toString();
+		const unlocked = {
+			_id: feed,
+			$or: [
+				{ lockUntil: { $exists: false } },
+				{ lockUntil: null },
+				{ lockUntil: { $lt: /* @__PURE__ */ new Date() } }
+			]
+		};
+		const takeLease = () => counter.findOneAndUpdate(unlocked, { $set: {
+			lockUntil: new Date(Date.now() + SEQUENCER_LEASE_MS),
+			lockBy: token
+		} }, {
+			returnDocument: "after",
+			lean: true
+		}).exec();
+		let lease = await takeLease();
+		if (!lease) {
+			if (await counter.exists({ _id: feed }).exec()) return null;
+			await counter.create([{
+				_id: feed,
+				seq: 0
+			}]).catch((err) => {
+				if (err.code !== 11e3) throw err;
+			});
+			lease = await takeLease();
+			if (!lease) return null;
+		}
+		let seq = lease.seq ?? 0;
+		let numbered = 0;
+		try {
+			for (;;) {
+				const rows = await pending.find({}).sort({ _id: 1 }).limit(SEQUENCER_BATCH).lean().exec();
+				if (rows.length === 0) break;
+				const build = (from, list) => list.map((r, i) => {
+					const { _id, ...rest } = r;
+					return {
+						...rest,
+						seq: from + i + 1,
+						pendingId: String(_id)
+					};
+				});
+				try {
+					await entries.insertMany(build(seq, rows), { ordered: true });
+					seq += rows.length;
+					numbered += rows.length;
+				} catch (err) {
+					if (err.code !== 11e3) throw err;
+					const placed = new Set((await entries.find({ pendingId: { $in: rows.map((r) => String(r._id)) } }).select({ pendingId: 1 }).lean().exec()).map((e) => e.pendingId));
+					const fresh = rows.filter((r) => !placed.has(String(r._id)));
+					const top = await entries.findOne({}).sort({ seq: -1 }).select({ seq: 1 }).lean().exec();
+					seq = Math.max(seq, top?.seq ?? 0);
+					if (fresh.length > 0) {
+						await entries.insertMany(build(seq, fresh), { ordered: true });
+						seq += fresh.length;
+						numbered += fresh.length;
+					}
+				}
+				await counter.updateOne({
+					_id: feed,
+					lockBy: token
+				}, { $max: { seq } }).exec();
+				await pending.deleteMany({ _id: { $in: rows.map((r) => r._id) } }).exec();
+				if (rows.length < SEQUENCER_BATCH) break;
+			}
+		} finally {
+			await counter.updateOne({
+				_id: feed,
+				lockBy: token
+			}, { $set: {
+				lockUntil: null,
+				lockBy: null
+			} }).exec();
+		}
+		return numbered;
+	};
 	const sessionOf = (options) => options?.session ? { session: options.session } : {};
 	const filterOf = (after, options = {}) => {
 		const { tenantId, scopes, sharedScopes } = options;
@@ -104,8 +248,37 @@ function createChangeLogStore(models) {
 		cursor: toCursor(row.seq)
 	});
 	return {
+		sequencePending,
+		async stage(entry, options) {
+			if (!deferred || !pending) {
+				await this.append(entry, options);
+				return;
+			}
+			await pending.create([stagedRow(entry, /* @__PURE__ */ new Date())], sessionOf(options));
+		},
 		async append(entry, options) {
 			const session = sessionOf(options);
+			if (deferred && pending) {
+				const at = /* @__PURE__ */ new Date();
+				const [staged] = await pending.create([stagedRow(entry, at)], session);
+				if (options?.session) return {
+					...entry,
+					at,
+					cursor: ""
+				};
+				const stagedId = String(staged?._id);
+				for (let attempt = 0; attempt < 50; attempt++) {
+					await sequencePending();
+					const at2 = await entries.findOne({ pendingId: stagedId }).select({ seq: 1 }).lean().exec();
+					if (typeof at2?.seq === "number") return {
+						...entry,
+						at,
+						cursor: toCursor(at2.seq)
+					};
+					await new Promise((r) => setTimeout(r, 20));
+				}
+				throw new Error(`[mongokit:sync] staged entry ${stagedId} was not sequenced — a sequencer lease is stuck`);
+			}
 			const seq = (await counter.findOneAndUpdate({ _id: feed }, { $inc: { seq: 1 } }, {
 				upsert: true,
 				returnDocument: "after",
@@ -130,6 +303,7 @@ function createChangeLogStore(models) {
 			};
 		},
 		async since(cursor, options = {}) {
+			if (deferred) await sequencePending();
 			const limit = options.limit ?? DEFAULT_LIMIT;
 			const rows = await entries.find(filterOf(fromCursor(cursor), options)).sort({ seq: 1 }).limit(limit + 1).lean().exec();
 			const page = rows.slice(0, limit).map(toEntry);
@@ -141,6 +315,7 @@ function createChangeLogStore(models) {
 			};
 		},
 		async latestCursor(options = {}) {
+			if (deferred) await sequencePending();
 			const row = await entries.findOne(filterOf(0, options)).sort({ seq: -1 }).select({ seq: 1 }).lean().exec();
 			return row?.seq ? toCursor(row.seq) : "";
 		},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classytic/mongokit",
-  "version": "3.41.0",
+  "version": "3.42.0",
   "description": "Production-grade MongoDB repositories - smart pagination, events, and plugins. Built on @classytic/repo-core.",
   "type": "module",
   "sideEffects": false,
@@ -199,7 +199,7 @@
     "@better-auth/mongo-adapter": "^1.6.29",
     "@biomejs/biome": "^2.5.8",
     "@classytic/dev-tools": "^0.2.0",
-    "@classytic/repo-core": "^0.31.0",
+    "@classytic/repo-core": "^0.31.1",
     "@types/node": "^26.2.0",
     "@vitest/coverage-v8": "^4.1.10",
     "ajv": "^8.20.0",