@procedures/mongo 0.54.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +6 -0
- package/dist/storage/document-storage.d.ts +42 -1
- package/dist/storage/document-storage.d.ts.map +1 -1
- package/dist/storage/document-storage.js +125 -14
- package/dist/storage/document-storage.js.map +1 -1
- package/dist/storage/materializer.d.ts +1 -1
- package/dist/storage/materializer.d.ts.map +1 -1
- package/dist/storage/materializer.js +5 -4
- package/dist/storage/materializer.js.map +1 -1
- package/dist/storage/own-writes.d.ts +10 -4
- package/dist/storage/own-writes.d.ts.map +1 -1
- package/dist/storage/own-writes.js +13 -5
- package/dist/storage/own-writes.js.map +1 -1
- package/dist/storage/storage-hints.js +1 -1
- package/dist/storage/storage-hints.js.map +1 -1
- package/package.json +6 -5
- package/src/storage/document-storage.ts +160 -19
- package/src/storage/materializer.ts +6 -4
- package/src/storage/own-writes.ts +22 -9
- package/src/storage/storage-hints.ts +1 -1
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package",
|
|
3
3
|
"name": "@procedures/mongo",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.55.0",
|
|
5
5
|
"description": "MongoDB as typed procedures: documents, bulk writes, distinct, explain, collections, indexes, aggregation, change streams and database info, with safe update and delete targets.",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"mongodb",
|
|
@@ -66,10 +66,11 @@
|
|
|
66
66
|
"devDependencies": {
|
|
67
67
|
"@mark1russell7/cue": "^2.1.1",
|
|
68
68
|
"@mark1russell7/rigor-bench": "^0.2.33",
|
|
69
|
+
"@mark1russell7/rigor-chaos": "^0.1.0",
|
|
69
70
|
"@mark1russell7/rigor-generators": "^0.1.5",
|
|
70
71
|
"@mark1russell7/rigor-statistics": "^0.1.9",
|
|
71
|
-
"@procedures/peer": "0.
|
|
72
|
-
"@procedures/store": "0.
|
|
72
|
+
"@procedures/peer": "0.55.0",
|
|
73
|
+
"@procedures/store": "0.55.0",
|
|
73
74
|
"@types/node": "^22.0.0",
|
|
74
75
|
"typescript": "^7.0.2",
|
|
75
76
|
"vitest": "^5.0.3"
|
|
@@ -79,8 +80,8 @@
|
|
|
79
80
|
},
|
|
80
81
|
"dependencies": {
|
|
81
82
|
"@mark1russell7/collections": "^0.10.0",
|
|
82
|
-
"@procedures/client": "0.
|
|
83
|
-
"@procedures/collections": "0.
|
|
83
|
+
"@procedures/client": "0.55.0",
|
|
84
|
+
"@procedures/collections": "0.55.0",
|
|
84
85
|
"@mark1russell7/service": "^0.8.3",
|
|
85
86
|
"mongodb": "^7.0.0",
|
|
86
87
|
"zod": "^4.6.5",
|
|
@@ -230,6 +230,26 @@ export interface MongoDocumentStorageOptions<T = unknown> {
|
|
|
230
230
|
queryPolicy?: StorageQueryPolicy | undefined;
|
|
231
231
|
/** The collection of the sequence counters (default `__storage_counters`). */
|
|
232
232
|
counters?: string | undefined;
|
|
233
|
+
/**
|
|
234
|
+
* The sequence of the numbers of the writes (roadmap CLI-96). `collection` (the default): one counter for the collection.
|
|
235
|
+
* `partition`: one counter and one write queue for each partition, so the writes of two partitions share no document.
|
|
236
|
+
*
|
|
237
|
+
* - Give the same value to each store of a collection. A store refuses a collection that has the other sequence, with a clear error.
|
|
238
|
+
* - A store with `partition` refuses a collection with numbers on its counter of the collection. Use it for a new collection.
|
|
239
|
+
* - One partition keeps the limit of one collection with the default: approximately 400 writes a second, and a stated limit of 200.
|
|
240
|
+
*/
|
|
241
|
+
sequence?: "collection" | "partition" | undefined;
|
|
242
|
+
/**
|
|
243
|
+
* The most writes of each client at one time, over the stores with the same value (roadmap CLI-96). The default is 8.
|
|
244
|
+
* A write waits for a permit after its turn in the write queue of its counter.
|
|
245
|
+
*
|
|
246
|
+
* - With `sequence: "partition"`, each partition has its own queue. Without this bound, each partition of a process has a transaction open at once.
|
|
247
|
+
* - A whole number of 1 or more, or `Infinity` for no bound. Another value is a `RangeError`.
|
|
248
|
+
* - The sweeps of 2, 4, 8, 16 and `Infinity` at the bench cases `400-p2` and `4000-p2` set the default of 8.
|
|
249
|
+
* - With `sequence: "partition"`, `Infinity` costs approximately 30 % of the writes a second. The bench run 42428048767
|
|
250
|
+
* gives 638.5 against 916.5 of the default sequence at `4000-p2`. The store emits a process warning one time for it.
|
|
251
|
+
*/
|
|
252
|
+
writesInFlight?: number | undefined;
|
|
233
253
|
/**
|
|
234
254
|
* The time after which a pending number of a write no longer holds the watermark (default 60 s):
|
|
235
255
|
* a writer that stopped does not stop the change reads. A plain number is a `TypeError`.
|
|
@@ -286,13 +306,31 @@ interface PendingRange {
|
|
|
286
306
|
at: Date;
|
|
287
307
|
}
|
|
288
308
|
|
|
289
|
-
/** A document of the counters: the sequence of a collection, or the floor of the log of a partition. */
|
|
309
|
+
/** A document of the counters: the sequence of a collection or of a partition, or the floor of the log of a partition. */
|
|
290
310
|
interface CounterDoc {
|
|
291
311
|
_id: string;
|
|
292
312
|
seq?: number;
|
|
293
313
|
epoch?: string;
|
|
294
314
|
floor?: number;
|
|
295
315
|
pending?: PendingRange[];
|
|
316
|
+
/** `partition` on the counter of a collection whose stores have the option `sequence: "partition"` (roadmap CLI-96). */
|
|
317
|
+
sequence?: string;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* The checks of the sequence of each collection (roadmap CLI-96): by the client, then by the names of the database, the
|
|
322
|
+
* counters and the collection. A check that fails leaves no entry, so the next call tries again.
|
|
323
|
+
*/
|
|
324
|
+
const SEQUENCE_CHECKS = new WeakMap<object, Map<string, Promise<void>>>();
|
|
325
|
+
|
|
326
|
+
/** The error of a store whose sequence is not the sequence of its collection (roadmap CLI-96). */
|
|
327
|
+
function sequenceMismatch(collection: string, sequence: "collection" | "partition"): Error {
|
|
328
|
+
return sequence === "partition"
|
|
329
|
+
? new Error(
|
|
330
|
+
`The collection ${collection} has numbers on its counter of the collection, so the option sequence "partition" refuses it. ` +
|
|
331
|
+
"The option is for a collection with no writes. The move of a live collection is a step with all processes stopped."
|
|
332
|
+
)
|
|
333
|
+
: new Error(`The collection ${collection} has a counter for each partition. Give the option sequence "partition" to each store of it.`);
|
|
296
334
|
}
|
|
297
335
|
|
|
298
336
|
/** True when a client is on a replica set or a sharded cluster: it has transactions. */
|
|
@@ -328,6 +366,37 @@ const QUEUES = new WeakMap<object, Map<string, Promise<unknown>>>();
|
|
|
328
366
|
/** The change reads of each client at one time. A burst of reads leaves the pool to the other operations. */
|
|
329
367
|
const READS = new WeakMap<object, Semaphore>();
|
|
330
368
|
|
|
369
|
+
/** The default of the option `writesInFlight`: the most writes of each client at one time (roadmap CLI-96). */
|
|
370
|
+
export const DEFAULT_WRITES_IN_FLIGHT = 8;
|
|
371
|
+
|
|
372
|
+
/** The code of the process warning for a store with `sequence: "partition"` and no bound on its writes. */
|
|
373
|
+
export const UNBOUNDED_PARTITION_WARNING = "PROCEDURES_MONGO_UNBOUNDED_PARTITION";
|
|
374
|
+
|
|
375
|
+
let warnedUnboundedPartition = false;
|
|
376
|
+
|
|
377
|
+
/** This function emits the process warning of an unbounded store with a counter for each partition, one time. */
|
|
378
|
+
function warnUnboundedPartition(): void {
|
|
379
|
+
if (warnedUnboundedPartition) return;
|
|
380
|
+
warnedUnboundedPartition = true;
|
|
381
|
+
process.emitWarning(
|
|
382
|
+
'writesInFlight: Infinity with sequence: "partition" costs approximately 30 % of the writes a second. Use the default bound of 8.',
|
|
383
|
+
{ code: UNBOUNDED_PARTITION_WARNING },
|
|
384
|
+
);
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** The permits of the writes of each client, by the value of `writesInFlight`: the stores with one value share the permits. */
|
|
388
|
+
const WRITES = new WeakMap<object, Map<number, Semaphore>>();
|
|
389
|
+
|
|
390
|
+
/** This function gives the permits of the writes of a client for a bound. */
|
|
391
|
+
function writePermitsOf(db: Db, bound: number): Semaphore {
|
|
392
|
+
const owner = ownerOf(db);
|
|
393
|
+
let byBound = WRITES.get(owner);
|
|
394
|
+
if (!byBound) WRITES.set(owner, (byBound = new Map()));
|
|
395
|
+
let permits = byBound.get(bound);
|
|
396
|
+
if (!permits) byBound.set(bound, (permits = semaphore(bound)));
|
|
397
|
+
return permits;
|
|
398
|
+
}
|
|
399
|
+
|
|
331
400
|
/** This function starts a write after each earlier write of the collection. */
|
|
332
401
|
function enqueueWrite<R>(db: Db, name: string, task: () => Promise<R>): Promise<R> {
|
|
333
402
|
const owner = ownerOf(db);
|
|
@@ -639,6 +708,8 @@ export interface StorageWriteTarget {
|
|
|
639
708
|
withPendingNumbers<R>(db: Db, count: number, write: (first: number) => Promise<R>): Promise<R>;
|
|
640
709
|
/** This method writes the floor document of the partition once (roadmap O66). */
|
|
641
710
|
noteFloor(db: Db): Promise<void>;
|
|
711
|
+
/** The sequence of the numbers: one counter for the collection, or one for each partition (roadmap CLI-96). */
|
|
712
|
+
readonly sequence: "collection" | "partition";
|
|
642
713
|
/** This method drops the oldest tombstones over the limit, and raises the floor of the change log. */
|
|
643
714
|
prune(db: Db): Promise<void>;
|
|
644
715
|
}
|
|
@@ -756,10 +827,11 @@ function prefilter(selector: StorageSelector, options: SelectorPrefilterOptions,
|
|
|
756
827
|
return parts.length === 1 ? parts[0] : { $and: parts };
|
|
757
828
|
}
|
|
758
829
|
|
|
759
|
-
/** The sequence counter of a collection. */
|
|
830
|
+
/** The sequence counter of a collection or of a partition. */
|
|
760
831
|
interface Counter {
|
|
761
832
|
seq: number;
|
|
762
833
|
epoch: string;
|
|
834
|
+
sequence?: string;
|
|
763
835
|
}
|
|
764
836
|
|
|
765
837
|
/** The watermark of a collection, with the floor of the log of a partition. */
|
|
@@ -787,6 +859,8 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
787
859
|
private readonly maxTombstones: number;
|
|
788
860
|
private readonly countersName: string;
|
|
789
861
|
private readonly writeLeaseMs: number;
|
|
862
|
+
/** The most writes of each client at one time, over the stores with this value. `Infinity` is no bound. */
|
|
863
|
+
private readonly writesInFlight: number;
|
|
790
864
|
/** The storages of the partitions, for `forPartition`: the same storage for the same partition. */
|
|
791
865
|
private readonly partitions: Map<string, MongoDocumentStorage<T>>;
|
|
792
866
|
/** The stored form of the documents (roadmap CLI-76). A `ViewMaterializer` of the collection takes the same form. */
|
|
@@ -800,6 +874,12 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
800
874
|
private indexes: Promise<void> | undefined;
|
|
801
875
|
/** True after this storage wrote (or found) the floor document of its partition (roadmap O66). A failed write leaves it false. */
|
|
802
876
|
private floorNoted = false;
|
|
877
|
+
/** The sequence of the numbers (roadmap CLI-96). */
|
|
878
|
+
private readonly sequence: "collection" | "partition";
|
|
879
|
+
/** The `_id` of the counter of the numbers: the collection, or the partition with the option `sequence: "partition"`. */
|
|
880
|
+
private readonly counterId: string;
|
|
881
|
+
/** The partition of the notes of the own numbers with a counter for each partition, else `undefined` (roadmap CLI-96). */
|
|
882
|
+
private readonly ownScope: string | undefined;
|
|
803
883
|
private closed = false;
|
|
804
884
|
|
|
805
885
|
static {
|
|
@@ -818,6 +898,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
818
898
|
writeTargetOfStorage = <S>(storage: MongoDocumentStorage<S>): StorageWriteTarget => ({
|
|
819
899
|
collection: storage.collectionName,
|
|
820
900
|
counters: storage.countersName,
|
|
901
|
+
sequence: storage.sequence,
|
|
821
902
|
form: storage.form,
|
|
822
903
|
partition: storage.partition,
|
|
823
904
|
writeLeaseMs: storage.writeLeaseMs,
|
|
@@ -862,8 +943,18 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
862
943
|
this.maxTombstones = options.maxTombstones ?? 10_000;
|
|
863
944
|
if (!Number.isSafeInteger(this.maxTombstones) || this.maxTombstones < 0) throw new Error("maxTombstones must be a whole number of 0 or more");
|
|
864
945
|
this.countersName = options.counters ?? "__storage_counters";
|
|
946
|
+
const sequence = options.sequence ?? "collection";
|
|
947
|
+
if (sequence !== "collection" && sequence !== "partition") throw new TypeError(`sequence must be "collection" or "partition", not ${JSON.stringify(sequence)}`);
|
|
948
|
+
this.sequence = sequence;
|
|
949
|
+
this.counterId = sequence === "partition" ? `seq${SEPARATOR}${this.collectionName}${SEPARATOR}${this.partition ?? ""}` : this.collectionName;
|
|
950
|
+
this.ownScope = sequence === "partition" ? (this.partition ?? "") : undefined;
|
|
865
951
|
this.writeLeaseMs = Math.round(optionalMs(options.writeLease, "writeLease") ?? 60_000);
|
|
866
952
|
if (!Number.isSafeInteger(this.writeLeaseMs) || this.writeLeaseMs < 1) throw new Error("writeLease must be 1 ms or more");
|
|
953
|
+
this.writesInFlight = options.writesInFlight ?? DEFAULT_WRITES_IN_FLIGHT;
|
|
954
|
+
if (!(this.writesInFlight === Infinity || (Number.isSafeInteger(this.writesInFlight) && this.writesInFlight >= 1))) {
|
|
955
|
+
throw new RangeError(`writesInFlight must be a whole number of 1 or more, or Infinity, not ${String(options.writesInFlight)}`);
|
|
956
|
+
}
|
|
957
|
+
if (this.writesInFlight === Infinity && this.sequence === "partition") warnUnboundedPartition();
|
|
867
958
|
this.partitions = partitions ?? new Map();
|
|
868
959
|
}
|
|
869
960
|
|
|
@@ -1011,7 +1102,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
1011
1102
|
*/
|
|
1012
1103
|
private async allocate(db: Db, count: number): Promise<{ first: number; epoch: string }> {
|
|
1013
1104
|
const counter = (await db.collection<CounterDoc>(this.countersName).findOneAndUpdate(
|
|
1014
|
-
{ _id: this.
|
|
1105
|
+
{ _id: this.counterId },
|
|
1015
1106
|
[
|
|
1016
1107
|
{ $set: { seq: { $add: [{ $ifNull: ["$seq", 0] }, count] }, epoch: { $ifNull: ["$epoch", randomUUID()] } } },
|
|
1017
1108
|
{
|
|
@@ -1025,7 +1116,12 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
1025
1116
|
{ upsert: true, returnDocument: "after" }
|
|
1026
1117
|
)) as unknown as Counter | null;
|
|
1027
1118
|
if (!counter) throw new Error(`The sequence of ${this.collectionName} is not available`);
|
|
1028
|
-
|
|
1119
|
+
const first = counter.seq - count + 1;
|
|
1120
|
+
if (this.sequence === "collection" && counter.sequence === "partition") {
|
|
1121
|
+
await this.release(db, first).catch(() => undefined);
|
|
1122
|
+
throw sequenceMismatch(this.collectionName, "collection");
|
|
1123
|
+
}
|
|
1124
|
+
return { first, epoch: counter.epoch };
|
|
1029
1125
|
}
|
|
1030
1126
|
|
|
1031
1127
|
/**
|
|
@@ -1037,7 +1133,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
1037
1133
|
private async release(db: Db, first: number): Promise<boolean> {
|
|
1038
1134
|
const live = { $gt: ["$$this.at", { $subtract: ["$$NOW", this.writeLeaseMs] }] };
|
|
1039
1135
|
const after = (await db.collection<CounterDoc>(this.countersName).findOneAndUpdate(
|
|
1040
|
-
{ _id: this.
|
|
1136
|
+
{ _id: this.counterId },
|
|
1041
1137
|
[
|
|
1042
1138
|
{
|
|
1043
1139
|
$set: {
|
|
@@ -1093,19 +1189,22 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
1093
1189
|
own = true
|
|
1094
1190
|
): Promise<R> {
|
|
1095
1191
|
const noteOwn = own ? noteOwnNumbers : (): void => undefined;
|
|
1192
|
+
await this.checkSequence(db);
|
|
1096
1193
|
const client = await this.transactionClient(db);
|
|
1097
1194
|
if (client) {
|
|
1098
1195
|
const session = client.startSession();
|
|
1099
1196
|
try {
|
|
1100
1197
|
return await session.withTransaction(async () => {
|
|
1101
1198
|
const counter = (await db.collection<CounterDoc>(this.countersName).findOneAndUpdate(
|
|
1102
|
-
{ _id: this.
|
|
1199
|
+
{ _id: this.counterId },
|
|
1103
1200
|
{ $inc: { seq: count }, $setOnInsert: { epoch: randomUUID() } },
|
|
1104
1201
|
{ upsert: true, returnDocument: "after", session }
|
|
1105
1202
|
)) as unknown as Counter | null;
|
|
1106
1203
|
if (!counter) throw new Error(`The sequence of ${this.collectionName} is not available`);
|
|
1204
|
+
// The error aborts the transaction, so the counter keeps its number
|
|
1205
|
+
if (this.sequence === "collection" && counter.sequence === "partition") throw sequenceMismatch(this.collectionName, "collection");
|
|
1107
1206
|
// A watcher of the collection in this process skips the events of these numbers (roadmap CLI-77)
|
|
1108
|
-
noteOwn(db, this.collectionName, counter.seq - count + 1, count);
|
|
1207
|
+
noteOwn(db, this.collectionName, counter.seq - count + 1, count, this.ownScope);
|
|
1109
1208
|
return write(counter.seq - count + 1, session);
|
|
1110
1209
|
});
|
|
1111
1210
|
} finally {
|
|
@@ -1126,10 +1225,11 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
1126
1225
|
collection: Collection<Stored>,
|
|
1127
1226
|
count: number,
|
|
1128
1227
|
write: (first: number) => Promise<R>,
|
|
1129
|
-
noteOwn: (db: Db, collection: string, first: number, count: number) => void
|
|
1228
|
+
noteOwn: (db: Db, collection: string, first: number, count: number, scope?: string) => void
|
|
1130
1229
|
): Promise<R> {
|
|
1230
|
+
await this.checkSequence(db);
|
|
1131
1231
|
let { first } = await this.allocate(db, count);
|
|
1132
|
-
noteOwn(db, this.collectionName, first, count);
|
|
1232
|
+
noteOwn(db, this.collectionName, first, count, this.ownScope);
|
|
1133
1233
|
let result: R;
|
|
1134
1234
|
try {
|
|
1135
1235
|
result = await write(first);
|
|
@@ -1141,7 +1241,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
1141
1241
|
if (moves >= MAX_MOVES) throw new Error(`A write of ${this.collectionName} stayed late after ${MAX_MOVES} moves to new numbers`);
|
|
1142
1242
|
// Late: a reader can have passed these numbers. The write takes new ones.
|
|
1143
1243
|
const next = (await this.allocate(db, count)).first;
|
|
1144
|
-
noteOwn(db, this.collectionName, next, count);
|
|
1244
|
+
noteOwn(db, this.collectionName, next, count, this.ownScope);
|
|
1145
1245
|
await this.move(collection, first, count, next);
|
|
1146
1246
|
first = next;
|
|
1147
1247
|
}
|
|
@@ -1151,20 +1251,23 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
1151
1251
|
/**
|
|
1152
1252
|
* The watermark of the collection: the highest number below each pending number of a write whose
|
|
1153
1253
|
* lease has not ended, the epoch, and the floor of the partition. Each number up to it is written
|
|
1154
|
-
* or abandoned. One command reads them. Only the first read of a
|
|
1254
|
+
* or abandoned. One command reads them. Only the first read of a store writes: it makes the counter. With the
|
|
1255
|
+
* option `sequence: "partition"`, the first read of each process also marks the counter of the collection.
|
|
1155
1256
|
*/
|
|
1156
1257
|
private async watermark(db: Db): Promise<Watermark> {
|
|
1258
|
+
await this.checkSequence(db);
|
|
1157
1259
|
const counters = db.collection<CounterDoc>(this.countersName);
|
|
1158
1260
|
const floorId = this.floorId();
|
|
1159
|
-
const read = (): Promise<Array<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null }>> =>
|
|
1261
|
+
const read = (): Promise<Array<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null; sequence?: string }>> =>
|
|
1160
1262
|
counters
|
|
1161
|
-
.aggregate<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null }>([
|
|
1162
|
-
{ $match: { _id: { $in: [this.
|
|
1263
|
+
.aggregate<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null; sequence?: string }>([
|
|
1264
|
+
{ $match: { _id: { $in: [this.counterId, floorId] } } },
|
|
1163
1265
|
{
|
|
1164
1266
|
$project: {
|
|
1165
1267
|
seq: 1,
|
|
1166
1268
|
epoch: 1,
|
|
1167
1269
|
floor: 1,
|
|
1270
|
+
sequence: 1,
|
|
1168
1271
|
low: {
|
|
1169
1272
|
$min: {
|
|
1170
1273
|
$map: {
|
|
@@ -1180,20 +1283,54 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
1180
1283
|
])
|
|
1181
1284
|
.toArray();
|
|
1182
1285
|
let docs = await read();
|
|
1183
|
-
if (!docs.some((doc) => doc._id === this.
|
|
1286
|
+
if (!docs.some((doc) => doc._id === this.counterId)) {
|
|
1184
1287
|
// A duplicate key: another read made the counter at the same time
|
|
1185
|
-
await counters.updateOne({ _id: this.
|
|
1288
|
+
await counters.updateOne({ _id: this.counterId }, { $setOnInsert: { seq: 0, epoch: randomUUID() } }, { upsert: true }).catch((error: unknown) => {
|
|
1186
1289
|
if (!isDuplicateKey(error)) throw error;
|
|
1187
1290
|
});
|
|
1188
1291
|
docs = await read();
|
|
1189
1292
|
}
|
|
1190
|
-
const mark = docs.find((doc) => doc._id === this.
|
|
1293
|
+
const mark = docs.find((doc) => doc._id === this.counterId);
|
|
1191
1294
|
if (!mark) throw new Error(`The sequence of ${this.collectionName} is not available`);
|
|
1295
|
+
if (this.sequence === "collection" && mark.sequence === "partition") throw sequenceMismatch(this.collectionName, "collection");
|
|
1192
1296
|
const seq = typeof mark.low === "number" ? Math.min(mark.seq, mark.low - 1) : mark.seq;
|
|
1193
1297
|
const floor = docs.find((doc) => doc._id === floorId)?.floor;
|
|
1194
1298
|
return { seq, epoch: mark.epoch, floor: typeof floor === "number" ? floor : 0, held: seq < mark.seq };
|
|
1195
1299
|
}
|
|
1196
1300
|
|
|
1301
|
+
/**
|
|
1302
|
+
* This method checks once for each collection that a store with the option `sequence: "partition"` can use it (roadmap
|
|
1303
|
+
* CLI-96). One atomic step marks the counter of the collection, unless that counter has numbers. A store with the default
|
|
1304
|
+
* reads the mark in its own steps on that counter, so it needs no check of its own.
|
|
1305
|
+
*/
|
|
1306
|
+
private checkSequence(db: Db): Promise<void> {
|
|
1307
|
+
if (this.sequence === "collection") return Promise.resolve();
|
|
1308
|
+
const owner = ownerOf(db);
|
|
1309
|
+
let checks = SEQUENCE_CHECKS.get(owner);
|
|
1310
|
+
if (!checks) SEQUENCE_CHECKS.set(owner, (checks = new Map()));
|
|
1311
|
+
const key = `${db.databaseName}\u0000${this.countersName}\u0000${this.collectionName}`;
|
|
1312
|
+
let check = checks.get(key);
|
|
1313
|
+
if (!check) {
|
|
1314
|
+
const mark = async (): Promise<void> => {
|
|
1315
|
+
const counter = await db.collection<CounterDoc>(this.countersName).findOneAndUpdate(
|
|
1316
|
+
{ _id: this.collectionName },
|
|
1317
|
+
// A counter with numbers keeps its mark as it is: absent for a collection of the default sequence
|
|
1318
|
+
[{ $set: { sequence: { $cond: [{ $gt: [{ $ifNull: ["$seq", 0] }, 0] }, "$sequence", "partition"] } } }],
|
|
1319
|
+
{ upsert: true, returnDocument: "after" }
|
|
1320
|
+
);
|
|
1321
|
+
if (counter?.sequence !== "partition") throw sequenceMismatch(this.collectionName, "partition");
|
|
1322
|
+
};
|
|
1323
|
+
// Two processes can insert the counter at the same time: the second finds it at its next try
|
|
1324
|
+
check = mark().catch((error: unknown) => (isDuplicateKey(error) ? mark() : Promise.reject(error)));
|
|
1325
|
+
checks.set(key, check);
|
|
1326
|
+
const own = checks;
|
|
1327
|
+
check.catch(() => {
|
|
1328
|
+
if (own.get(key) === check) own.delete(key);
|
|
1329
|
+
});
|
|
1330
|
+
}
|
|
1331
|
+
return check;
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1197
1334
|
/** The id of the document of the oldest kept change of the partition. */
|
|
1198
1335
|
private async floorOf(db: Db): Promise<number> {
|
|
1199
1336
|
const doc = await db.collection<CounterDoc>(this.countersName).findOne({ _id: this.floorId() });
|
|
@@ -1460,11 +1597,15 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
1460
1597
|
|
|
1461
1598
|
private async write<R>(task: (db: Db, items: Collection<Stored>) => Promise<R>): Promise<R> {
|
|
1462
1599
|
const { db, items } = await this.collection();
|
|
1463
|
-
|
|
1600
|
+
const run = async (): Promise<R> => {
|
|
1464
1601
|
// The floor document of the partition, outside each transaction of the task (roadmap O66)
|
|
1465
1602
|
await this.noteFloor(db);
|
|
1466
1603
|
return task(db, items);
|
|
1467
|
-
}
|
|
1604
|
+
};
|
|
1605
|
+
// The permit comes after the turn in the queue, so a write that waits for an earlier write of its counter holds no permit
|
|
1606
|
+
if (this.writesInFlight === Infinity) return enqueueWrite(db, this.counterId, run);
|
|
1607
|
+
const permits = writePermitsOf(db, this.writesInFlight);
|
|
1608
|
+
return enqueueWrite(db, this.counterId, () => permits.run(run));
|
|
1468
1609
|
}
|
|
1469
1610
|
|
|
1470
1611
|
async set(id: string, value: T, options?: StorageItemWriteOptions): Promise<void> {
|
|
@@ -360,6 +360,7 @@ function contentHash(doc: Document): string {
|
|
|
360
360
|
interface StorageTargetValues {
|
|
361
361
|
readonly form: StoredForm;
|
|
362
362
|
readonly counters: string | undefined;
|
|
363
|
+
readonly sequence: "collection" | "partition" | undefined;
|
|
363
364
|
readonly writeLeaseMs: number | undefined;
|
|
364
365
|
readonly maxTombstones: number | undefined;
|
|
365
366
|
}
|
|
@@ -557,7 +558,7 @@ export class ViewMaterializer {
|
|
|
557
558
|
/**
|
|
558
559
|
* A target of `storageTargets` as a storage on the database of the materializer, so its writes
|
|
559
560
|
* and the transaction of a run use one client. A given storage gives its form, its counters,
|
|
560
|
-
* `writeLease` and `maxTombstones`.
|
|
561
|
+
* its sequence, `writeLease` and `maxTombstones`.
|
|
561
562
|
*/
|
|
562
563
|
private storageTargetOf(collection: string, entry: ViewStorageTargetEntry): StorageTargetEntry {
|
|
563
564
|
const wrapped = typeof entry === "object" && entry !== null && "target" in entry ? entry : { target: entry };
|
|
@@ -568,15 +569,15 @@ export class ViewMaterializer {
|
|
|
568
569
|
if (target instanceof MongoDocumentStorage) {
|
|
569
570
|
const parts = storageWriteTarget(target);
|
|
570
571
|
if (parts.collection !== collection) throw new TypeError(`The storage target ${collection} has a storage of the collection ${parts.collection}`);
|
|
571
|
-
values = { form: parts.form, counters: parts.counters, writeLeaseMs: parts.writeLeaseMs, maxTombstones: parts.maxTombstones };
|
|
572
|
+
values = { form: parts.form, counters: parts.counters, sequence: parts.sequence, writeLeaseMs: parts.writeLeaseMs, maxTombstones: parts.maxTombstones };
|
|
572
573
|
} else if (typeof target === "object" && target !== null && typeof (target as Partial<StoredForm>).storedId === "function") {
|
|
573
|
-
values = { form: target as StoredForm, counters: undefined, writeLeaseMs: undefined, maxTombstones: undefined };
|
|
574
|
+
values = { form: target as StoredForm, counters: undefined, sequence: undefined, writeLeaseMs: undefined, maxTombstones: undefined };
|
|
574
575
|
} else {
|
|
575
576
|
throw new TypeError(`The storage target ${collection} needs a MongoDocumentStorage, a stored form or a ViewStorageTarget`);
|
|
576
577
|
}
|
|
577
578
|
// The same values give the storage of the last run: its partitions keep their state (the floor document of O66)
|
|
578
579
|
const cached = this.storageTargetCache.get(collection);
|
|
579
|
-
if (cached && cached.form === values.form && cached.counters === values.counters && cached.writeLeaseMs === values.writeLeaseMs && cached.maxTombstones === values.maxTombstones) {
|
|
580
|
+
if (cached && cached.form === values.form && cached.counters === values.counters && cached.sequence === values.sequence && cached.writeLeaseMs === values.writeLeaseMs && cached.maxTombstones === values.maxTombstones) {
|
|
580
581
|
return { storage: cached.storage, storagePartitionOf };
|
|
581
582
|
}
|
|
582
583
|
const db = (): Db => {
|
|
@@ -589,6 +590,7 @@ export class ViewMaterializer {
|
|
|
589
590
|
form: values.form,
|
|
590
591
|
transactions: "never",
|
|
591
592
|
...(values.counters !== undefined ? { counters: values.counters } : {}),
|
|
593
|
+
...(values.sequence !== undefined ? { sequence: values.sequence } : {}),
|
|
592
594
|
...(values.writeLeaseMs !== undefined ? { writeLease: ms(values.writeLeaseMs) } : {}),
|
|
593
595
|
...(values.maxTombstones !== undefined ? { maxTombstones: values.maxTombstones } : {}),
|
|
594
596
|
});
|
|
@@ -35,8 +35,11 @@ const MAX_OWN_WRITES = 100_000;
|
|
|
35
35
|
|
|
36
36
|
/** The noted writes of one watcher of one collection. */
|
|
37
37
|
export interface OwnNumbers {
|
|
38
|
-
/**
|
|
39
|
-
|
|
38
|
+
/**
|
|
39
|
+
* True when a write of this process noted the number and no event took it yet. The number is then taken. With a counter for
|
|
40
|
+
* each partition (roadmap CLI-96), two partitions can have the same number, so give the partition of the event too.
|
|
41
|
+
*/
|
|
42
|
+
take(seq: number, partition?: string): boolean;
|
|
40
43
|
/** True when this process pruned the tombstone of the stored `_id` and no event took it yet. The entry is then taken. */
|
|
41
44
|
takePruned(storedId: unknown): boolean;
|
|
42
45
|
/** This method ends the tracking: the storage notes no more entries for it. */
|
|
@@ -52,7 +55,8 @@ export interface TrackOwnNumbersOptions {
|
|
|
52
55
|
}
|
|
53
56
|
|
|
54
57
|
interface Tracker {
|
|
55
|
-
|
|
58
|
+
/** The numbers: a number for the counter of the collection, and the partition with the number for a counter of each partition. */
|
|
59
|
+
readonly numbers: Map<number | string, number> | undefined;
|
|
56
60
|
readonly pruned: Map<string, number>;
|
|
57
61
|
readonly now: () => number;
|
|
58
62
|
}
|
|
@@ -121,8 +125,9 @@ export function trackOwnNumbers(db: Db, collection: string, options: TrackOwnNum
|
|
|
121
125
|
return until > tracker.now();
|
|
122
126
|
};
|
|
123
127
|
return {
|
|
124
|
-
take(seq: number): boolean {
|
|
125
|
-
|
|
128
|
+
take(seq: number, partition?: string): boolean {
|
|
129
|
+
const numbers = tracker.numbers as Map<unknown, number> | undefined;
|
|
130
|
+
return numbers !== undefined && (live(numbers, seq) || live(numbers, scopedKey(partition ?? "", seq)));
|
|
126
131
|
},
|
|
127
132
|
takePruned(storedId: unknown): boolean {
|
|
128
133
|
return live(tracker.pruned as Map<unknown, number>, idKey(storedId));
|
|
@@ -137,14 +142,22 @@ export function trackOwnNumbers(db: Db, collection: string, options: TrackOwnNum
|
|
|
137
142
|
};
|
|
138
143
|
}
|
|
139
144
|
|
|
140
|
-
/**
|
|
141
|
-
|
|
145
|
+
/** The key of a number of a counter of a partition (roadmap CLI-96). The partition of a store without a partition is the empty text. */
|
|
146
|
+
function scopedKey(partition: string, seq: number): string {
|
|
147
|
+
return `${partition}\u001f${String(seq)}`;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* This function notes the numbers `first` to `first + count - 1` of a write of this process, for each open tracker of the
|
|
152
|
+
* collection. `scope` is the partition of a counter of each partition (roadmap CLI-96), and absent for the counter of the collection.
|
|
153
|
+
*/
|
|
154
|
+
export function noteOwnNumbers(db: Db, collection: string, first: number, count: number, scope?: string): void {
|
|
142
155
|
const trackers = trackersOf(db, collection);
|
|
143
156
|
if (!trackers) return;
|
|
144
157
|
for (const tracker of trackers) {
|
|
145
158
|
if (!tracker.numbers) continue;
|
|
146
|
-
const numbers: number
|
|
147
|
-
for (let seq = first; seq < first + count; seq++) numbers.push(seq);
|
|
159
|
+
const numbers: Array<number | string> = [];
|
|
160
|
+
for (let seq = first; seq < first + count; seq++) numbers.push(scope === undefined ? seq : scopedKey(scope, seq));
|
|
148
161
|
noteAll(tracker.numbers, numbers, tracker.now());
|
|
149
162
|
}
|
|
150
163
|
}
|
|
@@ -868,7 +868,7 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
|
|
|
868
868
|
if (source.partition !== undefined && item.partition !== source.partition) return;
|
|
869
869
|
if (!item.raw) {
|
|
870
870
|
// The event of a write of this process: the collection procedures publish it
|
|
871
|
-
if (skipOwn && own && item.seq !== undefined && own.take(item.seq)) return;
|
|
871
|
+
if (skipOwn && own && item.seq !== undefined && own.take(item.seq, item.partition)) return;
|
|
872
872
|
return publishItem(item);
|
|
873
873
|
}
|
|
874
874
|
const before = asDocument(change["fullDocumentBeforeChange"]);
|