@procedures/mongo 0.53.1 → 0.54.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "$schema": "https://json.schemastore.org/package",
3
3
  "name": "@procedures/mongo",
4
- "version": "0.53.1",
4
+ "version": "0.54.1",
5
5
  "description": "MongoDB as typed procedures: documents, bulk writes, distinct, explain, collections, indexes, aggregation, change streams and database info, with safe update and delete targets.",
6
6
  "keywords": [
7
7
  "mongodb",
@@ -46,6 +46,7 @@
46
46
  },
47
47
  "files": [
48
48
  "dist",
49
+ "!dist/.tsbuildinfo",
49
50
  "src",
50
51
  "!src/**/*.test.ts",
51
52
  "!src/**/*.test.tsx",
@@ -65,10 +66,11 @@
65
66
  "devDependencies": {
66
67
  "@mark1russell7/cue": "^2.1.1",
67
68
  "@mark1russell7/rigor-bench": "^0.2.33",
69
+ "@mark1russell7/rigor-chaos": "^0.1.0",
68
70
  "@mark1russell7/rigor-generators": "^0.1.5",
69
71
  "@mark1russell7/rigor-statistics": "^0.1.9",
70
- "@procedures/peer": "0.53.1",
71
- "@procedures/store": "0.53.1",
72
+ "@procedures/peer": "0.54.1",
73
+ "@procedures/store": "0.54.1",
72
74
  "@types/node": "^22.0.0",
73
75
  "typescript": "^7.0.2",
74
76
  "vitest": "^5.0.3"
@@ -78,8 +80,8 @@
78
80
  },
79
81
  "dependencies": {
80
82
  "@mark1russell7/collections": "^0.10.0",
81
- "@procedures/client": "0.53.1",
82
- "@procedures/collections": "0.53.1",
83
+ "@procedures/client": "0.54.1",
84
+ "@procedures/collections": "0.54.1",
83
85
  "@mark1russell7/service": "^0.8.3",
84
86
  "mongodb": "^7.0.0",
85
87
  "zod": "^4.6.5",
@@ -230,6 +230,26 @@ export interface MongoDocumentStorageOptions<T = unknown> {
230
230
  queryPolicy?: StorageQueryPolicy | undefined;
231
231
  /** The collection of the sequence counters (default `__storage_counters`). */
232
232
  counters?: string | undefined;
233
+ /**
234
+ * The sequence of the numbers of the writes (roadmap CLI-96). `collection` (the default): one counter for the collection.
235
+ * `partition`: one counter and one write queue for each partition, so the writes of two partitions share no document.
236
+ *
237
+ * - Give the same value to each store of a collection. A store refuses a collection that has the other sequence, with a clear error.
238
+ * - A store with `partition` refuses a collection with numbers on its counter of the collection. Use it for a new collection.
239
+ * - One partition keeps the limit of one collection with the default: approximately 400 writes a second, and a stated limit of 200.
240
+ */
241
+ sequence?: "collection" | "partition" | undefined;
242
+ /**
243
+ * The most writes of each client at one time, over the stores with the same value (roadmap CLI-96). The default is 8.
244
+ * A write waits for a permit after its turn in the write queue of its counter.
245
+ *
246
+ * - With `sequence: "partition"`, each partition has its own queue. Without this bound, each partition of a process has a transaction open at once.
247
+ * - A whole number of 1 or more, or `Infinity` for no bound. Another value is a `RangeError`.
248
+ * - The sweeps of 2, 4, 8, 16 and `Infinity` at the bench cases `400-p2` and `4000-p2` set the default of 8.
249
+ * - With `sequence: "partition"`, `Infinity` costs approximately 30 % of the writes a second. The bench run 42428048767
250
+ * gives 638.5 against 916.5 of the default sequence at `4000-p2`. The store emits a process warning one time for it.
251
+ */
252
+ writesInFlight?: number | undefined;
233
253
  /**
234
254
  * The time after which a pending number of a write no longer holds the watermark (default 60 s):
235
255
  * a writer that stopped does not stop the change reads. A plain number is a `TypeError`.
@@ -286,13 +306,31 @@ interface PendingRange {
286
306
  at: Date;
287
307
  }
288
308
 
289
- /** A document of the counters: the sequence of a collection, or the floor of the log of a partition. */
309
+ /** A document of the counters: the sequence of a collection or of a partition, or the floor of the log of a partition. */
290
310
  interface CounterDoc {
291
311
  _id: string;
292
312
  seq?: number;
293
313
  epoch?: string;
294
314
  floor?: number;
295
315
  pending?: PendingRange[];
316
+ /** `partition` on the counter of a collection whose stores have the option `sequence: "partition"` (roadmap CLI-96). */
317
+ sequence?: string;
318
+ }
319
+
320
+ /**
321
+ * The checks of the sequence of each collection (roadmap CLI-96): by the client, then by the names of the database, the
322
+ * counters and the collection. A check that fails leaves no entry, so the next call tries again.
323
+ */
324
+ const SEQUENCE_CHECKS = new WeakMap<object, Map<string, Promise<void>>>();
325
+
326
+ /** The error of a store whose sequence is not the sequence of its collection (roadmap CLI-96). */
327
+ function sequenceMismatch(collection: string, sequence: "collection" | "partition"): Error {
328
+ return sequence === "partition"
329
+ ? new Error(
330
+ `The collection ${collection} has numbers on its counter of the collection, so the option sequence "partition" refuses it. ` +
331
+ "The option is for a collection with no writes. The move of a live collection is a step with all processes stopped."
332
+ )
333
+ : new Error(`The collection ${collection} has a counter for each partition. Give the option sequence "partition" to each store of it.`);
296
334
  }
297
335
 
298
336
  /** True when a client is on a replica set or a sharded cluster: it has transactions. */
@@ -328,6 +366,37 @@ const QUEUES = new WeakMap<object, Map<string, Promise<unknown>>>();
328
366
  /** The change reads of each client at one time. A burst of reads leaves the pool to the other operations. */
329
367
  const READS = new WeakMap<object, Semaphore>();
330
368
 
369
+ /** The default of the option `writesInFlight`: the most writes of each client at one time (roadmap CLI-96). */
370
+ export const DEFAULT_WRITES_IN_FLIGHT = 8;
371
+
372
+ /** The code of the process warning for a store with `sequence: "partition"` and no bound on its writes. */
373
+ export const UNBOUNDED_PARTITION_WARNING = "PROCEDURES_MONGO_UNBOUNDED_PARTITION";
374
+
375
+ let warnedUnboundedPartition = false;
376
+
377
+ /** This function emits the process warning of an unbounded store with a counter for each partition, one time. */
378
+ function warnUnboundedPartition(): void {
379
+ if (warnedUnboundedPartition) return;
380
+ warnedUnboundedPartition = true;
381
+ process.emitWarning(
382
+ 'writesInFlight: Infinity with sequence: "partition" costs approximately 30 % of the writes a second. Use the default bound of 8.',
383
+ { code: UNBOUNDED_PARTITION_WARNING },
384
+ );
385
+ }
386
+
387
+ /** The permits of the writes of each client, by the value of `writesInFlight`: the stores with one value share the permits. */
388
+ const WRITES = new WeakMap<object, Map<number, Semaphore>>();
389
+
390
+ /** This function gives the permits of the writes of a client for a bound. */
391
+ function writePermitsOf(db: Db, bound: number): Semaphore {
392
+ const owner = ownerOf(db);
393
+ let byBound = WRITES.get(owner);
394
+ if (!byBound) WRITES.set(owner, (byBound = new Map()));
395
+ let permits = byBound.get(bound);
396
+ if (!permits) byBound.set(bound, (permits = semaphore(bound)));
397
+ return permits;
398
+ }
399
+
331
400
  /** This function starts a write after each earlier write of the collection. */
332
401
  function enqueueWrite<R>(db: Db, name: string, task: () => Promise<R>): Promise<R> {
333
402
  const owner = ownerOf(db);
@@ -639,6 +708,8 @@ export interface StorageWriteTarget {
639
708
  withPendingNumbers<R>(db: Db, count: number, write: (first: number) => Promise<R>): Promise<R>;
640
709
  /** This method writes the floor document of the partition once (roadmap O66). */
641
710
  noteFloor(db: Db): Promise<void>;
711
+ /** The sequence of the numbers: one counter for the collection, or one for each partition (roadmap CLI-96). */
712
+ readonly sequence: "collection" | "partition";
642
713
  /** This method drops the oldest tombstones over the limit, and raises the floor of the change log. */
643
714
  prune(db: Db): Promise<void>;
644
715
  }
@@ -756,10 +827,11 @@ function prefilter(selector: StorageSelector, options: SelectorPrefilterOptions,
756
827
  return parts.length === 1 ? parts[0] : { $and: parts };
757
828
  }
758
829
 
759
- /** The sequence counter of a collection. */
830
+ /** The sequence counter of a collection or of a partition. */
760
831
  interface Counter {
761
832
  seq: number;
762
833
  epoch: string;
834
+ sequence?: string;
763
835
  }
764
836
 
765
837
  /** The watermark of a collection, with the floor of the log of a partition. */
@@ -787,6 +859,8 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
787
859
  private readonly maxTombstones: number;
788
860
  private readonly countersName: string;
789
861
  private readonly writeLeaseMs: number;
862
+ /** The most writes of each client at one time, over the stores with this value. `Infinity` is no bound. */
863
+ private readonly writesInFlight: number;
790
864
  /** The storages of the partitions, for `forPartition`: the same storage for the same partition. */
791
865
  private readonly partitions: Map<string, MongoDocumentStorage<T>>;
792
866
  /** The stored form of the documents (roadmap CLI-76). A `ViewMaterializer` of the collection takes the same form. */
@@ -800,6 +874,12 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
800
874
  private indexes: Promise<void> | undefined;
801
875
  /** True after this storage wrote (or found) the floor document of its partition (roadmap O66). A failed write leaves it false. */
802
876
  private floorNoted = false;
877
+ /** The sequence of the numbers (roadmap CLI-96). */
878
+ private readonly sequence: "collection" | "partition";
879
+ /** The `_id` of the counter of the numbers: the collection, or the partition with the option `sequence: "partition"`. */
880
+ private readonly counterId: string;
881
+ /** The partition of the notes of the own numbers with a counter for each partition, else `undefined` (roadmap CLI-96). */
882
+ private readonly ownScope: string | undefined;
803
883
  private closed = false;
804
884
 
805
885
  static {
@@ -818,6 +898,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
818
898
  writeTargetOfStorage = <S>(storage: MongoDocumentStorage<S>): StorageWriteTarget => ({
819
899
  collection: storage.collectionName,
820
900
  counters: storage.countersName,
901
+ sequence: storage.sequence,
821
902
  form: storage.form,
822
903
  partition: storage.partition,
823
904
  writeLeaseMs: storage.writeLeaseMs,
@@ -862,8 +943,18 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
862
943
  this.maxTombstones = options.maxTombstones ?? 10_000;
863
944
  if (!Number.isSafeInteger(this.maxTombstones) || this.maxTombstones < 0) throw new Error("maxTombstones must be a whole number of 0 or more");
864
945
  this.countersName = options.counters ?? "__storage_counters";
946
+ const sequence = options.sequence ?? "collection";
947
+ if (sequence !== "collection" && sequence !== "partition") throw new TypeError(`sequence must be "collection" or "partition", not ${JSON.stringify(sequence)}`);
948
+ this.sequence = sequence;
949
+ this.counterId = sequence === "partition" ? `seq${SEPARATOR}${this.collectionName}${SEPARATOR}${this.partition ?? ""}` : this.collectionName;
950
+ this.ownScope = sequence === "partition" ? (this.partition ?? "") : undefined;
865
951
  this.writeLeaseMs = Math.round(optionalMs(options.writeLease, "writeLease") ?? 60_000);
866
952
  if (!Number.isSafeInteger(this.writeLeaseMs) || this.writeLeaseMs < 1) throw new Error("writeLease must be 1 ms or more");
953
+ this.writesInFlight = options.writesInFlight ?? DEFAULT_WRITES_IN_FLIGHT;
954
+ if (!(this.writesInFlight === Infinity || (Number.isSafeInteger(this.writesInFlight) && this.writesInFlight >= 1))) {
955
+ throw new RangeError(`writesInFlight must be a whole number of 1 or more, or Infinity, not ${String(options.writesInFlight)}`);
956
+ }
957
+ if (this.writesInFlight === Infinity && this.sequence === "partition") warnUnboundedPartition();
867
958
  this.partitions = partitions ?? new Map();
868
959
  }
869
960
 
@@ -1011,7 +1102,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
1011
1102
  */
1012
1103
  private async allocate(db: Db, count: number): Promise<{ first: number; epoch: string }> {
1013
1104
  const counter = (await db.collection<CounterDoc>(this.countersName).findOneAndUpdate(
1014
- { _id: this.collectionName },
1105
+ { _id: this.counterId },
1015
1106
  [
1016
1107
  { $set: { seq: { $add: [{ $ifNull: ["$seq", 0] }, count] }, epoch: { $ifNull: ["$epoch", randomUUID()] } } },
1017
1108
  {
@@ -1025,7 +1116,12 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
1025
1116
  { upsert: true, returnDocument: "after" }
1026
1117
  )) as unknown as Counter | null;
1027
1118
  if (!counter) throw new Error(`The sequence of ${this.collectionName} is not available`);
1028
- return { first: counter.seq - count + 1, epoch: counter.epoch };
1119
+ const first = counter.seq - count + 1;
1120
+ if (this.sequence === "collection" && counter.sequence === "partition") {
1121
+ await this.release(db, first).catch(() => undefined);
1122
+ throw sequenceMismatch(this.collectionName, "collection");
1123
+ }
1124
+ return { first, epoch: counter.epoch };
1029
1125
  }
1030
1126
 
1031
1127
  /**
@@ -1037,7 +1133,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
1037
1133
  private async release(db: Db, first: number): Promise<boolean> {
1038
1134
  const live = { $gt: ["$$this.at", { $subtract: ["$$NOW", this.writeLeaseMs] }] };
1039
1135
  const after = (await db.collection<CounterDoc>(this.countersName).findOneAndUpdate(
1040
- { _id: this.collectionName },
1136
+ { _id: this.counterId },
1041
1137
  [
1042
1138
  {
1043
1139
  $set: {
@@ -1093,19 +1189,22 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
1093
1189
  own = true
1094
1190
  ): Promise<R> {
1095
1191
  const noteOwn = own ? noteOwnNumbers : (): void => undefined;
1192
+ await this.checkSequence(db);
1096
1193
  const client = await this.transactionClient(db);
1097
1194
  if (client) {
1098
1195
  const session = client.startSession();
1099
1196
  try {
1100
1197
  return await session.withTransaction(async () => {
1101
1198
  const counter = (await db.collection<CounterDoc>(this.countersName).findOneAndUpdate(
1102
- { _id: this.collectionName },
1199
+ { _id: this.counterId },
1103
1200
  { $inc: { seq: count }, $setOnInsert: { epoch: randomUUID() } },
1104
1201
  { upsert: true, returnDocument: "after", session }
1105
1202
  )) as unknown as Counter | null;
1106
1203
  if (!counter) throw new Error(`The sequence of ${this.collectionName} is not available`);
1204
+ // The error aborts the transaction, so the counter keeps its number
1205
+ if (this.sequence === "collection" && counter.sequence === "partition") throw sequenceMismatch(this.collectionName, "collection");
1107
1206
  // A watcher of the collection in this process skips the events of these numbers (roadmap CLI-77)
1108
- noteOwn(db, this.collectionName, counter.seq - count + 1, count);
1207
+ noteOwn(db, this.collectionName, counter.seq - count + 1, count, this.ownScope);
1109
1208
  return write(counter.seq - count + 1, session);
1110
1209
  });
1111
1210
  } finally {
@@ -1126,10 +1225,11 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
1126
1225
  collection: Collection<Stored>,
1127
1226
  count: number,
1128
1227
  write: (first: number) => Promise<R>,
1129
- noteOwn: (db: Db, collection: string, first: number, count: number) => void
1228
+ noteOwn: (db: Db, collection: string, first: number, count: number, scope?: string) => void
1130
1229
  ): Promise<R> {
1230
+ await this.checkSequence(db);
1131
1231
  let { first } = await this.allocate(db, count);
1132
- noteOwn(db, this.collectionName, first, count);
1232
+ noteOwn(db, this.collectionName, first, count, this.ownScope);
1133
1233
  let result: R;
1134
1234
  try {
1135
1235
  result = await write(first);
@@ -1141,7 +1241,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
1141
1241
  if (moves >= MAX_MOVES) throw new Error(`A write of ${this.collectionName} stayed late after ${MAX_MOVES} moves to new numbers`);
1142
1242
  // Late: a reader can have passed these numbers. The write takes new ones.
1143
1243
  const next = (await this.allocate(db, count)).first;
1144
- noteOwn(db, this.collectionName, next, count);
1244
+ noteOwn(db, this.collectionName, next, count, this.ownScope);
1145
1245
  await this.move(collection, first, count, next);
1146
1246
  first = next;
1147
1247
  }
@@ -1151,20 +1251,23 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
1151
1251
  /**
1152
1252
  * The watermark of the collection: the highest number below each pending number of a write whose
1153
1253
  * lease has not ended, the epoch, and the floor of the partition. Each number up to it is written
1154
- * or abandoned. One command reads them. Only the first read of a collection writes: it makes the counter.
1254
+ * or abandoned. One command reads them. Only the first read of a store writes: it makes the counter. With the
1255
+ * option `sequence: "partition"`, the first read of each process also marks the counter of the collection.
1155
1256
  */
1156
1257
  private async watermark(db: Db): Promise<Watermark> {
1258
+ await this.checkSequence(db);
1157
1259
  const counters = db.collection<CounterDoc>(this.countersName);
1158
1260
  const floorId = this.floorId();
1159
- const read = (): Promise<Array<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null }>> =>
1261
+ const read = (): Promise<Array<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null; sequence?: string }>> =>
1160
1262
  counters
1161
- .aggregate<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null }>([
1162
- { $match: { _id: { $in: [this.collectionName, floorId] } } },
1263
+ .aggregate<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null; sequence?: string }>([
1264
+ { $match: { _id: { $in: [this.counterId, floorId] } } },
1163
1265
  {
1164
1266
  $project: {
1165
1267
  seq: 1,
1166
1268
  epoch: 1,
1167
1269
  floor: 1,
1270
+ sequence: 1,
1168
1271
  low: {
1169
1272
  $min: {
1170
1273
  $map: {
@@ -1180,20 +1283,54 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
1180
1283
  ])
1181
1284
  .toArray();
1182
1285
  let docs = await read();
1183
- if (!docs.some((doc) => doc._id === this.collectionName)) {
1286
+ if (!docs.some((doc) => doc._id === this.counterId)) {
1184
1287
  // A duplicate key: another read made the counter at the same time
1185
- await counters.updateOne({ _id: this.collectionName }, { $setOnInsert: { seq: 0, epoch: randomUUID() } }, { upsert: true }).catch((error: unknown) => {
1288
+ await counters.updateOne({ _id: this.counterId }, { $setOnInsert: { seq: 0, epoch: randomUUID() } }, { upsert: true }).catch((error: unknown) => {
1186
1289
  if (!isDuplicateKey(error)) throw error;
1187
1290
  });
1188
1291
  docs = await read();
1189
1292
  }
1190
- const mark = docs.find((doc) => doc._id === this.collectionName);
1293
+ const mark = docs.find((doc) => doc._id === this.counterId);
1191
1294
  if (!mark) throw new Error(`The sequence of ${this.collectionName} is not available`);
1295
+ if (this.sequence === "collection" && mark.sequence === "partition") throw sequenceMismatch(this.collectionName, "collection");
1192
1296
  const seq = typeof mark.low === "number" ? Math.min(mark.seq, mark.low - 1) : mark.seq;
1193
1297
  const floor = docs.find((doc) => doc._id === floorId)?.floor;
1194
1298
  return { seq, epoch: mark.epoch, floor: typeof floor === "number" ? floor : 0, held: seq < mark.seq };
1195
1299
  }
1196
1300
 
1301
+ /**
1302
+ * This method checks once for each collection that a store with the option `sequence: "partition"` can use it (roadmap
1303
+ * CLI-96). One atomic step marks the counter of the collection, unless that counter has numbers. A store with the default
1304
+ * reads the mark in its own steps on that counter, so it needs no check of its own.
1305
+ */
1306
+ private checkSequence(db: Db): Promise<void> {
1307
+ if (this.sequence === "collection") return Promise.resolve();
1308
+ const owner = ownerOf(db);
1309
+ let checks = SEQUENCE_CHECKS.get(owner);
1310
+ if (!checks) SEQUENCE_CHECKS.set(owner, (checks = new Map()));
1311
+ const key = `${db.databaseName}\u0000${this.countersName}\u0000${this.collectionName}`;
1312
+ let check = checks.get(key);
1313
+ if (!check) {
1314
+ const mark = async (): Promise<void> => {
1315
+ const counter = await db.collection<CounterDoc>(this.countersName).findOneAndUpdate(
1316
+ { _id: this.collectionName },
1317
+ // A counter with numbers keeps its mark as it is: absent for a collection of the default sequence
1318
+ [{ $set: { sequence: { $cond: [{ $gt: [{ $ifNull: ["$seq", 0] }, 0] }, "$sequence", "partition"] } } }],
1319
+ { upsert: true, returnDocument: "after" }
1320
+ );
1321
+ if (counter?.sequence !== "partition") throw sequenceMismatch(this.collectionName, "partition");
1322
+ };
1323
+ // Two processes can insert the counter at the same time: the second finds it at its next try
1324
+ check = mark().catch((error: unknown) => (isDuplicateKey(error) ? mark() : Promise.reject(error)));
1325
+ checks.set(key, check);
1326
+ const own = checks;
1327
+ check.catch(() => {
1328
+ if (own.get(key) === check) own.delete(key);
1329
+ });
1330
+ }
1331
+ return check;
1332
+ }
1333
+
1197
1334
  /** The id of the document of the oldest kept change of the partition. */
1198
1335
  private async floorOf(db: Db): Promise<number> {
1199
1336
  const doc = await db.collection<CounterDoc>(this.countersName).findOne({ _id: this.floorId() });
@@ -1460,11 +1597,15 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
1460
1597
 
1461
1598
  private async write<R>(task: (db: Db, items: Collection<Stored>) => Promise<R>): Promise<R> {
1462
1599
  const { db, items } = await this.collection();
1463
- return enqueueWrite(db, this.collectionName, async () => {
1600
+ const run = async (): Promise<R> => {
1464
1601
  // The floor document of the partition, outside each transaction of the task (roadmap O66)
1465
1602
  await this.noteFloor(db);
1466
1603
  return task(db, items);
1467
- });
1604
+ };
1605
+ // The permit comes after the turn in the queue, so a write that waits for an earlier write of its counter holds no permit
1606
+ if (this.writesInFlight === Infinity) return enqueueWrite(db, this.counterId, run);
1607
+ const permits = writePermitsOf(db, this.writesInFlight);
1608
+ return enqueueWrite(db, this.counterId, () => permits.run(run));
1468
1609
  }
1469
1610
 
1470
1611
  async set(id: string, value: T, options?: StorageItemWriteOptions): Promise<void> {
@@ -360,6 +360,7 @@ function contentHash(doc: Document): string {
360
360
  interface StorageTargetValues {
361
361
  readonly form: StoredForm;
362
362
  readonly counters: string | undefined;
363
+ readonly sequence: "collection" | "partition" | undefined;
363
364
  readonly writeLeaseMs: number | undefined;
364
365
  readonly maxTombstones: number | undefined;
365
366
  }
@@ -557,7 +558,7 @@ export class ViewMaterializer {
557
558
  /**
558
559
  * A target of `storageTargets` as a storage on the database of the materializer, so its writes
559
560
  * and the transaction of a run use one client. A given storage gives its form, its counters,
560
- * `writeLease` and `maxTombstones`.
561
+ * its sequence, `writeLease` and `maxTombstones`.
561
562
  */
562
563
  private storageTargetOf(collection: string, entry: ViewStorageTargetEntry): StorageTargetEntry {
563
564
  const wrapped = typeof entry === "object" && entry !== null && "target" in entry ? entry : { target: entry };
@@ -568,15 +569,15 @@ export class ViewMaterializer {
568
569
  if (target instanceof MongoDocumentStorage) {
569
570
  const parts = storageWriteTarget(target);
570
571
  if (parts.collection !== collection) throw new TypeError(`The storage target ${collection} has a storage of the collection ${parts.collection}`);
571
- values = { form: parts.form, counters: parts.counters, writeLeaseMs: parts.writeLeaseMs, maxTombstones: parts.maxTombstones };
572
+ values = { form: parts.form, counters: parts.counters, sequence: parts.sequence, writeLeaseMs: parts.writeLeaseMs, maxTombstones: parts.maxTombstones };
572
573
  } else if (typeof target === "object" && target !== null && typeof (target as Partial<StoredForm>).storedId === "function") {
573
- values = { form: target as StoredForm, counters: undefined, writeLeaseMs: undefined, maxTombstones: undefined };
574
+ values = { form: target as StoredForm, counters: undefined, sequence: undefined, writeLeaseMs: undefined, maxTombstones: undefined };
574
575
  } else {
575
576
  throw new TypeError(`The storage target ${collection} needs a MongoDocumentStorage, a stored form or a ViewStorageTarget`);
576
577
  }
577
578
  // The same values give the storage of the last run: its partitions keep their state (the floor document of O66)
578
579
  const cached = this.storageTargetCache.get(collection);
579
- if (cached && cached.form === values.form && cached.counters === values.counters && cached.writeLeaseMs === values.writeLeaseMs && cached.maxTombstones === values.maxTombstones) {
580
+ if (cached && cached.form === values.form && cached.counters === values.counters && cached.sequence === values.sequence && cached.writeLeaseMs === values.writeLeaseMs && cached.maxTombstones === values.maxTombstones) {
580
581
  return { storage: cached.storage, storagePartitionOf };
581
582
  }
582
583
  const db = (): Db => {
@@ -589,6 +590,7 @@ export class ViewMaterializer {
589
590
  form: values.form,
590
591
  transactions: "never",
591
592
  ...(values.counters !== undefined ? { counters: values.counters } : {}),
593
+ ...(values.sequence !== undefined ? { sequence: values.sequence } : {}),
592
594
  ...(values.writeLeaseMs !== undefined ? { writeLease: ms(values.writeLeaseMs) } : {}),
593
595
  ...(values.maxTombstones !== undefined ? { maxTombstones: values.maxTombstones } : {}),
594
596
  });
@@ -35,8 +35,11 @@ const MAX_OWN_WRITES = 100_000;
35
35
 
36
36
  /** The noted writes of one watcher of one collection. */
37
37
  export interface OwnNumbers {
38
- /** True when a write of this process noted the number and no event took it yet. The number is then taken. */
39
- take(seq: number): boolean;
38
+ /**
39
+ * True when a write of this process noted the number and no event took it yet. The number is then taken. With a counter for
40
+ * each partition (roadmap CLI-96), two partitions can have the same number, so give the partition of the event too.
41
+ */
42
+ take(seq: number, partition?: string): boolean;
40
43
  /** True when this process pruned the tombstone of the stored `_id` and no event took it yet. The entry is then taken. */
41
44
  takePruned(storedId: unknown): boolean;
42
45
  /** This method ends the tracking: the storage notes no more entries for it. */
@@ -52,7 +55,8 @@ export interface TrackOwnNumbersOptions {
52
55
  }
53
56
 
54
57
  interface Tracker {
55
- readonly numbers: Map<number, number> | undefined;
58
+ /** The numbers: a number for the counter of the collection, and the partition with the number for a counter of each partition. */
59
+ readonly numbers: Map<number | string, number> | undefined;
56
60
  readonly pruned: Map<string, number>;
57
61
  readonly now: () => number;
58
62
  }
@@ -121,8 +125,9 @@ export function trackOwnNumbers(db: Db, collection: string, options: TrackOwnNum
121
125
  return until > tracker.now();
122
126
  };
123
127
  return {
124
- take(seq: number): boolean {
125
- return tracker.numbers !== undefined && live(tracker.numbers as Map<unknown, number>, seq);
128
+ take(seq: number, partition?: string): boolean {
129
+ const numbers = tracker.numbers as Map<unknown, number> | undefined;
130
+ return numbers !== undefined && (live(numbers, seq) || live(numbers, scopedKey(partition ?? "", seq)));
126
131
  },
127
132
  takePruned(storedId: unknown): boolean {
128
133
  return live(tracker.pruned as Map<unknown, number>, idKey(storedId));
@@ -137,14 +142,22 @@ export function trackOwnNumbers(db: Db, collection: string, options: TrackOwnNum
137
142
  };
138
143
  }
139
144
 
140
- /** This function notes the numbers `first` to `first + count - 1` of a write of this process, for each open tracker of the collection. */
141
- export function noteOwnNumbers(db: Db, collection: string, first: number, count: number): void {
145
+ /** The key of a number of a counter of a partition (roadmap CLI-96). The partition of a store without a partition is the empty text. */
146
+ function scopedKey(partition: string, seq: number): string {
147
+ return `${partition}\u001f${String(seq)}`;
148
+ }
149
+
150
+ /**
151
+ * This function notes the numbers `first` to `first + count - 1` of a write of this process, for each open tracker of the
152
+ * collection. `scope` is the partition of a counter of each partition (roadmap CLI-96), and absent for the counter of the collection.
153
+ */
154
+ export function noteOwnNumbers(db: Db, collection: string, first: number, count: number, scope?: string): void {
142
155
  const trackers = trackersOf(db, collection);
143
156
  if (!trackers) return;
144
157
  for (const tracker of trackers) {
145
158
  if (!tracker.numbers) continue;
146
- const numbers: number[] = [];
147
- for (let seq = first; seq < first + count; seq++) numbers.push(seq);
159
+ const numbers: Array<number | string> = [];
160
+ for (let seq = first; seq < first + count; seq++) numbers.push(scope === undefined ? seq : scopedKey(scope, seq));
148
161
  noteAll(tracker.numbers, numbers, tracker.now());
149
162
  }
150
163
  }
@@ -868,7 +868,7 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
868
868
  if (source.partition !== undefined && item.partition !== source.partition) return;
869
869
  if (!item.raw) {
870
870
  // The event of a write of this process: the collection procedures publish it
871
- if (skipOwn && own && item.seq !== undefined && own.take(item.seq)) return;
871
+ if (skipOwn && own && item.seq !== undefined && own.take(item.seq, item.partition)) return;
872
872
  return publishItem(item);
873
873
  }
874
874
  const before = asDocument(change["fullDocumentBeforeChange"]);