@procedures/mongo 0.48.1 → 0.49.0

package/src/storage/document-storage.ts

@@ -68,6 +68,12 @@
  *   its lease moves its documents to new numbers, so a reader that passed the old numbers gets
  *   it. One case stays on a single server: a process that stops between such a late write and the
  *   move of its documents. Then a reader that passed the old numbers does not get that change.
+ * - A reader can get a write two times, with the same value or a later value. A full answer takes
+ *   its mark before it lists the items, so it can hold a write above its mark, and the next delta
+ *   gives the write again. A reader can also read a late write at its old numbers, and then the move
+ *   gives the write new numbers and a new version. The second case occurs only for a write that ends
+ *   after `writeLease`.
+ * - A version that stops below a pending number has `held`, so a reader knows to read again.
  * - The storage uses a transaction only with a client: the option `client`, or the default
  *   connection when `db` is absent. With `db` and no `client`, each write uses the watermark, also
  *   on a replica set.
@@ -113,10 +119,11 @@ import {
   type StorageWriteOptions,
   type VersionConflict,
 } from "@procedures/collections";
-import { optionalMs, toMs, type Duration } from "@procedures/client";
+import { optionalMs, Semaphore, toMs, type Duration } from "@procedures/client";
 import { ensureConnection } from "../connection.js";
 import { noteOwnNumbers, notePrunedIds, ownerOf } from "./own-writes.js";
 import { boundPipeline, cutResults, DEFAULT_PIPELINE_LIMITS, scopePipeline, type PipelineLimits } from "./pipeline-scope.js";
+import { CHANGE_READ_DIVISOR, poolShare } from "./pool-share.js";
 
 /** The fields of the storage in a stored document. */
 const KEY = "_key";
@@ -259,6 +266,13 @@ function isDuplicateKey(error: unknown): boolean {
   return code === 11000 || [writeErrors].flat().some((entry) => (entry as { code?: unknown } | null)?.code === 11000);
 }
 
+/** The place in an ordered bulk write of the write that a duplicate key stopped, or `undefined` when the error does not give one. */
+function duplicateIndex(error: unknown): number | undefined {
+  const { writeErrors } = (error ?? {}) as { writeErrors?: unknown };
+  const stopped = [writeErrors].flat().find((entry) => (entry as { code?: unknown } | null)?.code === 11000) as { index?: unknown } | undefined;
+  return typeof stopped?.index === "number" ? stopped.index : undefined;
+}
+
 /** A stored document. Its `_id` comes from `storedId`: a text by default, any value but `undefined` and `null` in another form. */
 type Stored = Document;
 
@@ -299,27 +313,70 @@ function hasTransactions(db: Db): Promise<boolean> {
 /** The most moves of a late write to new numbers. */
 const MAX_MOVES = 10;
 
+/** The ends of the work of one collection. These promises do not reject. */
+interface WorkQueue {
+  /** The end of each earlier write and change read. */
+  all: Promise<unknown>;
+  /** The end of the last write. */
+  writes: Promise<unknown>;
+}
+
 /**
- * The writes and the change reads of each collection, one at a time: by the client, then by the names of the
- * database and the collection. A new `Db` object of the same client has the same queue (roadmap CLI-93).
+ * The writes and the change reads of each collection: by the client, then by the names of the database and the
+ * collection. A new `Db` object of the same client has the same queue (roadmap CLI-93). A write goes alone, in the
+ * order of the calls. A change read goes after each earlier write, next to the other change reads. So a read and a
+ * write keep the order of their calls, and a read does not wait for a read (roadmap CLI-91).
  */
-const QUEUES = new WeakMap<object, Map<string, Promise<unknown>>>();
+const QUEUES = new WeakMap<object, Map<string, WorkQueue>>();
+
+/** The change reads of each client at one time. A burst of reads leaves the pool to the other operations. */
+const READS = new WeakMap<object, Semaphore>();
 
-function enqueue<R>(db: Db, name: string, task: () => Promise<R>): Promise<R> {
+function queueOf(db: Db, name: string): { queues: Map<string, WorkQueue>; collection: string; queue: WorkQueue } {
   const owner = ownerOf(db);
   let queues = QUEUES.get(owner);
   if (!queues) QUEUES.set(owner, (queues = new Map()));
   const collection = `${db.databaseName}\u0000${name}`;
-  const before = queues.get(collection) ?? Promise.resolve();
-  const run = before.then(task, task);
-  const settled = run.then(
+  let queue = queues.get(collection);
+  if (!queue) queues.set(collection, (queue = { all: Promise.resolve(), writes: Promise.resolve() }));
+  return { queues, collection, queue };
+}
+
+/** This function drops the queue of a collection when this end is the last of its work. */
+function dropWhenLast(queues: Map<string, WorkQueue>, collection: string, queue: WorkQueue, end: Promise<unknown>): void {
+  void end.then(() => {
+    if (queue.all === end && queues.get(collection) === queue) queues.delete(collection);
+  });
+}
+
+const settledOf = (run: Promise<unknown>): Promise<unknown> =>
+  run.then(
     () => undefined,
     () => undefined
   );
-  queues.set(collection, settled);
-  void settled.then(() => {
-    if (queues.get(collection) === settled) queues.delete(collection);
-  });
+
+/** This function starts a write after each earlier write and change read of the collection. */
+function enqueueWrite<R>(db: Db, name: string, task: () => Promise<R>): Promise<R> {
+  const { queues, collection, queue } = queueOf(db, name);
+  const run = queue.all.then(task);
+  const settled = settledOf(run);
+  queue.all = settled;
+  queue.writes = settled;
+  dropWhenLast(queues, collection, queue, settled);
+  return run;
+}
+
+/** This function starts a change read after each earlier write of the collection, next to the other change reads. */
+function enqueueRead<R>(db: Db, name: string, task: () => Promise<R>): Promise<R> {
+  const { queues, collection, queue } = queueOf(db, name);
+  const owner = ownerOf(db);
+  let reads = READS.get(owner);
+  if (!reads) READS.set(owner, (reads = new Semaphore(poolShare(db, CHANGE_READ_DIVISOR))));
+  const permits = reads;
+  const run = queue.writes.then(() => permits.withPermit(task));
+  const end = Promise.all([queue.all, settledOf(run)]);
+  queue.all = end;
+  dropWhenLast(queues, collection, queue, end);
   return run;
 }
 
@@ -720,6 +777,14 @@ interface Counter {
   epoch: string;
 }
 
+/** The watermark of a collection, with the floor of the log of a partition. */
+interface Watermark extends Counter {
+  /** The floor of the log of the partition: 0 when the partition has no floor document. */
+  floor: number;
+  /** True when a pending write holds the watermark below the counter. */
+  readonly held: boolean;
+}
+
 /**
  * A `CollectionStorage` over MongoDB with flat documents, a change log, versions, queries in the
  * database and partitions (roadmap CLI-24).
@@ -1100,35 +1165,48 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
 
   /**
    * The watermark of the collection: the highest number below each pending number of a write whose
-   * lease has not ended, and the epoch. Each number up to it is written or abandoned.
+   * lease has not ended, the epoch, and the floor of the partition. Each number up to it is written
+   * or abandoned. One command reads them. Only the first read of a collection writes: it makes the counter.
    */
-  private async watermark(db: Db): Promise<Counter> {
+  private async watermark(db: Db): Promise<Watermark> {
     const counters = db.collection<CounterDoc>(this.countersName);
-    await counters.updateOne({ _id: this.collectionName }, { $setOnInsert: { seq: 0, epoch: randomUUID() } }, { upsert: true });
-    const [mark] = await counters
-      .aggregate<{ seq: number; epoch: string; low: number | null }>([
-        { $match: { _id: this.collectionName } },
-        {
-          $project: {
-            seq: 1,
-            epoch: 1,
-            low: {
-              $min: {
-                $map: {
-                  input: {
-                    $filter: { input: { $ifNull: ["$pending", []] }, cond: { $gt: ["$$this.at", { $subtract: ["$$NOW", this.writeLeaseMs] }] } },
+    const floorId = this.floorId();
+    const read = (): Promise<Array<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null }>> =>
+      counters
+        .aggregate<{ _id: string; seq: number; epoch: string; floor?: number; low: number | null }>([
+          { $match: { _id: { $in: [this.collectionName, floorId] } } },
+          {
+            $project: {
+              seq: 1,
+              epoch: 1,
+              floor: 1,
+              low: {
+                $min: {
+                  $map: {
+                    input: {
+                      $filter: { input: { $ifNull: ["$pending", []] }, cond: { $gt: ["$$this.at", { $subtract: ["$$NOW", this.writeLeaseMs] }] } },
+                    },
+                    in: "$$this.from",
                   },
-                  in: "$$this.from",
                 },
               },
             },
           },
-        },
-      ])
-      .toArray();
+        ])
+        .toArray();
+    let docs = await read();
+    if (!docs.some((doc) => doc._id === this.collectionName)) {
+      // A duplicate key: another read made the counter at the same time
+      await counters.updateOne({ _id: this.collectionName }, { $setOnInsert: { seq: 0, epoch: randomUUID() } }, { upsert: true }).catch((error: unknown) => {
+        if (!isDuplicateKey(error)) throw error;
+      });
+      docs = await read();
+    }
+    const mark = docs.find((doc) => doc._id === this.collectionName);
     if (!mark) throw new Error(`The sequence of ${this.collectionName} is not available`);
     const seq = typeof mark.low === "number" ? Math.min(mark.seq, mark.low - 1) : mark.seq;
-    return { seq, epoch: mark.epoch };
+    const floor = docs.find((doc) => doc._id === floorId)?.floor;
+    return { seq, epoch: mark.epoch, floor: typeof floor === "number" ? floor : 0, held: seq < mark.seq };
   }
 
   /** The id of the document of the oldest kept change of the partition. */
@@ -1314,19 +1392,23 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
 
   async getChangesSince(since?: string): Promise<StorageChanges<T>> {
     const { db, items } = await this.collection();
-    return enqueue(db, this.collectionName, async () => {
+    return enqueueRead(db, this.collectionName, async () => {
       const counter = await this.watermark(db);
       const version = `${counter.epoch}.${counter.seq}`;
+      const held = counter.held ? { held: true } : {};
       const full = async (): Promise<StorageChanges<T>> => ({
         version,
         full: true,
         changed: this.decodeAll(await items.find(this.liveFilter()).sort({ [ORD]: 1, _id: 1 }).toArray()),
         deleted: [],
+        ...held,
       });
       const match = since === undefined ? null : /^([^.]+)\.(\d+)$/.exec(since);
       if (!match || match[1] !== counter.epoch) return full();
       const from = Number(match[2]);
-      if (from < (await this.floorOf(db)) || from > counter.seq) return full();
+      if (from < counter.floor || from > counter.seq) return full();
+      // No number is above this version: the read needs no query of the items
+      if (from === counter.seq) return { version, full: false, changed: [], deleted: [], ...held };
       // A number above the watermark can belong to a write that is not visible yet: the next read gives it
       const docs = await items
         .find({ ...this.partitionFilter(), [SEQ]: { $gt: from, $lte: counter.seq } } as Filter<Stored>)
@@ -1342,7 +1424,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
         if (doc[TOMB] === true) deleted.push(id);
         else changed.push(this.decode(doc, id)!);
       }
-      return { version, full: false, changed, deleted };
+      return { version, full: false, changed, deleted, ...held };
     });
   }
 
@@ -1352,7 +1434,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
 
   private async write<R>(task: (db: Db, items: Collection<Stored>) => Promise<R>): Promise<R> {
     const { db, items } = await this.collection();
-    return enqueue(db, this.collectionName, async () => {
+    return enqueueWrite(db, this.collectionName, async () => {
       // The floor document of the partition, outside each transaction of the task (roadmap O66)
       await this.noteFloor(db);
       return task(db, items);
@@ -1380,24 +1462,38 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
       JSON.stringify(value);
     }
     await this.write(async (db, collection) => {
-      const places = await this.placesOf(collection, items.map(([id]) => id));
+      const ids = items.map(([id]) => id);
+      const places = await this.placesOf(collection, ids);
       await this.withNumbers(db, collection, items.length, async (first, session) => {
         const guards = await this.checkVersions(collection, expected, session);
-        const placed = new Map(places);
-        const operations: AnyBulkWriteOperation<Stored>[] = items.map(([id, value], index) => {
-          const seq = first + index;
-          // A new id takes its place at its first write; a later write of the batch keeps it
-          if (!placed.has(id)) placed.set(id, seq);
-          // The guard of an id with a precondition: the write fails when another writer came first
-          const guard = guards.get(id);
-          guards.delete(id);
-          return { replaceOne: { filter: { ...this.writeFilter(id), ...guard }, replacement: this.encode(id, value, seq, placed.get(id)!), upsert: true } };
-        });
-        try {
-          await collection.bulkWrite(operations, { ordered: true, ...(session ? { session } : {}) });
-        } catch (error) {
-          if (expected.length > 0 && !session && isDuplicateKey(error)) throw await this.lateConflict(collection, expected);
-          throw error;
+        const operationsAt = (known: ReadonlyMap<string, number>): AnyBulkWriteOperation<Stored>[] => {
+          const placed = new Map(known);
+          return items.map(([id, value], index) => {
+            const seq = first + index;
+            // A new id takes its place at its first write; a later write of the batch keeps it
+            if (!placed.has(id)) placed.set(id, seq);
+            // The guard of an id with a precondition: the write fails when another writer came first
+            const guard = guards.get(id);
+            guards.delete(id);
+            return { replaceOne: { filter: { ...this.writeFilter(id), ...guard }, replacement: this.encode(id, value, seq, placed.get(id)!), upsert: true } };
+          });
+        };
+        // The race of a new id: another process made it after the query of this upsert, so a new try from the
+        // stopped write is a rewrite, with the same number. These numbers stay pending, so no reader passes them.
+        // A real duplicate stops one write two times, and it reaches the caller. So a batch has at most one try more than it has writes.
+        let known = places;
+        for (let from = 0, tries = 0; ; tries++) {
+          try {
+            await collection.bulkWrite(operationsAt(known).slice(from), { ordered: true, ...(session ? { session } : {}) });
+            return;
+          } catch (error) {
+            if (session || !isDuplicateKey(error)) throw error;
+            if (expected.length > 0) throw await this.lateConflict(collection, expected);
+            const at = duplicateIndex(error);
+            if (at === undefined || (tries > 0 && at === 0)) throw error;
+            from += at;
+            known = await this.placesOf(collection, ids);
+          }
         }
       });
     });
@@ -1410,9 +1506,17 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
       const next = applyPatch(this.decode(doc, id)![1], fields) as T;
       // The encoding is checked before a number of the sequence goes
       this.flatOf(id, next);
-      await this.withNumbers(db, collection, 1, (first, session) =>
-        collection.replaceOne(this.writeFilter(id), this.encode(id, next, first, this.placeOf(doc)), { upsert: true, ...(session ? { session } : {}) })
-      );
+      await this.withNumbers(db, collection, 1, async (first, session) => {
+        const write = (): Promise<unknown> =>
+          collection.replaceOne(this.writeFilter(id), this.encode(id, next, first, this.placeOf(doc)), { upsert: true, ...(session ? { session } : {}) });
+        try {
+          await write();
+        } catch (error) {
+          // The document went after the read, and another process made it again: one new try is a rewrite
+          if (session || !isDuplicateKey(error)) throw error;
+          await write();
+        }
+      });
       return next;
     });
   }

package/src/storage/materializer.ts

@@ -62,6 +62,7 @@ import { BSON, ObjectId, type AnyBulkWriteOperation, type ChangeStream, type Cli
 import { collectionChangeTopic, ms, optionalMs, realClock, Semaphore, writeLog, type Cancel, type Clock, type CollectionChangeEvent, type Duration, type LogFields, type Logger } from "@procedures/client";
 import { DEFAULT_STORED_FORM, DOCUMENT_STORAGE_FIELDS, MongoDocumentStorage, TOMBSTONE_FIELD, storageWriteTarget, type StorageWriteTarget, type StoredForm } from "./document-storage.js";
 import { scopePipeline } from "./pipeline-scope.js";
+import { poolShare } from "./pool-share.js";
 import type { StorageHintPublisher } from "./storage-hints.js";
 
 /** The fields of a storage that a row never keeps: its sequence number and its tombstone flag (roadmap CLI-77). */
@@ -319,9 +320,7 @@ const MOST_PASSES = 32;
  * a time, so two thirds of the pool stay free for the other operations of the process.
  */
 function passLimit(db: Db): number {
-  // A fake database of a test has no client
-  const pool = (db.client as Db["client"] | undefined)?.options.maxPoolSize;
-  return pool ? Math.max(1, Math.min(MOST_PASSES, Math.floor(pool / 3))) : MOST_PASSES;
+  return poolShare(db, 3, MOST_PASSES);
 }
 
 /** The `_id` of a row in the result, from the stored `_id` of `partitionTargetId`. */
@@ -792,7 +791,8 @@ export class ViewMaterializer {
       void this.track(this.onViewChange(change)).catch((error: unknown) => this.log("error", "A change of a view failed", { collection: this.viewsName, operationType: change["operationType"], error }, `a change of a view failed: ${String((error as Error)?.message ?? error)}`));
     });
     stream.on("error", (error: Error) => rewatch(`failed: ${error.message}`));
-    stream.on("close", () => rewatch("closed"));
+    // The cursor of the driver tells its close before the error of a failed read. The close waits one turn, so the log gives the error (roadmap CLI-107)
+    stream.on("close", () => setImmediate(() => rewatch("closed")));
     return openOnServer(stream);
   }
 
@@ -1024,7 +1024,8 @@ export class ViewMaterializer {
       if (watch.stream === stream) this.route(change, watch);
     });
     stream.on("error", (error: Error) => rewatch(`failed: ${error.message}`));
-    stream.on("close", () => rewatch("closed"));
+    // The cursor of the driver tells its close before the error of a failed read. The close waits one turn, so the log gives the error (roadmap CLI-107)
+    stream.on("close", () => setImmediate(() => rewatch("closed")));
     return openOnServer(stream);
   }
 

package/src/storage/pool-share.ts

@@ -0,0 +1,30 @@
+import type { Db } from "mongodb";
+
+/** The pool size of the driver when the client does not tell its size (the default of `maxPoolSize`). */
+export const DEFAULT_POOL_SIZE = 100;
+
+/**
+ * The part of the pool for the change reads of one client: one twelfth, so 8 reads at the default pool.
+ * A burst of change reads waits on round trips over a network. So a share of the pool, not a fixed count, sets the limit.
+ *
+ * The objective: in a burst, the p95 of other queries of the pool stays within 1.5 round trips.
+ * The share of the pool controls that p95 only when the connections wait on the network.
+ * So the suite `read-bound` of `stream-cost.yml` gates the objective at a delay of 20 ms.
+ * There, 8 reads at one time gave 1.01 to 1.44 round trips (run 42105731536). 16 reads gave up to 1.86, and 4 reads gave a burst twice as long.
+ *
+ * At a short round trip, the work of the server sets the p95 of each client of the server.
+ * A limit for one pool only spreads that work. At 5 ms, 8 reads gave up to 3 round trips, and 4 reads gave 2.2 (run 42041953313).
+ * There the suite only reports, with the p95 over the quiet p95 of the same probe.
+ */
+export const CHANGE_READ_DIVISOR = 12;
+
+/**
+ * This function gives the limit of operations of one kind at one time on a pool: the pool size over `divisor`,
+ * from 1 to `most`. Each operation uses one connection at a time. So the other operations of the process find a free
+ * connection in a burst, and a larger pool gives a larger limit.
+ */
+export function poolShare(db: Db, divisor: number, most: number = Number.POSITIVE_INFINITY): number {
+  // A fake database of a test has no client. A pool size of 0 has no limit in the driver: the share then uses the default size
+  const pool = (db.client as Db["client"] | undefined)?.options.maxPoolSize || DEFAULT_POOL_SIZE;
+  return Math.max(1, Math.min(most, Math.floor(pool / divisor)));
+}

package/src/storage/split-events.ts

@@ -0,0 +1,135 @@
+/**
+ * The change events over the BSON limit of 16 MiB (roadmap CLI-107). An event holds the images of
+ * the document before and after its write. Thus a write of a small field can make an event over the
+ * limit. The server then fails the read of that event, and a stream that opens again at the same
+ * token fails again.
+ *
+ * - The stage `$changeStreamSplitLargeEvent` (MongoDB 7.0, and 6.0.9 or later) splits such an event
+ *   into fragments. Each fragment has its own `_id` and the field `splitEvent: { fragment, of }`.
+ *   The stage is the last stage of the pipeline.
+ * - `eventJoiner()` joins the fragments of one event into one event. The joined event has the `_id`
+ *   of its last fragment, and no field `splitEvent`.
+ * - A stream that starts after a fragment gets the fragments after it. The joiner then tells that the
+ *   first fragments are missing.
+ * - `sameTokenFaults()` counts the errors with one code at one resume token. A fault that stays at one
+ *   event comes again at each open. A network error and an error that the driver resumes do not count.
+ */
+import type { Document } from "mongodb";
+import { codeOf } from "./storage-hints.js";
+
+/** The stage that splits a change event over the BSON limit into fragments. It is the last stage of a pipeline. */
+export const SPLIT_LARGE_EVENT_STAGE: Readonly<Document> = Object.freeze({ $changeStreamSplitLargeEvent: {} });
+
+/** The code of a server that does not know a stage of the pipeline: a server before 6.0.9 does not know the split stage. */
+export const UNKNOWN_STAGE_CODE = 40324;
+
+/** The code of a BSON object over the limit: the server cannot give the event. */
+export const OBJECT_TOO_LARGE_CODE = 10334;
+
+/** The count of errors with one code at one token after which the fault does not go away. */
+export const SAME_TOKEN_LIMIT = 5;
+
+/**
+ * The codes of the errors that a getMore of a change stream gives in an election or a network fault.
+ * The driver resumes the stream after these codes, and the server gives them at each token.
+ */
+const RESUMABLE_CODES: ReadonlySet<number> = new Set([6, 7, 63, 89, 91, 133, 150, 189, 234, 262, 9001, 10107, 11600, 11602, 13388, 13435, 13436]);
+
+/** The result of one event of the stream: a whole event, a fragment that waits for the others, or an event without its first fragments. */
+export type JoinResult = { kind: "event"; change: Document } | { kind: "part" } | { kind: "missing"; change: Document; missing: number };
+
+/** The joiner of the fragments of a split event. */
+export interface EventJoiner {
+  /** This method takes the next event of the stream, and gives the event when it is whole. */
+  push(change: Document): JoinResult;
+  /** True while fragments of an event wait for the others: the stream is then in the middle of an event. */
+  readonly partial: boolean;
+}
+
+/** The fragment fields of an event, or undefined for an event that is not split. */
+function splitOf(change: Document): { fragment: number; of: number } | undefined {
+  const split: unknown = change["splitEvent"];
+  if (typeof split !== "object" || split === null) return undefined;
+  const { fragment, of } = split as Document;
+  return typeof fragment === "number" && typeof of === "number" ? { fragment, of } : undefined;
+}
+
+/** This function joins fragments into one event: the fields of each fragment, the `_id` of the last, and no `splitEvent`. */
+function joined(parts: readonly Document[]): Document {
+  const event: Document = Object.assign({}, ...parts);
+  event["_id"] = parts[parts.length - 1]!["_id"];
+  delete event["splitEvent"];
+  return event;
+}
+
+/** This function gives a new joiner. Use one joiner for each stream: a new stream starts with no fragments. */
+export function eventJoiner(): EventJoiner {
+  let parts: Document[] = [];
+  /** The fragments that the stream did not give: the fragments before the first fragment that came. */
+  let missing = 0;
+  return {
+    get partial() {
+      return parts.length > 0;
+    },
+    push(change) {
+      const split = splitOf(change);
+      if (split === undefined) {
+        // An event that is not split ends fragments that wait: the stream cannot give them again
+        parts = [];
+        missing = 0;
+        return { kind: "event", change };
+      }
+      if (parts.length === 0 || split.fragment !== parts.length + missing + 1) {
+        parts = [];
+        missing = split.fragment - 1;
+      }
+      parts.push(change);
+      if (split.fragment < split.of) return { kind: "part" };
+      const event = joined(parts);
+      const lost = missing;
+      parts = [];
+      missing = 0;
+      if (lost > 0) return { kind: "missing", change: { ...event, splitEvent: { missing: lost, of: split.of } }, missing: lost };
+      return { kind: "event", change: event };
+    },
+  };
+}
+
+/** True for an error that the server gives at each token in an election or a network fault. */
+export function isResumable(error: unknown): boolean {
+  const labels = (error as { errorLabels?: unknown } | undefined)?.errorLabels;
+  if (Array.isArray(labels) && labels.includes("ResumableChangeStreamError")) return true;
+  const code = codeOf(error);
+  return code === undefined || RESUMABLE_CODES.has(code);
+}
+
+/** The counter of the errors with one code at one token. */
+export interface SameTokenFaults {
+  /** This method counts an error at a token, and gives the count of errors in a row with its code at that token. A resumable error gives 0. */
+  count(error: unknown, token: unknown): number;
+  /** This method starts the count again: the stream passed the token. */
+  reset(): void;
+}
+
+/** This function gives a new counter of the errors with one code at one token. */
+export function sameTokenFaults(): SameTokenFaults {
+  let key: string | undefined;
+  let repeats = 0;
+  return {
+    count(error, token) {
+      if (isResumable(error)) {
+        key = undefined;
+        repeats = 0;
+        return 0;
+      }
+      const next = `${String(codeOf(error))} ${JSON.stringify(token ?? null)}`;
+      repeats = next === key ? repeats + 1 : 1;
+      key = next;
+      return repeats;
+    },
+    reset() {
+      key = undefined;
+      repeats = 0;
+    },
+  };
+}

package/src/storage/storage-hints.ts

@@ -92,6 +92,7 @@ import {
   type RawStorageWrite,
   type StoredForm,
 } from "./document-storage.js";
+import { SAME_TOKEN_LIMIT, SPLIT_LARGE_EVENT_STAGE, UNKNOWN_STAGE_CODE, eventJoiner, sameTokenFaults } from "./split-events.js";
 import { trackOwnNumbers, type OwnNumbers } from "./own-writes.js";
 
 /** The change of one item of a change event of a `MongoDocumentStorage` collection, from `storageChangeOf()`. */
@@ -197,6 +198,13 @@ export interface WatchStorageChangesOptions {
    * @default { initialDelay: ms(250), maxDelay: seconds(30), backoffMultiplier: 2, jitter: 1 }
    */
   backoff?: ResubscribeBackoff | undefined;
+  /**
+   * True to end the pipeline of the stream with the stage `$changeStreamSplitLargeEvent`. The watcher
+   * then joins the fragments of an event over 16 MiB. With false, such an event fails at each open,
+   * and after 5 failures at its token the subscribers get a gap.
+   * @default true
+   */
+  splitLargeEvents?: boolean | undefined;
   /** The clock of the timers and of the time limit of the own writes (default the clock of the platform). The timers do not keep a process alive. */
   clock?: Clock | undefined;
   /**
@@ -513,6 +521,9 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
   const unplaced = new Set<string>();
   /** The partitions that got a hint: the topics of a gap. */
   const partitions = new Set<string | undefined>();
+  /** True while the pipeline ends with the split stage. A server that does not know the stage sets it to false. */
+  let split = options.splitLargeEvents !== false;
+  const faults = sameTokenFaults();
   let markReady: () => void = () => undefined;
   let failReady: (error: unknown) => void = () => undefined;
   const ready = new Promise<void>((resolve, reject) => {
@@ -827,6 +838,7 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
   const onChange = (change: Document): void => {
     token = change["_id"];
     attempt = 0;
+    faults.reset();
     const type = change["operationType"];
     if (type === "drop" || type === "rename" || type === "dropDatabase") return gap(`the collection ${source.collection} had a ${String(type)}`, { operationType: type, resumeToken: true });
     const ns = asDocument(change["ns"]);
@@ -917,7 +929,7 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
     if (token === undefined && opened) gap(`the stream of ${source.collection} opens again without a resume token`, { resumeToken: false });
     let next: ChangeStream;
     try {
-      next = db.collection(source.collection).watch([], {
+      next = db.collection(source.collection).watch(split ? [SPLIT_LARGE_EVENT_STAGE] : [], {
         fullDocument: "updateLookup",
         fullDocumentBeforeChange: "whenAvailable",
         ...(token !== undefined && { startAfter: token }),
@@ -927,6 +939,9 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
       return schedule(`the stream cannot open (${messageOf(error)})`, error);
     }
     stream = next;
+    const joiner = eventJoiner();
+    /** The token at the open: a reopen at the same token is no success, so the backoff does not start again. */
+    const openToken = JSON.stringify(token ?? null);
     let ended = false;
     const end = (reason: string, error?: unknown): void => {
       if (ended) return;
@@ -942,13 +957,27 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
         log("warn", "The server does not know the resume token", { collection: source.collection, code, resumeToken: true }, `the server does not know the resume token of ${source.collection} (code ${code})`);
         token = undefined;
       }
+      if (code === UNKNOWN_STAGE_CODE && split) {
+        // A server before 6.0.9: the stream opens again without the split stage
+        split = false;
+        log("warn", "The server does not know the stage $changeStreamSplitLargeEvent", { collection: source.collection, code }, `the server does not know the stage $changeStreamSplitLargeEvent: an event of ${source.collection} over 16 MiB gives a gap`);
+      } else if (error !== undefined) {
+        const repeats = faults.count(error, token);
+        if (repeats >= SAME_TOKEN_LIMIT) {
+          // The fault stays at one event: the next stream starts now without the token, and its open gives the gap
+          faults.reset();
+          token = undefined;
+          log("warn", "The stream fails at one token: it opens again without the token", { collection: source.collection, code, repeats }, `the stream of ${source.collection} failed ${repeats} times at one token with the code ${String(code)}: it opens again without the token`);
+        }
+      }
       schedule(`the stream ${reason}`, error);
     };
     next.on("resumeTokenChanged", (resumeToken: unknown) => {
       if (resumeToken === undefined || resumeToken === null) return;
-      token = resumeToken;
+      // In the middle of a split event, the token stays before the event: a stream that opens again gets all its fragments
+      if (!joiner.partial) token = resumeToken;
       opened = true;
-      attempt = 0;
+      if (JSON.stringify(resumeToken) !== openToken) attempt = 0;
       markReady();
       live = true;
       afterQueue();
@@ -958,15 +987,23 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
       markReady();
       live = true;
       afterQueue();
-      if (change["operationType"] === "invalidate") {
+      const result = joiner.push(change);
+      if (result.kind === "part") return;
+      if (result.kind === "missing") {
+        // The stream started after a fragment: the event cannot be read, so the subscribers read the collection in full
+        token = result.change["_id"];
+        return gap(`a split event of ${source.collection} came without its first ${result.missing} fragments`, { missing: result.missing, resumeToken: true });
+      }
+      if (result.change["operationType"] === "invalidate") {
         // The collection was dropped or renamed: the stream starts after the invalidate event
-        token = change["_id"];
+        token = result.change["_id"];
         return end("ended (invalidate)");
       }
-      onChange(change);
+      onChange(result.change);
     });
     next.on("error", (error: Error) => end(`failed: ${error.message}`, error));
-    next.on("close", () => end("closed"));
+    // The cursor of the driver tells its close before the error of a failed read. The end waits one turn, so the error comes first and counts
+    next.on("close", () => setImmediate(() => end("closed")));
   };
 
   void open();