@procedures/mongo 0.48.2 → 0.49.1

package/src/storage/materializer.ts

@@ -62,6 +62,7 @@ import { BSON, ObjectId, type AnyBulkWriteOperation, type ChangeStream, type Cli
 import { collectionChangeTopic, ms, optionalMs, realClock, Semaphore, writeLog, type Cancel, type Clock, type CollectionChangeEvent, type Duration, type LogFields, type Logger } from "@procedures/client";
 import { DEFAULT_STORED_FORM, DOCUMENT_STORAGE_FIELDS, MongoDocumentStorage, TOMBSTONE_FIELD, storageWriteTarget, type StorageWriteTarget, type StoredForm } from "./document-storage.js";
 import { scopePipeline } from "./pipeline-scope.js";
+import { poolShare } from "./pool-share.js";
 import type { StorageHintPublisher } from "./storage-hints.js";
 
 /** The fields of a storage that a row never keeps: its sequence number and its tombstone flag (roadmap CLI-77). */
@@ -319,9 +320,7 @@ const MOST_PASSES = 32;
  * a time, so two thirds of the pool stay free for the other operations of the process.
  */
 function passLimit(db: Db): number {
-  // A fake database of a test has no client
-  const pool = (db.client as Db["client"] | undefined)?.options.maxPoolSize;
-  return pool ? Math.max(1, Math.min(MOST_PASSES, Math.floor(pool / 3))) : MOST_PASSES;
+  return poolShare(db, 3, MOST_PASSES);
 }
 
 /** The `_id` of a row in the result, from the stored `_id` of `partitionTargetId`. */
@@ -792,7 +791,8 @@ export class ViewMaterializer {
       void this.track(this.onViewChange(change)).catch((error: unknown) => this.log("error", "A change of a view failed", { collection: this.viewsName, operationType: change["operationType"], error }, `a change of a view failed: ${String((error as Error)?.message ?? error)}`));
     });
     stream.on("error", (error: Error) => rewatch(`failed: ${error.message}`));
-    stream.on("close", () => rewatch("closed"));
+    // The cursor of the driver tells its close before the error of a failed read. The close waits one turn, so the log gives the error (roadmap CLI-107)
+    stream.on("close", () => setImmediate(() => rewatch("closed")));
     return openOnServer(stream);
   }
 
@@ -1024,7 +1024,8 @@ export class ViewMaterializer {
       if (watch.stream === stream) this.route(change, watch);
     });
     stream.on("error", (error: Error) => rewatch(`failed: ${error.message}`));
-    stream.on("close", () => rewatch("closed"));
+    // The cursor of the driver tells its close before the error of a failed read. The close waits one turn, so the log gives the error (roadmap CLI-107)
+    stream.on("close", () => setImmediate(() => rewatch("closed")));
     return openOnServer(stream);
   }
 

package/src/storage/pool-share.ts

@@ -0,0 +1,30 @@
+import type { Db } from "mongodb";
+
+/** The pool size of the driver when the client does not tell its size (the default of `maxPoolSize`). */
+export const DEFAULT_POOL_SIZE = 100;
+
+/**
+ * The part of the pool for the change reads of one client: one twelfth, so 8 reads at the default pool.
+ * A burst of change reads waits on round trips over a network. So a share of the pool, not a fixed count, sets the limit.
+ *
+ * The objective: in a burst, the p95 of other queries of the pool stays within 1.5 round trips.
+ * The share of the pool controls that p95 only when the connections wait on the network.
+ * So the suite `read-bound` of `stream-cost.yml` gates the objective at a delay of 20 ms.
+ * There, 8 reads at one time gave 1.01 to 1.44 round trips (run 42105731536). 16 reads gave up to 1.86, and 4 reads gave a burst twice as long.
+ *
+ * At a short round trip, the work of the server sets the p95 of each client of the server.
+ * A limit for one pool only spreads that work. At 5 ms, 8 reads gave up to 3 round trips, and 4 reads gave 2.2 (run 42041953313).
+ * There the suite only reports, with the p95 over the quiet p95 of the same probe.
+ */
+export const CHANGE_READ_DIVISOR = 12;
+
+/**
+ * This function gives the limit of operations of one kind at one time on a pool: the pool size over `divisor`,
+ * from 1 to `most`. Each operation uses one connection at a time. So the other operations of the process find a free
+ * connection in a burst, and a larger pool gives a larger limit.
+ */
+export function poolShare(db: Db, divisor: number, most: number = Number.POSITIVE_INFINITY): number {
+  // A fake database of a test has no client. A pool size of 0 has no limit in the driver: the share then uses the default size
+  const pool = (db.client as Db["client"] | undefined)?.options.maxPoolSize || DEFAULT_POOL_SIZE;
+  return Math.max(1, Math.min(most, Math.floor(pool / divisor)));
+}

package/src/storage/split-events.ts

@@ -0,0 +1,135 @@
+/**
+ * The change events over the BSON limit of 16 MiB (roadmap CLI-107). An event holds the images of
+ * the document before and after its write. Thus a write of a small field can make an event over the
+ * limit. The server then fails the read of that event, and a stream that opens again at the same
+ * token fails again.
+ *
+ * - The stage `$changeStreamSplitLargeEvent` (MongoDB 7.0, and 6.0.9 or later) splits such an event
+ *   into fragments. Each fragment has its own `_id` and the field `splitEvent: { fragment, of }`.
+ *   The stage is the last stage of the pipeline.
+ * - `eventJoiner()` joins the fragments of one event into one event. The joined event has the `_id`
+ *   of its last fragment, and no field `splitEvent`.
+ * - A stream that starts after a fragment gets the fragments after it. The joiner then tells that the
+ *   first fragments are missing.
+ * - `sameTokenFaults()` counts the errors with one code at one resume token. A fault that stays at one
+ *   event comes again at each open. A network error and an error that the driver resumes do not count.
+ */
+import type { Document } from "mongodb";
+import { codeOf } from "./storage-hints.js";
+
+/** The stage that splits a change event over the BSON limit into fragments. It is the last stage of a pipeline. */
+export const SPLIT_LARGE_EVENT_STAGE: Readonly<Document> = Object.freeze({ $changeStreamSplitLargeEvent: {} });
+
+/** The code of a server that does not know a stage of the pipeline: a server before 6.0.9 does not know the split stage. */
+export const UNKNOWN_STAGE_CODE = 40324;
+
+/** The code of a BSON object over the limit: the server cannot give the event. */
+export const OBJECT_TOO_LARGE_CODE = 10334;
+
+/** The count of errors with one code at one token after which the fault does not go away. */
+export const SAME_TOKEN_LIMIT = 5;
+
+/**
+ * The codes of the errors that a getMore of a change stream gives in an election or a network fault.
+ * The driver resumes the stream after these codes, and the server gives them at each token.
+ */
+const RESUMABLE_CODES: ReadonlySet<number> = new Set([6, 7, 63, 89, 91, 133, 150, 189, 234, 262, 9001, 10107, 11600, 11602, 13388, 13435, 13436]);
+
+/** The result of one event of the stream: a whole event, a fragment that waits for the others, or an event without its first fragments. */
+export type JoinResult = { kind: "event"; change: Document } | { kind: "part" } | { kind: "missing"; change: Document; missing: number };
+
+/** The joiner of the fragments of a split event. */
+export interface EventJoiner {
+  /** This method takes the next event of the stream, and gives the event when it is whole. */
+  push(change: Document): JoinResult;
+  /** True while fragments of an event wait for the others: the stream is then in the middle of an event. */
+  readonly partial: boolean;
+}
+
+/** The fragment fields of an event, or undefined for an event that is not split. */
+function splitOf(change: Document): { fragment: number; of: number } | undefined {
+  const split: unknown = change["splitEvent"];
+  if (typeof split !== "object" || split === null) return undefined;
+  const { fragment, of } = split as Document;
+  return typeof fragment === "number" && typeof of === "number" ? { fragment, of } : undefined;
+}
+
+/** This function joins fragments into one event: the fields of each fragment, the `_id` of the last, and no `splitEvent`. */
+function joined(parts: readonly Document[]): Document {
+  const event: Document = Object.assign({}, ...parts);
+  event["_id"] = parts[parts.length - 1]!["_id"];
+  delete event["splitEvent"];
+  return event;
+}
+
+/** This function gives a new joiner. Use one joiner for each stream: a new stream starts with no fragments. */
+export function eventJoiner(): EventJoiner {
+  let parts: Document[] = [];
+  /** The fragments that the stream did not give: the fragments before the first fragment that came. */
+  let missing = 0;
+  return {
+    get partial() {
+      return parts.length > 0;
+    },
+    push(change) {
+      const split = splitOf(change);
+      if (split === undefined) {
+        // An event that is not split ends fragments that wait: the stream cannot give them again
+        parts = [];
+        missing = 0;
+        return { kind: "event", change };
+      }
+      if (parts.length === 0 || split.fragment !== parts.length + missing + 1) {
+        parts = [];
+        missing = split.fragment - 1;
+      }
+      parts.push(change);
+      if (split.fragment < split.of) return { kind: "part" };
+      const event = joined(parts);
+      const lost = missing;
+      parts = [];
+      missing = 0;
+      if (lost > 0) return { kind: "missing", change: { ...event, splitEvent: { missing: lost, of: split.of } }, missing: lost };
+      return { kind: "event", change: event };
+    },
+  };
+}
+
+/** True for an error that the server gives at each token in an election or a network fault. */
+export function isResumable(error: unknown): boolean {
+  const labels = (error as { errorLabels?: unknown } | undefined)?.errorLabels;
+  if (Array.isArray(labels) && labels.includes("ResumableChangeStreamError")) return true;
+  const code = codeOf(error);
+  return code === undefined || RESUMABLE_CODES.has(code);
+}
+
+/** The counter of the errors with one code at one token. */
+export interface SameTokenFaults {
+  /** This method counts an error at a token, and gives the count of errors in a row with its code at that token. A resumable error gives 0. */
+  count(error: unknown, token: unknown): number;
+  /** This method starts the count again: the stream passed the token. */
+  reset(): void;
+}
+
+/** This function gives a new counter of the errors with one code at one token. */
+export function sameTokenFaults(): SameTokenFaults {
+  let key: string | undefined;
+  let repeats = 0;
+  return {
+    count(error, token) {
+      if (isResumable(error)) {
+        key = undefined;
+        repeats = 0;
+        return 0;
+      }
+      const next = `${String(codeOf(error))} ${JSON.stringify(token ?? null)}`;
+      repeats = next === key ? repeats + 1 : 1;
+      key = next;
+      return repeats;
+    },
+    reset() {
+      key = undefined;
+      repeats = 0;
+    },
+  };
+}

package/src/storage/storage-hints.ts

@@ -92,6 +92,7 @@ import {
   type RawStorageWrite,
   type StoredForm,
 } from "./document-storage.js";
+import { SAME_TOKEN_LIMIT, SPLIT_LARGE_EVENT_STAGE, UNKNOWN_STAGE_CODE, eventJoiner, sameTokenFaults } from "./split-events.js";
 import { trackOwnNumbers, type OwnNumbers } from "./own-writes.js";
 
 /** The change of one item of a change event of a `MongoDocumentStorage` collection, from `storageChangeOf()`. */
@@ -197,6 +198,13 @@ export interface WatchStorageChangesOptions {
    * @default { initialDelay: ms(250), maxDelay: seconds(30), backoffMultiplier: 2, jitter: 1 }
    */
   backoff?: ResubscribeBackoff | undefined;
+  /**
+   * True to end the pipeline of the stream with the stage `$changeStreamSplitLargeEvent`. The watcher
+   * then joins the fragments of an event over 16 MiB. With false, such an event fails at each open,
+   * and after 5 failures at its token the subscribers get a gap.
+   * @default true
+   */
+  splitLargeEvents?: boolean | undefined;
   /** The clock of the timers and of the time limit of the own writes (default the clock of the platform). The timers do not keep a process alive. */
   clock?: Clock | undefined;
   /**
@@ -513,6 +521,9 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
   const unplaced = new Set<string>();
   /** The partitions that got a hint: the topics of a gap. */
   const partitions = new Set<string | undefined>();
+  /** True while the pipeline ends with the split stage. A server that does not know the stage sets it to false. */
+  let split = options.splitLargeEvents !== false;
+  const faults = sameTokenFaults();
   let markReady: () => void = () => undefined;
   let failReady: (error: unknown) => void = () => undefined;
   const ready = new Promise<void>((resolve, reject) => {
@@ -827,6 +838,7 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
   const onChange = (change: Document): void => {
     token = change["_id"];
     attempt = 0;
+    faults.reset();
     const type = change["operationType"];
     if (type === "drop" || type === "rename" || type === "dropDatabase") return gap(`the collection ${source.collection} had a ${String(type)}`, { operationType: type, resumeToken: true });
     const ns = asDocument(change["ns"]);
@@ -917,7 +929,7 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
     if (token === undefined && opened) gap(`the stream of ${source.collection} opens again without a resume token`, { resumeToken: false });
     let next: ChangeStream;
     try {
-      next = db.collection(source.collection).watch([], {
+      next = db.collection(source.collection).watch(split ? [SPLIT_LARGE_EVENT_STAGE] : [], {
         fullDocument: "updateLookup",
         fullDocumentBeforeChange: "whenAvailable",
         ...(token !== undefined && { startAfter: token }),
@@ -927,6 +939,9 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
       return schedule(`the stream cannot open (${messageOf(error)})`, error);
     }
     stream = next;
+    const joiner = eventJoiner();
+    /** The token at the open: a reopen at the same token is no success, so the backoff does not start again. */
+    const openToken = JSON.stringify(token ?? null);
     let ended = false;
     const end = (reason: string, error?: unknown): void => {
       if (ended) return;
@@ -942,13 +957,27 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
         log("warn", "The server does not know the resume token", { collection: source.collection, code, resumeToken: true }, `the server does not know the resume token of ${source.collection} (code ${code})`);
         token = undefined;
       }
+      if (code === UNKNOWN_STAGE_CODE && split) {
+        // A server before 6.0.9: the stream opens again without the split stage
+        split = false;
+        log("warn", "The server does not know the stage $changeStreamSplitLargeEvent", { collection: source.collection, code }, `the server does not know the stage $changeStreamSplitLargeEvent: an event of ${source.collection} over 16 MiB gives a gap`);
+      } else if (error !== undefined) {
+        const repeats = faults.count(error, token);
+        if (repeats >= SAME_TOKEN_LIMIT) {
+          // The fault stays at one event: the next stream starts now without the token, and its open gives the gap
+          faults.reset();
+          token = undefined;
+          log("warn", "The stream fails at one token: it opens again without the token", { collection: source.collection, code, repeats }, `the stream of ${source.collection} failed ${repeats} times at one token with the code ${String(code)}: it opens again without the token`);
+        }
+      }
       schedule(`the stream ${reason}`, error);
     };
     next.on("resumeTokenChanged", (resumeToken: unknown) => {
       if (resumeToken === undefined || resumeToken === null) return;
-      token = resumeToken;
+      // In the middle of a split event, the token stays before the event: a stream that opens again gets all its fragments
+      if (!joiner.partial) token = resumeToken;
       opened = true;
-      attempt = 0;
+      if (JSON.stringify(resumeToken) !== openToken) attempt = 0;
       markReady();
       live = true;
       afterQueue();
@@ -958,15 +987,23 @@ export function watchStorageChanges<T>(storage: MongoDocumentStorage<T>, publish
       markReady();
       live = true;
       afterQueue();
-      if (change["operationType"] === "invalidate") {
+      const result = joiner.push(change);
+      if (result.kind === "part") return;
+      if (result.kind === "missing") {
+        // The stream started after a fragment: the event cannot be read, so the subscribers read the collection in full
+        token = result.change["_id"];
+        return gap(`a split event of ${source.collection} came without its first ${result.missing} fragments`, { missing: result.missing, resumeToken: true });
+      }
+      if (result.change["operationType"] === "invalidate") {
         // The collection was dropped or renamed: the stream starts after the invalidate event
-        token = change["_id"];
+        token = result.change["_id"];
         return end("ended (invalidate)");
       }
-      onChange(change);
+      onChange(result.change);
     });
     next.on("error", (error: Error) => end(`failed: ${error.message}`, error));
-    next.on("close", () => end("closed"));
+    // The cursor of the driver tells its close before the error of a failed read. The end waits one turn, so the error comes first and counts
+    next.on("close", () => setImmediate(() => end("closed")));
   };
 
   void open();