@procedures/mongo 0.34.0 → 0.35.1

package/dist/storage/storage-hints.js

@@ -0,0 +1,876 @@
+/**
+ * The change hints of a `MongoDocumentStorage` collection (roadmap CLI-77). `watchStorageChanges()`
+ * watches the change stream of the collection, and publishes each change as a
+ * `CollectionChangeEvent` on the topic of the collection procedures. Thus a subscriber also gets
+ * the writes that did not go through the procedures: a raw write, a script, another process, a
+ * view.
+ *
+ * - The topic is `collectionChangeTopic(name, partition)`, as the procedures publish it: the
+ *   partition of a document comes from `storage.form.partitionOf()`.
+ * - The event is a hint, without `since` and without `version` (the version of the change log): the
+ *   subscriber reads the changes with `changes`, as for each event without `since`. The event has
+ *   the id in `changed` or in `deleted`, and the version of the item in `versions` (a field of
+ *   `CollectionChangeEvent`): the version that `storage.versionOf(id)` gives after the write.
+ * - A write through the storage gives the document a new `_seq`. The watcher reads that from the
+ *   event, never from the `fullDocument` of the lookup: an update with `_seq` in
+ *   `updateDescription.updatedFields`, an insert or a replace with `_seq`. With the image before
+ *   the change, a replace whose `_seq` is the `_seq` before is a raw write (a copy). Without the
+ *   image, a replace with `_seq` counts as a write of the storage.
+ * - A raw write (D39), with `rawWrites: "adopt"` (the default): the watcher publishes nothing for
+ *   its event. The storage adopts it: one conditional write through the path of each write gives
+ *   the document a new `_seq` (and the storage fields and partition fields that a raw replace took
+ *   away), so `changes(since)` has it. The event of the adoption is a write of the storage: its
+ *   hint goes out, with the new version. When two watchers adopt the same write, or a write of the
+ *   storage comes first, one write wins and the others do nothing. A raw write never gets the old
+ *   version. With `rawWrites: "ignore"` (for example for a user that can only read), the hint of a
+ *   raw write goes out without a version, and `changes(since)` does not have the write.
+ * - The adoptions wait in one queue for each watcher: one adoption at a time, and the raw writes
+ *   that wait together take one block of numbers for each partition. An adoption that fails with
+ *   an error that can go away tries again after the backoff. When the queue is full, the hint of
+ *   the next raw write goes out without a version. The watcher keeps the ids of the raw writes that
+ *   did not fit or that failed (at most 10,000), and adopts them again when the queue is empty,
+ *   after a sweep. An adoption that fails for good (after its last try, or a second time), or an id
+ *   over that limit, gives its partition a reset hint (`reset: true`): its subscribers read it in full.
+ * - The adoption is exact only with the images before the change (`changeStreamPreAndPostImages`).
+ *   An insert and a replace show their document, but an update shows only its fields: without the
+ *   image, the `_seq` that the update kept comes from the lookup. When a write of the storage comes
+ *   after the raw update, the adoption can then give that write a newer version, and a client that
+ *   holds the version of that write gets a conflict of its precondition.
+ * - The sweep: after each gap (`sweep`, on by default with `rawWrites: "adopt"`), after a full
+ *   queue, and when the stream opens (`sweepAtOpen`, off by default), the watcher sends each
+ *   document without `_seq` to the same queue, in pages. Thus a document without `_seq` gets its
+ *   number. A raw update of a document with `_seq` keeps that `_seq`: no sweep finds it, so a gap
+ *   publishes a reset hint. The scan walks the index of `_id` and reads each document of the
+ *   collection: a partial index cannot express `$exists: false`. A document that the sweep cannot
+ *   place (no id, or another partition) goes to the log once, and the next sweeps leave it.
+ * - A raw hard delete of an item: with the image before the change, the storage adopts it as a
+ *   tombstone. Without the image (no `changeStreamPreAndPostImages` on the collection), the hint
+ *   goes out without a version, and the delete stays out of `changes(since)`.
+ * - A delete of a tombstone (the storage prunes the oldest ones) deletes no item: no hint, no
+ *   adoption. The image before the change tells it. Without the image, the watcher knows the
+ *   prunes of this process only.
+ * - The writes of this process through the storage: with `ownWrites: "skip"` (the default), the
+ *   watcher skips their events, because the collection procedures publish them already, with
+ *   `since`. Thus a subscriber gets one event for each write. With `ownWrites: "publish"`, the
+ *   watcher publishes each change. Use it when the writes of this process do not go through the
+ *   procedures, or when the procedures publish no events (`changeEvents: false`). An adoption is
+ *   never an own write.
+ * - After an error of the stream, the watcher opens the stream again after a backoff, from the
+ *   resume token of the last event: it loses no change. When the server does not have the token
+ *   any more, or the collection was dropped or renamed, the watcher cannot know what it missed: it
+ *   publishes a reset hint (a gap: no ids, `reset: true`) on the plain topic and on the topic of
+ *   each partition that it knows: the partitions of the counters, of `forPartition()`, of the
+ *   hints, of the subscribed topics of the bus, and of `gapPartitions`. The counters have a floor
+ *   document for each partition with a write through a storage. The subscribers read the
+ *   collection in full.
+ * - The pipeline of the stream is `[]`: the storage of a partition (`forPartition()`) watches the
+ *   whole collection and drops the events of other partitions.
+ * - Change streams need a replica set or a sharded cluster. The watcher asks the server (`hello`)
+ *   before the first open: on a single server, `ready` rejects (code 40573).
+ */
+import { MongoAPIError, MongoParseError, BSON } from "mongodb";
+import { collectionChangeTopic, collectionTopicPartition, optionalMs, realClock, } from "@procedures/client";
+import { DEFAULT_STORED_FORM, TOMBSTONE_FIELD, storageChangeSource, versionOfStored, } from "./document-storage.js";
+import { trackOwnNumbers } from "./own-writes.js";
+/** The fields of the storage that the change of an item reads. */
+const SEQ = "_seq";
+const KEY = "_key";
+/** The codes of the errors after which the server does not know the resume token: the watcher opens a new stream and publishes a gap. */
+const LOST_TOKEN_CODES = new Set([260, 280, 286]);
+/** The codes of a refused access: Unauthorized, AuthenticationFailed, and the access error of Atlas. */
+const REFUSED_CODES = new Set([13, 18, 8000]);
+/** The codes of the errors of a first open that do not go away: a refused access, no change streams on a single server (40573), a command that the server does not know (115). */
+const PERMANENT_CODES = new Set([...REFUSED_CODES, 40573, 115]);
+/** The most partitions whose topics get the hint of a gap. */
+const MAX_GAP_PARTITIONS = 10_000;
+/** The most raw writes of one adoption: they take one block of numbers for each partition. */
+const ADOPT_BATCH = 100;
+/** The most raw writes that wait for an adoption. The hint of a raw write over the limit goes out without a version. */
+const MAX_ADOPT_QUEUE = 10_000;
+/** The tries of an adoption that fails with an error that can go away. */
+const ADOPT_TRIES = 5;
+/** The documents of one page of the sweep. */
+const SWEEP_PAGE = 500;
+/** The most raw writes that the watcher keeps to adopt again. Over the limit, the partition gets a reset hint. */
+const MAX_ADOPT_REDO = 10_000;
+/** The most documents that the sweep cannot place and that the next sweeps leave. */
+const MAX_UNPLACED = 10_000;
+/** The hint of a gap: no ids, and the subscribers read the collection in full. */
+function resetHint() {
+    return { changed: [], deleted: [], reset: true };
+}
+/** The code of a command that the server does not know (an old server without `hello`). */
+const COMMAND_NOT_FOUND = 59;
+function asDocument(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
+}
+function seqOf(doc) {
+    const seq = doc?.[SEQ];
+    return typeof seq === "number" ? seq : undefined;
+}
+/** The new `_seq` of a write of the storage, read from the event, or undefined for a raw write. */
+function writtenSeq(type, change, after, before) {
+    if (type === "update")
+        return seqOf(asDocument(asDocument(change["updateDescription"])?.["updatedFields"]));
+    const seq = seqOf(after);
+    // A replace that copies the `_seq` of the document before it is a raw write. It needs the image before the change.
+    if (seq !== undefined && type === "replace" && before !== undefined && seqOf(before) === seq)
+        return undefined;
+    return seq;
+}
+/** The `_seq` of the document after a raw write: the state that the event saw. */
+function keptSeq(type, change, after, before) {
+    if (type !== "update")
+        return seqOf(after);
+    const removed = asDocument(change["updateDescription"])?.["removedFields"];
+    if (Array.isArray(removed) && removed.includes(SEQ))
+        return undefined;
+    // An update that did not touch `_seq` keeps the `_seq` before it. Without the image, the lookup tells it.
+    return seqOf(before ?? after);
+}
+/**
+ * This function reads the change of an item from a change event of a `MongoDocumentStorage`
+ * collection (roadmap CLI-77), for an app that watches the collection itself. It gives `undefined`
+ * for an event of no item: another operation (for example `drop`), an update whose document is
+ * gone (its delete event follows), the delete of a tombstone, or a document whose id neither
+ * `idOf` nor `_key` tells.
+ *
+ * - The partition is `form.partitionOf()` of `fullDocumentBeforeChange`, else of `fullDocument`,
+ *   else of `documentKey`: the first whose partition gives the id with `form.idOf()` of the stored
+ *   `_id`. Else the id is `_key`, in the partition of the first document.
+ * - A tombstone and a delete are deletes.
+ * - `raw` tells a raw write, from the event only (see the module comment). A write of the storage
+ *   has the version of its new `_seq`. A raw write has no version.
+ *
+ * @param change - A change event of the collection, with `fullDocument: "updateLookup"` for the updates
+ * @param form - The stored form of the storage, for example `storage.form` (default `DEFAULT_STORED_FORM`)
+ */
+export function storageChangeOf(change, form = DEFAULT_STORED_FORM) {
+    const type = change["operationType"];
+    if (type !== "insert" && type !== "replace" && type !== "update" && type !== "delete")
+        return undefined;
+    const after = type === "delete" ? undefined : asDocument(change["fullDocument"]);
+    const before = asDocument(change["fullDocumentBeforeChange"]);
+    const key = asDocument(change["documentKey"]);
+    // The partition: of the image before the change, else of the document after it, else of the stored _id (the order of the adoption)
+    const places = [before, after, key].filter((doc) => doc !== undefined);
+    if (places.length === 0)
+        return undefined;
+    const storedId = (key ?? places[0])["_id"];
+    let partition;
+    let id;
+    for (const doc of places) {
+        const place = form.partitionOf(doc);
+        const fromId = form.idOf(storedId, place);
+        if (typeof fromId === "string") {
+            partition = place;
+            id = fromId;
+            break;
+        }
+    }
+    if (id === undefined) {
+        // No partition tells the id through the form: the id is `_key`, in the first partition
+        const keyField = (after ?? before)?.[KEY];
+        if (typeof keyField !== "string")
+            return undefined;
+        partition = form.partitionOf(places[0]);
+        id = keyField;
+    }
+    if (!after) {
+        // An update whose document is gone: the delete event follows. A delete of a tombstone deletes no item.
+        if (type !== "delete" || before?.[TOMBSTONE_FIELD] === true)
+            return undefined;
+        // The storage deletes an item with a tombstone: a delete of an item is a raw write
+        return { id, partition, deleted: true, raw: true };
+    }
+    const deleted = after[TOMBSTONE_FIELD] === true;
+    const written = writtenSeq(type, change, after, before);
+    if (written === undefined) {
+        const seq = keptSeq(type, change, after, before);
+        return { id, partition, deleted, raw: true, ...(seq !== undefined && { seq }) };
+    }
+    if (deleted)
+        return { id, partition, deleted: true, raw: false, seq: written };
+    const version = versionOfStored({ ...after, [SEQ]: written }, form, partition) ?? String(written);
+    return { id, partition, deleted: false, raw: false, version, seq: written };
+}
+/** The delay before try `attempt` (0 for the first) of a backoff. */
+function backoffOf(options) {
+    const initial = optionalMs(options?.initialDelay, "backoff.initialDelay") ?? 250;
+    const max = optionalMs(options?.maxDelay, "backoff.maxDelay") ?? 30_000;
+    const factor = options?.backoffMultiplier ?? 2;
+    const jitter = options?.jitter ?? 1;
+    if (!(initial >= 0) || !(max >= initial))
+        throw new RangeError("backoff: initialDelay must be 0 or more, and maxDelay must be initialDelay or more");
+    if (!(factor >= 1))
+        throw new RangeError("backoff: backoffMultiplier must be 1 or more");
+    if (!(jitter >= 0 && jitter <= 1))
+        throw new RangeError("backoff: jitter must be from 0 to 1");
+    return (attempt) => {
+        const full = Math.min(max, initial * factor ** attempt);
+        return full - full * jitter * Math.random();
+    };
+}
+/** The message of an error. */
+function messageOf(error) {
+    return String(error?.message ?? error);
+}
+function codeOf(error) {
+    const code = error?.code;
+    return typeof code === "number" ? code : undefined;
+}
+/** True for a refused access: the user may not write (or read) the collection. */
+function isRefused(error) {
+    const code = codeOf(error);
+    return code !== undefined && REFUSED_CODES.has(code);
+}
+/** True for an error of a first open that does not go away: a single server, a refused access, a wrong option. */
+function isPermanent(error) {
+    const code = codeOf(error);
+    if (code !== undefined && PERMANENT_CODES.has(code))
+        return true;
+    return error instanceof MongoAPIError || error instanceof MongoParseError;
+}
+/** A stored `_id` for the log. */
+function describeId(storedId) {
+    try {
+        return BSON.EJSON.stringify({ _id: storedId }, { relaxed: false });
+    }
+    catch {
+        return String(storedId);
+    }
+}
+/** The key of a stored `_id` in the sets of the watcher, or undefined without an id. */
+function idKey(storedId) {
+    return storedId === undefined || storedId === null ? undefined : describeId(storedId);
+}
+/**
+ * This function watches the change stream of the collection of a `MongoDocumentStorage`, and
+ * publishes a hint of each change of an item on the topic of the collection procedures (roadmap
+ * CLI-77). The writes that do not go through the procedures (a raw write, another process, a
+ * script) then reach the subscribers too, and the storage adopts each raw write (D39). See the
+ * module comment for the rules.
+ *
+ * - The storage of a partition (`forPartition()`) publishes only the documents of its partition.
+ * - The watcher reads only the collection of the items. The documents of the counters
+ *   (`__storage_counters`) are in another collection, and a change there is never a hint.
+ * - An error of the publisher goes to the log. It never stops the watcher.
+ * - The adoption of raw writes needs write access. A refused adoption goes to the log once, and
+ *   the hint of that write goes out without a version: set `rawWrites: "ignore"` for a user that
+ *   can only read.
+ * - The adoption is exact only with `changeStreamPreAndPostImages` on the collection. Without the
+ *   image, a raw update and a later write of the storage can give that write an extra version.
+ *
+ * @param storage - The storage, for its collection, its database and its stored form
+ * @param publish - The function or the bus of the events, for example the bus of the procedures
+ * @param options - The topic, the own writes, the raw writes, the gap, the backoff, the clock and the log
+ * @returns The handle: `ready` and `stop()`
+ *
+ * @example
+ * ```typescript
+ * const notes = new MongoDocumentStorage<Note>({ collection: "notes" });
+ * const watch = watchStorageChanges(notes, bus); // the bus of the procedures
+ * await watch.ready;
+ * // ... a raw write of a note reaches the subscribers of collections.notes.<partition>
+ * await watch.stop();
+ * ```
+ */
+export function watchStorageChanges(storage, publish, options = {}) {
+    const source = storageChangeSource(storage);
+    const name = options.collection ?? source.collection;
+    const topicOf = options.topic ?? ((collection, partition) => collectionChangeTopic(collection, partition));
+    const clock = options.clock ?? realClock;
+    const log = options.log ?? ((level, message) => console[level === "info" ? "log" : level](`[storage-hints] ${message}`));
+    const emit = typeof publish === "function" ? publish : publish.emit.bind(publish);
+    // The subscribed topics of the bus name the partitions of a gap. A custom topic rule cannot be read back.
+    const channels = typeof publish === "object" && typeof publish.channels === "function" && options.topic === undefined ? publish.channels.bind(publish) : undefined;
+    const delayOf = backoffOf(options.backoff);
+    if (options.ownWrites !== undefined && options.ownWrites !== "skip" && options.ownWrites !== "publish")
+        throw new TypeError('ownWrites must be "skip" or "publish"');
+    if (options.rawWrites !== undefined && options.rawWrites !== "adopt" && options.rawWrites !== "ignore")
+        throw new TypeError('rawWrites must be "adopt" or "ignore"');
+    const skipOwn = (options.ownWrites ?? "skip") === "skip";
+    const adoptRaw = (options.rawWrites ?? "adopt") === "adopt";
+    if (options.sweep !== undefined && typeof options.sweep !== "boolean")
+        throw new TypeError("sweep must be a boolean");
+    if (options.sweep === true && !adoptRaw)
+        throw new TypeError('sweep needs rawWrites: "adopt"');
+    if (options.sweepAtOpen !== undefined && typeof options.sweepAtOpen !== "boolean")
+        throw new TypeError("sweepAtOpen must be a boolean");
+    if (options.sweepAtOpen === true && !adoptRaw)
+        throw new TypeError('sweepAtOpen needs rawWrites: "adopt"');
+    const sweepOn = options.sweep ?? adoptRaw;
+    const queueLimit = options.adoptQueueLimit ?? MAX_ADOPT_QUEUE;
+    if (!Number.isInteger(queueLimit) || queueLimit < 1)
+        throw new RangeError("adoptQueueLimit must be a whole number of 1 or more");
+    const redoLimit = options.adoptRedoLimit ?? MAX_ADOPT_REDO;
+    if (!Number.isInteger(redoLimit) || redoLimit < 0)
+        throw new RangeError("adoptRedoLimit must be a whole number of 0 or more");
+    const sweepPage = Math.min(SWEEP_PAGE, queueLimit);
+    const redoChunk = Math.min(ADOPT_BATCH, queueLimit);
+    let stopped = false;
+    let stream;
+    let token;
+    /** True after a stream was open: a new stream without a token can then miss changes. */
+    let opened = false;
+    let attempt = 0;
+    let timer;
+    let own;
+    /** True after the first refused adoption: the log tells it once. */
+    let refused = false;
+    /** True after the server said that it has change streams. */
+    let checked = false;
+    /** The raw writes that wait for an adoption, and true while one adoption runs. */
+    const queue = [];
+    let adopting = false;
+    /** True after the queue was full, until it is empty: the log tells it once, and the sweep waits. */
+    let full = false;
+    /** The timers of the adoptions that try again. */
+    const retries = new Map();
+    /** True when a sweep must run when the stream is open and the queue is empty, and true while one runs. */
+    let sweepWanted = options.sweepAtOpen ?? false;
+    let sweeping = false;
+    /** True while a stream is open: it saw its first resume token or change. */
+    let live = false;
+    /** The raw writes to adopt again when the queue is empty, by the key of the stored id, and true while that runs. */
+    const redo = new Map();
+    let redoing = false;
+    /** True after the set of the redo was full, until it is empty: the log tells it once. */
+    let redoFull = false;
+    /** The keys of the documents that the sweep cannot place: the next sweeps leave them. */
+    const unplaced = new Set();
+    /** The partitions that got a hint: the topics of a gap. */
+    const partitions = new Set();
+    let markReady = () => undefined;
+    let failReady = () => undefined;
+    const ready = new Promise((resolve, reject) => {
+        markReady = resolve;
+        failReady = reject;
+    });
+    // A caller that does not wait for `ready` gets no unhandled rejection: the log tells the error
+    ready.catch(() => undefined);
+    const send = (partition, event) => {
+        let topic;
+        try {
+            topic = topicOf(name, partition);
+            if (topic === undefined)
+                return;
+            const result = emit(topic, event);
+            if (result instanceof Promise)
+                result.catch((error) => log("error", `the publish on ${topic} failed: ${messageOf(error)}`));
+        }
+        catch (error) {
+            log("error", `the publish on ${topic ?? name} failed: ${messageOf(error)}`);
+        }
+    };
+    /** The hint of a change of an item. A raw write that the storage does not adopt has no version. */
+    const publishItem = (item) => {
+        if (stopped)
+            return;
+        if (partitions.size < MAX_GAP_PARTITIONS)
+            partitions.add(item.partition);
+        if (item.deleted)
+            return send(item.partition, { changed: [], deleted: [item.id] });
+        send(item.partition, item.version === undefined ? { changed: [item.id], deleted: [] } : { changed: [item.id], deleted: [], versions: [{ id: item.id, version: item.version }] });
+    };
+    /** The partitions of the subscribed topics of the bus and of `gapPartitions`. */
+    const extraPartitions = (into) => {
+        try {
+            for (const topic of channels?.() ?? []) {
+                const parsed = collectionTopicPartition(topic);
+                if (parsed?.collection === name)
+                    into.add(parsed.partition);
+            }
+        }
+        catch (error) {
+            log("warn", `the bus does not tell its channels: ${messageOf(error)}`);
+        }
+        try {
+            for (const partition of options.gapPartitions?.() ?? [])
+                into.add(partition);
+        }
+        catch (error) {
+            log("warn", `gapPartitions failed: ${messageOf(error)}`);
+        }
+    };
+    /** The reset hint of a gap: the subscribers read the collection in full. */
+    const gap = (reason) => {
+        log("warn", `${reason}: the subscribers read the collection in full`);
+        // The raw writes of the gap have no event: the sweep adopts the documents without _seq
+        if (sweepOn) {
+            sweepWanted = true;
+            afterQueue();
+        }
+        if (source.partition !== undefined)
+            return send(source.partition, resetHint());
+        const topics = new Set([undefined, ...partitions]);
+        void source
+            .partitions()
+            .then((known) => {
+            for (const partition of known)
+                topics.add(partition);
+        }, (error) => log("warn", `the counters of ${source.collection} cannot be read (${messageOf(error)}): the gap goes to the partitions of the hints and of the subscribers only`))
+            .then(() => {
+            if (stopped)
+                return;
+            extraPartitions(topics);
+            let count = 0;
+            for (const partition of topics) {
+                if (count++ >= MAX_GAP_PARTITIONS)
+                    break;
+                send(partition, resetHint());
+            }
+        });
+    };
+    /** The reset hint of each partition of the raw writes: their subscribers read the collection in full. */
+    const resetPartitions = (jobs) => {
+        if (stopped)
+            return;
+        const done = new Set();
+        for (const job of jobs) {
+            if (done.has(job.item.partition))
+                continue;
+            done.add(job.item.partition);
+            send(job.item.partition, resetHint());
+        }
+    };
+    /**
+     * It keeps a raw write to adopt again when the queue is empty, with the newest write of each
+     * document. False when the watcher cannot keep it: no id, or the set is full.
+     */
+    const remember = (job) => {
+        const key = idKey(job.raw.storedId);
+        if (key === undefined)
+            return false;
+        if (redo.has(key) || redo.size < redoLimit) {
+            redo.set(key, { item: job.item, raw: job.raw, tries: 0 });
+            return true;
+        }
+        if (!redoFull)
+            log("warn", `more than ${redoLimit} raw writes of ${source.collection} wait to be adopted again: the partitions of the next ones get a reset hint, and a sweep runs`);
+        redoFull = true;
+        return false;
+    };
+    /** The count and the first of the documents that the current sweep cannot place. */
+    let leftCount = 0;
+    let leftFirst = "";
+    /** A document that the sweep cannot place: the next sweeps leave it. */
+    const leave = (storedId, why) => {
+        const key = idKey(storedId);
+        if (key !== undefined && unplaced.size < MAX_UNPLACED)
+            unplaced.add(key);
+        if (leftCount++ === 0)
+            leftFirst = `${describeId(storedId)} (${why})`;
+    };
+    /** The end of one raw write of an adoption. The change event of an adoption is its hint. */
+    const settle = (job, result, failed) => {
+        if (typeof result === "string") {
+            // `adopted`: the event of the adoption is the hint. `lost`: a newer write won, and its event is the hint.
+            if (result === "unknown") {
+                if (job.kind === "sweep")
+                    leave(job.raw.storedId, "neither its id nor its partition");
+                else {
+                    log("warn", `the raw write of ${describeId(job.raw.storedId)} in ${source.collection} tells neither its id nor its partition: the storage leaves it`);
+                    publishItem(job.item);
+                }
+            }
+            return job.done?.();
+        }
+        const error = result.error;
+        if (isRefused(error)) {
+            if (!refused)
+                log("error", `the adoption of a raw write of ${source.collection} was refused (${messageOf(error)}): the watcher needs write access, or set rawWrites: "ignore". The hints of raw writes go out without a version`);
+            refused = true;
+            publishItem(job.item);
+            return job.done?.();
+        }
+        if (isPermanent(error) || job.tries + 1 >= ADOPT_TRIES) {
+            failed.push({ job, error });
+            return job.done?.();
+        }
+        // An error that can go away: the adoption tries again after the backoff
+        const retry = clock.after(delayOf(job.tries), () => {
+            retries.delete(retry);
+            enqueue({ ...job, tries: job.tries + 1 });
+        });
+        retry.unref?.();
+        retries.set(retry, job);
+    };
+    /** One adoption at a time: the raw writes that wait together take one block of numbers for each storage. */
+    const work = async () => {
+        adopting = true;
+        try {
+            while (!stopped && queue.length > 0) {
+                const batch = queue.splice(0, ADOPT_BATCH);
+                let results;
+                try {
+                    results = await source.adoptAll(batch.map((job) => job.raw));
+                }
+                catch (error) {
+                    results = batch.map(() => ({ error }));
+                }
+                if (stopped) {
+                    for (const job of batch)
+                        job.done?.();
+                    break;
+                }
+                const failed = [];
+                batch.forEach((job, index) => settle(job, results[index] ?? "unknown", failed));
+                const first = failed[0];
+                if (first) {
+                    // The last try failed: a reset hint for each partition. The redo tries once more; a second failure is for good.
+                    const names = [...new Set(failed.map(({ job }) => job.item.partition))].map((partition) => (partition === undefined ? "(none)" : JSON.stringify(partition)));
+                    log("warn", `the adoption of ${failed.length} raw writes of ${source.collection} failed (${messageOf(first.error)}): the partitions ${names.join(", ")} get a reset hint`);
+                    for (const { job } of failed)
+                        if (job.kind !== "redo" && !remember(job))
+                            sweepWanted = true;
+                    resetPartitions(failed.map(({ job }) => job));
+                }
+            }
+            if (queue.length === 0)
+                full = false;
+        }
+        finally {
+            adopting = false;
+        }
+        if (stopped)
+            for (const job of queue.splice(0))
+                job.done?.();
+        // The queue is empty: the sweep, then the redo, can start
+        else
+            afterQueue();
+    };
+    const enqueue = (job) => {
+        if (stopped)
+            return job.done?.();
+        if (queue.length >= queueLimit) {
+            if (!full)
+                log("warn", `more than ${queueLimit} raw writes of ${source.collection} wait for an adoption: the next ones are adopted again when the queue is empty`);
+            full = true;
+            if (remember(job)) {
+                // The hint of a raw write of a change event goes out now, without a version. Its adoption gives the version.
+                if (job.kind === undefined)
+                    publishItem(job.item);
+            }
+            else {
+                // The sweep finds the documents without _seq. A write of the sweep had no hint.
+                sweepWanted = true;
+                if (job.kind !== "sweep")
+                    resetPartitions([job]);
+            }
+            return job.done?.();
+        }
+        queue.push(job);
+        if (!adopting)
+            void work();
+    };
+    /** The adoption of a raw write (D39), through the queue. */
+    const adopt = (item, change, before) => {
+        const after = change["operationType"] === "delete" ? undefined : asDocument(change["fullDocument"]);
+        // A hard delete without the image before the change: the storage cannot write its tombstone
+        if (!after && !before)
+            return publishItem(item);
+        const storedId = asDocument(change["documentKey"])?.["_id"];
+        enqueue({ item, raw: { storedId, seq: item.seq, after, before }, tries: 0 });
+    };
+    /** The sweep: each document without `_seq` goes to the queue of the adoptions, one page at a time. */
+    const sweep = async () => {
+        let last;
+        let count = 0;
+        leftCount = 0;
+        try {
+            for (;;) {
+                if (stopped)
+                    return;
+                const page = await source.rawDocuments(last, sweepPage);
+                const end = page[page.length - 1];
+                if (!end)
+                    break;
+                last = { id: end["_id"] };
+                const waits = [];
+                for (const doc of page) {
+                    const key = idKey(doc["_id"]);
+                    if (key !== undefined && unplaced.has(key))
+                        continue;
+                    let item;
+                    try {
+                        item = storageChangeOf({ operationType: "insert", fullDocument: doc, documentKey: { _id: doc["_id"] } }, source.form);
+                    }
+                    catch (error) {
+                        leave(doc["_id"], messageOf(error));
+                        continue;
+                    }
+                    if (!item) {
+                        leave(doc["_id"], "no id");
+                        continue;
+                    }
+                    if (source.partition !== undefined && item.partition !== source.partition) {
+                        leave(doc["_id"], "another partition");
+                        continue;
+                    }
+                    if (!item.raw)
+                        continue;
+                    count++;
+                    const job = { item, raw: { storedId: doc["_id"], seq: undefined, after: doc }, tries: 0, kind: "sweep" };
+                    waits.push(new Promise((resolve) => enqueue({ ...job, done: resolve })));
+                }
+                await Promise.all(waits);
+                if (page.length < sweepPage)
+                    break;
+            }
+            if (stopped)
+                return;
+            if (count > 0)
+                log("info", `the sweep of ${source.collection} sent ${count} documents without _seq to the adoption`);
+            if (leftCount > 0)
+                log("warn", `the sweep of ${source.collection} cannot place ${leftCount} documents without _seq, for example ${leftFirst}: the next sweeps leave them`);
+        }
+        catch (error) {
+            if (!stopped)
+                log("warn", `the sweep of the documents without _seq of ${source.collection} failed: ${messageOf(error)}`);
+        }
+    };
+    /** The redo: each raw write that did not fit or that failed goes to the queue again, with its document now. */
+    const adoptAgain = async () => {
+        const jobs = [...redo.values()];
+        redo.clear();
+        redoFull = false;
+        for (let start = 0; start < jobs.length; start += redoChunk) {
+            if (stopped)
+                return;
+            const part = jobs.slice(start, start + redoChunk);
+            let docs;
+            try {
+                docs = await source.storedDocuments(part.map((job) => job.raw.storedId));
+            }
+            catch (error) {
+                if (stopped)
+                    return;
+                log("warn", `the documents of ${part.length} raw writes of ${source.collection} to adopt again cannot be read (${messageOf(error)}): their partitions get a reset hint`);
+                resetPartitions(part);
+                continue;
+            }
+            const current = new Map(docs.map((doc) => [idKey(doc["_id"]), doc]));
+            const waits = [];
+            for (const job of part) {
+                const doc = current.get(idKey(job.raw.storedId));
+                // A newer write (a write of the storage, or a delete) has its own event: no redo
+                let raw;
+                if (!doc)
+                    raw = job.raw.after === undefined ? job.raw : undefined;
+                else
+                    raw = seqOf(doc) === job.raw.seq ? { ...job.raw, after: doc } : undefined;
+                if (!raw)
+                    continue;
+                const again = { item: job.item, raw, tries: 0, kind: "redo" };
+                waits.push(new Promise((resolve) => enqueue({ ...again, done: resolve })));
+            }
+            await Promise.all(waits);
+        }
+    };
+    /** The work after an empty queue: the sweep when a stream is open, then the redo. */
+    const afterQueue = () => {
+        if (stopped || full || adopting || queue.length > 0 || sweeping || redoing)
+            return;
+        if (sweepWanted && live) {
+            sweepWanted = false;
+            sweeping = true;
+            void sweep().finally(() => {
+                sweeping = false;
+                afterQueue();
+            });
+            return;
+        }
+        if (redo.size === 0)
+            return;
+        redoing = true;
+        void adoptAgain()
+            .catch((error) => log("warn", `the redo of the raw writes of ${source.collection} failed: ${messageOf(error)}`))
+            .finally(() => {
+            redoing = false;
+            afterQueue();
+        });
+    };
+    const onChange = (change) => {
+        token = change["_id"];
+        attempt = 0;
+        const type = change["operationType"];
+        if (type === "drop" || type === "rename" || type === "dropDatabase")
+            return gap(`the collection ${source.collection} had a ${String(type)}`);
+        const ns = asDocument(change["ns"]);
+        if (ns?.["coll"] === source.counters)
+            return;
+        let item;
+        try {
+            item = storageChangeOf(change, source.form);
+        }
+        catch (error) {
+            log("warn", `a change of ${source.collection} has no item: ${messageOf(error)}`);
+            return;
+        }
+        if (!item)
+            return;
+        if (source.partition !== undefined && item.partition !== source.partition)
+            return;
+        if (!item.raw) {
+            // The event of a write of this process: the collection procedures publish it
+            if (skipOwn && own && item.seq !== undefined && own.take(item.seq))
+                return;
+            return publishItem(item);
+        }
+        const before = asDocument(change["fullDocumentBeforeChange"]);
+        // The delete of a tombstone that this process pruned: no item. With the image, storageChangeOf() tells it.
+        if (type === "delete" && !before && own?.takePruned(asDocument(change["documentKey"])?.["_id"]))
+            return;
+        if (!adoptRaw)
+            return publishItem(item);
+        adopt(item, change, before);
+    };
+    const schedule = (reason) => {
+        if (stopped)
+            return;
+        const delay = delayOf(attempt++);
+        log("warn", `${reason}: the stream of ${source.collection} opens again in ${Math.round(delay)} ms`);
+        timer = clock.after(delay, () => {
+            timer = undefined;
+            void open();
+        });
+        timer.unref?.();
+    };
+    /** The end of the adoptions that wait: a stopped watcher adopts nothing. */
+    const endAdoptions = () => {
+        for (const [retry, job] of retries) {
+            retry();
+            job.done?.();
+        }
+        retries.clear();
+        for (const job of queue.splice(0))
+            job.done?.();
+        redo.clear();
+    };
+    /** The end of a watcher whose first open failed with an error that does not go away. */
+    const fail = (error) => {
+        log("error", `the stream of ${source.collection} cannot open (${messageOf(error)}): the watcher stops`);
+        stopped = true;
+        endAdoptions();
+        own?.close();
+        own = undefined;
+        failReady(error);
+    };
+    const open = async () => {
+        if (stopped)
+            return;
+        let db;
+        try {
+            db = await source.db();
+        }
+        catch (error) {
+            if (!opened && isPermanent(error))
+                return fail(error);
+            return schedule(`the database is not available (${messageOf(error)})`);
+        }
+        if (stopped)
+            return;
+        if (!checked) {
+            // On a single server, the driver can wait without an error: the watcher asks the server first
+            let hello;
+            try {
+                hello = await db.command({ hello: 1 });
+            }
+            catch (error) {
+                // An old server without hello: the stream tells the error
+                if (codeOf(error) !== COMMAND_NOT_FOUND) {
+                    if (!opened && isPermanent(error))
+                        return fail(error);
+                    return schedule(`the server does not answer hello (${messageOf(error)})`);
+                }
+            }
+            if (stopped)
+                return;
+            if (hello && typeof hello["setName"] !== "string" && hello["msg"] !== "isdbgrid") {
+                return fail(Object.assign(new Error("change streams need a replica set or a sharded cluster, and the server is a single server"), { code: 40573 }));
+            }
+            checked = true;
+        }
+        own ??= trackOwnNumbers(db, source.collection, { numbers: skipOwn, clock });
+        if (token === undefined && opened)
+            gap(`the stream of ${source.collection} opens again without a resume token`);
+        let next;
+        try {
+            next = db.collection(source.collection).watch([], {
+                fullDocument: "updateLookup",
+                fullDocumentBeforeChange: "whenAvailable",
+                ...(token !== undefined && { startAfter: token }),
+            });
+        }
+        catch (error) {
+            if (!opened && isPermanent(error))
+                return fail(error);
+            return schedule(`the stream cannot open (${messageOf(error)})`);
+        }
+        stream = next;
+        let ended = false;
+        const end = (reason, error) => {
+            if (ended)
+                return;
+            ended = true;
+            next.close().catch(() => undefined);
+            if (stream !== next || stopped)
+                return;
+            stream = undefined;
+            live = false;
+            if (!opened && error !== undefined && isPermanent(error))
+                return fail(error);
+            const code = codeOf(error);
+            if (code !== undefined && LOST_TOKEN_CODES.has(code) && token !== undefined) {
+                // The next stream starts now, without the token: it publishes a gap
+                log("warn", `the server does not know the resume token of ${source.collection} (code ${code})`);
+                token = undefined;
+            }
+            schedule(`the stream ${reason}`);
+        };
+        next.on("resumeTokenChanged", (resumeToken) => {
+            if (resumeToken === undefined || resumeToken === null)
+                return;
+            token = resumeToken;
+            opened = true;
+            attempt = 0;
+            markReady();
+            live = true;
+            afterQueue();
+        });
+        next.on("change", (change) => {
+            opened = true;
+            markReady();
+            live = true;
+            afterQueue();
+            if (change["operationType"] === "invalidate") {
+                // The collection was dropped or renamed: the stream starts after the invalidate event
+                token = change["_id"];
+                return end("ended (invalidate)");
+            }
+            onChange(change);
+        });
+        next.on("error", (error) => end(`failed: ${error.message}`, error));
+        next.on("close", () => end("closed"));
+    };
+    void open();
+    return {
+        ready,
+        async stop() {
+            if (stopped)
+                return;
+            stopped = true;
+            timer?.();
+            timer = undefined;
+            endAdoptions();
+            own?.close();
+            own = undefined;
+            markReady();
+            const current = stream;
+            stream = undefined;
+            await current?.close().catch(() => undefined);
+        },
+    };
+}
+//# sourceMappingURL=storage-hints.js.map