@procedures/mongo 0.36.0 → 0.38.0

package/dist/storage/materializer.js

@@ -33,6 +33,8 @@
  *   1000 rows. It keeps nothing between passes.
  * - A change stream that ends (for example after `dropDatabase`) opens again after `rewatch`,
  *   and a pass catches up on the changes that the stream missed.
+ * - Each stream is open on the server before the materializer reads what it watches: the views
+ *   before the read of the views, the sources before a run. A change between the two reaches both.
  * - The status of each view (`ok`, `errored`, `cycle`) goes into the field `statusField` of its
  *   document. The materializer ignores an update of only that field.
  *
@@ -44,15 +46,17 @@
  * that moves a document to another partition also reaches the views of its old partition.
  *
  * Change streams and transactions need a replica set (MongoDB 6 or later). The target of a view is
- * a plain collection that only the materializer writes.
+ * a plain collection that only the materializer writes. A target of `storageTargets` is in the
+ * stored form of a `MongoDocumentStorage` instead (roadmap CLI-81): see `ViewStorageTarget`.
  *
  * The stop has the parts of a transport of a service: `stopIntake()`, `drain()`, `inFlight()` and
  * `close()`. A host composes them with no adapter. The timers come from the injected clock.
  */
 import { createHash } from "node:crypto";
+import { isDeepStrictEqual } from "node:util";
 import { BSON, ObjectId } from "mongodb";
-import { optionalMs, realClock } from "@procedures/client";
-import { DEFAULT_STORED_FORM, TOMBSTONE_FIELD } from "./document-storage.js";
+import { collectionChangeTopic, ms, optionalMs, realClock } from "@procedures/client";
+import { DEFAULT_STORED_FORM, DOCUMENT_STORAGE_FIELDS, MongoDocumentStorage, TOMBSTONE_FIELD, storageWriteTarget } from "./document-storage.js";
 import { scopePipeline } from "./pipeline-scope.js";
 /** The fields of a storage that a row never keeps: its sequence number and its tombstone flag (roadmap CLI-77). */
 const ROW_DROPPED_FIELDS = ["_seq", TOMBSTONE_FIELD];
@@ -94,14 +98,60 @@ function groups(items) {
         result.push(items.slice(start, start + WRITE_GROUP));
     return result;
 }
+/** The most tries of a run in a transaction after a write conflict (code 112). After them, the run gives way, and comes back after `rewatch`. */
+const CONFLICT_RETRIES = 3;
+/** The most ids of a hint of a target not in a stored form. More ids give a reset hint. */
+const MAX_HINT_IDS = 10_000;
+/** The `_id` of a row in the result, from the stored `_id` of `partitionTargetId`. */
+export function defaultRowId(storedId) {
+    return typeof storedId === "object" && storedId !== null && "id" in storedId ? storedId.id : storedId;
+}
+/**
+ * The id of a row of a target in a stored form, from the `_id` of the row in the result: a text as
+ * it is, another value as its relaxed Extended JSON text. The text of a value is the same at each
+ * run (the order of the fields of an object is the order of the result). A row without `_id` has
+ * no id.
+ */
+export function defaultStorageRowId(id) {
+    if (typeof id === "string")
+        return id;
+    if (id === undefined)
+        throw new TypeError("the row has no _id");
+    return BSON.EJSON.stringify(id, { relaxed: true });
+}
+/** The hash of a stored document without its numbers (`_seq` and `_ord`): two documents with the same content give the same hash. */
+function contentHash(doc) {
+    const { _seq: _number, _ord: _place, ...rest } = doc;
+    return createHash("sha1")
+        .update(canonical(BSON.EJSON.serialize(rest, { relaxed: false })))
+        .digest("hex");
+}
 /** The partition of the views is the partition of the storage. */
 export const identityPartition = (storagePartition) => storagePartition;
+/**
+ * This function waits until a change stream is open on the server: its first resume token. From
+ * then on, the stream gets each change. The driver sends the `aggregate` of a stream only after
+ * `watch()` returns, so a change before that point does not reach the stream. An error, an end or a
+ * close of the stream also ends the wait: its rewatch catches up (roadmap O63).
+ */
+function openOnServer(stream) {
+    return new Promise((resolve) => {
+        const events = ["resumeTokenChanged", "error", "end", "close"];
+        const done = () => {
+            for (const event of events)
+                stream.off(event, done);
+            resolve();
+        };
+        for (const event of events)
+            stream.on(event, done);
+    });
+}
 /** A path without its positions in arrays (the segments of digits only): `lines.0.price` gives `lines.price`. */
 function withoutPositions(path) {
-    return path
-        .split(".")
-        .filter((segment) => !/^\d+$/.test(segment))
-        .join(".");
+    // path-check: a field path of a MongoDB document, not a procedure path.
+    const segments = path.split(".");
+    // path-check: a field path of a MongoDB document, not a procedure path.
+    return segments.filter((segment) => !/^\d+$/.test(segment)).join(".");
 }
 /** The key of a row id for the signatures: type-aware, so an ObjectId and its text are different keys. */
 function idKey(id) {
@@ -153,11 +203,19 @@ export class ViewMaterializer {
     graceMs;
     leaseMs;
     storageSources;
+    /** The storages that the materializer made for the targets in a stored form, by collection, with the values that made them. */
+    storageTargetCache = new Map();
+    /** The partition of a storage target that each running view writes (by the key of the view), so a second view of it is refused. */
+    storageClaims = new Map();
+    storageRowIdOf;
+    rowIdOf;
     /** False after `stopIntake()`: a change starts no pass. */
     intake = false;
     /** The work of each partition, one at a time: the passes and the rebalances. */
     chains = new Map();
     pending = new Set();
+    /** The changes of the set of views (a resync, a change of a view document), one at a time, in the order of the views stream. */
+    viewChain = Promise.resolve();
     stopped = true;
     constructor(options) {
         this.options = options;
@@ -180,6 +238,68 @@ export class ViewMaterializer {
             }
             return [source, normal];
         }));
+        // A record of targets is examined at once: a wrong entry is an error of the options
+        const targets = options.storageTargets;
+        if (targets !== undefined && typeof targets !== "function") {
+            for (const [collection, entry] of Object.entries(targets))
+                this.storageTargetOf(collection, entry);
+        }
+        this.storageRowIdOf = options.storageRowIdOf ?? defaultStorageRowId;
+        this.rowIdOf = options.rowIdOf ?? defaultRowId;
+    }
+    /**
+     * A target of `storageTargets` as a storage on the database of the materializer, so its writes
+     * and the transaction of a run use one client. A given storage gives its form, its counters,
+     * `writeLease` and `maxTombstones`.
+     */
+    storageTargetOf(collection, entry) {
+        const wrapped = typeof entry === "object" && entry !== null && "target" in entry ? entry : { target: entry };
+        const storagePartitionOf = wrapped.storagePartitionOf ?? ((partition) => partition);
+        if (typeof storagePartitionOf !== "function")
+            throw new TypeError(`The storage target ${collection} needs a function storagePartitionOf`);
+        const target = wrapped.target;
+        let values;
+        if (target instanceof MongoDocumentStorage) {
+            const parts = storageWriteTarget(target);
+            if (parts.collection !== collection)
+                throw new TypeError(`The storage target ${collection} has a storage of the collection ${parts.collection}`);
+            values = { form: parts.form, counters: parts.counters, writeLeaseMs: parts.writeLeaseMs, maxTombstones: parts.maxTombstones };
+        }
+        else if (typeof target === "object" && target !== null && typeof target.storedId === "function") {
+            values = { form: target, counters: undefined, writeLeaseMs: undefined, maxTombstones: undefined };
+        }
+        else {
+            throw new TypeError(`The storage target ${collection} needs a MongoDocumentStorage, a stored form or a ViewStorageTarget`);
+        }
+        // The same values give the storage of the last run: its partitions keep their state (the floor document of O66)
+        const cached = this.storageTargetCache.get(collection);
+        if (cached && cached.form === values.form && cached.counters === values.counters && cached.writeLeaseMs === values.writeLeaseMs && cached.maxTombstones === values.maxTombstones) {
+            return { storage: cached.storage, storagePartitionOf };
+        }
+        const db = () => {
+            if (!this.db)
+                throw new Error("The materializer has not started");
+            return this.db;
+        };
+        const storage = new MongoDocumentStorage({
+            collection,
+            db,
+            form: values.form,
+            transactions: "never",
+            ...(values.counters !== undefined ? { counters: values.counters } : {}),
+            ...(values.writeLeaseMs !== undefined ? { writeLease: ms(values.writeLeaseMs) } : {}),
+            ...(values.maxTombstones !== undefined ? { maxTombstones: values.maxTombstones } : {}),
+        });
+        this.storageTargetCache.set(collection, { ...values, storage });
+        return { storage, storagePartitionOf };
+    }
+    /** The target in a stored form of a collection at this time, or undefined. The function of `storageTargets` runs at each call. */
+    storageTargetFor(collection) {
+        const targets = this.options.storageTargets;
+        if (targets === undefined)
+            return undefined;
+        const entry = typeof targets === "function" ? targets(collection) : Object.hasOwn(targets, collection) ? targets[collection] : undefined;
+        return entry === undefined || entry === null ? undefined : this.storageTargetOf(collection, entry);
     }
     log(level, message) {
         if (this.options.log)
@@ -223,8 +343,9 @@ export class ViewMaterializer {
         this.stopped = false;
         this.intake = true;
         this.db = await this.options.db();
+        // The stream of the views opens before the read of the views: a change between the two reaches both (roadmap O63)
+        await this.watchViews();
         const count = await this.resync();
-        this.watchViews();
         this.log("info", `started: ${count} view(s)`);
     }
     /** This method stops the passes of new changes: a change starts no pass, and a waiting pass does not start. */
@@ -296,26 +417,42 @@ export class ViewMaterializer {
         const partition = view[this.partitionField];
         return typeof partition === "string" && partition !== "" ? partition : undefined;
     }
-    /** This method closes each view, and opens each view of the views collection again: a full catch-up. */
+    /**
+     * This method runs a change of the set of views after the changes before it. A change of a view
+     * document that comes during a resync thus applies after the read of the resync, not before it.
+     */
+    inViewOrder(work) {
+        const run = this.viewChain.then(work);
+        this.viewChain = run.catch(() => undefined);
+        return run;
+    }
+    /**
+     * This method closes each view, and opens each view of the views collection again: a full
+     * catch-up. Open the stream of the views first: then a change after the read reaches the stream.
+     */
     async resync() {
-        for (const key of [...this.running.keys()])
-            this.tearDown(key);
-        const docs = (await this.db.collection(this.viewsName).find({}).toArray());
-        for (const view of docs)
-            await this.spinUp(view);
-        return docs.length;
+        const { count, registered } = await this.inViewOrder(async () => {
+            const docs = (await this.db.collection(this.viewsName).find({}).toArray());
+            for (const key of [...this.running.keys()])
+                this.tearDown(key);
+            return { count: docs.length, registered: docs.filter((view) => this.register(view)) };
+        });
+        for (const view of registered)
+            await this.activate(view);
+        return count;
     }
+    /** This method opens the stream of the views. The promise resolves when the stream is open on the server. */
     watchViews() {
         if (this.stopped || !this.db)
-            return;
+            return Promise.resolve();
         let stream;
         try {
             stream = this.db.collection(this.viewsName).watch([], { fullDocument: "updateLookup" });
         }
         catch (error) {
             this.log("error", `cannot watch ${this.viewsName}: ${String(error?.message ?? error)}`);
-            this.later(() => this.watchViews());
-            return;
+            this.later(() => void this.watchViews());
+            return Promise.resolve();
         }
         const previous = this.viewsStream;
         if (previous && previous !== stream)
@@ -329,8 +466,8 @@ export class ViewMaterializer {
             this.log("warn", `the stream of ${this.viewsName} ${reason}: it opens again in ${this.rewatchMs} ms`);
             stream.close().catch(() => undefined);
             this.later(() => {
-                this.watchViews();
-                void this.track(this.resync()).catch((error) => this.log("error", `the resync failed: ${String(error?.message ?? error)}`));
+                // The changes while the stream was closed are lost: a resync catches up, after the new stream is open
+                void this.track(this.watchViews().then(() => this.resync())).catch((error) => this.log("error", `the resync failed: ${String(error?.message ?? error)}`));
             });
         };
         stream.on("change", (change) => {
@@ -349,6 +486,7 @@ export class ViewMaterializer {
         });
         stream.on("error", (error) => rewatch(`failed: ${error.message}`));
         stream.on("close", () => rewatch("closed"));
+        return openOnServer(stream);
     }
     async onViewChange(change) {
         const type = change["operationType"];
@@ -356,41 +494,68 @@ export class ViewMaterializer {
             const view = change["fullDocument"];
             if (!view)
                 return;
-            const partition = this.partitionOf(view);
-            if (partition !== undefined)
-                this.tearDown(this.keyOf(partition, view._id));
-            await this.spinUp(view);
+            const registered = await this.inViewOrder(() => {
+                const partition = this.partitionOf(view);
+                if (partition !== undefined)
+                    this.tearDown(this.keyOf(partition, view._id));
+                return this.register(view);
+            });
+            if (registered)
+                await this.activate(view);
         }
         else if (type === "delete") {
             const id = String(change["documentKey"]["_id"]);
-            const partitions = new Set();
-            for (const [key, view] of [...this.running]) {
-                if (view.view._id === id) {
-                    partitions.add(view.partition);
-                    this.tearDown(key);
+            const partitions = await this.inViewOrder(() => {
+                const found = new Set();
+                for (const [key, view] of [...this.running]) {
+                    if (view.view._id === id) {
+                        found.add(view.partition);
+                        this.tearDown(key);
+                    }
                 }
-            }
+                return found;
+            });
             // A removed view can break a cycle of the other views
             for (const partition of partitions)
                 await this.rebalance(partition);
         }
     }
-    async spinUp(view) {
+    /** This method adds a view to the views that the materializer knows. It gives false for a view that cannot run. */
+    register(view) {
         const partition = this.partitionOf(view);
         if (partition === undefined) {
             this.log("warn", `the view ${String(view._id)} has no ${this.partitionField}: it does not run`);
-            return;
+            return false;
         }
         if (!Array.isArray(view.collections) || view.collections.length === 0 || typeof view.targetCollection !== "string" || !Array.isArray(view.pipeline)) {
             this.log("warn", `the view ${view._id} needs collections, a pipeline and a targetCollection: it does not run`);
-            return;
+            return false;
         }
         this.running.set(this.keyOf(partition, view._id), { view, partition, state: "active", streams: [] });
+        return true;
+    }
+    /** This method makes the index of the target of a registered view, and rebalances its partition: a new view runs. */
+    async activate(view) {
+        const partition = this.partitionOf(view);
         // The readers, the counts and the sweeps read the rows of a partition in a generation
         await this.db
             .collection(view.targetCollection)
             .createIndex({ [this.partitionField]: 1, [VIEW_GENERATION_FIELD]: 1 })
             .catch((error) => this.log("error", `the index of ${view.targetCollection} failed: ${String(error?.message ?? error)}`));
+        // A target in a stored form also has the indexes of the storage: the change reads, the order and the tombstones
+        let storageTarget;
+        try {
+            storageTarget = this.storageTargetFor(view.targetCollection);
+        }
+        catch (error) {
+            // The run of the view gives the error to its status
+            this.log("error", `the storage target ${view.targetCollection}: ${String(error?.message ?? error)}`);
+        }
+        if (storageTarget) {
+            await storageWriteTarget(storageTarget.storage)
+                .ready()
+                .catch((error) => this.log("error", `the indexes of the storage ${view.targetCollection} failed: ${String(error?.message ?? error)}`));
+        }
         await this.rebalance(partition);
     }
     tearDown(key) {
@@ -400,6 +565,7 @@ export class ViewMaterializer {
         for (const stream of view.streams)
             stream.close().catch(() => undefined);
         this.running.delete(key);
+        this.storageClaims.delete(key);
     }
     /**
      * The filter of the change stream of a source for a partition. It only removes the events that
@@ -486,9 +652,10 @@ export class ViewMaterializer {
     isCurrent(running) {
         return this.running.get(this.keyOf(running.partition, running.view._id)) === running;
     }
+    /** This method opens the stream of a source for a view. The promise resolves when the stream is open on the server. */
     watchSource(running, source) {
         if (this.stopped || !this.db)
-            return;
+            return Promise.resolve();
         let stream;
         try {
             stream = this.db
@@ -499,9 +666,9 @@ export class ViewMaterializer {
             this.log("error", `cannot watch ${source} for the view ${running.view._id}: ${String(error?.message ?? error)}`);
             this.later(() => {
                 if (this.isCurrent(running) && running.state === "active")
-                    this.watchSource(running, source);
+                    void this.watchSource(running, source);
             });
-            return;
+            return Promise.resolve();
         }
         running.streams.push(stream);
         let again = false;
@@ -519,9 +686,8 @@ export class ViewMaterializer {
             this.later(() => {
                 if (!this.isCurrent(running) || running.state !== "active")
                     return;
-                this.watchSource(running, source);
-                // The changes while the stream was closed are lost: a pass catches up
-                this.schedule(running.partition, source);
+                // The changes while the stream was closed are lost: a pass catches up, after the new stream is open
+                void this.watchSource(running, source).then(() => this.schedule(running.partition, source));
             });
         };
         stream.on("change", (change) => {
@@ -535,6 +701,7 @@ export class ViewMaterializer {
         });
         stream.on("error", (error) => rewatch(`failed: ${error.message}`));
         stream.on("close", () => rewatch("closed"));
+        return openOnServer(stream);
     }
     /** This method runs a function after `rewatch`, unless the materializer stops. The timer does not keep the process alive. */
     later(work) {
@@ -611,8 +778,11 @@ export class ViewMaterializer {
                 }
                 else if (running.state === "cyclic" || running.streams.length === 0) {
                     running.state = "active";
-                    for (const source of view.collections)
-                        this.watchSource(running, source);
+                    // The streams open before the run reads the sources: a change between the two reaches both (roadmap O63)
+                    await Promise.all(view.collections.map((source) => this.watchSource(running, source)));
+                    // A change of the view document during the wait replaced this view: the rebalance of the new one runs it
+                    if (!this.isCurrent(running))
+                        continue;
                     await this.runAndWriteStatus(view, partition);
                 }
             }
@@ -661,28 +831,102 @@ export class ViewMaterializer {
      */
     async run(view, partition) {
         const target = this.db.collection(view.targetCollection);
+        const storage = this.targetPartition(view, partition);
         const { status, now } = await this.statusOf(view, partition);
         if (status?.lease && status.lease.until instanceof Date && now < status.lease.until)
             throw new PassSuperseded("a pass of another process holds the view");
-        await this.dropOldGenerations(target, partition, status, now);
+        // The rows of a target in a stored form have no generation: a removal is a tombstone, never a delete of an old generation
+        if (!storage)
+            await this.dropOldGenerations(target, partition, status, now);
         const current = status?.generation;
         const generations = await this.currentGenerations(partition);
-        const size = current ? await target.countDocuments({ [this.partitionField]: partition, [VIEW_GENERATION_FIELD]: current }) : 0;
+        // The floor document of the partition of the storage, before the first write (roadmap O66)
+        if (storage)
+            await storage.writer.noteFloor(this.db);
+        const size = storage
+            ? await target.countDocuments({ ...storage.filter, [TOMBSTONE_FIELD]: { $ne: true } })
+            : current
+                ? await target.countDocuments({ [this.partitionField]: partition, [VIEW_GENERATION_FIELD]: current })
+                : 0;
         if (size <= this.maxTransactionRows) {
-            try {
-                return await this.runInTransaction(view, partition, target, current, generations);
-            }
-            catch (error) {
-                if (error instanceof ViewRunError || error instanceof PassSuperseded)
-                    throw error;
-                if (isWriteConflict(error))
-                    throw new PassSuperseded(`another process wrote the view at the same time (${String(error?.message ?? error)})`);
-                this.log("warn", `${view._id}: the transaction failed (${String(error?.message ?? error)}): the pass writes a new generation in groups`);
+            for (let attempt = 1;; attempt++) {
+                try {
+                    return storage
+                        ? await this.runStorageInTransaction(view, partition, target, storage, current, generations)
+                        : await this.runInTransaction(view, partition, target, current, generations);
+                }
+                catch (error) {
+                    if (error instanceof ViewRunError || error instanceof PassSuperseded)
+                        throw error;
+                    if (isWriteConflict(error)) {
+                        // A write conflict tries again a few times after a short wait, so a steady stream of writes does not starve the view
+                        if (attempt < CONFLICT_RETRIES) {
+                            await this.pause(25 * 2 ** attempt * (1 + Math.random()));
+                            continue;
+                        }
+                        throw new PassSuperseded(`another process wrote the view at the same time (${String(error?.message ?? error)})`);
+                    }
+                    this.log("warn", `${view._id}: the transaction failed (${String(error?.message ?? error)}): the pass writes ${storage ? "in groups" : "a new generation in groups"}`);
+                    break;
+                }
             }
         }
+        if (storage)
+            return this.runStorageInGroups(view, partition, target, storage, current, generations);
         await this.runInGroups(view, partition, target, current, generations);
         return true;
     }
+    /** This method waits on the clock of the materializer. The timer does not keep the process alive. */
+    pause(delay) {
+        return new Promise((resolve) => {
+            const cancel = this.clock.after(Math.round(delay), resolve);
+            cancel.unref?.();
+        });
+    }
+    /**
+     * The storage of the partition of a view in its target, or undefined for a target that is not in
+     * `storageTargets`. A partition that the storage refuses, and a partition field of the form with
+     * another value than the partition of the view, are an error of the view.
+     */
+    targetPartition(view, partition) {
+        let entry;
+        try {
+            entry = this.storageTargetFor(view.targetCollection);
+        }
+        catch (error) {
+            throw new ViewRunError(`the storage target ${view.targetCollection}: ${String(error?.message ?? error)}`);
+        }
+        if (!entry)
+            return undefined;
+        let writer;
+        let storagePartition;
+        try {
+            storagePartition = entry.storagePartitionOf(partition);
+            writer = storageWriteTarget(storagePartition === undefined ? entry.storage : entry.storage.forPartition(storagePartition));
+        }
+        catch (error) {
+            throw new ViewRunError(`the partition of the storage ${view.targetCollection}: ${String(error?.message ?? error)}`);
+        }
+        // One view writes a partition of a storage target: its run makes a tombstone of each other document of it
+        const claim = JSON.stringify([view.targetCollection, storagePartition ?? null]);
+        const key = this.keyOf(partition, view._id);
+        for (const [other, held] of this.storageClaims) {
+            if (other !== key && held.claim === claim && this.running.has(other)) {
+                const place = storagePartition === undefined ? "the storage" : `the partition ${storagePartition} of the storage`;
+                throw new ViewRunError(`the view ${held.view} (partition ${held.partition}) writes ${place} ${view.targetCollection} already: one view writes a partition of a storage target`);
+            }
+        }
+        this.storageClaims.set(key, { claim, view: view._id, partition });
+        const field = this.partitionField;
+        if (writer.partitionFields.has(field) && !isDeepStrictEqual(writer.partitionFields.get(field), partition)) {
+            throw new ViewRunError(`the stored form of ${view.targetCollection} gives the field ${field} another value than the partition ${partition}`);
+        }
+        return {
+            writer,
+            filter: { ...writer.partitionFilter(), [field]: partition },
+            partitionInContent: !writer.partitionFields.has(field) && !DOCUMENT_STORAGE_FIELDS.includes(field),
+        };
+    }
     /**
      * The status of a view in the database (the current generation, and the previous one), and the
      * time of the server: the times of the generations are times of the server.
@@ -724,7 +968,7 @@ export class ViewMaterializer {
         }
     }
     /** The rows of the result, in groups of 1000. A failure of the pipeline, a row without `_id` and too many rows are a `ViewRunError`. */
-    async *resultGroups(view, partition, generations, generation) {
+    async *resultGroups(view, partition, generations, generation, storage) {
         let pipeline;
         try {
             pipeline = this.scoped(view.pipeline, partition, view, generations);
@@ -750,6 +994,14 @@ export class ViewMaterializer {
                     break;
                 if (++count > this.maxRows)
                     throw new ViewRunError(`the result has more than ${this.maxRows} rows`);
+                if (storage) {
+                    group.push(this.storageRowOf(row, partition, storage));
+                    if (group.length === WRITE_GROUP) {
+                        yield group;
+                        group = [];
+                    }
+                    continue;
+                }
                 // The fields of the materializer, for a row that comes from a target: not in the content
                 const { _id, [this.partitionField]: _partition, [VIEW_GENERATION_FIELD]: _generation, [VIEW_HASH_FIELD]: _hash, ...content } = row;
                 if (_id === undefined || _id === null)
@@ -779,6 +1031,51 @@ export class ViewMaterializer {
             await cursor.close().catch(() => undefined);
         }
     }
+    /**
+     * A row of the result as a document of a target in a stored form (roadmap CLI-81). The content is
+     * the row without the fields of the materializer and of a storage: a pass-through row of a source
+     * in a stored form loses its `_key`, `_kid`, `_kpf`, `_ord` and `_json`, and the encoding of the
+     * storage makes them again from the row (`_json` from the content of the row). A field of the row
+     * with the name of a partition field of the form takes the value of the partition of the target.
+     * The numbers (`_seq`, `_ord`) come at the write.
+     */
+    storageRowOf(row, partition, storage) {
+        const { _id: rowId, [VIEW_GENERATION_FIELD]: _generation, [VIEW_HASH_FIELD]: _hash, ...content } = row;
+        let _id;
+        try {
+            _id = this.storageRowIdOf(rowId);
+        }
+        catch (error) {
+            throw new ViewRunError(`the id of a row of a target in a stored form: ${String(error?.message ?? error)}`);
+        }
+        if (typeof _id !== "string")
+            throw new ViewRunError(`storageRowIdOf must give a text, not ${typeof _id}`);
+        const field = this.partitionField;
+        delete content[field];
+        for (const name of DOCUMENT_STORAGE_FIELDS)
+            delete content[name];
+        for (const [name, value] of storage.writer.partitionFields) {
+            if (!(name in content))
+                continue;
+            if (value === undefined)
+                delete content[name];
+            else
+                content[name] = value;
+        }
+        if (storage.partitionInContent)
+            content[field] = partition;
+        let doc;
+        try {
+            doc = storage.writer.encode(_id, { _id, ...content }, 0, 0);
+        }
+        catch (error) {
+            throw new ViewRunError(`the row ${JSON.stringify(_id)}: ${String(error?.message ?? error)}`);
+        }
+        // A view that reads the target finds the row by the partition field, also in an item stored as JSON text
+        if (doc[field] === undefined)
+            doc[field] = partition;
+        return { id: _id, key: idKey(doc["_id"]), hash: contentHash(doc), doc };
+    }
     replaceOf(row, partition) {
         return { replaceOne: { filter: { _id: row.doc["_id"], [this.partitionField]: partition }, replacement: row.doc, upsert: true } };
     }
@@ -796,6 +1093,7 @@ export class ViewMaterializer {
         const generation = current ?? new ObjectId();
         const session = this.db.client.startSession();
         let changed = current === undefined;
+        const hint = { changed: [], deleted: [] };
         try {
             session.startTransaction();
             // In the transaction: a change of generation by another process since the start supersedes this pass
@@ -809,9 +1107,12 @@ export class ViewMaterializer {
                     .find({ _id: { $in: group.map((row) => row.doc["_id"]) }, [this.partitionField]: partition }, { projection: { [VIEW_HASH_FIELD]: 1 }, session })
                     .toArray();
                 const hashes = new Map(stored.map((doc) => [idKey(doc["_id"]), doc[VIEW_HASH_FIELD]]));
-                const writes = group.filter((row) => hashes.get(row.key) !== row.hash).map((row) => this.replaceOf(row, partition));
+                const rows = group.filter((row) => hashes.get(row.key) !== row.hash);
+                const writes = rows.map((row) => this.replaceOf(row, partition));
                 for (const row of group)
                     seen.add(row.key);
+                for (const row of rows)
+                    hint.changed.push(row.id);
                 if (writes.length > 0) {
                     await target.bulkWrite(writes, { ordered: false, session });
                     changed = true;
@@ -831,6 +1132,8 @@ export class ViewMaterializer {
                 await target.deleteMany({ [this.partitionField]: partition, _id: { $in: group } }, { session });
             if (stale.length > 0)
                 changed = true;
+            for (const storedId of stale)
+                hint.deleted.push(this.rowIdOf(storedId));
             if (!current && !(await this.setGeneration(view, partition, generation, { expected: undefined, session }))) {
                 throw new PassSuperseded("another process named the first generation of the view");
             }
@@ -843,8 +1146,37 @@ export class ViewMaterializer {
         finally {
             await session.endSession().catch(() => undefined);
         }
+        if (hint.changed.length > 0 || hint.deleted.length > 0)
+            this.publishHint(view, partition, hint);
         return changed;
     }
+    /**
+     * This method publishes the hint of a run of a target that is not in a stored form: one hint
+     * for the run, with the ids of the changed rows and of the removed rows. Without ids (a change
+     * of generation), with an id that is not a text, or with more than `MAX_HINT_IDS` ids, it is a
+     * reset hint. A failure of the publisher goes to the log: the run does not fail.
+     */
+    publishHint(view, partition, ids) {
+        const publish = this.options.publish;
+        if (!publish)
+            return;
+        const fail = (error) => this.log("error", `the hint of ${view._id} failed: ${String(error?.message ?? error)}`);
+        try {
+            const topic = this.options.hintTopic ? this.options.hintTopic(view, partition) : collectionChangeTopic(view._id);
+            if (topic === undefined)
+                return;
+            const texts = (values) => values.every((value) => typeof value === "string");
+            const event = ids && ids.changed.length + ids.deleted.length <= MAX_HINT_IDS && texts(ids.changed) && texts(ids.deleted)
+                ? { changed: [...new Set(ids.changed)], deleted: [...new Set(ids.deleted)] }
+                : { changed: [], deleted: [], reset: true };
+            const result = typeof publish === "function" ? publish(topic, event) : publish.emit(topic, event);
+            if (result instanceof Promise)
+                result.catch(fail);
+        }
+        catch (error) {
+            fail(error);
+        }
+    }
     /**
      * A run in groups, outside a transaction: a whole new generation, then the change of the status to
      * it. The readers of `viewFilter` move to it at once. The rows of no generation go at once: no
@@ -884,6 +1216,8 @@ export class ViewMaterializer {
             await this.releaseLease(view, partition, owner).catch(() => undefined);
             throw error;
         }
+        // The readers of the view moved to the new generation at once: a gap, so the subscribers read the view in full
+        this.publishHint(view, partition, undefined);
         await target.deleteMany({ [this.partitionField]: partition, [VIEW_GENERATION_FIELD]: { $exists: false } });
         // At the end of the grace time, the previous generation goes. The time of the server decides it, so a timer that ends early removes nothing
         const key = this.keyOf(partition, view._id);
@@ -899,6 +1233,204 @@ export class ViewMaterializer {
         cancel.unref?.();
         this.retries.add(cancel);
     }
+    /**
+     * The rows of a group that a run writes in a target in a stored form: each row whose stored
+     * document is absent, is a tombstone, or has other content (the numbers do not count). A row
+     * keeps the place (`_ord`) of its live stored document. Two rows with the same `_id` in a result
+     * are an error of the view.
+     */
+    async storageWrites(target, storage, group, seen, session) {
+        for (const row of group) {
+            if (seen.has(row.key))
+                throw new ViewRunError(`the result has two rows with the _id ${JSON.stringify(row.id)}`);
+            seen.add(row.key);
+        }
+        const stored = await target
+            .find({ _id: { $in: group.map((row) => row.doc["_id"]) }, ...storage.writer.partitionFilter() }, session ? { session } : {})
+            .toArray();
+        const byKey = new Map(stored.map((doc) => [idKey(doc["_id"]), doc]));
+        const writes = [];
+        for (const row of group) {
+            const doc = byKey.get(row.key);
+            const live = doc !== undefined && doc[TOMBSTONE_FIELD] !== true;
+            if (live && contentHash(doc) === row.hash)
+                continue;
+            writes.push({ row, ord: live && typeof doc["_ord"] === "number" ? doc["_ord"] : undefined });
+        }
+        return writes;
+    }
+    /**
+     * The live documents of the partition of a view in a target in a stored form that the result
+     * does not have. A document whose id the result does not have becomes the tombstone of that id. A
+     * document that tells no id, or the id of a row of the result (under another stored `_id`),
+     * becomes a tombstone without `_key`: no reader reads a delete of an item from it. More than
+     * `limit` documents is an error, so a transaction goes in groups.
+     */
+    async *storageRemovals(target, storage, seen, session, limit = Number.POSITIVE_INFINITY) {
+        let group = [];
+        let count = 0;
+        const cursor = target.find({ ...storage.filter, [TOMBSTONE_FIELD]: { $ne: true } }, { projection: { _id: 1, _key: 1 }, ...(session ? { session } : {}) });
+        try {
+            for await (const doc of cursor) {
+                if (seen.has(idKey(doc["_id"])))
+                    continue;
+                if (++count > limit)
+                    throw new Error(`more than ${limit} rows to remove`);
+                let id = storage.writer.idOfDoc(doc);
+                try {
+                    if (id !== undefined && seen.has(idKey(storage.writer.storedId(id))))
+                        id = undefined;
+                }
+                catch {
+                    id = undefined;
+                }
+                group.push({ storedId: doc["_id"], id });
+                if (group.length === WRITE_GROUP) {
+                    yield group;
+                    group = [];
+                }
+            }
+            if (group.length > 0)
+                yield group;
+        }
+        finally {
+            await cursor.close().catch(() => undefined);
+        }
+    }
+    /**
+     * The writes of a target in a stored form with their numbers from `first`: the tombstones first,
+     * then the rows. A new row takes its own number as its place. The filter of a write holds the
+     * partition: a row of another partition of the view fails with a duplicate key.
+     */
+    storageOperations(storage, partition, writes, removals, first) {
+        const field = this.partitionField;
+        const operations = [];
+        let seq = first;
+        for (const removal of removals) {
+            const number = seq++;
+            const tomb = removal.id === undefined ? { _id: removal.storedId, _seq: number, [TOMBSTONE_FIELD]: true } : storage.writer.tombstone(removal.id, number);
+            const replacement = { ...tomb, _id: removal.storedId };
+            for (const [name, value] of storage.writer.partitionFields)
+                if (value !== undefined)
+                    replacement[name] = value;
+            replacement[field] = partition;
+            operations.push({ replaceOne: { filter: { _id: removal.storedId, ...storage.filter, [TOMBSTONE_FIELD]: { $ne: true } }, replacement } });
+        }
+        for (const { row, ord } of writes) {
+            const number = seq++;
+            const filter = { _id: row.doc["_id"], ...storage.writer.partitionFilter(), $or: [{ [field]: partition }, { [TOMBSTONE_FIELD]: true }, { [field]: { $exists: false } }] };
+            operations.push({ replaceOne: { filter, replacement: { ...row.doc, _seq: number, _ord: ord ?? number }, upsert: true } });
+        }
+        return operations;
+    }
+    /**
+     * A run of a target in a stored form in one transaction (roadmap CLI-81): the rows that changed,
+     * and a tombstone for each row that the result does not have. The numbers are pending numbers of
+     * the storage: the run takes them after its reads, writes with them in the transaction, and ends
+     * them after the commit. A late run moves its documents to new numbers. The transaction does not
+     * write the counters, so the writes of the storage do not conflict with it.
+     */
+    async runStorageInTransaction(view, partition, target, storage, current, generations) {
+        const generation = current ?? new ObjectId();
+        const db = this.db;
+        const session = db.client.startSession();
+        let count = 0;
+        let removed = 0;
+        try {
+            session.startTransaction();
+            const fresh = await db.collection(this.viewsName).findOne({ _id: view._id, [this.partitionField]: partition }, { projection: { [this.statusField]: 1 }, session });
+            const now = fresh?.[this.statusField]?.generation;
+            if (!(now === undefined ? current === undefined : current !== undefined && now.equals(current)))
+                throw new PassSuperseded("another process changed the generation of the view");
+            const seen = new Set();
+            const writes = [];
+            for await (const group of this.resultGroups(view, partition, generations, generation, storage)) {
+                writes.push(...(await this.storageWrites(target, storage, group, seen, session)));
+                if (writes.length > this.maxTransactionRows)
+                    throw new Error(`more than ${this.maxTransactionRows} rows to write`);
+            }
+            const removals = [];
+            for await (const group of this.storageRemovals(target, storage, seen, session, this.maxTransactionRows - writes.length))
+                removals.push(...group);
+            count = writes.length + removals.length;
+            removed = removals.length;
+            const commit = async (first) => {
+                for (const operations of groups(this.storageOperations(storage, partition, writes, removals, first)))
+                    await target.bulkWrite(operations, { ordered: true, session });
+                if (!current && !(await this.setGeneration(view, partition, generation, { expected: undefined, session }))) {
+                    throw new PassSuperseded("another process named the first generation of the view");
+                }
+                await session.commitTransaction();
+            };
+            if (count === 0)
+                await commit(0);
+            else
+                await storage.writer.withPendingNumbers(db, count, commit);
+        }
+        catch (error) {
+            if (session.inTransaction())
+                await session.abortTransaction().catch(() => undefined);
+            throw error;
+        }
+        finally {
+            await session.endSession().catch(() => undefined);
+        }
+        if (removed > 0)
+            await storage.writer.prune(db);
+        return count > 0 || current === undefined;
+    }
+    /**
+     * A run of a target in a stored form in groups, outside a transaction (roadmap CLI-81): the rows
+     * of each group that changed, then the tombstones of the rows that the result does not have, with
+     * the pending numbers of the storage. The run writes in place, so a reader can see a part of it.
+     * It holds the lease of the view, and renews it after each group. The first run names a
+     * generation in the status of the view: the mark of a view that ran.
+     */
+    async runStorageInGroups(view, partition, target, storage, current, generations) {
+        const generation = current ?? new ObjectId();
+        const owner = new ObjectId().toHexString();
+        const db = this.db;
+        if (!(await this.takeLease(view, partition, owner, generation)))
+            throw new PassSuperseded("a pass of another process holds the view");
+        let changed = current === undefined;
+        let removed = false;
+        try {
+            const seen = new Set();
+            for await (const group of this.resultGroups(view, partition, generations, generation, storage)) {
+                const writes = await this.storageWrites(target, storage, group, seen, undefined);
+                if (writes.length > 0) {
+                    changed = true;
+                    await storage.writer.withPendingNumbers(db, writes.length, (first) => target.bulkWrite(this.storageOperations(storage, partition, writes, [], first), { ordered: true }));
+                }
+                if (!(await this.renewLease(view, partition, owner)))
+                    throw new PassSuperseded("the pass lost its lease");
+            }
+            for await (const removals of this.storageRemovals(target, storage, seen, undefined)) {
+                changed = true;
+                removed = true;
+                await storage.writer.withPendingNumbers(db, removals.length, (first) => target.bulkWrite(this.storageOperations(storage, partition, [], removals, first), { ordered: true }));
+                if (!(await this.renewLease(view, partition, owner)))
+                    throw new PassSuperseded("the pass lost its lease");
+            }
+            if (current === undefined) {
+                if (!(await this.setGeneration(view, partition, generation, { expected: undefined, owner }))) {
+                    throw new PassSuperseded("the pass lost its lease, or another process named the first generation of the view");
+                }
+            }
+            else {
+                await this.releaseLease(view, partition, owner);
+            }
+        }
+        catch (error) {
+            await this.releaseLease(view, partition, owner).catch(() => undefined);
+            throw error;
+        }
+        finally {
+            if (removed)
+                await storage.writer.prune(db).catch((error) => this.log("error", `${view._id}: the prune of the tombstones failed: ${String(error?.message ?? error)}`));
+        }
+        return changed;
+    }
     /** True while the view is the one that the materializer runs for this key. */
     isCurrentView(key, view) {
         return this.running.get(key)?.view === view;