@procedures/mongo 0.36.0 → 0.37.0

package/src/storage/materializer.ts

@@ -33,6 +33,8 @@
  *   1000 rows. It keeps nothing between passes.
  * - A change stream that ends (for example after `dropDatabase`) opens again after `rewatch`,
  *   and a pass catches up on the changes that the stream missed.
+ * - Each stream is open on the server before the materializer reads what it watches: the views
+ *   before the read of the views, the sources before a run. A change between the two reaches both.
  * - The status of each view (`ok`, `errored`, `cycle`) goes into the field `statusField` of its
  *   document. The materializer ignores an update of only that field.
  *
@@ -44,17 +46,20 @@
  * that moves a document to another partition also reaches the views of its old partition.
  *
  * Change streams and transactions need a replica set (MongoDB 6 or later). The target of a view is
- * a plain collection that only the materializer writes.
+ * a plain collection that only the materializer writes. A target of `storageTargets` is in the
+ * stored form of a `MongoDocumentStorage` instead (roadmap CLI-81): see `ViewStorageTarget`.
  *
  * The stop has the parts of a transport of a service: `stopIntake()`, `drain()`, `inFlight()` and
  * `close()`. A host composes them with no adapter. The timers come from the injected clock.
  */
 
 import { createHash } from "node:crypto";
-import { BSON, ObjectId, type ChangeStream, type ClientSession, type Collection, type Db, type Document } from "mongodb";
-import { optionalMs, realClock, type Cancel, type Clock, type Duration } from "@procedures/client";
-import { DEFAULT_STORED_FORM, TOMBSTONE_FIELD, type StoredForm } from "./document-storage.js";
+import { isDeepStrictEqual } from "node:util";
+import { BSON, ObjectId, type AnyBulkWriteOperation, type ChangeStream, type ClientSession, type Collection, type Db, type Document } from "mongodb";
+import { collectionChangeTopic, ms, optionalMs, realClock, type Cancel, type Clock, type CollectionChangeEvent, type Duration } from "@procedures/client";
+import { DEFAULT_STORED_FORM, DOCUMENT_STORAGE_FIELDS, MongoDocumentStorage, TOMBSTONE_FIELD, storageWriteTarget, type StorageWriteTarget, type StoredForm } from "./document-storage.js";
 import { scopePipeline } from "./pipeline-scope.js";
+import type { StorageHintPublisher } from "./storage-hints.js";
 
 /** The fields of a storage that a row never keeps: its sequence number and its tombstone flag (roadmap CLI-77). */
 const ROW_DROPPED_FIELDS = ["_seq", TOMBSTONE_FIELD] as const;
@@ -155,6 +160,82 @@ export interface ViewMaterializerOptions {
    * When the form or the function gives `undefined`, the change reaches the views of each partition.
    */
   storageSources?: Readonly<Record<string, ViewPartitionOf | ViewStorageSource>> | undefined;
+  /**
+   * The targets in the stored form of a `MongoDocumentStorage` (roadmap CLI-81), by the name of the
+   * target collection: the storage of the target, its stored form, or a `ViewStorageTarget`. The
+   * rows of such a target are documents of the storage, so a reader reads them through a
+   * `MongoDocumentStorage` with the same form, and `changes(since)` and `watchStorageChanges()`
+   * have each write of a run. See `ViewStorageTarget` for the rules. Give the same targets to each
+   * materializer of the views.
+   *
+   * A function gives the entry of a target collection at each run, or undefined for a target that
+   * is not in a stored form: an app can name its targets at run time. The materializer keeps the
+   * storage that it makes for a collection while the entry gives the same form, counters,
+   * `writeLease` and `maxTombstones`. An error of the function is an error of the view.
+   */
+  storageTargets?: Readonly<Record<string, ViewStorageTargetEntry>> | ((collection: string) => ViewStorageTargetEntry | undefined) | undefined;
+  /**
+   * The id of a row of a target in a stored form, from the `_id` of the row in the result. It must
+   * give a text, and give the same text for the same `_id` at each run.
+   * @default `defaultStorageRowId`: a text as it is, another value as its relaxed Extended JSON text
+   */
+  storageRowIdOf?: ((id: unknown) => string) | undefined;
+  /**
+   * The publisher of the hints of the targets that are not in `storageTargets`, for example the bus
+   * of the procedures. After each run that changes rows, the materializer publishes one hint with
+   * the ids of the changed rows and of the removed rows. A change of generation, a row id that is
+   * not a text and more than 10000 ids give a reset hint (`reset: true`): the subscribers read the
+   * view in full. The materializer publishes no hint of a target in a stored form: a watcher of
+   * the storage (`watchStorageChanges()`) does.
+   */
+  publish?: StorageHintPublisher | undefined;
+  /** The topic of the hints of a view, or undefined for no hint (default `collectionChangeTopic(view._id)`). */
+  hintTopic?: ((view: MaterializedView, partition: string) => string | undefined) | undefined;
+  /**
+   * The `_id` of a row in the result, from its stored `_id`, for the ids of a hint. It reverses
+   * `targetId`. The default reads the field `id` of the form of `partitionTargetId`.
+   */
+  rowIdOf?: ((storedId: unknown) => unknown) | undefined;
+}
+
+/**
+ * A storage with items of any type. The type of the items does not change the stored form, and a
+ * `MongoDocumentStorage<T>` of a given `T` is not a `MongoDocumentStorage<unknown>`.
+ */
+type AnyDocumentStorage = MongoDocumentStorage<any>;
+
+/** An entry of `storageTargets`: the storage of a target, its stored form, or a `ViewStorageTarget`. */
+export type ViewStorageTargetEntry = AnyDocumentStorage | StoredForm | ViewStorageTarget;
+
+/**
+ * A target of `storageTargets` (roadmap CLI-81).
+ *
+ * - The rows are documents of the storage: each row gets `_key`, `_seq`, `_ord`, the partition
+ *   fields of the form, and `_kid`, `_kpf` or `_json` as a write of the storage makes them. The
+ *   materializer encodes each row from its own content: the fields of a storage in a row (for
+ *   example of a source document that the pipeline passes through) do not go into the target.
+ * - The id of a row is `storageRowIdOf(_id)`: by default the `_id` of the row when it is a text,
+ *   else its relaxed Extended JSON text (`null` gives `"null"`, `5` gives `"5"`, an ObjectId gives
+ *   `{"$oid":"…"}`). The item has that id as its `_id`. Two rows of a result must not have the
+ *   same id.
+ * - A row also gets the partition field of the materializer with the partition of the view, so a
+ *   view that reads the target finds it. When that field is a partition field of the form, the
+ *   form must give it the partition of the view.
+ * - A run takes the numbers of its writes from the sequence of the storage, and a late run moves
+ *   to new numbers, as a write of the storage does. A removed row becomes a tombstone, and the
+ *   prune of the tombstones raises the floor of the change log.
+ * - A partition of the storage holds only the rows of one view. The run makes a tombstone of each
+ *   other live document in that partition. The materializer refuses a second view of the same
+ *   target collection and partition of the storage: that view has an error.
+ * - The materializer gives the target no images before the change. Turn on
+ *   `changeStreamPreAndPostImages` on the target when raw hard deletes can occur.
+ * - The rows have no generation, so `viewFilter()` and a `MongoViewStorage` do not read them.
+ */
+export interface ViewStorageTarget {
+  /** The storage of the target, or its stored form. The materializer writes with its own database, and takes the counters, `writeLease` and `maxTombstones` of a storage. */
+  target: AnyDocumentStorage | StoredForm;
+  /** The partition of the storage from the partition of a view (default the same partition). Undefined is the storage without a partition. */
+  storagePartitionOf?: ((viewPartition: string) => string | undefined) | undefined;
 }
 
 /** The function that gives the stored `_id` of a row of a view. */
@@ -212,6 +293,72 @@ function groups<T>(items: readonly T[]): T[][] {
   return result;
 }
 
+/** The most tries of a run in a transaction after a write conflict (code 112). After them, the run gives way, and comes back after `rewatch`. */
+const CONFLICT_RETRIES = 3;
+
+/** The most ids of a hint of a target not in a stored form. More ids give a reset hint. */
+const MAX_HINT_IDS = 10_000;
+
+/** The `_id` of a row in the result, from the stored `_id` of `partitionTargetId`. */
+export function defaultRowId(storedId: unknown): unknown {
+  return typeof storedId === "object" && storedId !== null && "id" in storedId ? (storedId as { id: unknown }).id : storedId;
+}
+
+/**
+ * The id of a row of a target in a stored form, from the `_id` of the row in the result: a text as
+ * it is, another value as its relaxed Extended JSON text. The text of a value is the same at each
+ * run (the order of the fields of an object is the order of the result). A row without `_id` has
+ * no id.
+ */
+export function defaultStorageRowId(id: unknown): string {
+  if (typeof id === "string") return id;
+  if (id === undefined) throw new TypeError("the row has no _id");
+  return BSON.EJSON.stringify(id as Document, { relaxed: true });
+}
+
+/** The hash of a stored document without its numbers (`_seq` and `_ord`): two documents with the same content give the same hash. */
+function contentHash(doc: Document): string {
+  const { _seq: _number, _ord: _place, ...rest } = doc;
+  return createHash("sha1")
+    .update(canonical(BSON.EJSON.serialize(rest, { relaxed: false })))
+    .digest("hex");
+}
+
+/** The values of a target in a stored form that make its storage. */
+interface StorageTargetValues {
+  readonly form: StoredForm;
+  readonly counters: string | undefined;
+  readonly writeLeaseMs: number | undefined;
+  readonly maxTombstones: number | undefined;
+}
+
+/** A target of `storageTargets`, ready to write: a storage on the database of the materializer. */
+interface StorageTargetEntry {
+  readonly storage: MongoDocumentStorage<unknown>;
+  readonly storagePartitionOf: (viewPartition: string) => string | undefined;
+}
+
+/** The storage of the partition of a view in its target (roadmap CLI-81). */
+interface TargetPartition {
+  readonly writer: StorageWriteTarget;
+  /** The filter of the rows of the view: the partition of the storage and the partition of the view. */
+  readonly filter: Document;
+  /** True when the partition field of the materializer goes in the content of an item (not a field of the storage or of the form). */
+  readonly partitionInContent: boolean;
+}
+
+/** A row of a target in a stored form that a run writes, with the place of its stored document (`_ord`) when it has one. */
+interface StorageWrite {
+  readonly row: StoredRow;
+  readonly ord: number | undefined;
+}
+
+/** A row of a target in a stored form that a run removes: its stored `_id`, and its id when the stored document tells one that the result does not have. */
+interface StorageRemoval {
+  readonly storedId: unknown;
+  readonly id: string | undefined;
+}
+
 /** The function that gives the partition of the views from the partition of a storage, or `undefined` when it does not know. */
 export type ViewPartitionOf = (storagePartition: string) => string | undefined;
 
@@ -226,6 +373,23 @@ export interface ViewStorageSource {
   viewPartitionOf: ViewPartitionOf;
 }
 
+/**
+ * This function waits until a change stream is open on the server: its first resume token. From
+ * then on, the stream gets each change. The driver sends the `aggregate` of a stream only after
+ * `watch()` returns, so a change before that point does not reach the stream. An error, an end or a
+ * close of the stream also ends the wait: its rewatch catches up (roadmap O63).
+ */
+function openOnServer(stream: ChangeStream): Promise<void> {
+  return new Promise((resolve) => {
+    const events = ["resumeTokenChanged", "error", "end", "close"] as const;
+    const done = (): void => {
+      for (const event of events) stream.off(event, done);
+      resolve();
+    };
+    for (const event of events) stream.on(event, done);
+  });
+}
+
 /** A view that the materializer knows. */
 interface RunningView {
   view: MaterializedView;
@@ -293,11 +457,19 @@ export class ViewMaterializer {
   private readonly graceMs: number;
   private readonly leaseMs: number;
   private readonly storageSources: ReadonlyMap<string, ViewStorageSource>;
+  /** The storages that the materializer made for the targets in a stored form, by collection, with the values that made them. */
+  private readonly storageTargetCache = new Map<string, StorageTargetValues & { storage: MongoDocumentStorage<unknown> }>();
+  /** The partition of a storage target that each running view writes (by the key of the view), so a second view of it is refused. */
+  private readonly storageClaims = new Map<string, { claim: string; view: string; partition: string }>();
+  private readonly storageRowIdOf: (id: unknown) => string;
+  private readonly rowIdOf: (storedId: unknown) => unknown;
   /** False after `stopIntake()`: a change starts no pass. */
   private intake = false;
   /** The work of each partition, one at a time: the passes and the rebalances. */
   private readonly chains = new Map<string, Promise<void>>();
   private readonly pending = new Set<Promise<unknown>>();
+  /** The changes of the set of views (a resync, a change of a view document), one at a time, in the order of the views stream. */
+  private viewChain: Promise<unknown> = Promise.resolve();
   private stopped = true;
 
   constructor(options: ViewMaterializerOptions) {
@@ -323,6 +495,63 @@ export class ViewMaterializer {
         return [source, normal];
       })
     );
+    // A record of targets is examined at once: a wrong entry is an error of the options
+    const targets = options.storageTargets;
+    if (targets !== undefined && typeof targets !== "function") {
+      for (const [collection, entry] of Object.entries(targets)) this.storageTargetOf(collection, entry);
+    }
+    this.storageRowIdOf = options.storageRowIdOf ?? defaultStorageRowId;
+    this.rowIdOf = options.rowIdOf ?? defaultRowId;
+  }
+
+  /**
+   * A target of `storageTargets` as a storage on the database of the materializer, so its writes
+   * and the transaction of a run use one client. A given storage gives its form, its counters,
+   * `writeLease` and `maxTombstones`.
+   */
+  private storageTargetOf(collection: string, entry: ViewStorageTargetEntry): StorageTargetEntry {
+    const wrapped = typeof entry === "object" && entry !== null && "target" in entry ? entry : { target: entry };
+    const storagePartitionOf = wrapped.storagePartitionOf ?? ((partition: string): string | undefined => partition);
+    if (typeof storagePartitionOf !== "function") throw new TypeError(`The storage target ${collection} needs a function storagePartitionOf`);
+    const target = wrapped.target;
+    let values: StorageTargetValues;
+    if (target instanceof MongoDocumentStorage) {
+      const parts = storageWriteTarget(target);
+      if (parts.collection !== collection) throw new TypeError(`The storage target ${collection} has a storage of the collection ${parts.collection}`);
+      values = { form: parts.form, counters: parts.counters, writeLeaseMs: parts.writeLeaseMs, maxTombstones: parts.maxTombstones };
+    } else if (typeof target === "object" && target !== null && typeof (target as Partial<StoredForm>).storedId === "function") {
+      values = { form: target as StoredForm, counters: undefined, writeLeaseMs: undefined, maxTombstones: undefined };
+    } else {
+      throw new TypeError(`The storage target ${collection} needs a MongoDocumentStorage, a stored form or a ViewStorageTarget`);
+    }
+    // The same values give the storage of the last run: its partitions keep their state (the floor document of O66)
+    const cached = this.storageTargetCache.get(collection);
+    if (cached && cached.form === values.form && cached.counters === values.counters && cached.writeLeaseMs === values.writeLeaseMs && cached.maxTombstones === values.maxTombstones) {
+      return { storage: cached.storage, storagePartitionOf };
+    }
+    const db = (): Db => {
+      if (!this.db) throw new Error("The materializer has not started");
+      return this.db;
+    };
+    const storage = new MongoDocumentStorage<unknown>({
+      collection,
+      db,
+      form: values.form,
+      transactions: "never",
+      ...(values.counters !== undefined ? { counters: values.counters } : {}),
+      ...(values.writeLeaseMs !== undefined ? { writeLease: ms(values.writeLeaseMs) } : {}),
+      ...(values.maxTombstones !== undefined ? { maxTombstones: values.maxTombstones } : {}),
+    });
+    this.storageTargetCache.set(collection, { ...values, storage });
+    return { storage, storagePartitionOf };
+  }
+
+  /** The target in a stored form of a collection at this time, or undefined. The function of `storageTargets` runs at each call. */
+  private storageTargetFor(collection: string): StorageTargetEntry | undefined {
+    const targets = this.options.storageTargets;
+    if (targets === undefined) return undefined;
+    const entry = typeof targets === "function" ? targets(collection) : Object.hasOwn(targets, collection) ? targets[collection] : undefined;
+    return entry === undefined || entry === null ? undefined : this.storageTargetOf(collection, entry);
   }
 
   private log(level: "info" | "warn" | "error", message: string): void {
@@ -368,8 +597,9 @@ export class ViewMaterializer {
     this.stopped = false;
     this.intake = true;
     this.db = await this.options.db();
+    // The stream of the views opens before the read of the views: a change between the two reaches both (roadmap O63)
+    await this.watchViews();
     const count = await this.resync();
-    this.watchViews();
     this.log("info", `started: ${count} view(s)`);
   }
 
@@ -452,23 +682,40 @@ export class ViewMaterializer {
     return typeof partition === "string" && partition !== "" ? partition : undefined;
   }
 
-  /** This method closes each view, and opens each view of the views collection again: a full catch-up. */
+  /**
+   * This method runs a change of the set of views after the changes before it. A change of a view
+   * document that comes during a resync thus applies after the read of the resync, not before it.
+   */
+  private inViewOrder<R>(work: () => R | Promise<R>): Promise<R> {
+    const run = this.viewChain.then(work);
+    this.viewChain = run.catch(() => undefined);
+    return run;
+  }
+
+  /**
+   * This method closes each view, and opens each view of the views collection again: a full
+   * catch-up. Open the stream of the views first: then a change after the read reaches the stream.
+   */
   private async resync(): Promise<number> {
-    for (const key of [...this.running.keys()]) this.tearDown(key);
-    const docs = (await this.db!.collection<MaterializedView>(this.viewsName).find({}).toArray()) as MaterializedView[];
-    for (const view of docs) await this.spinUp(view);
-    return docs.length;
+    const { count, registered } = await this.inViewOrder(async () => {
+      const docs = (await this.db!.collection<MaterializedView>(this.viewsName).find({}).toArray()) as MaterializedView[];
+      for (const key of [...this.running.keys()]) this.tearDown(key);
+      return { count: docs.length, registered: docs.filter((view) => this.register(view)) };
+    });
+    for (const view of registered) await this.activate(view);
+    return count;
   }
 
-  private watchViews(): void {
-    if (this.stopped || !this.db) return;
+  /** This method opens the stream of the views. The promise resolves when the stream is open on the server. */
+  private watchViews(): Promise<void> {
+    if (this.stopped || !this.db) return Promise.resolve();
     let stream: ChangeStream;
     try {
       stream = this.db.collection(this.viewsName).watch([], { fullDocument: "updateLookup" });
     } catch (error) {
       this.log("error", `cannot watch ${this.viewsName}: ${String((error as Error)?.message ?? error)}`);
-      this.later(() => this.watchViews());
-      return;
+      this.later(() => void this.watchViews());
+      return Promise.resolve();
     }
     const previous = this.viewsStream;
     if (previous && previous !== stream) previous.close().catch(() => undefined);
@@ -481,8 +728,8 @@ export class ViewMaterializer {
       this.log("warn", `the stream of ${this.viewsName} ${reason}: it opens again in ${this.rewatchMs} ms`);
       stream.close().catch(() => undefined);
       this.later(() => {
-        this.watchViews();
-        void this.track(this.resync()).catch((error: unknown) => this.log("error", `the resync failed: ${String((error as Error)?.message ?? error)}`));
+        // The changes while the stream was closed are lost: a resync catches up, after the new stream is open
+        void this.track(this.watchViews().then(() => this.resync())).catch((error: unknown) => this.log("error", `the resync failed: ${String((error as Error)?.message ?? error)}`));
       });
     };
 
@@ -499,6 +746,7 @@ export class ViewMaterializer {
     });
     stream.on("error", (error: Error) => rewatch(`failed: ${error.message}`));
     stream.on("close", () => rewatch("closed"));
+    return openOnServer(stream);
   }
 
   private async onViewChange(change: Document): Promise<void> {
@@ -506,39 +754,65 @@ export class ViewMaterializer {
     if (type === "insert" || type === "replace" || type === "update") {
       const view = change["fullDocument"] as MaterializedView | null;
       if (!view) return;
-      const partition = this.partitionOf(view);
-      if (partition !== undefined) this.tearDown(this.keyOf(partition, view._id));
-      await this.spinUp(view);
+      const registered = await this.inViewOrder(() => {
+        const partition = this.partitionOf(view);
+        if (partition !== undefined) this.tearDown(this.keyOf(partition, view._id));
+        return this.register(view);
+      });
+      if (registered) await this.activate(view);
     } else if (type === "delete") {
       const id = String((change["documentKey"] as Document)["_id"]);
-      const partitions = new Set<string>();
-      for (const [key, view] of [...this.running]) {
-        if (view.view._id === id) {
-          partitions.add(view.partition);
-          this.tearDown(key);
+      const partitions = await this.inViewOrder(() => {
+        const found = new Set<string>();
+        for (const [key, view] of [...this.running]) {
+          if (view.view._id === id) {
+            found.add(view.partition);
+            this.tearDown(key);
+          }
         }
-      }
+        return found;
+      });
       // A removed view can break a cycle of the other views
       for (const partition of partitions) await this.rebalance(partition);
     }
   }
 
-  private async spinUp(view: MaterializedView): Promise<void> {
+  /** This method adds a view to the views that the materializer knows. It gives false for a view that cannot run. */
+  private register(view: MaterializedView): boolean {
     const partition = this.partitionOf(view);
     if (partition === undefined) {
       this.log("warn", `the view ${String(view._id)} has no ${this.partitionField}: it does not run`);
-      return;
+      return false;
     }
     if (!Array.isArray(view.collections) || view.collections.length === 0 || typeof view.targetCollection !== "string" || !Array.isArray(view.pipeline)) {
       this.log("warn", `the view ${view._id} needs collections, a pipeline and a targetCollection: it does not run`);
-      return;
+      return false;
     }
     this.running.set(this.keyOf(partition, view._id), { view, partition, state: "active", streams: [] });
+    return true;
+  }
+
+  /** This method makes the index of the target of a registered view, and rebalances its partition: a new view runs. */
+  private async activate(view: MaterializedView): Promise<void> {
+    const partition = this.partitionOf(view)!;
     // The readers, the counts and the sweeps read the rows of a partition in a generation
     await this.db!
       .collection(view.targetCollection)
       .createIndex({ [this.partitionField]: 1, [VIEW_GENERATION_FIELD]: 1 })
       .catch((error: unknown) => this.log("error", `the index of ${view.targetCollection} failed: ${String((error as Error)?.message ?? error)}`));
+    // A target in a stored form also has the indexes of the storage: the change reads, the order and the tombstones
+    let storageTarget: StorageTargetEntry | undefined;
+    try {
+      storageTarget = this.storageTargetFor(view.targetCollection);
+    } catch (error) {
+      // The run of the view gives the error to its status
+      this.log("error", `the storage target ${view.targetCollection}: ${String((error as Error)?.message ?? error)}`);
+    }
+    if (storageTarget) {
+      await storageWriteTarget(storageTarget.storage)
+        .ready()
+        .catch((error: unknown) => this.log("error", `the indexes of the storage ${view.targetCollection} failed: ${String((error as Error)?.message ?? error)}`));
+    }
     await this.rebalance(partition);
   }
 
@@ -547,6 +821,7 @@ export class ViewMaterializer {
     if (!view) return;
     for (const stream of view.streams) stream.close().catch(() => undefined);
     this.running.delete(key);
+    this.storageClaims.delete(key);
   }
 
   /**
@@ -630,8 +905,9 @@ export class ViewMaterializer {
     return this.running.get(this.keyOf(running.partition, running.view._id)) === running;
   }
 
-  private watchSource(running: RunningView, source: string): void {
-    if (this.stopped || !this.db) return;
+  /** This method opens the stream of a source for a view. The promise resolves when the stream is open on the server. */
+  private watchSource(running: RunningView, source: string): Promise<void> {
+    if (this.stopped || !this.db) return Promise.resolve();
     let stream: ChangeStream;
     try {
       stream = this.db
@@ -640,9 +916,9 @@ export class ViewMaterializer {
     } catch (error) {
       this.log("error", `cannot watch ${source} for the view ${running.view._id}: ${String((error as Error)?.message ?? error)}`);
       this.later(() => {
-        if (this.isCurrent(running) && running.state === "active") this.watchSource(running, source);
+        if (this.isCurrent(running) && running.state === "active") void this.watchSource(running, source);
       });
-      return;
+      return Promise.resolve();
     }
     running.streams.push(stream);
     let again = false;
@@ -657,9 +933,8 @@ export class ViewMaterializer {
       this.log("warn", `the stream of ${source} for the view ${running.view._id} ${reason}: it opens again in ${this.rewatchMs} ms`);
       this.later(() => {
         if (!this.isCurrent(running) || running.state !== "active") return;
-        this.watchSource(running, source);
-        // The changes while the stream was closed are lost: a pass catches up
-        this.schedule(running.partition, source);
+        // The changes while the stream was closed are lost: a pass catches up, after the new stream is open
+        void this.watchSource(running, source).then(() => this.schedule(running.partition, source));
       });
     };
     stream.on("change", (change: Document) => {
@@ -670,6 +945,7 @@ export class ViewMaterializer {
     });
     stream.on("error", (error: Error) => rewatch(`failed: ${error.message}`));
     stream.on("close", () => rewatch("closed"));
+    return openOnServer(stream);
   }
 
   /** This method runs a function after `rewatch`, unless the materializer stops. The timer does not keep the process alive. */
@@ -744,7 +1020,10 @@ export class ViewMaterializer {
           }
         } else if (running.state === "cyclic" || running.streams.length === 0) {
           running.state = "active";
-          for (const source of view.collections) this.watchSource(running, source);
+          // The streams open before the run reads the sources: a change between the two reaches both (roadmap O63)
+          await Promise.all(view.collections.map((source) => this.watchSource(running, source)));
+          // A change of the view document during the wait replaced this view: the rebalance of the new one runs it
+          if (!this.isCurrent(running)) continue;
           await this.runAndWriteStatus(view, partition);
         }
       }
@@ -792,25 +1071,96 @@ export class ViewMaterializer {
    */
   private async run(view: MaterializedView, partition: string): Promise<boolean> {
     const target = this.db!.collection(view.targetCollection);
+    const storage = this.targetPartition(view, partition);
     const { status, now } = await this.statusOf(view, partition);
     if (status?.lease && status.lease.until instanceof Date && now < status.lease.until) throw new PassSuperseded("a pass of another process holds the view");
-    await this.dropOldGenerations(target, partition, status, now);
+    // The rows of a target in a stored form have no generation: a removal is a tombstone, never a delete of an old generation
+    if (!storage) await this.dropOldGenerations(target, partition, status, now);
     const current = status?.generation;
     const generations = await this.currentGenerations(partition);
-    const size = current ? await target.countDocuments({ [this.partitionField]: partition, [VIEW_GENERATION_FIELD]: current }) : 0;
+    // The floor document of the partition of the storage, before the first write (roadmap O66)
+    if (storage) await storage.writer.noteFloor(this.db!);
+    const size = storage
+      ? await target.countDocuments({ ...storage.filter, [TOMBSTONE_FIELD]: { $ne: true } })
+      : current
+        ? await target.countDocuments({ [this.partitionField]: partition, [VIEW_GENERATION_FIELD]: current })
+        : 0;
     if (size <= this.maxTransactionRows) {
-      try {
-        return await this.runInTransaction(view, partition, target, current, generations);
-      } catch (error) {
-        if (error instanceof ViewRunError || error instanceof PassSuperseded) throw error;
-        if (isWriteConflict(error)) throw new PassSuperseded(`another process wrote the view at the same time (${String((error as Error)?.message ?? error)})`);
-        this.log("warn", `${view._id}: the transaction failed (${String((error as Error)?.message ?? error)}): the pass writes a new generation in groups`);
+      for (let attempt = 1; ; attempt++) {
+        try {
+          return storage
+            ? await this.runStorageInTransaction(view, partition, target, storage, current, generations)
+            : await this.runInTransaction(view, partition, target, current, generations);
+        } catch (error) {
+          if (error instanceof ViewRunError || error instanceof PassSuperseded) throw error;
+          if (isWriteConflict(error)) {
+            // A write conflict tries again a few times after a short wait, so a steady stream of writes does not starve the view
+            if (attempt < CONFLICT_RETRIES) {
+              await this.pause(25 * 2 ** attempt * (1 + Math.random()));
+              continue;
+            }
+            throw new PassSuperseded(`another process wrote the view at the same time (${String((error as Error)?.message ?? error)})`);
+          }
+          this.log("warn", `${view._id}: the transaction failed (${String((error as Error)?.message ?? error)}): the pass writes ${storage ? "in groups" : "a new generation in groups"}`);
+          break;
+        }
       }
     }
+    if (storage) return this.runStorageInGroups(view, partition, target, storage, current, generations);
     await this.runInGroups(view, partition, target, current, generations);
     return true;
   }
 
+  /** This method waits on the clock of the materializer. The timer does not keep the process alive. */
+  private pause(delay: number): Promise<void> {
+    return new Promise((resolve) => {
+      const cancel = this.clock.after(Math.round(delay), resolve);
+      cancel.unref?.();
+    });
+  }
+
+  /**
+   * The storage of the partition of a view in its target, or undefined for a target that is not in
+   * `storageTargets`. A partition that the storage refuses, and a partition field of the form with
+   * another value than the partition of the view, are an error of the view.
+   */
+  private targetPartition(view: MaterializedView, partition: string): TargetPartition | undefined {
+    let entry: StorageTargetEntry | undefined;
+    try {
+      entry = this.storageTargetFor(view.targetCollection);
+    } catch (error) {
+      throw new ViewRunError(`the storage target ${view.targetCollection}: ${String((error as Error)?.message ?? error)}`);
+    }
+    if (!entry) return undefined;
+    let writer: StorageWriteTarget;
+    let storagePartition: string | undefined;
+    try {
+      storagePartition = entry.storagePartitionOf(partition);
+      writer = storageWriteTarget(storagePartition === undefined ? entry.storage : entry.storage.forPartition(storagePartition));
+    } catch (error) {
+      throw new ViewRunError(`the partition of the storage ${view.targetCollection}: ${String((error as Error)?.message ?? error)}`);
+    }
+    // One view writes a partition of a storage target: its run makes a tombstone of each other document of it
+    const claim = JSON.stringify([view.targetCollection, storagePartition ?? null]);
+    const key = this.keyOf(partition, view._id);
+    for (const [other, held] of this.storageClaims) {
+      if (other !== key && held.claim === claim && this.running.has(other)) {
+        const place = storagePartition === undefined ? "the storage" : `the partition ${storagePartition} of the storage`;
+        throw new ViewRunError(`the view ${held.view} (partition ${held.partition}) writes ${place} ${view.targetCollection} already: one view writes a partition of a storage target`);
+      }
+    }
+    this.storageClaims.set(key, { claim, view: view._id, partition });
+    const field = this.partitionField;
+    if (writer.partitionFields.has(field) && !isDeepStrictEqual(writer.partitionFields.get(field), partition)) {
+      throw new ViewRunError(`the stored form of ${view.targetCollection} gives the field ${field} another value than the partition ${partition}`);
+    }
+    return {
+      writer,
+      filter: { ...writer.partitionFilter(), [field]: partition },
+      partitionInContent: !writer.partitionFields.has(field) && !DOCUMENT_STORAGE_FIELDS.includes(field),
+    };
+  }
+
   /**
    * The status of a view in the database (the current generation, and the previous one), and the
    * time of the server: the times of the generations are times of the server.
@@ -852,7 +1202,7 @@ export class ViewMaterializer {
   }
 
   /** The rows of the result, in groups of 1000. A failure of the pipeline, a row without `_id` and too many rows are a `ViewRunError`. */
-  private async *resultGroups(view: MaterializedView, partition: string, generations: readonly ObjectId[], generation: ObjectId): AsyncGenerator<StoredRow[]> {
+  private async *resultGroups(view: MaterializedView, partition: string, generations: readonly ObjectId[], generation: ObjectId, storage?: TargetPartition): AsyncGenerator<StoredRow[]> {
     let pipeline: Document[];
     try {
       pipeline = this.scoped(view.pipeline, partition, view, generations);
@@ -874,6 +1224,14 @@ export class ViewMaterializer {
         }
         if (row === null) break;
         if (++count > this.maxRows) throw new ViewRunError(`the result has more than ${this.maxRows} rows`);
+        if (storage) {
+          group.push(this.storageRowOf(row, partition, storage));
+          if (group.length === WRITE_GROUP) {
+            yield group;
+            group = [];
+          }
+          continue;
+        }
         // The fields of the materializer, for a row that comes from a target: not in the content
         const { _id, [this.partitionField]: _partition, [VIEW_GENERATION_FIELD]: _generation, [VIEW_HASH_FIELD]: _hash, ...content } = row;
         if (_id === undefined || _id === null) throw new ViewRunError("each row of the result needs an _id");
@@ -900,6 +1258,43 @@ export class ViewMaterializer {
     }
   }
 
+  /**
+   * A row of the result as a document of a target in a stored form (roadmap CLI-81). The content is
+   * the row without the fields of the materializer and of a storage: a pass-through row of a source
+   * in a stored form loses its `_key`, `_kid`, `_kpf`, `_ord` and `_json`, and the encoding of the
+   * storage makes them again from the row (`_json` from the content of the row). A field of the row
+   * with the name of a partition field of the form takes the value of the partition of the target.
+   * The numbers (`_seq`, `_ord`) come at the write.
+   */
+  private storageRowOf(row: Document, partition: string, storage: TargetPartition): StoredRow {
+    const { _id: rowId, [VIEW_GENERATION_FIELD]: _generation, [VIEW_HASH_FIELD]: _hash, ...content } = row;
+    let _id: string;
+    try {
+      _id = this.storageRowIdOf(rowId);
+    } catch (error) {
+      throw new ViewRunError(`the id of a row of a target in a stored form: ${String((error as Error)?.message ?? error)}`);
+    }
+    if (typeof _id !== "string") throw new ViewRunError(`storageRowIdOf must give a text, not ${typeof _id}`);
+    const field = this.partitionField;
+    delete content[field];
+    for (const name of DOCUMENT_STORAGE_FIELDS) delete content[name];
+    for (const [name, value] of storage.writer.partitionFields) {
+      if (!(name in content)) continue;
+      if (value === undefined) delete content[name];
+      else content[name] = value;
+    }
+    if (storage.partitionInContent) content[field] = partition;
+    let doc: Document;
+    try {
+      doc = storage.writer.encode(_id, { _id, ...content }, 0, 0);
+    } catch (error) {
+      throw new ViewRunError(`the row ${JSON.stringify(_id)}: ${String((error as Error)?.message ?? error)}`);
+    }
+    // A view that reads the target finds the row by the partition field, also in an item stored as JSON text
+    if (doc[field] === undefined) doc[field] = partition;
+    return { id: _id, key: idKey(doc["_id"]), hash: contentHash(doc), doc };
+  }
+
   private replaceOf(row: StoredRow, partition: string): Document {
     return { replaceOne: { filter: { _id: row.doc["_id"], [this.partitionField]: partition }, replacement: row.doc, upsert: true } };
   }
@@ -918,6 +1313,7 @@ export class ViewMaterializer {
     const generation = current ?? new ObjectId();
     const session = this.db!.client.startSession();
     let changed = current === undefined;
+    const hint = { changed: [] as unknown[], deleted: [] as unknown[] };
     try {
       session.startTransaction();
       // In the transaction: a change of generation by another process since the start supersedes this pass
@@ -930,8 +1326,10 @@ export class ViewMaterializer {
           .find({ _id: { $in: group.map((row) => row.doc["_id"]) }, [this.partitionField]: partition } as Document, { projection: { [VIEW_HASH_FIELD]: 1 }, session })
           .toArray();
         const hashes = new Map(stored.map((doc) => [idKey(doc["_id"]), doc[VIEW_HASH_FIELD]]));
-        const writes = group.filter((row) => hashes.get(row.key) !== row.hash).map((row) => this.replaceOf(row, partition));
+        const rows = group.filter((row) => hashes.get(row.key) !== row.hash);
+        const writes = rows.map((row) => this.replaceOf(row, partition));
         for (const row of group) seen.add(row.key);
+        for (const row of rows) hint.changed.push(row.id);
         if (writes.length > 0) {
           await target.bulkWrite(writes as never, { ordered: false, session });
           changed = true;
@@ -947,6 +1345,7 @@ export class ViewMaterializer {
       }
       for (const group of groups(stale)) await target.deleteMany({ [this.partitionField]: partition, _id: { $in: group } } as Document, { session });
       if (stale.length > 0) changed = true;
+      for (const storedId of stale) hint.deleted.push(this.rowIdOf(storedId));
       if (!current && !(await this.setGeneration(view, partition, generation, { expected: undefined, session }))) {
         throw new PassSuperseded("another process named the first generation of the view");
       }
@@ -957,9 +1356,35 @@ export class ViewMaterializer {
     } finally {
       await session.endSession().catch(() => undefined);
     }
+    if (hint.changed.length > 0 || hint.deleted.length > 0) this.publishHint(view, partition, hint);
     return changed;
   }
 
+  /**
+   * This method publishes the hint of a run of a target that is not in a stored form: one hint
+   * for the run, with the ids of the changed rows and of the removed rows. Without ids (a change
+   * of generation), with an id that is not a text, or with more than `MAX_HINT_IDS` ids, it is a
+   * reset hint. A failure of the publisher goes to the log: the run does not fail.
+   */
+  private publishHint(view: MaterializedView, partition: string, ids: { changed: readonly unknown[]; deleted: readonly unknown[] } | undefined): void {
+    const publish = this.options.publish;
+    if (!publish) return;
+    const fail = (error: unknown): void => this.log("error", `the hint of ${view._id} failed: ${String((error as Error)?.message ?? error)}`);
+    try {
+      const topic = this.options.hintTopic ? this.options.hintTopic(view, partition) : collectionChangeTopic(view._id);
+      if (topic === undefined) return;
+      const texts = (values: readonly unknown[]): values is readonly string[] => values.every((value) => typeof value === "string");
+      const event: CollectionChangeEvent =
+        ids && ids.changed.length + ids.deleted.length <= MAX_HINT_IDS && texts(ids.changed) && texts(ids.deleted)
+          ? { changed: [...new Set(ids.changed)], deleted: [...new Set(ids.deleted)] }
+          : { changed: [], deleted: [], reset: true };
+      const result = typeof publish === "function" ? publish(topic, event) : publish.emit(topic, event);
+      if (result instanceof Promise) result.catch(fail);
+    } catch (error) {
+      fail(error);
+    }
+  }
+
   /**
    * A run in groups, outside a transaction: a whole new generation, then the change of the status to
    * it. The readers of `viewFilter` move to it at once. The rows of no generation go at once: no
@@ -995,6 +1420,8 @@ export class ViewMaterializer {
       await this.releaseLease(view, partition, owner).catch(() => undefined);
       throw error;
     }
+    // The readers of the view moved to the new generation at once: a gap, so the subscribers read the view in full
+    this.publishHint(view, partition, undefined);
     await target.deleteMany({ [this.partitionField]: partition, [VIEW_GENERATION_FIELD]: { $exists: false } });
     // At the end of the grace time, the previous generation goes. The time of the server decides it, so a timer that ends early removes nothing
     const key = this.keyOf(partition, view._id);
@@ -1010,6 +1437,202 @@ export class ViewMaterializer {
     this.retries.add(cancel);
   }
 
+  /**
+   * The rows of a group that a run writes in a target in a stored form: each row whose stored
+   * document is absent, is a tombstone, or has other content (the numbers do not count). A row
+   * keeps the place (`_ord`) of its live stored document. Two rows with the same `_id` in a result
+   * are an error of the view.
+   */
+  private async storageWrites(target: Collection, storage: TargetPartition, group: readonly StoredRow[], seen: Set<string>, session: ClientSession | undefined): Promise<StorageWrite[]> {
+    for (const row of group) {
+      if (seen.has(row.key)) throw new ViewRunError(`the result has two rows with the _id ${JSON.stringify(row.id)}`);
+      seen.add(row.key);
+    }
+    const stored = await target
+      .find({ _id: { $in: group.map((row) => row.doc["_id"]) }, ...storage.writer.partitionFilter() } as Document, session ? { session } : {})
+      .toArray();
+    const byKey = new Map(stored.map((doc) => [idKey(doc["_id"]), doc]));
+    const writes: StorageWrite[] = [];
+    for (const row of group) {
+      const doc = byKey.get(row.key);
+      const live = doc !== undefined && doc[TOMBSTONE_FIELD] !== true;
+      if (live && contentHash(doc) === row.hash) continue;
+      writes.push({ row, ord: live && typeof doc["_ord"] === "number" ? doc["_ord"] : undefined });
+    }
+    return writes;
+  }
+
+  /**
+   * The live documents of the partition of a view in a target in a stored form that the result
+   * does not have. A document whose id the result does not have becomes the tombstone of that id. A
+   * document that tells no id, or the id of a row of the result (under another stored `_id`),
+   * becomes a tombstone without `_key`: no reader reads a delete of an item from it. More than
+   * `limit` documents is an error, so a transaction goes in groups.
+   */
+  private async *storageRemovals(
+    target: Collection,
+    storage: TargetPartition,
+    seen: ReadonlySet<string>,
+    session: ClientSession | undefined,
+    limit = Number.POSITIVE_INFINITY
+  ): AsyncGenerator<StorageRemoval[]> {
+    let group: StorageRemoval[] = [];
+    let count = 0;
+    const cursor = target.find({ ...storage.filter, [TOMBSTONE_FIELD]: { $ne: true } } as Document, { projection: { _id: 1, _key: 1 }, ...(session ? { session } : {}) });
+    try {
+      for await (const doc of cursor) {
+        if (seen.has(idKey(doc["_id"]))) continue;
+        if (++count > limit) throw new Error(`more than ${limit} rows to remove`);
+        let id = storage.writer.idOfDoc(doc);
+        try {
+          if (id !== undefined && seen.has(idKey(storage.writer.storedId(id)))) id = undefined;
+        } catch {
+          id = undefined;
+        }
+        group.push({ storedId: doc["_id"], id });
+        if (group.length === WRITE_GROUP) {
+          yield group;
+          group = [];
+        }
+      }
+      if (group.length > 0) yield group;
+    } finally {
+      await cursor.close().catch(() => undefined);
+    }
+  }
+
+  /**
+   * The writes of a target in a stored form with their numbers from `first`: the tombstones first,
+   * then the rows. A new row takes its own number as its place. The filter of a write holds the
+   * partition: a row of another partition of the view fails with a duplicate key.
+   */
+  private storageOperations(storage: TargetPartition, partition: string, writes: readonly StorageWrite[], removals: readonly StorageRemoval[], first: number): AnyBulkWriteOperation<Document>[] {
+    const field = this.partitionField;
+    const operations: AnyBulkWriteOperation<Document>[] = [];
+    let seq = first;
+    for (const removal of removals) {
+      const number = seq++;
+      const tomb: Document = removal.id === undefined ? { _id: removal.storedId, _seq: number, [TOMBSTONE_FIELD]: true } : storage.writer.tombstone(removal.id, number);
+      const replacement: Document = { ...tomb, _id: removal.storedId };
+      for (const [name, value] of storage.writer.partitionFields) if (value !== undefined) replacement[name] = value;
+      replacement[field] = partition;
+      operations.push({ replaceOne: { filter: { _id: removal.storedId, ...storage.filter, [TOMBSTONE_FIELD]: { $ne: true } } as Document, replacement } });
+    }
+    for (const { row, ord } of writes) {
+      const number = seq++;
+      const filter = { _id: row.doc["_id"], ...storage.writer.partitionFilter(), $or: [{ [field]: partition }, { [TOMBSTONE_FIELD]: true }, { [field]: { $exists: false } }] };
+      operations.push({ replaceOne: { filter, replacement: { ...row.doc, _seq: number, _ord: ord ?? number }, upsert: true } });
+    }
+    return operations;
+  }
+
+  /**
+   * A run of a target in a stored form in one transaction (roadmap CLI-81): the rows that changed,
+   * and a tombstone for each row that the result does not have. The numbers are pending numbers of
+   * the storage: the run takes them after its reads, writes with them in the transaction, and ends
+   * them after the commit. A late run moves its documents to new numbers. The transaction does not
+   * write the counters, so the writes of the storage do not conflict with it.
+   */
+  private async runStorageInTransaction(
+    view: MaterializedView,
+    partition: string,
+    target: Collection,
+    storage: TargetPartition,
+    current: ObjectId | undefined,
+    generations: readonly ObjectId[]
+  ): Promise<boolean> {
+    const generation = current ?? new ObjectId();
+    const db = this.db!;
+    const session = db.client.startSession();
+    let count = 0;
+    let removed = 0;
+    try {
+      session.startTransaction();
+      const fresh = await db.collection(this.viewsName).findOne({ _id: view._id, [this.partitionField]: partition } as Document, { projection: { [this.statusField]: 1 }, session });
+      const now = (fresh?.[this.statusField] as Partial<MaterializedViewStatus> | undefined)?.generation;
+      if (!(now === undefined ? current === undefined : current !== undefined && now.equals(current))) throw new PassSuperseded("another process changed the generation of the view");
+      const seen = new Set<string>();
+      const writes: StorageWrite[] = [];
+      for await (const group of this.resultGroups(view, partition, generations, generation, storage)) {
+        writes.push(...(await this.storageWrites(target, storage, group, seen, session)));
+        if (writes.length > this.maxTransactionRows) throw new Error(`more than ${this.maxTransactionRows} rows to write`);
+      }
+      const removals: StorageRemoval[] = [];
+      for await (const group of this.storageRemovals(target, storage, seen, session, this.maxTransactionRows - writes.length)) removals.push(...group);
+      count = writes.length + removals.length;
+      removed = removals.length;
+      const commit = async (first: number): Promise<void> => {
+        for (const operations of groups(this.storageOperations(storage, partition, writes, removals, first))) await target.bulkWrite(operations, { ordered: true, session });
+        if (!current && !(await this.setGeneration(view, partition, generation, { expected: undefined, session }))) {
+          throw new PassSuperseded("another process named the first generation of the view");
+        }
+        await session.commitTransaction();
+      };
+      if (count === 0) await commit(0);
+      else await storage.writer.withPendingNumbers(db, count, commit);
+    } catch (error) {
+      if (session.inTransaction()) await session.abortTransaction().catch(() => undefined);
+      throw error;
+    } finally {
+      await session.endSession().catch(() => undefined);
+    }
+    if (removed > 0) await storage.writer.prune(db);
+    return count > 0 || current === undefined;
+  }
+
+  /**
+   * A run of a target in a stored form in groups, outside a transaction (roadmap CLI-81): the rows
+   * of each group that changed, then the tombstones of the rows that the result does not have, with
+   * the pending numbers of the storage. The run writes in place, so a reader can see a part of it.
+   * It holds the lease of the view, and renews it after each group. The first run names a
+   * generation in the status of the view: the mark of a view that ran.
+   */
+  private async runStorageInGroups(
+    view: MaterializedView,
+    partition: string,
+    target: Collection,
+    storage: TargetPartition,
+    current: ObjectId | undefined,
+    generations: readonly ObjectId[]
+  ): Promise<boolean> {
+    const generation = current ?? new ObjectId();
+    const owner = new ObjectId().toHexString();
+    const db = this.db!;
+    if (!(await this.takeLease(view, partition, owner, generation))) throw new PassSuperseded("a pass of another process holds the view");
+    let changed = current === undefined;
+    let removed = false;
+    try {
+      const seen = new Set<string>();
+      for await (const group of this.resultGroups(view, partition, generations, generation, storage)) {
+        const writes = await this.storageWrites(target, storage, group, seen, undefined);
+        if (writes.length > 0) {
+          changed = true;
+          await storage.writer.withPendingNumbers(db, writes.length, (first) => target.bulkWrite(this.storageOperations(storage, partition, writes, [], first), { ordered: true }));
+        }
+        if (!(await this.renewLease(view, partition, owner))) throw new PassSuperseded("the pass lost its lease");
+      }
+      for await (const removals of this.storageRemovals(target, storage, seen, undefined)) {
+        changed = true;
+        removed = true;
+        await storage.writer.withPendingNumbers(db, removals.length, (first) => target.bulkWrite(this.storageOperations(storage, partition, [], removals, first), { ordered: true }));
+        if (!(await this.renewLease(view, partition, owner))) throw new PassSuperseded("the pass lost its lease");
+      }
+      if (current === undefined) {
+        if (!(await this.setGeneration(view, partition, generation, { expected: undefined, owner }))) {
+          throw new PassSuperseded("the pass lost its lease, or another process named the first generation of the view");
+        }
+      } else {
+        await this.releaseLease(view, partition, owner);
+      }
+    } catch (error) {
+      await this.releaseLease(view, partition, owner).catch(() => undefined);
+      throw error;
+    } finally {
+      if (removed) await storage.writer.prune(db).catch((error: unknown) => this.log("error", `${view._id}: the prune of the tombstones failed: ${String((error as Error)?.message ?? error)}`));
+    }
+    return changed;
+  }
+
   /** True while the view is the one that the materializer runs for this key. */
   private isCurrentView(key: string, view: MaterializedView): boolean {
     return this.running.get(key)?.view === view;