@procedures/mongo 0.34.0 → 0.35.1

package/src/storage/index.ts

@@ -4,8 +4,20 @@
 
 export { MongoStorage } from "./mongo-storage.js";
 export type { MongoStorageOptions } from "./mongo-storage.js";
-export { MongoDocumentStorage, selectorPrefilter } from "./document-storage.js";
-export type { MongoDocumentStorageOptions } from "./document-storage.js";
+// The stored form of a MongoDocumentStorage is an option (roadmap CLI-76)
+export {
+  MongoDocumentStorage,
+  selectorPrefilter,
+  DOCUMENT_STORAGE_FIELDS,
+  TOMBSTONE_FIELD,
+  DEFAULT_STORED_FORM,
+  decodeStoredDocument,
+  versionOfStored,
+} from "./document-storage.js";
+export type { MongoDocumentStorageOptions, SelectorPrefilterOptions, StoredForm, DecodeStoredDocumentOptions, StoredDocumentEntry } from "./document-storage.js";
+// The change hints of a storage collection from its change stream (roadmap CLI-77)
+export { watchStorageChanges, storageChangeOf } from "./storage-hints.js";
+export type { StorageChange, StorageChangeHint, StorageChangesWatch, StorageHintPublisher, WatchStorageChangesOptions } from "./storage-hints.js";
 export {
   BLOCKED_PIPELINE_OPERATORS,
   ALLOWED_PIPELINE_STAGES,
@@ -19,7 +31,7 @@ export {
 export type { PipelineLimits, PipelineScope } from "./pipeline-scope.js";
 // Materialized views driven by change streams (Graph's split plan, P4)
 export { ViewMaterializer, identityPartition, partitionTargetId, viewFilter, VIEW_GENERATION_FIELD, VIEW_HASH_FIELD } from "./materializer.js";
-export type { MaterializedView, MaterializedViewStatus, ViewMaterializerOptions, ViewPartitionOf, ViewPipelineScope, ViewTargetId } from "./materializer.js";
+export type { MaterializedView, MaterializedViewStatus, ViewMaterializerOptions, ViewPartitionOf, ViewPipelineScope, ViewStorageSource, ViewTargetId } from "./materializer.js";
 // A read-only storage of a view for the collection procedures: one generation for each read (roadmap CLI-38)
 export { MongoViewStorage } from "./view-storage.js";
 export type { MongoViewStorageOptions } from "./view-storage.js";

package/src/storage/materializer.ts

@@ -38,8 +38,9 @@
  *
  * A change reaches only the views of its partition when its partition is known. A delete has no
  * document: its partition comes from the pre-image (a collection with
- * `changeStreamPreAndPostImages`), or from the id in a source of `storageSources`, through its
- * mapping. Else it reaches the views of each partition of its source. With pre-images, an update
+ * `changeStreamPreAndPostImages`), or, in a source of `storageSources`, from `partitionOf` of the
+ * stored form and the mapping of the source (roadmap CLI-76). Else it reaches the views of each
+ * partition of its source. With pre-images, an update
  * that moves a document to another partition also reaches the views of its old partition.
  *
  * Change streams and transactions need a replica set (MongoDB 6 or later). The target of a view is
@@ -52,9 +53,12 @@
 import { createHash } from "node:crypto";
 import { BSON, ObjectId, type ChangeStream, type ClientSession, type Collection, type Db, type Document } from "mongodb";
 import { optionalMs, realClock, type Cancel, type Clock, type Duration } from "@procedures/client";
-import { SEPARATOR } from "./document-storage.js";
+import { DEFAULT_STORED_FORM, TOMBSTONE_FIELD, type StoredForm } from "./document-storage.js";
 import { scopePipeline } from "./pipeline-scope.js";
 
+/** The fields of a storage that a row never keeps: its sequence number and its tombstone flag (roadmap CLI-77). */
+const ROW_DROPPED_FIELDS = ["_seq", TOMBSTONE_FIELD] as const;
+
 /** The definition of a view, as a document of the views collection. */
 export interface MaterializedView {
   _id: string;
@@ -142,13 +146,15 @@ export interface ViewMaterializerOptions {
    */
   passLease?: Duration | undefined;
   /**
-   * The sources that are collections of a `MongoDocumentStorage`, each with the function that
-   * gives the partition of the views from the partition of the storage. The stored id of a document
-   * starts with its storage partition and U+001F. A delete without a pre-image then reaches only the
-   * views of its partition. Give `identityPartition` when the two partitions are the same. Without a
-   * function, or when it gives `undefined`, the delete reaches the views of each partition.
+   * The sources that are collections of a `MongoDocumentStorage`. For each one, a function that
+   * gives the partition of the views from the partition of the storage (the storage has the default
+   * form), or a `ViewStorageSource` with the stored form of the storage (`storage.form`) and that
+   * function. A change without a pre-image then reaches only the views of its partition: the
+   * `partitionOf` of the form reads the storage partition from the document of the change (for a
+   * delete, only `documentKey`). Give `identityPartition` when the two partitions are the same.
+   * When the form or the function gives `undefined`, the change reaches the views of each partition.
    */
-  storageSources?: Readonly<Record<string, ViewPartitionOf>> | undefined;
+  storageSources?: Readonly<Record<string, ViewPartitionOf | ViewStorageSource>> | undefined;
 }
 
 /** The function that gives the stored `_id` of a row of a view. */
@@ -212,6 +218,14 @@ export type ViewPartitionOf = (storagePartition: string) => string | undefined;
 /** The partition of the views is the partition of the storage. */
 export const identityPartition: ViewPartitionOf = (storagePartition) => storagePartition;
 
+/** A source of `storageSources` with its stored form (roadmap CLI-76). */
+export interface ViewStorageSource {
+  /** The stored form of the storage, for example `storage.form`. Its `partitionOf` reads the storage partition of a document. */
+  form: Pick<StoredForm, "partitionOf">;
+  /** The partition of the views from the partition of the storage. */
+  viewPartitionOf: ViewPartitionOf;
+}
+
 /** A view that the materializer knows. */
 interface RunningView {
   view: MaterializedView;
@@ -278,7 +292,7 @@ export class ViewMaterializer {
   private readonly maxTransactionRows: number;
   private readonly graceMs: number;
   private readonly leaseMs: number;
-  private readonly storageSources: ReadonlyMap<string, ViewPartitionOf>;
+  private readonly storageSources: ReadonlyMap<string, ViewStorageSource>;
   /** False after `stopIntake()`: a change starts no pass. */
   private intake = false;
   /** The work of each partition, one at a time: the passes and the rebalances. */
@@ -300,7 +314,15 @@ export class ViewMaterializer {
     this.maxTransactionRows = options.maxTransactionRows ?? 10_000;
     this.graceMs = optionalMs(options.generationGrace, "generationGrace") ?? 60_000;
     this.leaseMs = Math.round(optionalMs(options.passLease, "passLease") ?? 60_000);
-    this.storageSources = new Map(Object.entries(options.storageSources ?? {}));
+    this.storageSources = new Map(
+      Object.entries(options.storageSources ?? {}).map(([source, entry]): [string, ViewStorageSource] => {
+        const normal = typeof entry === "function" ? { form: DEFAULT_STORED_FORM, viewPartitionOf: entry } : entry;
+        if (typeof normal?.form?.partitionOf !== "function" || typeof normal.viewPartitionOf !== "function") {
+          throw new TypeError(`The storage source ${source} needs a function, or a form with partitionOf and a viewPartitionOf`);
+        }
+        return [source, normal];
+      })
+    );
   }
 
   private log(level: "info" | "warn" | "error", message: string): void {
@@ -310,11 +332,13 @@ export class ViewMaterializer {
 
   /**
    * The guard of a pipeline of a view. A target of another view counts only in its current
-   * generation. A source document has no generation.
+   * generation. A source document has no generation. A tombstone of a `MongoDocumentStorage` (a
+   * deleted item) is not a row of a source (roadmap CLI-79).
    */
   private scoped(pipeline: Document[], partition: string, view: MaterializedView, generations: readonly ObjectId[]): Document[] {
     const match = {
       [this.partitionField]: partition,
+      [TOMBSTONE_FIELD]: { $ne: true },
       $or: [{ [VIEW_GENERATION_FIELD]: { $exists: false } }, { [VIEW_GENERATION_FIELD]: { $in: [...generations] } }],
     };
     if (this.options.scope) return this.options.scope(pipeline, partition, view, match);
@@ -570,21 +594,32 @@ export class ViewMaterializer {
 
   /**
    * True for a change without a pre-image (a delete, a replace, an update), in a source of
-   * `storageSources`, whose id names a storage partition that maps to another partition of the
-   * views. A document that is in the partition after the change, an id without the separator, a
-   * mapping that gives `undefined` and a mapping that fails do not tell another partition: false.
+   * `storageSources`, whose document names a storage partition that maps to another partition of
+   * the views. The document is `fullDocument`, else `fullDocumentBeforeChange`, else `documentKey`,
+   * and `partitionOf` of the form reads its storage partition. A document that is in the partition
+   * after the change, a document that does not tell its partition, a key (`documentKey`) that tells
+   * another storage partition than the document after the change (a move), a mapping that gives
+   * `undefined` and a function that fails do not tell another partition: false.
    */
   private isOtherPartition(change: Document, source: string, partition: string): boolean {
-    const viewPartitionOf = this.storageSources.get(source);
-    if (!viewPartitionOf || change["fullDocumentBeforeChange"]) return false;
+    const storageSource = this.storageSources.get(source);
+    if (!storageSource || change["fullDocumentBeforeChange"]) return false;
     if (!["delete", "replace", "update"].includes(change["operationType"] as string)) return false;
-    if ((change["fullDocument"] as Document | null | undefined)?.[this.partitionField] === partition) return false;
-    const id = (change["documentKey"] as Document | undefined)?.["_id"];
-    const at = typeof id === "string" ? id.indexOf(SEPARATOR) : -1;
-    if (at <= 0) return false;
+    const after = change["fullDocument"] as Document | null | undefined;
+    if (after?.[this.partitionField] === partition) return false;
+    const key = change["documentKey"] as Document | null | undefined;
+    const doc = (after ?? (change["fullDocumentBeforeChange"] as Document | null | undefined) ?? key) as Document | null | undefined;
+    if (!doc) return false;
     let viewPartition: string | undefined;
     try {
-      viewPartition = viewPartitionOf((id as string).slice(0, at));
+      const storagePartition = storageSource.form.partitionOf(doc);
+      if (typeof storagePartition !== "string") return false;
+      // A document whose key tells another partition than the document after the change can have moved out of this partition
+      if (after && key) {
+        const before = storageSource.form.partitionOf(key);
+        if (before !== undefined && before !== storagePartition) return false;
+      }
+      viewPartition = storageSource.viewPartitionOf(storagePartition);
     } catch {
       return false;
     }
@@ -842,6 +877,10 @@ export class ViewMaterializer {
         // The fields of the materializer, for a row that comes from a target: not in the content
         const { _id, [this.partitionField]: _partition, [VIEW_GENERATION_FIELD]: _generation, [VIEW_HASH_FIELD]: _hash, ...content } = row;
         if (_id === undefined || _id === null) throw new ViewRunError("each row of the result needs an _id");
+        // The sequence number and the tombstone flag of a MongoDocumentStorage, for a row that passes a
+        // source document through: a row is no write of a storage, so a watcher of the target must not
+        // read them (roadmap CLI-77). The other fields, for example _json and _key, are the content of the row.
+        for (const name of ROW_DROPPED_FIELDS) delete content[name];
         const stored = this.targetId(_id, partition, generation);
         const hash = createHash("sha1").update(canonical(content)).digest("hex");
         group.push({

package/src/storage/own-writes.ts

@@ -0,0 +1,157 @@
+/**
+ * The writes of this process (roadmap CLI-77). Each write of a `MongoDocumentStorage` takes
+ * numbers of the sequence of its collection (`_seq`). While a watcher of the collection
+ * (`watchStorageChanges`) tracks the collection, the storage notes here each number that it takes,
+ * and the stored `_id` of each tombstone that it prunes. The watcher then knows the change events
+ * of the writes of this process.
+ *
+ * - The numbers: the collection procedures of this process publish the events of these writes
+ *   already, so a watcher with `ownWrites: "skip"` skips them.
+ * - The pruned tombstones: the delete of a pruned tombstone deletes no item. Without the image
+ *   before the change, the event of the delete has only the stored `_id`, so the watcher reads it
+ *   here, and skips the event.
+ * - Each watcher has its own tracker. A tracker takes each noted entry once: the event of the
+ *   write takes it. A later event with the same `_seq` (for example a raw `$set` that keeps the
+ *   field) is not of this process, and the watcher reads it as a raw write.
+ * - The storage notes entries only while a tracker of the collection is open, so a process without
+ *   a watcher keeps nothing.
+ * - The trackers of a collection are by the `MongoClient` of the database and the names of the
+ *   database and of the collection: two `Db` objects of one database share them.
+ * - An entry without an event (for example of an aborted transaction) goes after
+ *   `OWN_WRITE_TTL_MS` on the clock of the tracker, and a tracker keeps at most `MAX_OWN_WRITES`
+ *   entries of each kind: the oldest go first. An entry that goes too early makes a second hint at
+ *   most, never a lost hint.
+ * - The numbers of an adoption of a raw write (D39) are never noted: the watcher publishes the
+ *   hint of the adoption.
+ */
+import { BSON, type Db } from "mongodb";
+import type { Clock } from "@procedures/client";
+
+/** The longest time that a noted entry waits for its change event: 5 minutes. */
+const OWN_WRITE_TTL_MS = 5 * 60_000;
+
+/** The most noted entries of each kind of one tracker. */
+const MAX_OWN_WRITES = 100_000;
+
+/** The noted writes of one watcher of one collection. */
+export interface OwnNumbers {
+  /** True when a write of this process noted the number and no event took it yet. The number is then taken. */
+  take(seq: number): boolean;
+  /** True when this process pruned the tombstone of the stored `_id` and no event took it yet. The entry is then taken. */
+  takePruned(storedId: unknown): boolean;
+  /** This method ends the tracking: the storage notes no more entries for it. */
+  close(): void;
+}
+
+/** The options of `trackOwnNumbers()`. */
+export interface TrackOwnNumbersOptions {
+  /** False: the tracker keeps only the pruned tombstones, not the numbers (default true). */
+  readonly numbers?: boolean | undefined;
+  /** The clock of the time limit of the entries (default `Date.now`). */
+  readonly clock?: Pick<Clock, "now"> | undefined;
+}
+
+interface Tracker {
+  readonly numbers: Map<number, number> | undefined;
+  readonly pruned: Map<string, number>;
+  readonly now: () => number;
+}
+
+/** The trackers of each collection: by the client (else the `Db` object), then by the names of the database and the collection. */
+const TRACKERS = new WeakMap<object, Map<string, Set<Tracker>>>();
+
+function ownerOf(db: Db): object {
+  const client = (db as { client?: unknown }).client;
+  return typeof client === "object" && client !== null ? client : db;
+}
+
+function nameOf(db: Db, collection: string): string {
+  return `${db.databaseName}\u0000${collection}`;
+}
+
+function trackersOf(db: Db, collection: string): Set<Tracker> | undefined {
+  return TRACKERS.get(ownerOf(db))?.get(nameOf(db, collection));
+}
+
+/** The key of a stored `_id` in a tracker: its extended JSON, so an `ObjectId` and a text never meet. */
+function idKey(storedId: unknown): string {
+  return BSON.EJSON.stringify({ v: storedId as never }, { relaxed: false });
+}
+
+/** This function notes some keys in a map with a time limit, and drops the old ones. */
+function noteAll<K>(map: Map<K, number>, keys: Iterable<K>, now: number): void {
+  for (const key of keys) {
+    // A key noted again (an aborted transaction gives its numbers again) moves to the end
+    map.delete(key);
+    map.set(key, now + OWN_WRITE_TTL_MS);
+  }
+  // The map keeps the order of the notes: the oldest keys are first
+  for (const [key, until] of map) {
+    if (until > now && map.size <= MAX_OWN_WRITES) break;
+    map.delete(key);
+  }
+}
+
+/** This function starts a tracker of the writes of this process in a collection. */
+export function trackOwnNumbers(db: Db, collection: string, options: TrackOwnNumbersOptions = {}): OwnNumbers {
+  const owner = ownerOf(db);
+  const name = nameOf(db, collection);
+  const clock = options.clock;
+  const tracker: Tracker = {
+    numbers: options.numbers === false ? undefined : new Map(),
+    pruned: new Map(),
+    now: clock ? () => clock.now() : Date.now,
+  };
+  let byName = TRACKERS.get(owner);
+  if (!byName) {
+    byName = new Map();
+    TRACKERS.set(owner, byName);
+  }
+  let trackers = byName.get(name);
+  if (!trackers) {
+    trackers = new Set();
+    byName.set(name, trackers);
+  }
+  trackers.add(tracker);
+  const live = (map: Map<unknown, number>, key: unknown): boolean => {
+    const until = map.get(key);
+    if (until === undefined) return false;
+    map.delete(key);
+    return until > tracker.now();
+  };
+  return {
+    take(seq: number): boolean {
+      return tracker.numbers !== undefined && live(tracker.numbers as Map<unknown, number>, seq);
+    },
+    takePruned(storedId: unknown): boolean {
+      return live(tracker.pruned as Map<unknown, number>, idKey(storedId));
+    },
+    close(): void {
+      const set = TRACKERS.get(owner)?.get(name);
+      set?.delete(tracker);
+      if (set?.size === 0) TRACKERS.get(owner)?.delete(name);
+      tracker.numbers?.clear();
+      tracker.pruned.clear();
+    },
+  };
+}
+
+/** This function notes the numbers `first` to `first + count - 1` of a write of this process, for each open tracker of the collection. */
+export function noteOwnNumbers(db: Db, collection: string, first: number, count: number): void {
+  const trackers = trackersOf(db, collection);
+  if (!trackers) return;
+  for (const tracker of trackers) {
+    if (!tracker.numbers) continue;
+    const numbers: number[] = [];
+    for (let seq = first; seq < first + count; seq++) numbers.push(seq);
+    noteAll(tracker.numbers, numbers, tracker.now());
+  }
+}
+
+/** This function notes the stored `_id` of each tombstone that this process prunes, for each open tracker of the collection. Call it before the delete. */
+export function notePrunedIds(db: Db, collection: string, storedIds: readonly unknown[]): void {
+  const trackers = trackersOf(db, collection);
+  if (!trackers) return;
+  const keys = storedIds.map(idKey);
+  for (const tracker of trackers) noteAll(tracker.pruned, keys, tracker.now());
+}