@procedures/mongo 0.35.0 → 0.35.1

package/dist/storage/document-storage.js

@@ -92,9 +92,11 @@
  */
 import { randomUUID } from "node:crypto";
 import { isDeepStrictEqual } from "node:util";
+import { BSON } from "mongodb";
 import { VersionConflictError, applyPatch, preconditionNotSupported, queryEntries, versionConflict, } from "@procedures/collections";
 import { optionalMs, toMs } from "@procedures/client";
 import { ensureConnection } from "../connection.js";
+import { noteOwnNumbers, notePrunedIds } from "./own-writes.js";
 import { boundPipeline, cutResults, DEFAULT_PIPELINE_LIMITS, scopePipeline } from "./pipeline-scope.js";
 /** The fields of the storage in a stored document. */
 const KEY = "_key";
@@ -270,6 +272,58 @@ export function decodeStoredDocument(doc, options = {}) {
     const entries = Object.entries(form.partitionFields(partition));
     return { id, partition, item: itemOf(doc, id, entries, options.addId ?? true) };
 }
+/**
+ * This function gives the version of a raw stored document of a `MongoDocumentStorage`: the
+ * version that `storage.versionOf(id)` gives for the same document (roadmap CLI-77). The storage
+ * uses it too, so the two never differ.
+ *
+ * - The version is the sequence number of the last write through the storage (`_seq`), as text.
+ * - A document without `_seq` (an old document, or one that a raw replace wrote) has the version `0`.
+ * - A tombstone, a value that is not a plain object, and a document whose id neither `idOf` nor
+ *   `_key` tells give `undefined`: they are no items, as for `decodeStoredDocument`.
+ *
+ * @param doc - The stored document, for example the `fullDocument` of a change event
+ * @param form - The stored form of the storage, for example `storage.form` (default `DEFAULT_STORED_FORM`)
+ * @param partition - The partition of the document. The default is `form.partitionOf(doc)`.
+ *
+ * @example
+ * ```typescript
+ * const version = versionOfStored(change.fullDocument, notes.form); // the same as await notes.versionOf(id)
+ * ```
+ */
+export function versionOfStored(doc, form = DEFAULT_STORED_FORM, partition) {
+    if (!isPlainObject(doc) || doc[TOMB] === true)
+        return undefined;
+    const place = partition ?? form.partitionOf(doc);
+    if (typeof form.idOf(doc["_id"], place) !== "string" && typeof doc[KEY] !== "string")
+        return undefined;
+    return seqVersion(doc);
+}
+/** The version of the `_seq` of a stored document: the number as text, or `0` without it. */
+function seqVersion(doc) {
+    const seq = doc[SEQ];
+    return typeof seq === "number" ? String(seq) : "0";
+}
+/** True when two values have the same BSON. */
+function sameBson(a, b) {
+    try {
+        return Buffer.from(BSON.serialize({ v: a })).equals(Buffer.from(BSON.serialize({ v: b })));
+    }
+    catch {
+        return false;
+    }
+}
+/** The most floor documents that `partitions()` reads. */
+const MAX_KNOWN_PARTITIONS = 10_000;
+function escapeRegex(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/** The reader of the parts of a storage. The class sets it, so it can read its private fields. */
+let sourceOfStorage;
+/** This function gives the parts of a storage that `watchStorageChanges` reads. The package does not export it. */
+export function storageChangeSource(storage) {
+    return sourceOfStorage(storage);
+}
 /** True for a plain object. */
 function isPlainObject(value) {
     if (typeof value !== "object" || value === null || Array.isArray(value))
@@ -405,7 +459,23 @@ export class MongoDocumentStorage {
     /** The fields that only the storage writes: its fields and the partition fields. */
     reserved;
     indexes;
+    /** True after this storage wrote (or found) the floor document of its partition (roadmap O66). A failed write leaves it false. */
+    floorNoted = false;
     closed = false;
+    static {
+        sourceOfStorage = (storage) => ({
+            db: () => storage.database(),
+            collection: storage.collectionName,
+            counters: storage.countersName,
+            form: storage.form,
+            partition: storage.partition,
+            adopt: (raw) => storage.adoptRaw(raw),
+            adoptAll: (raws) => storage.adoptRawAll(raws),
+            rawDocuments: (after, limit) => storage.rawDocuments(after, limit),
+            storedDocuments: (ids) => storage.storedDocuments(ids),
+            partitions: () => storage.knownPartitions(),
+        });
+    }
     constructor(options, partitions) {
         if (options.partition !== undefined && (options.partition === "" || options.partition.includes(SEPARATOR))) {
             throw new Error("A partition must be a text without the character U+001F");
@@ -648,9 +718,11 @@ export class MongoDocumentStorage {
     /**
      * This method takes numbers and writes with them. On a replica set, the two are one
      * transaction. Else the numbers stay pending until the write ends, and a late write moves its
-     * documents to new numbers.
+     * documents to new numbers. With `own` false (an adoption of a raw write, D39), a watcher of this
+     * process does not take the numbers as its own.
      */
-    async withNumbers(db, collection, count, write) {
+    async withNumbers(db, collection, count, write, own = true) {
+        const noteOwn = own ? noteOwnNumbers : () => undefined;
         const client = await this.transactionClient(db);
         if (client) {
             const session = client.startSession();
@@ -659,6 +731,8 @@ export class MongoDocumentStorage {
                     const counter = (await db.collection(this.countersName).findOneAndUpdate({ _id: this.collectionName }, { $inc: { seq: count }, $setOnInsert: { epoch: randomUUID() } }, { upsert: true, returnDocument: "after", session }));
                     if (!counter)
                         throw new Error(`The sequence of ${this.collectionName} is not available`);
+                    // A watcher of the collection in this process skips the events of these numbers (roadmap CLI-77)
+                    noteOwn(db, this.collectionName, counter.seq - count + 1, count);
                     return write(counter.seq - count + 1, session);
                 });
             }
@@ -667,6 +741,7 @@ export class MongoDocumentStorage {
             }
         }
         let { first } = await this.allocate(db, count);
+        noteOwn(db, this.collectionName, first, count);
         let result;
         try {
             result = await write(first);
@@ -680,6 +755,7 @@ export class MongoDocumentStorage {
                 throw new Error(`A write of ${this.collectionName} stayed late after ${MAX_MOVES} moves to new numbers`);
             // Late: a reader can have passed these numbers. The write takes new ones.
             const next = (await this.allocate(db, count)).first;
+            noteOwn(db, this.collectionName, next, count);
             await this.move(collection, first, count, next);
             first = next;
         }
@@ -722,6 +798,33 @@ export class MongoDocumentStorage {
     floorId() {
         return `floor${SEPARATOR}${this.collectionName}${SEPARATOR}${this.partition ?? ""}`;
     }
+    /**
+     * This method writes the floor document of the partition once for this storage, before its first
+     * write (roadmap O66). Then `knownPartitions()` of each process finds the partition, also without
+     * a prune, and a gap of `watchStorageChanges()` sends it a reset hint.
+     *
+     * - The floor `0` is neutral: `getChangesSince` gives a full result only for a version below the
+     *   floor, and a version is never below `0`. A missing floor document also reads as `0`. So the
+     *   document changes no result of the change log.
+     * - `$setOnInsert` writes only a new document: a floor of a prune (`$max`) stays as it is.
+     * - A duplicate key (another process inserted the same document first) is a success.
+     * - The upsert runs in the queue of the writes, before the task, so never in the transaction of
+     *   `withNumbers()`: it adds no write conflict to the transaction of the user, and a transaction
+     *   that tries again does not write it again.
+     * - A failure does not fail the write of the user. The flag stays false, so the next write tries again.
+     */
+    async noteFloor(db) {
+        if (this.floorNoted)
+            return;
+        try {
+            await db.collection(this.countersName).updateOne({ _id: this.floorId() }, { $setOnInsert: { floor: 0 } }, { upsert: true });
+            this.floorNoted = true;
+        }
+        catch (error) {
+            if (isDuplicateKey(error))
+                this.floorNoted = true;
+        }
+    }
     /** This method drops the oldest tombstones of the partition over the limit, and keeps the floor of the log. */
     async pruneTombstones(db, items) {
         const count = await items.countDocuments({ ...this.partitionFilter(), [TOMB]: true });
@@ -735,6 +838,8 @@ export class MongoDocumentStorage {
             .toArray();
         if (oldest.length === 0)
             return;
+        // A watcher of this process reads the delete of a pruned tombstone without the image before the change (roadmap CLI-77)
+        notePrunedIds(db, this.collectionName, oldest.map((doc) => doc._id));
         await items.deleteMany({ _id: { $in: oldest.map((doc) => doc._id) } });
         const floor = Math.max(...oldest.map((doc) => Number(doc[SEQ])));
         await db.collection(this.countersName).updateOne({ _id: this.floorId() }, { $max: { floor } }, { upsert: true });
@@ -848,14 +953,14 @@ export class MongoDocumentStorage {
     }
     /**
      * The version of an item: the sequence number of its last write, or undefined for a missing id.
-     * An item without a write through the storage (no `_seq`) has the version `0`.
+     * An item without a write through the storage (no `_seq`) has the version `0`. It is the version
+     * of `versionOfStored()` of the stored document (roadmap CLI-77): the two share one rule.
      */
     async versionOf(id) {
         const { items } = await this.collection();
         const doc = await items.findOne(this.liveFilter({ _id: this.storedId(id) }), { projection: { [SEQ]: 1 } });
-        if (!doc)
-            return undefined;
-        return typeof doc[SEQ] === "number" ? String(doc[SEQ]) : "0";
+        // The filter finds the document by the stored `_id` of the id in the partition: it is the item
+        return doc ? seqVersion(doc) : undefined;
     }
     async getChangesSince(since) {
         const { db, items } = await this.collection();
@@ -900,7 +1005,11 @@ export class MongoDocumentStorage {
     // ===========================================================================
     async write(task) {
         const { db, items } = await this.collection();
-        return enqueue(db, this.collectionName, () => task(db, items));
+        return enqueue(db, this.collectionName, async () => {
+            // The floor document of the partition, outside each transaction of the task (roadmap O66)
+            await this.noteFloor(db);
+            return task(db, items);
+        });
     }
     async set(id, value, options) {
         await this.setBatch([[id, value]], options?.expectedVersion === undefined ? undefined : { expectedVersions: [[id, options.expectedVersion]] });
@@ -1024,6 +1133,215 @@ export class MongoDocumentStorage {
         });
         await this.pruneTombstones(db, collection);
     }
+    // ===========================================================================
+    // The raw writes (D39, roadmap CLI-77)
+    // ===========================================================================
+    /**
+     * This method adopts a raw write: one conditional write through the path of each write gives the
+     * document a new `_seq` (see `RawStorageWrite`). It is for `watchStorageChanges()`, and it is no
+     * general setter of `_seq`.
+     */
+    async adoptRaw(raw) {
+        const [result] = await this.adoptRawAll([raw]);
+        if (result === undefined || typeof result !== "string")
+            throw result?.error;
+        return result;
+    }
+    /**
+     * This method adopts several raw writes. The writes of each partition take one block of numbers
+     * through the path of each write. A number of a write that does not win stays unused.
+     */
+    async adoptRawAll(raws) {
+        const results = raws.map(() => "unknown");
+        const groups = new Map();
+        raws.forEach((raw, index) => {
+            const target = this.adoptionTarget(raw);
+            if (!target)
+                return;
+            let group = groups.get(target.storage);
+            if (!group)
+                groups.set(target.storage, (group = []));
+            group.push({ index, raw, id: target.id });
+        });
+        for (const [storage, group] of groups) {
+            try {
+                const done = await storage.adoptGroup(group);
+                group.forEach((entry, position) => (results[entry.index] = done[position] ?? "lost"));
+            }
+            catch (error) {
+                for (const entry of group)
+                    results[entry.index] = { error };
+            }
+        }
+        return results;
+    }
+    /** The adoption of raw writes of this partition, with one block of numbers. */
+    async adoptGroup(group) {
+        return this.write(async (db, collection) => {
+            const results = await this.withNumbers(db, collection, group.length, async (first, session) => {
+                const out = [];
+                for (const [position, { raw, id }] of group.entries()) {
+                    const seq = first + position;
+                    out.push(raw.after === undefined ? await this.adoptDelete(collection, raw, id, seq, session) : await this.adoptWrite(collection, raw, raw.after, id, seq, session));
+                }
+                return out;
+            }, false);
+            if (results.some((result, position) => result === "adopted" && group[position].raw.after === undefined))
+                await this.pruneTombstones(db, collection);
+            return results;
+        });
+    }
+    /** A raw hard delete: a tombstone with the id and the partition of the image before the change. */
+    async adoptDelete(collection, raw, id, seq, session) {
+        // In a transaction, a duplicate key would end the transaction: the read comes first
+        if (session && (await collection.findOne({ _id: raw.storedId }, { projection: { _id: 1 }, session })))
+            return "lost";
+        const tomb = { _id: raw.storedId, [KEY]: id, [SEQ]: seq, [TOMB]: true };
+        this.putPartition(tomb);
+        try {
+            await collection.insertOne(tomb, session ? { session } : {});
+            return "adopted";
+        }
+        catch (error) {
+            // A raw insert of the same _id came after the delete: that insert is the newer state
+            if (isDuplicateKey(error))
+                return "lost";
+            throw error;
+        }
+    }
+    /** A raw insert, replace or update: one conditional write on the exact state of the event. */
+    async adoptWrite(collection, raw, after, id, seq, session) {
+        const fields = this.restoredFields(raw, after, id);
+        // The exact state of the event: the same `_seq` (or none), and no field that the adoption sets
+        const filter = { _id: raw.storedId, [SEQ]: raw.seq ?? { $exists: false } };
+        for (const name of Object.keys(fields))
+            filter[name] = { $exists: false };
+        const result = await collection.updateOne(filter, { $set: { ...fields, [SEQ]: seq } }, session ? { session } : {});
+        return result.matchedCount === 1 ? "adopted" : "lost";
+    }
+    /** One page of the documents without `_seq`, in the order of `_id` (the sweep of `watchStorageChanges()`). */
+    async rawDocuments(after, limit) {
+        const db = await this.database();
+        // The scan walks the index of _id from the last _id: `min` bounds the scan, in the order of the
+        // BSON types. `min` includes that _id, so the page skips it. No index can find the documents
+        // without a field (a partial index cannot express $exists: false): the scan reads each document.
+        let cursor = db.collection(this.collectionName).find({ [SEQ]: { $exists: false } }).hint({ _id: 1 });
+        if (after)
+            cursor = cursor.min({ _id: after.id });
+        const docs = await cursor.limit(after ? limit + 1 : limit).toArray();
+        if (after && docs.length > 0 && sameBson(docs[0]["_id"], after.id))
+            docs.shift();
+        return docs.slice(0, limit);
+    }
+    /** The stored documents of these stored `_id`s, for the adoptions that `watchStorageChanges()` does again. */
+    async storedDocuments(ids) {
+        const db = await this.database();
+        const filter = { _id: { $in: [...ids] } };
+        return db.collection(this.collectionName).find(filter).toArray();
+    }
+    /**
+     * The storage of the partition of a raw write and the id of its document, or undefined when
+     * neither the documents of the event nor the stored `_id` tell them. The partition comes from
+     * `form.partitionOf()` of the image before the change, else of the document after the write,
+     * else of the stored `_id` (the order of `storageChangeOf()`). The document must not have a
+     * partition field with another value.
+     */
+    adoptionTarget(raw) {
+        const doc = raw.after ?? raw.before;
+        if (!doc)
+            return undefined;
+        const places = new Set();
+        for (const source of [raw.before, raw.after, { _id: raw.storedId }])
+            if (isPlainObject(source))
+                places.add(this.form.partitionOf(source));
+        for (const place of places) {
+            if (place !== this.partition && (this.partition !== undefined || place === undefined))
+                continue;
+            let storage;
+            try {
+                storage = place === this.partition ? this : this.forPartition(place);
+            }
+            catch {
+                continue;
+            }
+            if (!storage.holds(doc))
+                continue;
+            const fromId = this.form.idOf(raw.storedId, place);
+            const key = raw.after?.[KEY] ?? raw.before?.[KEY];
+            const id = typeof fromId === "string" ? fromId : typeof key === "string" ? key : undefined;
+            if (id === undefined || id.includes(SEPARATOR))
+                continue;
+            return { storage, id };
+        }
+        return undefined;
+    }
+    /** True when each partition field of a document has the value of this partition, or is absent (a raw replace can drop it). */
+    holds(doc) {
+        for (const [name, value] of this.partitionEntries) {
+            if (!(name in doc))
+                continue;
+            if (value === undefined || !isDeepStrictEqual(doc[name], value))
+                return false;
+        }
+        return true;
+    }
+    /**
+     * The storage fields and the partition fields that a raw write took from the document, for the
+     * adoption to set again. No field of the item is in it.
+     *
+     * - The partition fields of this partition that the document does not have.
+     * - `_key` (with `_kid`) of the image before the change. Without the image, `_key` only when
+     *   `form.idOf()` does not tell the id, with `_kid`: the item keeps its `_id`, as a document from
+     *   outside the storage.
+     * - `_ord` of the image before the change, else `0`, the place of a document without `_ord`
+     *   (the rule of the old documents).
+     * - The names of `_kpf` of the image before the change that the document has with the value of
+     *   the partition.
+     */
+    restoredFields(raw, after, id) {
+        const before = raw.before;
+        const fields = {};
+        for (const [name, value] of this.partitionEntries)
+            if (value !== undefined && !(name in after))
+                fields[name] = value;
+        if (after[KEY] === undefined) {
+            if (typeof before?.[KEY] === "string") {
+                fields[KEY] = before[KEY];
+                if (before[KEEP_ID] === true && after[KEEP_ID] === undefined)
+                    fields[KEEP_ID] = true;
+            }
+            else if (this.form.idOf(raw.storedId, this.partition) !== id) {
+                fields[KEY] = id;
+                if (after[KEEP_ID] === undefined)
+                    fields[KEEP_ID] = true;
+            }
+        }
+        if (after[ORD] === undefined)
+            fields[ORD] = typeof before?.[ORD] === "number" ? before[ORD] : 0;
+        const kept = before?.[KEEP_PART];
+        if (after[KEEP_PART] === undefined && Array.isArray(kept)) {
+            const names = kept.filter((name) => typeof name === "string" && this.partitionValues.has(name) && isDeepStrictEqual(after[name], this.partitionValues.get(name)));
+            if (names.length > 0)
+                fields[KEEP_PART] = names;
+        }
+        return fields;
+    }
+    /** The partitions that the storage knows: the floor documents of the counters (of the first writes and of the prunes), and the storages of `forPartition()`. */
+    async knownPartitions() {
+        const db = await this.database();
+        const prefix = `floor${SEPARATOR}${this.collectionName}${SEPARATOR}`;
+        const floors = await db
+            .collection(this.countersName)
+            .find({ _id: { $regex: `^${escapeRegex(prefix)}` } }, { projection: { _id: 1 } })
+            .limit(MAX_KNOWN_PARTITIONS)
+            .toArray();
+        const found = new Set(this.partitions.keys());
+        for (const floor of floors) {
+            const rest = String(floor._id).slice(prefix.length);
+            found.add(rest === "" ? undefined : rest);
+        }
+        return [...found];
+    }
     /**
      * The preconditions of a write (roadmap CLI-16), after a check of their ids. A storage without
      * `versionOf` throws a `TypeError` before the write starts.