@procedures/mongo 0.33.0 → 0.35.0

package/dist/storage/document-storage.js

@@ -4,10 +4,13 @@
  * `{ _id, data }` documents.
  *
  * The documents:
- * - An item that is a plain object with safe keys is stored flat: its fields beside the fields of
- *   the storage. Another item (null, a primitive, an array, or an object with a key of the storage,
- *   a key that starts with `$` or a key with a dot) is stored as JSON text in `_json`. Both ways
- *   give the item back as it was.
+ * - An item that is a plain object is stored flat: its fields beside the fields of the storage. A
+ *   key with `$` or a dot below the top level, and a top-level key with a dot, stay as they are
+ *   (MongoDB 5.0 or later keeps them). Another item is stored as JSON text in `_json`: null, a
+ *   primitive, an array, an object with a top-level key of the storage or a top-level key that
+ *   starts with `$`, a key `__proto__`, or a class instance other than `Date`. Both ways give the
+ *   item back as it was. An object with a text `$ref` and an `$id` is refused (`TypeError`):
+ *   MongoDB gives it back as a `DBRef`.
  * - An item whose `_id` is its id keeps it: the storage gives it back.
  * - The fields of the storage: `_key` (the id), `_part` (the partition), `_seq` (the sequence
  *   number of the last write), `_ord` (the number of the first write: the order of the entries, as
@@ -16,6 +19,40 @@
  *   character U+001F. Each partition reads and writes only its documents: each write filter has the
  *   partition too, so a write of a stored `_id` of another partition fails and changes nothing.
  *
+ * The stored form (roadmap CLI-76): an app with its own form of documents keeps it with the option
+ * `form`, one `StoredForm` object. `DEFAULT_STORED_FORM` gives the form above, byte for byte.
+ * - `storedId(partition, id)` gives the stored `_id`, and `idOf(storedId, partition)` is its inverse.
+ *   The stored `_id` can be any value but `undefined` and `null`, for example `tenant:key`.
+ * - `partitionFields(partition)` gives the fields that name the partition (default `{ _part }`).
+ *   Each document of the partition has them, and they are the filter of the partition. A field with
+ *   the value `undefined` is absent: the document does not have it, and the filter is
+ *   `{ $exists: false }`. An item can have a partition field (one that is not a field of the
+ *   storage) with the value of its partition: the write leaves it out and notes it in `_kpf`, and
+ *   the read gives it back. The same field with another value is refused (`TypeError`).
+ * - `partitionOf(doc)` gives the partition of a stored document, for `ViewMaterializer`. A delete
+ *   event has only the `documentKey`: so the `_id` must tell the partition, or the collection needs
+ *   `changeStreamPreAndPostImages`.
+ * - The storage adds its fields (`_key`, `_seq`, `_ord`, `_tomb`, and `_kid` or `_json` as needed)
+ *   in each form. A reader of the raw documents can leave them out (`DOCUMENT_STORAGE_FIELDS`), and
+ *   must leave out the tombstones (`TOMBSTONE_FIELD` is `true`). `decodeStoredDocument(doc)` does
+ *   this, as the storage reads a document.
+ * - A document without the fields of the storage (for example one written before the app used the
+ *   storage) is an item too. Its id comes from `idOf`, else from `_key`, and its item gets that id
+ *   as `_id`. Its partition fields follow the rule of a written document without `_kpf`: a field
+ *   with the value of the partition stays out, and a field with another value stays in. So the
+ *   item has the same form before and after its first write. A document whose id neither
+ *   tells (for example an `ObjectId` that `idOf` does not name) is not an item: the reads and
+ *   `size()` leave it out. Its version is `0`, and it comes before the other items in the order of
+ *   the entries (then by `_id`). A full result of the change log has it, and a delta has it after
+ *   its first write through the storage. That write gives it its fields and a version, and keeps
+ *   its place.
+ * - Write through the storage. A raw write (for example `findOneAndReplace` or a script) takes no
+ *   number of the sequence, so a delta of the change log does not give it, and a raw replace drops
+ *   `_seq`.
+ * - A raw writer must not hard-delete an item while the storage writes it. The write of a document
+ *   without `_seq` is an upsert with `{ _seq: { $exists: false } }`: after a hard delete between
+ *   the read and the write, the upsert inserts the document again.
+ *
  * The change log:
  * - Each write gets the next number of the sequence of the collection. `_seq` is also the version
  *   of the item (`versionOf`).
@@ -31,6 +68,9 @@
  *   its lease moves its documents to new numbers, so a reader that passed the old numbers gets
  *   it. One case stays on a single server: a process that stops between such a late write and the
  *   move of its documents. Then a reader that passed the old numbers does not get that change.
+ * - The storage uses a transaction only with a client: the option `client`, or the default
+ *   connection when `db` is absent. With `db` and no `client`, each write uses the watermark, also
+ *   on a replica set.
  *
  * The queries: `query()` sends the conditions that MongoDB tests in the same way (the equality of
  * a string, a number or a boolean, and `$in` of them) as a filter. `queryEntries()` of the core
@@ -45,8 +85,13 @@
  * precondition has the `_seq` that the test read in its filter (a compare-and-set). Another writer
  * between the test and the write makes the write fail with `VersionConflictError`. Then a batch of
  * more than one item can keep a part of its writes: only a transaction makes a batch all or nothing.
+ * One case stays on a single server: a writer outside the storage that writes a document without
+ * `_seq` between the test and the write. The guard of such a document is `{ _seq: { $exists:
+ * false } }`, so it does not see that writer. On a replica set, the test and the write are one
+ * transaction, and this case does not occur.
  */
 import { randomUUID } from "node:crypto";
+import { isDeepStrictEqual } from "node:util";
 import { VersionConflictError, applyPatch, preconditionNotSupported, queryEntries, versionConflict, } from "@procedures/collections";
 import { optionalMs, toMs } from "@procedures/client";
 import { ensureConnection } from "../connection.js";
@@ -61,9 +106,47 @@ const TOMB = "_tomb";
 const JSON_FIELD = "_json";
 /** True in a stored document whose item had `_id` equal to its id. */
 const KEEP_ID = "_kid";
-const STORAGE_FIELDS = new Set(["_id", KEY, PART, SEQ, ORD, TOMB, JSON_FIELD, KEEP_ID]);
+/** The partition fields that the item had with the value of its partition (roadmap CLI-76). The write leaves them out of the fields. */
+const KEEP_PART = "_kpf";
+const STORAGE_FIELDS = new Set(["_id", KEY, PART, SEQ, ORD, TOMB, JSON_FIELD, KEEP_ID, KEEP_PART]);
 /** The separator of the partition and the id in a stored `_id`. */
 export const SEPARATOR = "\u001f";
+/**
+ * The fields that a `MongoDocumentStorage` can add to a stored document, in each stored form
+ * (roadmap CLI-76). A reader of the raw documents can leave them out. The partition fields are
+ * not in the list.
+ */
+export const DOCUMENT_STORAGE_FIELDS = Object.freeze([KEY, PART, SEQ, ORD, TOMB, JSON_FIELD, KEEP_ID, KEEP_PART]);
+/** The field that is `true` in a tombstone: the document of a deleted item. A reader of the raw documents leaves them out. */
+export const TOMBSTONE_FIELD = TOMB;
+/**
+ * The default stored form, the form of 0.34.0. The stored `_id` is the id, or `<partition>` +
+ * U+001F + `<id>` with a partition. The partition field is `_part` (absent without a partition).
+ * `partitionOf` reads `_part`, else the part of `_id` before U+001F.
+ */
+export const DEFAULT_STORED_FORM = Object.freeze({
+    storedId(partition, id) {
+        return partition === undefined ? id : `${partition}${SEPARATOR}${id}`;
+    },
+    idOf(storedId, partition) {
+        if (typeof storedId !== "string")
+            return undefined;
+        if (partition === undefined)
+            return storedId.includes(SEPARATOR) ? undefined : storedId;
+        const prefix = `${partition}${SEPARATOR}`;
+        return storedId.startsWith(prefix) ? storedId.slice(prefix.length) : undefined;
+    },
+    partitionFields(partition) {
+        return { [PART]: partition };
+    },
+    partitionOf(doc) {
+        if (typeof doc[PART] === "string")
+            return doc[PART];
+        const id = doc["_id"];
+        const at = typeof id === "string" ? id.indexOf(SEPARATOR) : -1;
+        return at > 0 ? id.slice(0, at) : undefined;
+    },
+});
 /** True for the error of a duplicate key: an insert of an upsert whose filter found no document. */
 function isDuplicateKey(error) {
     // A bulk write error has the code of its first write error, and the list of the write errors
@@ -102,25 +185,91 @@ function enqueue(db, collection, task) {
     });
     return run;
 }
-/** True for a plain object whose keys a flat document can keep, at each depth. */
-function isFlatSafe(value, top) {
+/**
+ * True for a value of a field that BSON gives back as it is: a key with `$` or a dot stays as it is
+ * (MongoDB 5.0 or later). False when only JSON text keeps it: a class instance other than `Date`,
+ * or a key `__proto__`. An object that BSON gives back as a `DBRef` adds a problem.
+ */
+function isNativeValue(value, path, problems) {
     if (Array.isArray(value))
-        return value.every((element) => isFlatSafe(element, false));
+        return value.every((element, index) => isNativeValue(element, `${path}.${index}`, problems));
     if (typeof value !== "object" || value === null)
         return true;
     const proto = Object.getPrototypeOf(value);
     if (proto !== Object.prototype && proto !== null)
         return value instanceof Date;
-    for (const [key, field] of Object.entries(value)) {
-        if (key.startsWith("$") || key.includes(".") || key === "__proto__")
-            return false;
-        if (top && STORAGE_FIELDS.has(key) && key !== "_id")
+    const object = value;
+    // The test of the BSON reader: a text `$ref`, an `$id`, and no `$db` or a text `$db`
+    if (typeof object["$ref"] === "string" && object["$id"] !== undefined && object["$id"] !== null && (object["$db"] === undefined || typeof object["$db"] === "string")) {
+        problems.push(`the field ${path} has $ref and $id, and MongoDB gives it back as a DBRef`);
+    }
+    for (const [key, field] of Object.entries(object)) {
+        if (key === "__proto__")
             return false;
-        if (!isFlatSafe(field, false))
+        if (!isNativeValue(field, `${path}.${key}`, problems))
             return false;
     }
     return true;
 }
+/**
+ * The item of a stored document (not a tombstone) whose id is known. `_json` gives the item as it
+ * was. Else the fields of the storage stay out, the item gets `_id` when it had it (`_kid`), when
+ * the document is from outside the storage (no `_key`) or with `addId`, and it gets back the
+ * partition fields of `_kpf`. A partition field with the value of the partition stays out (for a
+ * document from outside the storage too); a field with another value stays in.
+ */
+function itemOf(doc, id, partitionEntries, addId) {
+    if (typeof doc[JSON_FIELD] === "string") {
+        const item = JSON.parse(doc[JSON_FIELD]);
+        if (addId && isPlainObject(item) && !("_id" in item))
+            return { _id: id, ...item };
+        return item;
+    }
+    const outside = doc[KEY] === undefined;
+    const value = {};
+    if (addId || outside || doc[KEEP_ID] === true)
+        value["_id"] = id;
+    const partitionValues = new Map(partitionEntries);
+    for (const [key, field] of Object.entries(doc)) {
+        if (STORAGE_FIELDS.has(key))
+            continue;
+        // The value of the partition comes back only through `_kpf`, so the item has one form at each age of its document
+        if (partitionValues.has(key) && isDeepStrictEqual(field, partitionValues.get(key)))
+            continue;
+        put(value, key, field);
+    }
+    const kept = doc[KEEP_PART];
+    if (Array.isArray(kept)) {
+        for (const [name, field] of partitionEntries)
+            if (kept.includes(name) && !STORAGE_FIELDS.has(name))
+                put(value, name, field);
+    }
+    return value;
+}
+/**
+ * This function reads a raw document of a `MongoDocumentStorage` collection as the storage reads
+ * it (roadmap CLI-76), for an app that reads the collection directly (a query, a view, an
+ * aggregation). It gives `undefined` for a tombstone and for a document whose id neither `idOf`
+ * nor `_key` tells. Else it parses `_json`, leaves out the fields of the storage
+ * (`DOCUMENT_STORAGE_FIELDS`) and gives the item its id as `_id` (see `addId`).
+ *
+ * @example
+ * ```typescript
+ * const items = docs.flatMap((doc) => decodeStoredDocument<Note>(doc, { form: notes.form })?.item ?? []);
+ * ```
+ */
+export function decodeStoredDocument(doc, options = {}) {
+    if (!isPlainObject(doc) || doc[TOMB] === true)
+        return undefined;
+    const form = options.form ?? DEFAULT_STORED_FORM;
+    const partition = options.partition ?? form.partitionOf(doc);
+    const fromId = form.idOf(doc["_id"], partition);
+    const id = typeof fromId === "string" ? fromId : typeof doc[KEY] === "string" ? doc[KEY] : undefined;
+    if (id === undefined)
+        return undefined;
+    const entries = Object.entries(form.partitionFields(partition));
+    return { id, partition, item: itemOf(doc, id, entries, options.addId ?? true) };
+}
 /** True for a plain object. */
 function isPlainObject(value) {
     if (typeof value !== "object" || value === null || Array.isArray(value))
@@ -137,17 +286,47 @@ function isScalar(value) {
     return typeof value === "string" || typeof value === "boolean" || (typeof value === "number" && Number.isFinite(value));
 }
 /** The field of a stored document for a path of a selector, or undefined when the filter cannot test it. */
-function storedPath(path) {
+function storedPath(path, reserved) {
     if (path === "_id")
         return KEY;
-    const first = path.split(".")[0];
-    if (first === "_id" || STORAGE_FIELDS.has(first))
+    const parts = path.split(".");
+    const first = parts[0];
+    if (first === "_id" || STORAGE_FIELDS.has(first) || reserved.has(first))
+        return undefined;
+    // A part with `$` is a field name of an item, but an operator for MongoDB
+    if (parts.some((part) => part.startsWith("$")))
         return undefined;
     return path;
 }
+/**
+ * The filter of a condition on `_id` against the stored `_id`, or undefined. A flat item with an
+ * `_id` has its id as `_id`, and an item stored as JSON passes the filter of `query()` anyway.
+ */
+function idConditionFilter(condition, storedId) {
+    try {
+        if (typeof condition === "string")
+            return { _id: storedId(condition) };
+        if (!isPlainObject(condition))
+            return undefined;
+        const operators = {};
+        const eq = condition["$eq"];
+        if (typeof eq === "string")
+            operators["$eq"] = storedId(eq);
+        const among = condition["$in"];
+        if (Array.isArray(among) && among.length > 0 && among.every((value) => typeof value === "string"))
+            operators["$in"] = among.map((value) => storedId(value));
+        return Object.keys(operators).length > 0 ? { _id: operators } : undefined;
+    }
+    catch {
+        // An id that the storage refuses: no condition is a correct filter (it can pass more)
+        return undefined;
+    }
+}
 /** The filter of one condition, or undefined when the filter cannot test it in the same way. */
-function conditionFilter(path, condition) {
-    const field = storedPath(path);
+function conditionFilter(path, condition, options, reserved) {
+    if (path === "_id" && options.storedId)
+        return idConditionFilter(condition, options.storedId);
+    const field = storedPath(path, reserved);
     if (field === undefined)
         return undefined;
     if (isScalar(condition))
@@ -167,26 +346,29 @@ function conditionFilter(path, condition) {
  * This function gives a filter that each item of the selector passes (a filter can pass more),
  * or undefined for no condition. Only conditions that MongoDB tests in the same way go in it.
  */
-export function selectorPrefilter(selector) {
+export function selectorPrefilter(selector, options = {}) {
+    return prefilter(selector, options, new Set(options.storageFields ?? []));
+}
+function prefilter(selector, options, reserved) {
     const parts = [];
     for (const [key, condition] of Object.entries(selector)) {
         if (condition === undefined)
             continue;
         if (key === "$and" && Array.isArray(condition)) {
             for (const branch of condition) {
-                const filter = isPlainObject(branch) ? selectorPrefilter(branch) : undefined;
+                const filter = isPlainObject(branch) ? prefilter(branch, options, reserved) : undefined;
                 if (filter)
                     parts.push(filter);
             }
         }
         else if (key === "$or" && Array.isArray(condition) && condition.length > 0) {
-            const branches = condition.map((branch) => (isPlainObject(branch) ? selectorPrefilter(branch) : undefined));
+            const branches = condition.map((branch) => (isPlainObject(branch) ? prefilter(branch, options, reserved) : undefined));
             // A branch without a condition passes each item: then the $or tests nothing
             if (branches.every((branch) => branch !== undefined))
                 parts.push({ $or: branches });
         }
         else if (!key.startsWith("$")) {
-            const filter = conditionFilter(key, condition);
+            const filter = conditionFilter(key, condition, options, reserved);
             if (filter)
                 parts.push(filter);
         }
@@ -214,12 +396,37 @@ export class MongoDocumentStorage {
     writeLeaseMs;
     /** The storages of the partitions, for `forPartition`: the same storage for the same partition. */
     partitions;
+    /** The stored form of the documents (roadmap CLI-76). A `ViewMaterializer` of the collection takes the same form. */
+    form;
+    /** The fields that name the partition, in their order. An `undefined` value is an absent field. */
+    partitionEntries;
+    /** The values of the partition fields that are not fields of the storage, by name. */
+    partitionValues;
+    /** The fields that only the storage writes: its fields and the partition fields. */
+    reserved;
     indexes;
     closed = false;
     constructor(options, partitions) {
         if (options.partition !== undefined && (options.partition === "" || options.partition.includes(SEPARATOR))) {
             throw new Error("A partition must be a text without the character U+001F");
         }
+        const form = options.form ?? DEFAULT_STORED_FORM;
+        for (const name of ["storedId", "idOf", "partitionFields", "partitionOf"]) {
+            if (typeof form[name] !== "function")
+                throw new TypeError(`The stored form must have the function ${name}`);
+        }
+        this.form = form;
+        const fields = form.partitionFields(options.partition);
+        if (!isPlainObject(fields) || Object.keys(fields).length === 0)
+            throw new TypeError("partitionFields must give an object with at least one field");
+        for (const name of Object.keys(fields)) {
+            if (name === "_id" || name.startsWith("$") || name.includes(".") || name === "__proto__" || (STORAGE_FIELDS.has(name) && name !== PART)) {
+                throw new TypeError(`partitionFields cannot have the field ${JSON.stringify(name)}`);
+            }
+        }
+        this.partitionEntries = Object.entries(fields);
+        this.partitionValues = new Map(this.partitionEntries.filter(([name]) => !STORAGE_FIELDS.has(name)));
+        this.reserved = new Set([...STORAGE_FIELDS, ...Object.keys(fields)]);
         this.options = options;
         this.collectionName = options.collection;
         this.partition = options.partition;
@@ -251,8 +458,9 @@ export class MongoDocumentStorage {
     async collection() {
         const db = await this.database();
         const items = db.collection(this.collectionName);
+        const prefix = Object.fromEntries(this.partitionEntries.map(([name]) => [name, 1]));
         this.indexes ??= items
-            .createIndexes([{ key: { [PART]: 1, [SEQ]: 1 } }, { key: { [PART]: 1, [ORD]: 1 } }, { key: { [PART]: 1, [TOMB]: 1 } }])
+            .createIndexes([{ key: { ...prefix, [SEQ]: 1 } }, { key: { ...prefix, [ORD]: 1 } }, { key: { ...prefix, [TOMB]: 1 } }])
             .then(() => undefined)
             .catch((error) => {
             this.indexes = undefined;
@@ -263,7 +471,15 @@ export class MongoDocumentStorage {
     }
     /** The filter of the documents of the partition. */
     partitionFilter() {
-        return this.partition === undefined ? { [PART]: { $exists: false } } : { [PART]: this.partition };
+        // An object value is an equality too, never a set of operators
+        const condition = (value) => (value === undefined ? { $exists: false } : typeof value === "object" && value !== null ? { $eq: value } : value);
+        return Object.fromEntries(this.partitionEntries.map(([name, value]) => [name, condition(value)]));
+    }
+    /** This method puts the partition fields on a stored document, in their order. An absent field stays absent. */
+    putPartition(doc) {
+        for (const [name, value] of this.partitionEntries)
+            if (value !== undefined)
+                put(doc, name, value);
     }
     /** The filter of the items (not the tombstones) of the partition. */
     liveFilter(extra) {
@@ -273,7 +489,23 @@ export class MongoDocumentStorage {
     storedId(id) {
         if (typeof id !== "string" || id.includes(SEPARATOR))
             throw new Error("An id must be a text without the character U+001F");
-        return this.partition === undefined ? id : `${this.partition}${SEPARATOR}${id}`;
+        const stored = this.form.storedId(this.partition, id);
+        if (stored === undefined || stored === null)
+            throw new TypeError(`storedId gave ${String(stored)} for the id ${JSON.stringify(id)}`);
+        return stored;
+    }
+    /** The id of a stored document: the inverse of its stored `_id`, else `_key`, or undefined when neither tells it. */
+    idOfDoc(doc) {
+        const id = this.form.idOf(doc["_id"], this.partition);
+        if (typeof id === "string")
+            return id;
+        const key = doc[KEY];
+        return typeof key === "string" ? key : undefined;
+    }
+    /** The place of a stored document in the order of the entries. A document without `_ord` comes first. */
+    placeOf(doc) {
+        const ord = doc[ORD];
+        return typeof ord === "number" ? ord : 0;
     }
     /** The filter of a write of one stored document: its `_id` and the partition. */
     writeFilter(id) {
@@ -282,33 +514,72 @@ export class MongoDocumentStorage {
     /** The stored document of an item, with the number of this write and the number of its place. */
     encode(id, value, seq, ord) {
         const doc = { _id: this.storedId(id), [KEY]: id, [SEQ]: seq, [ORD]: ord };
-        if (this.partition !== undefined)
-            doc[PART] = this.partition;
-        const keepsId = isPlainObject(value) && value["_id"] === id;
-        if (isPlainObject(value) && isFlatSafe(value, true) && (!("_id" in value) || keepsId)) {
-            if (keepsId)
+        this.putPartition(doc);
+        const flat = this.flatOf(id, value);
+        if (flat) {
+            if (flat.keepsId)
                 doc[KEEP_ID] = true;
-            for (const [key, field] of Object.entries(value))
-                if (key !== "_id")
-                    put(doc, key, field);
+            if (flat.kept.length > 0)
+                doc[KEEP_PART] = flat.kept;
+            for (const [key, field] of flat.fields)
+                put(doc, key, field);
         }
         else {
             doc[JSON_FIELD] = JSON.stringify(value);
         }
         return doc;
     }
-    /** The id and the item of a stored document. */
-    decode(doc) {
-        const id = String(doc[KEY]);
-        if (typeof doc[JSON_FIELD] === "string")
-            return [id, JSON.parse(doc[JSON_FIELD])];
-        const value = {};
-        if (doc[KEEP_ID] === true)
-            value["_id"] = id;
-        for (const [key, field] of Object.entries(doc))
-            if (!STORAGE_FIELDS.has(key))
-                put(value, key, field);
-        return [id, value];
+    /**
+     * The fields of the flat document of an item, or undefined when only JSON text keeps it: an item
+     * that is not a plain object, an `_id` other than its id, a top-level key of the storage or with
+     * `$`, a key `__proto__`, a class instance other than `Date`. It throws a `TypeError` for a flat
+     * item that a document cannot keep: a partition field with another value than the partition, or
+     * an object that MongoDB gives back as a `DBRef`.
+     */
+    flatOf(id, value) {
+        if (!isPlainObject(value))
+            return undefined;
+        if ("_id" in value && value["_id"] !== id)
+            return undefined;
+        const flat = { keepsId: "_id" in value, kept: [], fields: [] };
+        const problems = [];
+        for (const [key, field] of Object.entries(value)) {
+            if (key === "_id")
+                continue;
+            if (key.startsWith("$") || key === "__proto__" || STORAGE_FIELDS.has(key))
+                return undefined;
+            if (this.partitionValues.has(key)) {
+                // The partition field restores it on read; another value would put the document in another partition
+                if (isDeepStrictEqual(field, this.partitionValues.get(key)))
+                    flat.kept.push(key);
+                else
+                    problems.push(`the field ${key} names the partition and has another value than the partition`);
+                continue;
+            }
+            if (!isNativeValue(field, key, problems))
+                return undefined;
+            flat.fields.push([key, field]);
+        }
+        if (problems.length > 0)
+            throw new TypeError(`The storage cannot keep the item ${JSON.stringify(id)}: ${problems.join("; ")}`);
+        return flat;
+    }
+    /** The id and the item of a stored document, or undefined when the document does not tell its id. */
+    decode(doc, known) {
+        const id = this.idOfDoc(doc) ?? known;
+        if (id === undefined)
+            return undefined;
+        return [id, itemOf(doc, id, this.partitionEntries, false)];
+    }
+    /** The entries of stored documents, in their order. A document that does not tell its id is left out. */
+    decodeAll(docs) {
+        const entries = [];
+        for (const doc of docs) {
+            const entry = this.decode(doc);
+            if (entry)
+                entries.push(entry);
+        }
+        return entries;
     }
     /**
      * The next `count` numbers of the sequence of the collection. They stay pending in the counter
@@ -468,17 +739,33 @@ export class MongoDocumentStorage {
         const floor = Math.max(...oldest.map((doc) => Number(doc[SEQ])));
         await db.collection(this.countersName).updateOne({ _id: this.floorId() }, { $max: { floor } }, { upsert: true });
     }
+    /** The ids of stored documents, in their order. A document that does not tell its id is left out. */
+    idsOf(docs) {
+        const ids = [];
+        for (const doc of docs) {
+            const id = this.idOfDoc(doc);
+            if (id !== undefined)
+                ids.push(id);
+        }
+        return ids;
+    }
     /** The items of the partition, in the order of the entries. */
     async liveDocs(extra) {
         const { items } = await this.collection();
-        return items.find(this.liveFilter(extra)).sort({ [ORD]: 1 }).toArray();
+        return items.find(this.liveFilter(extra)).sort({ [ORD]: 1, _id: 1 }).toArray();
     }
     /** The places (`_ord`) of the items of some ids that the partition has. */
     async placesOf(collection, ids) {
         const docs = await collection
             .find(this.liveFilter({ _id: { $in: [...new Set(ids)].map((id) => this.storedId(id)) } }), { projection: { [KEY]: 1, [ORD]: 1 } })
             .toArray();
-        return new Map(docs.map((doc) => [String(doc[KEY]), Number(doc[ORD])]));
+        const places = new Map();
+        for (const doc of docs) {
+            const id = this.idOfDoc(doc);
+            if (id !== undefined)
+                places.set(id, this.placeOf(doc));
+        }
+        return places;
     }
     // ===========================================================================
     // Reads
@@ -486,18 +773,18 @@ export class MongoDocumentStorage {
     async get(id) {
         const { items } = await this.collection();
         const doc = await items.findOne(this.liveFilter({ _id: this.storedId(id) }));
-        return doc ? this.decode(doc)[1] : undefined;
+        return doc ? this.decode(doc, id)?.[1] : undefined;
     }
     async getAll() {
-        return (await this.liveDocs()).map((doc) => this.decode(doc)[1]);
+        return this.decodeAll(await this.liveDocs()).map(([, value]) => value);
     }
     async getEntries() {
-        return (await this.liveDocs()).map((doc) => this.decode(doc));
+        return this.decodeAll(await this.liveDocs());
     }
     async getIds() {
         const { items } = await this.collection();
-        const docs = await items.find(this.liveFilter(), { projection: { [KEY]: 1 } }).sort({ [ORD]: 1 }).toArray();
-        return docs.map((doc) => String(doc[KEY]));
+        const docs = await items.find(this.liveFilter(), { projection: { _id: 1, [KEY]: 1 } }).sort({ [ORD]: 1, _id: 1 }).toArray();
+        return this.idsOf(docs);
     }
     async find(predicate) {
         return (await this.getAll()).filter(predicate);
@@ -506,9 +793,9 @@ export class MongoDocumentStorage {
         const rule = policy ?? this.options.queryPolicy;
         // The reference checks the selector and the options first, without a read of the database
         queryEntries([], selector, options, rule);
-        const filter = selectorPrefilter(selector);
+        const filter = selectorPrefilter(selector, { storedId: (id) => this.storedId(id), storageFields: this.reserved });
         const docs = await this.liveDocs(filter ? { $or: [{ [JSON_FIELD]: { $exists: true } }, filter] } : undefined);
-        return queryEntries(docs.map((doc) => this.decode(doc)), selector, options, rule);
+        return queryEntries(this.decodeAll(docs), selector, options, rule);
     }
     async has(id) {
         const { items } = await this.collection();
@@ -516,7 +803,12 @@ export class MongoDocumentStorage {
     }
     async size() {
         const { items } = await this.collection();
-        return items.countDocuments(this.liveFilter());
+        // A document with `_key` tells its id. A document from outside the storage counts only when `idOf` names it, as in getAll
+        const [all, outside] = await Promise.all([
+            items.countDocuments(this.liveFilter()),
+            items.find(this.liveFilter({ [KEY]: { $exists: false } }), { projection: { _id: 1 } }).toArray(),
+        ]);
+        return all - (outside.length - this.idsOf(outside).length);
     }
     async getBatch(ids) {
         const result = new Map();
@@ -526,7 +818,7 @@ export class MongoDocumentStorage {
         }
         const { items } = await this.collection();
         const docs = await items.find(this.liveFilter({ _id: { $in: ids.map((id) => this.storedId(id)) } })).toArray();
-        const byId = new Map(docs.map((doc) => this.decode(doc)));
+        const byId = new Map(this.decodeAll(docs));
         // The order of the ids of the call
         for (const id of ids)
             if (byId.has(id))
@@ -554,11 +846,16 @@ export class MongoDocumentStorage {
         const results = await items.aggregate(bounded.pipeline, { maxTimeMS }).toArray();
         return cutResults(results, maxResults, bounded.limited);
     }
-    /** The version of an item: the sequence number of its last write, or undefined for a missing id. */
+    /**
+     * The version of an item: the sequence number of its last write, or undefined for a missing id.
+     * An item without a write through the storage (no `_seq`) has the version `0`.
+     */
     async versionOf(id) {
         const { items } = await this.collection();
         const doc = await items.findOne(this.liveFilter({ _id: this.storedId(id) }), { projection: { [SEQ]: 1 } });
-        return doc ? String(doc[SEQ]) : undefined;
+        if (!doc)
+            return undefined;
+        return typeof doc[SEQ] === "number" ? String(doc[SEQ]) : "0";
     }
     async getChangesSince(since) {
         const { db, items } = await this.collection();
@@ -568,7 +865,7 @@ export class MongoDocumentStorage {
             const full = async () => ({
                 version,
                 full: true,
-                changed: (await items.find(this.liveFilter()).sort({ [ORD]: 1 }).toArray()).map((doc) => this.decode(doc)),
+                changed: this.decodeAll(await items.find(this.liveFilter()).sort({ [ORD]: 1, _id: 1 }).toArray()),
                 deleted: [],
             });
             const match = since === undefined ? null : /^([^.]+)\.(\d+)$/.exec(since);
@@ -587,10 +884,13 @@ export class MongoDocumentStorage {
             const changed = [];
             const deleted = [];
             for (const doc of docs) {
+                const id = this.idOfDoc(doc);
+                if (id === undefined)
+                    continue;
                 if (doc[TOMB] === true)
-                    deleted.push(String(doc[KEY]));
+                    deleted.push(id);
                 else
-                    changed.push(this.decode(doc));
+                    changed.push(this.decode(doc, id));
             }
             return { version, full: false, changed, deleted };
         });
@@ -618,6 +918,7 @@ export class MongoDocumentStorage {
         // The ids and the encoding are checked before a number of the sequence goes
         for (const [id, value] of items) {
             this.storedId(id);
+            this.flatOf(id, value);
             JSON.stringify(value);
         }
         await this.write(async (db, collection) => {
@@ -651,8 +952,10 @@ export class MongoDocumentStorage {
             const doc = await collection.findOne(this.liveFilter({ _id: this.storedId(id) }));
             if (!doc)
                 return undefined;
-            const next = applyPatch(this.decode(doc)[1], fields);
-            await this.withNumbers(db, collection, 1, (first, session) => collection.replaceOne(this.writeFilter(id), this.encode(id, next, first, Number(doc[ORD])), { upsert: true, ...(session ? { session } : {}) }));
+            const next = applyPatch(this.decode(doc, id)[1], fields);
+            // The encoding is checked before a number of the sequence goes
+            this.flatOf(id, next);
+            await this.withNumbers(db, collection, 1, (first, session) => collection.replaceOne(this.writeFilter(id), this.encode(id, next, first, this.placeOf(doc)), { upsert: true, ...(session ? { session } : {}) }));
             return next;
         });
     }
@@ -687,7 +990,7 @@ export class MongoDocumentStorage {
                 .toArray();
             if (live.length === 0)
                 return 0;
-            const liveIds = new Set(live.map((doc) => String(doc[KEY])));
+            const liveIds = new Set(this.idsOf(live));
             const deleted = unique.filter((id) => liveIds.has(id));
             await this.tombstone(db, collection, deleted);
             return deleted.length;
@@ -695,10 +998,11 @@ export class MongoDocumentStorage {
     }
     async clear(_options) {
         await this.write(async (db, collection) => {
-            const live = await collection.find(this.liveFilter(), { projection: { [KEY]: 1 } }).sort({ [ORD]: 1 }).toArray();
-            if (live.length === 0)
+            const live = await collection.find(this.liveFilter(), { projection: { _id: 1, [KEY]: 1 } }).sort({ [ORD]: 1, _id: 1 }).toArray();
+            const ids = this.idsOf(live);
+            if (ids.length === 0)
                 return;
-            await this.tombstone(db, collection, live.map((doc) => String(doc[KEY])));
+            await this.tombstone(db, collection, ids);
         });
     }
     /**
@@ -710,8 +1014,7 @@ export class MongoDocumentStorage {
             const guards = await this.checkVersions(collection, expected, session);
             const operations = ids.map((id, index) => {
                 const tomb = { _id: this.storedId(id), [KEY]: id, [SEQ]: first + index, [TOMB]: true };
-                if (this.partition !== undefined)
-                    tomb[PART] = this.partition;
+                this.putPartition(tomb);
                 return { replaceOne: { filter: { ...this.writeFilter(id), ...guards.get(id) }, replacement: tomb } };
             });
             const result = await collection.bulkWrite(operations, { ordered: true, ...(session ? { session } : {}) });
@@ -739,11 +1042,18 @@ export class MongoDocumentStorage {
         const docs = await collection
             .find({ ...this.partitionFilter(), _id: { $in: [...new Set(ids)].map((id) => this.storedId(id)) } }, session ? { session } : {})
             .toArray();
-        return new Map(docs.map((doc) => [String(doc[KEY]), doc]));
+        const byId = new Map();
+        for (const doc of docs) {
+            const id = this.idOfDoc(doc);
+            if (id !== undefined)
+                byId.set(id, doc);
+        }
+        return byId;
     }
     /** The current item of a stored document: no item for a tombstone or a missing document. */
     currentOf(doc) {
-        return doc && doc[TOMB] !== true ? { found: true, value: this.decode(doc)[1] } : { found: false };
+        const entry = doc && doc[TOMB] !== true ? this.decode(doc) : undefined;
+        return entry ? { found: true, value: entry[1] } : { found: false };
     }
     /**
      * This method tests the expected versions (roadmap CLI-16). A conflict throws
@@ -768,7 +1078,8 @@ export class MongoDocumentStorage {
             return guards;
         for (const [id] of expected) {
             const doc = docs.get(id);
-            guards.set(id, doc ? { [SEQ]: doc[SEQ] } : { [SEQ]: { $exists: false } });
+            // A document without `_seq` (not written through the storage yet) keeps that state until the write
+            guards.set(id, doc && typeof doc[SEQ] === "number" ? { [SEQ]: doc[SEQ] } : { [SEQ]: { $exists: false } });
         }
         return guards;
     }