@procedures/mongo 0.34.0 → 0.35.0

package/src/storage/document-storage.ts

@@ -4,10 +4,13 @@
  * `{ _id, data }` documents.
  *
  * The documents:
- * - An item that is a plain object with safe keys is stored flat: its fields beside the fields of
- *   the storage. Another item (null, a primitive, an array, or an object with a key of the storage,
- *   a key that starts with `$` or a key with a dot) is stored as JSON text in `_json`. Both ways
- *   give the item back as it was.
+ * - An item that is a plain object is stored flat: its fields beside the fields of the storage. A
+ *   key with `$` or a dot below the top level, and a top-level key with a dot, stay as they are
+ *   (MongoDB 5.0 or later keeps them). Another item is stored as JSON text in `_json`: null, a
+ *   primitive, an array, an object with a top-level key of the storage or a top-level key that
+ *   starts with `$`, a key `__proto__`, or a class instance other than `Date`. Both ways give the
+ *   item back as it was. An object with a text `$ref` and an `$id` is refused (`TypeError`):
+ *   MongoDB gives it back as a `DBRef`.
  * - An item whose `_id` is its id keeps it: the storage gives it back.
  * - The fields of the storage: `_key` (the id), `_part` (the partition), `_seq` (the sequence
  *   number of the last write), `_ord` (the number of the first write: the order of the entries, as
@@ -16,6 +19,40 @@
  *   character U+001F. Each partition reads and writes only its documents: each write filter has the
  *   partition too, so a write of a stored `_id` of another partition fails and changes nothing.
  *
+ * The stored form (roadmap CLI-76): an app with its own form of documents keeps it with the option
+ * `form`, one `StoredForm` object. `DEFAULT_STORED_FORM` gives the form above, byte for byte.
+ * - `storedId(partition, id)` gives the stored `_id`, and `idOf(storedId, partition)` is its inverse.
+ *   The stored `_id` can be any value but `undefined` and `null`, for example `tenant:key`.
+ * - `partitionFields(partition)` gives the fields that name the partition (default `{ _part }`).
+ *   Each document of the partition has them, and they are the filter of the partition. A field with
+ *   the value `undefined` is absent: the document does not have it, and the filter is
+ *   `{ $exists: false }`. An item can have a partition field (one that is not a field of the
+ *   storage) with the value of its partition: the write leaves it out and notes it in `_kpf`, and
+ *   the read gives it back. The same field with another value is refused (`TypeError`).
+ * - `partitionOf(doc)` gives the partition of a stored document, for `ViewMaterializer`. A delete
+ *   event has only the `documentKey`: so the `_id` must tell the partition, or the collection needs
+ *   `changeStreamPreAndPostImages`.
+ * - The storage adds its fields (`_key`, `_seq`, `_ord`, `_tomb`, and `_kid` or `_json` as needed)
+ *   in each form. A reader of the raw documents can leave them out (`DOCUMENT_STORAGE_FIELDS`), and
+ *   must leave out the tombstones (`TOMBSTONE_FIELD` is `true`). `decodeStoredDocument(doc)` does
+ *   this, as the storage reads a document.
+ * - A document without the fields of the storage (for example one written before the app used the
+ *   storage) is an item too. Its id comes from `idOf`, else from `_key`, and its item gets that id
+ *   as `_id`. Its partition fields follow the rule of a written document without `_kpf`: a field
+ *   with the value of the partition stays out, and a field with another value stays in. So the
+ *   item has the same form before and after its first write. A document whose id neither
+ *   tells (for example an `ObjectId` that `idOf` does not name) is not an item: the reads and
+ *   `size()` leave it out. Its version is `0`, and it comes before the other items in the order of
+ *   the entries (then by `_id`). A full result of the change log has it, and a delta has it after
+ *   its first write through the storage. That write gives it its fields and a version, and keeps
+ *   its place.
+ * - Write through the storage. A raw write (for example `findOneAndReplace` or a script) takes no
+ *   number of the sequence, so a delta of the change log does not give it, and a raw replace drops
+ *   `_seq`.
+ * - A raw writer must not hard-delete an item while the storage writes it. The write of a document
+ *   without `_seq` is an upsert with `{ _seq: { $exists: false } }`: after a hard delete between
+ *   the read and the write, the upsert inserts the document again.
+ *
  * The change log:
  * - Each write gets the next number of the sequence of the collection. `_seq` is also the version
  *   of the item (`versionOf`).
@@ -31,6 +68,9 @@
  *   its lease moves its documents to new numbers, so a reader that passed the old numbers gets
  *   it. One case stays on a single server: a process that stops between such a late write and the
  *   move of its documents. Then a reader that passed the old numbers does not get that change.
+ * - The storage uses a transaction only with a client: the option `client`, or the default
+ *   connection when `db` is absent. With `db` and no `client`, each write uses the watermark, also
+ *   on a replica set.
  *
  * The queries: `query()` sends the conditions that MongoDB tests in the same way (the equality of
  * a string, a number or a boolean, and `$in` of them) as a filter. `queryEntries()` of the core
@@ -45,9 +85,14 @@
  * precondition has the `_seq` that the test read in its filter (a compare-and-set). Another writer
  * between the test and the write makes the write fail with `VersionConflictError`. Then a batch of
  * more than one item can keep a part of its writes: only a transaction makes a batch all or nothing.
+ * One case stays on a single server: a writer outside the storage that writes a document without
+ * `_seq` between the test and the write. The guard of such a document is `{ _seq: { $exists:
+ * false } }`, so it does not see that writer. On a replica set, the test and the write are one
+ * transaction, and this case does not occur.
  */
 
 import { randomUUID } from "node:crypto";
+import { isDeepStrictEqual } from "node:util";
 import type { AnyBulkWriteOperation, ClientSession, Collection, Db, Document, Filter, MongoClient } from "mongodb";
 import {
   VersionConflictError,
@@ -82,10 +127,81 @@ const TOMB = "_tomb";
 const JSON_FIELD = "_json";
 /** True in a stored document whose item had `_id` equal to its id. */
 const KEEP_ID = "_kid";
-const STORAGE_FIELDS: ReadonlySet<string> = new Set(["_id", KEY, PART, SEQ, ORD, TOMB, JSON_FIELD, KEEP_ID]);
+/** The partition fields that the item had with the value of its partition (roadmap CLI-76). The write leaves them out of the fields. */
+const KEEP_PART = "_kpf";
+const STORAGE_FIELDS: ReadonlySet<string> = new Set(["_id", KEY, PART, SEQ, ORD, TOMB, JSON_FIELD, KEEP_ID, KEEP_PART]);
 /** The separator of the partition and the id in a stored `_id`. */
 export const SEPARATOR = "\u001f";
 
+/**
+ * The fields that a `MongoDocumentStorage` can add to a stored document, in each stored form
+ * (roadmap CLI-76). A reader of the raw documents can leave them out. The partition fields are
+ * not in the list.
+ */
+export const DOCUMENT_STORAGE_FIELDS: readonly string[] = Object.freeze([KEY, PART, SEQ, ORD, TOMB, JSON_FIELD, KEEP_ID, KEEP_PART]);
+
+/** The field that is `true` in a tombstone: the document of a deleted item. A reader of the raw documents leaves them out. */
+export const TOMBSTONE_FIELD: "_tomb" = TOMB;
+
+/**
+ * The stored form of a `MongoDocumentStorage` (roadmap CLI-76): how an id and a partition become a
+ * stored document, and back. Give the same object to the storage (`form`) and to a
+ * `ViewMaterializer` (`storageSources`), or read it from `storage.form`.
+ */
+export interface StoredForm {
+  /**
+   * The stored `_id` of an id in a partition. It must give a value other than `undefined` and
+   * `null`, and two ids of one partition must not give the same value.
+   */
+  storedId(partition: string | undefined, id: string): unknown;
+  /** The inverse of `storedId`: the id of a stored `_id`, or `undefined` when the `_id` is not of the partition. */
+  idOf(storedId: unknown, partition: string | undefined): string | undefined;
+  /**
+   * The fields that name a partition. Each document of the partition has them, and they are the
+   * filter of the partition. A field with the value `undefined` is absent: the document does not
+   * have it, and the filter is `{ $exists: false }`. Give the same names for each partition. A
+   * top-level field of an item with one of these names and the value of the partition is left out
+   * of the stored fields and noted in `_kpf`, and the read gives it back. Another value is refused
+   * with a `TypeError`.
+   */
+  partitionFields(partition: string | undefined): Readonly<Record<string, unknown>>;
+  /**
+   * The partition of a stored document, or `undefined` when the document does not tell it. A
+   * `ViewMaterializer` gives the document of a change event: `fullDocument`, else
+   * `fullDocumentBeforeChange`, else `documentKey`. A delete event has only `documentKey`, so with
+   * another `storedId`, the `_id` must tell the partition, or the collection needs
+   * `changeStreamPreAndPostImages`. The storage writes a delete as a tombstone (a replace event with
+   * the whole document), so the gap is only for a raw hard delete.
+   */
+  partitionOf(doc: Document): string | undefined;
+}
+
+/**
+ * The default stored form, the form of 0.34.0. The stored `_id` is the id, or `<partition>` +
+ * U+001F + `<id>` with a partition. The partition field is `_part` (absent without a partition).
+ * `partitionOf` reads `_part`, else the part of `_id` before U+001F.
+ */
+export const DEFAULT_STORED_FORM: StoredForm = Object.freeze({
+  storedId(partition: string | undefined, id: string): string {
+    return partition === undefined ? id : `${partition}${SEPARATOR}${id}`;
+  },
+  idOf(storedId: unknown, partition: string | undefined): string | undefined {
+    if (typeof storedId !== "string") return undefined;
+    if (partition === undefined) return storedId.includes(SEPARATOR) ? undefined : storedId;
+    const prefix = `${partition}${SEPARATOR}`;
+    return storedId.startsWith(prefix) ? storedId.slice(prefix.length) : undefined;
+  },
+  partitionFields(partition: string | undefined): Record<string, unknown> {
+    return { [PART]: partition };
+  },
+  partitionOf(doc: Document): string | undefined {
+    if (typeof doc[PART] === "string") return doc[PART];
+    const id: unknown = doc["_id"];
+    const at = typeof id === "string" ? id.indexOf(SEPARATOR) : -1;
+    return at > 0 ? (id as string).slice(0, at) : undefined;
+  },
+});
+
 /** The options of a `MongoDocumentStorage`. */
 export interface MongoDocumentStorageOptions<T = unknown> {
   /** The MongoDB collection. */
@@ -107,7 +223,8 @@ export interface MongoDocumentStorageOptions<T = unknown> {
   writeLease?: Duration | undefined;
   /**
    * The client of the database, for the transactions. The default is the client of the default
-   * connection when `db` is absent. Without a client, the writes use the watermark.
+   * connection when `db` is absent. Without a client, the writes use the watermark: with `db` and
+   * no `client`, a write is not a transaction, also on a replica set.
    */
   client?: (() => MongoClient | Promise<MongoClient>) | undefined;
   /**
@@ -123,6 +240,11 @@ export interface MongoDocumentStorageOptions<T = unknown> {
    * the sequence number of the last write.
    */
   versionOf?: ((item: T) => string | undefined) | undefined;
+  /**
+   * The stored form (roadmap CLI-76): the stored `_id`, its inverse, the partition fields and the
+   * partition of a document. The default `DEFAULT_STORED_FORM` writes the documents of 0.34.0.
+   */
+  form?: StoredForm | undefined;
 }
 
 /** True for the error of a duplicate key: an insert of an upsert whose filter found no document. */
@@ -132,8 +254,8 @@ function isDuplicateKey(error: unknown): boolean {
   return code === 11000 || [writeErrors].flat().some((entry) => (entry as { code?: unknown } | null)?.code === 11000);
 }
 
-/** A stored document. */
-type Stored = Document & { _id: string };
+/** A stored document. Its `_id` comes from `storedId`: a text by default, any value but `undefined` and `null` in another form. */
+type Stored = Document;
 
 /** A range of numbers that a write took and has not written yet. */
 interface PendingRange {
@@ -190,20 +312,113 @@ function enqueue<R>(db: Db, collection: string, task: () => Promise<R>): Promise
   return run;
 }
 
-/** True for a plain object whose keys a flat document can keep, at each depth. */
-function isFlatSafe(value: unknown, top: boolean): boolean {
-  if (Array.isArray(value)) return value.every((element) => isFlatSafe(element, false));
+/**
+ * True for a value of a field that BSON gives back as it is: a key with `$` or a dot stays as it is
+ * (MongoDB 5.0 or later). False when only JSON text keeps it: a class instance other than `Date`,
+ * or a key `__proto__`. An object that BSON gives back as a `DBRef` adds a problem.
+ */
+function isNativeValue(value: unknown, path: string, problems: string[]): boolean {
+  if (Array.isArray(value)) return value.every((element, index) => isNativeValue(element, `${path}.${index}`, problems));
   if (typeof value !== "object" || value === null) return true;
   const proto = Object.getPrototypeOf(value) as unknown;
   if (proto !== Object.prototype && proto !== null) return value instanceof Date;
-  for (const [key, field] of Object.entries(value)) {
-    if (key.startsWith("$") || key.includes(".") || key === "__proto__") return false;
-    if (top && STORAGE_FIELDS.has(key) && key !== "_id") return false;
-    if (!isFlatSafe(field, false)) return false;
+  const object = value as Record<string, unknown>;
+  // The test of the BSON reader: a text `$ref`, an `$id`, and no `$db` or a text `$db`
+  if (typeof object["$ref"] === "string" && object["$id"] !== undefined && object["$id"] !== null && (object["$db"] === undefined || typeof object["$db"] === "string")) {
+    problems.push(`the field ${path} has $ref and $id, and MongoDB gives it back as a DBRef`);
+  }
+  for (const [key, field] of Object.entries(object)) {
+    if (key === "__proto__") return false;
+    if (!isNativeValue(field, `${path}.${key}`, problems)) return false;
   }
   return true;
 }
 
+/** The fields of the flat document of an item. */
+interface FlatItem {
+  /** True when the item has `_id`, equal to its id. */
+  keepsId: boolean;
+  /** The partition fields that the item has with the value of its partition: the write leaves them out. */
+  kept: string[];
+  fields: Array<[string, unknown]>;
+}
+
+/**
+ * The item of a stored document (not a tombstone) whose id is known. `_json` gives the item as it
+ * was. Else the fields of the storage stay out, the item gets `_id` when it had it (`_kid`), when
+ * the document is from outside the storage (no `_key`) or with `addId`, and it gets back the
+ * partition fields of `_kpf`. A partition field with the value of the partition stays out (for a
+ * document from outside the storage too); a field with another value stays in.
+ */
+function itemOf(doc: Document, id: string, partitionEntries: ReadonlyArray<readonly [string, unknown]>, addId: boolean): unknown {
+  if (typeof doc[JSON_FIELD] === "string") {
+    const item = JSON.parse(doc[JSON_FIELD]) as unknown;
+    if (addId && isPlainObject(item) && !("_id" in item)) return { _id: id, ...item };
+    return item;
+  }
+  const outside = doc[KEY] === undefined;
+  const value: Record<string, unknown> = {};
+  if (addId || outside || doc[KEEP_ID] === true) value["_id"] = id;
+  const partitionValues = new Map(partitionEntries);
+  for (const [key, field] of Object.entries(doc)) {
+    if (STORAGE_FIELDS.has(key)) continue;
+    // The value of the partition comes back only through `_kpf`, so the item has one form at each age of its document
+    if (partitionValues.has(key) && isDeepStrictEqual(field, partitionValues.get(key))) continue;
+    put(value, key, field);
+  }
+  const kept = doc[KEEP_PART];
+  if (Array.isArray(kept)) {
+    for (const [name, field] of partitionEntries) if (kept.includes(name) && !STORAGE_FIELDS.has(name)) put(value, name, field);
+  }
+  return value;
+}
+
+/** The options of `decodeStoredDocument`. */
+export interface DecodeStoredDocumentOptions {
+  /** The stored form of the storage, for example `storage.form` (default `DEFAULT_STORED_FORM`). */
+  form?: StoredForm | undefined;
+  /** The partition of the document. The default is `form.partitionOf(doc)`. */
+  partition?: string | undefined;
+  /**
+   * True (the default): an item that is a plain object without `_id` gets its id as `_id`. False:
+   * the item is as the storage gives it, with `_id` only when it had it.
+   */
+  addId?: boolean | undefined;
+}
+
+/** An item of a stored document, from `decodeStoredDocument`. */
+export interface StoredDocumentEntry<T> {
+  /** The id of the item. */
+  id: string;
+  /** The partition of the document. */
+  partition: string | undefined;
+  /** The item. */
+  item: T;
+}
+
+/**
+ * This function reads a raw document of a `MongoDocumentStorage` collection as the storage reads
+ * it (roadmap CLI-76), for an app that reads the collection directly (a query, a view, an
+ * aggregation). It gives `undefined` for a tombstone and for a document whose id neither `idOf`
+ * nor `_key` tells. Else it parses `_json`, leaves out the fields of the storage
+ * (`DOCUMENT_STORAGE_FIELDS`) and gives the item its id as `_id` (see `addId`).
+ *
+ * @example
+ * ```typescript
+ * const items = docs.flatMap((doc) => decodeStoredDocument<Note>(doc, { form: notes.form })?.item ?? []);
+ * ```
+ */
+export function decodeStoredDocument<T = Document>(doc: Document, options: DecodeStoredDocumentOptions = {}): StoredDocumentEntry<T> | undefined {
+  if (!isPlainObject(doc) || doc[TOMB] === true) return undefined;
+  const form = options.form ?? DEFAULT_STORED_FORM;
+  const partition = options.partition ?? form.partitionOf(doc);
+  const fromId = form.idOf(doc["_id"], partition);
+  const id = typeof fromId === "string" ? fromId : typeof doc[KEY] === "string" ? doc[KEY] : undefined;
+  if (id === undefined) return undefined;
+  const entries = Object.entries(form.partitionFields(partition));
+  return { id, partition, item: itemOf(doc, id, entries, options.addId ?? true) as T };
+}
+
 /** True for a plain object. */
 function isPlainObject(value: unknown): value is Record<string, unknown> {
   if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
@@ -221,17 +436,52 @@ function isScalar(value: unknown): value is string | number | boolean {
   return typeof value === "string" || typeof value === "boolean" || (typeof value === "number" && Number.isFinite(value));
 }
 
+/** The options of `selectorPrefilter`. */
+export interface SelectorPrefilterOptions {
+  /**
+   * The stored `_id` of an id. With it, a condition on `_id` with text values tests the stored
+   * `_id` (and uses its index). Without it, the condition tests `_key`.
+   */
+  storedId?: ((id: string) => unknown) | undefined;
+  /** More fields that only the storage writes, for example the partition fields: a condition on one of them does not go in the filter. */
+  storageFields?: Iterable<string> | undefined;
+}
+
 /** The field of a stored document for a path of a selector, or undefined when the filter cannot test it. */
-function storedPath(path: string): string | undefined {
+function storedPath(path: string, reserved: ReadonlySet<string>): string | undefined {
   if (path === "_id") return KEY;
-  const first = path.split(".")[0]!;
-  if (first === "_id" || STORAGE_FIELDS.has(first)) return undefined;
+  const parts = path.split(".");
+  const first = parts[0]!;
+  if (first === "_id" || STORAGE_FIELDS.has(first) || reserved.has(first)) return undefined;
+  // A part with `$` is a field name of an item, but an operator for MongoDB
+  if (parts.some((part) => part.startsWith("$"))) return undefined;
   return path;
 }
 
+/**
+ * The filter of a condition on `_id` against the stored `_id`, or undefined. A flat item with an
+ * `_id` has its id as `_id`, and an item stored as JSON passes the filter of `query()` anyway.
+ */
+function idConditionFilter(condition: unknown, storedId: (id: string) => unknown): Document | undefined {
+  try {
+    if (typeof condition === "string") return { _id: storedId(condition) };
+    if (!isPlainObject(condition)) return undefined;
+    const operators: Document = {};
+    const eq = condition["$eq"];
+    if (typeof eq === "string") operators["$eq"] = storedId(eq);
+    const among = condition["$in"];
+    if (Array.isArray(among) && among.length > 0 && among.every((value) => typeof value === "string")) operators["$in"] = among.map((value: string) => storedId(value));
+    return Object.keys(operators).length > 0 ? { _id: operators } : undefined;
+  } catch {
+    // An id that the storage refuses: no condition is a correct filter (it can pass more)
+    return undefined;
+  }
+}
+
 /** The filter of one condition, or undefined when the filter cannot test it in the same way. */
-function conditionFilter(path: string, condition: unknown): Document | undefined {
-  const field = storedPath(path);
+function conditionFilter(path: string, condition: unknown, options: SelectorPrefilterOptions, reserved: ReadonlySet<string>): Document | undefined {
+  if (path === "_id" && options.storedId) return idConditionFilter(condition, options.storedId);
+  const field = storedPath(path, reserved);
   if (field === undefined) return undefined;
   if (isScalar(condition)) return { [field]: condition };
   if (!isPlainObject(condition)) return undefined;
@@ -247,21 +497,25 @@ function conditionFilter(path: string, condition: unknown): Document | undefined
  * This function gives a filter that each item of the selector passes (a filter can pass more),
  * or undefined for no condition. Only conditions that MongoDB tests in the same way go in it.
  */
-export function selectorPrefilter(selector: StorageSelector): Document | undefined {
+export function selectorPrefilter(selector: StorageSelector, options: SelectorPrefilterOptions = {}): Document | undefined {
+  return prefilter(selector, options, new Set(options.storageFields ?? []));
+}
+
+function prefilter(selector: StorageSelector, options: SelectorPrefilterOptions, reserved: ReadonlySet<string>): Document | undefined {
   const parts: Document[] = [];
   for (const [key, condition] of Object.entries(selector as Record<string, unknown>)) {
     if (condition === undefined) continue;
     if (key === "$and" && Array.isArray(condition)) {
       for (const branch of condition) {
-        const filter = isPlainObject(branch) ? selectorPrefilter(branch as StorageSelector) : undefined;
+        const filter = isPlainObject(branch) ? prefilter(branch as StorageSelector, options, reserved) : undefined;
         if (filter) parts.push(filter);
       }
     } else if (key === "$or" && Array.isArray(condition) && condition.length > 0) {
-      const branches = condition.map((branch) => (isPlainObject(branch) ? selectorPrefilter(branch as StorageSelector) : undefined));
+      const branches = condition.map((branch) => (isPlainObject(branch) ? prefilter(branch as StorageSelector, options, reserved) : undefined));
       // A branch without a condition passes each item: then the $or tests nothing
       if (branches.every((branch): branch is Document => branch !== undefined)) parts.push({ $or: branches });
     } else if (!key.startsWith("$")) {
-      const filter = conditionFilter(key, condition);
+      const filter = conditionFilter(key, condition, options, reserved);
       if (filter) parts.push(filter);
     }
   }
@@ -294,6 +548,14 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
   private readonly writeLeaseMs: number;
   /** The storages of the partitions, for `forPartition`: the same storage for the same partition. */
   private readonly partitions: Map<string, MongoDocumentStorage<T>>;
+  /** The stored form of the documents (roadmap CLI-76). A `ViewMaterializer` of the collection takes the same form. */
+  readonly form: StoredForm;
+  /** The fields that name the partition, in their order. An `undefined` value is an absent field. */
+  private readonly partitionEntries: ReadonlyArray<readonly [string, unknown]>;
+  /** The values of the partition fields that are not fields of the storage, by name. */
+  private readonly partitionValues: ReadonlyMap<string, unknown>;
+  /** The fields that only the storage writes: its fields and the partition fields. */
+  private readonly reserved: ReadonlySet<string>;
   private indexes: Promise<void> | undefined;
   private closed = false;
 
@@ -301,6 +563,21 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     if (options.partition !== undefined && (options.partition === "" || options.partition.includes(SEPARATOR))) {
       throw new Error("A partition must be a text without the character U+001F");
     }
+    const form = options.form ?? DEFAULT_STORED_FORM;
+    for (const name of ["storedId", "idOf", "partitionFields", "partitionOf"] as const) {
+      if (typeof form[name] !== "function") throw new TypeError(`The stored form must have the function ${name}`);
+    }
+    this.form = form;
+    const fields = form.partitionFields(options.partition);
+    if (!isPlainObject(fields) || Object.keys(fields).length === 0) throw new TypeError("partitionFields must give an object with at least one field");
+    for (const name of Object.keys(fields)) {
+      if (name === "_id" || name.startsWith("$") || name.includes(".") || name === "__proto__" || (STORAGE_FIELDS.has(name) && name !== PART)) {
+        throw new TypeError(`partitionFields cannot have the field ${JSON.stringify(name)}`);
+      }
+    }
+    this.partitionEntries = Object.entries(fields);
+    this.partitionValues = new Map(this.partitionEntries.filter(([name]) => !STORAGE_FIELDS.has(name)));
+    this.reserved = new Set([...STORAGE_FIELDS, ...Object.keys(fields)]);
     this.options = options;
     this.collectionName = options.collection;
     this.partition = options.partition;
@@ -334,8 +611,9 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
   private async collection(): Promise<{ db: Db; items: Collection<Stored> }> {
     const db = await this.database();
     const items = db.collection<Stored>(this.collectionName);
+    const prefix = Object.fromEntries(this.partitionEntries.map(([name]) => [name, 1]));
     this.indexes ??= items
-      .createIndexes([{ key: { [PART]: 1, [SEQ]: 1 } }, { key: { [PART]: 1, [ORD]: 1 } }, { key: { [PART]: 1, [TOMB]: 1 } }])
+      .createIndexes([{ key: { ...prefix, [SEQ]: 1 } }, { key: { ...prefix, [ORD]: 1 } }, { key: { ...prefix, [TOMB]: 1 } }])
       .then(() => undefined)
       .catch((error: unknown) => {
         this.indexes = undefined;
@@ -347,7 +625,14 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
 
   /** The filter of the documents of the partition. */
   private partitionFilter(): Document {
-    return this.partition === undefined ? { [PART]: { $exists: false } } : { [PART]: this.partition };
+    // An object value is an equality too, never a set of operators
+    const condition = (value: unknown): unknown => (value === undefined ? { $exists: false } : typeof value === "object" && value !== null ? { $eq: value } : value);
+    return Object.fromEntries(this.partitionEntries.map(([name, value]) => [name, condition(value)]));
+  }
+
+  /** This method puts the partition fields on a stored document, in their order. An absent field stays absent. */
+  private putPartition(doc: Stored): void {
+    for (const [name, value] of this.partitionEntries) if (value !== undefined) put(doc, name, value);
   }
 
   /** The filter of the items (not the tombstones) of the partition. */
@@ -356,9 +641,25 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
   }
 
   /** The stored `_id` of an id. An id with the separator could name an item of a partition: it is refused. */
-  private storedId(id: string): string {
+  private storedId(id: string): unknown {
     if (typeof id !== "string" || id.includes(SEPARATOR)) throw new Error("An id must be a text without the character U+001F");
-    return this.partition === undefined ? id : `${this.partition}${SEPARATOR}${id}`;
+    const stored = this.form.storedId(this.partition, id);
+    if (stored === undefined || stored === null) throw new TypeError(`storedId gave ${String(stored)} for the id ${JSON.stringify(id)}`);
+    return stored;
+  }
+
+  /** The id of a stored document: the inverse of its stored `_id`, else `_key`, or undefined when neither tells it. */
+  private idOfDoc(doc: Document): string | undefined {
+    const id = this.form.idOf(doc["_id"], this.partition);
+    if (typeof id === "string") return id;
+    const key = doc[KEY];
+    return typeof key === "string" ? key : undefined;
+  }
+
+  /** The place of a stored document in the order of the entries. A document without `_ord` comes first. */
+  private placeOf(doc: Document): number {
+    const ord = doc[ORD];
+    return typeof ord === "number" ? ord : 0;
   }
 
   /** The filter of a write of one stored document: its `_id` and the partition. */
@@ -369,25 +670,61 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
   /** The stored document of an item, with the number of this write and the number of its place. */
   private encode(id: string, value: T, seq: number, ord: number): Stored {
     const doc: Stored = { _id: this.storedId(id), [KEY]: id, [SEQ]: seq, [ORD]: ord };
-    if (this.partition !== undefined) doc[PART] = this.partition;
-    const keepsId = isPlainObject(value) && value["_id"] === id;
-    if (isPlainObject(value) && isFlatSafe(value, true) && (!("_id" in value) || keepsId)) {
-      if (keepsId) doc[KEEP_ID] = true;
-      for (const [key, field] of Object.entries(value)) if (key !== "_id") put(doc, key, field);
+    this.putPartition(doc);
+    const flat = this.flatOf(id, value);
+    if (flat) {
+      if (flat.keepsId) doc[KEEP_ID] = true;
+      if (flat.kept.length > 0) doc[KEEP_PART] = flat.kept;
+      for (const [key, field] of flat.fields) put(doc, key, field);
     } else {
       doc[JSON_FIELD] = JSON.stringify(value);
     }
     return doc;
   }
 
-  /** The id and the item of a stored document. */
-  private decode(doc: Document): [string, T] {
-    const id = String(doc[KEY]);
-    if (typeof doc[JSON_FIELD] === "string") return [id, JSON.parse(doc[JSON_FIELD]) as T];
-    const value: Record<string, unknown> = {};
-    if (doc[KEEP_ID] === true) value["_id"] = id;
-    for (const [key, field] of Object.entries(doc)) if (!STORAGE_FIELDS.has(key)) put(value, key, field);
-    return [id, value as T];
+  /**
+   * The fields of the flat document of an item, or undefined when only JSON text keeps it: an item
+   * that is not a plain object, an `_id` other than its id, a top-level key of the storage or with
+   * `$`, a key `__proto__`, a class instance other than `Date`. It throws a `TypeError` for a flat
+   * item that a document cannot keep: a partition field with another value than the partition, or
+   * an object that MongoDB gives back as a `DBRef`.
+   */
+  private flatOf(id: string, value: T): FlatItem | undefined {
+    if (!isPlainObject(value)) return undefined;
+    if ("_id" in value && value["_id"] !== id) return undefined;
+    const flat: FlatItem = { keepsId: "_id" in value, kept: [], fields: [] };
+    const problems: string[] = [];
+    for (const [key, field] of Object.entries(value)) {
+      if (key === "_id") continue;
+      if (key.startsWith("$") || key === "__proto__" || STORAGE_FIELDS.has(key)) return undefined;
+      if (this.partitionValues.has(key)) {
+        // The partition field restores it on read; another value would put the document in another partition
+        if (isDeepStrictEqual(field, this.partitionValues.get(key))) flat.kept.push(key);
+        else problems.push(`the field ${key} names the partition and has another value than the partition`);
+        continue;
+      }
+      if (!isNativeValue(field, key, problems)) return undefined;
+      flat.fields.push([key, field]);
+    }
+    if (problems.length > 0) throw new TypeError(`The storage cannot keep the item ${JSON.stringify(id)}: ${problems.join("; ")}`);
+    return flat;
+  }
+
+  /** The id and the item of a stored document, or undefined when the document does not tell its id. */
+  private decode(doc: Document, known?: string): [string, T] | undefined {
+    const id = this.idOfDoc(doc) ?? known;
+    if (id === undefined) return undefined;
+    return [id, itemOf(doc, id, this.partitionEntries, false) as T];
+  }
+
+  /** The entries of stored documents, in their order. A document that does not tell its id is left out. */
+  private decodeAll(docs: readonly Document[]): Array<[string, T]> {
+    const entries: Array<[string, T]> = [];
+    for (const doc of docs) {
+      const entry = this.decode(doc);
+      if (entry) entries.push(entry);
+    }
+    return entries;
   }
 
   /**
@@ -559,10 +896,20 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     await db.collection<CounterDoc>(this.countersName).updateOne({ _id: this.floorId() }, { $max: { floor } }, { upsert: true });
   }
 
+  /** The ids of stored documents, in their order. A document that does not tell its id is left out. */
+  private idsOf(docs: readonly Document[]): string[] {
+    const ids: string[] = [];
+    for (const doc of docs) {
+      const id = this.idOfDoc(doc);
+      if (id !== undefined) ids.push(id);
+    }
+    return ids;
+  }
+
   /** The items of the partition, in the order of the entries. */
   private async liveDocs(extra?: Document): Promise<Document[]> {
     const { items } = await this.collection();
-    return items.find(this.liveFilter(extra)).sort({ [ORD]: 1 }).toArray();
+    return items.find(this.liveFilter(extra)).sort({ [ORD]: 1, _id: 1 }).toArray();
   }
 
   /** The places (`_ord`) of the items of some ids that the partition has. */
@@ -570,7 +917,12 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     const docs = await collection
       .find(this.liveFilter({ _id: { $in: [...new Set(ids)].map((id) => this.storedId(id)) } }), { projection: { [KEY]: 1, [ORD]: 1 } })
       .toArray();
-    return new Map(docs.map((doc) => [String(doc[KEY]), Number(doc[ORD])]));
+    const places = new Map<string, number>();
+    for (const doc of docs) {
+      const id = this.idOfDoc(doc);
+      if (id !== undefined) places.set(id, this.placeOf(doc));
+    }
+    return places;
   }
 
   // ===========================================================================
@@ -580,21 +932,21 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
   async get(id: string): Promise<T | undefined> {
     const { items } = await this.collection();
     const doc = await items.findOne(this.liveFilter({ _id: this.storedId(id) }));
-    return doc ? this.decode(doc)[1] : undefined;
+    return doc ? this.decode(doc, id)?.[1] : undefined;
   }
 
   async getAll(): Promise<T[]> {
-    return (await this.liveDocs()).map((doc) => this.decode(doc)[1]);
+    return this.decodeAll(await this.liveDocs()).map(([, value]) => value);
   }
 
   async getEntries(): Promise<Array<[string, T]>> {
-    return (await this.liveDocs()).map((doc) => this.decode(doc));
+    return this.decodeAll(await this.liveDocs());
   }
 
   async getIds(): Promise<string[]> {
     const { items } = await this.collection();
-    const docs = await items.find(this.liveFilter(), { projection: { [KEY]: 1 } }).sort({ [ORD]: 1 }).toArray();
-    return docs.map((doc) => String(doc[KEY]));
+    const docs = await items.find(this.liveFilter(), { projection: { _id: 1, [KEY]: 1 } }).sort({ [ORD]: 1, _id: 1 }).toArray();
+    return this.idsOf(docs);
   }
 
   async find(predicate: (item: T) => boolean): Promise<T[]> {
@@ -605,9 +957,9 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     const rule = policy ?? this.options.queryPolicy;
     // The reference checks the selector and the options first, without a read of the database
     queryEntries([], selector, options, rule);
-    const filter = selectorPrefilter(selector);
+    const filter = selectorPrefilter(selector, { storedId: (id) => this.storedId(id), storageFields: this.reserved });
     const docs = await this.liveDocs(filter ? { $or: [{ [JSON_FIELD]: { $exists: true } }, filter] } : undefined);
-    return queryEntries(docs.map((doc) => this.decode(doc)), selector, options, rule) as Array<[string, T]>;
+    return queryEntries(this.decodeAll(docs), selector, options, rule) as Array<[string, T]>;
   }
 
   async has(id: string): Promise<boolean> {
@@ -617,7 +969,12 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
 
   async size(): Promise<number> {
     const { items } = await this.collection();
-    return items.countDocuments(this.liveFilter());
+    // A document with `_key` tells its id. A document from outside the storage counts only when `idOf` names it, as in getAll
+    const [all, outside] = await Promise.all([
+      items.countDocuments(this.liveFilter()),
+      items.find(this.liveFilter({ [KEY]: { $exists: false } }), { projection: { _id: 1 } }).toArray(),
+    ]);
+    return all - (outside.length - this.idsOf(outside).length);
   }
 
   async getBatch(ids: string[]): Promise<Map<string, T>> {
@@ -628,7 +985,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     }
     const { items } = await this.collection();
     const docs = await items.find(this.liveFilter({ _id: { $in: ids.map((id) => this.storedId(id)) } })).toArray();
-    const byId = new Map(docs.map((doc) => this.decode(doc)));
+    const byId = new Map(this.decodeAll(docs));
     // The order of the ids of the call
     for (const id of ids) if (byId.has(id)) result.set(id, byId.get(id)!);
     return result;
@@ -654,11 +1011,15 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     return cutResults(results, maxResults, bounded.limited);
   }
 
-  /** The version of an item: the sequence number of its last write, or undefined for a missing id. */
+  /**
+   * The version of an item: the sequence number of its last write, or undefined for a missing id.
+   * An item without a write through the storage (no `_seq`) has the version `0`.
+   */
   async versionOf(id: string): Promise<string | undefined> {
     const { items } = await this.collection();
     const doc = await items.findOne(this.liveFilter({ _id: this.storedId(id) }), { projection: { [SEQ]: 1 } });
-    return doc ? String(doc[SEQ]) : undefined;
+    if (!doc) return undefined;
+    return typeof doc[SEQ] === "number" ? String(doc[SEQ]) : "0";
   }
 
   async getChangesSince(since?: string): Promise<StorageChanges<T>> {
@@ -669,7 +1030,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
       const full = async (): Promise<StorageChanges<T>> => ({
         version,
         full: true,
-        changed: (await items.find(this.liveFilter()).sort({ [ORD]: 1 }).toArray()).map((doc) => this.decode(doc)),
+        changed: this.decodeAll(await items.find(this.liveFilter()).sort({ [ORD]: 1, _id: 1 }).toArray()),
         deleted: [],
       });
       const match = since === undefined ? null : /^([^.]+)\.(\d+)$/.exec(since);
@@ -686,8 +1047,10 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
       const changed: Array<[string, T]> = [];
       const deleted: string[] = [];
       for (const doc of docs) {
-        if (doc[TOMB] === true) deleted.push(String(doc[KEY]));
-        else changed.push(this.decode(doc));
+        const id = this.idOfDoc(doc);
+        if (id === undefined) continue;
+        if (doc[TOMB] === true) deleted.push(id);
+        else changed.push(this.decode(doc, id)!);
       }
       return { version, full: false, changed, deleted };
     });
@@ -719,6 +1082,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     // The ids and the encoding are checked before a number of the sequence goes
     for (const [id, value] of items) {
       this.storedId(id);
+      this.flatOf(id, value);
       JSON.stringify(value);
     }
     await this.write(async (db, collection) => {
@@ -749,9 +1113,11 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     return this.write(async (db, collection) => {
       const doc = await collection.findOne(this.liveFilter({ _id: this.storedId(id) }));
       if (!doc) return undefined;
-      const next = applyPatch(this.decode(doc)[1], fields) as T;
+      const next = applyPatch(this.decode(doc, id)![1], fields) as T;
+      // The encoding is checked before a number of the sequence goes
+      this.flatOf(id, next);
       await this.withNumbers(db, collection, 1, (first, session) =>
-        collection.replaceOne(this.writeFilter(id), this.encode(id, next, first, Number(doc[ORD])), { upsert: true, ...(session ? { session } : {}) })
+        collection.replaceOne(this.writeFilter(id), this.encode(id, next, first, this.placeOf(doc)), { upsert: true, ...(session ? { session } : {}) })
       );
       return next;
     });
@@ -787,7 +1153,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
         .find(this.liveFilter({ _id: { $in: unique.map((id) => this.storedId(id)) } }), { projection: { _id: 1, [KEY]: 1 } })
         .toArray();
       if (live.length === 0) return 0;
-      const liveIds = new Set(live.map((doc) => String(doc[KEY])));
+      const liveIds = new Set(this.idsOf(live));
       const deleted = unique.filter((id) => liveIds.has(id));
       await this.tombstone(db, collection, deleted);
       return deleted.length;
@@ -796,13 +1162,10 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
 
   async clear(_options?: StorageWriteOptions): Promise<void> {
     await this.write(async (db, collection) => {
-      const live = await collection.find(this.liveFilter(), { projection: { [KEY]: 1 } }).sort({ [ORD]: 1 }).toArray();
-      if (live.length === 0) return;
-      await this.tombstone(
-        db,
-        collection,
-        live.map((doc) => String(doc[KEY]))
-      );
+      const live = await collection.find(this.liveFilter(), { projection: { _id: 1, [KEY]: 1 } }).sort({ [ORD]: 1, _id: 1 }).toArray();
+      const ids = this.idsOf(live);
+      if (ids.length === 0) return;
+      await this.tombstone(db, collection, ids);
     });
   }
 
@@ -815,7 +1178,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
       const guards = await this.checkVersions(collection, expected, session);
       const operations: AnyBulkWriteOperation<Stored>[] = ids.map((id, index) => {
         const tomb: Stored = { _id: this.storedId(id), [KEY]: id, [SEQ]: first + index, [TOMB]: true };
-        if (this.partition !== undefined) tomb[PART] = this.partition;
+        this.putPartition(tomb);
         return { replaceOne: { filter: { ...this.writeFilter(id), ...guards.get(id) }, replacement: tomb } };
       });
       const result = await collection.bulkWrite(operations, { ordered: true, ...(session ? { session } : {}) });
@@ -841,12 +1204,18 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     const docs = await collection
       .find({ ...this.partitionFilter(), _id: { $in: [...new Set(ids)].map((id) => this.storedId(id)) } } as Filter<Stored>, session ? { session } : {})
       .toArray();
-    return new Map(docs.map((doc) => [String(doc[KEY]), doc]));
+    const byId = new Map<string, Document>();
+    for (const doc of docs) {
+      const id = this.idOfDoc(doc);
+      if (id !== undefined) byId.set(id, doc);
+    }
+    return byId;
   }
 
   /** The current item of a stored document: no item for a tombstone or a missing document. */
   private currentOf(doc: Document | undefined): CurrentItem<T> {
-    return doc && doc[TOMB] !== true ? { found: true, value: this.decode(doc)[1] } : { found: false };
+    const entry = doc && doc[TOMB] !== true ? this.decode(doc) : undefined;
+    return entry ? { found: true, value: entry[1] } : { found: false };
   }
 
   /**
@@ -868,7 +1237,8 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
     if (session) return guards;
     for (const [id] of expected) {
       const doc = docs.get(id);
-      guards.set(id, doc ? { [SEQ]: doc[SEQ] } : { [SEQ]: { $exists: false } });
+      // A document without `_seq` (not written through the storage yet) keeps that state until the write
+      guards.set(id, doc && typeof doc[SEQ] === "number" ? { [SEQ]: doc[SEQ] } : { [SEQ]: { $exists: false } });
     }
     return guards;
   }