mindoodb 0.0.65 → 0.0.67

package/dist/core/BaseMindooDB.js

@@ -5,6 +5,7 @@ exports.BaseMindooDB = exports.CUSTOM_DOC_ID_INITIAL_CHANGE_BYTES = void 0;
 // React Native: native Rust (react-native-automerge-generated)
 // Browser/Node.js: WASM (@automerge/automerge/slim)
 const automerge_adapter_1 = require("./automerge-adapter");
+const values_1 = require("./values");
 const storeEntryTime_1 = require("./storeEntryTime");
 const uuid_1 = require("uuid");
 const types_1 = require("./types");
@@ -150,6 +151,23 @@ const DEFAULT_WARMER_SCHEDULER = {
  * with a v2 header.
  */
 const DOC_CACHE_HEADER_VERSION = 2;
+/**
+ * How many quarantine records one document keeps in the audit log.
+ *
+ * One refused entry usually taints its causal dependents too (the
+ * `cascade_dependent` reason), so a single bad entry near the start of a long
+ * history can quarantine everything after it. The log exists to say which
+ * document is affected and why; a bounded sample of entry ids answers that as
+ * well as an unbounded one, and keeps one broken document from crowding every
+ * other out of the log — and out of the checkpoint it is written to.
+ */
+const MAX_QUARANTINE_RECORDS_PER_DOC = 200;
+/**
+ * How many documents the audit log covers at once. Oldest-first eviction: a
+ * document that started failing long ago has had its chance to be noticed,
+ * whereas dropping the newest would hide the problem the user is looking at.
+ */
+const MAX_QUARANTINE_DOCS = 500;
 /**
  * Hard-coded initial Automerge change bytes that seed every document created
  * with a caller-provided id (`createDocument({ id })`).
@@ -216,10 +234,19 @@ class BaseMindooDB {
     constructor(tenant, store, attachmentStore, attachmentConfig, documentCacheConfig, snapshotConfig, adminOnlyDb = false, logger, performanceCallback, timeTravelDate = null) {
         /**
          * Per-tenant quarantine/audit log of entries rejected during materialization
-         * (docs/accesscontrol.md §10). In-memory; surfaced via {@link getQuarantineLog}
-         * for Haven's audit view.
+         * (docs/accesscontrol.md §10), grouped by document and persisted in the
+         * metadata checkpoint. Surfaced via {@link getQuarantineLog} for Haven's
+         * audit view.
+         *
+         * Grouped by document because that is the unit a verdict is recomputed in: a
+         * materialization pass re-evaluates every entry of one document and knows the
+         * complete set of quarantined ones when it ends. Replacing that document's
+         * group with the pass result is what makes recording idempotent — the log
+         * would otherwise gain a duplicate of every record on every re-materialization
+         * — and what lets it heal once trust is restored, since an entry that is
+         * accepted this time is simply absent from the new group.
          */
-        this.quarantineLog = [];
+        this.quarantineLog = new Map();
         /**
          * Short-TTL cache of "is access control active for this tenant?" so the
          * materialization fast-path does not re-query the directory on every load.
@@ -446,6 +473,16 @@ class BaseMindooDB {
         /** Whether the setup document was already probed for FTS auto-activation. */
         this.fulltextSetupProbed = false;
         this.dirtyDocIds = new Set();
+        /**
+         * Per-document counter bumped by {@link updateIndex}, i.e. whenever new
+         * entries of a document are indexed. A materialization compares it before
+         * caching its result: if entries arrived while it was reading (a sync
+         * ingested the next page while a changefeed consumer was materializing),
+         * the result reflects an older entry set and must not land in L1 — the
+         * index already points past it, so a cached copy would be served (and
+         * flushed to L2 under the new changeSeq) as if it were current.
+         */
+        this.docEntryGeneration = new Map();
         this.cacheMetaDirty = false;
         /**
          * Per-document fingerprint (`changeSeq:automergeHeads`) of the state last
@@ -586,7 +623,7 @@ class BaseMindooDB {
     usernameHashFromRecord(payload) {
         if (!payload || typeof payload !== "object")
             return null;
-        const value = payload.username_hash;
+        const value = this.readAutomergeScalar(payload.username_hash);
         return typeof value === "string" && value.length > 0 ? value : null;
     }
     usernameHashFromInternalDoc(internalDoc) {
@@ -837,7 +874,7 @@ class BaseMindooDB {
         // flush-before-remove step.
         this.dirtyDocIds.clear();
         this.cacheMetaDirty = false;
-        this.summaryStore?.clearDirty();
+        this.summaryStore?.discardDirtyState();
         this.fulltextIndex?.clearDirty();
         void cacheManager.deregister(this);
         if (this.summaryStore) {
@@ -1594,6 +1631,14 @@ class BaseMindooDB {
         // the checkpoint as tally-aware, so restore can tell "nothing hidden"
         // apart from "written before this existed, count unknown".
         checkpoint.inaccessibleDocIds = Array.from(this.inaccessibleDocIds);
+        // Persist the materialization quarantine log (additive field, no version
+        // bump: a missing log restores as empty, which is exactly what the log was
+        // before it was persisted at all — and bumping the version would force a
+        // full metadata rebuild on every existing installation to gain nothing but
+        // an audit trail).
+        if (this.quarantineLog.size > 0) {
+            checkpoint.quarantineLog = Object.fromEntries(this.quarantineLog);
+        }
         return new TextEncoder().encode(JSON.stringify(checkpoint));
     }
     /**
@@ -1659,6 +1704,31 @@ class BaseMindooDB {
             else {
                 this.legacyCheckpointNeedsVisibilityTally = true;
             }
+            // Materialization quarantine log. Absent in checkpoints written before it
+            // was persisted, and absent whenever nothing is quarantined — both mean
+            // an empty log, and no heal flag is needed: the next materialization of
+            // an affected document re-derives its verdict anyway.
+            this.quarantineLog.clear();
+            if (checkpoint.quarantineLog &&
+                typeof checkpoint.quarantineLog === "object") {
+                for (const [docId, records] of Object.entries(checkpoint.quarantineLog)) {
+                    if (!Array.isArray(records))
+                        continue;
+                    const valid = records
+                        .filter((rec) => Boolean(rec &&
+                        typeof rec === "object" &&
+                        typeof rec.entryId === "string" &&
+                        typeof rec.reason === "string"))
+                        // Re-apply the caps on read: this file is on disk and may have been
+                        // written by a build with different limits.
+                        .slice(0, MAX_QUARANTINE_RECORDS_PER_DOC);
+                    if (valid.length === 0)
+                        continue;
+                    if (this.quarantineLog.size >= MAX_QUARANTINE_DOCS)
+                        break;
+                    this.quarantineLog.set(docId, valid);
+                }
+            }
             this.syncScanCursors.clear();
             if (checkpoint.syncScanCursors &&
                 typeof checkpoint.syncScanCursors === "object") {
@@ -1799,12 +1869,18 @@ class BaseMindooDB {
      * fall back to the normal materialization path when they encounter
      * one. Throws on truly malformed records (caught by callers).
      */
-    deserializeDoc(value) {
+    /** Split an L2 record into its JSON header and the Automerge binary. */
+    parseDocCacheRecord(value) {
         const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
         const headerLen = view.getUint32(0, false);
         const headerBytes = value.slice(4, 4 + headerLen);
-        const amBinary = value.slice(4 + headerLen);
-        const header = JSON.parse(new TextDecoder().decode(headerBytes));
+        return {
+            header: JSON.parse(new TextDecoder().decode(headerBytes)),
+            amBinary: value.slice(4 + headerLen),
+        };
+    }
+    deserializeDoc(value) {
+        const { header, amBinary } = this.parseDocCacheRecord(value);
         if (header.version !== DOC_CACHE_HEADER_VERSION) {
             // v1 records (no `version`) and unknown future versions: skip
             // gracefully so they get re-flushed with the current header on
@@ -2027,6 +2103,7 @@ class BaseMindooDB {
         // Every path that writes entries for a document lands here, so this is
         // where retained metadata for it goes stale (see scannedEntryMetadata).
         this.forgetScannedEntryMetadata(docId);
+        this.docEntryGeneration.set(docId, (this.docEntryGeneration.get(docId) ?? 0) + 1);
         // Same reasoning for the unreadable-document tally: every reveal and every
         // inaccessibility tombstone passes through here, so keeping it in step
         // needs no bookkeeping at the individual call sites. The one case this
@@ -3464,7 +3541,7 @@ class BaseMindooDB {
             },
         };
     }
-    async syncEntriesFromStore(sourceStore, targetStore, options) {
+    async syncEntriesFromStore(sourceStore, targetStore, options, onPageTransferred) {
         const signal = options?.signal;
         (0, syncStores_1.setSyncAbortSignalOnStore)(sourceStore, signal);
         (0, syncStores_1.setSyncAbortSignalOnStore)(targetStore, signal);
@@ -3482,6 +3559,7 @@ class BaseMindooDB {
                 logger: this.logger,
                 cursors: this.syncCursorStore(),
                 rejectionPolicy: "advance",
+                onPageTransferred,
             });
         }
         catch (error) {
@@ -3917,8 +3995,12 @@ class BaseMindooDB {
         // starting with "_", e.g. "_attachments") are managed by MindooDB and must
         // not be seeded by callers. `undefined` values are treated as "not set"
         // (Automerge cannot represent undefined).
+        // `MindooValue` typed values become CRDT values here (this also validates
+        // them before anything is written).
         const initialValueEntries = options.initialValues
-            ? Object.entries(options.initialValues).filter(([k, v]) => !k.startsWith("_") && v !== undefined)
+            ? Object.entries(options.initialValues)
+                .filter(([k, v]) => !k.startsWith("_") && v !== undefined)
+                .map(([k, v]) => [k, this.hydrateTypedValues(v)])
             : [];
         const hasInitialValues = initialValueEntries.length > 0;
         if (useCustomDocId) {
@@ -4181,8 +4263,12 @@ class BaseMindooDB {
         // Sanitize caller-provided initial values: internal/reserved fields are
         // managed by MindooDB and must not be seeded by callers. `undefined`
         // values are treated as "not set" (Automerge cannot represent undefined).
+        // `MindooValue` typed values become CRDT values here (this also validates
+        // them before anything is written).
         const initialValueEntries = options.initialValues
-            ? Object.entries(options.initialValues).filter(([k, v]) => !k.startsWith("_") && v !== undefined)
+            ? Object.entries(options.initialValues)
+                .filter(([k, v]) => !k.startsWith("_") && v !== undefined)
+                .map(([k, v]) => [k, this.hydrateTypedValues(v)])
             : [];
         const signerPublicKey = await this.resolveSignerPublicKey(useCustomSigningKey ? signingKeyPair : undefined);
         const createUsernameHash = this.usernameHashFromRecord(Object.fromEntries(initialValueEntries));
@@ -6836,13 +6922,11 @@ class BaseMindooDB {
             ...entryMetadata,
             encryptedData: encryptedPayload,
         };
-        // Client-side write-policy precheck (docs/accesscontrol.md §9). `internalDoc.doc`
-        // is still the pre-change state here (reassigned in
-        // applyLifecycleEntryLocally, after the store write). For deletes the rules
-        // default to the "before" state; both states are supplied so either `when`
-        // resolves. Throws AccessDeniedError if denied.
+        // Client-side write-policy precheck (docs/accesscontrol.md §9). For deletes
+        // the rules default to the "before" state; both states are supplied so either
+        // `when` resolves. Throws AccessDeniedError if denied.
         {
-            const beforeState = internalDoc.doc;
+            const beforeState = this.docAtHeads(internalDoc, newDoc, automergeDepHashes);
             await this.assertWriteAllowed({
                 op: entryType,
                 signerKey: createdByPublicKey,
@@ -7174,6 +7258,87 @@ class BaseMindooDB {
             spans: this.dehydrateRichTextSpans(spans),
         };
     }
+    async getTextCursors(doc, path, positions, options) {
+        this.validateJsonPath(path, "Text cursor");
+        if (!Array.isArray(positions)) {
+            throw new Error("Text cursor positions must be an array");
+        }
+        const { automergeDoc, heads } = await this.resolveTextCursorTarget(doc, options);
+        if (options?.move !== undefined &&
+            options.move !== "before" &&
+            options.move !== "after") {
+            throw new Error('Text cursor move must be "before" or "after"');
+        }
+        const source = this.textCursorSource(automergeDoc, heads);
+        const text = this.readTextForCursor(source, path);
+        const cursors = positions.map((position) => {
+            let target;
+            if (position === "start" || position === "end") {
+                target = position;
+            }
+            else if (typeof position === "number" && Number.isFinite(position)) {
+                target = this.clampIndex(position, text.length);
+            }
+            else {
+                throw new Error("Text cursor positions must be numbers, \"start\" or \"end\"");
+            }
+            return automerge_adapter_1.Automerge.getCursor(source, path, target, options?.move);
+        });
+        return {
+            path: [...path],
+            heads: heads ?? automerge_adapter_1.Automerge.getHeads(automergeDoc),
+            cursors,
+        };
+    }
+    async resolveTextCursors(doc, path, cursors, options) {
+        this.validateJsonPath(path, "Text cursor");
+        if (!Array.isArray(cursors) ||
+            cursors.some((cursor) => typeof cursor !== "string" || cursor === "")) {
+            throw new Error("Text cursors must be non-empty strings");
+        }
+        const { automergeDoc, heads } = await this.resolveTextCursorTarget(doc, options);
+        const source = this.textCursorSource(automergeDoc, heads);
+        this.readTextForCursor(source, path);
+        const positions = cursors.map((cursor) => automerge_adapter_1.Automerge.getCursorPosition(source, path, cursor));
+        return {
+            path: [...path],
+            heads: heads ?? automerge_adapter_1.Automerge.getHeads(automergeDoc),
+            positions,
+        };
+    }
+    async resolveTextCursorTarget(doc, options) {
+        const docId = doc.getId();
+        let internalDoc = this.wrappedInternalDocs.get(doc) ?? null;
+        if (!internalDoc) {
+            internalDoc = this.getCachedDocument(docId);
+        }
+        if (!internalDoc) {
+            const loadedDoc = await this.loadDocumentInternal(docId);
+            if (!loadedDoc) {
+                throw new errors_1.DocumentNotFoundError(docId);
+            }
+            internalDoc = loadedDoc;
+        }
+        if (internalDoc.isDeleted) {
+            throw new errors_1.DocumentDeletedError(docId);
+        }
+        const heads = options?.heads && options.heads.length > 0
+            ? options.heads
+            : undefined;
+        return { automergeDoc: internalDoc.doc, heads };
+    }
+    /** The document itself, or a read-only view of it at `heads`. */
+    textCursorSource(automergeDoc, heads) {
+        return heads ? automerge_adapter_1.Automerge.view(automergeDoc, heads) : automergeDoc;
+    }
+    /** Reads the text a cursor operates on, rejecting non-text fields early. */
+    readTextForCursor(source, path) {
+        const value = this.readValueAtPath(source, path);
+        if (typeof value !== "string") {
+            throw new Error(`Text cursors require an Automerge text field at '${path.map(String).join(".")}'`);
+        }
+        return value;
+    }
     async exportAutomergeSnapshot(doc) {
         const internalDoc = await this.resolveReadableInternalDoc(doc);
         const binary = automerge_adapter_1.Automerge.save(internalDoc.doc);
@@ -7385,6 +7550,9 @@ class BaseMindooDB {
         const pendingAttachmentAppends = new Map();
         // Map of attachmentId -> extraction result for setAttachmentExtractedText
         const pendingAttachmentTextUpdates = new Map();
+        // incrementCounter() calls, applied in call order after the field
+        // assignments so they can target counters assigned in the same callback.
+        const pendingCounterIncrements = [];
         // Reference to db for closures
         const db = this;
         // Guard flag to prevent operations after callback completes
@@ -7449,7 +7617,9 @@ class BaseMindooDB {
                         if (typeof prop === "string" && pendingChanges.has(prop)) {
                             return pendingChanges.get(prop);
                         }
-                        return target[prop];
+                        // Counters read as numbers and immutable strings as strings here
+                        // too; change counters with incrementCounter().
+                        return db.readAutomergeScalar(target[prop]);
                     },
                     has: (target, prop) => {
                         // If marked for deletion, it doesn't exist
@@ -7577,6 +7747,14 @@ class BaseMindooDB {
                     });
                 }
             },
+            // ========== Counter Write Methods ==========
+            incrementCounter: (path, delta) => {
+                throwIfCallbackInactive("incrementCounter");
+                const segments = typeof path === "string" ? [path] : [...path];
+                db.validateJsonPath(segments, "incrementCounter");
+                (0, values_1.assertCounterAmount)(delta, "incrementCounter delta");
+                pendingCounterIncrements.push({ path: segments, delta });
+            },
             // ========== Attachment Read Methods (also work in changeDoc context) ==========
             getAttachments: () => {
                 const payload = internalDoc.doc;
@@ -7612,12 +7790,15 @@ class BaseMindooDB {
             newDoc = this.runChangeWithOutdatedDocRecovery(internalDoc, (doc) => automerge_adapter_1.Automerge.change(doc, (automergeDoc) => {
                 // Apply all pending changes (sets/updates)
                 for (const [key, value] of pendingChanges) {
-                    automergeDoc[key] = value;
+                    automergeDoc[key] = this.hydrateTypedValues(value);
                 }
                 // Apply all pending deletions
                 for (const key of pendingDeletions) {
                     delete automergeDoc[key];
                 }
+                for (const { path, delta } of pendingCounterIncrements) {
+                    this.incrementCounterAtPath(automergeDoc, path, delta);
+                }
                 // Apply pending attachment changes
                 if (pendingAttachmentAdditions.length > 0 ||
                     pendingAttachmentRemovals.size > 0 ||
@@ -7741,12 +7922,14 @@ class BaseMindooDB {
             (patch.listInsert?.length ?? 0) +
             (patch.textSplice?.length ?? 0) +
             (patch.textMark?.length ?? 0) +
-            (patch.textUnmark?.length ?? 0);
+            (patch.textUnmark?.length ?? 0) +
+            (patch.counterIncrement?.length ?? 0);
         if (operationCount === 0) {
             throw new Error("JSON patch must include at least one operation");
         }
         for (const operation of patch.set ?? []) {
             this.validateJsonPath(operation.path, "JSON set");
+            this.hydrateTypedValues(operation.value);
         }
         for (const operation of patch.unset ?? []) {
             this.validateJsonPath(operation.path, "JSON unset");
@@ -7769,6 +7952,7 @@ class BaseMindooDB {
             if (!Array.isArray(operation.values)) {
                 throw new Error("JSON listInsert values must be an array");
             }
+            this.hydrateTypedValues(operation.values);
         }
         for (const operation of patch.textSplice ?? []) {
             this.validateJsonPath(operation.path, "JSON textSplice");
@@ -7811,6 +7995,134 @@ class BaseMindooDB {
                 throw new Error("JSON textUnmark names must be non-empty strings");
             }
         }
+        for (const operation of patch.counterIncrement ?? []) {
+            this.validateJsonPath(operation.path, "JSON counterIncrement");
+            (0, values_1.assertCounterAmount)(operation.delta, "JSON counterIncrement delta");
+        }
+    }
+    isAutomergeCounter(value) {
+        if (value === null || typeof value !== "object")
+            return false;
+        const automerge = automerge_adapter_1.Automerge;
+        if (typeof automerge.isCounter === "function") {
+            return automerge.isCounter(value);
+        }
+        return (typeof automerge.Counter === "function" &&
+            value instanceof automerge.Counter);
+    }
+    isAutomergeImmutableString(value) {
+        if (value === null || typeof value !== "object")
+            return false;
+        const check = automerge_adapter_1.Automerge
+            .isImmutableString;
+        return typeof check === "function" && check(value);
+    }
+    /**
+     * Plain-JS read view of an Automerge scalar wrapper: counters become numbers
+     * and immutable strings become strings. Everything else is returned as is.
+     */
+    readAutomergeScalar(value) {
+        if (this.isAutomergeCounter(value))
+            return value.value;
+        if (this.isAutomergeImmutableString(value))
+            return String(value);
+        return value;
+    }
+    /**
+     * Turns `MindooValue` typed values (tagged plain objects, see values.ts)
+     * anywhere inside `value` into their CRDT representation, and copies plain
+     * objects and arrays into fresh literals on the way. Values that already are
+     * CRDT scalars, dates or bytes pass through. Throws for malformed tags, so
+     * it doubles as validation.
+     */
+    hydrateTypedValues(value, depth = 0) {
+        if (value === null || typeof value !== "object")
+            return value;
+        if (depth > 100) {
+            throw new Error("Document value is nested too deeply");
+        }
+        if (value instanceof Uint8Array ||
+            value instanceof Date ||
+            this.isAutomergeCounter(value) ||
+            this.isAutomergeImmutableString(value)) {
+            return value;
+        }
+        if (Array.isArray(value)) {
+            return value.map((entry) => this.hydrateTypedValues(entry, depth + 1));
+        }
+        const record = value;
+        if (Object.prototype.hasOwnProperty.call(record, values_1.MINDOO_VALUE_TAG)) {
+            const typed = (0, values_1.parseTypedValue)(record);
+            switch (typed.$mindoo) {
+                case "atomic":
+                    return new automerge_adapter_1.Automerge.ImmutableString(typed.value);
+                case "counter":
+                    return new automerge_adapter_1.Automerge.Counter(typed.value);
+                case "timestamp":
+                    return (0, values_1.toTimestampDate)(typed.value, 'Typed value "timestamp"');
+            }
+        }
+        const result = {};
+        for (const [key, entry] of Object.entries(record)) {
+            if (entry !== undefined) {
+                result[key] = this.hydrateTypedValues(entry, depth + 1);
+            }
+        }
+        return result;
+    }
+    /**
+     * The plain read view of a value that may contain `MindooValue` tags, as
+     * `getData()` would return it once written (used where a precheck needs the
+     * "after" state before the value is in a document).
+     */
+    readTypedValues(value) {
+        return this.normalizeAutomergeScalars(this.hydrateTypedValues(value));
+    }
+    /**
+     * Replaces counters and atomic strings inside a materialized value with
+     * plain numbers and strings, in place where possible. Reads (getData, the
+     * query and summary indexes, write prechecks) then compare them like any
+     * other JSON value.
+     */
+    normalizeAutomergeScalars(value, depth = 0) {
+        if (value === null || typeof value !== "object")
+            return value;
+        if (this.isAutomergeCounter(value) || this.isAutomergeImmutableString(value)) {
+            return this.readAutomergeScalar(value);
+        }
+        if (value instanceof Uint8Array || value instanceof Date || depth > 100) {
+            return value;
+        }
+        if (Array.isArray(value)) {
+            for (let index = 0; index < value.length; index += 1) {
+                value[index] = this.normalizeAutomergeScalars(value[index], depth + 1);
+            }
+            return value;
+        }
+        const record = value;
+        for (const key of Object.keys(record)) {
+            record[key] = this.normalizeAutomergeScalars(record[key], depth + 1);
+        }
+        return record;
+    }
+    /**
+     * Must run inside an Automerge change callback. Initializes a missing field
+     * as a counter holding `delta`; rejects existing non-counter values rather
+     * than silently converting them (a plain number would lose its history and
+     * any concurrent plain-number write would still win).
+     */
+    incrementCounterAtPath(automergeDoc, path, delta) {
+        const parent = this.ensureJsonParentAtPath(automergeDoc, path);
+        const leaf = path[path.length - 1];
+        const existing = parent[leaf];
+        if (existing === undefined || existing === null) {
+            parent[leaf] = new automerge_adapter_1.Automerge.Counter(delta);
+            return;
+        }
+        if (!this.isAutomergeCounter(existing)) {
+            throw new Error(`Cannot increment non-counter value at '${path.map(String).join(".")}'; create it with MindooValue.counter() first`);
+        }
+        existing.increment(delta);
     }
     validateRichTextPatch(patch) {
         this.validateJsonPath(patch.path, "Rich-text patch");
@@ -7975,7 +8287,7 @@ class BaseMindooDB {
     }
     applyJsonPatchOperations(automergeDoc, patch) {
         for (const operation of patch.set ?? []) {
-            this.setJsonValueAtPath(automergeDoc, operation.path, structuredClone(operation.value));
+            this.setJsonValueAtPath(automergeDoc, operation.path, this.hydrateTypedValues(structuredClone(operation.value)));
         }
         for (const operation of patch.unset ?? []) {
             this.unsetJsonValueAtPath(automergeDoc, operation.path);
@@ -7986,7 +8298,7 @@ class BaseMindooDB {
         }
         for (const operation of patch.listInsert ?? []) {
             const list = this.ensureJsonListAtPath(automergeDoc, operation.path);
-            list.splice(operation.index, 0, ...structuredClone(operation.values));
+            list.splice(operation.index, 0, ...this.hydrateTypedValues(structuredClone(operation.values)));
         }
         for (const operation of patch.textSplice ?? []) {
             this.spliceJsonTextAtPath(automergeDoc, operation.path, operation.index, operation.deleteCount, operation.insert ?? "");
@@ -8005,6 +8317,9 @@ class BaseMindooDB {
                 this.unmarkJsonTextAtPath(automergeDoc, operation.path, operation.index, operation.length, name);
             }
         }
+        for (const operation of patch.counterIncrement ?? []) {
+            this.incrementCounterAtPath(automergeDoc, operation.path, operation.delta);
+        }
     }
     spliceJsonTextAtPath(automergeDoc, path, index, deleteCount, insert) {
         const value = this.readValueAtPath(automergeDoc, path);
@@ -8183,6 +8498,38 @@ class BaseMindooDB {
             return apply(internalDoc.doc);
         }
     }
+    /**
+     * The document as it stood at `heads`, for the "before" side of a Tier 2
+     * (`withfields`) write precheck.
+     *
+     * `internalDoc.doc` is normally the pre-change document (it is reassigned to
+     * the new document after the store write), but it is shared mutable state and
+     * is not guaranteed to still sit on the heads a change was computed from: an
+     * out-of-band Automerge operation, the clone recovery in
+     * {@link runChangeWithOutdatedDocRecovery}, or a concurrent op on the same
+     * cached document can move it. A "before" that is really the "after" silently
+     * flips every `when: "before"` verdict, so the caller passes the change's own
+     * dependency hashes, which are fixed at the moment the change was produced,
+     * and only falls back to the cache slot when it has none.
+     *
+     * `Automerge.clone` is load-bearing: a bare `Automerge.view` shares the source
+     * document's WASM handle, and {@link convertAutomergeToJS} materializes through
+     * that handle at ITS heads, not the view's — so materializing a view returns
+     * the post-change state. Cloning gives the snapshot its own handle.
+     */
+    docAtHeads(internalDoc, doc, heads) {
+        if (!heads || heads.length === 0)
+            return internalDoc.doc;
+        try {
+            return automerge_adapter_1.Automerge.clone(automerge_adapter_1.Automerge.view(doc, heads));
+        }
+        catch (error) {
+            // A snapshot we cannot build must not fail the write outright; the cache
+            // slot is the same value this code used before the snapshot existed.
+            this.logger.warn(`Could not snapshot document ${internalDoc.id} at its pre-change heads, using the cached state: ${error}`);
+            return internalDoc.doc;
+        }
+    }
     isOutdatedDocumentError(error) {
         if (!(error instanceof Error))
             return false;
@@ -8261,6 +8608,9 @@ class BaseMindooDB {
         // text patches, merges, ...) carries it consistently on each produced entry.
         const attachmentRefs = this.collectAttachmentRefs(newDoc);
         const entries = [];
+        // The dependencies of the first change ARE the heads the change was computed
+        // from, i.e. the document's "before" state for the write precheck below.
+        let headsBeforeFirstChange;
         for (let index = 0; index < resolvedChangeBytesList.length; index += 1) {
             const changeBytes = resolvedChangeBytesList[index];
             const entry = await this.buildDocChangeStoreEntry({
@@ -8278,23 +8628,23 @@ class BaseMindooDB {
             });
             entries.push(entry);
             const decodedChange = automerge_adapter_1.Automerge.decodeChange(changeBytes);
+            if (index === 0) {
+                headsBeforeFirstChange = decodedChange.deps;
+            }
             this.registerAutomergeHashMapping(docId, decodedChange.hash, entry.id);
         }
-        // Client-side write-policy precheck (docs/accesscontrol.md §9). `internalDoc.doc`
-        // is still the pre-change state here (it is reassigned to `newDoc` below,
-        // after the store write), so before/after are both available for Tier 2
-        // (`withfields`) evaluation. Throws AccessDeniedError when denied.
+        // Client-side write-policy precheck (docs/accesscontrol.md §9). Throws
+        // AccessDeniedError when denied.
         {
             const signerKey = useCustomKey
                 ? signingKeyPair.publicKey
                 : (await this.tenant.getCurrentUserId()).userSigningPublicKey;
-            const beforeState = internalDoc.doc;
             await this.assertWriteAllowed({
                 op: "doc_change",
                 signerKey,
                 isAuthor: await this.resolveIsAuthor(docId, signerKey, Number.MAX_SAFE_INTEGER),
                 bypass: bypassPrecheck,
-                getBeforeDoc: () => this.convertAutomergeToJS(beforeState),
+                getBeforeDoc: () => this.convertAutomergeToJS(this.docAtHeads(internalDoc, newDoc, headsBeforeFirstChange)),
                 getAfterDoc: () => this.convertAutomergeToJS(newDoc),
             });
         }
@@ -9474,8 +9824,12 @@ class BaseMindooDB {
             return !persistedHashes.has(parsed.automergeHash);
         });
         if (missingMetadata.length === 0) {
-            // No real deltas (changeSeq pointer drift only). Promote and return.
+            // No real deltas (changeSeq pointer drift only). Promote and return,
+            // marked dirty so the record is rewritten under the current changeSeq:
+            // otherwise every later read (and every warmer pass) would repeat
+            // this metadata scan + hash comparison for a record that is current.
             await this.storeCachedDocument(internal);
+            this.markDocDirty(docId);
             return internal;
         }
         let updated;
@@ -9490,8 +9844,9 @@ class BaseMindooDB {
             // applyNewEntriesToCachedDocument returns null when no signed entries
             // produced a head change. Treat the persisted doc as authoritative
             // and put it in L1 ourselves (the helper only caches when it
-            // actually applied work).
+            // actually applied work). Dirty for the same reason as above.
             await this.storeCachedDocument(internal);
+            this.markDocDirty(docId);
             return internal;
         }
         return updated;
@@ -9511,9 +9866,13 @@ class BaseMindooDB {
     /**
      * Start the L2 background warmer.
      *
-     * Walks a snapshot of the in-memory document index and, for every
-     * doc not already in L1, calls {@link tryLoadFromL2}. That path
-     * either:
+     * First waits until the changefeed consumers (summary buffer, an
+     * enabled full-text index) have caught up: they materialize every
+     * changed document anyway, so running next to them would do the same
+     * work twice. Then walks a snapshot of the in-memory document index and
+     * skips every doc that is in L1 or whose L2 record is already at the
+     * current changeSeq (checked from the record header, no deserialization).
+     * For the rest it calls {@link tryLoadFromL2}. That path either:
      *
      *  - hits L2 fresh and just promotes the doc to L1, or
      *  - hits L2 stale and applies missing deltas (re-flushed via the
@@ -9614,16 +9973,13 @@ class BaseMindooDB {
             this.logger.debug("Warmer skipped: no cacheManager attached, no L2 to warm.");
             return;
         }
-        // Snapshot the docId list so concurrent index updates don't shift
-        // our iteration mid-pass. Iterating the index directly would also
-        // work but a snapshot makes the loop's behavior easier to reason
-        // about under sync activity.
-        const docIds = this.index.map((entry) => entry.docId);
-        const total = docIds.length;
-        // Progress snapshot is replaced (not mutated in place) so polled
-        // readers see consistent values. We initialize even when total=0
-        // so the UI can display a "done" terminal state.
-        this.warmerProgress = { processed: 0, total, phase: "warming" };
+        // Changefeed consumers (summary buffer, full-text index) materialize
+        // every changed document anyway, and those loads persist to L2 through
+        // flush-before-evict / the periodic flush. Running the warmer next to
+        // them (it is typically started right after a sync, while their
+        // catch-up is still going) materialized the same documents twice. Let
+        // them finish first; the pass below then only has to pick up what they
+        // did not cover.
         const emitProgress = (progress) => {
             this.warmerProgress = progress;
             if (!onProgress)
@@ -9636,6 +9992,26 @@ class BaseMindooDB {
                 this.logger.warn(`Warmer onProgress callback threw: ${e}`);
             }
         };
+        this.warmerProgress = {
+            processed: 0,
+            total: this.index.length,
+            phase: "warming",
+        };
+        await this.awaitChangefeedConsumersForWarmer(signal);
+        if (signal.aborted) {
+            emitProgress({ processed: 0, total: this.index.length, phase: "cancelled" });
+            return;
+        }
+        // Snapshot the docId list so concurrent index updates don't shift
+        // our iteration mid-pass. Iterating the index directly would also
+        // work but a snapshot makes the loop's behavior easier to reason
+        // about under sync activity.
+        const docIds = this.index.map((entry) => entry.docId);
+        const total = docIds.length;
+        // Progress snapshot is replaced (not mutated in place) so polled
+        // readers see consistent values. We initialize even when total=0
+        // so the UI can display a "done" terminal state.
+        this.warmerProgress = { processed: 0, total, phase: "warming" };
         if (total === 0) {
             this.logger.debug("Warmer: nothing to warm, index is empty.");
             emitProgress({ processed: 0, total: 0, phase: "done" });
@@ -9645,6 +10021,7 @@ class BaseMindooDB {
         let processed = 0;
         let warmed = 0;
         let skippedAlreadyHot = 0;
+        let skippedFreshInL2 = 0;
         let errored = 0;
         try {
             for (const docId of docIds) {
@@ -9657,6 +10034,11 @@ class BaseMindooDB {
                 if (this.docCache.has(docId)) {
                     skippedAlreadyHot++;
                 }
+                else if (await this.isL2RecordCurrent(docId)) {
+                    // Nothing to do: loading it would only deserialize the doc into
+                    // L1 and evict it again (L1 is far smaller than a large DB).
+                    skippedFreshInL2++;
+                }
                 else {
                     try {
                         const fromL2 = await this.tryLoadFromL2(docId);
@@ -9682,7 +10064,7 @@ class BaseMindooDB {
                 }
             }
             this.logger.info(`Warmer finished in ${Date.now() - startedAt}ms: processed=${processed}, ` +
-                `warmed=${warmed}, alreadyHot=${skippedAlreadyHot}, errored=${errored}`);
+                `warmed=${warmed}, alreadyHot=${skippedAlreadyHot}, freshInL2=${skippedFreshInL2}, errored=${errored}`);
             emitProgress({ processed, total, phase: "done" });
         }
         catch (e) {
@@ -9693,6 +10075,82 @@ class BaseMindooDB {
             emitProgress({ processed, total, phase: "cancelled" });
         }
     }
+    /**
+     * Wait until the summary buffer and an enabled full-text index have
+     * consumed the changefeed (joining a running auto-follow run or starting
+     * a catch-up). Failures are logged: the warmer then simply does the
+     * remaining work itself.
+     */
+    async awaitChangefeedConsumersForWarmer(signal) {
+        const startedAt = Date.now();
+        const catchUp = (async () => {
+            await this.summaryStore?.update({ signal });
+            if (!signal.aborted && this.fulltextIndex?.isEnabled()) {
+                await this.fulltextIndex.update({ signal });
+            }
+        })();
+        // Joining an already running auto-follow run does not honor our signal,
+        // so a stop must not wait for it: the run just continues on its own.
+        let onAbort = () => { };
+        const aborted = new Promise((resolve) => {
+            onAbort = () => resolve();
+            if (signal.aborted)
+                resolve();
+            else
+                signal.addEventListener("abort", onAbort, { once: true });
+        });
+        try {
+            await Promise.race([catchUp, aborted]);
+        }
+        catch (e) {
+            this.logger.warn(`Warmer: changefeed consumer catch-up failed: ${e}`);
+        }
+        finally {
+            signal.removeEventListener("abort", onAbort);
+        }
+        catchUp.catch(() => { });
+        this.logger.debug(`Warmer: changefeed consumers caught up in ${Date.now() - startedAt}ms`);
+    }
+    /**
+     * Whether L2 already holds this document at its current changeSeq, so
+     * the warmer can skip it without deserializing. Answered from
+     * {@link lastFlushedDocState} for records written in this session,
+     * otherwise from the record header (no Automerge load).
+     */
+    async isL2RecordCurrent(docId) {
+        const indexEntryIdx = this.getDocIndexPosition(docId);
+        const store = this.cacheManager?.getStore() ?? null;
+        if (indexEntryIdx === undefined || !store) {
+            return false;
+        }
+        const entry = this.index[indexEntryIdx];
+        if (entry.accessState !== "visible") {
+            return false;
+        }
+        const currentChangeSeq = entry.changeSeq;
+        const flushed = this.lastFlushedDocState.get(docId);
+        if (flushed !== undefined) {
+            return flushed.startsWith(`${currentChangeSeq}:`);
+        }
+        try {
+            const bytes = await store.get("doc", `${this.getCachePrefix()}/${docId}`);
+            if (!bytes) {
+                return false;
+            }
+            const { header } = this.parseDocCacheRecord(bytes);
+            if (header.version !== DOC_CACHE_HEADER_VERSION ||
+                header.changeSeq !== currentChangeSeq ||
+                !Array.isArray(header.automergeHeads)) {
+                return false;
+            }
+            this.lastFlushedDocState.set(docId, `${currentChangeSeq}:${header.automergeHeads.join(",")}`);
+            return true;
+        }
+        catch {
+            // Unreadable record: let the regular L2 path evict/rebuild it.
+            return false;
+        }
+    }
     /**
      * The per-tenant quarantine/audit log: entries rejected during
      * materialization by the access-control layer (docs/accesscontrol.md §10).
@@ -9700,7 +10158,11 @@ class BaseMindooDB {
      * store but are excluded from materialized state and queries.
      */
     getQuarantineLog() {
-        return this.quarantineLog;
+        const flat = [];
+        for (const records of this.quarantineLog.values()) {
+            flat.push(...records);
+        }
+        return flat;
     }
     /**
      * Whether access control is active for this tenant and should gate
@@ -9900,8 +10362,9 @@ class BaseMindooDB {
         const afterValues = { _attachments: [] };
         if (options.initialValues) {
             for (const [k, v] of Object.entries(options.initialValues)) {
-                if (!k.startsWith("_"))
-                    afterValues[k] = v;
+                if (!k.startsWith("_") && v !== undefined) {
+                    afterValues[k] = this.readTypedValues(v);
+                }
             }
         }
         const decision = await this.evaluateClientWriteAccess({
@@ -10224,10 +10687,52 @@ class BaseMindooDB {
         const info = await this.resolveCreatorInfo(docId);
         return info?.signingKey ?? null;
     }
-    /** Append a record to the quarantine/audit log (and log it). */
-    recordQuarantine(rec) {
-        this.quarantineLog.push(rec);
-        console.log(`[MindooDB][ACL] Quarantined entry ${rec.entryId} (doc ${rec.docId}): ${rec.reason} — ${rec.detail}`);
+    /**
+     * Replace one document's quarantine records with the verdict of the pass that
+     * just ended.
+     *
+     * Whole-group replacement rather than appending, because a pass re-derives the
+     * complete verdict for the document: appending would duplicate every record
+     * on every re-materialization, and would never drop the record of an entry
+     * that has since become acceptable.
+     *
+     * Writes nothing when the verdict is unchanged. That is not just thrift — the
+     * checkpoint is rewritten whenever this marks it dirty, and a document is
+     * re-materialized often, so marking dirty on an unchanged verdict would turn
+     * every read of a quarantined document into a checkpoint write.
+     */
+    commitQuarantineForDoc(docId, records) {
+        const previous = this.quarantineLog.get(docId);
+        if (records.length === 0) {
+            if (previous) {
+                this.quarantineLog.delete(docId);
+                this.cacheMetaDirty = true;
+            }
+            return;
+        }
+        const kept = records.length > MAX_QUARANTINE_RECORDS_PER_DOC
+            ? records.slice(0, MAX_QUARANTINE_RECORDS_PER_DOC)
+            : records;
+        // Compare the verdict, not the record objects: `recordedAt` is a wall-clock
+        // stamp that differs on every pass and would make every verdict look new.
+        const verdictOf = (list) => list.map((rec) => `${rec.entryId}\u0000${rec.reason}`).join("\u0001");
+        if (previous && verdictOf(previous) === verdictOf(kept)) {
+            return;
+        }
+        this.quarantineLog.set(docId, kept);
+        if (!previous) {
+            while (this.quarantineLog.size > MAX_QUARANTINE_DOCS) {
+                // Map iteration is insertion-ordered, so this is the oldest document.
+                const oldest = this.quarantineLog.keys().next();
+                if (oldest.done || oldest.value === docId)
+                    break;
+                this.quarantineLog.delete(oldest.value);
+            }
+        }
+        this.cacheMetaDirty = true;
+        for (const rec of kept) {
+            console.log(`[MindooDB][ACL] Quarantined entry ${rec.entryId} (doc ${rec.docId}): ${rec.reason} — ${rec.detail}`);
+        }
     }
     /**
      * Materialize a document through the access-control (Tier 2) path
@@ -10284,8 +10789,12 @@ class BaseMindooDB {
             .sort((a, b) => (a.receivedAt ?? a.createdAt) === (b.receivedAt ?? b.createdAt)
             ? a.id.localeCompare(b.id)
             : (a.receivedAt ?? a.createdAt) - (b.receivedAt ?? b.createdAt));
-        if (dataEntries.length === 0)
+        if (dataEntries.length === 0) {
+            // Nothing left to judge (e.g. the history was purged), so no verdict can
+            // stand either.
+            this.commitQuarantineForDoc(docId, []);
             return { doc: null, cacheable };
+        }
         const createMeta = dataEntries.find((m) => m.entryType === "doc_create");
         const creatorKey = createMeta?.createdByPublicKey ?? null;
         const creatorTrustedTime = createMeta
@@ -10299,9 +10808,13 @@ class BaseMindooDB {
         let decryptionKeyId = "default";
         const acceptedMeta = [];
         const quarantined = new Set();
+        // Collected for the whole pass and committed once at the end, so the log
+        // holds this document's current verdict rather than the union of every
+        // verdict ever reached for it. See commitQuarantineForDoc.
+        const quarantineRecords = [];
         const quarantine = (meta, reason, detail) => {
             quarantined.add(meta.id);
-            this.recordQuarantine({
+            quarantineRecords.push({
                 entryId: meta.id,
                 docId,
                 dbid,
@@ -10458,6 +10971,9 @@ class BaseMindooDB {
                 this.registerAutomergeHashMapping(docId, parsed.automergeHash, entry.id);
             }
         }
+        // The pass is over and `quarantineRecords` is this document's full verdict,
+        // including the empty verdict that retires an earlier one.
+        this.commitQuarantineForDoc(docId, quarantineRecords);
         if (currentDoc === null) {
             // The create was quarantined or absent: the document is logically absent.
             return { doc: null, cacheable };
@@ -10485,6 +11001,22 @@ class BaseMindooDB {
     /**
      * Internal method to load a document from the content-addressed store
      */
+    /** Whether no new entries of `docId` were indexed since `generation` was read. */
+    isEntryGenerationCurrent(docId, generation) {
+        return (this.docEntryGeneration.get(docId) ?? 0) === generation;
+    }
+    /**
+     * The L2 path caches inside {@link tryLoadFromL2}; when entries were
+     * indexed while it ran, take the (outdated) result back out of L1.
+     */
+    dropCachedLoadIfSuperseded(docId, generation, loaded) {
+        if (this.isEntryGenerationCurrent(docId, generation) ||
+            this.docCache.get(docId) !== loaded) {
+            return;
+        }
+        this.docCache.delete(docId);
+        this.dirtyDocIds.delete(docId);
+    }
     async loadDocumentInternal(docId) {
         const startedAt = Date.now();
         const cacheCheckStartedAt = Date.now();
@@ -10528,8 +11060,10 @@ class BaseMindooDB {
         // the full signature-verify + decrypt + Automerge-replay pipeline.
         // The L2 path either returns a ready-to-use InternalDoc (already
         // promoted into L1) or null - in which case we proceed below.
+        const entryGeneration = this.docEntryGeneration.get(docId) ?? 0;
         const l2Doc = await this.tryLoadFromL2(docId);
         if (l2Doc !== null) {
+            this.dropCachedLoadIfSuperseded(docId, entryGeneration, l2Doc);
             this.performanceCallback?.onDocumentLoad?.({
                 docId,
                 cacheHit: true,
@@ -10613,7 +11147,9 @@ class BaseMindooDB {
             const aclDoc = aclResult.doc;
             // Only cache a fully-validated result. A transient validation failure
             // (audit finding #2) leaves `cacheable` false so the next load retries.
-            if (aclDoc && aclResult.cacheable) {
+            if (aclDoc &&
+                aclResult.cacheable &&
+                this.isEntryGenerationCurrent(docId, entryGeneration)) {
                 await this.storeCachedDocument(aclDoc);
                 this.markDocDirty(docId);
             }
@@ -10886,10 +11422,16 @@ class BaseMindooDB {
             witnessed: witnessState.witnessed,
             sealedRecipients: (0, recipients_1.newestRecipientBlock)(allEntryMetadata),
         };
-        // Update cache
-        await this.storeCachedDocument(internalDoc);
-        this.markDocDirty(docId);
-        this.logger.debug(`===== Successfully loaded document ${docId} and cached it =====`);
+        // Update cache (unless newer entries were indexed meanwhile: then this
+        // result is already outdated and the next read materializes again)
+        if (this.isEntryGenerationCurrent(docId, entryGeneration)) {
+            await this.storeCachedDocument(internalDoc);
+            this.markDocDirty(docId);
+            this.logger.debug(`===== Successfully loaded document ${docId} and cached it =====`);
+        }
+        else {
+            this.logger.debug(`===== Loaded document ${docId}; not cached, newer entries were indexed during the load =====`);
+        }
         this.performanceCallback?.onDocumentLoad?.({
             docId,
             cacheHit: false,
@@ -10920,8 +11462,10 @@ class BaseMindooDB {
         if (meta && meta.handle && typeof meta.handle.materialize === "function") {
             // Use native materialize() which properly converts Text objects to strings
             try {
+                // materialize() returns fresh objects, so the scalar wrappers can be
+                // replaced in place.
                 const materialized = meta.handle.materialize("/");
-                return materialized;
+                return this.normalizeAutomergeScalars(materialized);
             }
             catch (error) {
                 console.error("[MindooDB] Failed to materialize document:", error);
@@ -10942,6 +11486,11 @@ class BaseMindooDB {
             return value;
         if (value instanceof Uint8Array)
             return value;
+        if (value instanceof Date)
+            return new Date(value.getTime());
+        if (this.isAutomergeCounter(value) || this.isAutomergeImmutableString(value)) {
+            return this.readAutomergeScalar(value);
+        }
         if (depth > 40)
             return null;
         const automerge = automerge_adapter_1.Automerge;
@@ -10987,6 +11536,18 @@ class BaseMindooDB {
                 },
                 get: (target, prop) => {
                     const value = target[prop];
+                    // Counters read as plain numbers and immutable strings as strings;
+                    // their wrappers would otherwise surface as `{ value }` / `{ val }`
+                    // after structured cloning.
+                    if (db.isAutomergeCounter(value) ||
+                        db.isAutomergeImmutableString(value)) {
+                        return db.readAutomergeScalar(value);
+                    }
+                    // Hand out a copy: a Proxy around a Date breaks every Date method
+                    // ("this is not a Date object").
+                    if (value instanceof Date) {
+                        return new Date(value.getTime());
+                    }
                     // Recursively wrap nested objects (but not arrays or special types)
                     if (value !== null &&
                         typeof value === "object" &&
@@ -11044,6 +11605,9 @@ class BaseMindooDB {
             setAttachmentExtractedText: () => {
                 throw new Error("setAttachmentExtractedText() can only be called within changeDoc() callback");
             },
+            incrementCounter: () => {
+                throw new Error("incrementCounter() can only be called within changeDoc() callback; use applyJsonPatch() with counterIncrement outside of it");
+            },
             // ========== Attachment Read Methods ==========
             // These work in the read-only wrapper
             getAttachments: () => {
@@ -11851,7 +12415,14 @@ class BaseMindooDB {
         this.logger.info(`Pulling entries from remote store ${remoteStore.getId()}/${remoteStore.getStoreKind()}`);
         const restoreAuthOverride = await this.applyNetworkAuthOverrideForSync(remoteStore, options);
         try {
-            const syncResult = await this.syncEntriesFromStore(remoteStore, localStore, options);
+            // Docs pulls process each transferred scan page right away: the local
+            // index advances chunk by chunk, and the change event it emits lets the
+            // summary buffer / full-text index materialize and extract documents
+            // while the next page is still on the wire, instead of all at once
+            // after the transfer (see docs/adhoc-queries.md, "Cold start").
+            const syncResult = await this.syncEntriesFromStore(remoteStore, localStore, options, storeKind === types_2.StoreKind.docs
+                ? () => this.syncStoreChanges()
+                : undefined);
             const transferredBytes = syncResult.transferredBytes ?? 0;
             this.logger.debug(`Transferred ${syncResult.transferred} entries from remote store`);
             if (syncResult.cancelled) {