mindoodb 0.0.84 → 0.0.86

package/dist/core/BaseMindooDB.js

@@ -5,6 +5,7 @@ exports.BaseMindooDB = exports.CUSTOM_DOC_ID_INITIAL_CHANGE_BYTES = void 0;
 // React Native: native Rust (react-native-automerge-generated)
 // Browser/Node.js: WASM (@automerge/automerge/slim)
 const automerge_adapter_1 = require("./automerge-adapter");
+const ChangeDocDraft_1 = require("./ChangeDocDraft");
 const values_1 = require("./values");
 const storeEntryTime_1 = require("./storeEntryTime");
 const changeAttribution_1 = require("./changeAttribution");
@@ -293,6 +294,8 @@ class BaseMindooDB {
         this.nextChangeSeq = 1;
         // Cache of loaded documents: Map<docId, InternalDoc>
         this.docCache = new Map();
+        /** Document loads in progress; see {@link loadDocumentInternal}. */
+        this.documentLoadsInFlight = new Map();
         this.wrappedInternalDocs = new WeakMap();
         /**
          * Per-document count of local writes since the last snapshot eligibility
@@ -362,6 +365,25 @@ class BaseMindooDB {
          */
         this.revisionFeedDocCache = new Map();
         this.revisionFeedCacheLimit = 256;
+        /**
+         * Revisions of documents whose fold suffix holds un-witnessed entries, keyed
+         * by the exact fold (entry order, witness state and prefix). The feed
+         * re-discovers such entries on every resume, since their trusted time is the
+         * provisional `now`; while nothing about the document changed, only that
+         * trusted time differs, so the revisions are re-emitted from here instead
+         * of being loaded, verified, decrypted and folded again. Without it, every
+         * pass re-folded every un-witnessed document: registering N users offline
+         * cost O(N) per registration. An entry is replaced on any change and
+         * dropped once the document has no un-witnessed entries left.
+         *
+         * Each revision holds its own document state, so the cache is bounded: a
+         * document with a longer suffix, or one arriving when the cache is full, is
+         * folded every pass as before.
+         */
+        this.provisionalRevisionCache = new Map();
+        this.provisionalRevisionCacheSize = 0;
+        this.provisionalRevisionCacheLimit = 4096;
+        this.provisionalRevisionCacheDocLimit = 32;
         /**
          * Window size for batching encrypted-payload loads while folding a document's
          * revision suffix ({@link foldDocRevisions}): one {@link ContentAddressedStore.getEntries}
@@ -5174,6 +5196,27 @@ class BaseMindooDB {
         if (firstChangedIdx < 0) {
             return;
         }
+        // Unchanged document whose suffix is still provisional: re-emit the
+        // previous fold with the current trusted times (see provisionalRevisionCache).
+        const suffix = ttSorted.slice(firstChangedIdx);
+        const hasProvisional = suffix.some((r) => !r.witnessed);
+        const foldKey = [
+            ttSorted.slice(0, firstChangedIdx).map((r) => r.meta.id).join(","),
+            suffix.map((r) => `${r.meta.id}:${r.meta.receivedAt ?? "-"}`).join(","),
+        ].join("|");
+        const cachedFold = this.provisionalRevisionCache.get(docId);
+        if (cachedFold && cachedFold.key === foldKey) {
+            for (let i = 0; i < cachedFold.revisions.length; i++) {
+                yield { ...cachedFold.revisions[i], trustedTime: suffix[i].tt, cursor: resumeCursor };
+            }
+            return;
+        }
+        if (cachedFold) {
+            this.provisionalRevisionCache.delete(docId);
+            this.provisionalRevisionCacheSize -= cachedFold.revisions.length;
+        }
+        const foldedRevisions = [];
+        let foldComplete = true;
         // Origin time and key id come from the document's create entry, independent
         // of fold order (under the trusted-time rule un-witnessed entries can sort
         // before the create, so we must not rely on seeing the create first).
@@ -5223,6 +5266,7 @@ class BaseMindooDB {
                 // Absent from the loaded window means missing in the store or it failed
                 // signature verification (already logged during the parallel prepare).
                 this.logger.warn(`Revision entry ${meta.id} unavailable (missing or invalid), skipping`);
+                foldComplete = false;
                 continue;
             }
             const entryData = ready.entry;
@@ -5247,16 +5291,24 @@ class BaseMindooDB {
                 // this document's replay entries seen up to and including this one.
                 isDeleted: this.computeIsDeletedFromMetadata([...seenMeta]),
             };
-            yield {
+            const revision = {
                 docId,
                 entryId: meta.id,
                 doc: this.wrapDocument(internalDoc),
                 entryType: meta.entryType,
-                trustedTime: rev.tt,
                 witnessed: rev.witnessed,
                 createdByPublicKey: meta.createdByPublicKey,
-                cursor: resumeCursor,
             };
+            foldedRevisions.push(revision);
+            yield { ...revision, trustedTime: rev.tt, cursor: resumeCursor };
+        }
+        // Reached only when the consumer took every revision of this document.
+        if (hasProvisional &&
+            foldComplete &&
+            foldedRevisions.length <= this.provisionalRevisionCacheDocLimit &&
+            this.provisionalRevisionCacheSize + foldedRevisions.length <= this.provisionalRevisionCacheLimit) {
+            this.provisionalRevisionCache.set(docId, { key: foldKey, revisions: foldedRevisions });
+            this.provisionalRevisionCacheSize += foldedRevisions.length;
         }
         // Cache the doc's full merged state for the next incremental advance.
         this.putRevisionFeedCache(docId, {
@@ -8135,8 +8187,6 @@ class BaseMindooDB {
         //
         // We use a two-phase approach: the callback can do async work and collect
         // what needs to be changed, then we apply those changes in Automerge.change()
-        const pendingChanges = new Map();
-        const pendingDeletions = new Set();
         // Track pending attachment operations
         const pendingAttachmentAdditions = [];
         const pendingAttachmentRemovals = new Set();
@@ -8144,19 +8194,6 @@ class BaseMindooDB {
         const pendingAttachmentAppends = new Map();
         // Map of attachmentId -> extraction result for setAttachmentExtractedText
         const pendingAttachmentTextUpdates = new Map();
-        // incrementCounter() calls, applied in call order after the field
-        // assignments so they can target counters assigned in the same callback.
-        const pendingCounterIncrements = [];
-        // Assignments and deletions are applied before the increments, so an
-        // assignment or deletion of a field supersedes the increments queued on it
-        // (or below it) earlier in the callback; later increments still apply.
-        const dropPendingCounterIncrements = (field) => {
-            for (let i = pendingCounterIncrements.length - 1; i >= 0; i--) {
-                if (pendingCounterIncrements[i].path[0] === field) {
-                    pendingCounterIncrements.splice(i, 1);
-                }
-            }
-        };
         // Reference to db for closures
         const db = this;
         // Guard flag to prevent operations after callback completes
@@ -8167,6 +8204,15 @@ class BaseMindooDB {
                 throw new Error(`${methodName}() cannot be called after changeDoc() callback has completed. Document changes can only be made within the callback.`);
             }
         };
+        // getData() edits a lazily built shadow copy and logs every edit, at any
+        // depth, for replay inside Automerge.change() below.
+        const draft = new ChangeDocDraft_1.ChangeDocDraft(internalDoc.doc, {
+            scalarKind: (value) => db.automergeScalarKind(value),
+            readScalar: (value) => db.readAutomergeScalar(value),
+            makeCounter: (value) => new automerge_adapter_1.Automerge.Counter(value),
+            isRichTextInBase: (path) => db.isRichTextAt(internalDoc.doc, path),
+            assertActive: throwIfCallbackInactive,
+        });
         // Create a document wrapper that collects changes
         const collectingDoc = {
             getDatabase: () => this,
@@ -8184,62 +8230,7 @@ class BaseMindooDB {
             isEncryptedFor: (users) => (0, sealedTypes_1.isSealedKeyId)(internalDoc.decryptionKeyId) &&
                 (0, recipients_1.isPayloadEncryptedFor)(internalDoc.doc, internalDoc.sealedRecipients, Array.isArray(users) ? users : [users]),
             getHeads: () => automerge_adapter_1.Automerge.getHeads(internalDoc.doc),
-            getData: () => {
-                // Return a proxy that collects property assignments and deletions
-                const currentData = internalDoc.doc;
-                return new Proxy(currentData, {
-                    set: (target, prop, value) => {
-                        throwIfCallbackInactive("set property");
-                        if (typeof prop === "string") {
-                            dropPendingCounterIncrements(prop);
-                            // If this property was marked for deletion, remove it from deletions
-                            pendingDeletions.delete(prop);
-                            // Track the change
-                            pendingChanges.set(prop, value);
-                            // NOTE: Don't set on target immediately - Automerge requires changes
-                            // to be made inside Automerge.change(). Pending changes are applied later.
-                        }
-                        return true;
-                    },
-                    deleteProperty: (target, prop) => {
-                        throwIfCallbackInactive("delete property");
-                        if (typeof prop === "string") {
-                            dropPendingCounterIncrements(prop);
-                            // Mark for deletion
-                            pendingDeletions.add(prop);
-                            // Remove from pending changes if it was there
-                            pendingChanges.delete(prop);
-                            // NOTE: Don't delete from target immediately - Automerge requires changes
-                            // to be made inside Automerge.change(). Pending deletions are applied later.
-                        }
-                        return true;
-                    },
-                    get: (target, prop) => {
-                        // If marked for deletion, return undefined
-                        if (typeof prop === "string" && pendingDeletions.has(prop)) {
-                            return undefined;
-                        }
-                        // Check pending changes first, then target
-                        if (typeof prop === "string" && pendingChanges.has(prop)) {
-                            return pendingChanges.get(prop);
-                        }
-                        // Counters read as numbers and immutable strings as strings here
-                        // too; change counters with incrementCounter().
-                        return db.readAutomergeScalar(target[prop]);
-                    },
-                    has: (target, prop) => {
-                        // If marked for deletion, it doesn't exist
-                        if (typeof prop === "string" && pendingDeletions.has(prop)) {
-                            return false;
-                        }
-                        // Check pending changes first, then target
-                        if (typeof prop === "string" && pendingChanges.has(prop)) {
-                            return true;
-                        }
-                        return prop in target;
-                    },
-                });
-            },
+            getData: () => draft.getRoot(),
             // ========== Attachment Write Methods (work in changeDoc context) ==========
             addAttachment: async (fileData, fileName, mimeType, keyId) => {
                 throwIfCallbackInactive("addAttachment");
@@ -8354,12 +8345,25 @@ class BaseMindooDB {
                 }
             },
             // ========== Counter Write Methods ==========
+            updateText: (path, value) => {
+                const segments = typeof path === "string" ? [path] : [...path];
+                db.validateJsonPath(segments, "updateText");
+                if (draft.getFieldType(segments) === "richText") {
+                    throw new Error(`updateText: '${segments.map(String).join(".")}' is rich text (marks or blocks); use applyRichTextPatch() or applyRichTextStepsPatch()`);
+                }
+                draft.updateText(segments, value);
+            },
+            getFieldType: (path) => {
+                const segments = typeof path === "string" ? [path] : [...path];
+                db.validateJsonPath(segments, "getFieldType");
+                return draft.getFieldType(segments);
+            },
             incrementCounter: (path, delta) => {
                 throwIfCallbackInactive("incrementCounter");
                 const segments = typeof path === "string" ? [path] : [...path];
                 db.validateJsonPath(segments, "incrementCounter");
                 (0, values_1.assertCounterAmount)(delta, "incrementCounter delta");
-                pendingCounterIncrements.push({ path: segments, delta });
+                draft.incrementCounter(segments, delta);
             },
             // ========== Attachment Read Methods (also work in changeDoc context) ==========
             getAttachments: () => {
@@ -8394,20 +8398,14 @@ class BaseMindooDB {
         let newDoc;
         try {
             newDoc = this.runChangeWithOutdatedDocRecovery(internalDoc, (doc) => automerge_adapter_1.Automerge.change(doc, { message: changeMessage }, (automergeDoc) => {
-                // Apply all pending changes (sets/updates)
-                for (const [key, value] of pendingChanges) {
-                    this.assignHydrated(automergeDoc, key, this.hydrateTypedValues(value));
-                }
-                // Apply all pending deletions
-                for (const key of pendingDeletions) {
-                    delete automergeDoc[key];
-                }
+                // Replay the callback's field edits and counter increments in the
+                // order they were made: an assignment after an increment replaces
+                // it, an increment after an assignment adds to the new counter, and
+                // a list index means the element it meant at that moment.
+                this.applyChangeDocDraftOps(automergeDoc, draft.ops);
                 // Engine-internal edits that need the raw Automerge document, e.g.
                 // restoreDocumentToEntry's rich-text spans.
                 internalMutation?.(automergeDoc);
-                for (const { path, delta } of pendingCounterIncrements) {
-                    this.incrementCounterAtPath(automergeDoc, path, delta);
-                }
                 // Apply pending attachment changes
                 if (pendingAttachmentAdditions.length > 0 ||
                     pendingAttachmentRemovals.size > 0 ||
@@ -8512,6 +8510,96 @@ class BaseMindooDB {
         }
         await Promise.all(pendingAttachmentAdditions.map((ref) => this.clearPendingAttachmentUploadLedger(ref.attachmentId)));
     }
+    /** "counter" / "atomic" for the CRDT scalars that read as plain values, otherwise null. */
+    automergeScalarKind(value) {
+        if (this.isAutomergeCounter(value))
+            return "counter";
+        if (this.isAutomergeImmutableString(value))
+            return "atomic";
+        return null;
+    }
+    /** Whether the text at `path` carries marks or block markers (rich text). */
+    isRichTextAt(automergeDoc, path) {
+        if (typeof this.readValueAtPath(automergeDoc, path) !== "string")
+            return false;
+        const spans = automerge_adapter_1.Automerge.spans(automergeDoc, path);
+        return spans.some((span) => span.type === "block" || (span.marks !== undefined && Object.keys(span.marks).length > 0));
+    }
+    fieldTypeAt(automergeDoc, path) {
+        if (path.length === 0)
+            return "map";
+        let parent = automergeDoc;
+        for (const segment of path.slice(0, -1)) {
+            parent = parent?.[segment];
+            if (parent === null || typeof parent !== "object" || this.automergeScalarKind(parent) !== null)
+                return undefined;
+        }
+        const key = path[path.length - 1];
+        if (!Object.prototype.hasOwnProperty.call(parent, key))
+            return undefined;
+        const value = parent[key];
+        if (typeof value === "string")
+            return this.isRichTextAt(automergeDoc, path) ? "richText" : "text";
+        return (0, ChangeDocDraft_1.fieldTypeOf)(value, { scalarKind: (v) => this.automergeScalarKind(v) });
+    }
+    async getFieldType(doc, path) {
+        const segments = typeof path === "string" ? [path] : [...path];
+        this.validateJsonPath(segments, "getFieldType");
+        const docId = doc.getId();
+        let internalDoc = this.wrappedInternalDocs.get(doc) ?? this.getCachedDocument(docId);
+        if (!internalDoc) {
+            const loadedDoc = await this.loadDocumentInternal(docId);
+            if (!loadedDoc) {
+                throw new errors_1.DocumentNotFoundError(docId);
+            }
+            internalDoc = loadedDoc;
+        }
+        return this.fieldTypeAt(internalDoc.doc, segments);
+    }
+    /** Replays the edits a changeDoc() callback made through getData(), see ChangeDocDraft. */
+    applyChangeDocDraftOps(automergeDoc, ops) {
+        const containerAt = (path) => {
+            let current = automergeDoc;
+            for (const segment of path)
+                current = current[segment];
+            return current;
+        };
+        for (const op of ops) {
+            if (op.kind === "counter") {
+                this.incrementCounterAtPath(automergeDoc, op.path, op.delta);
+                continue;
+            }
+            if (op.kind === "splice") {
+                const list = containerAt(op.path);
+                const values = this.hydrateTypedValues(op.items);
+                list.splice(op.index, op.deleteCount, ...this.withoutNestedFloats(values));
+                values.forEach((value, offset) => {
+                    if (value instanceof automerge_adapter_1.Automerge.Float64)
+                        list[op.index + offset] = new automerge_adapter_1.Automerge.Float64(value.value);
+                    else
+                        this.restoreNestedFloats(list[op.index + offset], value);
+                });
+                continue;
+            }
+            const parent = containerAt(op.path.slice(0, -1));
+            const key = op.path[op.path.length - 1];
+            if (op.kind === "delete") {
+                delete parent[key];
+                continue;
+            }
+            const current = parent[key];
+            if (op.kind === "text" && typeof current === "string") {
+                // Splice only the changed middle, so the edit merges with concurrent
+                // typing instead of replacing the whole text.
+                const splice = (0, ChangeDocDraft_1.diffTextSplice)(current, op.value);
+                if (splice) {
+                    automerge_adapter_1.Automerge.splice(automergeDoc, op.path, splice.index, splice.deleteCount, splice.insert);
+                }
+                continue;
+            }
+            this.assignHydrated(parent, key, this.hydrateTypedValues(op.value));
+        }
+    }
     validateTextPatch(patch) {
         if (!Array.isArray(patch.path) || patch.path.length === 0) {
             throw new Error("Text patch path must contain at least one segment");
@@ -11751,7 +11839,33 @@ class BaseMindooDB {
         this.docCache.delete(docId);
         this.dirtyDocIds.delete(docId);
     }
-    async loadDocumentInternal(docId) {
+    /**
+     * Materialize a document, sharing a load that is already running for it.
+     *
+     * Several readers often want the same document at the same moment: the
+     * summary buffer and the full-text index follow the same changefeed after
+     * every sync page, each with its own prefetch window. Without sharing, both
+     * windows miss the L1 cache for documents the other is still loading and
+     * each materializes them, so a first sync with both indexes paid every
+     * document twice. A load is only shared while no newer entries for the
+     * document were indexed since it started, so no caller gets a state older
+     * than its call.
+     */
+    loadDocumentInternal(docId) {
+        const generation = this.docEntryGeneration.get(docId) ?? 0;
+        const inFlight = this.documentLoadsInFlight.get(docId);
+        if (inFlight && inFlight.generation === generation) {
+            return inFlight.promise;
+        }
+        const promise = this.loadDocumentUncoalesced(docId).finally(() => {
+            if (this.documentLoadsInFlight.get(docId)?.promise === promise) {
+                this.documentLoadsInFlight.delete(docId);
+            }
+        });
+        this.documentLoadsInFlight.set(docId, { generation, promise });
+        return promise;
+    }
+    async loadDocumentUncoalesced(docId) {
         const startedAt = Date.now();
         const cacheCheckStartedAt = Date.now();
         // Short-circuit on the visibility layer: a doc marked
@@ -12339,6 +12453,14 @@ class BaseMindooDB {
             setAttachmentExtractedText: () => {
                 throw new Error("setAttachmentExtractedText() can only be called within changeDoc() callback");
             },
+            getFieldType: (path) => {
+                const segments = typeof path === "string" ? [path] : [...path];
+                this.validateJsonPath(segments, "getFieldType");
+                return this.fieldTypeAt(internalDoc.doc, segments);
+            },
+            updateText: () => {
+                throw new Error("updateText() can only be called within changeDoc() callback; use applyJsonPatch() with textSplice outside of it");
+            },
             incrementCounter: () => {
                 throw new Error("incrementCounter() can only be called within changeDoc() callback; use applyJsonPatch() with counterIncrement outside of it");
             },