mindoodb 0.0.67 → 0.0.70

package/dist/core/BaseMindooDB.js

@@ -7,7 +7,6 @@ exports.BaseMindooDB = exports.CUSTOM_DOC_ID_INITIAL_CHANGE_BYTES = void 0;
 const automerge_adapter_1 = require("./automerge-adapter");
 const values_1 = require("./values");
 const storeEntryTime_1 = require("./storeEntryTime");
-const uuid_1 = require("uuid");
 const types_1 = require("./types");
 const types_2 = require("./appendonlystores/types");
 const syncStores_1 = require("./appendonlystores/syncStores");
@@ -483,6 +482,19 @@ class BaseMindooDB {
          * flushed to L2 under the new changeSeq) as if it were current.
          */
         this.docEntryGeneration = new Map();
+        /**
+         * Ids of entries this instance wrote itself (local writes) that the next
+         * {@link syncStoreChanges} run has not consumed yet. Local writes update
+         * the index directly but do not advance the processed-entry cursor, so
+         * that run (at the latest on the next open) re-reads them as "new". For a
+         * cached doc that is a no-op; for an uncached one the metadata-first path
+         * used to force a changeSeq bump, re-emitting unchanged documents to every
+         * changefeed consumer (summary, views, full-text) and invalidating their
+         * L2 records. Entries listed here are known to be indexed already, so the
+         * bump is only taken when the recomputed index state actually differs.
+         * Persisted in the metadata checkpoint (the restart case).
+         */
+        this.locallyIndexedEntryIds = new Set();
         this.cacheMetaDirty = false;
         /**
          * Per-document fingerprint (`changeSeq:automergeHeads`) of the state last
@@ -492,6 +504,15 @@ class BaseMindooDB {
          * that were merely loaded and cached, not changed).
          */
         this.lastFlushedDocState = new Map();
+        /**
+         * Per-document `changeSeq` of the record known to be in the L2 cache.
+         * Unlike {@link lastFlushedDocState} it is persisted in the metadata
+         * checkpoint, so after a restart the background warmer can tell which
+         * documents need work without reading every L2 record. A missing entry
+         * means "unknown" (legacy checkpoint): the record header is read once.
+         * Maintained only via {@link rememberL2Record} / {@link forgetL2Record}.
+         */
+        this.l2RecordChangeSeqs = new Map();
         /**
          * Tenant KeyBag fingerprint observed at the time the in-memory index
          * was last reconciled with the underlying store.
@@ -1583,7 +1604,29 @@ class BaseMindooDB {
         value.set(headerBytes, 4);
         value.set(amBinary, 4 + headerBytes.length);
         await store.put("doc", `${prefix}/${docId}`, value);
+        this.rememberL2Record(docId, changeSeq, flushKey);
+    }
+    /**
+     * Record that the L2 cache holds `docId` at `changeSeq` (with the given
+     * `changeSeq:automergeHeads` fingerprint). Marks the metadata checkpoint
+     * dirty when the persisted per-doc changeSeq moved, so the next flush
+     * carries it (flush-before-evict writes records outside a flush).
+     */
+    rememberL2Record(docId, changeSeq, flushKey) {
         this.lastFlushedDocState.set(docId, flushKey);
+        if (this.l2RecordChangeSeqs.get(docId) !== changeSeq) {
+            this.l2RecordChangeSeqs.set(docId, changeSeq);
+            this.cacheMetaDirty = true;
+            this.cacheManager?.markDirty();
+        }
+    }
+    /** Forget everything known about the L2 record of `docId`. */
+    forgetL2Record(docId) {
+        this.lastFlushedDocState.delete(docId);
+        if (this.l2RecordChangeSeqs.delete(docId)) {
+            this.cacheMetaDirty = true;
+            this.cacheManager?.markDirty();
+        }
     }
     exportMetadataCheckpoint() {
         const checkpoint = {
@@ -1607,6 +1650,16 @@ class BaseMindooDB {
         if (!this.supportsCursorScan(this.store)) {
             checkpoint.processedEntryIds = this.processedEntryIds;
         }
+        // Additive field: older checkpoints restore an empty set (the next
+        // syncStoreChanges then re-emits those local writes once, as before).
+        if (this.locallyIndexedEntryIds.size > 0) {
+            checkpoint.locallyIndexedEntryIds = Array.from(this.locallyIndexedEntryIds);
+        }
+        // Additive field: older checkpoints restore an empty map, which makes
+        // the warmer read each record header once (the previous behavior).
+        if (this.l2RecordChangeSeqs.size > 0) {
+            checkpoint.l2RecordChangeSeqs = Object.fromEntries(this.l2RecordChangeSeqs);
+        }
         // Persist the KeyBag fingerprint observed at the last reconciliation
         // so the next restore can decide whether to skip the visibility
         // metadata scan. `null` is acceptable - older checkpoints simply
@@ -1665,6 +1718,18 @@ class BaseMindooDB {
             if (checkpoint.processedEntryIds) {
                 this.processedEntryIds = checkpoint.processedEntryIds;
             }
+            this.locallyIndexedEntryIds = new Set(Array.isArray(checkpoint.locallyIndexedEntryIds)
+                ? checkpoint.locallyIndexedEntryIds.filter((id) => typeof id === "string")
+                : []);
+            this.l2RecordChangeSeqs.clear();
+            if (checkpoint.l2RecordChangeSeqs &&
+                typeof checkpoint.l2RecordChangeSeqs === "object") {
+                for (const [docId, changeSeq] of Object.entries(checkpoint.l2RecordChangeSeqs)) {
+                    if (typeof changeSeq === "number") {
+                        this.l2RecordChangeSeqs.set(docId, changeSeq);
+                    }
+                }
+            }
             this.lastReconciledKeyBagFingerprint =
                 typeof checkpoint.keyBagFingerprint === "string"
                     ? checkpoint.keyBagFingerprint
@@ -1786,6 +1851,17 @@ class BaseMindooDB {
             const docIds = await store.list("doc");
             const docPrefix = `${prefix}/`;
             const allOwnedKeys = docIds.filter((id) => id.startsWith(docPrefix));
+            // The persisted per-doc L2 state must not outlive its record (a
+            // record lost outside this instance would otherwise be skipped by the
+            // warmer forever).
+            if (this.l2RecordChangeSeqs.size > 0) {
+                const ownedDocIds = new Set(allOwnedKeys.map((key) => key.slice(docPrefix.length)));
+                for (const docId of Array.from(this.l2RecordChangeSeqs.keys())) {
+                    if (!ownedDocIds.has(docId)) {
+                        this.forgetL2Record(docId);
+                    }
+                }
+            }
             if (this.restoreToL2) {
                 // L2-only restore: leave L1 empty; let lazy reads via
                 // tryLoadFromL2 promote individual docs as they are needed. We
@@ -1836,9 +1912,14 @@ class BaseMindooDB {
                             const docId = batch[i].slice(docPrefix.length);
                             await store.delete("doc", batch[i]);
                             this.docCache.delete(docId);
-                            this.lastFlushedDocState.delete(docId);
+                            this.forgetL2Record(docId);
                             continue;
                         }
+                        // Legacy checkpoints lack the per-doc L2 state; the header
+                        // just read provides it for free.
+                        if (!this.l2RecordChangeSeqs.has(deserialized.internal.id)) {
+                            this.rememberL2Record(deserialized.internal.id, deserialized.persistedChangeSeq, `${deserialized.persistedChangeSeq}:${deserialized.persistedHeads.join(",")}`);
+                        }
                         await this.storeCachedDocument(deserialized.internal);
                         restoredDocs++;
                     }
@@ -2223,6 +2304,17 @@ class BaseMindooDB {
         }
         this.indexLookupStaleFrom = null;
     }
+    /**
+     * Write entries created by this instance to the docs store. The callers
+     * index the affected documents right away (see
+     * {@link locallyIndexedEntryIds}).
+     */
+    async putLocalEntries(entries) {
+        await this.store.putEntries(entries);
+        for (const entry of entries) {
+            this.locallyIndexedEntryIds.add(entry.id);
+        }
+    }
     /**
      * Register an automerge hash -> entry ID mapping for a document.
      * This is used to resolve Automerge dependency hashes to entry IDs.
@@ -2637,7 +2729,7 @@ class BaseMindooDB {
     async purgeMaterializedDocument(docId) {
         this.docCache.delete(docId);
         this.dirtyDocIds.delete(docId);
-        this.lastFlushedDocState.delete(docId);
+        this.forgetL2Record(docId);
         this.automergeHashToEntryId.delete(docId);
         this.forgetScannedEntryMetadata(docId);
         // Plaintext summary values must not outlive the purge either.
@@ -2964,10 +3056,12 @@ class BaseMindooDB {
         this.indexLookupStaleFrom = null;
         this.docCache.clear();
         this.lastFlushedDocState.clear();
+        this.l2RecordChangeSeqs.clear();
         this.automergeHashToEntryId.clear();
         this.clearScannedEntryMetadata();
         this.processedEntryIds = [];
         this.processedEntryCursor = null;
+        this.locallyIndexedEntryIds.clear();
         this.nextChangeSeq = 1;
         this.cacheMetaDirty = true;
         this.cacheManager?.markDirty();
@@ -3153,7 +3247,11 @@ class BaseMindooDB {
                             // Force the re-emit only when the batch contains actual replay
                             // entries (create/change/delete/undelete): a snapshot-only batch
                             // changes nothing observable and may keep the idempotent skip.
-                            const batchHasReplayEntries = docLifecycleEntries.some((e) => this.isDocumentReplayEntry(e));
+                            // Replay entries this instance wrote itself are already indexed
+                            // (the local write bumped changeSeq), so they only re-emit when
+                            // the recomputed state differs.
+                            const batchHasReplayEntries = docLifecycleEntries.some((e) => this.isDocumentReplayEntry(e) &&
+                                !this.locallyIndexedEntryIds.has(e.id));
                             this.updateIndex(docId, lastModified, isDeleted, representativeEntry.decryptionKeyId, "visible", awaitingWitness, witnessed, batchHasReplayEntries);
                             // `allDocMetadata` is this document's complete history in both
                             // branches above. Keeping it lets the materialization that
@@ -3261,6 +3359,9 @@ class BaseMindooDB {
         if (!this.supportsCursorScan(this.store)) {
             this.processedEntryIds.push(...newEntryMetadata.map((em) => em.id));
         }
+        for (const entryMeta of newEntryMetadata) {
+            this.locallyIndexedEntryIds.delete(entryMeta.id);
+        }
         this.processedEntryCursor = nextCursor;
         this.cacheMetaDirty = true;
         this.cacheManager?.markDirty();
@@ -3888,7 +3989,7 @@ class BaseMindooDB {
             allEntries.push(...preparedDoc.entries);
         }
         if (allEntries.length > 0) {
-            await this.store.putEntries(allEntries);
+            await this.putLocalEntries(allEntries);
         }
         for (const { index, internalDoc } of prepared) {
             await this.storeCachedDocument(internalDoc);
@@ -4204,7 +4305,7 @@ class BaseMindooDB {
             encryptedData: encryptedPayload,
         };
         // Store entry
-        await this.store.putEntries([fullEntry]);
+        await this.putLocalEntries([fullEntry]);
         // Register automerge hash -> entry ID mapping
         this.registerAutomergeHashMapping(docId, automergeHash, entryId);
         // A freshly created entry is versioned and un-witnessed, so the document is
@@ -6796,7 +6897,7 @@ class BaseMindooDB {
         if (prepared.length === 0) {
             return;
         }
-        await this.store.putEntries(prepared.map((p) => p.fullEntry));
+        await this.putLocalEntries(prepared.map((p) => p.fullEntry));
         for (const p of prepared) {
             await this.applyLifecycleEntryLocally(p, { markDirty: false });
         }
@@ -6829,7 +6930,7 @@ class BaseMindooDB {
      */
     async writeLifecycleEntry(internalDoc, entryType, signingKeyPair, signingKeyPassword, bypassPrecheck) {
         const prepared = await this.prepareLifecycleEntry(internalDoc, entryType, signingKeyPair, signingKeyPassword, bypassPrecheck);
-        await this.store.putEntries([prepared.fullEntry]);
+        await this.putLocalEntries([prepared.fullEntry]);
         await this.applyLifecycleEntryLocally(prepared, { markDirty: true });
     }
     /**
@@ -8648,7 +8749,7 @@ class BaseMindooDB {
                 getAfterDoc: () => this.convertAutomergeToJS(newDoc),
             });
         }
-        await this.store.putEntries(entries);
+        await this.putLocalEntries(entries);
         internalDoc.doc = newDoc;
         internalDoc.lastModified = now;
         // The change entries just written are versioned and un-witnessed, so the
@@ -8775,7 +8876,7 @@ class BaseMindooDB {
             const snapshotBytes = automerge_adapter_1.Automerge.save(internalDoc.doc);
             const encryptedPayload = await this.tenant.encryptPayload(snapshotBytes, internalDoc.decryptionKeyId);
             const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedPayload, this.getSubtle());
-            const pseudoSnapshotHash = `snapshot-${(0, uuid_1.v7)()}`;
+            const pseudoSnapshotHash = `snapshot-${(0, idGeneration_1.generateRandomUuid)()}`;
             const entryId = await (0, idGeneration_1.generateDocEntryId)(docId, pseudoSnapshotHash, headHashes, this.getSubtle());
             let signature;
             if (options.signingKeyPair && options.signingKeyPassword) {
@@ -8814,7 +8915,7 @@ class BaseMindooDB {
                         signingKeyPassword: options.signingKeyPassword,
                     }
                     : undefined);
-            await this.store.putEntries([snapshotEntry]);
+            await this.putLocalEntries([snapshotEntry]);
             this.writesSinceSnapshotCheck.set(docId, 0);
             this.logger.debug(`Created snapshot for document ${docId} with ${headHashes.length} heads and ${changesSinceSnapshot} changes since previous snapshot`);
         }
@@ -9743,7 +9844,7 @@ class BaseMindooDB {
             this.logger.warn(`L2 deserialize failed for ${docId}, evicting stale record: ${e}`);
             try {
                 await store.delete("doc", key);
-                this.lastFlushedDocState.delete(docId);
+                this.forgetL2Record(docId);
             }
             catch (deleteError) {
                 this.logger.warn(`Failed to evict corrupt L2 record for ${docId}: ${deleteError}`);
@@ -9770,7 +9871,7 @@ class BaseMindooDB {
             // L2 record is orphaned. Drop and fall through.
             try {
                 await store.delete("doc", key);
-                this.lastFlushedDocState.delete(docId);
+                this.forgetL2Record(docId);
             }
             catch (deleteError) {
                 this.logger.warn(`Failed to evict orphaned L2 record for ${docId}: ${deleteError}`);
@@ -9782,7 +9883,7 @@ class BaseMindooDB {
         if (persistedChangeSeq === currentChangeSeq) {
             // Seed the flush-skip fingerprint with the state just read from L2 so
             // the next periodic flush doesn't rewrite an identical record.
-            this.lastFlushedDocState.set(docId, `${persistedChangeSeq}:${automerge_adapter_1.Automerge.getHeads(internal.doc).join(",")}`);
+            this.rememberL2Record(docId, persistedChangeSeq, `${persistedChangeSeq}:${automerge_adapter_1.Automerge.getHeads(internal.doc).join(",")}`);
             await this.storeCachedDocument(internal);
             return internal;
         }
@@ -9869,9 +9970,11 @@ class BaseMindooDB {
      * First waits until the changefeed consumers (summary buffer, an
      * enabled full-text index) have caught up: they materialize every
      * changed document anyway, so running next to them would do the same
-     * work twice. Then walks a snapshot of the in-memory document index and
-     * skips every doc that is in L1 or whose L2 record is already at the
-     * current changeSeq (checked from the record header, no deserialization).
+     * work twice. Then walks the visible documents whose L2 record is not
+     * known to be at the current changeSeq (see {@link l2RecordChangeSeqs},
+     * persisted across restarts, so a pass after a small sync only visits
+     * what changed). It skips every doc that is in L1 or turns out current
+     * (record header check for docs with unknown state, no deserialization).
      * For the rest it calls {@link tryLoadFromL2}. That path either:
      *
      *  - hits L2 fresh and just promotes the doc to L1, or
@@ -9994,26 +10097,38 @@ class BaseMindooDB {
         };
         this.warmerProgress = {
             processed: 0,
-            total: this.index.length,
+            // Unknown until the consumers caught up and the pass is planned.
+            total: 0,
             phase: "warming",
         };
         await this.awaitChangefeedConsumersForWarmer(signal);
         if (signal.aborted) {
-            emitProgress({ processed: 0, total: this.index.length, phase: "cancelled" });
+            emitProgress({ processed: 0, total: 0, phase: "cancelled" });
             return;
         }
-        // Snapshot the docId list so concurrent index updates don't shift
-        // our iteration mid-pass. Iterating the index directly would also
-        // work but a snapshot makes the loop's behavior easier to reason
-        // about under sync activity.
-        const docIds = this.index.map((entry) => entry.docId);
+        // Snapshot the docs that may need work, so concurrent index updates
+        // don't shift our iteration mid-pass. Docs whose L2 record is known to
+        // be at the current changeSeq are left out up front: after a small sync
+        // the pass (and its progress total) covers only what actually changed
+        // instead of the whole database. Docs held in L1 are left out as well:
+        // loads and changes mark them dirty, so the next flush (or
+        // flush-before-evict) persists them anyway. Unknown docs (legacy
+        // checkpoint) stay in and are checked against their record header below.
+        const docIds = [];
+        for (const entry of this.index) {
+            if (entry.accessState === "visible" &&
+                this.l2RecordChangeSeqs.get(entry.docId) !== entry.changeSeq &&
+                !this.docCache.has(entry.docId)) {
+                docIds.push(entry.docId);
+            }
+        }
         const total = docIds.length;
         // Progress snapshot is replaced (not mutated in place) so polled
         // readers see consistent values. We initialize even when total=0
         // so the UI can display a "done" terminal state.
         this.warmerProgress = { processed: 0, total, phase: "warming" };
         if (total === 0) {
-            this.logger.debug("Warmer: nothing to warm, index is empty.");
+            this.logger.debug("Warmer: nothing to warm, every L2 record is current.");
             emitProgress({ processed: 0, total: 0, phase: "done" });
             return;
         }
@@ -10113,9 +10228,9 @@ class BaseMindooDB {
     }
     /**
      * Whether L2 already holds this document at its current changeSeq, so
-     * the warmer can skip it without deserializing. Answered from
-     * {@link lastFlushedDocState} for records written in this session,
-     * otherwise from the record header (no Automerge load).
+     * the warmer can skip it without deserializing. Answered from the
+     * persisted {@link l2RecordChangeSeqs}; only when that has no entry for
+     * the doc (legacy checkpoint) from the record header (no Automerge load).
      */
     async isL2RecordCurrent(docId) {
         const indexEntryIdx = this.getDocIndexPosition(docId);
@@ -10128,10 +10243,12 @@ class BaseMindooDB {
             return false;
         }
         const currentChangeSeq = entry.changeSeq;
-        const flushed = this.lastFlushedDocState.get(docId);
-        if (flushed !== undefined) {
-            return flushed.startsWith(`${currentChangeSeq}:`);
+        const knownChangeSeq = this.l2RecordChangeSeqs.get(docId);
+        if (knownChangeSeq !== undefined) {
+            return knownChangeSeq === currentChangeSeq;
         }
+        // Unknown (checkpoint from before the per-doc state was persisted):
+        // read the record header once and remember the answer.
         try {
             const bytes = await store.get("doc", `${this.getCachePrefix()}/${docId}`);
             if (!bytes) {
@@ -10143,7 +10260,7 @@ class BaseMindooDB {
                 !Array.isArray(header.automergeHeads)) {
                 return false;
             }
-            this.lastFlushedDocState.set(docId, `${currentChangeSeq}:${header.automergeHeads.join(",")}`);
+            this.rememberL2Record(docId, currentChangeSeq, `${currentChangeSeq}:${header.automergeHeads.join(",")}`);
             return true;
         }
         catch {
@@ -11852,7 +11969,7 @@ class BaseMindooDB {
         this.logger.debug(`Adding attachment to document ${docId}: ${fileName} (${fileData.length} bytes)`);
         const store = this.getEffectiveAttachmentStore();
         const currentUser = await this.tenant.getCurrentUserId();
-        const attachmentId = (0, idGeneration_1.generateFileUuid7)();
+        const attachmentId = (0, idGeneration_1.generateAttachmentId)();
         // Chunk the file
         const chunks = [];
         let prevChunkId = null;
@@ -11983,7 +12100,7 @@ class BaseMindooDB {
         this.logger.debug(`Adding streaming attachment to document ${docId}: ${fileName}`);
         const store = this.getEffectiveAttachmentStore();
         const currentUser = await this.tenant.getCurrentUserId();
-        const attachmentId = (0, idGeneration_1.generateFileUuid7)();
+        const attachmentId = (0, idGeneration_1.generateAttachmentId)();
         let totalSize = 0;
         let prevChunkId = null;
         let lastChunkId = "";
@@ -12315,7 +12432,7 @@ class BaseMindooDB {
             signingKeyPassword: signing?.signingKeyPassword,
             attachmentRefs: this.collectAttachmentRefs(newDoc),
         });
-        await this.store.putEntries([entry]);
+        await this.putLocalEntries([entry]);
         this.registerAutomergeHashMapping(docId, automerge_adapter_1.Automerge.decodeChange(changeBytes).hash, entry.id);
         internalDoc.doc = newDoc;
         internalDoc.lastModified = now;