mindoodb 0.0.3 → 0.0.5

package/dist/core/BaseMindooDB.js

@@ -7,6 +7,8 @@ exports.BaseMindooDB = void 0;
 const automerge_adapter_1 = require("./automerge-adapter");
 const uuid_1 = require("uuid");
 const bloom_1 = require("./appendonlystores/bloom");
+const MaterializationPlanner_1 = require("./appendonlystores/MaterializationPlanner");
+const AttachmentReadPlanner_1 = require("./appendonlystores/AttachmentReadPlanner");
 const idGeneration_1 = require("./utils/idGeneration");
 const errors_1 = require("./errors");
 const logging_1 = require("./logging");
@@ -14,6 +16,10 @@ const logging_1 = require("./logging");
  * Default chunk size for attachments: 256KB
  */
 const DEFAULT_CHUNK_SIZE_BYTES = 256 * 1024;
+const DEFAULT_SNAPSHOT_MIN_CHANGES = 100;
+const DEFAULT_SNAPSHOT_COOLDOWN_MS = 10 * 60 * 1000;
+const DEFAULT_MAX_CACHED_DOCS = 128;
+const DEFAULT_ITERATE_PREFETCH_WINDOW_DOCS = 0;
 /**
  * BaseMindooDB is a platform-agnostic implementation of MindooDB
  * that works in both browser and server environments.
@@ -34,12 +40,13 @@ const DEFAULT_CHUNK_SIZE_BYTES = 256 * 1024;
  * - Automerge.applyChanges(doc, changes) - verify method signature
  */
 class BaseMindooDB {
-    constructor(tenant, store, attachmentStore, attachmentConfig, adminOnlyDb = false, logger, performanceCallback) {
-        // Internal index: sorted array of document entries, maintained in order by (lastModified, docId)
-        // This allows efficient incremental processing without sorting on each call
+    constructor(tenant, store, attachmentStore, attachmentConfig, documentCacheConfig, snapshotConfig, adminOnlyDb = false, logger, performanceCallback) {
+        // Internal changefeed index: sorted by (changeSeq, docId) for deterministic iteration.
+        // lastModified remains available for UX metadata but is not the primary cursor key.
         this.index = [];
         // Lookup map for O(1) access to index entries by docId
         this.indexLookup = new Map(); // Map<docId, arrayIndex>
+        this.nextChangeSeq = 1;
         // Cache of loaded documents: Map<docId, InternalDoc>
         this.docCache = new Map();
         // Track which entry IDs we've already processed
@@ -51,11 +58,23 @@ class BaseMindooDB {
         // Cache for imported public keys (CryptoKey objects) to avoid re-importing the same key
         // Map<publicKeyPEM, CryptoKey>
         this.publicKeyCache = new Map();
+        // Local cache support
+        this.cacheManager = null;
+        this.cachePrefix = null;
+        this.dirtyDocIds = new Set();
+        this.cacheMetaDirty = false;
         this.tenant = tenant;
         this.store = store;
         this.attachmentStore = attachmentStore;
         this.chunkSizeBytes = attachmentConfig?.chunkSizeBytes ?? DEFAULT_CHUNK_SIZE_BYTES;
         this._isAdminOnlyDb = adminOnlyDb;
+        this.maxCachedDocs = Math.max(1, Math.floor(documentCacheConfig?.maxEntries ?? DEFAULT_MAX_CACHED_DOCS));
+        this.iteratePrefetchWindowDocs = Math.max(0, Math.floor(documentCacheConfig?.iteratePrefetchWindowDocs ??
+            DEFAULT_ITERATE_PREFETCH_WINDOW_DOCS));
+        this.cacheRestoreLimit = Math.max(1, Math.floor(documentCacheConfig?.restoreLimit ?? this.maxCachedDocs));
+        this.reconcileRestoredIndexOnInit = documentCacheConfig?.reconcileRestoredIndexOnInit ?? false;
+        this.snapshotMinChanges = Math.max(1, Math.floor(snapshotConfig?.minChanges ?? DEFAULT_SNAPSHOT_MIN_CHANGES));
+        this.snapshotCooldownMs = Math.max(0, Math.floor(snapshotConfig?.cooldownMs ?? DEFAULT_SNAPSHOT_COOLDOWN_MS));
         // Create logger if not provided (for backward compatibility)
         this.logger =
             logger ||
@@ -72,6 +91,208 @@ class BaseMindooDB {
     isAdminOnlyDb() {
         return this._isAdminOnlyDb;
     }
+    // ---------------------------------------------------------------------------
+    // Local cache support (ICacheable)
+    // ---------------------------------------------------------------------------
+    /**
+     * Attach a CacheManager so this DB participates in periodic cache flushing.
+     */
+    setCacheManager(cacheManager) {
+        this.cacheManager = cacheManager;
+        const cacheIdentity = this.store.getCacheIdentity?.() ?? this.store.getId();
+        this.cachePrefix = `${this.tenant.getId()}/${cacheIdentity}`;
+        cacheManager.register(this);
+    }
+    getCachePrefix() {
+        return this.cachePrefix ?? `${this.tenant.getId()}/${this.store.getId()}`;
+    }
+    hasDirtyState() {
+        return this.dirtyDocIds.size > 0 || this.cacheMetaDirty;
+    }
+    clearDirty() {
+        this.dirtyDocIds.clear();
+        this.cacheMetaDirty = false;
+        this.evictCachedDocsIfNeeded();
+    }
+    markDocDirty(docId) {
+        this.dirtyDocIds.add(docId);
+        this.cacheMetaDirty = true;
+        this.cacheManager?.markDirty();
+    }
+    getCachedDocument(docId) {
+        const cached = this.docCache.get(docId) ?? null;
+        if (!cached) {
+            return null;
+        }
+        this.touchCachedDocument(docId, cached);
+        return cached;
+    }
+    touchCachedDocument(docId, internalDoc) {
+        this.docCache.delete(docId);
+        this.docCache.set(docId, internalDoc);
+    }
+    storeCachedDocument(internalDoc) {
+        this.touchCachedDocument(internalDoc.id, internalDoc);
+        this.evictCachedDocsIfNeeded(new Set([internalDoc.id]));
+    }
+    evictCachedDocsIfNeeded(protectedDocIds) {
+        if (this.docCache.size <= this.maxCachedDocs) {
+            return;
+        }
+        for (const [docId] of this.docCache) {
+            if (this.docCache.size <= this.maxCachedDocs) {
+                break;
+            }
+            if (protectedDocIds?.has(docId)) {
+                continue;
+            }
+            this.docCache.delete(docId);
+            this.dirtyDocIds.delete(docId);
+        }
+    }
+    /**
+     * Flush dirty documents and metadata to the cache store.
+     */
+    async flushToCache(store) {
+        const prefix = this.getCachePrefix();
+        let written = 0;
+        for (const docId of this.dirtyDocIds) {
+            const internal = this.docCache.get(docId);
+            if (!internal)
+                continue;
+            const amBinary = automerge_adapter_1.Automerge.save(internal.doc);
+            const header = JSON.stringify({
+                id: internal.id,
+                createdAt: internal.createdAt,
+                lastModified: internal.lastModified,
+                decryptionKeyId: internal.decryptionKeyId,
+                isDeleted: internal.isDeleted,
+            });
+            const headerBytes = new TextEncoder().encode(header);
+            // Format: 4-byte header length (big-endian) + header JSON + Automerge binary
+            const value = new Uint8Array(4 + headerBytes.length + amBinary.length);
+            const view = new DataView(value.buffer);
+            view.setUint32(0, headerBytes.length, false);
+            value.set(headerBytes, 4);
+            value.set(amBinary, 4 + headerBytes.length);
+            await store.put("doc", `${prefix}/${docId}`, value);
+            written++;
+        }
+        // Write metadata checkpoint
+        const meta = this.exportMetadataCheckpoint();
+        await store.put("db-meta", prefix, meta);
+        written++;
+        return written;
+    }
+    exportMetadataCheckpoint() {
+        const checkpoint = {
+            version: 2,
+            processedEntryCursor: this.processedEntryCursor,
+            index: this.index,
+            nextChangeSeq: this.nextChangeSeq,
+        };
+        // Serialize automergeHashToEntryId: Map<string, Map<string,string>> -> nested object
+        const hashMap = {};
+        for (const [docId, inner] of this.automergeHashToEntryId) {
+            hashMap[docId] = Object.fromEntries(inner);
+        }
+        checkpoint.automergeHashToEntryId = hashMap;
+        // For stores without cursor scan, include processedEntryIds
+        if (!this.supportsCursorScan(this.store)) {
+            checkpoint.processedEntryIds = this.processedEntryIds;
+        }
+        return new TextEncoder().encode(JSON.stringify(checkpoint));
+    }
+    /**
+     * Attempt to restore state from cache. Returns true on success.
+     */
+    async restoreFromCache(store) {
+        const prefix = this.getCachePrefix();
+        try {
+            // 1. Load metadata checkpoint
+            const metaBytes = await store.get("db-meta", prefix);
+            if (!metaBytes) {
+                this.logger.debug("No cache metadata found, will do full rebuild");
+                return false;
+            }
+            const checkpoint = JSON.parse(new TextDecoder().decode(metaBytes));
+            if (checkpoint.version !== 2) {
+                this.logger.warn(`Unknown cache version ${checkpoint.version}, ignoring cache`);
+                return false;
+            }
+            // 2. Restore metadata
+            this.processedEntryCursor = checkpoint.processedEntryCursor ?? null;
+            this.index = checkpoint.index ?? [];
+            this.nextChangeSeq = checkpoint.nextChangeSeq ?? 1;
+            if (checkpoint.processedEntryIds) {
+                this.processedEntryIds = checkpoint.processedEntryIds;
+            }
+            // Rebuild indexLookup from index
+            this.indexLookup.clear();
+            for (let i = 0; i < this.index.length; i++) {
+                if (typeof this.index[i].changeSeq !== "number") {
+                    this.index[i].changeSeq = i + 1;
+                }
+                this.indexLookup.set(this.index[i].docId, i);
+            }
+            if (this.index.length > 0 && this.nextChangeSeq <= this.index.length) {
+                this.nextChangeSeq = Math.max(...this.index.map(e => e.changeSeq)) + 1;
+            }
+            // Restore automergeHashToEntryId
+            if (checkpoint.automergeHashToEntryId) {
+                this.automergeHashToEntryId.clear();
+                for (const [docId, inner] of Object.entries(checkpoint.automergeHashToEntryId)) {
+                    this.automergeHashToEntryId.set(docId, new Map(Object.entries(inner)));
+                }
+            }
+            // 3. Load cached documents
+            const docIds = await store.list("doc");
+            const docPrefix = `${prefix}/`;
+            let restoredDocs = 0;
+            for (const id of docIds) {
+                if (restoredDocs >= this.cacheRestoreLimit) {
+                    break;
+                }
+                if (!id.startsWith(docPrefix))
+                    continue;
+                const docId = id.slice(docPrefix.length);
+                const docBytes = await store.get("doc", id);
+                if (!docBytes)
+                    continue;
+                try {
+                    const internal = this.deserializeDoc(docBytes);
+                    this.storeCachedDocument(internal);
+                    restoredDocs++;
+                }
+                catch (e) {
+                    this.logger.warn(`Failed to restore cached doc ${docId}, will reload from store: ${e}`);
+                }
+            }
+            this.logger.info(`Restored ${restoredDocs} cached documents for ${this.store.getId()}`);
+            this.logger.info(`Restored ${restoredDocs} documents from cache for ${prefix}`);
+            return true;
+        }
+        catch (e) {
+            this.logger.warn(`Cache restore failed, will do full rebuild: ${e}`);
+            return false;
+        }
+    }
+    deserializeDoc(value) {
+        const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
+        const headerLen = view.getUint32(0, false);
+        const headerBytes = value.slice(4, 4 + headerLen);
+        const amBinary = value.slice(4 + headerLen);
+        const header = JSON.parse(new TextDecoder().decode(headerBytes));
+        const doc = automerge_adapter_1.Automerge.load(amBinary);
+        return {
+            id: header.id,
+            doc,
+            createdAt: header.createdAt,
+            lastModified: header.lastModified,
+            decryptionKeyId: header.decryptionKeyId,
+            isDeleted: header.isDeleted,
+        };
+    }
     /**
      * Get or import a CryptoKey for signature verification, with caching.
      * This avoids re-importing the same public key multiple times.
@@ -113,18 +334,18 @@ class BaseMindooDB {
     /**
      * Compare two index entries for sorting.
      * Returns negative if a < b, positive if a > b, 0 if equal.
-     * Sorts by lastModified first, then by docId for uniqueness.
+     * Sorts by deterministic changeSeq first, then by docId for uniqueness.
      */
     compareIndexEntries(a, b) {
-        if (a.lastModified !== b.lastModified) {
-            return a.lastModified - b.lastModified;
+        if (a.changeSeq !== b.changeSeq) {
+            return a.changeSeq - b.changeSeq;
         }
         return a.docId.localeCompare(b.docId);
     }
     /**
      * Update the index entry for a document.
-     * When a document changes, it's removed from its current position and inserted
-     * at the correct sorted position to maintain order by (lastModified, docId).
+     * When a document changes, it gets a new monotonic sequence and is moved to
+     * maintain order by (changeSeq, docId).
      *
      * Optimized to only update lookup map for entries that actually moved.
      *
@@ -133,12 +354,14 @@ class BaseMindooDB {
      * @param isDeleted Whether the document is deleted
      */
     updateIndex(docId, lastModified, isDeleted) {
-        const newEntry = { docId, lastModified, isDeleted };
+        const startedAt = Date.now();
+        const assignedSeq = this.nextChangeSeq;
+        const newEntry = { docId, changeSeq: assignedSeq, lastModified, isDeleted };
         const existingIndex = this.indexLookup.get(docId);
         // Check if the entry already exists and hasn't changed position
         if (existingIndex !== undefined) {
             const existingEntry = this.index[existingIndex];
-            // If position hasn't changed (same lastModified and docId), no update needed
+            // If only metadata flags stayed identical and caller replays same state, skip.
             if (existingEntry.lastModified === lastModified && existingEntry.isDeleted === isDeleted) {
                 return; // No change needed
             }
@@ -171,11 +394,17 @@ class BaseMindooDB {
         }
         // Insert at the correct position
         this.index.splice(insertIndex, 0, newEntry);
+        this.nextChangeSeq = assignedSeq + 1;
         // Update lookup map for entries from insertion point onwards
         // Only update entries that actually moved (from insertIndex to end)
         for (let i = insertIndex; i < this.index.length; i++) {
             this.indexLookup.set(this.index[i].docId, i);
         }
+        this.performanceCallback?.onIndexUpdate?.({
+            docId,
+            operation: existingIndex === undefined ? "insert" : "update",
+            time: Date.now() - startedAt,
+        });
     }
     /**
      * Register an automerge hash -> entry ID mapping for a document.
@@ -223,24 +452,76 @@ class BaseMindooDB {
     }
     /**
      * Initialize the database instance.
+     * If a cache store is available, attempts to restore from cache first,
+     * then processes only the delta. Falls back to full rebuild on cache miss.
      */
     async initialize() {
         this.logger.info(`Initializing database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
+        const cacheStore = this.cacheManager?.getStore();
+        if (cacheStore) {
+            const restored = await this.restoreFromCache(cacheStore);
+            if (restored) {
+                this.logger.info("Restored from cache, processing delta only");
+                await this.syncStoreChanges();
+                if (this.reconcileRestoredIndexOnInit) {
+                    await this.reconcileRestoredIndexWithStore();
+                }
+                return;
+            }
+        }
         await this.syncStoreChanges();
     }
+    async reconcileRestoredIndexWithStore() {
+        const lifecycleMetadata = await this.scanAllMetadata(this.store);
+        const lifecycleDocIds = Array.from(new Set(lifecycleMetadata
+            .filter((entry) => entry.entryType === "doc_create"
+            || entry.entryType === "doc_change"
+            || entry.entryType === "doc_delete"
+            || entry.entryType === "doc_snapshot")
+            .map((entry) => entry.docId))).sort((left, right) => left.localeCompare(right));
+        const missingDocIds = lifecycleDocIds.filter((docId) => !this.indexLookup.has(docId));
+        if (missingDocIds.length === 0) {
+            return;
+        }
+        this.logger.warn(`Detected stale cache checkpoint for ${this.store.getId()} - ` +
+            `index is missing ${missingDocIds.length} document(s): ${missingDocIds.join(", ")}. ` +
+            `Rebuilding metadata from local store.`);
+        this.index = [];
+        this.indexLookup.clear();
+        this.docCache.clear();
+        this.automergeHashToEntryId.clear();
+        this.processedEntryIds = [];
+        this.processedEntryCursor = null;
+        this.nextChangeSeq = 1;
+        this.cacheMetaDirty = true;
+        this.cacheManager?.markDirty();
+        await this.syncStoreChanges();
+        this.logger.info(`Metadata rebuild complete for ${this.store.getId()} - ` +
+            `index now has ${this.index.length} document(s).`);
+    }
     /**
      * Sync changes from the content-addressed store by finding new entries and processing them.
      * This method can be called multiple times to incrementally sync new entries.
      * On first call (when processedEntryIds is empty), it will process all entries.
      */
     async syncStoreChanges() {
+        const syncStartedAt = Date.now();
         this.logger.debug(`Syncing store changes for database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
         this.logger.debug(`Already processed ${this.processedEntryIds.length} entry IDs`);
         // Find new entries that we haven't processed yet
-        const newEntryMetadata = await this.getNewEntryMetadataForSync();
+        const { entries: newEntryMetadata, nextCursor } = await this.getNewEntryMetadataForSync();
         this.logger.debug(`Found ${newEntryMetadata.length} new entries`);
         if (newEntryMetadata.length === 0) {
+            this.processedEntryCursor = nextCursor;
             this.logger.debug(`No new entries to process`);
+            this.performanceCallback?.onSyncOperation?.({
+                operation: "findNewEntries",
+                time: Date.now() - syncStartedAt,
+                details: {
+                    newEntryCount: 0,
+                    processedEntryCount: this.processedEntryIds.length,
+                },
+            });
             return;
         }
         // Group new entries by document ID
@@ -257,10 +538,11 @@ class BaseMindooDB {
         this.logger.debug(`Processing ${entriesByDoc.size} documents with new entries`);
         // Helper function to process a single document
         const processDocument = async (docId, entryMetadataList) => {
+            const processStartedAt = Date.now();
             try {
                 this.logger.debug(`===== Processing document ${docId} with ${entryMetadataList.length} new entry(s) in syncStoreChanges =====`);
                 // Check if document is cached
-                const cachedDoc = this.docCache.get(docId);
+                const cachedDoc = this.getCachedDocument(docId);
                 let updatedDoc = null;
                 if (cachedDoc) {
                     // Document is cached - try incremental update
@@ -288,16 +570,37 @@ class BaseMindooDB {
                     }
                 }
                 else {
-                    // Document not cached - load from scratch
-                    this.logger.debug(`Document ${docId} not in cache, loading from store`);
-                    updatedDoc = await this.loadDocumentInternal(docId);
-                    if (updatedDoc) {
-                        this.logger.debug(`Successfully loaded document ${docId}, updating index`);
-                        this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted);
-                        this.logger.debug(`Updated index for document ${docId} (lastModified: ${updatedDoc.lastModified}, isDeleted: ${updatedDoc.isDeleted})`);
+                    // Metadata-first startup: avoid full materialization for uncached docs.
+                    // Index based on doc-lifecycle entries (create/change/delete) plus
+                    // doc_snapshot which may be the sole entry after a dense sync.
+                    // Attachment-only batches should not trigger document index updates.
+                    const docLifecycleEntries = entryMetadataList.filter((e) => e.entryType === "doc_create" ||
+                        e.entryType === "doc_change" ||
+                        e.entryType === "doc_delete" ||
+                        e.entryType === "doc_snapshot");
+                    if (docLifecycleEntries.length === 0) {
+                        this.logger.debug(`Skipping metadata-first index for doc ${docId} — no document lifecycle entries (${entryMetadataList.length} attachment/other entries only)`);
                     }
                     else {
-                        this.logger.warn(`Document ${docId} returned null from loadDocumentInternal`);
+                        // Before indexing, verify the user has the decryption key so that
+                        // documents the user cannot access do not appear in getAllDocumentIds.
+                        const representativeEntry = docLifecycleEntries[0];
+                        const keyAvailable = await this.tenant.hasDecryptionKey(representativeEntry.decryptionKeyId);
+                        if (!keyAvailable) {
+                            this.logger.debug(`Skipping metadata-first index for doc ${docId} — decryption key "${representativeEntry.decryptionKeyId}" not available`);
+                        }
+                        else {
+                            const mutationEntries = docLifecycleEntries.filter((e) => e.entryType === "doc_create" ||
+                                e.entryType === "doc_change" ||
+                                e.entryType === "doc_delete");
+                            // Snapshots compact replay history but should not change the user-visible
+                            // modification time when the original create/change/delete entries still exist.
+                            const entriesForLastModified = mutationEntries.length > 0 ? mutationEntries : docLifecycleEntries;
+                            const lastModified = Math.max(...entriesForLastModified.map((e) => e.createdAt));
+                            const isDeleted = mutationEntries.some((e) => e.entryType === "doc_delete");
+                            this.updateIndex(docId, lastModified, isDeleted);
+                            this.logger.debug(`Metadata-first update for uncached doc ${docId} (lastModified: ${lastModified}, isDeleted: ${isDeleted})`);
+                        }
                     }
                 }
             }
@@ -313,31 +616,46 @@ class BaseMindooDB {
                 // Re-throw the error so we can see what's happening in the test
                 throw error;
             }
-        };
-        // Process documents in parallel with concurrency limit (default: 10)
-        const maxConcurrency = 10;
-        const documentEntries = Array.from(entriesByDoc.entries());
-        // Process in batches to limit concurrency
-        const results = [];
-        for (let i = 0; i < documentEntries.length; i += maxConcurrency) {
-            const batch = documentEntries.slice(i, i + maxConcurrency);
-            const batchResults = await Promise.allSettled(batch.map(([docId, entryMetadataList]) => processDocument(docId, entryMetadataList)));
-            results.push(...batchResults);
-        }
-        // Check for errors (excluding SymmetricKeyNotFoundError which is handled gracefully)
-        for (let i = 0; i < results.length; i++) {
-            const result = results[i];
-            if (result.status === 'rejected') {
-                const error = result.reason;
-                if (!(error instanceof errors_1.SymmetricKeyNotFoundError)) {
-                    // Re-throw non-key errors
-                    throw error;
-                }
+            finally {
+                this.performanceCallback?.onSyncOperation?.({
+                    operation: "processDocument",
+                    time: Date.now() - processStartedAt,
+                    details: {
+                        docId,
+                        entryCount: entryMetadataList.length,
+                        cacheHit: this.docCache.has(docId),
+                    },
+                });
             }
+        };
+        const documentEntries = Array.from(entriesByDoc.entries())
+            .sort((a, b) => {
+            const aMinCreatedAt = Math.min(...a[1].map((e) => e.createdAt));
+            const bMinCreatedAt = Math.min(...b[1].map((e) => e.createdAt));
+            if (aMinCreatedAt !== bMinCreatedAt) {
+                return aMinCreatedAt - bMinCreatedAt;
+            }
+            return a[0].localeCompare(b[0]);
+        });
+        // Deterministic sequential processing ensures stable changefeed ordering.
+        for (const [docId, entryMetadataList] of documentEntries) {
+            await processDocument(docId, entryMetadataList);
         }
         // Append new entry IDs to our processed list
         this.processedEntryIds.push(...newEntryMetadata.map(em => em.id));
+        this.processedEntryCursor = nextCursor;
+        this.cacheMetaDirty = true;
+        this.cacheManager?.markDirty();
         this.logger.debug(`Synced ${newEntryMetadata.length} new entries, index now has ${this.index.length} documents`);
+        this.performanceCallback?.onSyncOperation?.({
+            operation: "findNewEntries",
+            time: Date.now() - syncStartedAt,
+            details: {
+                newEntryCount: newEntryMetadata.length,
+                documentCount: entriesByDoc.size,
+                indexSize: this.index.length,
+            },
+        });
     }
     supportsCursorScan(store) {
         return typeof store.scanEntriesSince === "function";
@@ -362,8 +680,22 @@ class BaseMindooDB {
         return all;
     }
     async getNewEntryMetadataForSync() {
+        const startedAt = Date.now();
         if (!this.supportsCursorScan(this.store)) {
-            return this.store.findNewEntries(this.processedEntryIds);
+            const result = await this.store.findNewEntries(this.processedEntryIds);
+            this.performanceCallback?.onSyncOperation?.({
+                operation: "findNewEntries",
+                time: Date.now() - startedAt,
+                details: {
+                    mode: "knownIds",
+                    resultCount: result.length,
+                    processedEntryCount: this.processedEntryIds.length,
+                },
+            });
+            return {
+                entries: result,
+                nextCursor: this.processedEntryCursor,
+            };
         }
         const allNew = [];
         let cursor = this.processedEntryCursor;
@@ -375,68 +707,251 @@ class BaseMindooDB {
                 break;
             }
         }
-        this.processedEntryCursor = cursor;
-        return allNew;
+        this.performanceCallback?.onSyncOperation?.({
+            operation: "findNewEntries",
+            time: Date.now() - startedAt,
+            details: {
+                mode: "cursorScan",
+                resultCount: allNew.length,
+                cursor,
+            },
+        });
+        return {
+            entries: allNew,
+            nextCursor: cursor,
+        };
     }
-    async syncEntriesFromStore(sourceStore, targetStore) {
-        let transferred = 0;
-        let targetBloom = null;
-        if (typeof targetStore.getIdBloomSummary === "function") {
-            try {
-                targetBloom = await targetStore.getIdBloomSummary();
-            }
-            catch (error) {
-                this.logger.warn("Failed to get bloom summary from target store, falling back to exact checks", error);
+    /**
+     * Fetch the target store's bloom filter summary, returning null if
+     * unsupported or on error (callers fall back to exact checks).
+     */
+    async getTargetBloomSummary(targetStore) {
+        if (typeof targetStore.getIdBloomSummary !== "function") {
+            return null;
+        }
+        try {
+            return await targetStore.getIdBloomSummary();
+        }
+        catch (error) {
+            this.logger.warn("Failed to get bloom summary from target store, falling back to exact checks", error);
+            return null;
+        }
+    }
+    /**
+     * From a list of candidate IDs, return only those the target store is
+     * missing.  Uses bloom-filter pre-screening when available, then falls
+     * back to exact `hasEntries` for the uncertain set.
+     */
+    async filterMissingIds(targetStore, candidateIds, bloom) {
+        let definitelyMissing = [];
+        let maybeExisting = candidateIds;
+        if (bloom) {
+            definitelyMissing = [];
+            maybeExisting = [];
+            for (const id of candidateIds) {
+                if ((0, bloom_1.bloomMightContainId)(bloom, id)) {
+                    maybeExisting.push(id);
+                }
+                else {
+                    definitelyMissing.push(id);
+                }
             }
         }
+        let missingIds = definitelyMissing;
+        if (maybeExisting.length > 0) {
+            const existing = await targetStore.hasEntries(maybeExisting);
+            const existingSet = new Set(existing);
+            missingIds = missingIds.concat(maybeExisting.filter((id) => !existingSet.has(id)));
+        }
+        return missingIds;
+    }
+    async syncEntriesFromStore(sourceStore, targetStore, options) {
+        if (options?.mode === "dense") {
+            return this.syncEntriesFromStoreDense(sourceStore, targetStore, options);
+        }
+        let transferred = 0;
+        let scanned = 0;
+        const onProgress = options?.onProgress;
+        const pageSize = options?.pageSize ?? 1000;
+        const targetBloom = await this.getTargetBloomSummary(targetStore);
+        const totalSourceEstimate = targetBloom?.totalIds;
         if (this.supportsCursorScan(sourceStore)) {
+            onProgress?.({
+                phase: 'preparing',
+                message: 'Preparing to sync entries...',
+                transferredEntries: 0,
+                scannedEntries: 0,
+                totalSourceEntries: totalSourceEstimate,
+            });
             let cursor = null;
+            let currentPage = 0;
             while (true) {
-                const page = await sourceStore.scanEntriesSince(cursor, 1000);
+                if (options?.signal?.aborted) {
+                    return { transferred, scanned, cancelled: true };
+                }
+                const page = await sourceStore.scanEntriesSince(cursor, pageSize);
+                currentPage++;
+                scanned += page.entries.length;
                 if (page.entries.length > 0) {
                     const ids = page.entries.map((m) => m.id);
-                    let definitelyMissing = [];
-                    let maybeExisting = ids;
-                    if (targetBloom) {
-                        definitelyMissing = [];
-                        maybeExisting = [];
-                        for (const id of ids) {
-                            if ((0, bloom_1.bloomMightContainId)(targetBloom, id)) {
-                                maybeExisting.push(id);
-                            }
-                            else {
-                                definitelyMissing.push(id);
-                            }
-                        }
-                    }
-                    let missingIds = definitelyMissing;
-                    if (maybeExisting.length > 0) {
-                        const existing = await targetStore.hasEntries(maybeExisting);
-                        const existingSet = new Set(existing);
-                        const maybeMissing = maybeExisting.filter((id) => !existingSet.has(id));
-                        missingIds = missingIds.concat(maybeMissing);
-                    }
+                    const missingIds = await this.filterMissingIds(targetStore, ids, targetBloom);
                     if (missingIds.length > 0) {
                         const missingEntries = await sourceStore.getEntries(missingIds);
                         await targetStore.putEntries(missingEntries);
                         transferred += missingEntries.length;
                     }
                 }
+                onProgress?.({
+                    phase: 'transferring',
+                    message: `Transferred ${transferred} entries (page ${currentPage}, scanned ${scanned})`,
+                    transferredEntries: transferred,
+                    scannedEntries: scanned,
+                    totalSourceEntries: totalSourceEstimate,
+                    currentPage,
+                });
                 cursor = page.nextCursor;
                 if (!page.hasMore) {
                     break;
                 }
             }
-            return transferred;
+            return { transferred, scanned, cancelled: false };
         }
+        onProgress?.({
+            phase: 'preparing',
+            message: 'Finding new entries...',
+            transferredEntries: 0,
+            scannedEntries: 0,
+        });
         const targetIds = await targetStore.getAllIds();
         const sourceNewMetadata = await sourceStore.findNewEntries(targetIds);
         if (sourceNewMetadata.length === 0) {
-            return 0;
+            return { transferred: 0, scanned: 0, cancelled: false };
         }
+        if (options?.signal?.aborted) {
+            return { transferred: 0, scanned: 0, cancelled: true };
+        }
+        onProgress?.({
+            phase: 'transferring',
+            message: `Transferring ${sourceNewMetadata.length} entries...`,
+            transferredEntries: 0,
+            scannedEntries: sourceNewMetadata.length,
+            totalSourceEntries: sourceNewMetadata.length,
+        });
         const sourceEntries = await sourceStore.getEntries(sourceNewMetadata.map((m) => m.id));
         await targetStore.putEntries(sourceEntries);
-        return sourceEntries.length;
+        onProgress?.({
+            phase: 'transferring',
+            message: `Transferred ${sourceEntries.length} entries`,
+            transferredEntries: sourceEntries.length,
+            scannedEntries: sourceNewMetadata.length,
+            totalSourceEntries: sourceNewMetadata.length,
+        });
+        return {
+            transferred: sourceEntries.length,
+            scanned: sourceNewMetadata.length,
+            cancelled: false,
+        };
+    }
+    /**
+     * Dense sync: transfer only the entries required to reconstruct the latest
+     * state of each document, using the batch materialization planner to skip
+     * historical entries already superseded by snapshots.
+     *
+     * Algorithm:
+     * 1. Discover all documents on the source via `doc_create` metadata.
+     * 2. Also fetch `doc_delete` metadata so deletion markers are transferred.
+     * 3. Ask the source's batch planner for the optimal replay set per document.
+     * 4. Merge all entry IDs: doc_create + doc_delete + snapshot + uncovered changes.
+     * 5. Filter out IDs the target already has (bloom + exact check).
+     * 6. Transfer only the missing entries.
+     *
+     * Attachment chunks are intentionally skipped — they are not part of the
+     * Automerge document DAG and can be fetched on demand when accessed.
+     */
+    async syncEntriesFromStoreDense(sourceStore, targetStore, options) {
+        const onProgress = options?.onProgress;
+        onProgress?.({
+            phase: "preparing",
+            message: "Dense sync: discovering documents on source...",
+            transferredEntries: 0,
+            scannedEntries: 0,
+        });
+        if (options?.signal?.aborted) {
+            return { transferred: 0, scanned: 0, cancelled: true };
+        }
+        // ── Phase 1: discover documents ──────────────────────────────────
+        const docCreateEntries = await sourceStore.findEntries("doc_create", null, null);
+        const docDeleteEntries = await sourceStore.findEntries("doc_delete", null, null);
+        const docIds = [...new Set(docCreateEntries.map((e) => e.docId))];
+        this.logger.info(`Dense sync: found ${docIds.length} documents on source`);
+        if (docIds.length === 0) {
+            return { transferred: 0, scanned: 0, cancelled: false };
+        }
+        if (options?.signal?.aborted) {
+            return { transferred: 0, scanned: 0, cancelled: true };
+        }
+        // ── Phase 2: batch plan on source ────────────────────────────────
+        onProgress?.({
+            phase: "planning",
+            message: `Dense sync: computing materialization plans for ${docIds.length} documents...`,
+            transferredEntries: 0,
+            scannedEntries: docIds.length,
+        });
+        const batchPlan = await sourceStore.planDocumentMaterializationBatch(docIds);
+        // ── Phase 3: collect needed entry IDs ────────────────────────────
+        const neededIds = new Set();
+        for (const entry of docCreateEntries) {
+            neededIds.add(entry.id);
+        }
+        for (const entry of docDeleteEntries) {
+            neededIds.add(entry.id);
+        }
+        for (const plan of batchPlan.plans) {
+            if (plan.snapshotEntryId) {
+                neededIds.add(plan.snapshotEntryId);
+            }
+            for (const id of plan.entryIdsToApply) {
+                neededIds.add(id);
+            }
+        }
+        this.logger.info(`Dense sync: planner identified ${neededIds.size} required entries ` +
+            `(from ${docIds.length} documents)`);
+        if (options?.signal?.aborted) {
+            return { transferred: 0, scanned: neededIds.size, cancelled: true };
+        }
+        // ── Phase 4: filter out entries the target already has ───────────
+        const allNeededArray = Array.from(neededIds);
+        const targetBloom = await this.getTargetBloomSummary(targetStore);
+        const missingIds = await this.filterMissingIds(targetStore, allNeededArray, targetBloom);
+        this.logger.info(`Dense sync: ${missingIds.length} entries to transfer ` +
+            `(${allNeededArray.length - missingIds.length} already present)`);
+        if (missingIds.length === 0) {
+            return { transferred: 0, scanned: allNeededArray.length, cancelled: false };
+        }
+        if (options?.signal?.aborted) {
+            return { transferred: 0, scanned: allNeededArray.length, cancelled: true };
+        }
+        // ── Phase 5: transfer missing entries in pages ───────────────────
+        const pageSize = options?.pageSize ?? 500;
+        let transferred = 0;
+        for (let offset = 0; offset < missingIds.length; offset += pageSize) {
+            if (options?.signal?.aborted) {
+                return { transferred, scanned: allNeededArray.length, cancelled: true };
+            }
+            const batch = missingIds.slice(offset, offset + pageSize);
+            const entries = await sourceStore.getEntries(batch);
+            await targetStore.putEntries(entries);
+            transferred += entries.length;
+            onProgress?.({
+                phase: "transferring",
+                message: `Dense sync: transferred ${transferred}/${missingIds.length} entries`,
+                transferredEntries: transferred,
+                scannedEntries: allNeededArray.length,
+                totalSourceEntries: allNeededArray.length,
+            });
+        }
+        this.logger.info(`Dense sync complete: transferred ${transferred} entries`);
+        return { transferred, scanned: allNeededArray.length, cancelled: false };
     }
     getStore() {
         return this.store;
@@ -575,8 +1090,9 @@ class BaseMindooDB {
             isDeleted: false,
         };
         // Update cache and index
-        this.docCache.set(docId, internalDoc);
+        this.storeCachedDocument(internalDoc);
         this.updateIndex(docId, internalDoc.lastModified, false);
+        this.markDocDirty(docId);
         this.logger.info(`Document ${docId} created successfully`);
         this.logger.debug(`Document ${docId} cached and indexed (lastModified: ${internalDoc.lastModified})`);
         return this.wrapDocument(internalDoc);
@@ -592,53 +1108,109 @@ class BaseMindooDB {
         return this.wrapDocument(internalDoc);
     }
     async getDocumentAtTimestamp(docId, timestamp) {
+        const startedAt = Date.now();
         this.logger.debug(`Getting document ${docId} at timestamp ${timestamp}`);
         // Get all entry metadata for this document
         const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
-        // Filter entries up to the timestamp
+        // Filter to document replay metadata plus snapshots up to the timestamp.
+        // Attachment chunks are not part of the Automerge replay DAG and must not
+        // participate in historical materialization.
         const relevantEntries = allEntryMetadata
-            .filter(em => em.createdAt <= timestamp)
+            .filter((em) => em.createdAt <= timestamp
+            && (em.entryType === "doc_create"
+                || em.entryType === "doc_change"
+                || em.entryType === "doc_delete"
+                || em.entryType === "doc_snapshot"))
             .sort((a, b) => a.createdAt - b.createdAt);
         if (relevantEntries.length === 0) {
+            this.performanceCallback?.onHistoryOperation?.({
+                operation: "getDocumentAtTimestamp",
+                docId,
+                time: Date.now() - startedAt,
+                scannedEntries: 0,
+                bounded: false,
+            });
             return null; // Document didn't exist at that time
         }
-        // Load entries
-        const entries = await this.store.getEntries(relevantEntries.map(em => em.id));
-        // Apply changes in order
-        let doc = automerge_adapter_1.Automerge.init();
-        for (const entryData of entries) {
-            // Admin-only mode: only accept entries signed by the admin key
+        const metadataById = new Map(relevantEntries.map((meta) => [meta.id, meta]));
+        const materializationPlan = (0, MaterializationPlanner_1.computeDocumentMaterializationPlan)(docId, relevantEntries);
+        let startFromSnapshot = materializationPlan.snapshotEntryId !== null;
+        const snapshotMeta = materializationPlan.snapshotEntryId
+            ? (metadataById.get(materializationPlan.snapshotEntryId) || null)
+            : null;
+        if (startFromSnapshot && !snapshotMeta) {
+            this.logger.warn(`Planner referenced snapshot ${materializationPlan.snapshotEntryId} not found in metadata for ${docId}; falling back to replay without snapshot`);
+            startFromSnapshot = false;
+        }
+        let doc = undefined;
+        if (startFromSnapshot && snapshotMeta) {
+            const snapshotEntries = await this.store.getEntries([snapshotMeta.id]);
+            if (snapshotEntries.length > 0) {
+                const snapshotData = snapshotEntries[0];
+                let isValid = false;
+                if (this._isAdminOnlyDb && snapshotData.createdByPublicKey !== this.getAdminPublicKey()) {
+                    this.logger.warn(`Admin-only DB: skipping snapshot ${snapshotData.id} not signed by admin key`);
+                }
+                else {
+                    isValid = await this.tenant.verifySignature(snapshotData.encryptedData, snapshotData.signature, snapshotData.createdByPublicKey);
+                }
+                if (!isValid) {
+                    this.logger.warn(`Invalid signature for snapshot ${snapshotData.id}, falling back to replay without snapshot`);
+                    startFromSnapshot = false;
+                }
+                else {
+                    const decryptedSnapshot = await this.tenant.decryptPayload(snapshotData.encryptedData, snapshotData.decryptionKeyId);
+                    doc = automerge_adapter_1.Automerge.load(decryptedSnapshot);
+                    const parsed = (0, idGeneration_1.parseDocEntryId)(snapshotData.id);
+                    if (parsed) {
+                        this.registerAutomergeHashMapping(docId, parsed.automergeHash, snapshotData.id);
+                    }
+                }
+            }
+        }
+        if (!doc) {
+            doc = automerge_adapter_1.Automerge.init();
+        }
+        const entriesToApply = materializationPlan.entryIdsToApply
+            .map((id) => metadataById.get(id))
+            .filter((entry) => entry !== undefined);
+        const loadedEntries = entriesToApply.length > 0
+            ? await this.store.getEntries(entriesToApply.map((entry) => entry.id))
+            : [];
+        const entryById = new Map(loadedEntries.map((entry) => [entry.id, entry]));
+        for (const entryMeta of entriesToApply) {
+            const entryData = entryById.get(entryMeta.id);
+            if (!entryData) {
+                this.logger.warn(`Entry ${entryMeta.id} not found in store, skipping`);
+                continue;
+            }
             if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
                 this.logger.warn(`Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
                 continue;
             }
-            // Verify signature against the encrypted payload (no decryption needed)
-            // We sign the encrypted payload, so anyone can verify signatures without decryption keys
             const isValid = await this.tenant.verifySignature(entryData.encryptedData, entryData.signature, entryData.createdByPublicKey);
             if (!isValid) {
                 this.logger.warn(`Invalid signature for entry ${entryData.id}, skipping`);
                 continue;
             }
-            // Decrypt payload (only after signature verification passes)
             const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
-            // Apply change using loadIncremental - this is the recommended way for binary change data
             doc = automerge_adapter_1.Automerge.loadIncremental(doc, decryptedPayload);
-            // Register the automerge hash -> entry ID mapping for future dependency resolution
             const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
             if (parsed) {
                 this.registerAutomergeHashMapping(docId, parsed.automergeHash, entryData.id);
             }
         }
-        // Check if document was deleted at this timestamp
-        // Look for a "doc_delete" type entry in the relevant entries
-        const deleteEntry = relevantEntries.find(em => em.entryType === "doc_delete" && em.createdAt <= timestamp);
-        const isDeleted = deleteEntry !== undefined;
-        // Find the first entry to get createdAt and decryptionKeyId
-        const firstEntry = relevantEntries.length > 0 ? relevantEntries[0] : null;
-        const createdAt = firstEntry ? firstEntry.createdAt : timestamp;
-        const decryptionKeyId = firstEntry ? firstEntry.decryptionKeyId : "default";
-        // Use delete entry timestamp as lastModified if deleted, otherwise use the requested timestamp
-        const lastModified = isDeleted && deleteEntry ? deleteEntry.createdAt : timestamp;
+        const replayEntries = relevantEntries.filter((entry) => entry.entryType === "doc_create"
+            || entry.entryType === "doc_change"
+            || entry.entryType === "doc_delete");
+        const firstReplayEntry = replayEntries.length > 0 ? replayEntries[0] : null;
+        const lastReplayEntry = replayEntries.length > 0 ? replayEntries[replayEntries.length - 1] : null;
+        const createdAt = firstReplayEntry?.createdAt ?? snapshotMeta?.createdAt ?? timestamp;
+        const decryptionKeyId = firstReplayEntry?.decryptionKeyId
+            ?? snapshotMeta?.decryptionKeyId
+            ?? "default";
+        const isDeleted = lastReplayEntry?.entryType === "doc_delete";
+        const lastModified = lastReplayEntry?.createdAt ?? snapshotMeta?.createdAt ?? timestamp;
         const internalDoc = {
             id: docId,
             doc,
@@ -647,6 +1219,14 @@ class BaseMindooDB {
             decryptionKeyId,
             isDeleted,
         };
+        this.performanceCallback?.onHistoryOperation?.({
+            operation: "getDocumentAtTimestamp",
+            docId,
+            time: Date.now() - startedAt,
+            scannedEntries: relevantEntries.length,
+            returnedEntries: 1,
+            bounded: false,
+        });
         return this.wrapDocument(internalDoc);
     }
     async *iterateDocumentHistory(docId) {
@@ -750,6 +1330,46 @@ class BaseMindooDB {
             }
         }
     }
+    async getDocumentHistoryPage(docId, options) {
+        const startedAt = Date.now();
+        const limit = Math.max(1, Math.floor(options?.limit ?? 100));
+        const offset = Math.max(0, Math.floor(options?.cursor?.offset ?? 0));
+        // This API stays metadata-only so large history views can page cheaply
+        // without reconstructing every historical Automerge document state.
+        const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
+        const relevantEntries = allEntryMetadata
+            .filter((em) => em.entryType === "doc_create" ||
+            em.entryType === "doc_change" ||
+            em.entryType === "doc_delete")
+            .sort((a, b) => a.createdAt - b.createdAt);
+        // Cursor paging is offset-based because the result is a bounded timeline view
+        // over the doc's sorted change metadata, not a resumable store scan cursor.
+        const slice = relevantEntries.slice(offset, offset + limit);
+        const entries = slice.map((entry) => ({
+            entryId: entry.id,
+            entryType: entry.entryType,
+            changeCreatedAt: entry.createdAt,
+            changeCreatedByPublicKey: entry.createdByPublicKey,
+            dependencyIds: [...entry.dependencyIds],
+            isDeleted: entry.entryType === "doc_delete",
+        }));
+        const nextOffset = offset + entries.length;
+        const hasMore = nextOffset < relevantEntries.length;
+        const result = {
+            entries,
+            nextCursor: hasMore ? { offset: nextOffset } : null,
+            hasMore,
+        };
+        this.performanceCallback?.onHistoryOperation?.({
+            operation: "getDocumentHistoryPage",
+            docId,
+            time: Date.now() - startedAt,
+            scannedEntries: relevantEntries.length,
+            returnedEntries: entries.length,
+            bounded: true,
+        });
+        return result;
+    }
     async getAllDocumentIds() {
         // Return all non-deleted document IDs from index
         const docIds = [];
@@ -760,6 +1380,15 @@ class BaseMindooDB {
         }
         return docIds;
     }
+    async getDeletedDocumentIds() {
+        const docIds = [];
+        for (const entry of this.index) {
+            if (entry.isDeleted) {
+                docIds.push(entry.docId);
+            }
+        }
+        return docIds;
+    }
     async getAllDocumentIdsAtTimestamp(timestamp) {
         // findEntries() uses an exclusive upper bound (createdAt < creationDateUntil).
         // Query up to timestamp + 1 so entries created exactly at `timestamp` are included
@@ -836,6 +1465,7 @@ class BaseMindooDB {
         if (!internalDoc || internalDoc.isDeleted) {
             throw new Error(`Document ${docId} not found or already deleted`);
         }
+        const now = Date.now();
         // Create deletion change by clearing all fields from the document
         // This ensures Automerge produces actual change bytes
         // The deletion is also tracked via the "doc_delete" type entry in the append-only store
@@ -846,6 +1476,7 @@ class BaseMindooDB {
             }
             // Mark as deleted for clarity
             doc._deleted = true;
+            doc._lastModified = now;
         });
         // Get the change bytes from the document
         const changeBytes = automerge_adapter_1.Automerge.getLastLocalChange(newDoc);
@@ -885,7 +1516,7 @@ class BaseMindooDB {
             contentHash,
             docId,
             dependencyIds,
-            createdAt: Date.now(),
+            createdAt: now,
             createdByPublicKey,
             decryptionKeyId: internalDoc.decryptionKeyId,
             signature,
@@ -903,9 +1534,10 @@ class BaseMindooDB {
         this.registerAutomergeHashMapping(docId, automergeHash, entryId);
         // Update cache and index
         internalDoc.isDeleted = true;
-        internalDoc.lastModified = Date.now();
-        this.docCache.set(docId, internalDoc);
+        internalDoc.lastModified = now;
+        this.storeCachedDocument(internalDoc);
         this.updateIndex(docId, internalDoc.lastModified, true);
+        this.markDocDirty(docId);
         this.logger.info(`Document ${docId} deleted successfully`);
     }
     async changeDoc(doc, changeFunc) {
@@ -929,7 +1561,7 @@ class BaseMindooDB {
             }
         }
         // Get internal document from cache or load it
-        let internalDoc = this.docCache.get(docId);
+        let internalDoc = this.getCachedDocument(docId);
         if (!internalDoc) {
             this.logger.debug(`Document ${docId} not in cache, loading from store`);
             const loadedDoc = await this.loadDocumentInternal(docId);
@@ -1208,7 +1840,7 @@ class BaseMindooDB {
             contentHash,
             docId,
             dependencyIds,
-            createdAt: Date.now(),
+            createdAt: now,
             createdByPublicKey,
             decryptionKeyId: internalDoc.decryptionKeyId,
             signature,
@@ -1226,98 +1858,242 @@ class BaseMindooDB {
         this.registerAutomergeHashMapping(docId, automergeHash, entryId);
         // Update cache and index
         internalDoc.doc = newDoc;
-        internalDoc.lastModified = Date.now();
-        this.docCache.set(docId, internalDoc);
+        internalDoc.lastModified = now;
+        this.storeCachedDocument(internalDoc);
         this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted);
+        this.markDocDirty(docId);
         this.logger.info(`Document ${docId} ${useCustomKey ? 'changed with custom signing key' : 'changed'} successfully`);
+        // Snapshot creation runs as a best-effort background optimization.
+        await this.maybeWriteSnapshotForDocument(internalDoc, useCustomKey
+            ? { signingKeyPair, signingKeyPassword, createdByPublicKey }
+            : { createdByPublicKey });
     }
-    async *iterateChangesSince(cursor) {
-        // Default to initial cursor if null is provided
-        const actualCursor = cursor ?? { lastModified: 0, docId: "" };
-        this.logger.debug(`Starting iteration from cursor ${JSON.stringify(actualCursor)}`);
-        // Find starting position using binary search
-        // We want to find the first entry that is greater than the cursor
-        let startIndex = 0;
-        if (this.index.length > 0) {
-            let left = 0;
-            let right = this.index.length - 1;
-            while (left <= right) {
-                const mid = Math.floor((left + right) / 2);
-                const entry = this.index[mid];
-                const cmp = this.compareIndexEntries({ docId: actualCursor.docId, lastModified: actualCursor.lastModified }, entry);
-                if (cmp < 0) {
-                    right = mid - 1;
-                    startIndex = mid;
-                }
-                else {
-                    left = mid + 1;
-                    startIndex = mid + 1;
-                }
+    /**
+     * Best-effort snapshot scheduling and writing.
+     * A snapshot is written only when enough replay history has accumulated and
+     * a cooldown window has elapsed, to avoid snapshot churn on hot documents.
+     */
+    async maybeWriteSnapshotForDocument(internalDoc, options) {
+        const docId = internalDoc.id;
+        try {
+            const allMetadata = await this.scanAllMetadata(this.store, { docId });
+            const replayEntries = allMetadata.filter((em) => em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete");
+            const snapshots = allMetadata
+                .filter((em) => em.entryType === "doc_snapshot")
+                .sort((a, b) => b.createdAt - a.createdAt);
+            const latestSnapshot = snapshots[0] || null;
+            const latestSnapshotAt = latestSnapshot?.createdAt ?? 0;
+            const changesSinceSnapshot = replayEntries.filter((em) => em.createdAt > latestSnapshotAt).length;
+            if (changesSinceSnapshot < this.snapshotMinChanges) {
+                return;
+            }
+            if (latestSnapshot &&
+                Date.now() - latestSnapshot.createdAt < this.snapshotCooldownMs) {
+                return;
+            }
+            const headHashes = automerge_adapter_1.Automerge.getHeads(internalDoc.doc);
+            const headEntryIds = this.resolveAutomergeDepsToEntryIds(docId, headHashes);
+            const snapshotBytes = automerge_adapter_1.Automerge.save(internalDoc.doc);
+            const encryptedPayload = await this.tenant.encryptPayload(snapshotBytes, internalDoc.decryptionKeyId);
+            const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedPayload, this.getSubtle());
+            const pseudoSnapshotHash = `snapshot-${(0, uuid_1.v7)()}`;
+            const entryId = await (0, idGeneration_1.generateDocEntryId)(docId, pseudoSnapshotHash, headHashes, this.getSubtle());
+            let signature;
+            if (options.signingKeyPair && options.signingKeyPassword) {
+                signature = await this.tenant.signPayloadWithKey(encryptedPayload, options.signingKeyPair, options.signingKeyPassword);
             }
-            // If we found an exact match, start from the next entry
-            if (startIndex < this.index.length) {
-                const entry = this.index[startIndex];
-                if (entry.lastModified === actualCursor.lastModified && entry.docId === actualCursor.docId) {
-                    startIndex++;
-                }
+            else {
+                signature = await this.tenant.signPayload(encryptedPayload);
             }
+            const snapshotEntry = {
+                entryType: "doc_snapshot",
+                id: entryId,
+                contentHash,
+                docId,
+                dependencyIds: headEntryIds,
+                createdAt: Date.now(),
+                createdByPublicKey: options.createdByPublicKey,
+                decryptionKeyId: internalDoc.decryptionKeyId,
+                snapshotHeadHashes: headHashes,
+                snapshotHeadEntryIds: headEntryIds,
+                signature,
+                originalSize: snapshotBytes.length,
+                encryptedSize: encryptedPayload.length,
+                encryptedData: encryptedPayload,
+            };
+            await this.store.putEntries([snapshotEntry]);
+            this.logger.debug(`Created snapshot for document ${docId} with ${headHashes.length} heads and ${changesSinceSnapshot} changes since previous snapshot`);
+        }
+        catch (error) {
+            this.logger.warn(`Failed to create snapshot for document ${docId}, continuing without snapshot`, error);
         }
-        // Pre-check which documents are in cache to optimize iteration
-        const uncachedDocIds = [];
-        for (let i = startIndex; i < this.index.length; i++) {
-            const entry = this.index[i];
-            if (!this.docCache.has(entry.docId)) {
-                uncachedDocIds.push(entry.docId);
+    }
+    async prefetchIterationWindow(indexSnapshot, startIndex) {
+        if (this.iteratePrefetchWindowDocs <= 0) {
+            return 0;
+        }
+        const docIds = [];
+        const seen = new Set();
+        // Only look ahead a bounded number of uncached docs so iteration does not
+        // materialize the entire remaining changefeed tail up front.
+        for (let i = startIndex; i < indexSnapshot.length && docIds.length < this.iteratePrefetchWindowDocs; i++) {
+            const docId = indexSnapshot[i].docId;
+            if (seen.has(docId) || this.docCache.has(docId)) {
+                continue;
             }
+            seen.add(docId);
+            docIds.push(docId);
+        }
+        if (docIds.length === 0) {
+            return 0;
+        }
+        // Prefetch is opportunistic: iteration can still fall back to on-demand
+        // loads, so individual prefetch failures are logged and ignored here.
+        await Promise.all(docIds.map((docId) => this.loadDocumentInternal(docId).catch((err) => {
+            this.logger.warn(`Failed to prefetch document ${docId}:`, err);
+            return null;
+        })));
+        return docIds.length;
+    }
+    getStartIndexForCursor(indexSnapshot, actualCursor) {
+        let startIndex = 0;
+        if (indexSnapshot.length === 0) {
+            return startIndex;
         }
-        // Prefetch uncached documents in batches (if any)
-        if (uncachedDocIds.length > 0) {
-            this.logger.debug(`Prefetching ${uncachedDocIds.length} uncached documents for iteration`);
-            const prefetchBatchSize = 10;
-            for (let i = 0; i < uncachedDocIds.length; i += prefetchBatchSize) {
-                const batch = uncachedDocIds.slice(i, i + prefetchBatchSize);
-                await Promise.all(batch.map(docId => this.loadDocumentInternal(docId).catch(err => {
-                    this.logger.warn(`Failed to prefetch document ${docId}:`, err);
-                    return null;
-                })));
+        const cursorSeq = typeof actualCursor.changeSeq === "number" ? actualCursor.changeSeq : 0;
+        let left = 0;
+        let right = indexSnapshot.length - 1;
+        while (left <= right) {
+            const mid = Math.floor((left + right) / 2);
+            const entry = indexSnapshot[mid];
+            const cursorComparable = {
+                docId: actualCursor.docId,
+                changeSeq: cursorSeq,
+            };
+            const cmp = this.compareIndexEntries(cursorComparable, entry);
+            if (cmp < 0) {
+                right = mid - 1;
+                startIndex = mid;
+            }
+            else {
+                left = mid + 1;
+                startIndex = mid + 1;
             }
         }
-        // Iterate through the index and yield documents one at a time
-        // Documents should now be in cache from prefetching
-        for (let i = startIndex; i < this.index.length; i++) {
-            const entry = this.index[i];
-            try {
-                this.logger.debug(`Yielding document ${entry.docId} from index (lastModified: ${entry.lastModified}, isDeleted: ${entry.isDeleted})`);
-                // Check cache first (should be there after prefetching)
-                let internalDoc = this.docCache.get(entry.docId) || null;
-                if (!internalDoc) {
-                    // Fallback to loading if not in cache (shouldn't happen after prefetching)
-                    this.logger.debug(`Document ${entry.docId} not in cache, loading from store`);
-                    internalDoc = await this.loadDocumentInternal(entry.docId);
-                }
-                if (!internalDoc) {
-                    this.logger.warn(`Document ${entry.docId} not found, skipping`);
-                    continue;
-                }
-                // Wrap the document (works for both deleted and non-deleted documents)
-                const doc = this.wrapDocument(internalDoc);
-                this.logger.debug(`Successfully loaded document ${entry.docId} (isDeleted: ${doc.isDeleted()})`);
-                // Create cursor for current document
+        if (startIndex < indexSnapshot.length) {
+            const entry = indexSnapshot[startIndex];
+            if (entry.changeSeq === cursorSeq && entry.docId === actualCursor.docId) {
+                startIndex++;
+            }
+        }
+        return startIndex;
+    }
+    async *iterateChangeMetadataSince(cursor) {
+        const startedAt = Date.now();
+        const actualCursor = cursor ?? {
+            changeSeq: 0,
+            lastModified: 0,
+            docId: "",
+        };
+        const indexSnapshot = [...this.index];
+        const startIndex = this.getStartIndexForCursor(indexSnapshot, actualCursor);
+        let yieldedDocuments = 0;
+        try {
+            for (let i = startIndex; i < indexSnapshot.length; i++) {
+                const entry = indexSnapshot[i];
                 const currentCursor = {
+                    changeSeq: entry.changeSeq,
                     lastModified: entry.lastModified,
                     docId: entry.docId,
                 };
-                // Yield immediately - this allows the caller to break early after each document
-                // Deleted documents are included so external indexes can handle deletions
-                yield { doc, cursor: currentCursor };
+                yieldedDocuments++;
+                yield {
+                    docId: entry.docId,
+                    lastModified: entry.lastModified,
+                    isDeleted: entry.isDeleted,
+                    cursor: currentCursor,
+                };
             }
-            catch (error) {
-                this.logger.error(`Error processing document ${entry.docId}:`, error);
-                // Stop processing on error
-                throw error;
+        }
+        finally {
+            this.performanceCallback?.onSyncOperation?.({
+                operation: "iterateChangeMetadataSince",
+                time: Date.now() - startedAt,
+                details: {
+                    yieldedDocuments,
+                    startIndex,
+                },
+            });
+        }
+    }
+    async *iterateChangesSince(cursor) {
+        const startedAt = Date.now();
+        // Default to initial cursor if null is provided.
+        // Prefer deterministic sequence-based cursoring; keep legacy fallback compatibility.
+        const actualCursor = cursor ?? { changeSeq: 0, lastModified: 0, docId: "" };
+        this.logger.debug(`Starting iteration from cursor ${JSON.stringify(actualCursor)}`);
+        // Use a stable snapshot of the index for this generator run so concurrent
+        // updates do not reorder/skip entries while iterating.
+        const indexSnapshot = [...this.index];
+        // Find starting position using binary search
+        // We want to find the first entry that is greater than the cursor.
+        const startIndex = this.getStartIndexForCursor(indexSnapshot, actualCursor);
+        let prefetchedDocuments = await this.prefetchIterationWindow(indexSnapshot, startIndex);
+        let yieldedDocuments = 0;
+        let loadedDocuments = 0;
+        try {
+            // Iterate through the stable snapshot and yield documents one at a time.
+            for (let i = startIndex; i < indexSnapshot.length; i++) {
+                const entry = indexSnapshot[i];
+                try {
+                    this.logger.debug(`Yielding document ${entry.docId} from index (lastModified: ${entry.lastModified}, isDeleted: ${entry.isDeleted})`);
+                    let internalDoc = this.getCachedDocument(entry.docId);
+                    if (!internalDoc) {
+                        this.logger.debug(`Document ${entry.docId} not in cache, loading from store`);
+                        internalDoc = await this.loadDocumentInternal(entry.docId);
+                        if (internalDoc) {
+                            loadedDocuments++;
+                        }
+                    }
+                    if (!internalDoc) {
+                        this.logger.warn(`Document ${entry.docId} not found, skipping`);
+                        continue;
+                    }
+                    // Wrap the document (works for both deleted and non-deleted documents)
+                    const doc = this.wrapDocument(internalDoc);
+                    this.logger.debug(`Successfully loaded document ${entry.docId} (isDeleted: ${doc.isDeleted()})`);
+                    // Create cursor for current document
+                    const currentCursor = {
+                        changeSeq: entry.changeSeq,
+                        lastModified: entry.lastModified,
+                        docId: entry.docId,
+                    };
+                    // Yield immediately - this allows the caller to break early after each document
+                    // Deleted documents are included so external indexes can handle deletions
+                    yieldedDocuments++;
+                    yield { doc, cursor: currentCursor };
+                    prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1);
+                }
+                catch (error) {
+                    this.logger.error(`Error processing document ${entry.docId}:`, error);
+                    // Stop processing on error
+                    throw error;
+                }
             }
         }
-        this.logger.debug(`Iteration completed`);
+        finally {
+            this.logger.debug(`Iteration completed`);
+            this.performanceCallback?.onSyncOperation?.({
+                operation: "iterateChangesSince",
+                time: Date.now() - startedAt,
+                details: {
+                    yieldedDocuments,
+                    prefetchedDocuments,
+                    loadedDocuments,
+                    startIndex,
+                },
+            });
+        }
     }
     /**
      * Incrementally update a cached document with new entries.
@@ -1338,10 +2114,10 @@ class BaseMindooDB {
             this.logger.debug(`No change entries to apply for document ${docId}`);
             return null; // No changes
         }
-        // Sort entries by timestamp
-        entriesToApply.sort((a, b) => a.createdAt - b.createdAt);
+        const entriesById = new Map(entriesToApply.map((entry) => [entry.id, entry]));
+        const orderedEntryIds = (0, MaterializationPlanner_1.topologicalByDependencies)(new Set(entriesToApply.map((entry) => entry.id)), entriesById);
         // Load entries from store
-        const entries = await this.store.getEntries(entriesToApply.map(em => em.id));
+        const entries = await this.store.getEntries(orderedEntryIds);
         // Filter entries for admin-only mode first
         const validEntries = entries.filter(entryData => {
             if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
@@ -1429,9 +2205,9 @@ class BaseMindooDB {
         this.logger.debug(`Document ${docId} changed: heads before=${JSON.stringify(headsBefore)}, after=${JSON.stringify(headsAfter)}`);
         // Update metadata
         const payload = updatedDoc;
-        const lastEntry = entries[entries.length - 1];
+        const lastEntryCreatedAt = entries.reduce((maxCreatedAt, entry) => Math.max(maxCreatedAt, entry.createdAt), cachedDoc.lastModified);
         const lastModified = payload._lastModified ||
-            (lastEntry ? lastEntry.createdAt : cachedDoc.lastModified);
+            lastEntryCreatedAt;
         // Check if document was deleted
         const hasDeleteEntry = newEntryMetadata.some(em => em.entryType === "doc_delete");
         const isDeleted = hasDeleteEntry || cachedDoc.isDeleted;
@@ -1444,57 +2220,113 @@ class BaseMindooDB {
             isDeleted,
         };
         // Update cache
-        this.docCache.set(docId, updatedInternalDoc);
+        this.storeCachedDocument(updatedInternalDoc);
+        this.markDocDirty(docId);
         return updatedInternalDoc;
     }
     /**
      * Internal method to load a document from the content-addressed store
      */
     async loadDocumentInternal(docId) {
+        const startedAt = Date.now();
+        const cacheCheckStartedAt = Date.now();
         // Check cache first
         if (this.docCache.has(docId)) {
             this.logger.debug(`Document ${docId} found in cache, returning cached version`);
-            return this.docCache.get(docId);
+            const cached = this.getCachedDocument(docId);
+            this.performanceCallback?.onDocumentLoad?.({
+                docId,
+                cacheHit: true,
+                metadataEntriesScanned: 0,
+                replayEntriesLoaded: 0,
+                snapshotUsed: false,
+                cacheCheckTime: Date.now() - cacheCheckStartedAt,
+                storeQueryTime: 0,
+                entryLoadTime: 0,
+                signatureVerificationTime: 0,
+                decryptionTime: 0,
+                automergeTime: 0,
+                totalTime: Date.now() - startedAt,
+            });
+            return cached;
         }
         this.logger.debug(`===== Starting to load document ${docId} from store =====`);
+        const cacheCheckTime = Date.now() - cacheCheckStartedAt;
+        let storeQueryTime = 0;
+        let entryLoadTime = 0;
+        let signatureVerificationTime = 0;
+        let decryptionTime = 0;
+        let automergeTime = 0;
         // Get all entry metadata for this document
         // TODO: Implement loading from last snapshot if available
         this.logger.debug(`Getting all entry hashes for document ${docId}`);
+        const storeQueryStartedAt = Date.now();
         const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
+        storeQueryTime += Date.now() - storeQueryStartedAt;
         this.logger.debug(`Found ${allEntryMetadata.length} total entry hashes for document ${docId}`);
         if (allEntryMetadata.length === 0) {
             this.logger.debug(`No entry hashes found for document ${docId}, returning null`);
+            this.performanceCallback?.onDocumentLoad?.({
+                docId,
+                cacheHit: false,
+                metadataEntriesScanned: 0,
+                replayEntriesLoaded: 0,
+                snapshotUsed: false,
+                cacheCheckTime,
+                storeQueryTime,
+                entryLoadTime,
+                signatureVerificationTime,
+                decryptionTime,
+                automergeTime,
+                totalTime: Date.now() - startedAt,
+            });
             return null;
         }
         // Log all entry types
         const entryTypes = allEntryMetadata.map(em => `${em.entryType}@${em.createdAt}`).join(', ');
         this.logger.debug(`Entry types for ${docId}: ${entryTypes}`);
-        // Find the most recent snapshot (if any)
-        const snapshots = allEntryMetadata.filter(em => em.entryType === "doc_snapshot");
-        this.logger.debug(`Found ${snapshots.length} snapshot(s) for document ${docId}`);
-        let startFromSnapshot = false;
-        let snapshotMeta = null;
-        if (snapshots.length > 0) {
-            // Use the most recent snapshot
-            snapshots.sort((a, b) => b.createdAt - a.createdAt);
-            snapshotMeta = snapshots[0];
-            startFromSnapshot = true;
-            this.logger.debug(`Will start from snapshot ${snapshotMeta.id} created at ${snapshotMeta.createdAt}`);
+        const metadataById = new Map(allEntryMetadata.map((meta) => [meta.id, meta]));
+        const planStartedAt = Date.now();
+        const materializationPlan = (0, MaterializationPlanner_1.computeDocumentMaterializationPlan)(docId, allEntryMetadata, {
+            includeDiagnostics: true,
+        });
+        const planTime = Date.now() - planStartedAt;
+        this.performanceCallback?.onSyncOperation?.({
+            operation: "planDocumentMaterialization",
+            time: planTime,
+            details: {
+                docId,
+                metadataEntriesScanned: allEntryMetadata.length,
+                replayEntriesLoaded: materializationPlan.entryIdsToApply.length,
+                snapshotEntryId: materializationPlan.snapshotEntryId,
+                diagnostics: materializationPlan.diagnostics,
+            },
+        });
+        let startFromSnapshot = materializationPlan.snapshotEntryId !== null;
+        const snapshotMeta = materializationPlan.snapshotEntryId
+            ? (metadataById.get(materializationPlan.snapshotEntryId) || null)
+            : null;
+        if (startFromSnapshot && !snapshotMeta) {
+            this.logger.warn(`Planner referenced snapshot ${materializationPlan.snapshotEntryId} not found in metadata for ${docId}; falling back to replay without snapshot`);
+            startFromSnapshot = false;
+        }
+        if (startFromSnapshot && snapshotMeta) {
+            this.logger.debug(`Planner selected snapshot ${snapshotMeta.id} for ${docId}`);
         }
         else {
-            this.logger.debug(`No snapshot found, will start from scratch`);
-        }
-        // Get all entries (excluding snapshot entries - we'll handle delete separately)
-        // Include "doc_create", "doc_change", and "doc_delete" types as they all contain Automerge changes to apply
-        const entriesToLoad = startFromSnapshot
-            ? allEntryMetadata.filter(em => (em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete") && em.createdAt > snapshotMeta.createdAt)
-            : allEntryMetadata.filter(em => em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete");
-        this.logger.debug(`Will load ${entriesToLoad.length} entries for document ${docId} (after snapshot filter)`);
+            this.logger.debug(`Planner did not select a snapshot for ${docId}`);
+        }
+        const entriesToLoad = materializationPlan.entryIdsToApply
+            .map((id) => metadataById.get(id))
+            .filter((m) => m !== undefined);
+        this.logger.debug(`Planner returned ${entriesToLoad.length} replay entries for ${docId}; diagnostics=${JSON.stringify(materializationPlan.diagnostics || {})}`);
         // Load the snapshot first if we have one
         let doc = undefined;
         if (startFromSnapshot && snapshotMeta) {
             this.logger.debug(`Loading snapshot for document ${docId}`);
+            const snapshotLoadStartedAt = Date.now();
             const snapshotEntries = await this.store.getEntries([snapshotMeta.id]);
+            entryLoadTime += Date.now() - snapshotLoadStartedAt;
             this.logger.debug(`Retrieved ${snapshotEntries.length} snapshot entry(s) from store`);
             if (snapshotEntries.length > 0) {
                 const snapshotData = snapshotEntries[0];
@@ -1506,7 +2338,9 @@ class BaseMindooDB {
                 else {
                     // Verify signature against the encrypted snapshot (no decryption needed)
                     // We sign the encrypted payload, so anyone can verify signatures without decryption keys
+                    const signatureStartedAt = Date.now();
                     isValid = await this.tenant.verifySignature(snapshotData.encryptedData, snapshotData.signature, snapshotData.createdByPublicKey);
+                    signatureVerificationTime += Date.now() - signatureStartedAt;
                 }
                 if (!isValid) {
                     this.logger.warn(`Invalid signature for snapshot ${snapshotData.id}, falling back to loading from scratch`);
@@ -1516,13 +2350,17 @@ class BaseMindooDB {
                 else {
                     this.logger.debug(`Snapshot signature valid, decrypting snapshot`);
                     // Decrypt snapshot (only after signature verification passes)
+                    const decryptStartedAt = Date.now();
                     const decryptedSnapshot = await this.tenant.decryptPayload(snapshotData.encryptedData, snapshotData.decryptionKeyId);
+                    decryptionTime += Date.now() - decryptStartedAt;
                     this.logger.debug(`Decrypted snapshot (${snapshotData.encryptedData.length} -> ${decryptedSnapshot.length} bytes)`);
                     // Load snapshot using Automerge.load()
                     // This deserializes a full document snapshot from binary data
                     // According to Automerge docs: load() is equivalent to init() followed by loadIncremental()
                     this.logger.debug(`Loading snapshot into Automerge document`);
+                    const automergeStartedAt = Date.now();
                     doc = automerge_adapter_1.Automerge.load(decryptedSnapshot);
+                    automergeTime += Date.now() - automergeStartedAt;
                     this.logger.debug(`Successfully loaded snapshot, document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
                     // Register the snapshot's automerge hash -> entry ID mapping
                     const parsed = (0, idGeneration_1.parseDocEntryId)(snapshotData.id);
@@ -1538,12 +2376,11 @@ class BaseMindooDB {
             doc = automerge_adapter_1.Automerge.init();
             this.logger.debug(`Initialized empty document, heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
         }
-        // Sort entries by timestamp
-        entriesToLoad.sort((a, b) => a.createdAt - b.createdAt);
-        this.logger.debug(`Sorted ${entriesToLoad.length} entries by timestamp for document ${docId}`);
         // Load and apply all entries
         this.logger.debug(`Fetching ${entriesToLoad.length} entries from store for document ${docId}`);
+        const entryLoadStartedAt = Date.now();
         const entries = await this.store.getEntries(entriesToLoad.map(em => em.id));
+        entryLoadTime += Date.now() - entryLoadStartedAt;
         this.logger.debug(`Retrieved ${entries.length} entries from store for document ${docId}`);
         this.logger.debug(`Loading document ${docId}: found ${entries.length} entries to apply (${startFromSnapshot ? 'starting from snapshot' : 'starting from scratch'})`);
         // Log current document state before applying entries
@@ -1583,6 +2420,7 @@ class BaseMindooDB {
                 }
             }
             // Verify all signatures in parallel using cached keys
+            const signatureStartedAt = Date.now();
             const signatureVerificationResults = await Promise.all(validEntries.map(async (entryData) => {
                 const cryptoKey = keyMap.get(entryData.createdByPublicKey);
                 if (!cryptoKey) {
@@ -1592,6 +2430,7 @@ class BaseMindooDB {
                 const isValid = await this.verifySignatureWithKey(cryptoKey, entryData.encryptedData, entryData.signature);
                 return { entryData, isValid };
             }));
+            signatureVerificationTime += Date.now() - signatureStartedAt;
             // Filter out entries with invalid signatures
             const verifiedEntries = signatureVerificationResults
                 .filter(({ isValid }) => {
@@ -1608,10 +2447,12 @@ class BaseMindooDB {
                 // Parallel decryption - decrypt all entries concurrently
                 // Automerge handles dependency buffering internally, so we can decrypt all in parallel
                 this.logger.debug(`Decrypting ${verifiedEntries.length} entries in parallel`);
+                const decryptionStartedAt = Date.now();
                 const decryptionResults = await Promise.all(verifiedEntries.map(async (entryData) => {
                     const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
                     return { entryData, decryptedPayload };
                 }));
+                decryptionTime += Date.now() - decryptionStartedAt;
                 // Collect change bytes and register automerge hash mappings
                 const changeBytes = [];
                 for (const { entryData, decryptedPayload } of decryptionResults) {
@@ -1626,8 +2467,10 @@ class BaseMindooDB {
                 if (changeBytes.length > 0) {
                     this.logger.debug(`Applying ${changeBytes.length} changes to document ${docId} using batch applyChanges`);
                     try {
+                        const automergeStartedAt = Date.now();
                         const result = automerge_adapter_1.Automerge.applyChanges(doc, changeBytes);
                         doc = result[0];
+                        automergeTime += Date.now() - automergeStartedAt;
                         this.logger.debug(`Successfully applied ${changeBytes.length} changes to document ${docId}`);
                         this.logger.debug(`Document state after applying changes: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
                     }
@@ -1647,7 +2490,10 @@ class BaseMindooDB {
         const hasDeleteEntry = allEntryMetadata.some(em => em.entryType === "doc_delete");
         const isDeleted = hasDeleteEntry;
         this.logger.debug(`Document ${docId} isDeleted: ${isDeleted}`);
-        const decryptionKeyId = payload._decryptionKeyId || "default";
+        // The authoritative decryptionKeyId comes from the doc_create entry's metadata,
+        // not from the Automerge payload (which does not store encryption metadata).
+        const createEntry = allEntryMetadata.find(em => em.entryType === "doc_create");
+        const decryptionKeyId = createEntry ? createEntry.decryptionKeyId : "default";
         // Get lastModified from payload, or use the timestamp of the last entry
         const lastEntry = entries.length > 0 ? entries[entries.length - 1] : null;
         const lastModified = payload._lastModified ||
@@ -1665,8 +2511,23 @@ class BaseMindooDB {
             isDeleted,
         };
         // Update cache
-        this.docCache.set(docId, internalDoc);
+        this.storeCachedDocument(internalDoc);
+        this.markDocDirty(docId);
         this.logger.debug(`===== Successfully loaded document ${docId} and cached it =====`);
+        this.performanceCallback?.onDocumentLoad?.({
+            docId,
+            cacheHit: false,
+            metadataEntriesScanned: allEntryMetadata.length,
+            replayEntriesLoaded: entriesToLoad.length,
+            snapshotUsed: startFromSnapshot && snapshotMeta !== null,
+            cacheCheckTime,
+            storeQueryTime,
+            entryLoadTime,
+            signatureVerificationTime,
+            decryptionTime,
+            automergeTime,
+            totalTime: Date.now() - startedAt,
+        });
         return internalDoc;
     }
     /**
@@ -1785,7 +2646,7 @@ class BaseMindooDB {
      * Get an attachment reference by ID from a document's _attachments array.
      */
     getAttachmentRefInternal(docId, attachmentId) {
-        const internalDoc = this.docCache.get(docId);
+        const internalDoc = this.getCachedDocument(docId);
         if (!internalDoc) {
             throw new Error(`Document ${docId} not found in cache`);
         }
@@ -1797,6 +2658,39 @@ class BaseMindooDB {
         }
         return ref;
     }
+    async planAttachmentRead(store, ref, startByte, endByteExclusive) {
+        if (store.planAttachmentReadByWalkingMetadata) {
+            try {
+                // if the store has a planAttachmentReadByWalkingMetadata method, use it for less overhead through the network
+                return await store.planAttachmentReadByWalkingMetadata(ref.lastChunkId, ref.size, {
+                    startByte,
+                    endByteExclusive,
+                });
+            }
+            catch (error) {
+                this.logger.debug("Store-level attachment read planner failed, falling back to local metadata walk", { attachmentId: ref.attachmentId, error });
+            }
+        }
+        // fall back to local metadata walk
+        return (0, AttachmentReadPlanner_1.planAttachmentReadByWalkingMetadata)(store, ref.lastChunkId, ref.size, {
+            startByte,
+            endByteExclusive,
+        });
+    }
+    async decryptAttachmentChunk(chunk) {
+        if (this._isAdminOnlyDb && chunk.createdByPublicKey !== this.getAdminPublicKey()) {
+            throw new Error(`Admin-only DB: chunk ${chunk.id} not signed by admin key`);
+        }
+        const isValid = await this.tenant.verifySignature(chunk.encryptedData, chunk.signature, chunk.createdByPublicKey);
+        if (!isValid) {
+            throw new Error(`Invalid signature for chunk ${chunk.id}`);
+        }
+        const plaintext = await this.tenant.decryptAttachmentPayload(chunk.encryptedData, chunk.decryptionKeyId);
+        if (plaintext.length !== chunk.originalSize) {
+            throw new Error(`Attachment chunk ${chunk.id} decrypted to ${plaintext.length} bytes, expected ${chunk.originalSize}`);
+        }
+        return plaintext;
+    }
     /**
      * Internal method to fetch and concatenate all chunks for an attachment.
      */
@@ -1813,17 +2707,7 @@ class BaseMindooDB {
         const plaintextChunks = [];
         let totalSize = 0;
         for (const chunk of chunks) {
-            // Admin-only mode: only accept chunks signed by the admin key
-            if (this._isAdminOnlyDb && chunk.createdByPublicKey !== this.getAdminPublicKey()) {
-                throw new Error(`Admin-only DB: chunk ${chunk.id} not signed by admin key`);
-            }
-            // Verify signature
-            const isValid = await this.tenant.verifySignature(chunk.encryptedData, chunk.signature, chunk.createdByPublicKey);
-            if (!isValid) {
-                throw new Error(`Invalid signature for chunk ${chunk.id}`);
-            }
-            // Decrypt
-            const plaintext = await this.tenant.decryptAttachmentPayload(chunk.encryptedData, chunk.decryptionKeyId);
+            const plaintext = await this.decryptAttachmentChunk(chunk);
             plaintextChunks.push(plaintext);
             totalSize += plaintext.length;
         }
@@ -1846,37 +2730,16 @@ class BaseMindooDB {
             throw new Error(`Invalid byte range: [${startByte}, ${endByte})`);
         }
         const ref = this.getAttachmentRefInternal(docId, attachmentId);
-        if (endByte > ref.size) {
-            throw new Error(`End byte ${endByte} exceeds attachment size ${ref.size}`);
-        }
         const store = this.getEffectiveAttachmentStore();
-        const chunkSize = this.chunkSizeBytes;
-        // Calculate which chunks we need
-        const startChunkIndex = Math.floor(startByte / chunkSize);
-        const endChunkIndex = Math.floor((endByte - 1) / chunkSize);
-        // Resolve dependency chain to get all chunk IDs
-        const allChunkIds = await store.resolveDependencies(ref.lastChunkId, { includeStart: true });
-        // Get only the chunks we need
-        const neededChunkIds = allChunkIds.slice(startChunkIndex, endChunkIndex + 1);
+        const readPlan = await this.planAttachmentRead(store, ref, startByte, endByte);
+        const neededChunkIds = readPlan.chunkPlans.map((chunkPlan) => chunkPlan.id);
         const chunks = await store.getEntries(neededChunkIds);
         // Decrypt needed chunks
         const plaintextChunks = [];
         for (const chunk of chunks) {
-            // Admin-only mode: only accept chunks signed by the admin key
-            if (this._isAdminOnlyDb && chunk.createdByPublicKey !== this.getAdminPublicKey()) {
-                throw new Error(`Admin-only DB: chunk ${chunk.id} not signed by admin key`);
-            }
-            // Verify signature
-            const isValid = await this.tenant.verifySignature(chunk.encryptedData, chunk.signature, chunk.createdByPublicKey);
-            if (!isValid) {
-                throw new Error(`Invalid signature for chunk ${chunk.id}`);
-            }
-            // Decrypt
-            const plaintext = await this.tenant.decryptAttachmentPayload(chunk.encryptedData, chunk.decryptionKeyId);
+            const plaintext = await this.decryptAttachmentChunk(chunk);
             plaintextChunks.push(plaintext);
         }
-        // Calculate offsets within the fetched chunks
-        const offsetInFirstChunk = startByte - (startChunkIndex * chunkSize);
         const totalNeededBytes = endByte - startByte;
         // Extract the requested range
         const result = new Uint8Array(totalNeededBytes);
@@ -1884,7 +2747,7 @@ class BaseMindooDB {
         let bytesRemaining = totalNeededBytes;
         for (let i = 0; i < plaintextChunks.length && bytesRemaining > 0; i++) {
             const chunk = plaintextChunks[i];
-            const chunkStart = i === 0 ? offsetInFirstChunk : 0;
+            const chunkStart = i === 0 ? readPlan.offsetInFirstChunk : 0;
             const bytesToCopy = Math.min(chunk.length - chunkStart, bytesRemaining);
             result.set(chunk.slice(chunkStart, chunkStart + bytesToCopy), resultOffset);
             resultOffset += bytesToCopy;
@@ -1899,30 +2762,25 @@ class BaseMindooDB {
     async *streamAttachmentInternal(docId, attachmentId, startOffset) {
         this.logger.debug(`Streaming attachment ${attachmentId} from offset ${startOffset}`);
         const ref = this.getAttachmentRefInternal(docId, attachmentId);
+        if (startOffset < 0 || startOffset > ref.size) {
+            throw new Error(`Invalid stream offset ${startOffset} for attachment size ${ref.size}`);
+        }
+        if (ref.size === 0 || startOffset === ref.size) {
+            return;
+        }
         const store = this.getEffectiveAttachmentStore();
-        const chunkSize = this.chunkSizeBytes;
-        // Calculate starting chunk
-        const startChunkIndex = Math.floor(startOffset / chunkSize);
-        const offsetInStartChunk = startOffset % chunkSize;
-        // Resolve dependency chain to get all chunk IDs
-        const allChunkIds = await store.resolveDependencies(ref.lastChunkId, { includeStart: true });
+        const readPlan = await this.planAttachmentRead(store, ref, startOffset, ref.size);
         // Stream chunks starting from startChunkIndex
-        for (let i = startChunkIndex; i < allChunkIds.length; i++) {
-            const [chunk] = await store.getEntries([allChunkIds[i]]);
-            // Admin-only mode: only accept chunks signed by the admin key
-            if (this._isAdminOnlyDb && chunk.createdByPublicKey !== this.getAdminPublicKey()) {
-                throw new Error(`Admin-only DB: chunk ${chunk.id} not signed by admin key`);
+        for (let i = 0; i < readPlan.chunkPlans.length; i++) {
+            const chunkPlan = readPlan.chunkPlans[i];
+            const [chunk] = await store.getEntries([chunkPlan.id]);
+            if (!chunk) {
+                throw new Error(`Attachment chunk ${chunkPlan.id} not found in store`);
             }
-            // Verify signature
-            const isValid = await this.tenant.verifySignature(chunk.encryptedData, chunk.signature, chunk.createdByPublicKey);
-            if (!isValid) {
-                throw new Error(`Invalid signature for chunk ${chunk.id}`);
-            }
-            // Decrypt
-            const plaintext = await this.tenant.decryptAttachmentPayload(chunk.encryptedData, chunk.decryptionKeyId);
+            const plaintext = await this.decryptAttachmentChunk(chunk);
             // For first chunk, skip bytes before startOffset
-            if (i === startChunkIndex && offsetInStartChunk > 0) {
-                yield plaintext.slice(offsetInStartChunk);
+            if (i === 0 && readPlan.offsetInFirstChunk > 0) {
+                yield plaintext.slice(readPlan.offsetInFirstChunk);
             }
             else {
                 yield plaintext;
@@ -2126,7 +2984,64 @@ class BaseMindooDB {
         };
     }
     /**
-     * Pull changes from a remote content-addressed store.
+     * Resolve a sync target to a ContentAddressedStore.
+     * Accepts either a raw store or a MindooDB instance (calls getStore()).
+     */
+    resolveStore(remote) {
+        if ('getStore' in remote && typeof remote.getStore === 'function') {
+            return remote.getStore();
+        }
+        return remote;
+    }
+    /**
+     * Apply a temporary network-auth identity override for a single sync call.
+     *
+     * Purpose:
+     * - Allows per-call authentication as a different user (for example admin bootstrap)
+     *   without changing the tenant's default connected user.
+     * - Returns a cleanup callback so caller can always restore default auth state in `finally`.
+     */
+    async applyNetworkAuthOverrideForSync(remoteStore, options) {
+        // Per-call override: use an alternate identity (for example, admin bootstrap)
+        // only for this sync operation.
+        const override = options?.networkAuthOverride;
+        if (!override) {
+            // No override requested: return a no-op cleanup callback for unified call sites.
+            return () => { };
+        }
+        const overrideCapableStore = remoteStore;
+        if (typeof overrideCapableStore.setSyncAuthOverride !== "function") {
+            // Local/in-memory stores do not support network auth override; keep default auth.
+            this.logger.warn("networkAuthOverride was provided, but remote store does not support auth override");
+            return () => { };
+        }
+        const subtle = this.tenant.getCryptoAdapter().getSubtle();
+        // Decrypt and import override signing key (Ed25519) for challenge signing.
+        const signingKeyBuffer = await this.tenant.decryptPrivateKey(override.user.userSigningKeyPair.privateKey, override.password, "signing");
+        const signingKey = await subtle.importKey("pkcs8", signingKeyBuffer, { name: "Ed25519" }, false, ["sign"]);
+        // Decrypt and import override encryption key (RSA-OAEP) so encrypted network
+        // entries can be decrypted with the same override identity.
+        const encryptionKeyBuffer = await this.tenant.decryptPrivateKey(override.user.userEncryptionKeyPair.privateKey, override.password, "encryption");
+        const encryptionKey = await subtle.importKey("pkcs8", encryptionKeyBuffer, { name: "RSA-OAEP", hash: "SHA-256" }, false, ["decrypt"]);
+        // Activate temporary override on the remote store.
+        overrideCapableStore.setSyncAuthOverride({
+            username: override.user.username,
+            signingKey,
+            privateEncryptionKey: encryptionKey,
+        });
+        // Return cleanup callback so caller can always restore normal auth in finally.
+        return () => {
+            if (typeof overrideCapableStore.clearSyncAuthOverride === "function") {
+                overrideCapableStore.clearSyncAuthOverride();
+            }
+            else {
+                // Backward-compatible fallback if explicit clear API is not implemented.
+                overrideCapableStore.setSyncAuthOverride?.(null);
+            }
+        };
+    }
+    /**
+     * Pull changes from a remote content-addressed store or another MindooDB instance.
      *
      * This method:
      * 1. Finds entries in the remote store that we don't have locally
@@ -2134,48 +3049,109 @@ class BaseMindooDB {
      * 3. Stores them in our local store
      * 4. Syncs the local store to process the new entries
      *
-     * @param remoteStore The remote store to pull entries from
-     * @return A promise that resolves when the pull is complete
+     * @param remote The remote store or MindooDB instance to pull entries from
+     * @param options Optional sync options for progress tracking, paging, and cancellation
+     * @return A promise that resolves with the sync result
      */
-    async pullChangesFrom(remoteStore) {
+    async pullChangesFrom(remote, options) {
+        const remoteStore = this.resolveStore(remote);
         if (this.store.getId() !== remoteStore.getId()) {
             throw new Error(`[BaseMindooDB] Cannot pull entries from the incompatible store ${this.store.getId()}`);
         }
         this.logger.info(`Pulling entries from remote store ${remoteStore.getId()}`);
-        const transferred = await this.syncEntriesFromStore(remoteStore, this.store);
-        this.logger.debug(`Transferred ${transferred} entries from remote store`);
-        if (transferred === 0) {
-            this.logger.debug(`No new entries to pull`);
-            return;
+        const restoreAuthOverride = await this.applyNetworkAuthOverrideForSync(remoteStore, options);
+        try {
+            const syncResult = await this.syncEntriesFromStore(remoteStore, this.store, options);
+            this.logger.debug(`Transferred ${syncResult.transferred} entries from remote store`);
+            if (syncResult.cancelled) {
+                this.logger.info(`Pull cancelled after transferring ${syncResult.transferred} entries`);
+                return {
+                    transferredEntries: syncResult.transferred,
+                    scannedEntries: syncResult.scanned,
+                    cancelled: true,
+                };
+            }
+            if (syncResult.transferred === 0) {
+                this.logger.debug(`No new entries to pull`);
+                return { transferredEntries: 0, scannedEntries: syncResult.scanned, cancelled: false };
+            }
+            options?.onProgress?.({
+                phase: 'processing',
+                message: `Processing ${syncResult.transferred} new entries...`,
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.transferred,
+            });
+            // Sync the local store to process the new entries
+            // This will update the index, cache, and processedEntryIds
+            await this.syncStoreChanges();
+            this.logger.info(`Pull complete, synced ${syncResult.transferred} entries`);
+            options?.onProgress?.({
+                phase: 'complete',
+                message: `Pull complete: ${syncResult.transferred} entries synced`,
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.transferred,
+            });
+            return {
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
+                cancelled: false,
+            };
+        }
+        finally {
+            restoreAuthOverride();
         }
-        // Sync the local store to process the new entries
-        // This will update the index, cache, and processedEntryIds
-        await this.syncStoreChanges();
-        this.logger.info(`Pull complete, synced ${transferred} entries`);
     }
     /**
-     * Push changes to a remote content-addressed store.
+     * Push changes to a remote content-addressed store or another MindooDB instance.
      *
      * This method:
      * 1. Finds entries in our local store that the remote doesn't have
      * 2. Retrieves those entries from our local store
      * 3. Stores them in the remote store
      *
-     * @param remoteStore The remote store to push entries to
-     * @return A promise that resolves when the push is complete
+     * @param remote The remote store or MindooDB instance to push entries to
+     * @param options Optional sync options for progress tracking, paging, and cancellation
+     * @return A promise that resolves with the sync result
      */
-    async pushChangesTo(remoteStore) {
+    async pushChangesTo(remote, options) {
+        const remoteStore = this.resolveStore(remote);
         if (this.store.getId() !== remoteStore.getId()) {
             throw new Error(`[BaseMindooDB] Cannot push entries to the incompatible store ${this.store.getId()}`);
         }
         this.logger.info(`Pushing entries to remote store ${remoteStore.getId()}`);
-        const transferred = await this.syncEntriesFromStore(this.store, remoteStore);
-        this.logger.debug(`Transferred ${transferred} entries to remote store`);
-        if (transferred === 0) {
-            this.logger.debug(`No new entries to push`);
-            return;
+        const restoreAuthOverride = await this.applyNetworkAuthOverrideForSync(remoteStore, options);
+        try {
+            const syncResult = await this.syncEntriesFromStore(this.store, remoteStore, options);
+            this.logger.debug(`Transferred ${syncResult.transferred} entries to remote store`);
+            if (syncResult.cancelled) {
+                this.logger.info(`Push cancelled after transferring ${syncResult.transferred} entries`);
+                return {
+                    transferredEntries: syncResult.transferred,
+                    scannedEntries: syncResult.scanned,
+                    cancelled: true,
+                };
+            }
+            if (syncResult.transferred === 0) {
+                this.logger.debug(`No new entries to push`);
+            }
+            else {
+                this.logger.info(`Pushed ${syncResult.transferred} entries to remote store`);
+            }
+            options?.onProgress?.({
+                phase: 'complete',
+                message: `Push complete: ${syncResult.transferred} entries transferred`,
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.transferred,
+            });
+            return {
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
+                cancelled: false,
+            };
+        }
+        finally {
+            restoreAuthOverride();
         }
-        this.logger.info(`Pushed ${transferred} entries to remote store`);
     }
 }
 exports.BaseMindooDB = BaseMindooDB;