mindoodb 0.0.2 → 0.0.4

package/dist/core/BaseMindooDB.js

@@ -1,41 +1,13 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BaseMindooDB = void 0;
-const Automerge = __importStar(require("@automerge/automerge"));
+// Import platform-appropriate Automerge implementation
+// React Native: native Rust (react-native-automerge-generated)
+// Browser/Node.js: WASM (@automerge/automerge/slim)
+const automerge_adapter_1 = require("./automerge-adapter");
 const uuid_1 = require("uuid");
+const bloom_1 = require("./appendonlystores/bloom");
+const MaterializationPlanner_1 = require("./appendonlystores/MaterializationPlanner");
 const idGeneration_1 = require("./utils/idGeneration");
 const errors_1 = require("./errors");
 const logging_1 = require("./logging");
@@ -43,6 +15,10 @@ const logging_1 = require("./logging");
  * Default chunk size for attachments: 256KB
  */
 const DEFAULT_CHUNK_SIZE_BYTES = 256 * 1024;
+const DEFAULT_SNAPSHOT_MIN_CHANGES = 100;
+const DEFAULT_SNAPSHOT_COOLDOWN_MS = 10 * 60 * 1000;
+const DEFAULT_MAX_CACHED_DOCS = 128;
+const DEFAULT_ITERATE_PREFETCH_WINDOW_DOCS = 0;
 /**
  * BaseMindooDB is a platform-agnostic implementation of MindooDB
  * that works in both browser and server environments.
@@ -63,27 +39,40 @@ const DEFAULT_CHUNK_SIZE_BYTES = 256 * 1024;
  * - Automerge.applyChanges(doc, changes) - verify method signature
  */
 class BaseMindooDB {
-    constructor(tenant, store, attachmentStore, attachmentConfig, adminOnlyDb = false, logger, performanceCallback) {
-        // Internal index: sorted array of document entries, maintained in order by (lastModified, docId)
-        // This allows efficient incremental processing without sorting on each call
+    constructor(tenant, store, attachmentStore, attachmentConfig, documentCacheConfig, snapshotConfig, adminOnlyDb = false, logger, performanceCallback) {
+        // Internal changefeed index: sorted by (changeSeq, docId) for deterministic iteration.
+        // lastModified remains available for UX metadata but is not the primary cursor key.
         this.index = [];
         // Lookup map for O(1) access to index entries by docId
         this.indexLookup = new Map(); // Map<docId, arrayIndex>
+        this.nextChangeSeq = 1;
         // Cache of loaded documents: Map<docId, InternalDoc>
         this.docCache = new Map();
         // Track which entry IDs we've already processed
         this.processedEntryIds = [];
+        this.processedEntryCursor = null;
         // Index: automergeHash -> entryId for each document
         // Used for resolving Automerge dependency hashes to entry IDs
         this.automergeHashToEntryId = new Map(); // Map<docId, Map<automergeHash, entryId>>
         // Cache for imported public keys (CryptoKey objects) to avoid re-importing the same key
         // Map<publicKeyPEM, CryptoKey>
         this.publicKeyCache = new Map();
+        // Local cache support
+        this.cacheManager = null;
+        this.cachePrefix = null;
+        this.dirtyDocIds = new Set();
+        this.cacheMetaDirty = false;
         this.tenant = tenant;
         this.store = store;
         this.attachmentStore = attachmentStore;
         this.chunkSizeBytes = attachmentConfig?.chunkSizeBytes ?? DEFAULT_CHUNK_SIZE_BYTES;
         this._isAdminOnlyDb = adminOnlyDb;
+        this.maxCachedDocs = Math.max(1, Math.floor(documentCacheConfig?.maxEntries ?? DEFAULT_MAX_CACHED_DOCS));
+        this.iteratePrefetchWindowDocs = Math.max(0, Math.floor(documentCacheConfig?.iteratePrefetchWindowDocs ??
+            DEFAULT_ITERATE_PREFETCH_WINDOW_DOCS));
+        this.cacheRestoreLimit = Math.max(1, Math.floor(documentCacheConfig?.restoreLimit ?? this.maxCachedDocs));
+        this.snapshotMinChanges = Math.max(1, Math.floor(snapshotConfig?.minChanges ?? DEFAULT_SNAPSHOT_MIN_CHANGES));
+        this.snapshotCooldownMs = Math.max(0, Math.floor(snapshotConfig?.cooldownMs ?? DEFAULT_SNAPSHOT_COOLDOWN_MS));
         // Create logger if not provided (for backward compatibility)
         this.logger =
             logger ||
@@ -100,6 +89,207 @@ class BaseMindooDB {
     isAdminOnlyDb() {
         return this._isAdminOnlyDb;
     }
+    // ---------------------------------------------------------------------------
+    // Local cache support (ICacheable)
+    // ---------------------------------------------------------------------------
+    /**
+     * Attach a CacheManager so this DB participates in periodic cache flushing.
+     */
+    setCacheManager(cacheManager) {
+        this.cacheManager = cacheManager;
+        const cacheIdentity = this.store.getCacheIdentity?.() ?? this.store.getId();
+        this.cachePrefix = `${this.tenant.getId()}/${cacheIdentity}`;
+        cacheManager.register(this);
+    }
+    getCachePrefix() {
+        return this.cachePrefix ?? `${this.tenant.getId()}/${this.store.getId()}`;
+    }
+    hasDirtyState() {
+        return this.dirtyDocIds.size > 0 || this.cacheMetaDirty;
+    }
+    clearDirty() {
+        this.dirtyDocIds.clear();
+        this.cacheMetaDirty = false;
+        this.evictCachedDocsIfNeeded();
+    }
+    markDocDirty(docId) {
+        this.dirtyDocIds.add(docId);
+        this.cacheMetaDirty = true;
+        this.cacheManager?.markDirty();
+    }
+    getCachedDocument(docId) {
+        const cached = this.docCache.get(docId) ?? null;
+        if (!cached) {
+            return null;
+        }
+        this.touchCachedDocument(docId, cached);
+        return cached;
+    }
+    touchCachedDocument(docId, internalDoc) {
+        this.docCache.delete(docId);
+        this.docCache.set(docId, internalDoc);
+    }
+    storeCachedDocument(internalDoc) {
+        this.touchCachedDocument(internalDoc.id, internalDoc);
+        this.evictCachedDocsIfNeeded(new Set([internalDoc.id]));
+    }
+    evictCachedDocsIfNeeded(protectedDocIds) {
+        if (this.docCache.size <= this.maxCachedDocs) {
+            return;
+        }
+        for (const [docId] of this.docCache) {
+            if (this.docCache.size <= this.maxCachedDocs) {
+                break;
+            }
+            if (protectedDocIds?.has(docId)) {
+                continue;
+            }
+            this.docCache.delete(docId);
+            this.dirtyDocIds.delete(docId);
+        }
+    }
+    /**
+     * Flush dirty documents and metadata to the cache store.
+     */
+    async flushToCache(store) {
+        const prefix = this.getCachePrefix();
+        let written = 0;
+        for (const docId of this.dirtyDocIds) {
+            const internal = this.docCache.get(docId);
+            if (!internal)
+                continue;
+            const amBinary = automerge_adapter_1.Automerge.save(internal.doc);
+            const header = JSON.stringify({
+                id: internal.id,
+                createdAt: internal.createdAt,
+                lastModified: internal.lastModified,
+                decryptionKeyId: internal.decryptionKeyId,
+                isDeleted: internal.isDeleted,
+            });
+            const headerBytes = new TextEncoder().encode(header);
+            // Format: 4-byte header length (big-endian) + header JSON + Automerge binary
+            const value = new Uint8Array(4 + headerBytes.length + amBinary.length);
+            const view = new DataView(value.buffer);
+            view.setUint32(0, headerBytes.length, false);
+            value.set(headerBytes, 4);
+            value.set(amBinary, 4 + headerBytes.length);
+            await store.put("doc", `${prefix}/${docId}`, value);
+            written++;
+        }
+        // Write metadata checkpoint
+        const meta = this.exportMetadataCheckpoint();
+        await store.put("db-meta", prefix, meta);
+        written++;
+        return written;
+    }
+    exportMetadataCheckpoint() {
+        const checkpoint = {
+            version: 1,
+            processedEntryCursor: this.processedEntryCursor,
+            index: this.index,
+            nextChangeSeq: this.nextChangeSeq,
+        };
+        // Serialize automergeHashToEntryId: Map<string, Map<string,string>> -> nested object
+        const hashMap = {};
+        for (const [docId, inner] of this.automergeHashToEntryId) {
+            hashMap[docId] = Object.fromEntries(inner);
+        }
+        checkpoint.automergeHashToEntryId = hashMap;
+        // For stores without cursor scan, include processedEntryIds
+        if (!this.supportsCursorScan(this.store)) {
+            checkpoint.processedEntryIds = this.processedEntryIds;
+        }
+        return new TextEncoder().encode(JSON.stringify(checkpoint));
+    }
+    /**
+     * Attempt to restore state from cache. Returns true on success.
+     */
+    async restoreFromCache(store) {
+        const prefix = this.getCachePrefix();
+        try {
+            // 1. Load metadata checkpoint
+            const metaBytes = await store.get("db-meta", prefix);
+            if (!metaBytes) {
+                this.logger.debug("No cache metadata found, will do full rebuild");
+                return false;
+            }
+            const checkpoint = JSON.parse(new TextDecoder().decode(metaBytes));
+            if (checkpoint.version !== 1) {
+                this.logger.warn(`Unknown cache version ${checkpoint.version}, ignoring cache`);
+                return false;
+            }
+            // 2. Restore metadata
+            this.processedEntryCursor = checkpoint.processedEntryCursor ?? null;
+            this.index = checkpoint.index ?? [];
+            this.nextChangeSeq = checkpoint.nextChangeSeq ?? 1;
+            if (checkpoint.processedEntryIds) {
+                this.processedEntryIds = checkpoint.processedEntryIds;
+            }
+            // Rebuild indexLookup from index
+            this.indexLookup.clear();
+            for (let i = 0; i < this.index.length; i++) {
+                if (typeof this.index[i].changeSeq !== "number") {
+                    this.index[i].changeSeq = i + 1;
+                }
+                this.indexLookup.set(this.index[i].docId, i);
+            }
+            if (this.index.length > 0 && this.nextChangeSeq <= this.index.length) {
+                this.nextChangeSeq = Math.max(...this.index.map(e => e.changeSeq)) + 1;
+            }
+            // Restore automergeHashToEntryId
+            if (checkpoint.automergeHashToEntryId) {
+                this.automergeHashToEntryId.clear();
+                for (const [docId, inner] of Object.entries(checkpoint.automergeHashToEntryId)) {
+                    this.automergeHashToEntryId.set(docId, new Map(Object.entries(inner)));
+                }
+            }
+            // 3. Load cached documents
+            const docIds = await store.list("doc");
+            const docPrefix = `${prefix}/`;
+            let restoredDocs = 0;
+            for (const id of docIds) {
+                if (restoredDocs >= this.cacheRestoreLimit) {
+                    break;
+                }
+                if (!id.startsWith(docPrefix))
+                    continue;
+                const docId = id.slice(docPrefix.length);
+                const docBytes = await store.get("doc", id);
+                if (!docBytes)
+                    continue;
+                try {
+                    const internal = this.deserializeDoc(docBytes);
+                    this.storeCachedDocument(internal);
+                    restoredDocs++;
+                }
+                catch (e) {
+                    this.logger.warn(`Failed to restore cached doc ${docId}, will reload from store: ${e}`);
+                }
+            }
+            this.logger.info(`Restored ${restoredDocs} documents from cache for ${prefix}`);
+            return true;
+        }
+        catch (e) {
+            this.logger.warn(`Cache restore failed, will do full rebuild: ${e}`);
+            return false;
+        }
+    }
+    deserializeDoc(value) {
+        const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
+        const headerLen = view.getUint32(0, false);
+        const headerBytes = value.slice(4, 4 + headerLen);
+        const amBinary = value.slice(4 + headerLen);
+        const header = JSON.parse(new TextDecoder().decode(headerBytes));
+        const doc = automerge_adapter_1.Automerge.load(amBinary);
+        return {
+            id: header.id,
+            doc,
+            createdAt: header.createdAt,
+            lastModified: header.lastModified,
+            decryptionKeyId: header.decryptionKeyId,
+            isDeleted: header.isDeleted,
+        };
+    }
     /**
      * Get or import a CryptoKey for signature verification, with caching.
      * This avoids re-importing the same public key multiple times.
@@ -141,18 +331,18 @@ class BaseMindooDB {
     /**
      * Compare two index entries for sorting.
      * Returns negative if a < b, positive if a > b, 0 if equal.
-     * Sorts by lastModified first, then by docId for uniqueness.
+     * Sorts by deterministic changeSeq first, then by docId for uniqueness.
      */
     compareIndexEntries(a, b) {
-        if (a.lastModified !== b.lastModified) {
-            return a.lastModified - b.lastModified;
+        if (a.changeSeq !== b.changeSeq) {
+            return a.changeSeq - b.changeSeq;
         }
         return a.docId.localeCompare(b.docId);
     }
     /**
      * Update the index entry for a document.
-     * When a document changes, it's removed from its current position and inserted
-     * at the correct sorted position to maintain order by (lastModified, docId).
+     * When a document changes, it gets a new monotonic sequence and is moved to
+     * maintain order by (changeSeq, docId).
      *
      * Optimized to only update lookup map for entries that actually moved.
      *
@@ -161,12 +351,14 @@ class BaseMindooDB {
      * @param isDeleted Whether the document is deleted
      */
     updateIndex(docId, lastModified, isDeleted) {
-        const newEntry = { docId, lastModified, isDeleted };
+        const startedAt = Date.now();
+        const assignedSeq = this.nextChangeSeq;
+        const newEntry = { docId, changeSeq: assignedSeq, lastModified, isDeleted };
         const existingIndex = this.indexLookup.get(docId);
         // Check if the entry already exists and hasn't changed position
         if (existingIndex !== undefined) {
             const existingEntry = this.index[existingIndex];
-            // If position hasn't changed (same lastModified and docId), no update needed
+            // If only metadata flags stayed identical and caller replays same state, skip.
             if (existingEntry.lastModified === lastModified && existingEntry.isDeleted === isDeleted) {
                 return; // No change needed
             }
@@ -199,11 +391,17 @@ class BaseMindooDB {
         }
         // Insert at the correct position
         this.index.splice(insertIndex, 0, newEntry);
+        this.nextChangeSeq = assignedSeq + 1;
         // Update lookup map for entries from insertion point onwards
         // Only update entries that actually moved (from insertIndex to end)
         for (let i = insertIndex; i < this.index.length; i++) {
             this.indexLookup.set(this.index[i].docId, i);
         }
+        this.performanceCallback?.onIndexUpdate?.({
+            docId,
+            operation: existingIndex === undefined ? "insert" : "update",
+            time: Date.now() - startedAt,
+        });
     }
     /**
      * Register an automerge hash -> entry ID mapping for a document.
@@ -251,9 +449,20 @@ class BaseMindooDB {
     }
     /**
      * Initialize the database instance.
+     * If a cache store is available, attempts to restore from cache first,
+     * then processes only the delta. Falls back to full rebuild on cache miss.
      */
     async initialize() {
         this.logger.info(`Initializing database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
+        const cacheStore = this.cacheManager?.getStore();
+        if (cacheStore) {
+            const restored = await this.restoreFromCache(cacheStore);
+            if (restored) {
+                this.logger.info("Restored from cache, processing delta only");
+                await this.syncStoreChanges();
+                return;
+            }
+        }
         await this.syncStoreChanges();
     }
     /**
@@ -262,13 +471,22 @@ class BaseMindooDB {
      * On first call (when processedEntryIds is empty), it will process all entries.
      */
     async syncStoreChanges() {
+        const syncStartedAt = Date.now();
         this.logger.debug(`Syncing store changes for database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
         this.logger.debug(`Already processed ${this.processedEntryIds.length} entry IDs`);
         // Find new entries that we haven't processed yet
-        const newEntryMetadata = await this.store.findNewEntries(this.processedEntryIds);
+        const newEntryMetadata = await this.getNewEntryMetadataForSync();
         this.logger.debug(`Found ${newEntryMetadata.length} new entries`);
         if (newEntryMetadata.length === 0) {
             this.logger.debug(`No new entries to process`);
+            this.performanceCallback?.onSyncOperation?.({
+                operation: "findNewEntries",
+                time: Date.now() - syncStartedAt,
+                details: {
+                    newEntryCount: 0,
+                    processedEntryCount: this.processedEntryIds.length,
+                },
+            });
             return;
         }
         // Group new entries by document ID
@@ -285,10 +503,11 @@ class BaseMindooDB {
         this.logger.debug(`Processing ${entriesByDoc.size} documents with new entries`);
         // Helper function to process a single document
         const processDocument = async (docId, entryMetadataList) => {
+            const processStartedAt = Date.now();
             try {
                 this.logger.debug(`===== Processing document ${docId} with ${entryMetadataList.length} new entry(s) in syncStoreChanges =====`);
                 // Check if document is cached
-                const cachedDoc = this.docCache.get(docId);
+                const cachedDoc = this.getCachedDocument(docId);
                 let updatedDoc = null;
                 if (cachedDoc) {
                     // Document is cached - try incremental update
@@ -316,16 +535,37 @@ class BaseMindooDB {
                     }
                 }
                 else {
-                    // Document not cached - load from scratch
-                    this.logger.debug(`Document ${docId} not in cache, loading from store`);
-                    updatedDoc = await this.loadDocumentInternal(docId);
-                    if (updatedDoc) {
-                        this.logger.debug(`Successfully loaded document ${docId}, updating index`);
-                        this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted);
-                        this.logger.debug(`Updated index for document ${docId} (lastModified: ${updatedDoc.lastModified}, isDeleted: ${updatedDoc.isDeleted})`);
+                    // Metadata-first startup: avoid full materialization for uncached docs.
+                    // Index based on doc-lifecycle entries (create/change/delete) plus
+                    // doc_snapshot which may be the sole entry after a dense sync.
+                    // Attachment-only batches should not trigger document index updates.
+                    const docLifecycleEntries = entryMetadataList.filter((e) => e.entryType === "doc_create" ||
+                        e.entryType === "doc_change" ||
+                        e.entryType === "doc_delete" ||
+                        e.entryType === "doc_snapshot");
+                    if (docLifecycleEntries.length === 0) {
+                        this.logger.debug(`Skipping metadata-first index for doc ${docId} — no document lifecycle entries (${entryMetadataList.length} attachment/other entries only)`);
                     }
                     else {
-                        this.logger.warn(`Document ${docId} returned null from loadDocumentInternal`);
+                        // Before indexing, verify the user has the decryption key so that
+                        // documents the user cannot access do not appear in getAllDocumentIds.
+                        const representativeEntry = docLifecycleEntries[0];
+                        const keyAvailable = await this.tenant.hasDecryptionKey(representativeEntry.decryptionKeyId);
+                        if (!keyAvailable) {
+                            this.logger.debug(`Skipping metadata-first index for doc ${docId} — decryption key "${representativeEntry.decryptionKeyId}" not available`);
+                        }
+                        else {
+                            const mutationEntries = docLifecycleEntries.filter((e) => e.entryType === "doc_create" ||
+                                e.entryType === "doc_change" ||
+                                e.entryType === "doc_delete");
+                            // Snapshots compact replay history but should not change the user-visible
+                            // modification time when the original create/change/delete entries still exist.
+                            const entriesForLastModified = mutationEntries.length > 0 ? mutationEntries : docLifecycleEntries;
+                            const lastModified = Math.max(...entriesForLastModified.map((e) => e.createdAt));
+                            const isDeleted = mutationEntries.some((e) => e.entryType === "doc_delete");
+                            this.updateIndex(docId, lastModified, isDeleted);
+                            this.logger.debug(`Metadata-first update for uncached doc ${docId} (lastModified: ${lastModified}, isDeleted: ${isDeleted})`);
+                        }
                     }
                 }
             }
@@ -341,31 +581,332 @@ class BaseMindooDB {
                 // Re-throw the error so we can see what's happening in the test
                 throw error;
             }
+            finally {
+                this.performanceCallback?.onSyncOperation?.({
+                    operation: "processDocument",
+                    time: Date.now() - processStartedAt,
+                    details: {
+                        docId,
+                        entryCount: entryMetadataList.length,
+                        cacheHit: this.docCache.has(docId),
+                    },
+                });
+            }
         };
-        // Process documents in parallel with concurrency limit (default: 10)
-        const maxConcurrency = 10;
-        const documentEntries = Array.from(entriesByDoc.entries());
-        // Process in batches to limit concurrency
-        const results = [];
-        for (let i = 0; i < documentEntries.length; i += maxConcurrency) {
-            const batch = documentEntries.slice(i, i + maxConcurrency);
-            const batchResults = await Promise.allSettled(batch.map(([docId, entryMetadataList]) => processDocument(docId, entryMetadataList)));
-            results.push(...batchResults);
-        }
-        // Check for errors (excluding SymmetricKeyNotFoundError which is handled gracefully)
-        for (let i = 0; i < results.length; i++) {
-            const result = results[i];
-            if (result.status === 'rejected') {
-                const error = result.reason;
-                if (!(error instanceof errors_1.SymmetricKeyNotFoundError)) {
-                    // Re-throw non-key errors
-                    throw error;
-                }
+        const documentEntries = Array.from(entriesByDoc.entries())
+            .sort((a, b) => {
+            const aMinCreatedAt = Math.min(...a[1].map((e) => e.createdAt));
+            const bMinCreatedAt = Math.min(...b[1].map((e) => e.createdAt));
+            if (aMinCreatedAt !== bMinCreatedAt) {
+                return aMinCreatedAt - bMinCreatedAt;
             }
+            return a[0].localeCompare(b[0]);
+        });
+        // Deterministic sequential processing ensures stable changefeed ordering.
+        for (const [docId, entryMetadataList] of documentEntries) {
+            await processDocument(docId, entryMetadataList);
         }
         // Append new entry IDs to our processed list
         this.processedEntryIds.push(...newEntryMetadata.map(em => em.id));
+        this.cacheMetaDirty = true;
+        this.cacheManager?.markDirty();
         this.logger.debug(`Synced ${newEntryMetadata.length} new entries, index now has ${this.index.length} documents`);
+        this.performanceCallback?.onSyncOperation?.({
+            operation: "findNewEntries",
+            time: Date.now() - syncStartedAt,
+            details: {
+                newEntryCount: newEntryMetadata.length,
+                documentCount: entriesByDoc.size,
+                indexSize: this.index.length,
+            },
+        });
+    }
+    supportsCursorScan(store) {
+        return typeof store.scanEntriesSince === "function";
+    }
+    async scanAllMetadata(store, filters) {
+        if (!this.supportsCursorScan(store)) {
+            if (filters?.docId) {
+                return store.findNewEntriesForDoc([], filters.docId);
+            }
+            return store.findNewEntries([]);
+        }
+        const all = [];
+        let cursor = null;
+        while (true) {
+            const page = await store.scanEntriesSince(cursor, 1000, filters);
+            all.push(...page.entries);
+            cursor = page.nextCursor;
+            if (!page.hasMore) {
+                break;
+            }
+        }
+        return all;
+    }
+    async getNewEntryMetadataForSync() {
+        const startedAt = Date.now();
+        if (!this.supportsCursorScan(this.store)) {
+            const result = await this.store.findNewEntries(this.processedEntryIds);
+            this.performanceCallback?.onSyncOperation?.({
+                operation: "findNewEntries",
+                time: Date.now() - startedAt,
+                details: {
+                    mode: "knownIds",
+                    resultCount: result.length,
+                    processedEntryCount: this.processedEntryIds.length,
+                },
+            });
+            return result;
+        }
+        const allNew = [];
+        let cursor = this.processedEntryCursor;
+        while (true) {
+            const page = await this.store.scanEntriesSince(cursor, 1000);
+            allNew.push(...page.entries);
+            cursor = page.nextCursor;
+            if (!page.hasMore) {
+                break;
+            }
+        }
+        this.processedEntryCursor = cursor;
+        this.performanceCallback?.onSyncOperation?.({
+            operation: "findNewEntries",
+            time: Date.now() - startedAt,
+            details: {
+                mode: "cursorScan",
+                resultCount: allNew.length,
+                cursor,
+            },
+        });
+        return allNew;
+    }
+    /**
+     * Fetch the target store's bloom filter summary, returning null if
+     * unsupported or on error (callers fall back to exact checks).
+     */
+    async getTargetBloomSummary(targetStore) {
+        if (typeof targetStore.getIdBloomSummary !== "function") {
+            return null;
+        }
+        try {
+            return await targetStore.getIdBloomSummary();
+        }
+        catch (error) {
+            this.logger.warn("Failed to get bloom summary from target store, falling back to exact checks", error);
+            return null;
+        }
+    }
+    /**
+     * From a list of candidate IDs, return only those the target store is
+     * missing.  Uses bloom-filter pre-screening when available, then falls
+     * back to exact `hasEntries` for the uncertain set.
+     */
+    async filterMissingIds(targetStore, candidateIds, bloom) {
+        let definitelyMissing = [];
+        let maybeExisting = candidateIds;
+        if (bloom) {
+            definitelyMissing = [];
+            maybeExisting = [];
+            for (const id of candidateIds) {
+                if ((0, bloom_1.bloomMightContainId)(bloom, id)) {
+                    maybeExisting.push(id);
+                }
+                else {
+                    definitelyMissing.push(id);
+                }
+            }
+        }
+        let missingIds = definitelyMissing;
+        if (maybeExisting.length > 0) {
+            const existing = await targetStore.hasEntries(maybeExisting);
+            const existingSet = new Set(existing);
+            missingIds = missingIds.concat(maybeExisting.filter((id) => !existingSet.has(id)));
+        }
+        return missingIds;
+    }
+    async syncEntriesFromStore(sourceStore, targetStore, options) {
+        if (options?.mode === "dense") {
+            return this.syncEntriesFromStoreDense(sourceStore, targetStore, options);
+        }
+        let transferred = 0;
+        let scanned = 0;
+        const onProgress = options?.onProgress;
+        const pageSize = options?.pageSize ?? 1000;
+        const targetBloom = await this.getTargetBloomSummary(targetStore);
+        const totalSourceEstimate = targetBloom?.totalIds;
+        if (this.supportsCursorScan(sourceStore)) {
+            onProgress?.({
+                phase: 'preparing',
+                message: 'Preparing to sync entries...',
+                transferredEntries: 0,
+                scannedEntries: 0,
+                totalSourceEntries: totalSourceEstimate,
+            });
+            let cursor = null;
+            let currentPage = 0;
+            while (true) {
+                if (options?.signal?.aborted) {
+                    return { transferred, cancelled: true };
+                }
+                const page = await sourceStore.scanEntriesSince(cursor, pageSize);
+                currentPage++;
+                scanned += page.entries.length;
+                if (page.entries.length > 0) {
+                    const ids = page.entries.map((m) => m.id);
+                    const missingIds = await this.filterMissingIds(targetStore, ids, targetBloom);
+                    if (missingIds.length > 0) {
+                        const missingEntries = await sourceStore.getEntries(missingIds);
+                        await targetStore.putEntries(missingEntries);
+                        transferred += missingEntries.length;
+                    }
+                }
+                onProgress?.({
+                    phase: 'transferring',
+                    message: `Transferred ${transferred} entries (page ${currentPage}, scanned ${scanned})`,
+                    transferredEntries: transferred,
+                    scannedEntries: scanned,
+                    totalSourceEntries: totalSourceEstimate,
+                    currentPage,
+                });
+                cursor = page.nextCursor;
+                if (!page.hasMore) {
+                    break;
+                }
+            }
+            return { transferred, cancelled: false };
+        }
+        onProgress?.({
+            phase: 'preparing',
+            message: 'Finding new entries...',
+            transferredEntries: 0,
+            scannedEntries: 0,
+        });
+        const targetIds = await targetStore.getAllIds();
+        const sourceNewMetadata = await sourceStore.findNewEntries(targetIds);
+        if (sourceNewMetadata.length === 0) {
+            return { transferred: 0, cancelled: false };
+        }
+        if (options?.signal?.aborted) {
+            return { transferred: 0, cancelled: true };
+        }
+        onProgress?.({
+            phase: 'transferring',
+            message: `Transferring ${sourceNewMetadata.length} entries...`,
+            transferredEntries: 0,
+            scannedEntries: sourceNewMetadata.length,
+            totalSourceEntries: sourceNewMetadata.length,
+        });
+        const sourceEntries = await sourceStore.getEntries(sourceNewMetadata.map((m) => m.id));
+        await targetStore.putEntries(sourceEntries);
+        onProgress?.({
+            phase: 'transferring',
+            message: `Transferred ${sourceEntries.length} entries`,
+            transferredEntries: sourceEntries.length,
+            scannedEntries: sourceNewMetadata.length,
+            totalSourceEntries: sourceNewMetadata.length,
+        });
+        return { transferred: sourceEntries.length, cancelled: false };
+    }
+    /**
+     * Dense sync: transfer only the entries required to reconstruct the latest
+     * state of each document, using the batch materialization planner to skip
+     * historical entries already superseded by snapshots.
+     *
+     * Algorithm:
+     * 1. Discover all documents on the source via `doc_create` metadata.
+     * 2. Also fetch `doc_delete` metadata so deletion markers are transferred.
+     * 3. Ask the source's batch planner for the optimal replay set per document.
+     * 4. Merge all entry IDs: doc_create + doc_delete + snapshot + uncovered changes.
+     * 5. Filter out IDs the target already has (bloom + exact check).
+     * 6. Transfer only the missing entries.
+     *
+     * Attachment chunks are intentionally skipped — they are not part of the
+     * Automerge document DAG and can be fetched on demand when accessed.
+     */
+    async syncEntriesFromStoreDense(sourceStore, targetStore, options) {
+        const onProgress = options?.onProgress;
+        onProgress?.({
+            phase: "preparing",
+            message: "Dense sync: discovering documents on source...",
+            transferredEntries: 0,
+            scannedEntries: 0,
+        });
+        if (options?.signal?.aborted) {
+            return { transferred: 0, cancelled: true };
+        }
+        // ── Phase 1: discover documents ──────────────────────────────────
+        const docCreateEntries = await sourceStore.findEntries("doc_create", null, null);
+        const docDeleteEntries = await sourceStore.findEntries("doc_delete", null, null);
+        const docIds = [...new Set(docCreateEntries.map((e) => e.docId))];
+        this.logger.info(`Dense sync: found ${docIds.length} documents on source`);
+        if (docIds.length === 0) {
+            return { transferred: 0, cancelled: false };
+        }
+        if (options?.signal?.aborted) {
+            return { transferred: 0, cancelled: true };
+        }
+        // ── Phase 2: batch plan on source ────────────────────────────────
+        onProgress?.({
+            phase: "planning",
+            message: `Dense sync: computing materialization plans for ${docIds.length} documents...`,
+            transferredEntries: 0,
+            scannedEntries: docIds.length,
+        });
+        const batchPlan = await sourceStore.planDocumentMaterializationBatch(docIds);
+        // ── Phase 3: collect needed entry IDs ────────────────────────────
+        const neededIds = new Set();
+        for (const entry of docCreateEntries) {
+            neededIds.add(entry.id);
+        }
+        for (const entry of docDeleteEntries) {
+            neededIds.add(entry.id);
+        }
+        for (const plan of batchPlan.plans) {
+            if (plan.snapshotEntryId) {
+                neededIds.add(plan.snapshotEntryId);
+            }
+            for (const id of plan.entryIdsToApply) {
+                neededIds.add(id);
+            }
+        }
+        this.logger.info(`Dense sync: planner identified ${neededIds.size} required entries ` +
+            `(from ${docIds.length} documents)`);
+        if (options?.signal?.aborted) {
+            return { transferred: 0, cancelled: true };
+        }
+        // ── Phase 4: filter out entries the target already has ───────────
+        const allNeededArray = Array.from(neededIds);
+        const targetBloom = await this.getTargetBloomSummary(targetStore);
+        const missingIds = await this.filterMissingIds(targetStore, allNeededArray, targetBloom);
+        this.logger.info(`Dense sync: ${missingIds.length} entries to transfer ` +
+            `(${allNeededArray.length - missingIds.length} already present)`);
+        if (missingIds.length === 0) {
+            return { transferred: 0, cancelled: false };
+        }
+        if (options?.signal?.aborted) {
+            return { transferred: 0, cancelled: true };
+        }
+        // ── Phase 5: transfer missing entries in pages ───────────────────
+        const pageSize = options?.pageSize ?? 500;
+        let transferred = 0;
+        for (let offset = 0; offset < missingIds.length; offset += pageSize) {
+            if (options?.signal?.aborted) {
+                return { transferred, cancelled: true };
+            }
+            const batch = missingIds.slice(offset, offset + pageSize);
+            const entries = await sourceStore.getEntries(batch);
+            await targetStore.putEntries(entries);
+            transferred += entries.length;
+            onProgress?.({
+                phase: "transferring",
+                message: `Dense sync: transferred ${transferred}/${missingIds.length} entries`,
+                transferredEntries: transferred,
+                scannedEntries: allNeededArray.length,
+                totalSourceEntries: allNeededArray.length,
+            });
+        }
+        this.logger.info(`Dense sync complete: transferred ${transferred} entries`);
+        return { transferred, cancelled: false };
     }
     getStore() {
         return this.store;
@@ -407,18 +948,18 @@ class BaseMindooDB {
         const docId = (0, uuid_1.v7)();
         this.logger.debug(`Creating document ${docId} with key ${keyId}${useCustomSigningKey ? ' using custom signing key' : ''}`);
         // Create initial Automerge document
-        const initialDoc = Automerge.init();
+        const initialDoc = automerge_adapter_1.Automerge.init();
         // Create the first change
         const now = Date.now();
         this.logger.debug(`Creating initial Automerge change for document ${docId}`);
         let newDoc;
         try {
-            newDoc = Automerge.change(initialDoc, (doc) => {
+            newDoc = automerge_adapter_1.Automerge.change(initialDoc, (doc) => {
                 // Store metadata in the document payload
                 // We need to modify the document to ensure a change is created
                 doc._attachments = [];
             });
-            this.logger.debug(`Successfully created Automerge change, document heads: ${JSON.stringify(Automerge.getHeads(newDoc))}`);
+            this.logger.debug(`Successfully created Automerge change, document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
         }
         catch (error) {
             this.logger.error(`Error in Automerge.change for document ${docId}:`, error);
@@ -426,7 +967,7 @@ class BaseMindooDB {
         }
         // Get the change bytes from the document
         this.logger.debug(`Getting change bytes from document ${docId}`);
-        const changeBytes = Automerge.getLastLocalChange(newDoc);
+        const changeBytes = automerge_adapter_1.Automerge.getLastLocalChange(newDoc);
         if (!changeBytes) {
             throw new Error("Failed to get change bytes from Automerge document");
         }
@@ -435,7 +976,7 @@ class BaseMindooDB {
         this.logger.debug(`Decoding change to get hash and dependencies`);
         let decodedChange;
         try {
-            decodedChange = Automerge.decodeChange(changeBytes);
+            decodedChange = automerge_adapter_1.Automerge.decodeChange(changeBytes);
             this.logger.debug(`Successfully decoded change, hash: ${decodedChange.hash}, deps: ${decodedChange.deps?.length || 0}`);
         }
         catch (error) {
@@ -504,8 +1045,9 @@ class BaseMindooDB {
             isDeleted: false,
         };
         // Update cache and index
-        this.docCache.set(docId, internalDoc);
+        this.storeCachedDocument(internalDoc);
         this.updateIndex(docId, internalDoc.lastModified, false);
+        this.markDocDirty(docId);
         this.logger.info(`Document ${docId} created successfully`);
         this.logger.debug(`Document ${docId} cached and indexed (lastModified: ${internalDoc.lastModified})`);
         return this.wrapDocument(internalDoc);
@@ -521,20 +1063,28 @@ class BaseMindooDB {
         return this.wrapDocument(internalDoc);
     }
     async getDocumentAtTimestamp(docId, timestamp) {
+        const startedAt = Date.now();
         this.logger.debug(`Getting document ${docId} at timestamp ${timestamp}`);
         // Get all entry metadata for this document
-        const allEntryMetadata = await this.store.findNewEntriesForDoc([], docId);
+        const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
         // Filter entries up to the timestamp
         const relevantEntries = allEntryMetadata
             .filter(em => em.createdAt <= timestamp)
             .sort((a, b) => a.createdAt - b.createdAt);
         if (relevantEntries.length === 0) {
+            this.performanceCallback?.onHistoryOperation?.({
+                operation: "getDocumentAtTimestamp",
+                docId,
+                time: Date.now() - startedAt,
+                scannedEntries: 0,
+                bounded: false,
+            });
             return null; // Document didn't exist at that time
         }
         // Load entries
         const entries = await this.store.getEntries(relevantEntries.map(em => em.id));
         // Apply changes in order
-        let doc = Automerge.init();
+        let doc = automerge_adapter_1.Automerge.init();
         for (const entryData of entries) {
             // Admin-only mode: only accept entries signed by the admin key
             if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
@@ -551,7 +1101,7 @@ class BaseMindooDB {
             // Decrypt payload (only after signature verification passes)
             const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
             // Apply change using loadIncremental - this is the recommended way for binary change data
-            doc = Automerge.loadIncremental(doc, decryptedPayload);
+            doc = automerge_adapter_1.Automerge.loadIncremental(doc, decryptedPayload);
             // Register the automerge hash -> entry ID mapping for future dependency resolution
             const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
             if (parsed) {
@@ -576,12 +1126,20 @@ class BaseMindooDB {
             decryptionKeyId,
             isDeleted,
         };
+        this.performanceCallback?.onHistoryOperation?.({
+            operation: "getDocumentAtTimestamp",
+            docId,
+            time: Date.now() - startedAt,
+            scannedEntries: relevantEntries.length,
+            returnedEntries: 1,
+            bounded: false,
+        });
         return this.wrapDocument(internalDoc);
     }
     async *iterateDocumentHistory(docId) {
         this.logger.debug(`Iterating document history for ${docId}`);
         // Get all entry metadata for this document
-        const allEntryMetadata = await this.store.findNewEntriesForDoc([], docId);
+        const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
         // Filter to only doc_create, doc_change, and doc_delete entries (exclude snapshots)
         const relevantEntries = allEntryMetadata
             .filter(em => em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete")
@@ -623,17 +1181,17 @@ class BaseMindooDB {
                     this.logger.warn(`First entry is not doc_create, skipping`);
                     continue;
                 }
-                currentDoc = Automerge.init();
+                currentDoc = automerge_adapter_1.Automerge.init();
                 createdAt = entryMetadata.createdAt;
                 decryptionKeyId = entryMetadata.decryptionKeyId;
             }
             // Check document heads before applying change (for non-first entries)
-            const headsBefore = isFirstEntry ? null : (currentDoc ? Automerge.getHeads(currentDoc) : null);
+            const headsBefore = isFirstEntry ? null : (currentDoc ? automerge_adapter_1.Automerge.getHeads(currentDoc) : null);
             // Apply change using loadIncremental
             if (currentDoc === null) {
                 throw new Error("currentDoc should not be null at this point");
             }
-            currentDoc = Automerge.loadIncremental(currentDoc, decryptedPayload);
+            currentDoc = automerge_adapter_1.Automerge.loadIncremental(currentDoc, decryptedPayload);
             // Check if document actually changed by comparing heads
             // For first entry (doc_create), always yield since it's the initial creation
             // For delete entries, always yield
@@ -643,7 +1201,7 @@ class BaseMindooDB {
                 shouldYield = true;
             }
             else {
-                const headsAfter = Automerge.getHeads(currentDoc);
+                const headsAfter = automerge_adapter_1.Automerge.getHeads(currentDoc);
                 const headsChanged = headsBefore !== null && JSON.stringify(headsBefore) !== JSON.stringify(headsAfter);
                 shouldYield = headsChanged;
             }
@@ -655,7 +1213,7 @@ class BaseMindooDB {
             // Only yield if document actually changed (or if this is create/delete entry)
             if (shouldYield) {
                 // Clone the document to ensure independence
-                const clonedDoc = Automerge.clone(currentDoc);
+                const clonedDoc = automerge_adapter_1.Automerge.clone(currentDoc);
                 // Create internal doc representation
                 const internalDoc = {
                     id: docId,
@@ -679,6 +1237,46 @@ class BaseMindooDB {
             }
         }
     }
+    async getDocumentHistoryPage(docId, options) {
+        const startedAt = Date.now();
+        const limit = Math.max(1, Math.floor(options?.limit ?? 100));
+        const offset = Math.max(0, Math.floor(options?.cursor?.offset ?? 0));
+        // This API stays metadata-only so large history views can page cheaply
+        // without reconstructing every historical Automerge document state.
+        const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
+        const relevantEntries = allEntryMetadata
+            .filter((em) => em.entryType === "doc_create" ||
+            em.entryType === "doc_change" ||
+            em.entryType === "doc_delete")
+            .sort((a, b) => a.createdAt - b.createdAt);
+        // Cursor paging is offset-based because the result is a bounded timeline view
+        // over the doc's sorted change metadata, not a resumable store scan cursor.
+        const slice = relevantEntries.slice(offset, offset + limit);
+        const entries = slice.map((entry) => ({
+            entryId: entry.id,
+            entryType: entry.entryType,
+            changeCreatedAt: entry.createdAt,
+            changeCreatedByPublicKey: entry.createdByPublicKey,
+            dependencyIds: [...entry.dependencyIds],
+            isDeleted: entry.entryType === "doc_delete",
+        }));
+        const nextOffset = offset + entries.length;
+        const hasMore = nextOffset < relevantEntries.length;
+        const result = {
+            entries,
+            nextCursor: hasMore ? { offset: nextOffset } : null,
+            hasMore,
+        };
+        this.performanceCallback?.onHistoryOperation?.({
+            operation: "getDocumentHistoryPage",
+            docId,
+            time: Date.now() - startedAt,
+            scannedEntries: relevantEntries.length,
+            returnedEntries: entries.length,
+            bounded: true,
+        });
+        return result;
+    }
     async getAllDocumentIds() {
         // Return all non-deleted document IDs from index
         const docIds = [];
@@ -690,10 +1288,14 @@ class BaseMindooDB {
         return docIds;
     }
     async getAllDocumentIdsAtTimestamp(timestamp) {
+        // findEntries() uses an exclusive upper bound (createdAt < creationDateUntil).
+        // Query up to timestamp + 1 so entries created exactly at `timestamp` are included
+        // for the strict checks below (createTime < timestamp, deleteTime > timestamp).
+        const upperBoundExclusive = timestamp === Number.MAX_SAFE_INTEGER ? timestamp : timestamp + 1;
         // Efficiently query for doc_create and doc_delete entries before the timestamp
         const [creates, deletes] = await Promise.all([
-            this.store.findEntries("doc_create", null, timestamp),
-            this.store.findEntries("doc_delete", null, timestamp)
+            this.store.findEntries("doc_create", null, upperBoundExclusive),
+            this.store.findEntries("doc_delete", null, upperBoundExclusive)
         ]);
         // Build sets of docIds for efficient lookup
         const createdDocIds = new Set();
@@ -722,9 +1324,9 @@ class BaseMindooDB {
             const createTime = createTimes.get(docId);
             const deleteTime = deleteTimes.get(docId);
             // Document exists at timestamp if:
-            // 1. It was created before the timestamp
+            // 1. It was created at or before the timestamp
             // 2. Either it was never deleted, or it was deleted after the timestamp
-            if (createTime < timestamp) {
+            if (createTime <= timestamp) {
                 if (!deleteTime || deleteTime > timestamp) {
                     docIds.push(docId);
                 }
@@ -761,24 +1363,26 @@ class BaseMindooDB {
         if (!internalDoc || internalDoc.isDeleted) {
             throw new Error(`Document ${docId} not found or already deleted`);
         }
+        const now = Date.now();
         // Create deletion change by clearing all fields from the document
         // This ensures Automerge produces actual change bytes
         // The deletion is also tracked via the "doc_delete" type entry in the append-only store
-        const newDoc = Automerge.change(internalDoc.doc, (doc) => {
+        const newDoc = automerge_adapter_1.Automerge.change(internalDoc.doc, (doc) => {
             // Remove all fields from the document
             for (const key of Object.keys(doc)) {
                 delete doc[key];
             }
             // Mark as deleted for clarity
             doc._deleted = true;
+            doc._lastModified = now;
         });
         // Get the change bytes from the document
-        const changeBytes = Automerge.getLastLocalChange(newDoc);
+        const changeBytes = automerge_adapter_1.Automerge.getLastLocalChange(newDoc);
         if (!changeBytes) {
             throw new Error("Failed to get change bytes from Automerge document");
         }
         // Decode the change to get hash and dependencies
-        const decodedChange = Automerge.decodeChange(changeBytes);
+        const decodedChange = automerge_adapter_1.Automerge.decodeChange(changeBytes);
         const automergeHash = decodedChange.hash;
         const automergeDepHashes = decodedChange.deps || []; // Dependencies from the decoded change
         // Encrypt the change payload first
@@ -810,7 +1414,7 @@ class BaseMindooDB {
             contentHash,
             docId,
             dependencyIds,
-            createdAt: Date.now(),
+            createdAt: now,
             createdByPublicKey,
             decryptionKeyId: internalDoc.decryptionKeyId,
             signature,
@@ -828,9 +1432,10 @@ class BaseMindooDB {
         this.registerAutomergeHashMapping(docId, automergeHash, entryId);
         // Update cache and index
         internalDoc.isDeleted = true;
-        internalDoc.lastModified = Date.now();
-        this.docCache.set(docId, internalDoc);
+        internalDoc.lastModified = now;
+        this.storeCachedDocument(internalDoc);
         this.updateIndex(docId, internalDoc.lastModified, true);
+        this.markDocDirty(docId);
         this.logger.info(`Document ${docId} deleted successfully`);
     }
     async changeDoc(doc, changeFunc) {
@@ -854,7 +1459,7 @@ class BaseMindooDB {
             }
         }
         // Get internal document from cache or load it
-        let internalDoc = this.docCache.get(docId);
+        let internalDoc = this.getCachedDocument(docId);
         if (!internalDoc) {
             this.logger.debug(`Document ${docId} not in cache, loading from store`);
             const loadedDoc = await this.loadDocumentInternal(docId);
@@ -873,7 +1478,7 @@ class BaseMindooDB {
         // Apply the change function
         const now = Date.now();
         this.logger.debug(`Applying change function to document ${docId}`);
-        this.logger.debug(`Document state before change: heads=${JSON.stringify(Automerge.getHeads(internalDoc.doc))}`);
+        this.logger.debug(`Document state before change: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(internalDoc.doc))}`);
         // For async callbacks, we need to handle document modifications carefully.
         // Automerge.change() requires synchronous modifications, so we'll:
         // 1. Execute the async callback to do any async work (like signing)
@@ -916,8 +1521,8 @@ class BaseMindooDB {
                             pendingDeletions.delete(prop);
                             // Track the change
                             pendingChanges.set(prop, value);
-                            // Also set on the target for immediate access
-                            target[prop] = value;
+                            // NOTE: Don't set on target immediately - Automerge requires changes
+                            // to be made inside Automerge.change(). Pending changes are applied later.
                         }
                         return true;
                     },
@@ -928,8 +1533,8 @@ class BaseMindooDB {
                             pendingDeletions.add(prop);
                             // Remove from pending changes if it was there
                             pendingChanges.delete(prop);
-                            // Also delete from the target for immediate access
-                            delete target[prop];
+                            // NOTE: Don't delete from target immediately - Automerge requires changes
+                            // to be made inside Automerge.change(). Pending deletions are applied later.
                         }
                         return true;
                     },
@@ -1048,7 +1653,7 @@ class BaseMindooDB {
         // Now apply the collected changes synchronously in Automerge.change()
         let newDoc;
         try {
-            newDoc = Automerge.change(internalDoc.doc, (automergeDoc) => {
+            newDoc = automerge_adapter_1.Automerge.change(internalDoc.doc, (automergeDoc) => {
                 // Apply all pending changes (sets/updates)
                 for (const [key, value] of pendingChanges) {
                     automergeDoc[key] = value;
@@ -1087,7 +1692,7 @@ class BaseMindooDB {
                 // Update lastModified timestamp
                 automergeDoc._lastModified = now;
             });
-            this.logger.debug(`Successfully applied change function, new document heads: ${JSON.stringify(Automerge.getHeads(newDoc))}`);
+            this.logger.debug(`Successfully applied change function, new document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
         }
         catch (error) {
             this.logger.error(`Error in Automerge.change for document ${docId}:`, error);
@@ -1095,14 +1700,14 @@ class BaseMindooDB {
         }
         // Get the change bytes from the document
         this.logger.debug(`Getting change bytes from document ${docId}`);
-        const changeBytes = Automerge.getLastLocalChange(newDoc);
+        const changeBytes = automerge_adapter_1.Automerge.getLastLocalChange(newDoc);
         if (!changeBytes) {
             //TODO decide if we just exit here or throw an error
             throw new Error("Failed to get change bytes from Automerge document");
         }
         this.logger.debug(`Got change bytes: ${changeBytes.length} bytes`);
         // Decode the change to get hash and dependencies
-        const decodedChange = Automerge.decodeChange(changeBytes);
+        const decodedChange = automerge_adapter_1.Automerge.decodeChange(changeBytes);
         const automergeHash = decodedChange.hash;
         const automergeDepHashes = decodedChange.deps || []; // Dependencies from the decoded change
         // Encrypt the change payload first
@@ -1133,7 +1738,7 @@ class BaseMindooDB {
             contentHash,
             docId,
             dependencyIds,
-            createdAt: Date.now(),
+            createdAt: now,
             createdByPublicKey,
             decryptionKeyId: internalDoc.decryptionKeyId,
             signature,
@@ -1151,98 +1756,242 @@ class BaseMindooDB {
         this.registerAutomergeHashMapping(docId, automergeHash, entryId);
         // Update cache and index
         internalDoc.doc = newDoc;
-        internalDoc.lastModified = Date.now();
-        this.docCache.set(docId, internalDoc);
+        internalDoc.lastModified = now;
+        this.storeCachedDocument(internalDoc);
         this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted);
+        this.markDocDirty(docId);
         this.logger.info(`Document ${docId} ${useCustomKey ? 'changed with custom signing key' : 'changed'} successfully`);
+        // Snapshot creation runs as a best-effort background optimization.
+        await this.maybeWriteSnapshotForDocument(internalDoc, useCustomKey
+            ? { signingKeyPair, signingKeyPassword, createdByPublicKey }
+            : { createdByPublicKey });
     }
-    async *iterateChangesSince(cursor) {
-        // Default to initial cursor if null is provided
-        const actualCursor = cursor ?? { lastModified: 0, docId: "" };
-        this.logger.debug(`Starting iteration from cursor ${JSON.stringify(actualCursor)}`);
-        // Find starting position using binary search
-        // We want to find the first entry that is greater than the cursor
-        let startIndex = 0;
-        if (this.index.length > 0) {
-            let left = 0;
-            let right = this.index.length - 1;
-            while (left <= right) {
-                const mid = Math.floor((left + right) / 2);
-                const entry = this.index[mid];
-                const cmp = this.compareIndexEntries({ docId: actualCursor.docId, lastModified: actualCursor.lastModified }, entry);
-                if (cmp < 0) {
-                    right = mid - 1;
-                    startIndex = mid;
-                }
-                else {
-                    left = mid + 1;
-                    startIndex = mid + 1;
-                }
+    /**
+     * Best-effort snapshot scheduling and writing.
+     * A snapshot is written only when enough replay history has accumulated and
+     * a cooldown window has elapsed, to avoid snapshot churn on hot documents.
+     */
+    async maybeWriteSnapshotForDocument(internalDoc, options) {
+        const docId = internalDoc.id;
+        try {
+            const allMetadata = await this.scanAllMetadata(this.store, { docId });
+            const replayEntries = allMetadata.filter((em) => em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete");
+            const snapshots = allMetadata
+                .filter((em) => em.entryType === "doc_snapshot")
+                .sort((a, b) => b.createdAt - a.createdAt);
+            const latestSnapshot = snapshots[0] || null;
+            const latestSnapshotAt = latestSnapshot?.createdAt ?? 0;
+            const changesSinceSnapshot = replayEntries.filter((em) => em.createdAt > latestSnapshotAt).length;
+            if (changesSinceSnapshot < this.snapshotMinChanges) {
+                return;
             }
-            // If we found an exact match, start from the next entry
-            if (startIndex < this.index.length) {
-                const entry = this.index[startIndex];
-                if (entry.lastModified === actualCursor.lastModified && entry.docId === actualCursor.docId) {
-                    startIndex++;
-                }
+            if (latestSnapshot &&
+                Date.now() - latestSnapshot.createdAt < this.snapshotCooldownMs) {
+                return;
+            }
+            const headHashes = automerge_adapter_1.Automerge.getHeads(internalDoc.doc);
+            const headEntryIds = this.resolveAutomergeDepsToEntryIds(docId, headHashes);
+            const snapshotBytes = automerge_adapter_1.Automerge.save(internalDoc.doc);
+            const encryptedPayload = await this.tenant.encryptPayload(snapshotBytes, internalDoc.decryptionKeyId);
+            const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedPayload, this.getSubtle());
+            const pseudoSnapshotHash = `snapshot-${(0, uuid_1.v7)()}`;
+            const entryId = await (0, idGeneration_1.generateDocEntryId)(docId, pseudoSnapshotHash, headHashes, this.getSubtle());
+            let signature;
+            if (options.signingKeyPair && options.signingKeyPassword) {
+                signature = await this.tenant.signPayloadWithKey(encryptedPayload, options.signingKeyPair, options.signingKeyPassword);
             }
+            else {
+                signature = await this.tenant.signPayload(encryptedPayload);
+            }
+            const snapshotEntry = {
+                entryType: "doc_snapshot",
+                id: entryId,
+                contentHash,
+                docId,
+                dependencyIds: headEntryIds,
+                createdAt: Date.now(),
+                createdByPublicKey: options.createdByPublicKey,
+                decryptionKeyId: internalDoc.decryptionKeyId,
+                snapshotHeadHashes: headHashes,
+                snapshotHeadEntryIds: headEntryIds,
+                signature,
+                originalSize: snapshotBytes.length,
+                encryptedSize: encryptedPayload.length,
+                encryptedData: encryptedPayload,
+            };
+            await this.store.putEntries([snapshotEntry]);
+            this.logger.debug(`Created snapshot for document ${docId} with ${headHashes.length} heads and ${changesSinceSnapshot} changes since previous snapshot`);
+        }
+        catch (error) {
+            this.logger.warn(`Failed to create snapshot for document ${docId}, continuing without snapshot`, error);
+        }
+    }
+    async prefetchIterationWindow(indexSnapshot, startIndex) {
+        if (this.iteratePrefetchWindowDocs <= 0) {
+            return 0;
         }
-        // Pre-check which documents are in cache to optimize iteration
-        const uncachedDocIds = [];
-        for (let i = startIndex; i < this.index.length; i++) {
-            const entry = this.index[i];
-            if (!this.docCache.has(entry.docId)) {
-                uncachedDocIds.push(entry.docId);
+        const docIds = [];
+        const seen = new Set();
+        // Only look ahead a bounded number of uncached docs so iteration does not
+        // materialize the entire remaining changefeed tail up front.
+        for (let i = startIndex; i < indexSnapshot.length && docIds.length < this.iteratePrefetchWindowDocs; i++) {
+            const docId = indexSnapshot[i].docId;
+            if (seen.has(docId) || this.docCache.has(docId)) {
+                continue;
             }
+            seen.add(docId);
+            docIds.push(docId);
         }
-        // Prefetch uncached documents in batches (if any)
-        if (uncachedDocIds.length > 0) {
-            this.logger.debug(`Prefetching ${uncachedDocIds.length} uncached documents for iteration`);
-            const prefetchBatchSize = 10;
-            for (let i = 0; i < uncachedDocIds.length; i += prefetchBatchSize) {
-                const batch = uncachedDocIds.slice(i, i + prefetchBatchSize);
-                await Promise.all(batch.map(docId => this.loadDocumentInternal(docId).catch(err => {
-                    this.logger.warn(`Failed to prefetch document ${docId}:`, err);
-                    return null;
-                })));
+        if (docIds.length === 0) {
+            return 0;
+        }
+        // Prefetch is opportunistic: iteration can still fall back to on-demand
+        // loads, so individual prefetch failures are logged and ignored here.
+        await Promise.all(docIds.map((docId) => this.loadDocumentInternal(docId).catch((err) => {
+            this.logger.warn(`Failed to prefetch document ${docId}:`, err);
+            return null;
+        })));
+        return docIds.length;
+    }
+    getStartIndexForCursor(indexSnapshot, actualCursor) {
+        let startIndex = 0;
+        if (indexSnapshot.length === 0) {
+            return startIndex;
+        }
+        const cursorSeq = typeof actualCursor.changeSeq === "number" ? actualCursor.changeSeq : 0;
+        let left = 0;
+        let right = indexSnapshot.length - 1;
+        while (left <= right) {
+            const mid = Math.floor((left + right) / 2);
+            const entry = indexSnapshot[mid];
+            const cursorComparable = {
+                docId: actualCursor.docId,
+                changeSeq: cursorSeq,
+            };
+            const cmp = this.compareIndexEntries(cursorComparable, entry);
+            if (cmp < 0) {
+                right = mid - 1;
+                startIndex = mid;
+            }
+            else {
+                left = mid + 1;
+                startIndex = mid + 1;
             }
         }
-        // Iterate through the index and yield documents one at a time
-        // Documents should now be in cache from prefetching
-        for (let i = startIndex; i < this.index.length; i++) {
-            const entry = this.index[i];
-            try {
-                this.logger.debug(`Yielding document ${entry.docId} from index (lastModified: ${entry.lastModified}, isDeleted: ${entry.isDeleted})`);
-                // Check cache first (should be there after prefetching)
-                let internalDoc = this.docCache.get(entry.docId) || null;
-                if (!internalDoc) {
-                    // Fallback to loading if not in cache (shouldn't happen after prefetching)
-                    this.logger.debug(`Document ${entry.docId} not in cache, loading from store`);
-                    internalDoc = await this.loadDocumentInternal(entry.docId);
-                }
-                if (!internalDoc) {
-                    this.logger.warn(`Document ${entry.docId} not found, skipping`);
-                    continue;
-                }
-                // Wrap the document (works for both deleted and non-deleted documents)
-                const doc = this.wrapDocument(internalDoc);
-                this.logger.debug(`Successfully loaded document ${entry.docId} (isDeleted: ${doc.isDeleted()})`);
-                // Create cursor for current document
+        if (startIndex < indexSnapshot.length) {
+            const entry = indexSnapshot[startIndex];
+            if (entry.changeSeq === cursorSeq && entry.docId === actualCursor.docId) {
+                startIndex++;
+            }
+        }
+        return startIndex;
+    }
+    async *iterateChangeMetadataSince(cursor) {
+        const startedAt = Date.now();
+        const actualCursor = cursor ?? {
+            changeSeq: 0,
+            lastModified: 0,
+            docId: "",
+        };
+        const indexSnapshot = [...this.index];
+        const startIndex = this.getStartIndexForCursor(indexSnapshot, actualCursor);
+        let yieldedDocuments = 0;
+        try {
+            for (let i = startIndex; i < indexSnapshot.length; i++) {
+                const entry = indexSnapshot[i];
                 const currentCursor = {
+                    changeSeq: entry.changeSeq,
                     lastModified: entry.lastModified,
                     docId: entry.docId,
                 };
-                // Yield immediately - this allows the caller to break early after each document
-                // Deleted documents are included so external indexes can handle deletions
-                yield { doc, cursor: currentCursor };
+                yieldedDocuments++;
+                yield {
+                    docId: entry.docId,
+                    lastModified: entry.lastModified,
+                    isDeleted: entry.isDeleted,
+                    cursor: currentCursor,
+                };
             }
-            catch (error) {
-                this.logger.error(`Error processing document ${entry.docId}:`, error);
-                // Stop processing on error
-                throw error;
+        }
+        finally {
+            this.performanceCallback?.onSyncOperation?.({
+                operation: "iterateChangeMetadataSince",
+                time: Date.now() - startedAt,
+                details: {
+                    yieldedDocuments,
+                    startIndex,
+                },
+            });
+        }
+    }
+    async *iterateChangesSince(cursor) {
+        const startedAt = Date.now();
+        // Default to initial cursor if null is provided.
+        // Prefer deterministic sequence-based cursoring; keep legacy fallback compatibility.
+        const actualCursor = cursor ?? { changeSeq: 0, lastModified: 0, docId: "" };
+        this.logger.debug(`Starting iteration from cursor ${JSON.stringify(actualCursor)}`);
+        // Use a stable snapshot of the index for this generator run so concurrent
+        // updates do not reorder/skip entries while iterating.
+        const indexSnapshot = [...this.index];
+        // Find starting position using binary search
+        // We want to find the first entry that is greater than the cursor.
+        const startIndex = this.getStartIndexForCursor(indexSnapshot, actualCursor);
+        let prefetchedDocuments = await this.prefetchIterationWindow(indexSnapshot, startIndex);
+        let yieldedDocuments = 0;
+        let loadedDocuments = 0;
+        try {
+            // Iterate through the stable snapshot and yield documents one at a time.
+            for (let i = startIndex; i < indexSnapshot.length; i++) {
+                const entry = indexSnapshot[i];
+                try {
+                    this.logger.debug(`Yielding document ${entry.docId} from index (lastModified: ${entry.lastModified}, isDeleted: ${entry.isDeleted})`);
+                    let internalDoc = this.getCachedDocument(entry.docId);
+                    if (!internalDoc) {
+                        this.logger.debug(`Document ${entry.docId} not in cache, loading from store`);
+                        internalDoc = await this.loadDocumentInternal(entry.docId);
+                        if (internalDoc) {
+                            loadedDocuments++;
+                        }
+                    }
+                    if (!internalDoc) {
+                        this.logger.warn(`Document ${entry.docId} not found, skipping`);
+                        continue;
+                    }
+                    // Wrap the document (works for both deleted and non-deleted documents)
+                    const doc = this.wrapDocument(internalDoc);
+                    this.logger.debug(`Successfully loaded document ${entry.docId} (isDeleted: ${doc.isDeleted()})`);
+                    // Create cursor for current document
+                    const currentCursor = {
+                        changeSeq: entry.changeSeq,
+                        lastModified: entry.lastModified,
+                        docId: entry.docId,
+                    };
+                    // Yield immediately - this allows the caller to break early after each document
+                    // Deleted documents are included so external indexes can handle deletions
+                    yieldedDocuments++;
+                    yield { doc, cursor: currentCursor };
+                    prefetchedDocuments += await this.prefetchIterationWindow(indexSnapshot, i + 1);
+                }
+                catch (error) {
+                    this.logger.error(`Error processing document ${entry.docId}:`, error);
+                    // Stop processing on error
+                    throw error;
+                }
             }
         }
-        this.logger.debug(`Iteration completed`);
+        finally {
+            this.logger.debug(`Iteration completed`);
+            this.performanceCallback?.onSyncOperation?.({
+                operation: "iterateChangesSince",
+                time: Date.now() - startedAt,
+                details: {
+                    yieldedDocuments,
+                    prefetchedDocuments,
+                    loadedDocuments,
+                    startIndex,
+                },
+            });
+        }
     }
     /**
      * Incrementally update a cached document with new entries.
@@ -1256,7 +2005,7 @@ class BaseMindooDB {
         }
         this.logger.debug(`Applying ${newEntryMetadata.length} new entries to cached document ${docId}`);
         // Get current document heads to check if it changes
-        const headsBefore = Automerge.getHeads(cachedDoc.doc);
+        const headsBefore = automerge_adapter_1.Automerge.getHeads(cachedDoc.doc);
         // Filter entries to only include change entries (exclude snapshots)
         const entriesToApply = newEntryMetadata.filter(em => em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete");
         if (entriesToApply.length === 0) {
@@ -1340,12 +2089,12 @@ class BaseMindooDB {
         // Clone the cached document to avoid "outdated document" error
         // This happens when the document has been wrapped and returned to the user
         // Automerge marks documents as outdated when they're accessed, preventing direct mutation
-        const clonedDoc = Automerge.clone(cachedDoc.doc);
+        const clonedDoc = automerge_adapter_1.Automerge.clone(cachedDoc.doc);
         // Apply all changes at once (Automerge handles dependency ordering)
-        const result = Automerge.applyChanges(clonedDoc, changeBytes);
+        const result = automerge_adapter_1.Automerge.applyChanges(clonedDoc, changeBytes);
         const updatedDoc = result[0];
         // Check if document actually changed
-        const headsAfter = Automerge.getHeads(updatedDoc);
+        const headsAfter = automerge_adapter_1.Automerge.getHeads(updatedDoc);
         const headsChanged = JSON.stringify(headsBefore) !== JSON.stringify(headsAfter);
         if (!headsChanged) {
             this.logger.debug(`Document ${docId} heads unchanged after applying new entries`);
@@ -1369,57 +2118,113 @@ class BaseMindooDB {
             isDeleted,
         };
         // Update cache
-        this.docCache.set(docId, updatedInternalDoc);
+        this.storeCachedDocument(updatedInternalDoc);
+        this.markDocDirty(docId);
         return updatedInternalDoc;
     }
     /**
      * Internal method to load a document from the content-addressed store
      */
     async loadDocumentInternal(docId) {
+        const startedAt = Date.now();
+        const cacheCheckStartedAt = Date.now();
         // Check cache first
         if (this.docCache.has(docId)) {
             this.logger.debug(`Document ${docId} found in cache, returning cached version`);
-            return this.docCache.get(docId);
+            const cached = this.getCachedDocument(docId);
+            this.performanceCallback?.onDocumentLoad?.({
+                docId,
+                cacheHit: true,
+                metadataEntriesScanned: 0,
+                replayEntriesLoaded: 0,
+                snapshotUsed: false,
+                cacheCheckTime: Date.now() - cacheCheckStartedAt,
+                storeQueryTime: 0,
+                entryLoadTime: 0,
+                signatureVerificationTime: 0,
+                decryptionTime: 0,
+                automergeTime: 0,
+                totalTime: Date.now() - startedAt,
+            });
+            return cached;
         }
         this.logger.debug(`===== Starting to load document ${docId} from store =====`);
+        const cacheCheckTime = Date.now() - cacheCheckStartedAt;
+        let storeQueryTime = 0;
+        let entryLoadTime = 0;
+        let signatureVerificationTime = 0;
+        let decryptionTime = 0;
+        let automergeTime = 0;
         // Get all entry metadata for this document
         // TODO: Implement loading from last snapshot if available
         this.logger.debug(`Getting all entry hashes for document ${docId}`);
-        const allEntryMetadata = await this.store.findNewEntriesForDoc([], docId);
+        const storeQueryStartedAt = Date.now();
+        const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
+        storeQueryTime += Date.now() - storeQueryStartedAt;
         this.logger.debug(`Found ${allEntryMetadata.length} total entry hashes for document ${docId}`);
         if (allEntryMetadata.length === 0) {
             this.logger.debug(`No entry hashes found for document ${docId}, returning null`);
+            this.performanceCallback?.onDocumentLoad?.({
+                docId,
+                cacheHit: false,
+                metadataEntriesScanned: 0,
+                replayEntriesLoaded: 0,
+                snapshotUsed: false,
+                cacheCheckTime,
+                storeQueryTime,
+                entryLoadTime,
+                signatureVerificationTime,
+                decryptionTime,
+                automergeTime,
+                totalTime: Date.now() - startedAt,
+            });
             return null;
         }
         // Log all entry types
         const entryTypes = allEntryMetadata.map(em => `${em.entryType}@${em.createdAt}`).join(', ');
         this.logger.debug(`Entry types for ${docId}: ${entryTypes}`);
-        // Find the most recent snapshot (if any)
-        const snapshots = allEntryMetadata.filter(em => em.entryType === "doc_snapshot");
-        this.logger.debug(`Found ${snapshots.length} snapshot(s) for document ${docId}`);
-        let startFromSnapshot = false;
-        let snapshotMeta = null;
-        if (snapshots.length > 0) {
-            // Use the most recent snapshot
-            snapshots.sort((a, b) => b.createdAt - a.createdAt);
-            snapshotMeta = snapshots[0];
-            startFromSnapshot = true;
-            this.logger.debug(`Will start from snapshot ${snapshotMeta.id} created at ${snapshotMeta.createdAt}`);
+        const metadataById = new Map(allEntryMetadata.map((meta) => [meta.id, meta]));
+        const planStartedAt = Date.now();
+        const materializationPlan = (0, MaterializationPlanner_1.computeDocumentMaterializationPlan)(docId, allEntryMetadata, {
+            includeDiagnostics: true,
+        });
+        const planTime = Date.now() - planStartedAt;
+        this.performanceCallback?.onSyncOperation?.({
+            operation: "planDocumentMaterialization",
+            time: planTime,
+            details: {
+                docId,
+                metadataEntriesScanned: allEntryMetadata.length,
+                replayEntriesLoaded: materializationPlan.entryIdsToApply.length,
+                snapshotEntryId: materializationPlan.snapshotEntryId,
+                diagnostics: materializationPlan.diagnostics,
+            },
+        });
+        let startFromSnapshot = materializationPlan.snapshotEntryId !== null;
+        const snapshotMeta = materializationPlan.snapshotEntryId
+            ? (metadataById.get(materializationPlan.snapshotEntryId) || null)
+            : null;
+        if (startFromSnapshot && !snapshotMeta) {
+            this.logger.warn(`Planner referenced snapshot ${materializationPlan.snapshotEntryId} not found in metadata for ${docId}; falling back to replay without snapshot`);
+            startFromSnapshot = false;
+        }
+        if (startFromSnapshot && snapshotMeta) {
+            this.logger.debug(`Planner selected snapshot ${snapshotMeta.id} for ${docId}`);
         }
         else {
-            this.logger.debug(`No snapshot found, will start from scratch`);
-        }
-        // Get all entries (excluding snapshot entries - we'll handle delete separately)
-        // Include "doc_create", "doc_change", and "doc_delete" types as they all contain Automerge changes to apply
-        const entriesToLoad = startFromSnapshot
-            ? allEntryMetadata.filter(em => (em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete") && em.createdAt > snapshotMeta.createdAt)
-            : allEntryMetadata.filter(em => em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete");
-        this.logger.debug(`Will load ${entriesToLoad.length} entries for document ${docId} (after snapshot filter)`);
+            this.logger.debug(`Planner did not select a snapshot for ${docId}`);
+        }
+        const entriesToLoad = materializationPlan.entryIdsToApply
+            .map((id) => metadataById.get(id))
+            .filter((m) => m !== undefined);
+        this.logger.debug(`Planner returned ${entriesToLoad.length} replay entries for ${docId}; diagnostics=${JSON.stringify(materializationPlan.diagnostics || {})}`);
         // Load the snapshot first if we have one
         let doc = undefined;
         if (startFromSnapshot && snapshotMeta) {
             this.logger.debug(`Loading snapshot for document ${docId}`);
+            const snapshotLoadStartedAt = Date.now();
             const snapshotEntries = await this.store.getEntries([snapshotMeta.id]);
+            entryLoadTime += Date.now() - snapshotLoadStartedAt;
             this.logger.debug(`Retrieved ${snapshotEntries.length} snapshot entry(s) from store`);
             if (snapshotEntries.length > 0) {
                 const snapshotData = snapshotEntries[0];
@@ -1431,7 +2236,9 @@ class BaseMindooDB {
                 else {
                     // Verify signature against the encrypted snapshot (no decryption needed)
                     // We sign the encrypted payload, so anyone can verify signatures without decryption keys
+                    const signatureStartedAt = Date.now();
                     isValid = await this.tenant.verifySignature(snapshotData.encryptedData, snapshotData.signature, snapshotData.createdByPublicKey);
+                    signatureVerificationTime += Date.now() - signatureStartedAt;
                 }
                 if (!isValid) {
                     this.logger.warn(`Invalid signature for snapshot ${snapshotData.id}, falling back to loading from scratch`);
@@ -1441,14 +2248,18 @@ class BaseMindooDB {
                 else {
                     this.logger.debug(`Snapshot signature valid, decrypting snapshot`);
                     // Decrypt snapshot (only after signature verification passes)
+                    const decryptStartedAt = Date.now();
                     const decryptedSnapshot = await this.tenant.decryptPayload(snapshotData.encryptedData, snapshotData.decryptionKeyId);
+                    decryptionTime += Date.now() - decryptStartedAt;
                     this.logger.debug(`Decrypted snapshot (${snapshotData.encryptedData.length} -> ${decryptedSnapshot.length} bytes)`);
                     // Load snapshot using Automerge.load()
                     // This deserializes a full document snapshot from binary data
                     // According to Automerge docs: load() is equivalent to init() followed by loadIncremental()
                     this.logger.debug(`Loading snapshot into Automerge document`);
-                    doc = Automerge.load(decryptedSnapshot);
-                    this.logger.debug(`Successfully loaded snapshot, document heads: ${JSON.stringify(Automerge.getHeads(doc))}`);
+                    const automergeStartedAt = Date.now();
+                    doc = automerge_adapter_1.Automerge.load(decryptedSnapshot);
+                    automergeTime += Date.now() - automergeStartedAt;
+                    this.logger.debug(`Successfully loaded snapshot, document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
                     // Register the snapshot's automerge hash -> entry ID mapping
                     const parsed = (0, idGeneration_1.parseDocEntryId)(snapshotData.id);
                     if (parsed) {
@@ -1460,19 +2271,18 @@ class BaseMindooDB {
         // If we don't have a snapshot, start from scratch
         if (!doc) {
             this.logger.debug(`Initializing new Automerge document for ${docId}`);
-            doc = Automerge.init();
-            this.logger.debug(`Initialized empty document, heads: ${JSON.stringify(Automerge.getHeads(doc))}`);
+            doc = automerge_adapter_1.Automerge.init();
+            this.logger.debug(`Initialized empty document, heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
         }
-        // Sort entries by timestamp
-        entriesToLoad.sort((a, b) => a.createdAt - b.createdAt);
-        this.logger.debug(`Sorted ${entriesToLoad.length} entries by timestamp for document ${docId}`);
         // Load and apply all entries
         this.logger.debug(`Fetching ${entriesToLoad.length} entries from store for document ${docId}`);
+        const entryLoadStartedAt = Date.now();
         const entries = await this.store.getEntries(entriesToLoad.map(em => em.id));
+        entryLoadTime += Date.now() - entryLoadStartedAt;
         this.logger.debug(`Retrieved ${entries.length} entries from store for document ${docId}`);
         this.logger.debug(`Loading document ${docId}: found ${entries.length} entries to apply (${startFromSnapshot ? 'starting from snapshot' : 'starting from scratch'})`);
         // Log current document state before applying entries
-        this.logger.debug(`Document state before applying entries: heads=${JSON.stringify(Automerge.getHeads(doc))}`);
+        this.logger.debug(`Document state before applying entries: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
         // Filter entries for admin-only mode first
         const validEntries = entries.filter(entryData => {
             if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
@@ -1508,6 +2318,7 @@ class BaseMindooDB {
                 }
             }
             // Verify all signatures in parallel using cached keys
+            const signatureStartedAt = Date.now();
             const signatureVerificationResults = await Promise.all(validEntries.map(async (entryData) => {
                 const cryptoKey = keyMap.get(entryData.createdByPublicKey);
                 if (!cryptoKey) {
@@ -1517,6 +2328,7 @@ class BaseMindooDB {
                 const isValid = await this.verifySignatureWithKey(cryptoKey, entryData.encryptedData, entryData.signature);
                 return { entryData, isValid };
             }));
+            signatureVerificationTime += Date.now() - signatureStartedAt;
             // Filter out entries with invalid signatures
             const verifiedEntries = signatureVerificationResults
                 .filter(({ isValid }) => {
@@ -1533,10 +2345,12 @@ class BaseMindooDB {
                 // Parallel decryption - decrypt all entries concurrently
                 // Automerge handles dependency buffering internally, so we can decrypt all in parallel
                 this.logger.debug(`Decrypting ${verifiedEntries.length} entries in parallel`);
+                const decryptionStartedAt = Date.now();
                 const decryptionResults = await Promise.all(verifiedEntries.map(async (entryData) => {
                     const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
                     return { entryData, decryptedPayload };
                 }));
+                decryptionTime += Date.now() - decryptionStartedAt;
                 // Collect change bytes and register automerge hash mappings
                 const changeBytes = [];
                 for (const { entryData, decryptedPayload } of decryptionResults) {
@@ -1551,10 +2365,12 @@ class BaseMindooDB {
                 if (changeBytes.length > 0) {
                     this.logger.debug(`Applying ${changeBytes.length} changes to document ${docId} using batch applyChanges`);
                     try {
-                        const result = Automerge.applyChanges(doc, changeBytes);
+                        const automergeStartedAt = Date.now();
+                        const result = automerge_adapter_1.Automerge.applyChanges(doc, changeBytes);
                         doc = result[0];
+                        automergeTime += Date.now() - automergeStartedAt;
                         this.logger.debug(`Successfully applied ${changeBytes.length} changes to document ${docId}`);
-                        this.logger.debug(`Document state after applying changes: heads=${JSON.stringify(Automerge.getHeads(doc))}`);
+                        this.logger.debug(`Document state after applying changes: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
                     }
                     catch (error) {
                         this.logger.error(`Error applying changes to document ${docId}:`, error);
@@ -1566,13 +2382,16 @@ class BaseMindooDB {
         }
         // Extract metadata from document (doc is guaranteed to be defined at this point)
         this.logger.debug(`All entries applied successfully for document ${docId}`);
-        this.logger.debug(`Final document heads: ${JSON.stringify(Automerge.getHeads(doc))}`);
+        this.logger.debug(`Final document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
         const payload = doc;
         // Check if document was deleted by looking for a "doc_delete" type entry
         const hasDeleteEntry = allEntryMetadata.some(em => em.entryType === "doc_delete");
         const isDeleted = hasDeleteEntry;
         this.logger.debug(`Document ${docId} isDeleted: ${isDeleted}`);
-        const decryptionKeyId = payload._decryptionKeyId || "default";
+        // The authoritative decryptionKeyId comes from the doc_create entry's metadata,
+        // not from the Automerge payload (which does not store encryption metadata).
+        const createEntry = allEntryMetadata.find(em => em.entryType === "doc_create");
+        const decryptionKeyId = createEntry ? createEntry.decryptionKeyId : "default";
         // Get lastModified from payload, or use the timestamp of the last entry
         const lastEntry = entries.length > 0 ? entries[entries.length - 1] : null;
         const lastModified = payload._lastModified ||
@@ -1590,10 +2409,69 @@ class BaseMindooDB {
             isDeleted,
         };
         // Update cache
-        this.docCache.set(docId, internalDoc);
+        this.storeCachedDocument(internalDoc);
+        this.markDocDirty(docId);
         this.logger.debug(`===== Successfully loaded document ${docId} and cached it =====`);
+        this.performanceCallback?.onDocumentLoad?.({
+            docId,
+            cacheHit: false,
+            metadataEntriesScanned: allEntryMetadata.length,
+            replayEntriesLoaded: entriesToLoad.length,
+            snapshotUsed: startFromSnapshot && snapshotMeta !== null,
+            cacheCheckTime,
+            storeQueryTime,
+            entryLoadTime,
+            signatureVerificationTime,
+            decryptionTime,
+            automergeTime,
+            totalTime: Date.now() - startedAt,
+        });
         return internalDoc;
     }
+    /**
+     * Convert an Automerge document to a plain JS object, converting Text objects to strings.
+     * If using native Automerge (react-native-automerge-generated), this uses the native
+     * materialize() method which properly converts Text objects to strings.
+     * Falls back to direct access if native backend is not available.
+     */
+    convertAutomergeToJS(doc) {
+        // Check if this document has a native Automerge handle attached
+        // The native implementation attaches metadata with Symbol.for('_am_meta')
+        const STATE = Symbol.for('_am_meta');
+        const meta = doc[STATE];
+        if (meta && meta.handle && typeof meta.handle.materialize === 'function') {
+            // Use native materialize() which properly converts Text objects to strings
+            try {
+                const materialized = meta.handle.materialize('/');
+                return materialized;
+            }
+            catch (error) {
+                console.error('[MindooDB] Failed to materialize document:', error);
+                // Fall through to direct access
+            }
+        }
+        // Fallback: direct access (for WASM or if native fails)
+        const result = {};
+        const keys = Object.keys(doc);
+        for (const key of keys) {
+            const value = doc[key];
+            if (value !== null && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Uint8Array)) {
+                result[key] = this.convertAutomergeToJS(value);
+            }
+            else if (Array.isArray(value)) {
+                result[key] = value.map(item => {
+                    if (item !== null && typeof item === 'object' && !Array.isArray(item) && !(item instanceof Uint8Array)) {
+                        return this.convertAutomergeToJS(item);
+                    }
+                    return item;
+                });
+            }
+            else {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
     /**
      * Wrap an internal document in the MindooDoc interface.
      * This is the read-only wrapper - write methods throw errors.
@@ -1626,7 +2504,11 @@ class BaseMindooDB {
             getCreatedAt: () => internalDoc.createdAt,
             getLastModified: () => internalDoc.lastModified,
             isDeleted: () => internalDoc.isDeleted,
-            getData: () => createReadOnlyProxy(internalDoc.doc),
+            getData: () => {
+                // Convert Automerge document to plain JS object, converting Text objects to strings
+                const jsDoc = this.convertAutomergeToJS(internalDoc.doc);
+                return createReadOnlyProxy(jsDoc);
+            },
             // ========== Attachment Write Methods ==========
             // These throw errors in the read-only wrapper
             addAttachment: async () => {
@@ -1662,7 +2544,7 @@ class BaseMindooDB {
      * Get an attachment reference by ID from a document's _attachments array.
      */
     getAttachmentRefInternal(docId, attachmentId) {
-        const internalDoc = this.docCache.get(docId);
+        const internalDoc = this.getCachedDocument(docId);
         if (!internalDoc) {
             throw new Error(`Document ${docId} not found in cache`);
         }
@@ -2003,7 +2885,64 @@ class BaseMindooDB {
         };
     }
     /**
-     * Pull changes from a remote content-addressed store.
+     * Resolve a sync target to a ContentAddressedStore.
+     * Accepts either a raw store or a MindooDB instance (calls getStore()).
+     */
+    resolveStore(remote) {
+        if ('getStore' in remote && typeof remote.getStore === 'function') {
+            return remote.getStore();
+        }
+        return remote;
+    }
+    /**
+     * Apply a temporary network-auth identity override for a single sync call.
+     *
+     * Purpose:
+     * - Allows per-call authentication as a different user (for example admin bootstrap)
+     *   without changing the tenant's default connected user.
+     * - Returns a cleanup callback so caller can always restore default auth state in `finally`.
+     */
+    async applyNetworkAuthOverrideForSync(remoteStore, options) {
+        // Per-call override: use an alternate identity (for example, admin bootstrap)
+        // only for this sync operation.
+        const override = options?.networkAuthOverride;
+        if (!override) {
+            // No override requested: return a no-op cleanup callback for unified call sites.
+            return () => { };
+        }
+        const overrideCapableStore = remoteStore;
+        if (typeof overrideCapableStore.setSyncAuthOverride !== "function") {
+            // Local/in-memory stores do not support network auth override; keep default auth.
+            this.logger.warn("networkAuthOverride was provided, but remote store does not support auth override");
+            return () => { };
+        }
+        const subtle = this.tenant.getCryptoAdapter().getSubtle();
+        // Decrypt and import override signing key (Ed25519) for challenge signing.
+        const signingKeyBuffer = await this.tenant.decryptPrivateKey(override.user.userSigningKeyPair.privateKey, override.password, "signing");
+        const signingKey = await subtle.importKey("pkcs8", signingKeyBuffer, { name: "Ed25519" }, false, ["sign"]);
+        // Decrypt and import override encryption key (RSA-OAEP) so encrypted network
+        // entries can be decrypted with the same override identity.
+        const encryptionKeyBuffer = await this.tenant.decryptPrivateKey(override.user.userEncryptionKeyPair.privateKey, override.password, "encryption");
+        const encryptionKey = await subtle.importKey("pkcs8", encryptionKeyBuffer, { name: "RSA-OAEP", hash: "SHA-256" }, false, ["decrypt"]);
+        // Activate temporary override on the remote store.
+        overrideCapableStore.setSyncAuthOverride({
+            username: override.user.username,
+            signingKey,
+            privateEncryptionKey: encryptionKey,
+        });
+        // Return cleanup callback so caller can always restore normal auth in finally.
+        return () => {
+            if (typeof overrideCapableStore.clearSyncAuthOverride === "function") {
+                overrideCapableStore.clearSyncAuthOverride();
+            }
+            else {
+                // Backward-compatible fallback if explicit clear API is not implemented.
+                overrideCapableStore.setSyncAuthOverride?.(null);
+            }
+        };
+    }
+    /**
+     * Pull changes from a remote content-addressed store or another MindooDB instance.
      *
      * This method:
      * 1. Finds entries in the remote store that we don't have locally
@@ -2011,69 +2950,93 @@ class BaseMindooDB {
      * 3. Stores them in our local store
      * 4. Syncs the local store to process the new entries
      *
-     * @param remoteStore The remote store to pull entries from
-     * @return A promise that resolves when the pull is complete
+     * @param remote The remote store or MindooDB instance to pull entries from
+     * @param options Optional sync options for progress tracking, paging, and cancellation
+     * @return A promise that resolves with the sync result
      */
-    async pullChangesFrom(remoteStore) {
+    async pullChangesFrom(remote, options) {
+        const remoteStore = this.resolveStore(remote);
         if (this.store.getId() !== remoteStore.getId()) {
             throw new Error(`[BaseMindooDB] Cannot pull entries from the incompatible store ${this.store.getId()}`);
         }
         this.logger.info(`Pulling entries from remote store ${remoteStore.getId()}`);
-        // Get all entry IDs we already have in our local store
-        const localEntryIds = await this.store.getAllIds();
-        this.logger.debug(`Local store has ${localEntryIds.length} entry IDs`);
-        // Find entries in the remote store that we don't have
-        const newEntryMetadata = await remoteStore.findNewEntries(localEntryIds);
-        this.logger.debug(`Found ${newEntryMetadata.length} new entries in remote store`);
-        if (newEntryMetadata.length === 0) {
-            this.logger.debug(`No new entries to pull`);
-            return;
+        const restoreAuthOverride = await this.applyNetworkAuthOverrideForSync(remoteStore, options);
+        try {
+            const syncResult = await this.syncEntriesFromStore(remoteStore, this.store, options);
+            this.logger.debug(`Transferred ${syncResult.transferred} entries from remote store`);
+            if (syncResult.cancelled) {
+                this.logger.info(`Pull cancelled after transferring ${syncResult.transferred} entries`);
+                return { transferredEntries: syncResult.transferred, cancelled: true };
+            }
+            if (syncResult.transferred === 0) {
+                this.logger.debug(`No new entries to pull`);
+                return { transferredEntries: 0, cancelled: false };
+            }
+            options?.onProgress?.({
+                phase: 'processing',
+                message: `Processing ${syncResult.transferred} new entries...`,
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.transferred,
+            });
+            // Sync the local store to process the new entries
+            // This will update the index, cache, and processedEntryIds
+            await this.syncStoreChanges();
+            this.logger.info(`Pull complete, synced ${syncResult.transferred} entries`);
+            options?.onProgress?.({
+                phase: 'complete',
+                message: `Pull complete: ${syncResult.transferred} entries synced`,
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.transferred,
+            });
+            return { transferredEntries: syncResult.transferred, cancelled: false };
+        }
+        finally {
+            restoreAuthOverride();
         }
-        // Get the full entries from the remote store
-        const newEntries = await remoteStore.getEntries(newEntryMetadata.map(em => em.id));
-        this.logger.debug(`Retrieved ${newEntries.length} entries from remote store`);
-        // Store all entries in our local store
-        // The putEntries method handles deduplication (no-op if entry already exists)
-        await this.store.putEntries(newEntries);
-        this.logger.debug(`Stored ${newEntries.length} entries in local store`);
-        // Sync the local store to process the new entries
-        // This will update the index, cache, and processedEntryIds
-        await this.syncStoreChanges();
-        this.logger.info(`Pull complete, synced ${newEntries.length} entries`);
     }
     /**
-     * Push changes to a remote content-addressed store.
+     * Push changes to a remote content-addressed store or another MindooDB instance.
      *
      * This method:
      * 1. Finds entries in our local store that the remote doesn't have
      * 2. Retrieves those entries from our local store
      * 3. Stores them in the remote store
      *
-     * @param remoteStore The remote store to push entries to
-     * @return A promise that resolves when the push is complete
+     * @param remote The remote store or MindooDB instance to push entries to
+     * @param options Optional sync options for progress tracking, paging, and cancellation
+     * @return A promise that resolves with the sync result
      */
-    async pushChangesTo(remoteStore) {
+    async pushChangesTo(remote, options) {
+        const remoteStore = this.resolveStore(remote);
         if (this.store.getId() !== remoteStore.getId()) {
             throw new Error(`[BaseMindooDB] Cannot push entries to the incompatible store ${this.store.getId()}`);
         }
         this.logger.info(`Pushing entries to remote store ${remoteStore.getId()}`);
-        // Get all entry IDs the remote store already has
-        const remoteEntryIds = await remoteStore.getAllIds();
-        this.logger.debug(`Remote store has ${remoteEntryIds.length} entry IDs`);
-        // Find entries in our local store that the remote doesn't have
-        const newEntryMetadata = await this.store.findNewEntries(remoteEntryIds);
-        this.logger.debug(`Found ${newEntryMetadata.length} new entries to push`);
-        if (newEntryMetadata.length === 0) {
-            this.logger.debug(`No new entries to push`);
-            return;
+        const restoreAuthOverride = await this.applyNetworkAuthOverrideForSync(remoteStore, options);
+        try {
+            const syncResult = await this.syncEntriesFromStore(this.store, remoteStore, options);
+            this.logger.debug(`Transferred ${syncResult.transferred} entries to remote store`);
+            if (syncResult.cancelled) {
+                this.logger.info(`Push cancelled after transferring ${syncResult.transferred} entries`);
+                return { transferredEntries: syncResult.transferred, cancelled: true };
+            }
+            if (syncResult.transferred === 0) {
+                this.logger.debug(`No new entries to push`);
+            }
+            else {
+                this.logger.info(`Pushed ${syncResult.transferred} entries to remote store`);
+            }
+            options?.onProgress?.({
+                phase: 'complete',
+                message: `Push complete: ${syncResult.transferred} entries transferred`,
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.transferred,
+            });
+            return { transferredEntries: syncResult.transferred, cancelled: false };
+        }
+        finally {
+            restoreAuthOverride();
         }
-        // Get the full entries from our local store
-        const newEntries = await this.store.getEntries(newEntryMetadata.map(em => em.id));
-        this.logger.debug(`Retrieved ${newEntries.length} entries from local store`);
-        // Store all entries in the remote store
-        // The putEntries method handles deduplication (no-op if entry already exists)
-        await remoteStore.putEntries(newEntries);
-        this.logger.info(`Pushed ${newEntries.length} entries to remote store`);
     }
 }
 exports.BaseMindooDB = BaseMindooDB;