mindoodb 0.0.1 → 0.0.3

package/dist/core/BaseMindooDB.js

@@ -1,43 +1,15 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BaseMindooDB = void 0;
-const Automerge = __importStar(require("@automerge/automerge"));
+// Import platform-appropriate Automerge implementation
+// React Native: native Rust (react-native-automerge-generated)
+// Browser/Node.js: WASM (@automerge/automerge/slim)
+const automerge_adapter_1 = require("./automerge-adapter");
 const uuid_1 = require("uuid");
+const bloom_1 = require("./appendonlystores/bloom");
 const idGeneration_1 = require("./utils/idGeneration");
 const errors_1 = require("./errors");
+const logging_1 = require("./logging");
 /**
  * Default chunk size for attachments: 256KB
  */
@@ -62,7 +34,7 @@ const DEFAULT_CHUNK_SIZE_BYTES = 256 * 1024;
  * - Automerge.applyChanges(doc, changes) - verify method signature
  */
 class BaseMindooDB {
-    constructor(tenant, store, attachmentStore, attachmentConfig, adminOnlyDb = false) {
+    constructor(tenant, store, attachmentStore, attachmentConfig, adminOnlyDb = false, logger, performanceCallback) {
         // Internal index: sorted array of document entries, maintained in order by (lastModified, docId)
         // This allows efficient incremental processing without sorting on each call
         this.index = [];
@@ -72,14 +44,23 @@ class BaseMindooDB {
         this.docCache = new Map();
         // Track which entry IDs we've already processed
         this.processedEntryIds = [];
+        this.processedEntryCursor = null;
         // Index: automergeHash -> entryId for each document
         // Used for resolving Automerge dependency hashes to entry IDs
         this.automergeHashToEntryId = new Map(); // Map<docId, Map<automergeHash, entryId>>
+        // Cache for imported public keys (CryptoKey objects) to avoid re-importing the same key
+        // Map<publicKeyPEM, CryptoKey>
+        this.publicKeyCache = new Map();
         this.tenant = tenant;
         this.store = store;
         this.attachmentStore = attachmentStore;
         this.chunkSizeBytes = attachmentConfig?.chunkSizeBytes ?? DEFAULT_CHUNK_SIZE_BYTES;
         this._isAdminOnlyDb = adminOnlyDb;
+        // Create logger if not provided (for backward compatibility)
+        this.logger =
+            logger ||
+                new logging_1.MindooLogger((0, logging_1.getDefaultLogLevel)(), "BaseMindooDB", true);
+        this.performanceCallback = performanceCallback;
     }
     /**
      * Get the admin public key from the tenant.
@@ -91,6 +72,44 @@ class BaseMindooDB {
     isAdminOnlyDb() {
         return this._isAdminOnlyDb;
     }
+    /**
+     * Get or import a CryptoKey for signature verification, with caching.
+     * This avoids re-importing the same public key multiple times.
+     */
+    async getOrImportPublicKey(publicKey) {
+        // Check cache first
+        if (this.publicKeyCache.has(publicKey)) {
+            return this.publicKeyCache.get(publicKey);
+        }
+        // Validate the public key is trusted
+        const directory = await this.tenant.openDirectory();
+        const isTrusted = await directory.validatePublicSigningKey(publicKey);
+        if (!isTrusted) {
+            this.logger.warn(`Public key not trusted: ${publicKey}`);
+            return null;
+        }
+        // Import the key
+        const subtle = this.tenant.getCryptoAdapter().getSubtle();
+        const publicKeyBuffer = this.tenant.pemToArrayBuffer(publicKey);
+        const cryptoKey = await subtle.importKey("spki", publicKeyBuffer, {
+            name: "Ed25519",
+        }, false, // not extractable
+        ["verify"]);
+        // Cache the imported key
+        this.publicKeyCache.set(publicKey, cryptoKey);
+        return cryptoKey;
+    }
+    /**
+     * Verify a signature using a pre-imported CryptoKey.
+     * This bypasses the key import step for better performance when keys are cached.
+     */
+    async verifySignatureWithKey(cryptoKey, payload, signature) {
+        const subtle = this.tenant.getCryptoAdapter().getSubtle();
+        const isValid = await subtle.verify({
+            name: "Ed25519",
+        }, cryptoKey, signature.buffer, payload.buffer);
+        return isValid;
+    }
     /**
      * Compare two index entries for sorting.
      * Returns negative if a < b, positive if a > b, 0 if equal.
@@ -107,25 +126,33 @@ class BaseMindooDB {
      * When a document changes, it's removed from its current position and inserted
      * at the correct sorted position to maintain order by (lastModified, docId).
      *
+     * Optimized to only update lookup map for entries that actually moved.
+     *
      * @param docId The document ID
      * @param lastModified The new last modified timestamp
      * @param isDeleted Whether the document is deleted
      */
     updateIndex(docId, lastModified, isDeleted) {
-        // Remove existing entry if present
+        const newEntry = { docId, lastModified, isDeleted };
         const existingIndex = this.indexLookup.get(docId);
+        // Check if the entry already exists and hasn't changed position
         if (existingIndex !== undefined) {
-            // Remove from array
+            const existingEntry = this.index[existingIndex];
+            // If position hasn't changed (same lastModified and docId), no update needed
+            if (existingEntry.lastModified === lastModified && existingEntry.isDeleted === isDeleted) {
+                return; // No change needed
+            }
+            // Remove from current position
             this.index.splice(existingIndex, 1);
-            // Update lookup map for all entries after the removed one
-            for (let i = existingIndex; i < this.index.length; i++) {
+            // Update lookup map for entries that moved (only those after the removed position)
+            // We'll update the full range after insertion to be safe
+            const minAffectedIndex = Math.min(existingIndex, this.index.length);
+            for (let i = minAffectedIndex; i < this.index.length; i++) {
                 this.indexLookup.set(this.index[i].docId, i);
             }
-            // Remove the entry from lookup
             this.indexLookup.delete(docId);
         }
         // Find insertion point using binary search to maintain sorted order
-        const newEntry = { docId, lastModified, isDeleted };
         let insertIndex = this.index.length; // Default to end
         // Binary search for insertion point
         let left = 0;
@@ -144,7 +171,8 @@ class BaseMindooDB {
         }
         // Insert at the correct position
         this.index.splice(insertIndex, 0, newEntry);
-        // Update lookup map for all entries from insertion point onwards
+        // Update lookup map for entries from insertion point onwards
+        // Only update entries that actually moved (from insertIndex to end)
         for (let i = insertIndex; i < this.index.length; i++) {
             this.indexLookup.set(this.index[i].docId, i);
         }
@@ -182,7 +210,7 @@ class BaseMindooDB {
                 entryIds.push(entryId);
             }
             else {
-                console.warn(`[BaseMindooDB] Could not resolve automerge hash ${hash} to entry ID for doc ${docId}`);
+                this.logger.warn(`Could not resolve automerge hash ${hash} to entry ID for doc ${docId}`);
             }
         }
         return entryIds;
@@ -197,7 +225,7 @@ class BaseMindooDB {
      * Initialize the database instance.
      */
     async initialize() {
-        console.log(`[BaseMindooDB] Initializing database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
+        this.logger.info(`Initializing database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
         await this.syncStoreChanges();
     }
     /**
@@ -206,13 +234,13 @@ class BaseMindooDB {
      * On first call (when processedEntryIds is empty), it will process all entries.
      */
     async syncStoreChanges() {
-        console.log(`[BaseMindooDB] Syncing store changes for database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
-        console.log(`[BaseMindooDB] Already processed ${this.processedEntryIds.length} entry IDs`);
+        this.logger.debug(`Syncing store changes for database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
+        this.logger.debug(`Already processed ${this.processedEntryIds.length} entry IDs`);
         // Find new entries that we haven't processed yet
-        const newEntryMetadata = await this.store.findNewEntries(this.processedEntryIds);
-        console.log(`[BaseMindooDB] Found ${newEntryMetadata.length} new entries`);
+        const newEntryMetadata = await this.getNewEntryMetadataForSync();
+        this.logger.debug(`Found ${newEntryMetadata.length} new entries`);
         if (newEntryMetadata.length === 0) {
-            console.log(`[BaseMindooDB] No new entries to process`);
+            this.logger.debug(`No new entries to process`);
             return;
         }
         // Group new entries by document ID
@@ -224,49 +252,191 @@ class BaseMindooDB {
             entriesByDoc.get(entryMeta.docId).push(entryMeta);
         }
         // Process each document with new entries
-        // Reload documents to apply all changes (including new ones)
-        console.log(`[BaseMindooDB] Processing ${entriesByDoc.size} documents with new entries`);
-        for (const [docId, entryMetadataList] of entriesByDoc) {
+        // Use incremental cache updates when possible
+        // Process documents in parallel with concurrency limit
+        this.logger.debug(`Processing ${entriesByDoc.size} documents with new entries`);
+        // Helper function to process a single document
+        const processDocument = async (docId, entryMetadataList) => {
             try {
-                console.log(`[BaseMindooDB] ===== Processing document ${docId} with ${entryMetadataList.length} new entry(s) in syncStoreChanges =====`);
-                // Clear cache for this document so it gets reloaded with all entries
-                this.docCache.delete(docId);
-                console.log(`[BaseMindooDB] Cleared cache for document ${docId}`);
-                // Reload document (this will load all entries including new ones)
-                console.log(`[BaseMindooDB] About to call loadDocumentInternal for document ${docId}`);
-                const doc = await this.loadDocumentInternal(docId);
-                console.log(`[BaseMindooDB] loadDocumentInternal returned for document ${docId}, result: ${doc ? 'success' : 'null'}`);
-                if (doc) {
-                    console.log(`[BaseMindooDB] Successfully reloaded document ${docId}, updating index`);
-                    this.updateIndex(docId, doc.lastModified, doc.isDeleted);
-                    console.log(`[BaseMindooDB] Updated index for document ${docId} (lastModified: ${doc.lastModified}, isDeleted: ${doc.isDeleted})`);
+                this.logger.debug(`===== Processing document ${docId} with ${entryMetadataList.length} new entry(s) in syncStoreChanges =====`);
+                // Check if document is cached
+                const cachedDoc = this.docCache.get(docId);
+                let updatedDoc = null;
+                if (cachedDoc) {
+                    // Document is cached - try incremental update
+                    this.logger.debug(`Document ${docId} found in cache, attempting incremental update`);
+                    try {
+                        updatedDoc = await this.applyNewEntriesToCachedDocument(cachedDoc, entryMetadataList);
+                        if (updatedDoc) {
+                            this.logger.debug(`Successfully updated cached document ${docId} incrementally`);
+                            // Only update index if document actually changed
+                            this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted);
+                            this.logger.debug(`Updated index for document ${docId} (lastModified: ${updatedDoc.lastModified}, isDeleted: ${updatedDoc.isDeleted})`);
+                        }
+                        else {
+                            this.logger.debug(`Document ${docId} unchanged after applying new entries, skipping index update`);
+                        }
+                    }
+                    catch (error) {
+                        // If incremental update fails, fall back to full reload
+                        this.logger.warn(`Incremental update failed for document ${docId}, falling back to full reload:`, error);
+                        this.docCache.delete(docId);
+                        updatedDoc = await this.loadDocumentInternal(docId);
+                        if (updatedDoc) {
+                            this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted);
+                        }
+                    }
                 }
                 else {
-                    console.warn(`[BaseMindooDB] Document ${docId} returned null from loadDocumentInternal`);
+                    // Document not cached - load from scratch
+                    this.logger.debug(`Document ${docId} not in cache, loading from store`);
+                    updatedDoc = await this.loadDocumentInternal(docId);
+                    if (updatedDoc) {
+                        this.logger.debug(`Successfully loaded document ${docId}, updating index`);
+                        this.updateIndex(docId, updatedDoc.lastModified, updatedDoc.isDeleted);
+                        this.logger.debug(`Updated index for document ${docId} (lastModified: ${updatedDoc.lastModified}, isDeleted: ${updatedDoc.isDeleted})`);
+                    }
+                    else {
+                        this.logger.warn(`Document ${docId} returned null from loadDocumentInternal`);
+                    }
                 }
             }
             catch (error) {
                 // Handle missing symmetric key gracefully - skip documents we can't decrypt
                 if (error instanceof errors_1.SymmetricKeyNotFoundError) {
-                    console.log(`[BaseMindooDB] Skipping document ${docId} - missing key: ${error.keyId}`);
+                    this.logger.debug(`Skipping document ${docId} - missing key: ${error.keyId}`);
                     // Mark entry IDs as processed so we don't retry them
                     this.processedEntryIds.push(...entryMetadataList.map(em => em.id));
-                    continue; // Skip to next document
-                }
-                console.error(`[BaseMindooDB] ===== ERROR processing document ${docId} in syncStoreChanges =====`);
-                console.error(`[BaseMindooDB] Error type: ${error instanceof Error ? error.constructor.name : typeof error}`);
-                console.error(`[BaseMindooDB] Error message: ${error instanceof Error ? error.message : String(error)}`);
-                if (error instanceof Error && error.stack) {
-                    console.error(`[BaseMindooDB] Error stack: ${error.stack}`);
+                    return; // Skip this document
                 }
+                this.logger.error(`===== ERROR processing document ${docId} in syncStoreChanges =====`, error);
                 // Re-throw the error so we can see what's happening in the test
-                console.error(`[BaseMindooDB] Re-throwing error for document ${docId}`);
                 throw error;
             }
+        };
+        // Process documents in parallel with concurrency limit (default: 10)
+        const maxConcurrency = 10;
+        const documentEntries = Array.from(entriesByDoc.entries());
+        // Process in batches to limit concurrency
+        const results = [];
+        for (let i = 0; i < documentEntries.length; i += maxConcurrency) {
+            const batch = documentEntries.slice(i, i + maxConcurrency);
+            const batchResults = await Promise.allSettled(batch.map(([docId, entryMetadataList]) => processDocument(docId, entryMetadataList)));
+            results.push(...batchResults);
+        }
+        // Check for errors (excluding SymmetricKeyNotFoundError which is handled gracefully)
+        for (let i = 0; i < results.length; i++) {
+            const result = results[i];
+            if (result.status === 'rejected') {
+                const error = result.reason;
+                if (!(error instanceof errors_1.SymmetricKeyNotFoundError)) {
+                    // Re-throw non-key errors
+                    throw error;
+                }
+            }
         }
         // Append new entry IDs to our processed list
         this.processedEntryIds.push(...newEntryMetadata.map(em => em.id));
-        console.log(`[BaseMindooDB] Synced ${newEntryMetadata.length} new entries, index now has ${this.index.length} documents`);
+        this.logger.debug(`Synced ${newEntryMetadata.length} new entries, index now has ${this.index.length} documents`);
+    }
+    supportsCursorScan(store) {
+        return typeof store.scanEntriesSince === "function";
+    }
+    async scanAllMetadata(store, filters) {
+        if (!this.supportsCursorScan(store)) {
+            if (filters?.docId) {
+                return store.findNewEntriesForDoc([], filters.docId);
+            }
+            return store.findNewEntries([]);
+        }
+        const all = [];
+        let cursor = null;
+        while (true) {
+            const page = await store.scanEntriesSince(cursor, 1000, filters);
+            all.push(...page.entries);
+            cursor = page.nextCursor;
+            if (!page.hasMore) {
+                break;
+            }
+        }
+        return all;
+    }
+    async getNewEntryMetadataForSync() {
+        if (!this.supportsCursorScan(this.store)) {
+            return this.store.findNewEntries(this.processedEntryIds);
+        }
+        const allNew = [];
+        let cursor = this.processedEntryCursor;
+        while (true) {
+            const page = await this.store.scanEntriesSince(cursor, 1000);
+            allNew.push(...page.entries);
+            cursor = page.nextCursor;
+            if (!page.hasMore) {
+                break;
+            }
+        }
+        this.processedEntryCursor = cursor;
+        return allNew;
+    }
+    async syncEntriesFromStore(sourceStore, targetStore) {
+        let transferred = 0;
+        let targetBloom = null;
+        if (typeof targetStore.getIdBloomSummary === "function") {
+            try {
+                targetBloom = await targetStore.getIdBloomSummary();
+            }
+            catch (error) {
+                this.logger.warn("Failed to get bloom summary from target store, falling back to exact checks", error);
+            }
+        }
+        if (this.supportsCursorScan(sourceStore)) {
+            let cursor = null;
+            while (true) {
+                const page = await sourceStore.scanEntriesSince(cursor, 1000);
+                if (page.entries.length > 0) {
+                    const ids = page.entries.map((m) => m.id);
+                    let definitelyMissing = [];
+                    let maybeExisting = ids;
+                    if (targetBloom) {
+                        definitelyMissing = [];
+                        maybeExisting = [];
+                        for (const id of ids) {
+                            if ((0, bloom_1.bloomMightContainId)(targetBloom, id)) {
+                                maybeExisting.push(id);
+                            }
+                            else {
+                                definitelyMissing.push(id);
+                            }
+                        }
+                    }
+                    let missingIds = definitelyMissing;
+                    if (maybeExisting.length > 0) {
+                        const existing = await targetStore.hasEntries(maybeExisting);
+                        const existingSet = new Set(existing);
+                        const maybeMissing = maybeExisting.filter((id) => !existingSet.has(id));
+                        missingIds = missingIds.concat(maybeMissing);
+                    }
+                    if (missingIds.length > 0) {
+                        const missingEntries = await sourceStore.getEntries(missingIds);
+                        await targetStore.putEntries(missingEntries);
+                        transferred += missingEntries.length;
+                    }
+                }
+                cursor = page.nextCursor;
+                if (!page.hasMore) {
+                    break;
+                }
+            }
+            return transferred;
+        }
+        const targetIds = await targetStore.getAllIds();
+        const sourceNewMetadata = await sourceStore.findNewEntries(targetIds);
+        if (sourceNewMetadata.length === 0) {
+            return 0;
+        }
+        const sourceEntries = await sourceStore.getEntries(sourceNewMetadata.map((m) => m.id));
+        await targetStore.putEntries(sourceEntries);
+        return sourceEntries.length;
     }
     getStore() {
         return this.store;
@@ -306,72 +476,72 @@ class BaseMindooDB {
         }
         // Generate UUID7 for document ID
         const docId = (0, uuid_1.v7)();
-        console.log(`[BaseMindooDB] Creating document ${docId} with key ${keyId}${useCustomSigningKey ? ' using custom signing key' : ''}`);
+        this.logger.debug(`Creating document ${docId} with key ${keyId}${useCustomSigningKey ? ' using custom signing key' : ''}`);
         // Create initial Automerge document
-        const initialDoc = Automerge.init();
+        const initialDoc = automerge_adapter_1.Automerge.init();
         // Create the first change
         const now = Date.now();
-        console.log(`[BaseMindooDB] Creating initial Automerge change for document ${docId}`);
+        this.logger.debug(`Creating initial Automerge change for document ${docId}`);
         let newDoc;
         try {
-            newDoc = Automerge.change(initialDoc, (doc) => {
+            newDoc = automerge_adapter_1.Automerge.change(initialDoc, (doc) => {
                 // Store metadata in the document payload
                 // We need to modify the document to ensure a change is created
                 doc._attachments = [];
             });
-            console.log(`[BaseMindooDB] Successfully created Automerge change, document heads: ${JSON.stringify(Automerge.getHeads(newDoc))}`);
+            this.logger.debug(`Successfully created Automerge change, document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
         }
         catch (error) {
-            console.error(`[BaseMindooDB] Error in Automerge.change for document ${docId}:`, error);
+            this.logger.error(`Error in Automerge.change for document ${docId}:`, error);
             throw error;
         }
         // Get the change bytes from the document
-        console.log(`[BaseMindooDB] Getting change bytes from document ${docId}`);
-        const changeBytes = Automerge.getLastLocalChange(newDoc);
+        this.logger.debug(`Getting change bytes from document ${docId}`);
+        const changeBytes = automerge_adapter_1.Automerge.getLastLocalChange(newDoc);
         if (!changeBytes) {
             throw new Error("Failed to get change bytes from Automerge document");
         }
-        console.log(`[BaseMindooDB] Got change bytes: ${changeBytes.length} bytes`);
+        this.logger.debug(`Got change bytes: ${changeBytes.length} bytes`);
         // Decode the change to get hash and dependencies
-        console.log(`[BaseMindooDB] Decoding change to get hash and dependencies`);
+        this.logger.debug(`Decoding change to get hash and dependencies`);
         let decodedChange;
         try {
-            decodedChange = Automerge.decodeChange(changeBytes);
-            console.log(`[BaseMindooDB] Successfully decoded change, hash: ${decodedChange.hash}, deps: ${decodedChange.deps?.length || 0}`);
+            decodedChange = automerge_adapter_1.Automerge.decodeChange(changeBytes);
+            this.logger.debug(`Successfully decoded change, hash: ${decodedChange.hash}, deps: ${decodedChange.deps?.length || 0}`);
         }
         catch (error) {
-            console.error(`[BaseMindooDB] Error decoding change for document ${docId}:`, error);
+            this.logger.error(`Error decoding change for document ${docId}:`, error);
             throw error;
         }
         const automergeHash = decodedChange.hash;
         const automergeDepHashes = decodedChange.deps || []; // First change has no dependencies
         // Encrypt the change payload first
-        console.log(`[BaseMindooDB] Encrypting change payload for document ${docId}`);
+        this.logger.debug(`Encrypting change payload for document ${docId}`);
         const encryptedPayload = await this.tenant.encryptPayload(changeBytes, keyId);
-        console.log(`[BaseMindooDB] Encrypted payload: ${changeBytes.length} -> ${encryptedPayload.length} bytes`);
+        this.logger.debug(`Encrypted payload: ${changeBytes.length} -> ${encryptedPayload.length} bytes`);
         // Compute content hash from encrypted data
         const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedPayload, this.getSubtle());
-        console.log(`[BaseMindooDB] Computed content hash: ${contentHash.substring(0, 16)}...`);
+        this.logger.debug(`Computed content hash: ${contentHash.substring(0, 16)}...`);
         // Generate entry ID with blockchain-like chaining
         const entryId = await (0, idGeneration_1.generateDocEntryId)(docId, automergeHash, automergeDepHashes, this.getSubtle());
-        console.log(`[BaseMindooDB] Generated entry ID: ${entryId}`);
+        this.logger.debug(`Generated entry ID: ${entryId}`);
         // Resolve Automerge dependency hashes to entry IDs (empty for first change)
         const dependencyIds = this.resolveAutomergeDepsToEntryIds(docId, automergeDepHashes);
         // Sign the encrypted payload - either with custom key or current user's key
         let signature;
         let createdByPublicKey;
         if (useCustomSigningKey) {
-            console.log(`[BaseMindooDB] Signing encrypted payload for document ${docId} with provided key`);
+            this.logger.debug(`Signing encrypted payload for document ${docId} with provided key`);
             signature = await this.tenant.signPayloadWithKey(encryptedPayload, signingKeyPair, signingKeyPassword);
             createdByPublicKey = signingKeyPair.publicKey;
         }
         else {
-            console.log(`[BaseMindooDB] Signing encrypted payload for document ${docId}`);
+            this.logger.debug(`Signing encrypted payload for document ${docId}`);
             const currentUser = await this.tenant.getCurrentUserId();
             signature = await this.tenant.signPayload(encryptedPayload);
             createdByPublicKey = currentUser.userSigningPublicKey;
         }
-        console.log(`[BaseMindooDB] Signed payload, signature length: ${signature.length} bytes`);
+        this.logger.debug(`Signed payload, signature length: ${signature.length} bytes`);
         // Create entry metadata
         const entryMetadata = {
             entryType: "doc_create",
@@ -407,8 +577,8 @@ class BaseMindooDB {
         // Update cache and index
         this.docCache.set(docId, internalDoc);
         this.updateIndex(docId, internalDoc.lastModified, false);
-        console.log(`[BaseMindooDB] Document ${docId} created successfully`);
-        console.log(`[BaseMindooDB] Document ${docId} cached and indexed (lastModified: ${internalDoc.lastModified})`);
+        this.logger.info(`Document ${docId} created successfully`);
+        this.logger.debug(`Document ${docId} cached and indexed (lastModified: ${internalDoc.lastModified})`);
         return this.wrapDocument(internalDoc);
     }
     async getDocument(docId) {
@@ -422,9 +592,9 @@ class BaseMindooDB {
         return this.wrapDocument(internalDoc);
     }
     async getDocumentAtTimestamp(docId, timestamp) {
-        console.log(`[BaseMindooDB] Getting document ${docId} at timestamp ${timestamp}`);
+        this.logger.debug(`Getting document ${docId} at timestamp ${timestamp}`);
         // Get all entry metadata for this document
-        const allEntryMetadata = await this.store.findNewEntriesForDoc([], docId);
+        const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
         // Filter entries up to the timestamp
         const relevantEntries = allEntryMetadata
             .filter(em => em.createdAt <= timestamp)
@@ -435,24 +605,24 @@ class BaseMindooDB {
         // Load entries
         const entries = await this.store.getEntries(relevantEntries.map(em => em.id));
         // Apply changes in order
-        let doc = Automerge.init();
+        let doc = automerge_adapter_1.Automerge.init();
         for (const entryData of entries) {
             // Admin-only mode: only accept entries signed by the admin key
             if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
-                console.warn(`[BaseMindooDB] Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
+                this.logger.warn(`Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
                 continue;
             }
             // Verify signature against the encrypted payload (no decryption needed)
             // We sign the encrypted payload, so anyone can verify signatures without decryption keys
             const isValid = await this.tenant.verifySignature(entryData.encryptedData, entryData.signature, entryData.createdByPublicKey);
             if (!isValid) {
-                console.warn(`[BaseMindooDB] Invalid signature for entry ${entryData.id}, skipping`);
+                this.logger.warn(`Invalid signature for entry ${entryData.id}, skipping`);
                 continue;
             }
             // Decrypt payload (only after signature verification passes)
             const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
             // Apply change using loadIncremental - this is the recommended way for binary change data
-            doc = Automerge.loadIncremental(doc, decryptedPayload);
+            doc = automerge_adapter_1.Automerge.loadIncremental(doc, decryptedPayload);
             // Register the automerge hash -> entry ID mapping for future dependency resolution
             const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
             if (parsed) {
@@ -461,24 +631,125 @@ class BaseMindooDB {
         }
         // Check if document was deleted at this timestamp
         // Look for a "doc_delete" type entry in the relevant entries
-        const hasDeleteEntry = relevantEntries.some(em => em.entryType === "doc_delete" && em.createdAt <= timestamp);
-        if (hasDeleteEntry) {
-            return null; // Document was deleted at this time
-        }
+        const deleteEntry = relevantEntries.find(em => em.entryType === "doc_delete" && em.createdAt <= timestamp);
+        const isDeleted = deleteEntry !== undefined;
         // Find the first entry to get createdAt and decryptionKeyId
         const firstEntry = relevantEntries.length > 0 ? relevantEntries[0] : null;
         const createdAt = firstEntry ? firstEntry.createdAt : timestamp;
         const decryptionKeyId = firstEntry ? firstEntry.decryptionKeyId : "default";
+        // Use delete entry timestamp as lastModified if deleted, otherwise use the requested timestamp
+        const lastModified = isDeleted && deleteEntry ? deleteEntry.createdAt : timestamp;
         const internalDoc = {
             id: docId,
             doc,
             createdAt,
-            lastModified: timestamp,
+            lastModified,
             decryptionKeyId,
-            isDeleted: false,
+            isDeleted,
         };
         return this.wrapDocument(internalDoc);
     }
+    async *iterateDocumentHistory(docId) {
+        this.logger.debug(`Iterating document history for ${docId}`);
+        // Get all entry metadata for this document
+        const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
+        // Filter to only doc_create, doc_change, and doc_delete entries (exclude snapshots)
+        const relevantEntries = allEntryMetadata
+            .filter(em => em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete")
+            .sort((a, b) => a.createdAt - b.createdAt);
+        if (relevantEntries.length === 0) {
+            return; // Document has no history
+        }
+        // Load all entries
+        const entries = await this.store.getEntries(relevantEntries.map(em => em.id));
+        // Build a map for quick lookup
+        const entryMap = new Map(entries.map(e => [e.id, e]));
+        // Apply changes in order
+        let currentDoc = null;
+        let createdAt = null;
+        let decryptionKeyId = "default";
+        for (const entryMetadata of relevantEntries) {
+            const entryData = entryMap.get(entryMetadata.id);
+            if (!entryData) {
+                this.logger.warn(`Entry ${entryMetadata.id} not found in store, skipping`);
+                continue;
+            }
+            // Admin-only mode: only accept entries signed by the admin key
+            if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
+                this.logger.warn(`Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
+                continue;
+            }
+            // Verify signature
+            const isValid = await this.tenant.verifySignature(entryData.encryptedData, entryData.signature, entryData.createdByPublicKey);
+            if (!isValid) {
+                this.logger.warn(`Invalid signature for entry ${entryData.id}, skipping`);
+                continue;
+            }
+            // Decrypt payload
+            const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
+            // Initialize document if this is the first entry (doc_create)
+            const isFirstEntry = currentDoc === null;
+            if (isFirstEntry) {
+                if (entryMetadata.entryType !== "doc_create") {
+                    this.logger.warn(`First entry is not doc_create, skipping`);
+                    continue;
+                }
+                currentDoc = automerge_adapter_1.Automerge.init();
+                createdAt = entryMetadata.createdAt;
+                decryptionKeyId = entryMetadata.decryptionKeyId;
+            }
+            // Check document heads before applying change (for non-first entries)
+            const headsBefore = isFirstEntry ? null : (currentDoc ? automerge_adapter_1.Automerge.getHeads(currentDoc) : null);
+            // Apply change using loadIncremental
+            if (currentDoc === null) {
+                throw new Error("currentDoc should not be null at this point");
+            }
+            currentDoc = automerge_adapter_1.Automerge.loadIncremental(currentDoc, decryptedPayload);
+            // Check if document actually changed by comparing heads
+            // For first entry (doc_create), always yield since it's the initial creation
+            // For delete entries, always yield
+            // For other entries, only yield if heads changed
+            let shouldYield = false;
+            if (isFirstEntry || entryMetadata.entryType === "doc_delete") {
+                shouldYield = true;
+            }
+            else {
+                const headsAfter = automerge_adapter_1.Automerge.getHeads(currentDoc);
+                const headsChanged = headsBefore !== null && JSON.stringify(headsBefore) !== JSON.stringify(headsAfter);
+                shouldYield = headsChanged;
+            }
+            // Register automerge hash mapping
+            const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
+            if (parsed) {
+                this.registerAutomergeHashMapping(docId, parsed.automergeHash, entryData.id);
+            }
+            // Only yield if document actually changed (or if this is create/delete entry)
+            if (shouldYield) {
+                // Clone the document to ensure independence
+                const clonedDoc = automerge_adapter_1.Automerge.clone(currentDoc);
+                // Create internal doc representation
+                const internalDoc = {
+                    id: docId,
+                    doc: clonedDoc,
+                    createdAt: createdAt,
+                    lastModified: entryMetadata.createdAt,
+                    decryptionKeyId,
+                    isDeleted: entryMetadata.entryType === "doc_delete",
+                };
+                // Wrap and yield (including delete entries)
+                const wrappedDoc = this.wrapDocument(internalDoc);
+                yield {
+                    doc: wrappedDoc,
+                    changeCreatedAt: entryMetadata.createdAt,
+                    changeCreatedByPublicKey: entryMetadata.createdByPublicKey,
+                };
+            }
+            // Stop after delete entry (document is deleted, no more changes)
+            if (entryMetadata.entryType === "doc_delete") {
+                break;
+            }
+        }
+    }
     async getAllDocumentIds() {
         // Return all non-deleted document IDs from index
         const docIds = [];
@@ -489,6 +760,53 @@ class BaseMindooDB {
         }
         return docIds;
     }
+    async getAllDocumentIdsAtTimestamp(timestamp) {
+        // findEntries() uses an exclusive upper bound (createdAt < creationDateUntil).
+        // Query up to timestamp + 1 so entries created exactly at `timestamp` are included
+        // for the strict checks below (createTime < timestamp, deleteTime > timestamp).
+        const upperBoundExclusive = timestamp === Number.MAX_SAFE_INTEGER ? timestamp : timestamp + 1;
+        // Efficiently query for doc_create and doc_delete entries before the timestamp
+        const [creates, deletes] = await Promise.all([
+            this.store.findEntries("doc_create", null, upperBoundExclusive),
+            this.store.findEntries("doc_delete", null, upperBoundExclusive)
+        ]);
+        // Build sets of docIds for efficient lookup
+        const createdDocIds = new Set();
+        const deletedDocIds = new Set();
+        // Track earliest create time for each doc
+        const createTimes = new Map();
+        for (const entry of creates) {
+            const existingTime = createTimes.get(entry.docId);
+            if (!existingTime || entry.createdAt < existingTime) {
+                createTimes.set(entry.docId, entry.createdAt);
+                createdDocIds.add(entry.docId);
+            }
+        }
+        // Track earliest delete time for each doc
+        const deleteTimes = new Map();
+        for (const entry of deletes) {
+            const existingTime = deleteTimes.get(entry.docId);
+            if (!existingTime || entry.createdAt < existingTime) {
+                deleteTimes.set(entry.docId, entry.createdAt);
+                deletedDocIds.add(entry.docId);
+            }
+        }
+        // Find documents that existed at the timestamp
+        const docIds = [];
+        for (const docId of createdDocIds) {
+            const createTime = createTimes.get(docId);
+            const deleteTime = deleteTimes.get(docId);
+            // Document exists at timestamp if:
+            // 1. It was created at or before the timestamp
+            // 2. Either it was never deleted, or it was deleted after the timestamp
+            if (createTime <= timestamp) {
+                if (!deleteTime || deleteTime > timestamp) {
+                    docIds.push(docId);
+                }
+            }
+        }
+        return docIds;
+    }
     async deleteDocument(docId) {
         return this.deleteDocInternal(docId);
     }
@@ -502,7 +820,7 @@ class BaseMindooDB {
      */
     async deleteDocInternal(docId, signingKeyPair, signingKeyPassword) {
         const useCustomSigningKey = signingKeyPair !== undefined && signingKeyPassword !== undefined;
-        console.log(`[BaseMindooDB] Deleting document ${docId}${useCustomSigningKey ? ' using custom signing key' : ''}`);
+        this.logger.debug(`Deleting document ${docId}${useCustomSigningKey ? ' using custom signing key' : ''}`);
         // Admin-only validation: only admin key can modify data in admin-only databases
         if (this._isAdminOnlyDb) {
             const adminPublicKey = this.getAdminPublicKey();
@@ -521,7 +839,7 @@ class BaseMindooDB {
         // Create deletion change by clearing all fields from the document
         // This ensures Automerge produces actual change bytes
         // The deletion is also tracked via the "doc_delete" type entry in the append-only store
-        const newDoc = Automerge.change(internalDoc.doc, (doc) => {
+        const newDoc = automerge_adapter_1.Automerge.change(internalDoc.doc, (doc) => {
             // Remove all fields from the document
             for (const key of Object.keys(doc)) {
                 delete doc[key];
@@ -530,12 +848,12 @@ class BaseMindooDB {
             doc._deleted = true;
         });
         // Get the change bytes from the document
-        const changeBytes = Automerge.getLastLocalChange(newDoc);
+        const changeBytes = automerge_adapter_1.Automerge.getLastLocalChange(newDoc);
         if (!changeBytes) {
             throw new Error("Failed to get change bytes from Automerge document");
         }
         // Decode the change to get hash and dependencies
-        const decodedChange = Automerge.decodeChange(changeBytes);
+        const decodedChange = automerge_adapter_1.Automerge.decodeChange(changeBytes);
         const automergeHash = decodedChange.hash;
         const automergeDepHashes = decodedChange.deps || []; // Dependencies from the decoded change
         // Encrypt the change payload first
@@ -550,12 +868,12 @@ class BaseMindooDB {
         let signature;
         let createdByPublicKey;
         if (useCustomSigningKey) {
-            console.log(`[BaseMindooDB] Signing deletion for document ${docId} with provided key`);
+            this.logger.debug(`Signing deletion for document ${docId} with provided key`);
             signature = await this.tenant.signPayloadWithKey(encryptedPayload, signingKeyPair, signingKeyPassword);
             createdByPublicKey = signingKeyPair.publicKey;
         }
         else {
-            console.log(`[BaseMindooDB] Signing deletion for document ${docId} with current user's key`);
+            this.logger.debug(`Signing deletion for document ${docId} with current user's key`);
             const currentUser = await this.tenant.getCurrentUserId();
             signature = await this.tenant.signPayload(encryptedPayload);
             createdByPublicKey = currentUser.userSigningPublicKey;
@@ -588,7 +906,7 @@ class BaseMindooDB {
         internalDoc.lastModified = Date.now();
         this.docCache.set(docId, internalDoc);
         this.updateIndex(docId, internalDoc.lastModified, true);
-        console.log(`[BaseMindooDB] Document ${docId} deleted successfully`);
+        this.logger.info(`Document ${docId} deleted successfully`);
     }
     async changeDoc(doc, changeFunc) {
         return this.changeDocInternal(doc, changeFunc);
@@ -599,7 +917,7 @@ class BaseMindooDB {
     async changeDocInternal(doc, changeFunc, signingKeyPair, signingKeyPassword) {
         const docId = doc.getId();
         const useCustomKey = signingKeyPair && signingKeyPassword;
-        console.log(`[BaseMindooDB] ===== ${useCustomKey ? 'changeDocWithSigningKey' : 'changeDoc'} called for document ${docId} =====`);
+        this.logger.debug(`===== ${useCustomKey ? 'changeDocWithSigningKey' : 'changeDoc'} called for document ${docId} =====`);
         // Admin-only validation: only admin key can modify data in admin-only databases
         if (this._isAdminOnlyDb) {
             const adminPublicKey = this.getAdminPublicKey();
@@ -613,24 +931,24 @@ class BaseMindooDB {
         // Get internal document from cache or load it
         let internalDoc = this.docCache.get(docId);
         if (!internalDoc) {
-            console.log(`[BaseMindooDB] Document ${docId} not in cache, loading from store`);
+            this.logger.debug(`Document ${docId} not in cache, loading from store`);
             const loadedDoc = await this.loadDocumentInternal(docId);
             if (!loadedDoc) {
                 throw new Error(`Document ${docId} not found`);
             }
             internalDoc = loadedDoc;
-            console.log(`[BaseMindooDB] Successfully loaded document ${docId} from store for ${useCustomKey ? 'changeDocWithSigningKey' : 'changeDoc'}`);
+            this.logger.debug(`Successfully loaded document ${docId} from store for ${useCustomKey ? 'changeDocWithSigningKey' : 'changeDoc'}`);
         }
         else {
-            console.log(`[BaseMindooDB] Document ${docId} found in cache`);
+            this.logger.debug(`Document ${docId} found in cache`);
         }
         if (internalDoc.isDeleted) {
             throw new Error(`Document ${docId} has been deleted`);
         }
         // Apply the change function
         const now = Date.now();
-        console.log(`[BaseMindooDB] Applying change function to document ${docId}`);
-        console.log(`[BaseMindooDB] Document state before change: heads=${JSON.stringify(Automerge.getHeads(internalDoc.doc))}`);
+        this.logger.debug(`Applying change function to document ${docId}`);
+        this.logger.debug(`Document state before change: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(internalDoc.doc))}`);
         // For async callbacks, we need to handle document modifications carefully.
         // Automerge.change() requires synchronous modifications, so we'll:
         // 1. Execute the async callback to do any async work (like signing)
@@ -673,8 +991,8 @@ class BaseMindooDB {
                             pendingDeletions.delete(prop);
                             // Track the change
                             pendingChanges.set(prop, value);
-                            // Also set on the target for immediate access
-                            target[prop] = value;
+                            // NOTE: Don't set on target immediately - Automerge requires changes
+                            // to be made inside Automerge.change(). Pending changes are applied later.
                         }
                         return true;
                     },
@@ -685,8 +1003,8 @@ class BaseMindooDB {
                             pendingDeletions.add(prop);
                             // Remove from pending changes if it was there
                             pendingChanges.delete(prop);
-                            // Also delete from the target for immediate access
-                            delete target[prop];
+                            // NOTE: Don't delete from target immediately - Automerge requires changes
+                            // to be made inside Automerge.change(). Pending deletions are applied later.
                         }
                         return true;
                     },
@@ -805,7 +1123,7 @@ class BaseMindooDB {
         // Now apply the collected changes synchronously in Automerge.change()
         let newDoc;
         try {
-            newDoc = Automerge.change(internalDoc.doc, (automergeDoc) => {
+            newDoc = automerge_adapter_1.Automerge.change(internalDoc.doc, (automergeDoc) => {
                 // Apply all pending changes (sets/updates)
                 for (const [key, value] of pendingChanges) {
                     automergeDoc[key] = value;
@@ -844,22 +1162,22 @@ class BaseMindooDB {
                 // Update lastModified timestamp
                 automergeDoc._lastModified = now;
             });
-            console.log(`[BaseMindooDB] Successfully applied change function, new document heads: ${JSON.stringify(Automerge.getHeads(newDoc))}`);
+            this.logger.debug(`Successfully applied change function, new document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(newDoc))}`);
         }
         catch (error) {
-            console.error(`[BaseMindooDB] Error in Automerge.change for document ${docId}:`, error);
+            this.logger.error(`Error in Automerge.change for document ${docId}:`, error);
             throw error;
         }
         // Get the change bytes from the document
-        console.log(`[BaseMindooDB] Getting change bytes from document ${docId}`);
-        const changeBytes = Automerge.getLastLocalChange(newDoc);
+        this.logger.debug(`Getting change bytes from document ${docId}`);
+        const changeBytes = automerge_adapter_1.Automerge.getLastLocalChange(newDoc);
         if (!changeBytes) {
             //TODO decide if we just exit here or throw an error
             throw new Error("Failed to get change bytes from Automerge document");
         }
-        console.log(`[BaseMindooDB] Got change bytes: ${changeBytes.length} bytes`);
+        this.logger.debug(`Got change bytes: ${changeBytes.length} bytes`);
         // Decode the change to get hash and dependencies
-        const decodedChange = Automerge.decodeChange(changeBytes);
+        const decodedChange = automerge_adapter_1.Automerge.decodeChange(changeBytes);
         const automergeHash = decodedChange.hash;
         const automergeDepHashes = decodedChange.deps || []; // Dependencies from the decoded change
         // Encrypt the change payload first
@@ -868,7 +1186,7 @@ class BaseMindooDB {
         const contentHash = await (0, idGeneration_1.computeContentHash)(encryptedPayload, this.getSubtle());
         // Generate entry ID with blockchain-like chaining
         const entryId = await (0, idGeneration_1.generateDocEntryId)(docId, automergeHash, automergeDepHashes, this.getSubtle());
-        console.log(`[BaseMindooDB] Generated entry ID for change: ${entryId}`);
+        this.logger.debug(`Generated entry ID for change: ${entryId}`);
         // Resolve Automerge dependency hashes to entry IDs
         const dependencyIds = this.resolveAutomergeDepsToEntryIds(docId, automergeDepHashes);
         // Sign the encrypted payload - use custom key if provided, otherwise current user's key
@@ -911,12 +1229,12 @@ class BaseMindooDB {
         internalDoc.lastModified = Date.now();
         this.docCache.set(docId, internalDoc);
         this.updateIndex(docId, internalDoc.lastModified, internalDoc.isDeleted);
-        console.log(`[BaseMindooDB] Document ${docId} ${useCustomKey ? 'changed with custom signing key' : 'changed'} successfully`);
+        this.logger.info(`Document ${docId} ${useCustomKey ? 'changed with custom signing key' : 'changed'} successfully`);
     }
-    async processChangesSince(cursor, limit, callback) {
+    async *iterateChangesSince(cursor) {
         // Default to initial cursor if null is provided
         const actualCursor = cursor ?? { lastModified: 0, docId: "" };
-        console.log(`[BaseMindooDB] Processing changes since cursor ${JSON.stringify(actualCursor)} (limit: ${limit})`);
+        this.logger.debug(`Starting iteration from cursor ${JSON.stringify(actualCursor)}`);
         // Find starting position using binary search
         // We want to find the first entry that is greater than the cursor
         let startIndex = 0;
@@ -944,77 +1262,190 @@ class BaseMindooDB {
                 }
             }
         }
-        // Process documents in order from the starting position
-        let processedCount = 0;
-        let lastCursor = actualCursor;
-        for (let i = startIndex; i < this.index.length && processedCount < limit; i++) {
+        // Pre-check which documents are in cache to optimize iteration
+        const uncachedDocIds = [];
+        for (let i = startIndex; i < this.index.length; i++) {
             const entry = this.index[i];
-            // Skip deleted documents (they're still in index for tracking, but shouldn't be processed)
-            if (entry.isDeleted) {
-                continue;
+            if (!this.docCache.has(entry.docId)) {
+                uncachedDocIds.push(entry.docId);
             }
+        }
+        // Prefetch uncached documents in batches (if any)
+        if (uncachedDocIds.length > 0) {
+            this.logger.debug(`Prefetching ${uncachedDocIds.length} uncached documents for iteration`);
+            const prefetchBatchSize = 10;
+            for (let i = 0; i < uncachedDocIds.length; i += prefetchBatchSize) {
+                const batch = uncachedDocIds.slice(i, i + prefetchBatchSize);
+                await Promise.all(batch.map(docId => this.loadDocumentInternal(docId).catch(err => {
+                    this.logger.warn(`Failed to prefetch document ${docId}:`, err);
+                    return null;
+                })));
+            }
+        }
+        // Iterate through the index and yield documents one at a time
+        // Documents should now be in cache from prefetching
+        for (let i = startIndex; i < this.index.length; i++) {
+            const entry = this.index[i];
             try {
-                console.log(`[BaseMindooDB] Processing document ${entry.docId} from index (lastModified: ${entry.lastModified})`);
-                const doc = await this.getDocument(entry.docId);
-                console.log(`[BaseMindooDB] Successfully loaded document ${entry.docId}`);
+                this.logger.debug(`Yielding document ${entry.docId} from index (lastModified: ${entry.lastModified}, isDeleted: ${entry.isDeleted})`);
+                // Check cache first (should be there after prefetching)
+                let internalDoc = this.docCache.get(entry.docId) || null;
+                if (!internalDoc) {
+                    // Fallback to loading if not in cache (shouldn't happen after prefetching)
+                    this.logger.debug(`Document ${entry.docId} not in cache, loading from store`);
+                    internalDoc = await this.loadDocumentInternal(entry.docId);
+                }
+                if (!internalDoc) {
+                    this.logger.warn(`Document ${entry.docId} not found, skipping`);
+                    continue;
+                }
+                // Wrap the document (works for both deleted and non-deleted documents)
+                const doc = this.wrapDocument(internalDoc);
+                this.logger.debug(`Successfully loaded document ${entry.docId} (isDeleted: ${doc.isDeleted()})`);
                 // Create cursor for current document
                 const currentCursor = {
                     lastModified: entry.lastModified,
                     docId: entry.docId,
                 };
-                const shouldContinue = callback(doc, currentCursor);
-                // Update cursor to the last successfully processed document
-                lastCursor = currentCursor;
-                processedCount++;
-                // Stop if callback returns false
-                if (shouldContinue === false) {
-                    console.log(`[BaseMindooDB] Callback requested to stop processing`);
-                    break;
-                }
+                // Yield immediately - this allows the caller to break early after each document
+                // Deleted documents are included so external indexes can handle deletions
+                yield { doc, cursor: currentCursor };
             }
             catch (error) {
-                console.error(`[BaseMindooDB] Error processing document ${entry.docId}:`, error);
+                this.logger.error(`Error processing document ${entry.docId}:`, error);
                 // Stop processing on error
                 throw error;
             }
         }
-        console.log(`[BaseMindooDB] Processed ${processedCount} changes, last cursor: ${JSON.stringify(lastCursor)}`);
-        return lastCursor;
+        this.logger.debug(`Iteration completed`);
     }
-    async *iterateChangesSince(cursor, pageSize = 100) {
-        console.log(`[BaseMindooDB] Starting iteration from cursor ${JSON.stringify(cursor)} (pageSize: ${pageSize})`);
-        let currentCursor = cursor;
-        while (true) {
-            // Collect documents from this page
-            const pageResults = [];
-            let documentsProcessed = 0;
-            // Process one page of changes
-            const returnedCursor = await this.processChangesSince(currentCursor, pageSize, (doc, docCursor) => {
-                documentsProcessed++;
-                pageResults.push({ doc, cursor: docCursor });
-                // Always continue processing the page
-                return true;
-            });
-            // Yield all documents from this page
-            for (const result of pageResults) {
-                yield result;
+    /**
+     * Incrementally update a cached document with new entries.
+     * Returns the updated document, or null if document wasn't actually changed.
+     */
+    async applyNewEntriesToCachedDocument(cachedDoc, newEntryMetadata) {
+        const docId = cachedDoc.id;
+        if (newEntryMetadata.length === 0) {
+            this.logger.debug(`No new entries for cached document ${docId}`);
+            return null; // No changes
+        }
+        this.logger.debug(`Applying ${newEntryMetadata.length} new entries to cached document ${docId}`);
+        // Get current document heads to check if it changes
+        const headsBefore = automerge_adapter_1.Automerge.getHeads(cachedDoc.doc);
+        // Filter entries to only include change entries (exclude snapshots)
+        const entriesToApply = newEntryMetadata.filter(em => em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete");
+        if (entriesToApply.length === 0) {
+            this.logger.debug(`No change entries to apply for document ${docId}`);
+            return null; // No changes
+        }
+        // Sort entries by timestamp
+        entriesToApply.sort((a, b) => a.createdAt - b.createdAt);
+        // Load entries from store
+        const entries = await this.store.getEntries(entriesToApply.map(em => em.id));
+        // Filter entries for admin-only mode first
+        const validEntries = entries.filter(entryData => {
+            if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
+                this.logger.warn(`Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
+                return false;
             }
-            // If we processed fewer documents than the page size, we've reached the end
-            if (documentsProcessed < pageSize) {
-                console.log(`[BaseMindooDB] Reached end of documents, processed ${documentsProcessed} in last page`);
-                break;
+            return true;
+        });
+        if (validEntries.length === 0) {
+            this.logger.debug(`No valid entries to process for cached document ${docId}`);
+            return null;
+        }
+        // Batch signature verification with key caching
+        // Group entries by public key to import each key only once
+        const entriesByPublicKey = new Map();
+        for (const entryData of validEntries) {
+            if (!entriesByPublicKey.has(entryData.createdByPublicKey)) {
+                entriesByPublicKey.set(entryData.createdByPublicKey, []);
             }
-            // If the cursor didn't advance, we've reached the end
-            if (currentCursor !== null &&
-                returnedCursor.lastModified === currentCursor.lastModified &&
-                returnedCursor.docId === currentCursor.docId) {
-                console.log(`[BaseMindooDB] Cursor did not advance, reached end of documents`);
-                break;
+            entriesByPublicKey.get(entryData.createdByPublicKey).push(entryData);
+        }
+        // Import all unique public keys in parallel (with caching)
+        const keyImportResults = await Promise.all(Array.from(entriesByPublicKey.keys()).map(async (publicKey) => {
+            const cryptoKey = await this.getOrImportPublicKey(publicKey);
+            return { publicKey, cryptoKey };
+        }));
+        // Create a map of public key -> CryptoKey for quick lookup
+        const keyMap = new Map();
+        for (const { publicKey, cryptoKey } of keyImportResults) {
+            if (cryptoKey) {
+                keyMap.set(publicKey, cryptoKey);
+            }
+        }
+        // Verify all signatures in parallel using cached keys
+        const signatureVerificationResults = await Promise.all(validEntries.map(async (entryData) => {
+            const cryptoKey = keyMap.get(entryData.createdByPublicKey);
+            if (!cryptoKey) {
+                // Key was not trusted or failed to import
+                return { entryData, isValid: false };
+            }
+            const isValid = await this.verifySignatureWithKey(cryptoKey, entryData.encryptedData, entryData.signature);
+            return { entryData, isValid };
+        }));
+        // Filter out entries with invalid signatures
+        const verifiedEntries = signatureVerificationResults
+            .filter(({ isValid }) => isValid)
+            .map(({ entryData }) => entryData);
+        if (verifiedEntries.length === 0) {
+            this.logger.debug(`No entries with valid signatures for cached document ${docId}`);
+            return null;
+        }
+        // Parallel decryption - decrypt all entries concurrently
+        const decryptionResults = await Promise.all(verifiedEntries.map(async (entryData) => {
+            const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
+            return { entryData, decryptedPayload };
+        }));
+        // Collect change bytes and register automerge hash mappings
+        const changeBytes = [];
+        for (const { entryData, decryptedPayload } of decryptionResults) {
+            changeBytes.push(decryptedPayload);
+            // Register automerge hash -> entry ID mapping
+            const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
+            if (parsed) {
+                this.registerAutomergeHashMapping(docId, parsed.automergeHash, entryData.id);
             }
-            // Continue with the next page using the returned cursor
-            currentCursor = returnedCursor;
         }
-        console.log(`[BaseMindooDB] Iteration completed`);
+        if (changeBytes.length === 0) {
+            this.logger.debug(`No valid change bytes to apply for document ${docId}`);
+            return null; // No changes
+        }
+        // Clone the cached document to avoid "outdated document" error
+        // This happens when the document has been wrapped and returned to the user
+        // Automerge marks documents as outdated when they're accessed, preventing direct mutation
+        const clonedDoc = automerge_adapter_1.Automerge.clone(cachedDoc.doc);
+        // Apply all changes at once (Automerge handles dependency ordering)
+        const result = automerge_adapter_1.Automerge.applyChanges(clonedDoc, changeBytes);
+        const updatedDoc = result[0];
+        // Check if document actually changed
+        const headsAfter = automerge_adapter_1.Automerge.getHeads(updatedDoc);
+        const headsChanged = JSON.stringify(headsBefore) !== JSON.stringify(headsAfter);
+        if (!headsChanged) {
+            this.logger.debug(`Document ${docId} heads unchanged after applying new entries`);
+            return null; // Document didn't actually change
+        }
+        this.logger.debug(`Document ${docId} changed: heads before=${JSON.stringify(headsBefore)}, after=${JSON.stringify(headsAfter)}`);
+        // Update metadata
+        const payload = updatedDoc;
+        const lastEntry = entries[entries.length - 1];
+        const lastModified = payload._lastModified ||
+            (lastEntry ? lastEntry.createdAt : cachedDoc.lastModified);
+        // Check if document was deleted
+        const hasDeleteEntry = newEntryMetadata.some(em => em.entryType === "doc_delete");
+        const isDeleted = hasDeleteEntry || cachedDoc.isDeleted;
+        const updatedInternalDoc = {
+            id: docId,
+            doc: updatedDoc,
+            createdAt: cachedDoc.createdAt,
+            lastModified,
+            decryptionKeyId: cachedDoc.decryptionKeyId,
+            isDeleted,
+        };
+        // Update cache
+        this.docCache.set(docId, updatedInternalDoc);
+        return updatedInternalDoc;
     }
     /**
      * Internal method to load a document from the content-addressed store
@@ -1022,25 +1453,25 @@ class BaseMindooDB {
     async loadDocumentInternal(docId) {
         // Check cache first
         if (this.docCache.has(docId)) {
-            console.log(`[BaseMindooDB] Document ${docId} found in cache, returning cached version`);
+            this.logger.debug(`Document ${docId} found in cache, returning cached version`);
             return this.docCache.get(docId);
         }
-        console.log(`[BaseMindooDB] ===== Starting to load document ${docId} from store =====`);
+        this.logger.debug(`===== Starting to load document ${docId} from store =====`);
         // Get all entry metadata for this document
         // TODO: Implement loading from last snapshot if available
-        console.log(`[BaseMindooDB] Getting all entry hashes for document ${docId}`);
-        const allEntryMetadata = await this.store.findNewEntriesForDoc([], docId);
-        console.log(`[BaseMindooDB] Found ${allEntryMetadata.length} total entry hashes for document ${docId}`);
+        this.logger.debug(`Getting all entry hashes for document ${docId}`);
+        const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
+        this.logger.debug(`Found ${allEntryMetadata.length} total entry hashes for document ${docId}`);
         if (allEntryMetadata.length === 0) {
-            console.log(`[BaseMindooDB] No entry hashes found for document ${docId}, returning null`);
+            this.logger.debug(`No entry hashes found for document ${docId}, returning null`);
             return null;
         }
         // Log all entry types
         const entryTypes = allEntryMetadata.map(em => `${em.entryType}@${em.createdAt}`).join(', ');
-        console.log(`[BaseMindooDB] Entry types for ${docId}: ${entryTypes}`);
+        this.logger.debug(`Entry types for ${docId}: ${entryTypes}`);
         // Find the most recent snapshot (if any)
         const snapshots = allEntryMetadata.filter(em => em.entryType === "doc_snapshot");
-        console.log(`[BaseMindooDB] Found ${snapshots.length} snapshot(s) for document ${docId}`);
+        this.logger.debug(`Found ${snapshots.length} snapshot(s) for document ${docId}`);
         let startFromSnapshot = false;
         let snapshotMeta = null;
         if (snapshots.length > 0) {
@@ -1048,29 +1479,29 @@ class BaseMindooDB {
             snapshots.sort((a, b) => b.createdAt - a.createdAt);
             snapshotMeta = snapshots[0];
             startFromSnapshot = true;
-            console.log(`[BaseMindooDB] Will start from snapshot ${snapshotMeta.id} created at ${snapshotMeta.createdAt}`);
+            this.logger.debug(`Will start from snapshot ${snapshotMeta.id} created at ${snapshotMeta.createdAt}`);
         }
         else {
-            console.log(`[BaseMindooDB] No snapshot found, will start from scratch`);
+            this.logger.debug(`No snapshot found, will start from scratch`);
         }
         // Get all entries (excluding snapshot entries - we'll handle delete separately)
         // Include "doc_create", "doc_change", and "doc_delete" types as they all contain Automerge changes to apply
         const entriesToLoad = startFromSnapshot
             ? allEntryMetadata.filter(em => (em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete") && em.createdAt > snapshotMeta.createdAt)
             : allEntryMetadata.filter(em => em.entryType === "doc_create" || em.entryType === "doc_change" || em.entryType === "doc_delete");
-        console.log(`[BaseMindooDB] Will load ${entriesToLoad.length} entries for document ${docId} (after snapshot filter)`);
+        this.logger.debug(`Will load ${entriesToLoad.length} entries for document ${docId} (after snapshot filter)`);
         // Load the snapshot first if we have one
         let doc = undefined;
         if (startFromSnapshot && snapshotMeta) {
-            console.log(`[BaseMindooDB] Loading snapshot for document ${docId}`);
+            this.logger.debug(`Loading snapshot for document ${docId}`);
             const snapshotEntries = await this.store.getEntries([snapshotMeta.id]);
-            console.log(`[BaseMindooDB] Retrieved ${snapshotEntries.length} snapshot entry(s) from store`);
+            this.logger.debug(`Retrieved ${snapshotEntries.length} snapshot entry(s) from store`);
             if (snapshotEntries.length > 0) {
                 const snapshotData = snapshotEntries[0];
                 // Admin-only mode: only accept snapshots signed by admin
                 let isValid = false;
                 if (this._isAdminOnlyDb && snapshotData.createdByPublicKey !== this.getAdminPublicKey()) {
-                    console.warn(`[BaseMindooDB] Admin-only DB: skipping snapshot ${snapshotData.id} not signed by admin key`);
+                    this.logger.warn(`Admin-only DB: skipping snapshot ${snapshotData.id} not signed by admin key`);
                 }
                 else {
                     // Verify signature against the encrypted snapshot (no decryption needed)
@@ -1078,21 +1509,21 @@ class BaseMindooDB {
                     isValid = await this.tenant.verifySignature(snapshotData.encryptedData, snapshotData.signature, snapshotData.createdByPublicKey);
                 }
                 if (!isValid) {
-                    console.warn(`[BaseMindooDB] Invalid signature for snapshot ${snapshotData.id}, falling back to loading from scratch`);
+                    this.logger.warn(`Invalid signature for snapshot ${snapshotData.id}, falling back to loading from scratch`);
                     // Fall back to loading from scratch
                     startFromSnapshot = false;
                 }
                 else {
-                    console.log(`[BaseMindooDB] Snapshot signature valid, decrypting snapshot`);
+                    this.logger.debug(`Snapshot signature valid, decrypting snapshot`);
                     // Decrypt snapshot (only after signature verification passes)
                     const decryptedSnapshot = await this.tenant.decryptPayload(snapshotData.encryptedData, snapshotData.decryptionKeyId);
-                    console.log(`[BaseMindooDB] Decrypted snapshot (${snapshotData.encryptedData.length} -> ${decryptedSnapshot.length} bytes)`);
+                    this.logger.debug(`Decrypted snapshot (${snapshotData.encryptedData.length} -> ${decryptedSnapshot.length} bytes)`);
                     // Load snapshot using Automerge.load()
                     // This deserializes a full document snapshot from binary data
                     // According to Automerge docs: load() is equivalent to init() followed by loadIncremental()
-                    console.log(`[BaseMindooDB] Loading snapshot into Automerge document`);
-                    doc = Automerge.load(decryptedSnapshot);
-                    console.log(`[BaseMindooDB] Successfully loaded snapshot, document heads: ${JSON.stringify(Automerge.getHeads(doc))}`);
+                    this.logger.debug(`Loading snapshot into Automerge document`);
+                    doc = automerge_adapter_1.Automerge.load(decryptedSnapshot);
+                    this.logger.debug(`Successfully loaded snapshot, document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
                     // Register the snapshot's automerge hash -> entry ID mapping
                     const parsed = (0, idGeneration_1.parseDocEntryId)(snapshotData.id);
                     if (parsed) {
@@ -1103,96 +1534,119 @@ class BaseMindooDB {
         }
         // If we don't have a snapshot, start from scratch
         if (!doc) {
-            console.log(`[BaseMindooDB] Initializing new Automerge document for ${docId}`);
-            doc = Automerge.init();
-            console.log(`[BaseMindooDB] Initialized empty document, heads: ${JSON.stringify(Automerge.getHeads(doc))}`);
+            this.logger.debug(`Initializing new Automerge document for ${docId}`);
+            doc = automerge_adapter_1.Automerge.init();
+            this.logger.debug(`Initialized empty document, heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
         }
         // Sort entries by timestamp
         entriesToLoad.sort((a, b) => a.createdAt - b.createdAt);
-        console.log(`[BaseMindooDB] Sorted ${entriesToLoad.length} entries by timestamp for document ${docId}`);
+        this.logger.debug(`Sorted ${entriesToLoad.length} entries by timestamp for document ${docId}`);
         // Load and apply all entries
-        console.log(`[BaseMindooDB] Fetching ${entriesToLoad.length} entries from store for document ${docId}`);
+        this.logger.debug(`Fetching ${entriesToLoad.length} entries from store for document ${docId}`);
         const entries = await this.store.getEntries(entriesToLoad.map(em => em.id));
-        console.log(`[BaseMindooDB] Retrieved ${entries.length} entries from store for document ${docId}`);
-        console.log(`[BaseMindooDB] Loading document ${docId}: found ${entries.length} entries to apply (${startFromSnapshot ? 'starting from snapshot' : 'starting from scratch'})`);
+        this.logger.debug(`Retrieved ${entries.length} entries from store for document ${docId}`);
+        this.logger.debug(`Loading document ${docId}: found ${entries.length} entries to apply (${startFromSnapshot ? 'starting from snapshot' : 'starting from scratch'})`);
         // Log current document state before applying entries
-        console.log(`[BaseMindooDB] Document state before applying entries: heads=${JSON.stringify(Automerge.getHeads(doc))}`);
-        // Verify signatures, decrypt, and apply entries one at a time
-        // This ensures dependencies are handled correctly by Automerge
-        for (let i = 0; i < entries.length; i++) {
-            const entryData = entries[i];
-            console.log(`[BaseMindooDB] ===== Processing entry ${i + 1}/${entries.length} for document ${docId} =====`);
-            console.log(`[BaseMindooDB] Entry id: ${entryData.id}`);
-            console.log(`[BaseMindooDB] Entry type: ${entryData.entryType}`);
-            console.log(`[BaseMindooDB] Entry createdAt: ${entryData.createdAt}`);
-            console.log(`[BaseMindooDB] Entry dependencies: ${JSON.stringify(entryData.dependencyIds || [])}`);
-            console.log(`[BaseMindooDB] Entry payload size: ${entryData.encryptedData.length} bytes`);
-            // Admin-only mode: only accept entries signed by the admin key
-            // This prevents recursion and ensures security for the directory database
+        this.logger.debug(`Document state before applying entries: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
+        // Filter entries for admin-only mode first
+        const validEntries = entries.filter(entryData => {
             if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
-                console.warn(`[BaseMindooDB] Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
-                continue;
+                this.logger.warn(`Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
+                return false;
             }
-            // Verify signature against the encrypted payload (no decryption needed)
-            // We sign the encrypted payload, so anyone can verify signatures without decryption keys
-            console.log(`[BaseMindooDB] Verifying signature for entry ${entryData.id}`);
-            const isValid = await this.tenant.verifySignature(entryData.encryptedData, entryData.signature, entryData.createdByPublicKey);
-            if (!isValid) {
-                console.warn(`[BaseMindooDB] Invalid signature for entry ${entryData.id}, skipping`);
-                continue;
+            return true;
+        });
+        if (validEntries.length === 0) {
+            this.logger.debug(`No valid entries to process for document ${docId}`);
+        }
+        else {
+            // Batch signature verification with key caching
+            // Group entries by public key to import each key only once
+            this.logger.debug(`Verifying ${validEntries.length} signatures in parallel with key caching`);
+            const entriesByPublicKey = new Map();
+            for (const entryData of validEntries) {
+                if (!entriesByPublicKey.has(entryData.createdByPublicKey)) {
+                    entriesByPublicKey.set(entryData.createdByPublicKey, []);
+                }
+                entriesByPublicKey.get(entryData.createdByPublicKey).push(entryData);
             }
-            console.log(`[BaseMindooDB] Signature valid for entry ${entryData.id}`);
-            // Decrypt payload (only after signature verification passes)
-            console.log(`[BaseMindooDB] Decrypting payload for entry ${entryData.id} with key ${entryData.decryptionKeyId}`);
-            const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
-            console.log(`[BaseMindooDB] Decrypted payload: ${entryData.encryptedData.length} -> ${decryptedPayload.length} bytes`);
-            // Apply change using applyChanges with raw change bytes
-            // applyChanges expects an array of Uint8Array (raw change bytes)
-            try {
-                console.log(`[BaseMindooDB] Document state before applying entry ${i + 1}: heads=${JSON.stringify(Automerge.getHeads(doc))}`);
-                console.log(`[BaseMindooDB] Calling Automerge.applyChanges for entry ${i + 1}/${entries.length} on document ${docId}`);
-                console.log(`[BaseMindooDB] Decrypted payload length: ${decryptedPayload.length} bytes`);
-                const currentDoc = doc;
-                const result = Automerge.applyChanges(currentDoc, [decryptedPayload]);
-                doc = result[0];
-                console.log(`[BaseMindooDB] Successfully applied entry ${i + 1}/${entries.length}`);
-                console.log(`[BaseMindooDB] Document state after applying entry ${i + 1}: heads=${JSON.stringify(Automerge.getHeads(doc))}`);
-                // Register the automerge hash -> entry ID mapping for future dependency resolution
-                const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
-                if (parsed) {
-                    this.registerAutomergeHashMapping(docId, parsed.automergeHash, entryData.id);
+            // Import all unique public keys in parallel (with caching)
+            const keyImportResults = await Promise.all(Array.from(entriesByPublicKey.keys()).map(async (publicKey) => {
+                const cryptoKey = await this.getOrImportPublicKey(publicKey);
+                return { publicKey, cryptoKey };
+            }));
+            // Create a map of public key -> CryptoKey for quick lookup
+            const keyMap = new Map();
+            for (const { publicKey, cryptoKey } of keyImportResults) {
+                if (cryptoKey) {
+                    keyMap.set(publicKey, cryptoKey);
                 }
             }
-            catch (error) {
-                console.error(`[BaseMindooDB] Error applying entry ${entryData.id} to document ${docId}:`, error);
-                console.error(`[BaseMindooDB] Entry index: ${i + 1}/${entries.length}`);
-                console.error(`[BaseMindooDB] Entry dependencies:`, entryData.dependencyIds);
-                console.error(`[BaseMindooDB] Entry type:`, entryData.entryType);
-                console.error(`[BaseMindooDB] Entry createdAt:`, entryData.createdAt);
-                console.error(`[BaseMindooDB] Document state before entry:`, {
-                    hasDoc: !!doc,
-                    docHeads: doc ? Automerge.getHeads(doc) : null,
-                });
-                // Log previous entry for debugging dependency issues
-                if (i > 0) {
-                    const prevEntry = entries[i - 1];
-                    console.error(`[BaseMindooDB] Previous entry:`, {
-                        id: prevEntry.id,
-                        deps: prevEntry.dependencyIds,
-                        type: prevEntry.entryType,
-                    });
+            // Verify all signatures in parallel using cached keys
+            const signatureVerificationResults = await Promise.all(validEntries.map(async (entryData) => {
+                const cryptoKey = keyMap.get(entryData.createdByPublicKey);
+                if (!cryptoKey) {
+                    // Key was not trusted or failed to import
+                    return { entryData, isValid: false };
+                }
+                const isValid = await this.verifySignatureWithKey(cryptoKey, entryData.encryptedData, entryData.signature);
+                return { entryData, isValid };
+            }));
+            // Filter out entries with invalid signatures
+            const verifiedEntries = signatureVerificationResults
+                .filter(({ isValid }) => {
+                if (!isValid) {
+                    this.logger.warn(`Invalid signature for entry, skipping`);
+                }
+                return isValid;
+            })
+                .map(({ entryData }) => entryData);
+            if (verifiedEntries.length === 0) {
+                this.logger.debug(`No entries with valid signatures for document ${docId}`);
+            }
+            else {
+                // Parallel decryption - decrypt all entries concurrently
+                // Automerge handles dependency buffering internally, so we can decrypt all in parallel
+                this.logger.debug(`Decrypting ${verifiedEntries.length} entries in parallel`);
+                const decryptionResults = await Promise.all(verifiedEntries.map(async (entryData) => {
+                    const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
+                    return { entryData, decryptedPayload };
+                }));
+                // Collect change bytes and register automerge hash mappings
+                const changeBytes = [];
+                for (const { entryData, decryptedPayload } of decryptionResults) {
+                    changeBytes.push(decryptedPayload);
+                    // Register the automerge hash -> entry ID mapping for future dependency resolution
+                    const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
+                    if (parsed) {
+                        this.registerAutomergeHashMapping(docId, parsed.automergeHash, entryData.id);
+                    }
+                }
+                // Batch apply all changes at once - Automerge handles dependency ordering
+                if (changeBytes.length > 0) {
+                    this.logger.debug(`Applying ${changeBytes.length} changes to document ${docId} using batch applyChanges`);
+                    try {
+                        const result = automerge_adapter_1.Automerge.applyChanges(doc, changeBytes);
+                        doc = result[0];
+                        this.logger.debug(`Successfully applied ${changeBytes.length} changes to document ${docId}`);
+                        this.logger.debug(`Document state after applying changes: heads=${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
+                    }
+                    catch (error) {
+                        this.logger.error(`Error applying changes to document ${docId}:`, error);
+                        this.logger.error(`Number of changes: ${changeBytes.length}`);
+                        throw error;
+                    }
                 }
-                throw error;
             }
         }
         // Extract metadata from document (doc is guaranteed to be defined at this point)
-        console.log(`[BaseMindooDB] All entries applied successfully for document ${docId}`);
-        console.log(`[BaseMindooDB] Final document heads: ${JSON.stringify(Automerge.getHeads(doc))}`);
+        this.logger.debug(`All entries applied successfully for document ${docId}`);
+        this.logger.debug(`Final document heads: ${JSON.stringify(automerge_adapter_1.Automerge.getHeads(doc))}`);
         const payload = doc;
         // Check if document was deleted by looking for a "doc_delete" type entry
         const hasDeleteEntry = allEntryMetadata.some(em => em.entryType === "doc_delete");
         const isDeleted = hasDeleteEntry;
-        console.log(`[BaseMindooDB] Document ${docId} isDeleted: ${isDeleted}`);
+        this.logger.debug(`Document ${docId} isDeleted: ${isDeleted}`);
         const decryptionKeyId = payload._decryptionKeyId || "default";
         // Get lastModified from payload, or use the timestamp of the last entry
         const lastEntry = entries.length > 0 ? entries[entries.length - 1] : null;
@@ -1201,7 +1655,7 @@ class BaseMindooDB {
         // Get createdAt from the first entry
         const firstEntry = allEntryMetadata.length > 0 ? allEntryMetadata[0] : null;
         const createdAt = firstEntry ? firstEntry.createdAt : lastModified;
-        console.log(`[BaseMindooDB] Document ${docId} metadata: createdAt=${createdAt}, lastModified=${lastModified}, decryptionKeyId=${decryptionKeyId}`);
+        this.logger.debug(`Document ${docId} metadata: createdAt=${createdAt}, lastModified=${lastModified}, decryptionKeyId=${decryptionKeyId}`);
         const internalDoc = {
             id: docId,
             doc: doc, // doc is guaranteed to be defined at this point
@@ -1212,9 +1666,53 @@ class BaseMindooDB {
         };
         // Update cache
         this.docCache.set(docId, internalDoc);
-        console.log(`[BaseMindooDB] ===== Successfully loaded document ${docId} and cached it =====`);
+        this.logger.debug(`===== Successfully loaded document ${docId} and cached it =====`);
         return internalDoc;
     }
+    /**
+     * Convert an Automerge document to a plain JS object, converting Text objects to strings.
+     * If using native Automerge (react-native-automerge-generated), this uses the native
+     * materialize() method which properly converts Text objects to strings.
+     * Falls back to direct access if native backend is not available.
+     */
+    convertAutomergeToJS(doc) {
+        // Check if this document has a native Automerge handle attached
+        // The native implementation attaches metadata with Symbol.for('_am_meta')
+        const STATE = Symbol.for('_am_meta');
+        const meta = doc[STATE];
+        if (meta && meta.handle && typeof meta.handle.materialize === 'function') {
+            // Use native materialize() which properly converts Text objects to strings
+            try {
+                const materialized = meta.handle.materialize('/');
+                return materialized;
+            }
+            catch (error) {
+                console.error('[MindooDB] Failed to materialize document:', error);
+                // Fall through to direct access
+            }
+        }
+        // Fallback: direct access (for WASM or if native fails)
+        const result = {};
+        const keys = Object.keys(doc);
+        for (const key of keys) {
+            const value = doc[key];
+            if (value !== null && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Uint8Array)) {
+                result[key] = this.convertAutomergeToJS(value);
+            }
+            else if (Array.isArray(value)) {
+                result[key] = value.map(item => {
+                    if (item !== null && typeof item === 'object' && !Array.isArray(item) && !(item instanceof Uint8Array)) {
+                        return this.convertAutomergeToJS(item);
+                    }
+                    return item;
+                });
+            }
+            else {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
     /**
      * Wrap an internal document in the MindooDoc interface.
      * This is the read-only wrapper - write methods throw errors.
@@ -1247,7 +1745,11 @@ class BaseMindooDB {
             getCreatedAt: () => internalDoc.createdAt,
             getLastModified: () => internalDoc.lastModified,
             isDeleted: () => internalDoc.isDeleted,
-            getData: () => createReadOnlyProxy(internalDoc.doc),
+            getData: () => {
+                // Convert Automerge document to plain JS object, converting Text objects to strings
+                const jsDoc = this.convertAutomergeToJS(internalDoc.doc);
+                return createReadOnlyProxy(jsDoc);
+            },
             // ========== Attachment Write Methods ==========
             // These throw errors in the read-only wrapper
             addAttachment: async () => {
@@ -1299,12 +1801,12 @@ class BaseMindooDB {
      * Internal method to fetch and concatenate all chunks for an attachment.
      */
     async getAttachmentInternal(docId, attachmentId) {
-        console.log(`[BaseMindooDB] Getting attachment ${attachmentId} from document ${docId}`);
+        this.logger.debug(`Getting attachment ${attachmentId} from document ${docId}`);
         const ref = this.getAttachmentRefInternal(docId, attachmentId);
         const store = this.getEffectiveAttachmentStore();
         // Resolve dependency chain to get all chunk IDs in order (oldest first)
         const chunkIds = await store.resolveDependencies(ref.lastChunkId, { includeStart: true });
-        console.log(`[BaseMindooDB] Resolved ${chunkIds.length} chunks for attachment ${attachmentId}`);
+        this.logger.debug(`Resolved ${chunkIds.length} chunks for attachment ${attachmentId}`);
         // Fetch all chunks
         const chunks = await store.getEntries(chunkIds);
         // Verify signatures, decrypt, and collect plaintext chunks
@@ -1332,14 +1834,14 @@ class BaseMindooDB {
             result.set(chunk, offset);
             offset += chunk.length;
         }
-        console.log(`[BaseMindooDB] Retrieved attachment ${attachmentId}: ${result.length} bytes`);
+        this.logger.debug(`Retrieved attachment ${attachmentId}: ${result.length} bytes`);
         return result;
     }
     /**
      * Internal method to get a byte range from an attachment.
      */
     async getAttachmentRangeInternal(docId, attachmentId, startByte, endByte) {
-        console.log(`[BaseMindooDB] Getting attachment ${attachmentId} range [${startByte}, ${endByte}) from document ${docId}`);
+        this.logger.debug(`Getting attachment ${attachmentId} range [${startByte}, ${endByte}) from document ${docId}`);
         if (startByte < 0 || endByte <= startByte) {
             throw new Error(`Invalid byte range: [${startByte}, ${endByte})`);
         }
@@ -1388,14 +1890,14 @@ class BaseMindooDB {
             resultOffset += bytesToCopy;
             bytesRemaining -= bytesToCopy;
         }
-        console.log(`[BaseMindooDB] Retrieved attachment ${attachmentId} range: ${result.length} bytes`);
+        this.logger.debug(`Retrieved attachment ${attachmentId} range: ${result.length} bytes`);
         return result;
     }
     /**
      * Internal async generator to stream attachment data.
      */
     async *streamAttachmentInternal(docId, attachmentId, startOffset) {
-        console.log(`[BaseMindooDB] Streaming attachment ${attachmentId} from offset ${startOffset}`);
+        this.logger.debug(`Streaming attachment ${attachmentId} from offset ${startOffset}`);
         const ref = this.getAttachmentRefInternal(docId, attachmentId);
         const store = this.getEffectiveAttachmentStore();
         const chunkSize = this.chunkSizeBytes;
@@ -1426,7 +1928,7 @@ class BaseMindooDB {
                 yield plaintext;
             }
         }
-        console.log(`[BaseMindooDB] Finished streaming attachment ${attachmentId}`);
+        this.logger.debug(`Finished streaming attachment ${attachmentId}`);
     }
     /**
      * Get the effective attachment store (attachment store if configured, otherwise doc store).
@@ -1438,7 +1940,7 @@ class BaseMindooDB {
      * Internal method to add an attachment by chunking the file and storing chunks.
      */
     async addAttachmentInternal(docId, fileData, fileName, mimeType, decryptionKeyId, createdAt) {
-        console.log(`[BaseMindooDB] Adding attachment to document ${docId}: ${fileName} (${fileData.length} bytes)`);
+        this.logger.debug(`Adding attachment to document ${docId}: ${fileName} (${fileData.length} bytes)`);
         const store = this.getEffectiveAttachmentStore();
         const currentUser = await this.tenant.getCurrentUserId();
         const attachmentId = (0, idGeneration_1.generateFileUuid7)();
@@ -1477,7 +1979,7 @@ class BaseMindooDB {
         }
         // Store all chunks
         await store.putEntries(chunks);
-        console.log(`[BaseMindooDB] Stored ${chunks.length} chunks for attachment ${attachmentId}`);
+        this.logger.debug(`Stored ${chunks.length} chunks for attachment ${attachmentId}`);
         // Create attachment reference
         const ref = {
             attachmentId,
@@ -1496,7 +1998,7 @@ class BaseMindooDB {
      * Memory efficient - processes data chunk by chunk without loading entire file into memory.
      */
     async addAttachmentStreamInternal(docId, dataStream, fileName, mimeType, decryptionKeyId, createdAt) {
-        console.log(`[BaseMindooDB] Adding streaming attachment to document ${docId}: ${fileName}`);
+        this.logger.debug(`Adding streaming attachment to document ${docId}: ${fileName}`);
         const store = this.getEffectiveAttachmentStore();
         const currentUser = await this.tenant.getCurrentUserId();
         const attachmentId = (0, idGeneration_1.generateFileUuid7)();
@@ -1561,7 +2063,7 @@ class BaseMindooDB {
             lastChunkId = await storeChunk(buffer);
             totalSize += buffer.length;
         }
-        console.log(`[BaseMindooDB] Stored ${chunkCount} chunks for streaming attachment ${attachmentId} (${totalSize} bytes)`);
+        this.logger.debug(`Stored ${chunkCount} chunks for streaming attachment ${attachmentId} (${totalSize} bytes)`);
         // Create attachment reference
         const ref = {
             attachmentId,
@@ -1579,7 +2081,7 @@ class BaseMindooDB {
      * Internal method to append data to an existing attachment.
      */
     async appendToAttachmentInternal(docId, attachmentId, decryptionKeyId, prevLastChunkId, data, createdAt) {
-        console.log(`[BaseMindooDB] Appending ${data.length} bytes to attachment ${attachmentId}`);
+        this.logger.debug(`Appending ${data.length} bytes to attachment ${attachmentId}`);
         const store = this.getEffectiveAttachmentStore();
         const currentUser = await this.tenant.getCurrentUserId();
         // Chunk the data
@@ -1617,7 +2119,7 @@ class BaseMindooDB {
         }
         // Store all chunks
         await store.putEntries(chunks);
-        console.log(`[BaseMindooDB] Appended ${chunks.length} chunks to attachment ${attachmentId}`);
+        this.logger.debug(`Appended ${chunks.length} chunks to attachment ${attachmentId}`);
         return {
             lastChunkId,
             sizeIncrease: data.length,
@@ -1639,28 +2141,17 @@ class BaseMindooDB {
         if (this.store.getId() !== remoteStore.getId()) {
             throw new Error(`[BaseMindooDB] Cannot pull entries from the incompatible store ${this.store.getId()}`);
         }
-        console.log(`[BaseMindooDB] Pulling entries from remote store ${remoteStore.getId()}`);
-        // Get all entry IDs we already have in our local store
-        const localEntryIds = await this.store.getAllIds();
-        console.log(`[BaseMindooDB] Local store has ${localEntryIds.length} entry IDs`);
-        // Find entries in the remote store that we don't have
-        const newEntryMetadata = await remoteStore.findNewEntries(localEntryIds);
-        console.log(`[BaseMindooDB] Found ${newEntryMetadata.length} new entries in remote store`);
-        if (newEntryMetadata.length === 0) {
-            console.log(`[BaseMindooDB] No new entries to pull`);
+        this.logger.info(`Pulling entries from remote store ${remoteStore.getId()}`);
+        const transferred = await this.syncEntriesFromStore(remoteStore, this.store);
+        this.logger.debug(`Transferred ${transferred} entries from remote store`);
+        if (transferred === 0) {
+            this.logger.debug(`No new entries to pull`);
             return;
         }
-        // Get the full entries from the remote store
-        const newEntries = await remoteStore.getEntries(newEntryMetadata.map(em => em.id));
-        console.log(`[BaseMindooDB] Retrieved ${newEntries.length} entries from remote store`);
-        // Store all entries in our local store
-        // The putEntries method handles deduplication (no-op if entry already exists)
-        await this.store.putEntries(newEntries);
-        console.log(`[BaseMindooDB] Stored ${newEntries.length} entries in local store`);
         // Sync the local store to process the new entries
         // This will update the index, cache, and processedEntryIds
         await this.syncStoreChanges();
-        console.log(`[BaseMindooDB] Pull complete, synced ${newEntries.length} entries`);
+        this.logger.info(`Pull complete, synced ${transferred} entries`);
     }
     /**
      * Push changes to a remote content-addressed store.
@@ -1677,24 +2168,14 @@ class BaseMindooDB {
         if (this.store.getId() !== remoteStore.getId()) {
             throw new Error(`[BaseMindooDB] Cannot push entries to the incompatible store ${this.store.getId()}`);
         }
-        console.log(`[BaseMindooDB] Pushing entries to remote store ${remoteStore.getId()}`);
-        // Get all entry IDs the remote store already has
-        const remoteEntryIds = await remoteStore.getAllIds();
-        console.log(`[BaseMindooDB] Remote store has ${remoteEntryIds.length} entry IDs`);
-        // Find entries in our local store that the remote doesn't have
-        const newEntryMetadata = await this.store.findNewEntries(remoteEntryIds);
-        console.log(`[BaseMindooDB] Found ${newEntryMetadata.length} new entries to push`);
-        if (newEntryMetadata.length === 0) {
-            console.log(`[BaseMindooDB] No new entries to push`);
+        this.logger.info(`Pushing entries to remote store ${remoteStore.getId()}`);
+        const transferred = await this.syncEntriesFromStore(this.store, remoteStore);
+        this.logger.debug(`Transferred ${transferred} entries to remote store`);
+        if (transferred === 0) {
+            this.logger.debug(`No new entries to push`);
             return;
         }
-        // Get the full entries from our local store
-        const newEntries = await this.store.getEntries(newEntryMetadata.map(em => em.id));
-        console.log(`[BaseMindooDB] Retrieved ${newEntries.length} entries from local store`);
-        // Store all entries in the remote store
-        // The putEntries method handles deduplication (no-op if entry already exists)
-        await remoteStore.putEntries(newEntries);
-        console.log(`[BaseMindooDB] Pushed ${newEntries.length} entries to remote store`);
+        this.logger.info(`Pushed ${transferred} entries to remote store`);
     }
 }
 exports.BaseMindooDB = BaseMindooDB;