mindoodb 0.0.4 → 0.0.5

package/dist/core/BaseMindooDB.js

@@ -8,6 +8,7 @@ const automerge_adapter_1 = require("./automerge-adapter");
 const uuid_1 = require("uuid");
 const bloom_1 = require("./appendonlystores/bloom");
 const MaterializationPlanner_1 = require("./appendonlystores/MaterializationPlanner");
+const AttachmentReadPlanner_1 = require("./appendonlystores/AttachmentReadPlanner");
 const idGeneration_1 = require("./utils/idGeneration");
 const errors_1 = require("./errors");
 const logging_1 = require("./logging");
@@ -71,6 +72,7 @@ class BaseMindooDB {
         this.iteratePrefetchWindowDocs = Math.max(0, Math.floor(documentCacheConfig?.iteratePrefetchWindowDocs ??
             DEFAULT_ITERATE_PREFETCH_WINDOW_DOCS));
         this.cacheRestoreLimit = Math.max(1, Math.floor(documentCacheConfig?.restoreLimit ?? this.maxCachedDocs));
+        this.reconcileRestoredIndexOnInit = documentCacheConfig?.reconcileRestoredIndexOnInit ?? false;
         this.snapshotMinChanges = Math.max(1, Math.floor(snapshotConfig?.minChanges ?? DEFAULT_SNAPSHOT_MIN_CHANGES));
         this.snapshotCooldownMs = Math.max(0, Math.floor(snapshotConfig?.cooldownMs ?? DEFAULT_SNAPSHOT_COOLDOWN_MS));
         // Create logger if not provided (for backward compatibility)
@@ -184,7 +186,7 @@ class BaseMindooDB {
     }
     exportMetadataCheckpoint() {
         const checkpoint = {
-            version: 1,
+            version: 2,
             processedEntryCursor: this.processedEntryCursor,
             index: this.index,
             nextChangeSeq: this.nextChangeSeq,
@@ -214,7 +216,7 @@ class BaseMindooDB {
                 return false;
             }
             const checkpoint = JSON.parse(new TextDecoder().decode(metaBytes));
-            if (checkpoint.version !== 1) {
+            if (checkpoint.version !== 2) {
                 this.logger.warn(`Unknown cache version ${checkpoint.version}, ignoring cache`);
                 return false;
             }
@@ -266,6 +268,7 @@ class BaseMindooDB {
                     this.logger.warn(`Failed to restore cached doc ${docId}, will reload from store: ${e}`);
                 }
             }
+            this.logger.info(`Restored ${restoredDocs} cached documents for ${this.store.getId()}`);
             this.logger.info(`Restored ${restoredDocs} documents from cache for ${prefix}`);
             return true;
         }
@@ -460,11 +463,42 @@ class BaseMindooDB {
             if (restored) {
                 this.logger.info("Restored from cache, processing delta only");
                 await this.syncStoreChanges();
+                if (this.reconcileRestoredIndexOnInit) {
+                    await this.reconcileRestoredIndexWithStore();
+                }
                 return;
             }
         }
         await this.syncStoreChanges();
     }
+    async reconcileRestoredIndexWithStore() {
+        const lifecycleMetadata = await this.scanAllMetadata(this.store);
+        const lifecycleDocIds = Array.from(new Set(lifecycleMetadata
+            .filter((entry) => entry.entryType === "doc_create"
+            || entry.entryType === "doc_change"
+            || entry.entryType === "doc_delete"
+            || entry.entryType === "doc_snapshot")
+            .map((entry) => entry.docId))).sort((left, right) => left.localeCompare(right));
+        const missingDocIds = lifecycleDocIds.filter((docId) => !this.indexLookup.has(docId));
+        if (missingDocIds.length === 0) {
+            return;
+        }
+        this.logger.warn(`Detected stale cache checkpoint for ${this.store.getId()} - ` +
+            `index is missing ${missingDocIds.length} document(s): ${missingDocIds.join(", ")}. ` +
+            `Rebuilding metadata from local store.`);
+        this.index = [];
+        this.indexLookup.clear();
+        this.docCache.clear();
+        this.automergeHashToEntryId.clear();
+        this.processedEntryIds = [];
+        this.processedEntryCursor = null;
+        this.nextChangeSeq = 1;
+        this.cacheMetaDirty = true;
+        this.cacheManager?.markDirty();
+        await this.syncStoreChanges();
+        this.logger.info(`Metadata rebuild complete for ${this.store.getId()} - ` +
+            `index now has ${this.index.length} document(s).`);
+    }
     /**
      * Sync changes from the content-addressed store by finding new entries and processing them.
      * This method can be called multiple times to incrementally sync new entries.
@@ -475,9 +509,10 @@ class BaseMindooDB {
         this.logger.debug(`Syncing store changes for database ${this.store.getId()} in tenant ${this.tenant.getId()}`);
         this.logger.debug(`Already processed ${this.processedEntryIds.length} entry IDs`);
         // Find new entries that we haven't processed yet
-        const newEntryMetadata = await this.getNewEntryMetadataForSync();
+        const { entries: newEntryMetadata, nextCursor } = await this.getNewEntryMetadataForSync();
         this.logger.debug(`Found ${newEntryMetadata.length} new entries`);
         if (newEntryMetadata.length === 0) {
+            this.processedEntryCursor = nextCursor;
             this.logger.debug(`No new entries to process`);
             this.performanceCallback?.onSyncOperation?.({
                 operation: "findNewEntries",
@@ -608,6 +643,7 @@ class BaseMindooDB {
         }
         // Append new entry IDs to our processed list
         this.processedEntryIds.push(...newEntryMetadata.map(em => em.id));
+        this.processedEntryCursor = nextCursor;
         this.cacheMetaDirty = true;
         this.cacheManager?.markDirty();
         this.logger.debug(`Synced ${newEntryMetadata.length} new entries, index now has ${this.index.length} documents`);
@@ -656,7 +692,10 @@ class BaseMindooDB {
                     processedEntryCount: this.processedEntryIds.length,
                 },
             });
-            return result;
+            return {
+                entries: result,
+                nextCursor: this.processedEntryCursor,
+            };
         }
         const allNew = [];
         let cursor = this.processedEntryCursor;
@@ -668,7 +707,6 @@ class BaseMindooDB {
                 break;
             }
         }
-        this.processedEntryCursor = cursor;
         this.performanceCallback?.onSyncOperation?.({
             operation: "findNewEntries",
             time: Date.now() - startedAt,
@@ -678,7 +716,10 @@ class BaseMindooDB {
                 cursor,
             },
         });
-        return allNew;
+        return {
+            entries: allNew,
+            nextCursor: cursor,
+        };
     }
     /**
      * Fetch the target store's bloom filter summary, returning null if
@@ -746,7 +787,7 @@ class BaseMindooDB {
             let currentPage = 0;
             while (true) {
                 if (options?.signal?.aborted) {
-                    return { transferred, cancelled: true };
+                    return { transferred, scanned, cancelled: true };
                 }
                 const page = await sourceStore.scanEntriesSince(cursor, pageSize);
                 currentPage++;
@@ -773,7 +814,7 @@ class BaseMindooDB {
                     break;
                 }
             }
-            return { transferred, cancelled: false };
+            return { transferred, scanned, cancelled: false };
         }
         onProgress?.({
             phase: 'preparing',
@@ -784,10 +825,10 @@ class BaseMindooDB {
         const targetIds = await targetStore.getAllIds();
         const sourceNewMetadata = await sourceStore.findNewEntries(targetIds);
         if (sourceNewMetadata.length === 0) {
-            return { transferred: 0, cancelled: false };
+            return { transferred: 0, scanned: 0, cancelled: false };
         }
         if (options?.signal?.aborted) {
-            return { transferred: 0, cancelled: true };
+            return { transferred: 0, scanned: 0, cancelled: true };
         }
         onProgress?.({
             phase: 'transferring',
@@ -805,7 +846,11 @@ class BaseMindooDB {
             scannedEntries: sourceNewMetadata.length,
             totalSourceEntries: sourceNewMetadata.length,
         });
-        return { transferred: sourceEntries.length, cancelled: false };
+        return {
+            transferred: sourceEntries.length,
+            scanned: sourceNewMetadata.length,
+            cancelled: false,
+        };
     }
     /**
      * Dense sync: transfer only the entries required to reconstruct the latest
@@ -832,7 +877,7 @@ class BaseMindooDB {
             scannedEntries: 0,
         });
         if (options?.signal?.aborted) {
-            return { transferred: 0, cancelled: true };
+            return { transferred: 0, scanned: 0, cancelled: true };
         }
         // ── Phase 1: discover documents ──────────────────────────────────
         const docCreateEntries = await sourceStore.findEntries("doc_create", null, null);
@@ -840,10 +885,10 @@ class BaseMindooDB {
         const docIds = [...new Set(docCreateEntries.map((e) => e.docId))];
         this.logger.info(`Dense sync: found ${docIds.length} documents on source`);
         if (docIds.length === 0) {
-            return { transferred: 0, cancelled: false };
+            return { transferred: 0, scanned: 0, cancelled: false };
         }
         if (options?.signal?.aborted) {
-            return { transferred: 0, cancelled: true };
+            return { transferred: 0, scanned: 0, cancelled: true };
         }
         // ── Phase 2: batch plan on source ────────────────────────────────
         onProgress?.({
@@ -872,7 +917,7 @@ class BaseMindooDB {
         this.logger.info(`Dense sync: planner identified ${neededIds.size} required entries ` +
             `(from ${docIds.length} documents)`);
         if (options?.signal?.aborted) {
-            return { transferred: 0, cancelled: true };
+            return { transferred: 0, scanned: neededIds.size, cancelled: true };
         }
         // ── Phase 4: filter out entries the target already has ───────────
         const allNeededArray = Array.from(neededIds);
@@ -881,17 +926,17 @@ class BaseMindooDB {
         this.logger.info(`Dense sync: ${missingIds.length} entries to transfer ` +
             `(${allNeededArray.length - missingIds.length} already present)`);
         if (missingIds.length === 0) {
-            return { transferred: 0, cancelled: false };
+            return { transferred: 0, scanned: allNeededArray.length, cancelled: false };
         }
         if (options?.signal?.aborted) {
-            return { transferred: 0, cancelled: true };
+            return { transferred: 0, scanned: allNeededArray.length, cancelled: true };
         }
         // ── Phase 5: transfer missing entries in pages ───────────────────
         const pageSize = options?.pageSize ?? 500;
         let transferred = 0;
         for (let offset = 0; offset < missingIds.length; offset += pageSize) {
             if (options?.signal?.aborted) {
-                return { transferred, cancelled: true };
+                return { transferred, scanned: allNeededArray.length, cancelled: true };
             }
             const batch = missingIds.slice(offset, offset + pageSize);
             const entries = await sourceStore.getEntries(batch);
@@ -906,7 +951,7 @@ class BaseMindooDB {
             });
         }
         this.logger.info(`Dense sync complete: transferred ${transferred} entries`);
-        return { transferred, cancelled: false };
+        return { transferred, scanned: allNeededArray.length, cancelled: false };
     }
     getStore() {
         return this.store;
@@ -1067,9 +1112,15 @@ class BaseMindooDB {
         this.logger.debug(`Getting document ${docId} at timestamp ${timestamp}`);
         // Get all entry metadata for this document
         const allEntryMetadata = await this.scanAllMetadata(this.store, { docId });
-        // Filter entries up to the timestamp
+        // Filter to document replay metadata plus snapshots up to the timestamp.
+        // Attachment chunks are not part of the Automerge replay DAG and must not
+        // participate in historical materialization.
         const relevantEntries = allEntryMetadata
-            .filter(em => em.createdAt <= timestamp)
+            .filter((em) => em.createdAt <= timestamp
+            && (em.entryType === "doc_create"
+                || em.entryType === "doc_change"
+                || em.entryType === "doc_delete"
+                || em.entryType === "doc_snapshot"))
             .sort((a, b) => a.createdAt - b.createdAt);
         if (relevantEntries.length === 0) {
             this.performanceCallback?.onHistoryOperation?.({
@@ -1081,43 +1132,85 @@ class BaseMindooDB {
             });
             return null; // Document didn't exist at that time
         }
-        // Load entries
-        const entries = await this.store.getEntries(relevantEntries.map(em => em.id));
-        // Apply changes in order
-        let doc = automerge_adapter_1.Automerge.init();
-        for (const entryData of entries) {
-            // Admin-only mode: only accept entries signed by the admin key
+        const metadataById = new Map(relevantEntries.map((meta) => [meta.id, meta]));
+        const materializationPlan = (0, MaterializationPlanner_1.computeDocumentMaterializationPlan)(docId, relevantEntries);
+        let startFromSnapshot = materializationPlan.snapshotEntryId !== null;
+        const snapshotMeta = materializationPlan.snapshotEntryId
+            ? (metadataById.get(materializationPlan.snapshotEntryId) || null)
+            : null;
+        if (startFromSnapshot && !snapshotMeta) {
+            this.logger.warn(`Planner referenced snapshot ${materializationPlan.snapshotEntryId} not found in metadata for ${docId}; falling back to replay without snapshot`);
+            startFromSnapshot = false;
+        }
+        let doc = undefined;
+        if (startFromSnapshot && snapshotMeta) {
+            const snapshotEntries = await this.store.getEntries([snapshotMeta.id]);
+            if (snapshotEntries.length > 0) {
+                const snapshotData = snapshotEntries[0];
+                let isValid = false;
+                if (this._isAdminOnlyDb && snapshotData.createdByPublicKey !== this.getAdminPublicKey()) {
+                    this.logger.warn(`Admin-only DB: skipping snapshot ${snapshotData.id} not signed by admin key`);
+                }
+                else {
+                    isValid = await this.tenant.verifySignature(snapshotData.encryptedData, snapshotData.signature, snapshotData.createdByPublicKey);
+                }
+                if (!isValid) {
+                    this.logger.warn(`Invalid signature for snapshot ${snapshotData.id}, falling back to replay without snapshot`);
+                    startFromSnapshot = false;
+                }
+                else {
+                    const decryptedSnapshot = await this.tenant.decryptPayload(snapshotData.encryptedData, snapshotData.decryptionKeyId);
+                    doc = automerge_adapter_1.Automerge.load(decryptedSnapshot);
+                    const parsed = (0, idGeneration_1.parseDocEntryId)(snapshotData.id);
+                    if (parsed) {
+                        this.registerAutomergeHashMapping(docId, parsed.automergeHash, snapshotData.id);
+                    }
+                }
+            }
+        }
+        if (!doc) {
+            doc = automerge_adapter_1.Automerge.init();
+        }
+        const entriesToApply = materializationPlan.entryIdsToApply
+            .map((id) => metadataById.get(id))
+            .filter((entry) => entry !== undefined);
+        const loadedEntries = entriesToApply.length > 0
+            ? await this.store.getEntries(entriesToApply.map((entry) => entry.id))
+            : [];
+        const entryById = new Map(loadedEntries.map((entry) => [entry.id, entry]));
+        for (const entryMeta of entriesToApply) {
+            const entryData = entryById.get(entryMeta.id);
+            if (!entryData) {
+                this.logger.warn(`Entry ${entryMeta.id} not found in store, skipping`);
+                continue;
+            }
             if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
                 this.logger.warn(`Admin-only DB: skipping entry ${entryData.id} not signed by admin key`);
                 continue;
             }
-            // Verify signature against the encrypted payload (no decryption needed)
-            // We sign the encrypted payload, so anyone can verify signatures without decryption keys
             const isValid = await this.tenant.verifySignature(entryData.encryptedData, entryData.signature, entryData.createdByPublicKey);
             if (!isValid) {
                 this.logger.warn(`Invalid signature for entry ${entryData.id}, skipping`);
                 continue;
             }
-            // Decrypt payload (only after signature verification passes)
             const decryptedPayload = await this.tenant.decryptPayload(entryData.encryptedData, entryData.decryptionKeyId);
-            // Apply change using loadIncremental - this is the recommended way for binary change data
             doc = automerge_adapter_1.Automerge.loadIncremental(doc, decryptedPayload);
-            // Register the automerge hash -> entry ID mapping for future dependency resolution
             const parsed = (0, idGeneration_1.parseDocEntryId)(entryData.id);
             if (parsed) {
                 this.registerAutomergeHashMapping(docId, parsed.automergeHash, entryData.id);
             }
         }
-        // Check if document was deleted at this timestamp
-        // Look for a "doc_delete" type entry in the relevant entries
-        const deleteEntry = relevantEntries.find(em => em.entryType === "doc_delete" && em.createdAt <= timestamp);
-        const isDeleted = deleteEntry !== undefined;
-        // Find the first entry to get createdAt and decryptionKeyId
-        const firstEntry = relevantEntries.length > 0 ? relevantEntries[0] : null;
-        const createdAt = firstEntry ? firstEntry.createdAt : timestamp;
-        const decryptionKeyId = firstEntry ? firstEntry.decryptionKeyId : "default";
-        // Use delete entry timestamp as lastModified if deleted, otherwise use the requested timestamp
-        const lastModified = isDeleted && deleteEntry ? deleteEntry.createdAt : timestamp;
+        const replayEntries = relevantEntries.filter((entry) => entry.entryType === "doc_create"
+            || entry.entryType === "doc_change"
+            || entry.entryType === "doc_delete");
+        const firstReplayEntry = replayEntries.length > 0 ? replayEntries[0] : null;
+        const lastReplayEntry = replayEntries.length > 0 ? replayEntries[replayEntries.length - 1] : null;
+        const createdAt = firstReplayEntry?.createdAt ?? snapshotMeta?.createdAt ?? timestamp;
+        const decryptionKeyId = firstReplayEntry?.decryptionKeyId
+            ?? snapshotMeta?.decryptionKeyId
+            ?? "default";
+        const isDeleted = lastReplayEntry?.entryType === "doc_delete";
+        const lastModified = lastReplayEntry?.createdAt ?? snapshotMeta?.createdAt ?? timestamp;
         const internalDoc = {
             id: docId,
             doc,
@@ -1287,6 +1380,15 @@ class BaseMindooDB {
         }
         return docIds;
     }
+    async getDeletedDocumentIds() {
+        const docIds = [];
+        for (const entry of this.index) {
+            if (entry.isDeleted) {
+                docIds.push(entry.docId);
+            }
+        }
+        return docIds;
+    }
     async getAllDocumentIdsAtTimestamp(timestamp) {
         // findEntries() uses an exclusive upper bound (createdAt < creationDateUntil).
         // Query up to timestamp + 1 so entries created exactly at `timestamp` are included
@@ -2012,10 +2114,10 @@ class BaseMindooDB {
             this.logger.debug(`No change entries to apply for document ${docId}`);
             return null; // No changes
         }
-        // Sort entries by timestamp
-        entriesToApply.sort((a, b) => a.createdAt - b.createdAt);
+        const entriesById = new Map(entriesToApply.map((entry) => [entry.id, entry]));
+        const orderedEntryIds = (0, MaterializationPlanner_1.topologicalByDependencies)(new Set(entriesToApply.map((entry) => entry.id)), entriesById);
         // Load entries from store
-        const entries = await this.store.getEntries(entriesToApply.map(em => em.id));
+        const entries = await this.store.getEntries(orderedEntryIds);
         // Filter entries for admin-only mode first
         const validEntries = entries.filter(entryData => {
             if (this._isAdminOnlyDb && entryData.createdByPublicKey !== this.getAdminPublicKey()) {
@@ -2103,9 +2205,9 @@ class BaseMindooDB {
         this.logger.debug(`Document ${docId} changed: heads before=${JSON.stringify(headsBefore)}, after=${JSON.stringify(headsAfter)}`);
         // Update metadata
         const payload = updatedDoc;
-        const lastEntry = entries[entries.length - 1];
+        const lastEntryCreatedAt = entries.reduce((maxCreatedAt, entry) => Math.max(maxCreatedAt, entry.createdAt), cachedDoc.lastModified);
         const lastModified = payload._lastModified ||
-            (lastEntry ? lastEntry.createdAt : cachedDoc.lastModified);
+            lastEntryCreatedAt;
         // Check if document was deleted
         const hasDeleteEntry = newEntryMetadata.some(em => em.entryType === "doc_delete");
         const isDeleted = hasDeleteEntry || cachedDoc.isDeleted;
@@ -2556,6 +2658,39 @@ class BaseMindooDB {
         }
         return ref;
     }
+    async planAttachmentRead(store, ref, startByte, endByteExclusive) {
+        if (store.planAttachmentReadByWalkingMetadata) {
+            try {
+                // if the store has a planAttachmentReadByWalkingMetadata method, use it for less overhead through the network
+                return await store.planAttachmentReadByWalkingMetadata(ref.lastChunkId, ref.size, {
+                    startByte,
+                    endByteExclusive,
+                });
+            }
+            catch (error) {
+                this.logger.debug("Store-level attachment read planner failed, falling back to local metadata walk", { attachmentId: ref.attachmentId, error });
+            }
+        }
+        // fall back to local metadata walk
+        return (0, AttachmentReadPlanner_1.planAttachmentReadByWalkingMetadata)(store, ref.lastChunkId, ref.size, {
+            startByte,
+            endByteExclusive,
+        });
+    }
+    async decryptAttachmentChunk(chunk) {
+        if (this._isAdminOnlyDb && chunk.createdByPublicKey !== this.getAdminPublicKey()) {
+            throw new Error(`Admin-only DB: chunk ${chunk.id} not signed by admin key`);
+        }
+        const isValid = await this.tenant.verifySignature(chunk.encryptedData, chunk.signature, chunk.createdByPublicKey);
+        if (!isValid) {
+            throw new Error(`Invalid signature for chunk ${chunk.id}`);
+        }
+        const plaintext = await this.tenant.decryptAttachmentPayload(chunk.encryptedData, chunk.decryptionKeyId);
+        if (plaintext.length !== chunk.originalSize) {
+            throw new Error(`Attachment chunk ${chunk.id} decrypted to ${plaintext.length} bytes, expected ${chunk.originalSize}`);
+        }
+        return plaintext;
+    }
     /**
      * Internal method to fetch and concatenate all chunks for an attachment.
      */
@@ -2572,17 +2707,7 @@ class BaseMindooDB {
         const plaintextChunks = [];
         let totalSize = 0;
         for (const chunk of chunks) {
-            // Admin-only mode: only accept chunks signed by the admin key
-            if (this._isAdminOnlyDb && chunk.createdByPublicKey !== this.getAdminPublicKey()) {
-                throw new Error(`Admin-only DB: chunk ${chunk.id} not signed by admin key`);
-            }
-            // Verify signature
-            const isValid = await this.tenant.verifySignature(chunk.encryptedData, chunk.signature, chunk.createdByPublicKey);
-            if (!isValid) {
-                throw new Error(`Invalid signature for chunk ${chunk.id}`);
-            }
-            // Decrypt
-            const plaintext = await this.tenant.decryptAttachmentPayload(chunk.encryptedData, chunk.decryptionKeyId);
+            const plaintext = await this.decryptAttachmentChunk(chunk);
             plaintextChunks.push(plaintext);
             totalSize += plaintext.length;
         }
@@ -2605,37 +2730,16 @@ class BaseMindooDB {
             throw new Error(`Invalid byte range: [${startByte}, ${endByte})`);
         }
         const ref = this.getAttachmentRefInternal(docId, attachmentId);
-        if (endByte > ref.size) {
-            throw new Error(`End byte ${endByte} exceeds attachment size ${ref.size}`);
-        }
         const store = this.getEffectiveAttachmentStore();
-        const chunkSize = this.chunkSizeBytes;
-        // Calculate which chunks we need
-        const startChunkIndex = Math.floor(startByte / chunkSize);
-        const endChunkIndex = Math.floor((endByte - 1) / chunkSize);
-        // Resolve dependency chain to get all chunk IDs
-        const allChunkIds = await store.resolveDependencies(ref.lastChunkId, { includeStart: true });
-        // Get only the chunks we need
-        const neededChunkIds = allChunkIds.slice(startChunkIndex, endChunkIndex + 1);
+        const readPlan = await this.planAttachmentRead(store, ref, startByte, endByte);
+        const neededChunkIds = readPlan.chunkPlans.map((chunkPlan) => chunkPlan.id);
         const chunks = await store.getEntries(neededChunkIds);
         // Decrypt needed chunks
         const plaintextChunks = [];
         for (const chunk of chunks) {
-            // Admin-only mode: only accept chunks signed by the admin key
-            if (this._isAdminOnlyDb && chunk.createdByPublicKey !== this.getAdminPublicKey()) {
-                throw new Error(`Admin-only DB: chunk ${chunk.id} not signed by admin key`);
-            }
-            // Verify signature
-            const isValid = await this.tenant.verifySignature(chunk.encryptedData, chunk.signature, chunk.createdByPublicKey);
-            if (!isValid) {
-                throw new Error(`Invalid signature for chunk ${chunk.id}`);
-            }
-            // Decrypt
-            const plaintext = await this.tenant.decryptAttachmentPayload(chunk.encryptedData, chunk.decryptionKeyId);
+            const plaintext = await this.decryptAttachmentChunk(chunk);
             plaintextChunks.push(plaintext);
         }
-        // Calculate offsets within the fetched chunks
-        const offsetInFirstChunk = startByte - (startChunkIndex * chunkSize);
         const totalNeededBytes = endByte - startByte;
         // Extract the requested range
         const result = new Uint8Array(totalNeededBytes);
@@ -2643,7 +2747,7 @@ class BaseMindooDB {
         let bytesRemaining = totalNeededBytes;
         for (let i = 0; i < plaintextChunks.length && bytesRemaining > 0; i++) {
             const chunk = plaintextChunks[i];
-            const chunkStart = i === 0 ? offsetInFirstChunk : 0;
+            const chunkStart = i === 0 ? readPlan.offsetInFirstChunk : 0;
             const bytesToCopy = Math.min(chunk.length - chunkStart, bytesRemaining);
             result.set(chunk.slice(chunkStart, chunkStart + bytesToCopy), resultOffset);
             resultOffset += bytesToCopy;
@@ -2658,30 +2762,25 @@ class BaseMindooDB {
     async *streamAttachmentInternal(docId, attachmentId, startOffset) {
         this.logger.debug(`Streaming attachment ${attachmentId} from offset ${startOffset}`);
         const ref = this.getAttachmentRefInternal(docId, attachmentId);
+        if (startOffset < 0 || startOffset > ref.size) {
+            throw new Error(`Invalid stream offset ${startOffset} for attachment size ${ref.size}`);
+        }
+        if (ref.size === 0 || startOffset === ref.size) {
+            return;
+        }
         const store = this.getEffectiveAttachmentStore();
-        const chunkSize = this.chunkSizeBytes;
-        // Calculate starting chunk
-        const startChunkIndex = Math.floor(startOffset / chunkSize);
-        const offsetInStartChunk = startOffset % chunkSize;
-        // Resolve dependency chain to get all chunk IDs
-        const allChunkIds = await store.resolveDependencies(ref.lastChunkId, { includeStart: true });
+        const readPlan = await this.planAttachmentRead(store, ref, startOffset, ref.size);
         // Stream chunks starting from startChunkIndex
-        for (let i = startChunkIndex; i < allChunkIds.length; i++) {
-            const [chunk] = await store.getEntries([allChunkIds[i]]);
-            // Admin-only mode: only accept chunks signed by the admin key
-            if (this._isAdminOnlyDb && chunk.createdByPublicKey !== this.getAdminPublicKey()) {
-                throw new Error(`Admin-only DB: chunk ${chunk.id} not signed by admin key`);
-            }
-            // Verify signature
-            const isValid = await this.tenant.verifySignature(chunk.encryptedData, chunk.signature, chunk.createdByPublicKey);
-            if (!isValid) {
-                throw new Error(`Invalid signature for chunk ${chunk.id}`);
+        for (let i = 0; i < readPlan.chunkPlans.length; i++) {
+            const chunkPlan = readPlan.chunkPlans[i];
+            const [chunk] = await store.getEntries([chunkPlan.id]);
+            if (!chunk) {
+                throw new Error(`Attachment chunk ${chunkPlan.id} not found in store`);
             }
-            // Decrypt
-            const plaintext = await this.tenant.decryptAttachmentPayload(chunk.encryptedData, chunk.decryptionKeyId);
+            const plaintext = await this.decryptAttachmentChunk(chunk);
             // For first chunk, skip bytes before startOffset
-            if (i === startChunkIndex && offsetInStartChunk > 0) {
-                yield plaintext.slice(offsetInStartChunk);
+            if (i === 0 && readPlan.offsetInFirstChunk > 0) {
+                yield plaintext.slice(readPlan.offsetInFirstChunk);
             }
             else {
                 yield plaintext;
@@ -2966,11 +3065,15 @@ class BaseMindooDB {
             this.logger.debug(`Transferred ${syncResult.transferred} entries from remote store`);
             if (syncResult.cancelled) {
                 this.logger.info(`Pull cancelled after transferring ${syncResult.transferred} entries`);
-                return { transferredEntries: syncResult.transferred, cancelled: true };
+                return {
+                    transferredEntries: syncResult.transferred,
+                    scannedEntries: syncResult.scanned,
+                    cancelled: true,
+                };
             }
             if (syncResult.transferred === 0) {
                 this.logger.debug(`No new entries to pull`);
-                return { transferredEntries: 0, cancelled: false };
+                return { transferredEntries: 0, scannedEntries: syncResult.scanned, cancelled: false };
             }
             options?.onProgress?.({
                 phase: 'processing',
@@ -2988,7 +3091,11 @@ class BaseMindooDB {
                 transferredEntries: syncResult.transferred,
                 scannedEntries: syncResult.transferred,
             });
-            return { transferredEntries: syncResult.transferred, cancelled: false };
+            return {
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
+                cancelled: false,
+            };
         }
         finally {
             restoreAuthOverride();
@@ -3018,7 +3125,11 @@ class BaseMindooDB {
             this.logger.debug(`Transferred ${syncResult.transferred} entries to remote store`);
             if (syncResult.cancelled) {
                 this.logger.info(`Push cancelled after transferring ${syncResult.transferred} entries`);
-                return { transferredEntries: syncResult.transferred, cancelled: true };
+                return {
+                    transferredEntries: syncResult.transferred,
+                    scannedEntries: syncResult.scanned,
+                    cancelled: true,
+                };
             }
             if (syncResult.transferred === 0) {
                 this.logger.debug(`No new entries to push`);
@@ -3032,7 +3143,11 @@ class BaseMindooDB {
                 transferredEntries: syncResult.transferred,
                 scannedEntries: syncResult.transferred,
             });
-            return { transferredEntries: syncResult.transferred, cancelled: false };
+            return {
+                transferredEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
+                cancelled: false,
+            };
         }
         finally {
             restoreAuthOverride();