mindoodb 0.0.10 → 0.0.11

package/dist/core/BaseMindooDB.js

@@ -18,6 +18,7 @@ const logging_1 = require("./logging");
  * Default chunk size for attachments: 256KB
  */
 const DEFAULT_CHUNK_SIZE_BYTES = 256 * 1024;
+const DEFAULT_ATTACHMENT_STREAM_BATCH_SIZE = 4;
 const DEFAULT_SNAPSHOT_MIN_CHANGES = 100;
 const DEFAULT_SNAPSHOT_COOLDOWN_MS = 10 * 60 * 1000;
 const DEFAULT_MAX_CACHED_DOCS = 128;
@@ -778,6 +779,98 @@ class BaseMindooDB {
         }
         return missingIds;
     }
+    /**
+     * Determine how many entry IDs to fetch per `getEntries` call during sync.
+     *
+     * This is intentionally separate from the metadata scan `pageSize` so that
+     * scanning can page through large ID lists quickly while the heavier
+     * payload downloads use a smaller batch to keep progress responsive and
+     * cancellation timely.
+     *
+     * Priority: explicit option > attachment default (100) > pageSize fallback.
+     */
+    resolveTransferBatchSize(options) {
+        if (options?.transferBatchSize && options.transferBatchSize > 0) {
+            return options.transferBatchSize;
+        }
+        if (options?.storeKind === types_1.StoreKind.attachments) {
+            return 100;
+        }
+        return options?.pageSize ?? 1000;
+    }
+    /**
+     * Transfer a set of entry IDs from source to target in fixed-size batches,
+     * emitting progress and checking for cancellation between each batch.
+     *
+     * Callers (cursor-scan path and legacy path) collect the IDs that need
+     * transferring, then delegate to this method instead of issuing one
+     * monolithic `getEntries`.  This gives three benefits:
+     *
+     * 1. The UI receives frequent progress updates with batch metadata so long
+     *    transfers no longer look frozen.
+     * 2. Cancellation can interrupt work between batches rather than waiting
+     *    for one large HTTP response to complete.
+     * 3. Each server-side `getEntries` + RSA encryption unit is smaller,
+     *    reducing the risk of socket timeouts on large payloads.
+     *
+     * Returns partial progress on cancellation so callers can report how much
+     * was actually transferred before the abort.
+     */
+    async transferEntriesInBatches(sourceStore, targetStore, entryIds, options, state) {
+        if (entryIds.length === 0) {
+            return { transferred: state.transferred, cancelled: false };
+        }
+        const onProgress = options?.onProgress;
+        const signal = options?.signal;
+        const transferBatchSize = this.resolveTransferBatchSize(options);
+        const totalTransferBatches = Math.max(1, Math.ceil(entryIds.length / transferBatchSize));
+        let transferred = state.transferred;
+        for (let offset = 0; offset < entryIds.length; offset += transferBatchSize) {
+            if (signal?.aborted) {
+                return { transferred, cancelled: true };
+            }
+            const currentTransferBatch = Math.floor(offset / transferBatchSize) + 1;
+            const batchIds = entryIds.slice(offset, offset + transferBatchSize);
+            const pageSummary = state.currentPage ? `page ${state.currentPage}, ` : "";
+            onProgress?.({
+                phase: "transferring",
+                message: `Transferring batch ${currentTransferBatch}/${totalTransferBatches} (${batchIds.length} entries, ${pageSummary}scanned ${state.scanned})...`,
+                transferredEntries: transferred,
+                scannedEntries: state.scanned,
+                totalSourceEntries: state.totalSourceEntries,
+                currentPage: state.currentPage,
+                currentTransferBatch,
+                totalTransferBatches,
+                transferBatchSize,
+            });
+            try {
+                const batchEntries = await sourceStore.getEntries(batchIds);
+                if (signal?.aborted) {
+                    return { transferred, cancelled: true };
+                }
+                await targetStore.putEntries(batchEntries);
+                transferred += batchEntries.length;
+            }
+            catch (error) {
+                if (signal?.aborted) {
+                    return { transferred, cancelled: true };
+                }
+                throw error;
+            }
+            onProgress?.({
+                phase: "transferring",
+                message: `Transferred ${transferred} entries after batch ${currentTransferBatch}/${totalTransferBatches}`,
+                transferredEntries: transferred,
+                scannedEntries: state.scanned,
+                totalSourceEntries: state.totalSourceEntries,
+                currentPage: state.currentPage,
+                currentTransferBatch,
+                totalTransferBatches,
+                transferBatchSize,
+            });
+        }
+        return { transferred, cancelled: false };
+    }
     async syncEntriesFromStore(sourceStore, targetStore, options) {
         const signal = options?.signal;
         BaseMindooDB.setSyncAbortSignalOnStore(sourceStore, signal);
@@ -843,20 +936,16 @@ class BaseMindooDB {
                         return { transferred, scanned, cancelled: true };
                     }
                     if (missingIds.length > 0) {
-                        onProgress?.({
-                            phase: 'transferring',
-                            message: `Transferring ${missingIds.length} entries (page ${currentPage}, scanned ${scanned})...`,
-                            transferredEntries: transferred,
-                            scannedEntries: scanned,
+                        const transferResult = await this.transferEntriesInBatches(sourceStore, targetStore, missingIds, options, {
+                            transferred,
+                            scanned,
                             totalSourceEntries: totalSourceEstimate,
                             currentPage,
                         });
-                        const missingEntries = await sourceStore.getEntries(missingIds);
-                        if (signal?.aborted) {
+                        transferred = transferResult.transferred;
+                        if (transferResult.cancelled) {
                             return { transferred, scanned, cancelled: true };
                         }
-                        await targetStore.putEntries(missingEntries);
-                        transferred += missingEntries.length;
                     }
                 }
                 onProgress?.({
@@ -891,21 +980,28 @@ class BaseMindooDB {
         onProgress?.({
             phase: 'transferring',
             message: `Transferring ${sourceNewMetadata.length} entries...`,
-            transferredEntries: 0,
+            transferredEntries: transferred,
             scannedEntries: sourceNewMetadata.length,
             totalSourceEntries: sourceNewMetadata.length,
         });
-        const sourceEntries = await sourceStore.getEntries(sourceNewMetadata.map((m) => m.id));
-        await targetStore.putEntries(sourceEntries);
+        const transferResult = await this.transferEntriesInBatches(sourceStore, targetStore, sourceNewMetadata.map((m) => m.id), options, {
+            transferred,
+            scanned: sourceNewMetadata.length,
+            totalSourceEntries: sourceNewMetadata.length,
+        });
+        transferred = transferResult.transferred;
+        if (transferResult.cancelled) {
+            return { transferred, scanned: sourceNewMetadata.length, cancelled: true };
+        }
         onProgress?.({
             phase: 'transferring',
-            message: `Transferred ${sourceEntries.length} entries`,
-            transferredEntries: sourceEntries.length,
+            message: `Transferred ${transferred} entries`,
+            transferredEntries: transferred,
             scannedEntries: sourceNewMetadata.length,
             totalSourceEntries: sourceNewMetadata.length,
         });
         return {
-            transferred: sourceEntries.length,
+            transferred,
             scanned: sourceNewMetadata.length,
             cancelled: false,
         };
@@ -3199,10 +3295,10 @@ class BaseMindooDB {
     /**
      * Get an attachment reference by ID from a document's _attachments array.
      */
-    getAttachmentRefInternal(docId, attachmentId) {
-        const internalDoc = this.getCachedDocument(docId);
+    async getAttachmentRefInternal(docId, attachmentId) {
+        const internalDoc = this.getCachedDocument(docId) ?? await this.loadDocumentInternal(docId);
         if (!internalDoc) {
-            throw new Error(`Document ${docId} not found in cache`);
+            throw new Error(`Document ${docId} not found while reading attachment ${attachmentId}`);
         }
         const payload = internalDoc.doc;
         const attachments = payload._attachments || [];
@@ -3250,7 +3346,7 @@ class BaseMindooDB {
      */
     async getAttachmentInternal(docId, attachmentId) {
         this.logger.debug(`Getting attachment ${attachmentId} from document ${docId}`);
-        const ref = this.getAttachmentRefInternal(docId, attachmentId);
+        const ref = await this.getAttachmentRefInternal(docId, attachmentId);
         const store = this.getEffectiveAttachmentStore();
         // Resolve dependency chain to get all chunk IDs in order (oldest first)
         const chunkIds = await store.resolveDependencies(ref.lastChunkId, { includeStart: true });
@@ -3283,7 +3379,7 @@ class BaseMindooDB {
         if (startByte < 0 || endByte <= startByte) {
             throw new Error(`Invalid byte range: [${startByte}, ${endByte})`);
         }
-        const ref = this.getAttachmentRefInternal(docId, attachmentId);
+        const ref = await this.getAttachmentRefInternal(docId, attachmentId);
         const store = this.getEffectiveAttachmentStore();
         const readPlan = await this.planAttachmentRead(store, ref, startByte, endByte);
         const neededChunkIds = readPlan.chunkPlans.map((chunkPlan) => chunkPlan.id);
@@ -3315,7 +3411,7 @@ class BaseMindooDB {
      */
     async *streamAttachmentInternal(docId, attachmentId, startOffset) {
         this.logger.debug(`Streaming attachment ${attachmentId} from offset ${startOffset}`);
-        const ref = this.getAttachmentRefInternal(docId, attachmentId);
+        const ref = await this.getAttachmentRefInternal(docId, attachmentId);
         if (startOffset < 0 || startOffset > ref.size) {
             throw new Error(`Invalid stream offset ${startOffset} for attachment size ${ref.size}`);
         }
@@ -3324,20 +3420,25 @@ class BaseMindooDB {
         }
         const store = this.getEffectiveAttachmentStore();
         const readPlan = await this.planAttachmentRead(store, ref, startOffset, ref.size);
-        // Stream chunks starting from startChunkIndex
-        for (let i = 0; i < readPlan.chunkPlans.length; i++) {
-            const chunkPlan = readPlan.chunkPlans[i];
-            const [chunk] = await store.getEntries([chunkPlan.id]);
-            if (!chunk) {
-                throw new Error(`Attachment chunk ${chunkPlan.id} not found in store`);
-            }
-            const plaintext = await this.decryptAttachmentChunk(chunk);
-            // For first chunk, skip bytes before startOffset
-            if (i === 0 && readPlan.offsetInFirstChunk > 0) {
-                yield plaintext.slice(readPlan.offsetInFirstChunk);
-            }
-            else {
-                yield plaintext;
+        for (let i = 0; i < readPlan.chunkPlans.length; i += DEFAULT_ATTACHMENT_STREAM_BATCH_SIZE) {
+            const chunkPlansBatch = readPlan.chunkPlans.slice(i, i + DEFAULT_ATTACHMENT_STREAM_BATCH_SIZE);
+            const batchIds = chunkPlansBatch.map((chunkPlan) => chunkPlan.id);
+            const chunks = await store.getEntries(batchIds);
+            const chunkById = new Map(chunks.map((chunk) => [chunk.id, chunk]));
+            for (let batchIndex = 0; batchIndex < chunkPlansBatch.length; batchIndex++) {
+                const chunkPlan = chunkPlansBatch[batchIndex];
+                const chunk = chunkById.get(chunkPlan.id);
+                if (!chunk) {
+                    throw new Error(`Attachment chunk ${chunkPlan.id} not found in store. The document metadata exists, but the attachment payload may not be synced locally yet.`);
+                }
+                const plaintext = await this.decryptAttachmentChunk(chunk);
+                // For first chunk, skip bytes before startOffset
+                if (i === 0 && batchIndex === 0 && readPlan.offsetInFirstChunk > 0) {
+                    yield plaintext.slice(readPlan.offsetInFirstChunk);
+                }
+                else {
+                    yield plaintext;
+                }
             }
         }
         this.logger.debug(`Finished streaming attachment ${attachmentId}`);
@@ -3643,19 +3744,35 @@ class BaseMindooDB {
                 phase: 'processing',
                 message: `Processing ${syncResult.transferred} new entries...`,
                 transferredEntries: syncResult.transferred,
-                scannedEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
             });
             // Sync the local store to process the new entries
             // This will update the index, cache, and processedEntryIds
+            if (options?.signal?.aborted) {
+                this.logger.info(`Pull cancelled before local processing after transferring ${syncResult.transferred} entries`);
+                return {
+                    transferredEntries: syncResult.transferred,
+                    scannedEntries: syncResult.scanned,
+                    cancelled: true,
+                };
+            }
             if (storeKind === types_1.StoreKind.docs) {
                 await this.syncStoreChanges();
             }
+            if (options?.signal?.aborted) {
+                this.logger.info(`Pull cancelled after local processing for ${storeKind} entries`);
+                return {
+                    transferredEntries: syncResult.transferred,
+                    scannedEntries: syncResult.scanned,
+                    cancelled: true,
+                };
+            }
             this.logger.info(`Pull complete, synced ${syncResult.transferred} entries`);
             options?.onProgress?.({
                 phase: 'complete',
                 message: `Pull complete: ${syncResult.transferred} ${storeKind} entries synced`,
                 transferredEntries: syncResult.transferred,
-                scannedEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
             });
             return {
                 transferredEntries: syncResult.transferred,
@@ -3712,7 +3829,7 @@ class BaseMindooDB {
                 phase: 'complete',
                 message: `Push complete: ${syncResult.transferred} entries transferred`,
                 transferredEntries: syncResult.transferred,
-                scannedEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
             });
             return {
                 transferredEntries: syncResult.transferred,