mindoodb 0.0.9 → 0.0.11

package/dist/core/BaseMindooDB.js

@@ -6,6 +6,7 @@ exports.BaseMindooDB = void 0;
 // Browser/Node.js: WASM (@automerge/automerge/slim)
 const automerge_adapter_1 = require("./automerge-adapter");
 const uuid_1 = require("uuid");
+const types_1 = require("./appendonlystores/types");
 const bloom_1 = require("./appendonlystores/bloom");
 const MaterializationPlanner_1 = require("./appendonlystores/MaterializationPlanner");
 const DocumentDagAnalysis_1 = require("./DocumentDagAnalysis");
@@ -17,6 +18,7 @@ const logging_1 = require("./logging");
  * Default chunk size for attachments: 256KB
  */
 const DEFAULT_CHUNK_SIZE_BYTES = 256 * 1024;
+const DEFAULT_ATTACHMENT_STREAM_BATCH_SIZE = 4;
 const DEFAULT_SNAPSHOT_MIN_CHANGES = 100;
 const DEFAULT_SNAPSHOT_COOLDOWN_MS = 10 * 60 * 1000;
 const DEFAULT_MAX_CACHED_DOCS = 128;
@@ -64,6 +66,12 @@ class BaseMindooDB {
         this.cachePrefix = null;
         this.dirtyDocIds = new Set();
         this.cacheMetaDirty = false;
+        if (store.getStoreKind() !== types_1.StoreKind.docs) {
+            throw new Error(`[BaseMindooDB] Expected primary store kind ${types_1.StoreKind.docs} but received ${store.getStoreKind()}`);
+        }
+        if (attachmentStore.getStoreKind() !== types_1.StoreKind.attachments) {
+            throw new Error(`[BaseMindooDB] Expected attachment store kind ${types_1.StoreKind.attachments} but received ${attachmentStore.getStoreKind()}`);
+        }
         this.tenant = tenant;
         this.store = store;
         this.attachmentStore = attachmentStore;
@@ -738,6 +746,11 @@ class BaseMindooDB {
             return null;
         }
     }
+    static setSyncAbortSignalOnStore(store, signal) {
+        if ('setSyncAbortSignal' in store && typeof store.setSyncAbortSignal === 'function') {
+            store.setSyncAbortSignal(signal);
+        }
+    }
     /**
      * From a list of candidate IDs, return only those the target store is
      * missing.  Uses bloom-filter pre-screening when available, then falls
@@ -766,14 +779,126 @@ class BaseMindooDB {
         }
         return missingIds;
     }
+    /**
+     * Determine how many entry IDs to fetch per `getEntries` call during sync.
+     *
+     * This is intentionally separate from the metadata scan `pageSize` so that
+     * scanning can page through large ID lists quickly while the heavier
+     * payload downloads use a smaller batch to keep progress responsive and
+     * cancellation timely.
+     *
+     * Priority: explicit option > attachment default (100) > pageSize fallback.
+     */
+    resolveTransferBatchSize(options) {
+        if (options?.transferBatchSize && options.transferBatchSize > 0) {
+            return options.transferBatchSize;
+        }
+        if (options?.storeKind === types_1.StoreKind.attachments) {
+            return 100;
+        }
+        return options?.pageSize ?? 1000;
+    }
+    /**
+     * Transfer a set of entry IDs from source to target in fixed-size batches,
+     * emitting progress and checking for cancellation between each batch.
+     *
+     * Callers (cursor-scan path and legacy path) collect the IDs that need
+     * transferring, then delegate to this method instead of issuing one
+     * monolithic `getEntries`.  This gives three benefits:
+     *
+     * 1. The UI receives frequent progress updates with batch metadata so long
+     *    transfers no longer look frozen.
+     * 2. Cancellation can interrupt work between batches rather than waiting
+     *    for one large HTTP response to complete.
+     * 3. Each server-side `getEntries` + RSA encryption unit is smaller,
+     *    reducing the risk of socket timeouts on large payloads.
+     *
+     * Returns partial progress on cancellation so callers can report how much
+     * was actually transferred before the abort.
+     */
+    async transferEntriesInBatches(sourceStore, targetStore, entryIds, options, state) {
+        if (entryIds.length === 0) {
+            return { transferred: state.transferred, cancelled: false };
+        }
+        const onProgress = options?.onProgress;
+        const signal = options?.signal;
+        const transferBatchSize = this.resolveTransferBatchSize(options);
+        const totalTransferBatches = Math.max(1, Math.ceil(entryIds.length / transferBatchSize));
+        let transferred = state.transferred;
+        for (let offset = 0; offset < entryIds.length; offset += transferBatchSize) {
+            if (signal?.aborted) {
+                return { transferred, cancelled: true };
+            }
+            const currentTransferBatch = Math.floor(offset / transferBatchSize) + 1;
+            const batchIds = entryIds.slice(offset, offset + transferBatchSize);
+            const pageSummary = state.currentPage ? `page ${state.currentPage}, ` : "";
+            onProgress?.({
+                phase: "transferring",
+                message: `Transferring batch ${currentTransferBatch}/${totalTransferBatches} (${batchIds.length} entries, ${pageSummary}scanned ${state.scanned})...`,
+                transferredEntries: transferred,
+                scannedEntries: state.scanned,
+                totalSourceEntries: state.totalSourceEntries,
+                currentPage: state.currentPage,
+                currentTransferBatch,
+                totalTransferBatches,
+                transferBatchSize,
+            });
+            try {
+                const batchEntries = await sourceStore.getEntries(batchIds);
+                if (signal?.aborted) {
+                    return { transferred, cancelled: true };
+                }
+                await targetStore.putEntries(batchEntries);
+                transferred += batchEntries.length;
+            }
+            catch (error) {
+                if (signal?.aborted) {
+                    return { transferred, cancelled: true };
+                }
+                throw error;
+            }
+            onProgress?.({
+                phase: "transferring",
+                message: `Transferred ${transferred} entries after batch ${currentTransferBatch}/${totalTransferBatches}`,
+                transferredEntries: transferred,
+                scannedEntries: state.scanned,
+                totalSourceEntries: state.totalSourceEntries,
+                currentPage: state.currentPage,
+                currentTransferBatch,
+                totalTransferBatches,
+                transferBatchSize,
+            });
+        }
+        return { transferred, cancelled: false };
+    }
     async syncEntriesFromStore(sourceStore, targetStore, options) {
-        if (options?.mode === "dense") {
-            return this.syncEntriesFromStoreDense(sourceStore, targetStore, options);
+        const signal = options?.signal;
+        BaseMindooDB.setSyncAbortSignalOnStore(sourceStore, signal);
+        BaseMindooDB.setSyncAbortSignalOnStore(targetStore, signal);
+        try {
+            if (options?.mode === "dense") {
+                return await this.syncEntriesFromStoreDense(sourceStore, targetStore, options);
+            }
+            return await this.syncEntriesFromStoreImpl(sourceStore, targetStore, options);
+        }
+        catch (error) {
+            if (signal?.aborted) {
+                this.logger.info("Sync cancelled by abort signal");
+                return { transferred: 0, scanned: 0, cancelled: true };
+            }
+            throw error;
+        }
+        finally {
+            BaseMindooDB.setSyncAbortSignalOnStore(sourceStore, undefined);
+            BaseMindooDB.setSyncAbortSignalOnStore(targetStore, undefined);
         }
+    }
+    async syncEntriesFromStoreImpl(sourceStore, targetStore, options) {
         let transferred = 0;
         let scanned = 0;
         const onProgress = options?.onProgress;
         const pageSize = options?.pageSize ?? 1000;
+        const signal = options?.signal;
         const targetBloom = await this.getTargetBloomSummary(targetStore);
         const totalSourceEstimate = targetBloom?.totalIds;
         if (this.supportsCursorScan(sourceStore)) {
@@ -787,19 +912,40 @@ class BaseMindooDB {
             let cursor = null;
             let currentPage = 0;
             while (true) {
-                if (options?.signal?.aborted) {
+                if (signal?.aborted) {
                     return { transferred, scanned, cancelled: true };
                 }
                 const page = await sourceStore.scanEntriesSince(cursor, pageSize);
                 currentPage++;
                 scanned += page.entries.length;
+                if (signal?.aborted) {
+                    return { transferred, scanned, cancelled: true };
+                }
                 if (page.entries.length > 0) {
+                    onProgress?.({
+                        phase: 'transferring',
+                        message: `Scanned ${scanned} entries, checking for changes (page ${currentPage})...`,
+                        transferredEntries: transferred,
+                        scannedEntries: scanned,
+                        totalSourceEntries: totalSourceEstimate,
+                        currentPage,
+                    });
                     const ids = page.entries.map((m) => m.id);
                     const missingIds = await this.filterMissingIds(targetStore, ids, targetBloom);
+                    if (signal?.aborted) {
+                        return { transferred, scanned, cancelled: true };
+                    }
                     if (missingIds.length > 0) {
-                        const missingEntries = await sourceStore.getEntries(missingIds);
-                        await targetStore.putEntries(missingEntries);
-                        transferred += missingEntries.length;
+                        const transferResult = await this.transferEntriesInBatches(sourceStore, targetStore, missingIds, options, {
+                            transferred,
+                            scanned,
+                            totalSourceEntries: totalSourceEstimate,
+                            currentPage,
+                        });
+                        transferred = transferResult.transferred;
+                        if (transferResult.cancelled) {
+                            return { transferred, scanned, cancelled: true };
+                        }
                     }
                 }
                 onProgress?.({
@@ -834,21 +980,28 @@ class BaseMindooDB {
         onProgress?.({
             phase: 'transferring',
             message: `Transferring ${sourceNewMetadata.length} entries...`,
-            transferredEntries: 0,
+            transferredEntries: transferred,
             scannedEntries: sourceNewMetadata.length,
             totalSourceEntries: sourceNewMetadata.length,
         });
-        const sourceEntries = await sourceStore.getEntries(sourceNewMetadata.map((m) => m.id));
-        await targetStore.putEntries(sourceEntries);
+        const transferResult = await this.transferEntriesInBatches(sourceStore, targetStore, sourceNewMetadata.map((m) => m.id), options, {
+            transferred,
+            scanned: sourceNewMetadata.length,
+            totalSourceEntries: sourceNewMetadata.length,
+        });
+        transferred = transferResult.transferred;
+        if (transferResult.cancelled) {
+            return { transferred, scanned: sourceNewMetadata.length, cancelled: true };
+        }
         onProgress?.({
             phase: 'transferring',
-            message: `Transferred ${sourceEntries.length} entries`,
-            transferredEntries: sourceEntries.length,
+            message: `Transferred ${transferred} entries`,
+            transferredEntries: transferred,
             scannedEntries: sourceNewMetadata.length,
             totalSourceEntries: sourceNewMetadata.length,
         });
         return {
-            transferred: sourceEntries.length,
+            transferred,
             scanned: sourceNewMetadata.length,
             cancelled: false,
         };
@@ -3142,10 +3295,10 @@ class BaseMindooDB {
     /**
      * Get an attachment reference by ID from a document's _attachments array.
      */
-    getAttachmentRefInternal(docId, attachmentId) {
-        const internalDoc = this.getCachedDocument(docId);
+    async getAttachmentRefInternal(docId, attachmentId) {
+        const internalDoc = this.getCachedDocument(docId) ?? await this.loadDocumentInternal(docId);
         if (!internalDoc) {
-            throw new Error(`Document ${docId} not found in cache`);
+            throw new Error(`Document ${docId} not found while reading attachment ${attachmentId}`);
         }
         const payload = internalDoc.doc;
         const attachments = payload._attachments || [];
@@ -3193,7 +3346,7 @@ class BaseMindooDB {
      */
     async getAttachmentInternal(docId, attachmentId) {
         this.logger.debug(`Getting attachment ${attachmentId} from document ${docId}`);
-        const ref = this.getAttachmentRefInternal(docId, attachmentId);
+        const ref = await this.getAttachmentRefInternal(docId, attachmentId);
         const store = this.getEffectiveAttachmentStore();
         // Resolve dependency chain to get all chunk IDs in order (oldest first)
         const chunkIds = await store.resolveDependencies(ref.lastChunkId, { includeStart: true });
@@ -3226,7 +3379,7 @@ class BaseMindooDB {
         if (startByte < 0 || endByte <= startByte) {
             throw new Error(`Invalid byte range: [${startByte}, ${endByte})`);
         }
-        const ref = this.getAttachmentRefInternal(docId, attachmentId);
+        const ref = await this.getAttachmentRefInternal(docId, attachmentId);
         const store = this.getEffectiveAttachmentStore();
         const readPlan = await this.planAttachmentRead(store, ref, startByte, endByte);
         const neededChunkIds = readPlan.chunkPlans.map((chunkPlan) => chunkPlan.id);
@@ -3258,7 +3411,7 @@ class BaseMindooDB {
      */
     async *streamAttachmentInternal(docId, attachmentId, startOffset) {
         this.logger.debug(`Streaming attachment ${attachmentId} from offset ${startOffset}`);
-        const ref = this.getAttachmentRefInternal(docId, attachmentId);
+        const ref = await this.getAttachmentRefInternal(docId, attachmentId);
         if (startOffset < 0 || startOffset > ref.size) {
             throw new Error(`Invalid stream offset ${startOffset} for attachment size ${ref.size}`);
         }
@@ -3267,29 +3420,34 @@ class BaseMindooDB {
         }
         const store = this.getEffectiveAttachmentStore();
         const readPlan = await this.planAttachmentRead(store, ref, startOffset, ref.size);
-        // Stream chunks starting from startChunkIndex
-        for (let i = 0; i < readPlan.chunkPlans.length; i++) {
-            const chunkPlan = readPlan.chunkPlans[i];
-            const [chunk] = await store.getEntries([chunkPlan.id]);
-            if (!chunk) {
-                throw new Error(`Attachment chunk ${chunkPlan.id} not found in store`);
-            }
-            const plaintext = await this.decryptAttachmentChunk(chunk);
-            // For first chunk, skip bytes before startOffset
-            if (i === 0 && readPlan.offsetInFirstChunk > 0) {
-                yield plaintext.slice(readPlan.offsetInFirstChunk);
-            }
-            else {
-                yield plaintext;
+        for (let i = 0; i < readPlan.chunkPlans.length; i += DEFAULT_ATTACHMENT_STREAM_BATCH_SIZE) {
+            const chunkPlansBatch = readPlan.chunkPlans.slice(i, i + DEFAULT_ATTACHMENT_STREAM_BATCH_SIZE);
+            const batchIds = chunkPlansBatch.map((chunkPlan) => chunkPlan.id);
+            const chunks = await store.getEntries(batchIds);
+            const chunkById = new Map(chunks.map((chunk) => [chunk.id, chunk]));
+            for (let batchIndex = 0; batchIndex < chunkPlansBatch.length; batchIndex++) {
+                const chunkPlan = chunkPlansBatch[batchIndex];
+                const chunk = chunkById.get(chunkPlan.id);
+                if (!chunk) {
+                    throw new Error(`Attachment chunk ${chunkPlan.id} not found in store. The document metadata exists, but the attachment payload may not be synced locally yet.`);
+                }
+                const plaintext = await this.decryptAttachmentChunk(chunk);
+                // For first chunk, skip bytes before startOffset
+                if (i === 0 && batchIndex === 0 && readPlan.offsetInFirstChunk > 0) {
+                    yield plaintext.slice(readPlan.offsetInFirstChunk);
+                }
+                else {
+                    yield plaintext;
+                }
             }
         }
         this.logger.debug(`Finished streaming attachment ${attachmentId}`);
     }
     /**
-     * Get the effective attachment store (attachment store if configured, otherwise doc store).
+     * Get the attachment store for this database.
      */
     getEffectiveAttachmentStore() {
-        return this.attachmentStore || this.store;
+        return this.attachmentStore;
     }
     /**
      * Internal method to add an attachment by chunking the file and storing chunks.
@@ -3484,9 +3642,14 @@ class BaseMindooDB {
      * Resolve a sync target to a ContentAddressedStore.
      * Accepts either a raw store or a MindooDB instance (calls getStore()).
      */
-    resolveStore(remote) {
+    getStoreForKind(storeKind = types_1.StoreKind.docs) {
+        return storeKind === types_1.StoreKind.attachments ? this.attachmentStore : this.store;
+    }
+    resolveStore(remote, storeKind = types_1.StoreKind.docs) {
         if ('getStore' in remote && typeof remote.getStore === 'function') {
-            return remote.getStore();
+            return storeKind === types_1.StoreKind.attachments
+                ? remote.getAttachmentStore()
+                : remote.getStore();
         }
         return remote;
     }
@@ -3546,19 +3709,24 @@ class BaseMindooDB {
      * 3. Stores them in our local store
      * 4. Syncs the local store to process the new entries
      *
+     * The optional `storeKind` sync option selects which store is synced.
+     * By default, this method syncs the docs store only.
+     *
      * @param remote The remote store or MindooDB instance to pull entries from
-     * @param options Optional sync options for progress tracking, paging, and cancellation
+     * @param options Optional sync options for progress tracking, paging, cancellation, and store selection
      * @return A promise that resolves with the sync result
      */
     async pullChangesFrom(remote, options) {
-        const remoteStore = this.resolveStore(remote);
-        if (this.store.getId() !== remoteStore.getId()) {
-            throw new Error(`[BaseMindooDB] Cannot pull entries from the incompatible store ${this.store.getId()}`);
+        const storeKind = options?.storeKind ?? types_1.StoreKind.docs;
+        const localStore = this.getStoreForKind(storeKind);
+        const remoteStore = this.resolveStore(remote, storeKind);
+        if (localStore.getId() !== remoteStore.getId() || localStore.getStoreKind() !== remoteStore.getStoreKind()) {
+            throw new Error(`[BaseMindooDB] Cannot pull entries from the incompatible store ${localStore.getId()}/${localStore.getStoreKind()}`);
         }
-        this.logger.info(`Pulling entries from remote store ${remoteStore.getId()}`);
+        this.logger.info(`Pulling entries from remote store ${remoteStore.getId()}/${remoteStore.getStoreKind()}`);
         const restoreAuthOverride = await this.applyNetworkAuthOverrideForSync(remoteStore, options);
         try {
-            const syncResult = await this.syncEntriesFromStore(remoteStore, this.store, options);
+            const syncResult = await this.syncEntriesFromStore(remoteStore, localStore, options);
             this.logger.debug(`Transferred ${syncResult.transferred} entries from remote store`);
             if (syncResult.cancelled) {
                 this.logger.info(`Pull cancelled after transferring ${syncResult.transferred} entries`);
@@ -3576,17 +3744,35 @@ class BaseMindooDB {
                 phase: 'processing',
                 message: `Processing ${syncResult.transferred} new entries...`,
                 transferredEntries: syncResult.transferred,
-                scannedEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
             });
             // Sync the local store to process the new entries
             // This will update the index, cache, and processedEntryIds
-            await this.syncStoreChanges();
+            if (options?.signal?.aborted) {
+                this.logger.info(`Pull cancelled before local processing after transferring ${syncResult.transferred} entries`);
+                return {
+                    transferredEntries: syncResult.transferred,
+                    scannedEntries: syncResult.scanned,
+                    cancelled: true,
+                };
+            }
+            if (storeKind === types_1.StoreKind.docs) {
+                await this.syncStoreChanges();
+            }
+            if (options?.signal?.aborted) {
+                this.logger.info(`Pull cancelled after local processing for ${storeKind} entries`);
+                return {
+                    transferredEntries: syncResult.transferred,
+                    scannedEntries: syncResult.scanned,
+                    cancelled: true,
+                };
+            }
             this.logger.info(`Pull complete, synced ${syncResult.transferred} entries`);
             options?.onProgress?.({
                 phase: 'complete',
-                message: `Pull complete: ${syncResult.transferred} entries synced`,
+                message: `Pull complete: ${syncResult.transferred} ${storeKind} entries synced`,
                 transferredEntries: syncResult.transferred,
-                scannedEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
             });
             return {
                 transferredEntries: syncResult.transferred,
@@ -3606,19 +3792,24 @@ class BaseMindooDB {
      * 2. Retrieves those entries from our local store
      * 3. Stores them in the remote store
      *
+     * The optional `storeKind` sync option selects which store is synced.
+     * By default, this method syncs the docs store only.
+     *
      * @param remote The remote store or MindooDB instance to push entries to
-     * @param options Optional sync options for progress tracking, paging, and cancellation
+     * @param options Optional sync options for progress tracking, paging, cancellation, and store selection
      * @return A promise that resolves with the sync result
      */
     async pushChangesTo(remote, options) {
-        const remoteStore = this.resolveStore(remote);
-        if (this.store.getId() !== remoteStore.getId()) {
-            throw new Error(`[BaseMindooDB] Cannot push entries to the incompatible store ${this.store.getId()}`);
+        const storeKind = options?.storeKind ?? types_1.StoreKind.docs;
+        const localStore = this.getStoreForKind(storeKind);
+        const remoteStore = this.resolveStore(remote, storeKind);
+        if (localStore.getId() !== remoteStore.getId() || localStore.getStoreKind() !== remoteStore.getStoreKind()) {
+            throw new Error(`[BaseMindooDB] Cannot push entries to the incompatible store ${localStore.getId()}/${localStore.getStoreKind()}`);
         }
-        this.logger.info(`Pushing entries to remote store ${remoteStore.getId()}`);
+        this.logger.info(`Pushing entries to remote store ${remoteStore.getId()}/${remoteStore.getStoreKind()}`);
         const restoreAuthOverride = await this.applyNetworkAuthOverrideForSync(remoteStore, options);
         try {
-            const syncResult = await this.syncEntriesFromStore(this.store, remoteStore, options);
+            const syncResult = await this.syncEntriesFromStore(localStore, remoteStore, options);
             this.logger.debug(`Transferred ${syncResult.transferred} entries to remote store`);
             if (syncResult.cancelled) {
                 this.logger.info(`Push cancelled after transferring ${syncResult.transferred} entries`);
@@ -3638,7 +3829,7 @@ class BaseMindooDB {
                 phase: 'complete',
                 message: `Push complete: ${syncResult.transferred} entries transferred`,
                 transferredEntries: syncResult.transferred,
-                scannedEntries: syncResult.transferred,
+                scannedEntries: syncResult.scanned,
             });
             return {
                 transferredEntries: syncResult.transferred,