@spirobel/monero-wallet-api 0.5.3 → 0.5.5

package/dist/scanning-syncing/scanresult/scanCoordination.js

@@ -1,6 +1,6 @@
-import { setupBlocksBufferGenerator, ViewPair, handleConnectionStatusChanges, processScanResult, writeCacheToFile, sleep, findTipIndex, readWriteConnectionStatusFile, connectionStatusFilePath, } from "../../api";
-import { markWorkItemAsDone, } from "./scanLoop";
-import { makeWorkItem, scanLoop } from "./scanLoop";
+import { setupBlocksBufferGenerator, ViewPair, handleConnectionStatusChanges, processScanResult, writeCacheToFile, sleep, findTipIndex, readWriteConnectionStatusFile, applyWalletScanProgress, } from "../../api";
+import {} from "./scanLoop";
+import { makeWorkItem } from "./scanLoop";
 import { cullTooLargeScanHeight, getNonHaltedWallets, getPathPrefix, openScanSettingsFile, SCAN_SETTINGS_STORE_NAME_DEFAULT, walletSettingsPlusKeys, } from "../../api";
 import { findRange, initScanCacheFile, lastRange, makeCacheRangeForHeight, mergeRanges, } from "./scanCache";
 import { sendToCpuWorker } from "../worker-mains/cpubound-main";
@@ -166,19 +166,6 @@ export function reconcileWorkItemDone(blocksBuffer, workItemBuffer) {
         }
     }
 }
-/**
- * one round of the select race. takes two primed promises (blocks, scan),
- * races them, returns winner info. the loser stays in flight.
- */
-export async function raceStep(blocksPromise, scanPromise) {
-    const winner = await Promise.race([
-        blocksPromise.then((v) => ({ src: "blocks", value: v })),
-        scanPromise.then((v) => ({ src: "scan", value: v })),
-    ]);
-    const value = winner.value;
-    const done = !!winner.value.done;
-    return { src: winner.src, value, done };
-}
 /**
  * handle a yield from the blocks buffer fetch loop.
  */
@@ -189,45 +176,6 @@ export async function handleBlocksYield(value, scanSettingsPath) {
     await handleConnectionStatusChanges(value, scanSettingsPath);
     return { isBlocksBufferChanged: false };
 }
-/**
- * process a scan result for a completed work item.
- * updates the cache, writes to disk, marks work item done, reconciles.
- */
-export async function processScanResultForWorkItem(value, workBuffer, blocksBuffer, pathPrefix, secret_spend_key) {
-    if (value.type !== "Ready" || !value.result || !value.work_uuid)
-        return;
-    const item = await markWorkItemAsDone(value, workBuffer);
-    log("processScanResultForWorkItem", `item_work_uuid=${item?.work_uuid}`);
-    if (!item || item.status !== "scanwork_done")
-        throw new Error("[processScanResultForWorkItem] item not found or not scanwork not done. item_status=" +
-            item?.status);
-    const firstBlock = item.batch.get_blocks_result_meta.block_infos[item.from];
-    log("processScanResultForWorkItem", [
-        `block_infos.length=${item.batch.get_blocks_result_meta.block_infos.length} firstBlock=${firstBlock?.block_height} firstBlockHash=${firstBlock?.block_hash?.slice(0, 8)}`,
-    ]);
-    log("processScanResultForWorkItem", [
-        `block_infos.length=${item.batch.get_blocks_result_meta.block_infos.length} from=${item.from} to=${item.to}`,
-    ]);
-    const lastBlock = item.batch.get_blocks_result_meta.block_infos[item.to];
-    const cache = item.walletConfig.cache;
-    await processScanResult({
-        from_height: firstBlock.block_height,
-        to_height: lastBlock.block_height,
-        result: value.result,
-        scanCache: cache,
-        secret_spend_key,
-    });
-    await writeCacheToFile(cache, pathPrefix);
-    item.status = "process_result_done";
-    reconcileWorkItemDone(blocksBuffer, workBuffer);
-    //because cache is tied to the workitems by reference,
-    // the following workitems will have the most recent cache
-    //TODO: if we do cpu workers we need to be sure to wait with the processing at the top of this functon
-    // until all workitems for this wallet before it (to the left of it) are done
-    //  the cpu worker loop generator will have to ensure the order of this
-    // currently as a sideeffect of the work scheduling function on fetch result aka blocks buffer changed,
-    // this is already implicitly handled
-}
 /**
  * process a scan result for a completed work item.
  * updates the cache, writes to disk, marks work item done, reconciles blockbuffer with this.
@@ -272,6 +220,9 @@ export async function processWorkItem(item, workBuffer, blocksBuffer, pathPrefix
         throw error;
     }
     await writeCacheToFile(cache, pathPrefix);
+    // raw cpu ScanResult is fully merged into cache now. drop it so workBuffer
+    // does not pin all_key_images / hash strings until left-edge free.
+    item.result = undefined;
     item.status = "process_result_done";
     log("processWorkItem", `process_result_done work_uuid=${item?.work_uuid}`);
     // remove from blocksbuffer if no more work items reference it
@@ -323,132 +274,7 @@ function computeETA(cache, totalBlocksScanned, scanStartTime) {
     const etaMs = remaining / blocksPerMs;
     return msToHHMM(etaMs);
 }
-/**
- * coordinator main: async generator that drives the full scan cycle.
- * takes only scanSettingsPath, derives everything else internally.
- * yields events for some of the underlying generator events,
- * look at CoordinatorEvent, processing not decoupled from cpu bound scan work,
- * as this single threaded, there is no point
- */
-export async function* coordinatorMain(scanSettingsPath, pathPrefix) {
-    const ctx = await setupCoordinator(scanSettingsPath, pathPrefix);
-    if (!ctx)
-        throw new Error("[coordinatorMain] findWorkToBeDone returned false");
-    const work_to_be_done = ctx.work_to_be_done;
-    const blocksBuffer = ctx.blocksBuffer;
-    const workBuffer = ctx.workBuffer;
-    const blocksGenerator = ctx.blocksGenerator;
-    let totalBlocksScanned = 0;
-    let scanStartTime = Date.now();
-    let blocksPromise = blocksGenerator.next();
-    const scanGens = new Map();
-    const scanPromises = new Map();
-    for (const wc of work_to_be_done.wallet_configs) {
-        scanGens.set(wc.primary_address, scanLoop({
-            primary_address: wc.primary_address,
-            secret_view_key: wc.secret_view_key,
-            secret_spend_key: wc.secret_spend_key,
-            subaddress_index: wc.subaddress_index,
-        }));
-    }
-    // prime all scan generators
-    // ( technically no need to do this as loop + scan generator + workbuffer structure mean it will happen automatically)
-    for (const [, gen] of scanGens) {
-        await gen.next();
-    }
-    while (true) {
-        const races = [blocksPromise.then((v) => ({ src: "blocks", value: v }))];
-        for (const [addr, p] of scanPromises) {
-            races.push(p.then((v) => ({ src: "scan", addr, value: v })));
-        }
-        const winner = await Promise.race(races);
-        log("coordinatorMain", [
-            "winner=" + winner.src + " addr=" + String(winner.addr ?? "").slice(0, 8),
-        ]);
-        if (winner.src === "blocks") {
-            const result = winner.value.value;
-            const { isBlocksBufferChanged } = await handleBlocksYield(result, scanSettingsPath);
-            if (isBlocksBufferChanged) {
-                makeWorkItemsForAllWallets(work_to_be_done.wallet_configs, blocksBuffer, workBuffer);
-                // start scan promises for wallets that now have work
-                for (const wc of work_to_be_done.wallet_configs) {
-                    const addr = wc.primary_address;
-                    if (scanPromises.has(addr))
-                        continue;
-                    const gen = scanGens.get(addr);
-                    log("coordinatorMain", ["scanGens.get(addr)", gen]);
-                    if (!gen)
-                        continue;
-                    const item = workBuffer.find((x) => x.status === "fresh" && x.walletConfig.primary_address === addr);
-                    if (item) {
-                        item.status = "scanwork_in_progress";
-                        scanPromises.set(addr, gen.next(item));
-                    }
-                }
-                logBufStatus(blocksBuffer, workBuffer, scanPromises, "after_blocks");
-                yield { type: "blocks_buffer_changed" };
-            }
-            else {
-                logBufStatus(blocksBuffer, workBuffer, scanPromises, "fetch_conn");
-                yield { type: "connection_status", status: result };
-            }
-            blocksPromise = blocksGenerator.next();
-        }
-        else {
-            const addr = winner.addr;
-            const gen = scanGens.get(addr);
-            const value = winner.value.value;
-            if (value.type === "InProgress") {
-                scanPromises.set(addr, gen.next());
-            }
-            else if (value.type === "Ready") {
-                const wc = work_to_be_done.wallet_configs.find((x) => x.primary_address === addr);
-                if (!wc)
-                    throw new Error("[coordinatorMain] wallet config not found on process result");
-                // count blocks scanned in this work item for ETA
-                const doneItem = workBuffer.find((x) => x.work_uuid === value.work_uuid);
-                const blockCount = doneItem ? doneItem.to - doneItem.from + 1 : 0;
-                await processScanResultForWorkItem(value, workBuffer, blocksBuffer, getPathPrefix(scanSettingsPath, pathPrefix), wc?.secret_spend_key);
-                totalBlocksScanned += blockCount;
-                const eta = computeETA(wc.cache, totalBlocksScanned, scanStartTime);
-                // write connection status sync (includes eta)
-                if (eta) {
-                    await readWriteConnectionStatusFile((cs) => {
-                        cs.sync = {
-                            ...cs.sync,
-                            scanned_ranges: wc.cache.scanned_ranges,
-                            daemon_height: wc.cache.daemon_height,
-                            current_scan_height: lastRange(wc.cache.scanned_ranges)?.end || 0,
-                            eta,
-                            timestamp: new Date().toISOString(),
-                        };
-                    }, scanSettingsPath);
-                }
-                const nextItem = workBuffer.find((x) => x.status !== "process_result_done" &&
-                    x.walletConfig.primary_address === addr);
-                if (nextItem) {
-                    scanPromises.set(addr, gen.next(nextItem));
-                }
-                else {
-                    scanPromises.delete(addr);
-                }
-                logBufStatus(blocksBuffer, workBuffer, scanPromises, "after_scan");
-                yield {
-                    type: "scan_ready",
-                    address: addr,
-                    result: value,
-                    newCache: wc.cache,
-                    changed_outputs: [],
-                };
-                // signal idle when no scans active and nothing buffered
-                if (scanPromises.size === 0 && blocksBuffer.length === 0) {
-                    yield { type: "all_idle" };
-                }
-            }
-        }
-    }
-}
-export async function setupCoordinator(scanSettingsPath, pathPrefix) {
+export async function setupCoordinator(scanSettingsPath, pathPrefix, stopSync) {
     const work_to_be_done = await findWorkToBeDone(scanSettingsPath, pathPrefix);
     if (!work_to_be_done)
         return false;
@@ -457,6 +283,7 @@ export async function setupCoordinator(scanSettingsPath, pathPrefix) {
         startHeight: work_to_be_done.start_height,
         anchor_range: work_to_be_done.anchor_range,
         scanSettingsPath,
+        stopSync,
     });
     const workBuffer = [];
     //TODO: refactor to per wallet buffer
@@ -475,21 +302,46 @@ export async function setupCoordinator(scanSettingsPath, pathPrefix) {
         work_to_be_done,
     };
 }
+// flip inflight work back to fresh so a restart can reschedule it
+export function resetInProgressWorkItems(workBuffer) {
+    for (const item of workBuffer) {
+        if (item.status === "scanwork_in_progress") {
+            item.status = "fresh";
+            item.result = undefined;
+        }
+    }
+}
+// tell busy cpu ports to cancel and wait for their promises (with timeout)
+export async function cancelBusyCpuPorts(ports, timeoutMs = 3000) {
+    const busy = ports.filter((ps) => ps.promise);
+    for (const ps of busy) {
+        try {
+            sendToCpuWorker(ps.port, "cancel");
+        }
+        catch (err) {
+            log("cancelBusyCpuPorts", ["send cancel failed", String(err)]);
+        }
+    }
+    if (busy.length === 0)
+        return;
+    await Promise.race([
+        Promise.allSettled(busy.map((ps) => ps.promise)),
+        sleep(timeoutMs),
+    ]);
+    for (const ps of ports) {
+        ps.promise = null;
+    }
+}
 /**
  * coordinator main (multithreaded): dispatches scan work to CPU workers
  * via MessagePorts, processes results in order per wallet.
  * falls back to single-threaded coordinator if no cpuPorts provided.
  */
-export async function* coordinatorMainMultithreaded(scanSettingsPath, pathPrefix, cpuPorts) {
+export async function* coordinatorMainMultithreaded(scanSettingsPath, pathPrefix, cpuPorts, stopSync) {
     if (!cpuPorts || cpuPorts.length === 0) {
-        // throw new Error(
-        //   "[coordinatorMain Multithreaded] cpuPorts empty, there must be at least cpu worker",
-        // );
-        log("coordinatorMainMultithreaded", "fallback to single-threaded");
-        yield* coordinatorMain(scanSettingsPath, pathPrefix);
-        return;
+        throw new Error("[coordinatorMain Multithreaded] cpuPorts empty, there must be at least cpu worker");
     }
-    const ctx = await setupCoordinator(scanSettingsPath, pathPrefix);
+    const ctx = await setupCoordinator(scanSettingsPath, pathPrefix, stopSync);
     if (!ctx)
         throw new Error("[coordinatorMain Multithreaded] findWorkToBeDone returned false");
     const work_to_be_done = ctx.work_to_be_done;
@@ -509,6 +361,13 @@ export async function* coordinatorMainMultithreaded(scanSettingsPath, pathPrefix
     }
     let race_count = 0;
     while (true) {
+        if (stopSync?.aborted) {
+            log("coordinatorMainMultithreaded", ["shutdown signal, draining"]);
+            await cancelBusyCpuPorts(freePorts);
+            resetInProgressWorkItems(workBuffer);
+            yield { type: "shutdown_done" };
+            return;
+        }
         race_count++;
         log("coordinatorMainMultithreaded", ["race_count", race_count]);
         await scheduleWorkOnCpuPorts(freePorts, workBuffer);
@@ -526,6 +385,16 @@ export async function* coordinatorMainMultithreaded(scanSettingsPath, pathPrefix
         const winner = await Promise.race(races);
         log("coordinatorMainMultithreaded", "cpu ports status");
         if ("src" in winner && winner.src === "blocks") {
+            // fetch loop returned (done) on abort, treat as shutdown
+            if (winner.value.done) {
+                log("coordinatorMainMultithreaded", [
+                    "blocks generator done, draining",
+                ]);
+                await cancelBusyCpuPorts(freePorts);
+                resetInProgressWorkItems(workBuffer);
+                yield { type: "shutdown_done" };
+                return;
+            }
             const result = winner.value.value;
             const { isBlocksBufferChanged } = await handleBlocksYield(result, scanSettingsPath);
             if (isBlocksBufferChanged) {
@@ -558,30 +427,22 @@ export async function* coordinatorMainMultithreaded(scanSettingsPath, pathPrefix
                 continue;
             const blockCount = to_be_processed.to - to_be_processed.from + 1;
             totalBlocksScanned += blockCount;
-            const eta = computeETA(wallet.cache, totalBlocksScanned, scanStartTime);
-            // write connection status sync (includes eta)
-            if (eta) {
-                await readWriteConnectionStatusFile((cs) => {
-                    cs.sync = {
-                        ...cs.sync,
-                        scanned_ranges: wallet.cache.scanned_ranges,
-                        daemon_height: wallet.cache.daemon_height,
-                        current_scan_height: lastRange(wallet.cache.scanned_ranges)?.end || 0,
-                        eta,
-                        timestamp: new Date().toISOString(),
-                    };
-                }, scanSettingsPath);
-            }
             const res = await processWorkItem(to_be_processed, workBuffer, blocksBuffer, getPathPrefix(scanSettingsPath, pathPrefix), wallet.secret_spend_key);
+            // always persist wallet progress after process; eta only if we have a new one
+            // so missing eta does not wipe the previous value (no flicker)
+            const eta = computeETA(wallet.cache, totalBlocksScanned, scanStartTime);
+            await readWriteConnectionStatusFile((cs) => {
+                applyWalletScanProgress(cs, {
+                    current_scan_height: lastRange(wallet.cache.scanned_ranges)?.end || 0,
+                    scanned_ranges: wallet.cache.scanned_ranges,
+                    daemon_height: wallet.cache.daemon_height,
+                    eta,
+                });
+            }, scanSettingsPath);
             //  log("coordinatorMainMultithreaded", "processWorkItem result");
             yield {
                 type: "scan_ready",
                 address: wallet.primary_address,
-                result: {
-                    type: "Ready",
-                    work_uuid: to_be_processed.work_uuid,
-                    result: to_be_processed.result,
-                },
                 newCache: wallet.cache,
                 changed_outputs: res.changed_outputs,
             };
@@ -606,6 +467,17 @@ export async function scheduleWorkOnCpuPorts(ports, work_buffer) {
             const msg = event.data;
             // handle the result here:
             log("scheduleWorkOnCpuPorts", ["onmessage result", msg]);
+            // cancel ack may omit work_uuid when idle, but inflight cancel carries it
+            if (msg.type === "Canceled") {
+                if (item.status === "scanwork_in_progress") {
+                    item.status = "fresh";
+                    item.result = undefined;
+                }
+                port_status.promise = null;
+                // thin resolve: dont put full msg/result on the port promise on cancel
+                resolve_port({ type: "Canceled", work_uuid: item.work_uuid });
+                return;
+            }
             if (msg.work_uuid !== item.work_uuid) {
                 log("scheduleWorkOnCpuPorts", [
                     "wrong work_uuid in msg msg.work_uuid=",
@@ -623,10 +495,18 @@ export async function scheduleWorkOnCpuPorts(ports, work_buffer) {
                 log("scheduleWorkOnCpuPorts", "wrong status in msg");
                 throw new Error("[scheduleWorkOnCpuPorts] wrong status in msg");
             }
+            // fat ScanResult lives only on the work item until process clears it
             item.result = msg.result;
             item.status = "scanwork_done";
+            // cpu already has its own buffer copy; process only needs meta.
+            // drop work-item bytes now; blocksBuffer still owns the batch until left-edge free.
+            item.batch = {
+                ...item.batch,
+                data: new Uint8Array(0),
+            };
             port_status.promise = null;
-            resolve_port(msg);
+            // thin resolve: race/await must not retain all_key_images via promise value
+            resolve_port({ type: msg.type, work_uuid: msg.work_uuid });
         };
         port_status.port.onmessage = onmessage;
         item.status = "scanwork_in_progress";

package/dist/scanning-syncing/scanresult/scanLoop.d.ts

@@ -12,9 +12,8 @@ export declare function makeWorkItem(walletConfig: WalletConfigPlusCache, batch:
 export type ScanLoopIteratorResult = IteratorYieldResult<ScanLoopYield>;
 export type ScanLoopInput = WorkItem | "cancel" | undefined;
 export type ScanLoopYield = {
-    type: "Ready" | "InProgress";
+    type: "Ready" | "InProgress" | "Canceled";
     work_uuid?: string;
     result?: ScanResult;
 };
 export declare function scanLoop(wallet: WalletConfig): AsyncGenerator<ScanLoopYield, void, ScanLoopInput>;
-export declare function markWorkItemAsDone(loop_event: ScanLoopYield, work_buffer: WorkItem[]): Promise<WorkItem | undefined>;

package/dist/scanning-syncing/scanresult/scanLoop.js

@@ -101,16 +101,3 @@ export async function* scanLoop(wallet) {
         // will be done by processScanResult in the consumer
     }
 }
-//theoretically markWorkItemAsDone could be done in the generator as WorkItem is passed in by reference
-// but: eventually this has to be done accross CPU worker boundaries
-// so we act on the yielded scan result
-export async function markWorkItemAsDone(loop_event, work_buffer) {
-    if (loop_event.type === "Ready" && loop_event.work_uuid) {
-        const work_item = work_buffer.find((w) => w.work_uuid === loop_event.work_uuid);
-        if (work_item) {
-            work_item.result = loop_event.result;
-            work_item.status = "scanwork_done";
-            return work_item;
-        }
-    }
-}