@spirobel/monero-wallet-api 0.3.18 → 0.4.1

package/dist/scanning-syncing/scanresult/scanLoop.js

@@ -0,0 +1,116 @@
+import { sleep, ViewPair, } from "../../api";
+export function makeWorkItem(walletConfig, batch, from, to) {
+    if (to && to > batch.get_blocks_result_meta.block_infos.length - 1) {
+        // we could throw here but we dont. Off by one errors shouldnt break the application
+        // and we put so much work into making processResult idempotent
+        // so a caller bug scheduling too much work just leads to slightly worse performance
+        //throw new Error("to is out of bounds");
+        to = batch.get_blocks_result_meta.block_infos.length - 1;
+    }
+    if (typeof from === "number" && from < 0) {
+        //throw new Error("from is out of bounds");
+        from = 0;
+    }
+    return {
+        walletConfig,
+        batch: {
+            ...batch,
+            data: batch.data.slice(),
+        },
+        work_uuid: crypto.randomUUID(),
+        from: from ?? 0, // default is to start from the batch beginning
+        to: to ?? batch.get_blocks_result_meta.block_infos.length - 1, // default is to go to the batch end
+        status: "fresh",
+    };
+}
+export async function* scanLoop(wallet) {
+    let scanResult;
+    let work_uuid;
+    let workitem_batch_uuid;
+    let loaded_batch_uuid;
+    while (true) {
+        const item = yield {
+            type: "Ready",
+            work_uuid,
+            result: scanResult,
+        };
+        if (item === "cancel" || item === undefined)
+            continue;
+        work_uuid = String(item.work_uuid);
+        // 1.call loadGetBlocksBinResponse
+        workitem_batch_uuid = String(item.batch.local_uuid);
+        const viewpair = await ViewPair.create(item.walletConfig.primary_address, item.walletConfig.secret_view_key, item.walletConfig.subaddress_index);
+        await viewpair.loadGetBlocksBinResponse(item.batch.data);
+        loaded_batch_uuid = workitem_batch_uuid;
+        const first_block_meta = item.batch.get_blocks_result_meta.block_infos[0];
+        if (!first_block_meta)
+            throw new Error("no first block meta");
+        scanResult = undefined;
+        scanResult = {
+            outputs: [],
+            all_key_images: [],
+            new_height: first_block_meta.block_height + item.from,
+            primary_address: wallet.primary_address,
+            block_infos: item.batch.get_blocks_result_meta.block_infos,
+            daemon_height: item.batch.get_blocks_result_meta.daemon_height,
+        };
+        if (!(item.to >= item.from))
+            throw new Error("to must be >= from");
+        for (let i = item.from; i <= item.to; i++) {
+            // 2.call getBlocksBinScanOneBlock
+            const blockResult = await viewpair.getBlocksBinScanOneBlock(i);
+            if ("error" in blockResult)
+                throw new Error(blockResult.error);
+            //3. accumulate the scanresult
+            scanResult.outputs.push(...blockResult.outputs);
+            scanResult.all_key_images.push(...blockResult.all_key_images);
+            scanResult.new_height = first_block_meta.block_height + i;
+            //       //  update item.scanCache in memory will be done by the consumer
+            // it is saved on the workitem, but the workitemBuffer is managed by the consumer
+            // the consumer will reconcile the workitemBuffer on blocksbuffer changed yields from the fetchloop
+            // it will also reconcile the blocksbuffer with the workitembuffer when work was done,
+            // processScanResult will be called by the consumer and the scanresult will
+            // be persisted to disk by the consumer.
+            // if the workitem is at the left end of the workitemBuffer it will be shifted (popped from the left)
+            // if it is removed like this from the workItemBuffer and eventually no items in the workItemBuffer
+            // refercence the getBlocksBinBufferItem in the blocksbuffer, the blocksbufferitem is removed from the blocksbuffer
+            // this is how the reconiliaton workitembuffer -> blocksbuffer happens.
+            //
+            // the reconciliation in blocksbuffer -> workitembuffer starts in the fetchloop before it sends a yield blocksbuffer changed
+            // it is finished by the consumer of this generator after the yield blocksbuffer changed
+            // this happens through checking all workitemBuffer items,
+            //  if their GetBlocksBinBufferItem is in the blocksbuffer (via the local_uuid)
+            // if they are not in the blocksbuffer they are removed from the workitemBuffer
+            // then for all the blocksbuffer items that are not yet referenced by workbuffer items
+            // they need to be silced into workitems and added to the workitembuffer
+            //the reconciliation blocksbuffer -> workitembuffer is done on every blocks buffer changed event from the fetchloop
+            // the reconciliation workitembuffer -> blocksbuffer is done on every workitembuffer marked done at the end of the workbuffer (left end)
+            const input = yield {
+                type: "InProgress",
+                work_uuid,
+                result: scanResult,
+            };
+            if (input === "cancel")
+                break;
+            if (typeof input === "object")
+                throw new Error("cancel oldworkitem, before sending new workitem, while work in progress");
+            await sleep(10); // make sure the loop is not tight
+        }
+        // TODO: move scanCache forward with scanResult
+        // this can not be done here, has to be done by the consumer of the generator
+        // will be done by processScanResult in the consumer
+    }
+}
+//theoretically markWorkItemAsDone could be done in the generator as WorkItem is passed in by reference
+// but: eventually this has to be done accross CPU worker boundaries
+// so we act on the yielded scan result
+export async function markWorkItemAsDone(loop_event, work_buffer) {
+    if (loop_event.type === "Ready" && loop_event.work_uuid) {
+        const work_item = work_buffer.find((w) => w.work_uuid === loop_event.work_uuid);
+        if (work_item) {
+            work_item.result = loop_event.result;
+            work_item.status = "scanwork_done";
+            return work_item;
+        }
+    }
+}

package/dist/scanning-syncing/scanresult/scanResult.d.ts

@@ -1,4 +1,4 @@
-import type { BlockInfo, Output } from "../../api";
+import { type BlockInfo, type Output } from "../../api";
 import { type KeyImage } from "./computeKeyImage";
 import { type ErrorResponse } from "../../node-interaction/binaryEndpoints";
 import type { CacheChangedCallback, CacheRange, ChangedOutput, ScanCache } from "./scanCache";
@@ -11,7 +11,22 @@ export type ProcessScanResultParams = {
     pathPrefix?: string;
     use_master_current_range?: boolean;
 };
-export declare function processScanResult(params: ProcessScanResultParams): Promise<CacheRange>;
+export type ProcessScanResultParamsWithoutSideEffects = {
+    from_height: number;
+    secret_spend_key?: string;
+    result: ScanResult | ErrorResponse | undefined;
+    scanCache: ScanCache;
+    to_height?: number;
+};
+export type ProcessScanResult = {
+    current_range: CacheRange;
+    changed_outputs: ChangedOutput[];
+};
+export declare function processScanResult(params: ProcessScanResultParamsWithoutSideEffects): Promise<ProcessScanResult>;
+export declare function processResultReturnValue(cache: ScanCache, params: ProcessScanResultParamsWithoutSideEffects, changed_outputs: ChangedOutput[]): {
+    current_range: CacheRange;
+    changed_outputs: ChangedOutput[];
+};
 export type OnchainKeyImage = {
     key_image_hex: KeyImage;
     relative_index: number;
@@ -35,7 +50,6 @@ export type FastForward = number;
  * to look out for our own spends before proceeding the scan. This happens in the processScanResult function.
  */
 export type ScanResultCallback = ((result: ScanResult | ErrorResponse | EmptyScanResult) => FastForward | void) | ((result: ScanResult | ErrorResponse | EmptyScanResult) => Promise<FastForward | void>);
-export declare function updateScanHeight(current_range: CacheRange, result: ScanResult, cache: ScanCache): [CacheRange, ChangedOutput[]];
 export declare function makeNewRange(newRange: CacheRange, cache: ScanCache): CacheRange;
 export declare function detectOutputs(result: ScanResult, cache: ScanCache, spend_private_key?: string): Promise<ChangedOutput[]>;
 export declare function detectOwnspends(result: ScanResult, cache: ScanCache): ChangedOutput[];
@@ -62,3 +76,10 @@ export type Spendable = {
 export type OutputStatus = PrePending | Pending | Spent | Burnt | Reorged | Spendable;
 export declare function outputStatus(output: Output, cache: ScanCache, current_height: number): OutputStatus;
 export declare function spendable(output: Output, cache: ScanCache, current_height: number): boolean;
+export declare function findTipIndex(block_infos: BlockInfo[], oldTip: BlockInfo): number | "reorg_found" | "empty_blocks_array";
+export declare function selectAnchors(block_infos: BlockInfo[], tipIndex: number, oldRange: CacheRange, endIndex?: number): CacheRange;
+export declare function shouldReplaceAnchor(oldRange: CacheRange, last: BlockInfo): true | undefined;
+export declare function newAnchorCandidates(oldRange: CacheRange, block_infos: BlockInfo[], tipIndex: number, endIndex: number, last: BlockInfo): {
+    newCandidateAnchor: BlockInfo;
+    newAnchor: BlockInfo;
+};

package/dist/scanning-syncing/scanresult/scanResult.js

@@ -1,110 +1,128 @@
+import { CatastrophicReorgError, } from "../../api";
 import { computeKeyImage } from "./computeKeyImage";
-import { mergeRanges, findRange, readCacheFileDefaultLocation, lastRange, writeCacheToFile, } from "./scanCache";
+import { log } from "../../io/logging";
+import { mergeRanges, findRange, readCacheFileDefaultLocation, lastRange, writeCacheToFile, findRangeThrows, } from "./scanCache";
 import {} from "../../node-interaction/binaryEndpoints";
-import { handleReorg } from "./reorg";
 export async function processScanResult(params) {
-    let current_range = params.current_range;
-    const { result, cacheChanged, secret_spend_key, pathPrefix, use_master_current_range, } = params;
+    const { result, scanCache: cache, secret_spend_key } = params;
+    let changed_outputs = [];
     if (!(result && "primary_address" in result))
-        return current_range;
-    const cache = await readCacheFileDefaultLocation(result.primary_address, pathPrefix);
-    if (!cache)
-        throw new Error(`cache not found for primary address: ${result.primary_address} and path prefix: ${pathPrefix}`);
+        return processResultReturnValue(cache, params, changed_outputs);
     if (result && "daemon_height" in result)
         cache.daemon_height = result.daemon_height;
-    if (use_master_current_range) {
-        const last = lastRange(cache.scanned_ranges);
-        if (last) {
-            last.block_hashes = current_range.block_hashes;
-            last.end = current_range.end;
-        }
-        else {
-            cache.scanned_ranges.push(current_range);
-        }
-    }
-    try {
-        if (result && "new_height" in result) {
-            const [new_range, changed_outputs] = updateScanHeight(current_range, result, cache);
-            current_range = new_range;
-            changed_outputs.push(...(await detectOutputs(result, cache, secret_spend_key)));
-            if (secret_spend_key)
-                changed_outputs.push(...detectOwnspends(result, cache));
-            // write to cache
-            await writeCacheToFile(cache, params.pathPrefix);
-            await cacheChanged({
-                newCache: cache,
-                changed_outputs,
-            });
+    if (result && "new_height" in result) {
+        const oldRange = findRangeThrows(cache.scanned_ranges, params.from_height);
+        let tipIndex = findTipIndex(result.block_infos, oldRange.block_hashes[0]);
+        if (tipIndex === "empty_blocks_array")
+            return processResultReturnValue(cache, params, changed_outputs);
+        if (tipIndex === "reorg_found") {
+            let split_height_index = null;
+            for (const block_hash of oldRange.block_hashes) {
+                split_height_index = findTipIndex(result.block_infos, block_hash);
+                if (typeof split_height_index === "number")
+                    break;
+            }
+            if (typeof split_height_index !== "number")
+                throw new CatastrophicReorgError("could not locate anchors, should not try to processResult on stale workItems");
+            const split_height = result.block_infos[split_height_index];
+            // we found the split height & do the reorg
+            log("processScanResult", [
+                "SPLIT FOUND at height",
+                split_height.block_height,
+            ]);
+            if (!cache.reorg_info) {
+                cache.reorg_info = {
+                    split_heights: [split_height],
+                    removed_outputs: [],
+                    reverted_spends: [],
+                };
+            }
+            else {
+                cache.reorg_info.split_heights.push(split_height);
+            }
+            // First, collect reverted spends from all outputs (before any removals)
+            // so that outputs with block_height < split_height but spent_block_height >= split_height are captured
+            const reverted_outputs = Object.entries(cache.outputs).filter(([id, output]) => output.spent_block_height !== undefined &&
+                output.spent_block_height >= split_height.block_height);
+            const removed_outputs = Object.entries(cache.outputs).filter(([id, output]) => output.block_height >= split_height.block_height);
+            log("processScanResult", [
+                "removed_outputs count:",
+                removed_outputs.length,
+                "reverted count:",
+                reverted_outputs.length,
+            ]);
+            for (const [id, old_output_state] of removed_outputs) {
+                // 1. find key_image of output to be removed (as it was reorged)
+                const [key_image] = Object.entries(cache.own_key_images).find(([own_key_image, globalid]) => globalid === id) || [""]; // if this is viewonly the key_image will be empty
+                cache.reorg_info.removed_outputs.push({
+                    old_output_state,
+                    key_image,
+                    split_height,
+                });
+                // 2. remove from outputs and own_key_images
+                delete cache.outputs[id];
+                delete cache.own_key_images[key_image];
+                changed_outputs.push({
+                    output: old_output_state,
+                    change_reason: "reorged",
+                });
+            }
+            for (const [id, old_output_state_pointer] of reverted_outputs) {
+                const [key_image] = Object.entries(cache.own_key_images).find(([own_key_image, globalid]) => globalid === id) || [""]; // if this is viewonly the key_image will be empty
+                const old_output_state = Object.assign({}, old_output_state_pointer);
+                cache.reorg_info.reverted_spends.push({
+                    old_output_state,
+                    key_image, // in this case key_image only used here, does not get removed
+                    split_height,
+                });
+                // remove spend info from original cache (if output still exists
+                // it may have been deleted above in the removed_outputs loop)
+                if (cache.outputs[id]) {
+                    delete cache.outputs[id].spent_relative_index;
+                    delete cache.outputs[id].spent_in_tx_hash;
+                    delete cache.outputs[id].spent_block_height;
+                    delete cache.outputs[id].spent_block_timestamp;
+                }
+                changed_outputs.push({
+                    output: old_output_state,
+                    change_reason: "reorged_spent",
+                });
+            }
+            tipIndex = split_height_index;
         }
-        return current_range;
-    }
-    catch (e) {
-        params.catastrophic_reorg_cb();
-        throw e;
+        const to_height_index = result.block_infos.findIndex((b) => b.block_height === params.to_height);
+        const new_range = selectAnchors(result.block_infos, tipIndex, oldRange, to_height_index);
+        const last_height = typeof params.to_height === "number"
+            ? params.to_height
+            : result.block_infos[result.block_infos.length - 1].block_height;
+        //first check that no reorg occured by 1. finding oldrange, making sure oldrange end is in the meta blockinfos
+        // if that is not met call reorg handling code
+        //2. make sure reorg handling code works indepented of where the oldrange end is in the meta blockinfos
+        //3. make sure the anchor selection logic works independent of where the oldrange end is in the meta blockinfos
+        //const [new_range, changed] = updateScanHeight(current_range, result, cache);
+        makeNewRange(new_range, cache);
+        const tipHeight = result.block_infos[tipIndex].block_height;
+        const filteredOutputs = result.outputs.filter((o) => o.block_height >= tipHeight && o.block_height <= last_height);
+        const filteredKeyImages = result.all_key_images.filter((ki) => ki.block_height >= tipHeight && ki.block_height <= last_height);
+        changed_outputs.push(...(await detectOutputs({
+            ...result,
+            outputs: filteredOutputs,
+            all_key_images: filteredKeyImages,
+        }, cache, secret_spend_key)));
+        if (secret_spend_key)
+            changed_outputs.push(...detectOwnspends({
+                ...result,
+                outputs: filteredOutputs,
+                all_key_images: filteredKeyImages,
+            }, cache));
     }
+    return processResultReturnValue(cache, params, changed_outputs);
 }
-export function updateScanHeight(current_range, result, cache) {
-    let last_block_hash_of_result = result.block_infos.at(-1);
-    let current_blockhash = current_range?.block_hashes.at(0);
-    if (!current_blockhash)
-        throw new Error("current_range passed to updateScanHeight was malformed. block_hashes is empty");
-    if (!last_block_hash_of_result)
-        return [current_range, []]; // block_infos empty, no change (we are at tip and there was no new block)
-    // if last blockhash is undefined it means there was not reorg, we are at tip, block_infos is empty ( no new blocks )
-    const oldRange = findRange(cache.scanned_ranges, current_blockhash.block_height);
-    if (!oldRange)
-        throw new Error(`could not find scan range for height ${current_blockhash.block_height},
-       that means the blocks in the response from getBlocks.bin do not overlap
-       with the scanned ranges in the cache. This should not happen, as even if 
-       we are starting from a new start_height that has been supplied to scanWithCache,
-       it has been found as an existing range in the cache, or it has been
-       added as a new range before we started scannning.`);
-    // now we need to find the block_infos of old range in the new geblocksbin response result block_infos
-    // if we cant find the new range, there was a reorg and we need to clean all outputs after that and log what happened
-    let first_block_hash = result.block_infos.at(0);
-    if (!first_block_hash)
-        return [current_range, []]; // should never happen, if there is last_block_hash there should be first_block_hash
-    // if the first block hash in the response is not the same as the last block hash in the old range, there was a reorg
-    if (!(first_block_hash.block_hash === current_blockhash.block_hash)) {
-        return handleReorg(current_range, result, cache, oldRange);
-    }
-    // scan only happens in one direction,
-    // to scan earlier ranges: abort and recall with smaller start_height
-    // getblocksbin will return up to 1000 blocks at once
-    // so this should never happen, except if we just popped a block (but that case is handled above in the reorg case)
-    if (current_blockhash.block_height > last_block_hash_of_result.block_height)
-        throw new Error(`current scan height was larger than block height of last block from latest scan result. 
-       Most likely connected to faulty node / catastrophic reorg.
-       current height: ${current_blockhash.block_height}, new height: ${last_block_hash_of_result.block_height}`);
-    // 1. add new scanned range
-    let anchor = undefined;
-    let anchor_candidate = undefined;
-    if (oldRange.block_hashes.length >= 3) {
-        const old_anchor = oldRange.block_hashes.at(-1);
-        const old_anchor_candidate = oldRange.block_hashes.at(-2);
-        anchor = old_anchor;
-        anchor_candidate = old_anchor_candidate;
-        if (
-        // if the old range has an anchor, and the anchor is more than 200 blocks old
-        typeof old_anchor?.block_height === "number" &&
-            current_blockhash.block_height - old_anchor.block_height > 200) {
-            anchor = old_anchor_candidate; // use the anchor_candidate as anchor
-            // new anchor_candidate: is the one 100 blocks in, or the old scan tip
-            anchor_candidate =
-                result.block_infos.slice(-100)[0] || oldRange?.block_hashes.at(0); // use  use the old scan tip as anchor candidate
-        }
-    }
-    // if there is no old anchor, use the one 100 blocks in, or the last block hash
-    anchor =
-        anchor || result.block_infos.slice(-100)[0] || last_block_hash_of_result;
-    // carry over the old anchor candidate or use the last block
-    anchor_candidate = anchor_candidate || last_block_hash_of_result;
-    const newRange = {
-        start: current_blockhash.block_height,
-        end: last_block_hash_of_result.block_height,
-        block_hashes: [last_block_hash_of_result, anchor_candidate, anchor],
+export function processResultReturnValue(cache, params, changed_outputs) {
+    return {
+        current_range: findRangeThrows(cache.scanned_ranges, params.from_height),
+        changed_outputs,
     };
-    return [makeNewRange(newRange, cache), []];
 }
 export function makeNewRange(newRange, cache) {
     cache.scanned_ranges.push(newRange);
@@ -211,3 +229,47 @@ export function spendable(output, cache, current_height) {
     const status = outputStatus(output, cache, current_height);
     return status.status === "spendable";
 }
+export function findTipIndex(block_infos, oldTip) {
+    if (block_infos.length === 0)
+        return "empty_blocks_array";
+    const idx = block_infos.findIndex((b) => b.block_hash === oldTip.block_hash);
+    if (idx === -1)
+        return "reorg_found";
+    return idx;
+}
+export function selectAnchors(block_infos, tipIndex, oldRange, endIndex = block_infos.length - 1) {
+    const tip = block_infos[tipIndex];
+    const last = block_infos[endIndex];
+    const { newCandidateAnchor, newAnchor } = newAnchorCandidates(oldRange, block_infos, tipIndex, endIndex, last);
+    return {
+        start: tip.block_height,
+        end: last.block_height,
+        block_hashes: [last, newCandidateAnchor, newAnchor],
+        // newest, middle, oldest
+    };
+}
+export function shouldReplaceAnchor(oldRange, last) {
+    const oldAnchor = oldRange.block_hashes.at(2);
+    if (!oldAnchor)
+        throw new Error("old Range malformed");
+    if (
+    //if the old anchor is more than 200 blocks old
+    last.block_height - oldAnchor.block_height >
+        200) {
+        return true;
+    }
+}
+export function newAnchorCandidates(oldRange, block_infos, tipIndex, endIndex, last) {
+    const oldCandidateAnchor = oldRange.block_hashes.at(1);
+    const oldAnchor = oldRange.block_hashes.at(2);
+    if (!oldAnchor || !oldCandidateAnchor || !last)
+        throw new Error("old Range malformed");
+    if (shouldReplaceAnchor(oldRange, last)) {
+        const newCandidateAnchorIndex = Math.max(tipIndex, endIndex - 100);
+        const newCandidateAnchor = block_infos[newCandidateAnchorIndex];
+        if (!newCandidateAnchor)
+            throw new Error("tipIndex clamped candidate anchor index out of range");
+        return { newCandidateAnchor, newAnchor: oldCandidateAnchor };
+    }
+    return { newCandidateAnchor: oldCandidateAnchor, newAnchor: oldAnchor };
+}