@bsv/overlay 2.0.3 → 2.0.4

package/dist/esm/src/Engine.js

@@ -1,8 +1,11 @@
-import { Transaction, isBroadcastFailure, SHIPBroadcaster, HTTPSOverlayBroadcastFacilitator, LookupResolver } from '@bsv/sdk';
+import { Transaction, MerklePath, isBroadcastFailure, SHIPBroadcaster, HTTPSOverlayBroadcastFacilitator, LookupResolver } from '@bsv/sdk';
 import { GASP } from '@bsv/gasp';
 import { OverlayGASPRemote } from './GASP/OverlayGASPRemote.js';
 import { OverlayGASPStorage } from './GASP/OverlayGASPStorage.js';
+import { BASM_ZERO_HASH, computeBasmRoot, computeTac, extractMerkleProofMetadata } from './BASM.js';
+import { BASMRemote } from './BASMRemote.js';
 const DEFAULT_GASP_SYNC_LIMIT = 10000;
+const DEFAULT_BASM_RANGE_LIMIT = 1024;
 /**
  * An engine for running BSV Overlay Services (topic managers and lookup services).
  */
@@ -23,6 +26,9 @@ export class Engine {
     overlayBroadcastFacilitator;
     logger;
     suppressDefaultSyncAdvertisements;
+    topicAnchorHeaderResolver;
+    basmSyncEnabled;
+    unprovenEvictionBlocks;
     /**
      * Creates a new Overlay Services Engine
      * @param {[key: string]: TopicManager} managers - manages topic admittance
@@ -41,8 +47,11 @@ export class Engine {
      * @param {OverlayBroadcastFacilitator} overlayBroadcastFacilitator - Facilitator for propagation to other Overlay Services.
      * @param {typeof console} logger - The place where log entries are written.
      * @param {boolean} suppressDefaultSyncAdvertisements - Whether to suppress the default (SHIP/SLAP) sync advertisements.
+     * @param {TopicAnchorHeaderResolver} topicAnchorHeaderResolver - Resolves block hashes for BASM anchors.
+     * @param {boolean} basmSyncEnabled - Whether BASM sync should run automatically.
+     * @param {number} unprovenEvictionBlocks - Default block age for opt-in unproven state eviction.
      */
-    constructor(managers, lookupServices, storage, chainTracker, hostingURL, shipTrackers, slapTrackers, broadcaster, advertiser, syncConfiguration, logTime = false, logPrefix = '[OVERLAY_ENGINE] ', throwOnBroadcastFailure = false, overlayBroadcastFacilitator = new HTTPSOverlayBroadcastFacilitator(), logger = console, suppressDefaultSyncAdvertisements = true) {
+    constructor(managers, lookupServices, storage, chainTracker, hostingURL, shipTrackers, slapTrackers, broadcaster, advertiser, syncConfiguration, logTime = false, logPrefix = '[OVERLAY_ENGINE] ', throwOnBroadcastFailure = false, overlayBroadcastFacilitator = new HTTPSOverlayBroadcastFacilitator(), logger = console, suppressDefaultSyncAdvertisements = true, topicAnchorHeaderResolver, basmSyncEnabled = false, unprovenEvictionBlocks = 144) {
         this.managers = managers;
         this.lookupServices = lookupServices;
         this.storage = storage;
@@ -59,6 +68,9 @@ export class Engine {
         this.overlayBroadcastFacilitator = overlayBroadcastFacilitator;
         this.logger = logger;
         this.suppressDefaultSyncAdvertisements = suppressDefaultSyncAdvertisements;
+        this.topicAnchorHeaderResolver = topicAnchorHeaderResolver;
+        this.basmSyncEnabled = basmSyncEnabled;
+        this.unprovenEvictionBlocks = unprovenEvictionBlocks;
         // To encourage synchronization of overlay services, the SHIP sync strategy is used by default for all overlay topics, except for 'tm_ship' and 'tm_slap'.
         // For these two topics, any existing trackers are combined with the provided shipTrackers and slapTrackers omitting any duplicates.
         if (syncConfiguration === undefined) {
@@ -86,9 +98,7 @@ export class Engine {
             }
             else {
                 // Set undefined managers to 'SHIP' by default
-                if (this.syncConfiguration[managerName] === undefined) {
-                    this.syncConfiguration[managerName] = 'SHIP';
-                }
+                this.syncConfiguration[managerName] ??= 'SHIP';
             }
         }
     }
@@ -103,6 +113,276 @@ export class Engine {
             this.logger.timeEnd(`${this.logPrefix} ${label}`);
         }
     }
+    async currentHeightOrUndefined() {
+        if (this.chainTracker === 'scripts only') {
+            return undefined;
+        }
+        try {
+            return await this.chainTracker.currentHeight();
+        }
+        catch (error) {
+            this.logger.warn(`Unable to resolve current chain height for overlay metadata: ${error instanceof Error ? error.message : String(error)}`);
+            return undefined;
+        }
+    }
+    async resolveBlockHash(blockHeight, merkleRoot) {
+        try {
+            const header = await this.topicAnchorHeaderResolver?.(blockHeight);
+            if (header === undefined) {
+                return undefined;
+            }
+            if (header.merkleRoot !== undefined && merkleRoot !== undefined && header.merkleRoot !== merkleRoot) {
+                throw new Error(`Header merkle root ${header.merkleRoot} does not match proof root ${merkleRoot} at height ${blockHeight}`);
+            }
+            return header.blockHash;
+        }
+        catch (error) {
+            this.logger.warn(`Unable to resolve BASM block hash for height ${blockHeight}: ${error instanceof Error ? error.message : String(error)}`);
+            return undefined;
+        }
+    }
+    compactBEEFForStorage(tx, originalBEEF) {
+        return tx.merklePath === undefined ? originalBEEF : tx.toAtomicBEEF();
+    }
+    async recordTransactionData(tx, beef, blockHash) {
+        if (typeof this.storage.upsertTransactionRecord !== 'function') {
+            return;
+        }
+        const txid = tx.id('hex');
+        const metadata = extractMerkleProofMetadata(txid, tx.merklePath);
+        await this.storage.upsertTransactionRecord({
+            txid,
+            beef: this.compactBEEFForStorage(tx, beef),
+            rawTx: Array.from(tx.toBinary()),
+            merklePath: tx.merklePath?.toBinary(),
+            blockHeight: metadata?.blockHeight,
+            blockHash,
+            blockIndex: metadata?.blockIndex,
+            merkleRoot: metadata?.merkleRoot
+        });
+    }
+    async buildAppliedTransactionRecord(tx) {
+        const txid = tx.id('hex');
+        const metadata = extractMerkleProofMetadata(txid, tx.merklePath);
+        const [firstSeenHeight, blockHash] = await Promise.all([
+            this.currentHeightOrUndefined(),
+            metadata === undefined ? undefined : this.resolveBlockHash(metadata.blockHeight, metadata.merkleRoot)
+        ]);
+        return {
+            blockHeight: metadata?.blockHeight,
+            blockHash,
+            blockIndex: metadata?.blockIndex,
+            merkleRoot: metadata?.merkleRoot,
+            firstSeenHeight: firstSeenHeight ?? metadata?.blockHeight,
+            proven: metadata !== undefined
+        };
+    }
+    async recomputeTopicBlockAnchor(topic, blockHeight, blockHash) {
+        if (typeof this.storage.findAdmittedTransactionsForBlock !== 'function' ||
+            typeof this.storage.upsertTopicBlockAnchor !== 'function' ||
+            typeof this.storage.findTopicBlockAnchor !== 'function') {
+            return undefined;
+        }
+        const anchorBlockHash = blockHash ?? (await this.storage.findTopicBlockAnchor(topic, blockHeight))?.blockHash;
+        if (anchorBlockHash === undefined) {
+            return undefined;
+        }
+        // BRC-136 per-block completeness: establish the chain's genesis at the first
+        // admitted height, then keep every height from there to the tip contiguous so
+        // the cumulative TAC never resets across blocks with no admitted transactions.
+        // We rebuild [fromHeight, toHeight] rather than only the touched height so that
+        // an out-of-order proof (older height arriving after a newer one) can never
+        // leave a gap that silently breaks the chain.
+        const tip = await this.storage.findTopicAnchorTip?.(topic);
+        const tipHeight = tip !== undefined && tip.blockHeight >= 0 ? tip.blockHeight : undefined;
+        const fromHeight = tipHeight === undefined ? blockHeight : Math.min(blockHeight, tipHeight + 1);
+        const toHeight = tipHeight === undefined ? blockHeight : Math.max(blockHeight, tipHeight);
+        await this.rebuildTopicAnchorChain(topic, fromHeight, toHeight, new Map([[blockHeight, anchorBlockHash]]));
+        return await this.storage.findTopicBlockAnchor(topic, blockHeight);
+    }
+    /**
+     * Extends every configured topic's anchor chain forward with empty Topic Block
+     * Anchors (basmRoot = zero hash, admittedCount = 0) up to `toHeight`, so the
+     * cumulative TAC advances on every block even when a topic admits nothing —
+     * this is what lets a peer authoritatively confirm "this block contained no
+     * transactions for this topic". Chains with no first admission yet are left
+     * unstarted (genesis is the topic's first admitted height).
+     */
+    async advanceTopicAnchorChains(toHeight) {
+        if (typeof this.storage.findTopicAnchorTip !== 'function' ||
+            typeof this.storage.upsertTopicBlockAnchor !== 'function') {
+            return;
+        }
+        const targetHeight = toHeight ?? await this.currentHeightOrUndefined();
+        if (targetHeight === undefined) {
+            return;
+        }
+        for (const topic of Object.keys(this.managers)) {
+            const tip = await this.storage.findTopicAnchorTip(topic);
+            if (tip === undefined || tip.blockHeight < 0 || tip.blockHeight >= targetHeight) {
+                continue;
+            }
+            await this.rebuildTopicAnchorChain(topic, tip.blockHeight + 1, targetHeight);
+        }
+    }
+    /**
+     * Rebuilds a contiguous slice of a topic's anchor chain over [fromHeight,
+     * toHeight]. Each height uses its admitted transactions (empty -> zero basmRoot)
+     * and chains the cumulative TAC from the prior height. Missing heights are
+     * filled rather than skipped, so the chain stays gap-free. If a block hash
+     * cannot be resolved for some height the extension halts there to preserve
+     * contiguity instead of leaving a hole.
+     */
+    async rebuildTopicAnchorChain(topic, fromHeight, toHeight, blockHashHints = new Map(), forceResolve = false) {
+        if (typeof this.storage.findAdmittedTransactionsForBlock !== 'function' ||
+            typeof this.storage.upsertTopicBlockAnchor !== 'function' ||
+            typeof this.storage.findTopicBlockAnchor !== 'function' ||
+            toHeight < fromHeight) {
+            return;
+        }
+        if (toHeight - fromHeight + 1 > DEFAULT_BASM_RANGE_LIMIT) {
+            // Bound the work per pass; the next trigger resumes from the new tip.
+            this.logger.warn(`[BASM] capping anchor chain extension for "${topic}" at ${DEFAULT_BASM_RANGE_LIMIT} blocks (requested ${fromHeight}..${toHeight}); will continue on the next pass`);
+            toHeight = fromHeight + DEFAULT_BASM_RANGE_LIMIT - 1;
+        }
+        const previousAnchor = fromHeight > 0
+            ? await this.storage.findTopicBlockAnchor(topic, fromHeight - 1)
+            : undefined;
+        let prevTac = previousAnchor?.tac ?? BASM_ZERO_HASH;
+        for (let height = fromHeight; height <= toHeight; height++) {
+            const admitted = await this.storage.findAdmittedTransactionsForBlock(topic, height);
+            const existing = await this.storage.findTopicBlockAnchor(topic, height);
+            // On a reorg rebuild the existing anchor's block hash is stale, so force
+            // canonical re-resolution from the header resolver instead of reusing it.
+            const blockHash = blockHashHints.get(height) ?? (forceResolve ? undefined : existing?.blockHash) ?? await this.resolveBlockHash(height);
+            if (blockHash === undefined) {
+                this.logger.warn(`[BASM] unable to resolve block hash for "${topic}" at height ${height}; halting chain extension`);
+                return;
+            }
+            const basmRoot = computeBasmRoot(admitted);
+            const tac = computeTac(prevTac, blockHash, basmRoot);
+            await this.storage.upsertTopicBlockAnchor({
+                topic,
+                blockHeight: height,
+                blockHash,
+                basmRoot,
+                admittedCount: admitted.length,
+                tac
+            });
+            prevTac = tac;
+        }
+    }
+    /**
+     * Reconciles BASM anchors with a blockchain reorganization reported by the
+     * chain tracker (e.g. go-chaintracks `/v2/reorg/stream`). Proven topic
+     * transactions whose block was orphaned are demoted to unproven so they leave
+     * the admitted set, then every topic anchor chain intersecting the affected
+     * height range is rebuilt over the canonical block hashes. A reorg changes the
+     * canonical block hash for the affected heights, so topics with no demoted
+     * transaction are rebuilt too. Idempotent: a clean window demotes nothing and
+     * reproduces an identical TAC, so this is safe to invoke on every reorg event,
+     * SSE reconnect, and poll.
+     */
+    async handleReorg(input) {
+        const report = { perTopic: [] };
+        if (typeof this.storage.findProvenAppliedTransactionsByBlockHash !== 'function' ||
+            typeof this.storage.demoteAppliedTransactionToUnproven !== 'function' ||
+            typeof this.storage.findTopicBlockAnchors !== 'function' ||
+            typeof this.storage.upsertTopicBlockAnchor !== 'function') {
+            return report;
+        }
+        // 1) Demote proven admissions whose block was orphaned. Hashes are
+        //    normalized to lower-case display hex to match stored block hashes
+        //    (go-sdk chainhash.Hash marshals as reversed display hex).
+        const demotedByTopic = new Map();
+        for (const rawHash of input.orphanedBlockHashes) {
+            const blockHash = rawHash.toLowerCase();
+            const rows = await this.storage.findProvenAppliedTransactionsByBlockHash(blockHash);
+            for (const row of rows) {
+                await this.storage.demoteAppliedTransactionToUnproven(row.txid, row.topic);
+                const list = demotedByTopic.get(row.topic) ?? [];
+                list.push(row.txid);
+                demotedByTopic.set(row.topic, list);
+            }
+        }
+        // 2) Rebuild every topic anchor chain that intersects the reorged range,
+        //    forcing canonical block-hash re-resolution so stale hashes are replaced.
+        for (const topic of Object.keys(this.managers)) {
+            const existing = await this.storage.findTopicBlockAnchors(topic, input.rebuildFromHeight, input.newTipHeight);
+            if (existing.length === 0) {
+                continue;
+            }
+            const startHeight = Math.min(...existing.map(anchor => anchor.blockHeight));
+            await this.rebuildTopicAnchorChain(topic, startHeight, input.newTipHeight, new Map(), true);
+            report.perTopic.push({
+                topic,
+                demotedTxids: demotedByTopic.get(topic) ?? [],
+                rebuiltFrom: startHeight,
+                rebuiltTo: input.newTipHeight
+            });
+        }
+        return report;
+    }
+    /**
+     * Revalidation sweep: the reorg fallback for chain trackers without a reorg
+     * event stream, and the catch-up step on every reorg-SSE (re)connect (the
+     * go-chaintracks reorg stream carries no event ids, so a reconnect cannot
+     * replay events missed while disconnected). Scans proven applied transactions
+     * in `[tip - depth + 1, tip]`; any whose proof root no longer validates against
+     * the chain tracker, or whose block hash diverges from the canonical header, is
+     * treated as orphaned and reconciled via {@link handleReorg}.
+     */
+    async isProvenAnchorStale(row, chainTracker) {
+        if (row.merkleRoot !== undefined) {
+            try {
+                if (!(await chainTracker.isValidRootForHeight(row.merkleRoot, row.blockHeight))) {
+                    return true;
+                }
+            }
+            catch (error) {
+                this.logger.warn(`[BASM] root validation failed for ${row.txid} at height ${row.blockHeight}: ${error instanceof Error ? error.message : String(error)}`);
+                return undefined;
+            }
+        }
+        const canonical = await this.resolveBlockHash(row.blockHeight);
+        return canonical !== undefined && canonical.toLowerCase() !== row.blockHash?.toLowerCase();
+    }
+    async revalidateRecentAnchors(depth = 3) {
+        const chainTracker = this.chainTracker;
+        if (chainTracker === 'scripts only') {
+            this.logger.warn('[BASM] revalidation sweep requires a ChainTracker; skipping');
+            return undefined;
+        }
+        if (typeof this.storage.findProvenAppliedTransactionsInRange !== 'function') {
+            return undefined;
+        }
+        const tip = await this.currentHeightOrUndefined();
+        if (tip === undefined) {
+            return undefined;
+        }
+        const fromHeight = Math.max(0, tip - depth + 1);
+        const rows = await this.storage.findProvenAppliedTransactionsInRange(fromHeight, tip);
+        const orphaned = new Set();
+        let minAffected = Number.POSITIVE_INFINITY;
+        for (const row of rows) {
+            if (row.blockHash === undefined) {
+                continue;
+            }
+            const stale = await this.isProvenAnchorStale(row, chainTracker);
+            if (stale === true) {
+                orphaned.add(row.blockHash.toLowerCase());
+                minAffected = Math.min(minAffected, row.blockHeight);
+            }
+        }
+        if (orphaned.size === 0) {
+            return { perTopic: [] };
+        }
+        return await this.handleReorg({
+            orphanedBlockHashes: Array.from(orphaned),
+            rebuildFromHeight: minAffected,
+            newTipHeight: tip
+        });
+    }
     /**
      * Submits a transaction for processing by Overlay Services.
      * @param {TaggedBEEF} taggedBEEF - The transaction to process
@@ -156,7 +436,7 @@ export class Engine {
                 // Identify previous coins admitted to this specific topic
                 const previousCoins = [];
                 const outputPromises = tx.inputs.map(async (input, i) => {
-                    const previousTXID = input.sourceTXID !== undefined ? input.sourceTXID : input.sourceTransaction?.id('hex');
+                    const previousTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex');
                     if (previousTXID !== undefined) {
                         // Check if the previous output was admitted to this specific topic
                         const output = await this.storage.findOutput(previousTXID, input.sourceOutputIndex, topic);
@@ -343,17 +623,17 @@ export class Engine {
                     if (typeof previousTXID !== 'string')
                         continue;
                     const previousOutputIndex = tx.inputs[inputIndex].sourceOutputIndex;
-                    if (!admissibleOutputs.coinsToRetain.includes(inputIndex)) {
-                        outputsToMarkStale.push({
+                    if (admissibleOutputs.coinsToRetain.includes(inputIndex)) {
+                        outputsConsumed.push({
                             txid: previousTXID,
-                            previousOutputIndex,
-                            inputIndex
+                            outputIndex: previousOutputIndex
                         });
                     }
                     else {
-                        outputsConsumed.push({
+                        outputsToMarkStale.push({
                             txid: previousTXID,
-                            outputIndex: previousOutputIndex
+                            previousOutputIndex,
+                            inputIndex
                         });
                     }
                 }
@@ -381,10 +661,11 @@ export class Engine {
                         satoshis: tx.outputs[outputIndex].satoshis,
                         topic,
                         spent: false,
-                        beef: taggedBEEF.beef,
+                        beef: this.compactBEEFForStorage(tx, taggedBEEF.beef),
                         consumedBy: [],
                         outputsConsumed,
-                        score: Date.now()
+                        score: Date.now(),
+                        blockHeight: extractMerkleProofMetadata(txid, tx.merklePath)?.blockHeight
                     });
                     this.endTime(`insertNewOutput_${txid.substring(0, 10)}`);
                     newUTXOs.push({ txid, outputIndex });
@@ -424,6 +705,8 @@ export class Engine {
                 }));
                 this.startTime(`outputConsumed_${txid.substring(0, 10)}`);
                 // Update each output consumed to know who consumed it and insert applied transaction in parallel
+                const appliedRecord = await this.buildAppliedTransactionRecord(tx);
+                await this.recordTransactionData(tx, taggedBEEF.beef, appliedRecord.blockHash);
                 await Promise.all([
                     ...outputsConsumed.map(async (output) => {
                         const outputToUpdate = await this.storage.findOutput(output.txid, output.outputIndex, topic);
@@ -434,9 +717,13 @@ export class Engine {
                     }),
                     this.storage.insertAppliedTransaction({
                         txid,
-                        topic
+                        topic,
+                        ...appliedRecord
                     })
                 ]);
+                if (appliedRecord.blockHeight !== undefined && appliedRecord.blockHash !== undefined) {
+                    await this.recomputeTopicBlockAnchor(topic, appliedRecord.blockHeight, appliedRecord.blockHash);
+                }
                 this.endTime(`outputConsumed_${txid.substring(0, 10)}`);
             }
             catch (error) {
@@ -729,7 +1016,7 @@ export class Engine {
                             // Parse out the advertisements using the provided parser
                             const tx = Transaction.fromBEEF(output.beef);
                             const advertisement = this.advertiser?.parseAdvertisement(tx.outputs[output.outputIndex].lockingScript);
-                            if (advertisement !== undefined && advertisement !== null && advertisement.protocol === 'SHIP') {
+                            if (advertisement?.protocol === 'SHIP') {
                                 endpointSet.add(advertisement.domain);
                             }
                         }
@@ -767,6 +1054,282 @@ export class Engine {
             }
         }
     }
+    async resolveSyncEndpointsForTopic(topic) {
+        if (this.syncConfiguration === undefined) {
+            return [];
+        }
+        let syncEndpoints = this.syncConfiguration[topic];
+        if (syncEndpoints === false || syncEndpoints === undefined) {
+            return [];
+        }
+        if (syncEndpoints === 'SHIP') {
+            const resolverConfig = this.slapTrackers
+                ? { slapTrackers: this.slapTrackers }
+                : {};
+            const resolver = new LookupResolver(resolverConfig);
+            const lookupAnswer = await resolver.query({
+                service: 'ls_ship',
+                query: {
+                    topics: [topic]
+                }
+            });
+            const endpointSet = new Set();
+            if (lookupAnswer.type === 'output-list') {
+                lookupAnswer.outputs.forEach(output => {
+                    try {
+                        const tx = Transaction.fromBEEF(output.beef);
+                        const advertisement = this.advertiser?.parseAdvertisement(tx.outputs[output.outputIndex].lockingScript);
+                        if (advertisement?.protocol === 'SHIP') {
+                            endpointSet.add(advertisement.domain);
+                        }
+                    }
+                    catch (error) {
+                        this.logger.error('Failed to parse BASM advertisement output:', error);
+                    }
+                });
+            }
+            syncEndpoints = Array.from(endpointSet);
+        }
+        if (!Array.isArray(syncEndpoints)) {
+            return [];
+        }
+        return syncEndpoints.filter(endpoint => endpoint !== this.hostingURL);
+    }
+    async provideTopicAnchorTip(topic) {
+        const tip = await this.storage.findTopicAnchorTip?.(topic);
+        return tip ?? {
+            topic,
+            blockHeight: -1,
+            tac: BASM_ZERO_HASH
+        };
+    }
+    async provideTopicAnchorRange(topic, fromHeight, toHeight) {
+        if (typeof this.storage.findTopicBlockAnchors !== 'function') {
+            throw new TypeError('Storage does not support BASM topic anchor ranges');
+        }
+        if (!Number.isInteger(fromHeight) || !Number.isInteger(toHeight) || fromHeight < 0 || toHeight < fromHeight) {
+            throw new Error('Invalid topic anchor range');
+        }
+        if (toHeight - fromHeight + 1 > DEFAULT_BASM_RANGE_LIMIT) {
+            throw new Error(`Topic anchor range is capped at ${DEFAULT_BASM_RANGE_LIMIT} heights`);
+        }
+        return {
+            topic,
+            anchors: await this.storage.findTopicBlockAnchors(topic, fromHeight, toHeight)
+        };
+    }
+    async provideAdmittedList(topic, blockHeight, blockHash) {
+        if (typeof this.storage.findAdmittedTransactionsForBlock !== 'function') {
+            throw new TypeError('Storage does not support BASM admitted lists');
+        }
+        return {
+            topic,
+            blockHeight,
+            blockHash,
+            admitted: await this.storage.findAdmittedTransactionsForBlock(topic, blockHeight, blockHash)
+        };
+    }
+    async provideCompoundMerklePath(topic, blockHeight, txids) {
+        if (typeof this.storage.findTransactionMerklePaths !== 'function') {
+            throw new TypeError('Storage does not support direct Merkle path lookup');
+        }
+        if (txids.length === 0) {
+            throw new Error('At least one txid is required');
+        }
+        const admitted = await this.storage.findAdmittedTransactionsForBlock?.(topic, blockHeight);
+        if (admitted !== undefined) {
+            const admittedSet = new Set(admitted.map(item => item.txid));
+            const missingAdmissions = txids.filter(txid => !admittedSet.has(txid));
+            if (missingAdmissions.length > 0) {
+                throw new Error(`Requested txids are not admitted to topic ${topic} at height ${blockHeight}: ${missingAdmissions.join(',')}`);
+            }
+        }
+        const proofs = await this.storage.findTransactionMerklePaths(txids);
+        const proofByTxid = new Map(proofs.map(proof => [proof.txid, proof]));
+        const missing = txids.filter(txid => !proofByTxid.has(txid));
+        if (missing.length > 0) {
+            throw new Error(`No direct Merkle path found for txids: ${missing.join(',')}`);
+        }
+        let compound;
+        for (const txid of txids) {
+            const proof = proofByTxid.get(txid);
+            if (proof === undefined)
+                continue;
+            const path = MerklePath.fromHex(proof.merklePath);
+            if (path.blockHeight !== blockHeight) {
+                throw new Error(`Merkle path for ${txid} is at height ${path.blockHeight}, expected ${blockHeight}`);
+            }
+            if (compound === undefined) {
+                compound = path;
+            }
+            else {
+                compound.combine(path);
+            }
+        }
+        if (compound === undefined) {
+            throw new Error('Unable to build compound Merkle path');
+        }
+        return {
+            topic,
+            blockHeight,
+            txids,
+            merklePath: compound.toHex()
+        };
+    }
+    async provideRawTransactions(txids) {
+        if (typeof this.storage.findRawTransactions !== 'function') {
+            throw new TypeError('Storage does not support raw transaction lookup');
+        }
+        const transactions = await this.storage.findRawTransactions(txids);
+        const found = new Set(transactions.map(tx => tx.txid));
+        return {
+            transactions,
+            missing: txids.filter(txid => !found.has(txid))
+        };
+    }
+    async startBASMSync() {
+        if (this.syncConfiguration === undefined) {
+            throw new Error('Overlay Service Engine not configured for topical synchronization!');
+        }
+        const reports = [];
+        for (const topic of Object.keys(this.syncConfiguration)) {
+            const endpoints = await this.resolveSyncEndpointsForTopic(topic);
+            for (const endpoint of endpoints) {
+                reports.push(await this.reconcileBASMWithPeer(topic, endpoint));
+            }
+        }
+        return reports;
+    }
+    async reconcileBASMWithPeer(topic, endpoint) {
+        const report = {
+            topic,
+            endpoint,
+            status: 'skipped',
+            checkedHeights: [],
+            missingTxids: [],
+            fetchedTxCount: 0
+        };
+        try {
+            const remote = new BASMRemote(endpoint, topic);
+            const [localTip, remoteTip] = await Promise.all([
+                this.provideTopicAnchorTip(topic),
+                remote.requestTopicAnchorTip()
+            ]);
+            report.localTip = localTip;
+            report.remoteTip = remoteTip;
+            if (localTip.blockHeight >= remoteTip.blockHeight) {
+                report.status = localTip.tac === remoteTip.tac && localTip.blockHeight === remoteTip.blockHeight ? 'matched' : 'diverged';
+                report.message = report.status === 'matched'
+                    ? 'Topic anchor tips match'
+                    : 'Remote tip is not ahead; historical divergence needs manual or binary-search reconciliation';
+                return report;
+            }
+            const fromHeight = Math.max(localTip.blockHeight + 1, remoteTip.blockHeight - DEFAULT_BASM_RANGE_LIMIT + 1, 0);
+            const range = await remote.requestTopicAnchorRange(fromHeight, remoteTip.blockHeight);
+            for (const remoteAnchor of range.anchors) {
+                await this.reconcileRemoteAnchor(topic, remote, remoteAnchor, report);
+            }
+            const refreshedTip = await this.provideTopicAnchorTip(topic);
+            report.localTip = refreshedTip;
+            report.status = refreshedTip.blockHeight >= remoteTip.blockHeight && refreshedTip.tac === remoteTip.tac ? 'matched' : 'advanced';
+            return report;
+        }
+        catch (error) {
+            report.status = 'error';
+            report.message = error instanceof Error ? error.message : String(error);
+            this.logger.error(`[BASM SYNC] Sync failed for topic "${topic}" with peer "${endpoint}"`, error);
+            return report;
+        }
+    }
+    async reconcileRemoteAnchor(topic, remote, remoteAnchor, report) {
+        report.checkedHeights.push(remoteAnchor.blockHeight);
+        const localAnchor = await this.storage.findTopicBlockAnchor?.(topic, remoteAnchor.blockHeight, remoteAnchor.blockHash);
+        if (localAnchor?.tac === remoteAnchor.tac) {
+            return;
+        }
+        const admittedResponse = await remote.requestAdmittedList(remoteAnchor.blockHeight, remoteAnchor.blockHash);
+        const remoteBasmRoot = computeBasmRoot(admittedResponse.admitted);
+        if (remoteBasmRoot !== remoteAnchor.basmRoot ||
+            admittedResponse.admitted.length !== remoteAnchor.admittedCount) {
+            throw new Error(`Peer ${report.endpoint} supplied an admitted list inconsistent with its anchor at height ${remoteAnchor.blockHeight}`);
+        }
+        const localAdmitted = await this.storage.findAdmittedTransactionsForBlock?.(topic, remoteAnchor.blockHeight, remoteAnchor.blockHash) ?? [];
+        const localTxids = new Set(localAdmitted.map(item => item.txid));
+        const missingTxids = admittedResponse.admitted
+            .map(item => item.txid)
+            .filter(txid => !localTxids.has(txid));
+        report.missingTxids.push(...missingTxids);
+        if (missingTxids.length === 0) {
+            report.status = 'diverged';
+            return;
+        }
+        await this.fetchBASMMissingTransactions(remote, topic, remoteAnchor, missingTxids);
+        report.fetchedTxCount += missingTxids.length;
+    }
+    async fetchBASMMissingTransactions(remote, topic, anchor, txids) {
+        if (this.chainTracker === 'scripts only') {
+            throw new Error('BASM reconciliation requires a ChainTracker capable of validating BUMP proofs');
+        }
+        const proofResponse = await remote.requestCompoundMerklePath(anchor.blockHeight, txids);
+        const compoundPath = MerklePath.fromHex(proofResponse.merklePath);
+        for (const txid of txids) {
+            const valid = await compoundPath.verify(txid, this.chainTracker);
+            if (!valid) {
+                throw new Error(`Peer supplied invalid compound Merkle path for ${txid} at height ${anchor.blockHeight}`);
+            }
+        }
+        const rawResponse = await remote.requestRawTransactions(txids);
+        if (rawResponse.missing.length > 0) {
+            throw new Error(`Peer did not return raw transactions for txids: ${rawResponse.missing.join(',')}`);
+        }
+        for (const record of rawResponse.transactions) {
+            const tx = Transaction.fromHex(record.rawTx);
+            if (tx.id('hex') !== record.txid) {
+                throw new Error(`Raw transaction txid mismatch: expected ${record.txid}, got ${tx.id('hex')}`);
+            }
+            try {
+                tx.merklePath = compoundPath.extract([record.txid]);
+            }
+            catch {
+                tx.merklePath = compoundPath;
+            }
+            await this.submit({ beef: tx.toBEEF(), topics: [topic] }, undefined, 'historical-tx');
+        }
+    }
+    async evictUnprovenTransactions(options = {}) {
+        if (typeof this.storage.findUnprovenAppliedTransactions !== 'function') {
+            throw new TypeError('Storage does not support unproven transaction eviction');
+        }
+        if (this.chainTracker === 'scripts only') {
+            throw new Error('Unproven eviction requires a ChainTracker to determine block age');
+        }
+        const thresholdBlocks = options.thresholdBlocks ?? this.unprovenEvictionBlocks;
+        const currentHeight = await this.chainTracker.currentHeight();
+        const cutoffHeight = currentHeight - thresholdBlocks;
+        const candidates = await this.storage.findUnprovenAppliedTransactions(cutoffHeight, options.topic);
+        let evictedOutputs = 0;
+        for (const candidate of candidates) {
+            for (const output of candidate.outputs) {
+                for (const service of Object.values(this.lookupServices)) {
+                    try {
+                        await service.outputEvicted(output.txid, output.outputIndex);
+                    }
+                    catch (error) {
+                        this.logger.debug(`outputEvicted notification failed for ${output.txid}.${output.outputIndex}: ${error}`);
+                    }
+                }
+                await this.storage.deleteOutput(output.txid, output.outputIndex, candidate.topic);
+                evictedOutputs++;
+            }
+            await this.storage.deleteAppliedTransaction?.(candidate.txid, candidate.topic);
+        }
+        return {
+            cutoffHeight,
+            candidates: candidates.length,
+            evictedTransactions: candidates.length,
+            evictedOutputs
+        };
+    }
     /**
      * Given a GASP request, create an initial response.
      *
@@ -813,29 +1376,17 @@ export class Engine {
                     // For each input, look it up and recurse.
                     for (const input of tx.inputs) {
                         // We should always have a source transaction
-                        if (input.sourceTransaction !== undefined) {
-                            searchInput(input.sourceTransaction);
+                        if (input.sourceTransaction === undefined) {
+                            throw new Error('Incomplete SPV data!');
                         }
                         else {
-                            throw new Error('Incomplete SPV data!');
+                            searchInput(input.sourceTransaction);
                         }
                     }
                 }
             };
             searchInput(rootTx);
-            if (correctTx !== undefined) {
-                const rawTx = correctTx.toHex();
-                const node = {
-                    rawTx,
-                    graphID,
-                    outputIndex
-                };
-                if (correctTx.merklePath !== undefined) {
-                    node.proof = correctTx.merklePath.toHex();
-                }
-                return node;
-            }
-            else {
+            if (correctTx === undefined) {
                 // Recursively try to find a matching output
                 let foundNode;
                 for (const currentOutput of output.outputsConsumed) {
@@ -845,6 +1396,8 @@ export class Engine {
                         break;
                     }
                     catch (error) {
+                        // Best-effort: output may not be found or hydration fails for this candidate; try the next one
+                        this.logger.debug(`Unable to hydrate output ${currentOutput.txid}.${currentOutput.outputIndex}: ${error}`);
                         continue;
                     }
                 }
@@ -852,6 +1405,18 @@ export class Engine {
                     return foundNode;
                 }
             }
+            else {
+                const rawTx = correctTx.toHex();
+                const node = {
+                    rawTx,
+                    graphID,
+                    outputIndex
+                };
+                if (correctTx.merklePath !== undefined) {
+                    node.proof = correctTx.merklePath.toHex();
+                }
+                return node;
+            }
             throw new Error('Unable to find output associated with your request!');
         };
         const [rootTxid, rootOutputIndex] = graphID.split('.');
@@ -875,7 +1440,7 @@ export class Engine {
     */
     async getUTXOHistory(output, historySelector, currentDepth = 0, context = this.createUTXOHistoryHydrationContext()) {
         // If we have an output but no history selector, just return the output.
-        if (typeof historySelector === 'undefined') {
+        if (historySelector === undefined) {
             return output;
         }
         try {
@@ -914,7 +1479,10 @@ export class Engine {
                     try {
                         await l.outputNoLongerRetainedInHistory?.(output.txid, output.outputIndex, output.topic);
                     }
-                    catch (_) { }
+                    catch (e) {
+                        // Best-effort notification; lookup service failure must not abort UTXO deletion
+                        this.logger.debug(`outputNoLongerRetainedInHistory notification failed for ${output.txid}.${output.outputIndex}: ${e}`);
+                    }
                 }
             }
             // If there are no more consumed utxos, return
@@ -922,7 +1490,7 @@ export class Engine {
                 return;
             }
             // Delete any stale outputs that were consumed as inputs
-            output.outputsConsumed.map(async (outputIdentifier) => {
+            await Promise.all(output.outputsConsumed.map(async (outputIdentifier) => {
                 const staleOutput = await this.storage.findOutput(outputIdentifier.txid, outputIdentifier.outputIndex, output.topic);
                 // Make sure an output was found
                 if (staleOutput === null || staleOutput === undefined) {
@@ -936,7 +1504,7 @@ export class Engine {
                 }
                 // Find previousUTXO history
                 return await this.deleteUTXODeep(staleOutput);
-            });
+            }));
         }
         catch (error) {
             throw new Error(`Failed to delete all stale outputs: ${error} `);
@@ -1011,12 +1579,35 @@ export class Engine {
         if (outputs === undefined || outputs.length === 0) {
             throw new Error('Could not find matching transaction outputs for proof ingest!');
         }
+        const proofMetadata = extractMerkleProofMetadata(txid, proof);
+        const resolvedBlockHeight = blockHeight ?? proofMetadata?.blockHeight;
+        const resolvedBlockHash = resolvedBlockHeight === undefined
+            ? undefined
+            : await this.resolveBlockHash(resolvedBlockHeight, proofMetadata?.merkleRoot);
         for (const output of outputs) {
             await this.updateMerkleProof(output, txid, proof);
             // Add the associated blockHeight
-            if (blockHeight !== undefined) {
-                output.blockHeight = blockHeight;
-                await this.storage.updateOutputBlockHeight?.(output.txid, output.outputIndex, output.topic, blockHeight);
+            if (resolvedBlockHeight !== undefined) {
+                output.blockHeight = resolvedBlockHeight;
+                await this.storage.updateOutputBlockHeight?.(output.txid, output.outputIndex, output.topic, resolvedBlockHeight);
+            }
+            if (output.beef !== undefined) {
+                const tx = Transaction.fromBEEF(output.beef);
+                this.updateInputProofs(tx, txid, proof);
+                await this.recordTransactionData(tx, tx.toBEEF(), resolvedBlockHash);
+            }
+            if (resolvedBlockHeight !== undefined) {
+                await this.storage.updateAppliedTransactionProof?.({
+                    txid,
+                    topic: output.topic,
+                    blockHeight: resolvedBlockHeight,
+                    blockHash: resolvedBlockHash,
+                    blockIndex: proofMetadata?.blockIndex,
+                    merkleRoot: proofMetadata?.merkleRoot
+                });
+            }
+            if (resolvedBlockHeight !== undefined && resolvedBlockHash !== undefined) {
+                await this.recomputeTopicBlockAnchor(output.topic, resolvedBlockHeight, resolvedBlockHash);
             }
         }
     }
@@ -1032,7 +1623,7 @@ export class Engine {
                 result[t] = await this.managers[t].getMetaData();
             }
             catch (e) {
-                this.logger.warn(`Unable to get metadata for topic manager: ${t}`);
+                this.logger.warn(`Unable to get metadata for topic manager: ${t}: ${e}`);
                 result[t] = {
                     name: t,
                     shortDescription: 'No topical tagline.'
@@ -1053,7 +1644,7 @@ export class Engine {
                 result[ls] = await this.lookupServices[ls].getMetaData();
             }
             catch (e) {
-                this.logger.warn(`Unable to get metadata for lookup service: ${ls}`);
+                this.logger.warn(`Unable to get metadata for lookup service: ${ls}: ${e}`);
                 result[ls] = {
                     name: ls,
                     shortDescription: 'No lookup service tagline.'
@@ -1069,7 +1660,7 @@ export class Engine {
      */
     async getDocumentationForTopicManager(manager) {
         const documentation = await this.managers[manager]?.getDocumentation?.();
-        return documentation !== undefined ? documentation : 'No documentation found!';
+        return documentation ?? 'No documentation found!';
     }
     /**
      * Run a query to get the documentation for a particular lookup service
@@ -1078,7 +1669,7 @@ export class Engine {
      */
     async getDocumentationForLookupServiceProvider(provider) {
         const documentation = await this.lookupServices[provider]?.getDocumentation?.();
-        return documentation !== undefined ? documentation : 'No documentation found!';
+        return documentation ?? 'No documentation found!';
     }
     /**
      * Validates a URL to ensure it does not match disallowed patterns:
@@ -1108,7 +1699,7 @@ export class Engine {
                 /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/, // Loopback IPs
                 /^10\.\d{1,3}\.\d{1,3}\.\d{1,3}$/, // 10.x.x.x private IPs
                 /^192\.168\.\d{1,3}\.\d{1,3}$/, // 192.168.x.x private IPs
-                /^172\.(1[6-9]|2[0-9]|3[0-1])\.\d{1,3}\.\d{1,3}$/, // 172.16.x.x to 172.31.x.x private IPs
+                /^172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3}$/, // 172.16.x.x to 172.31.x.x private IPs
                 /^0\.0\.0\.0$/ // Non-routable address
             ];
             // Check for IPv4 matches
@@ -1122,8 +1713,8 @@ export class Engine {
             // If none of the disallowed conditions matched, the URL is valid
             return true;
         }
-        catch (error) {
-            // If the input is not a valid URL, return false
+        catch (_e) {
+            // URL constructor throws on malformed input — not a valid URL, return false
             return false;
         }
     }