@bsv/overlay 2.2.1 → 2.3.0

package/dist/esm/src/Engine.js

@@ -4,8 +4,15 @@ import { OverlayGASPRemote } from './GASP/OverlayGASPRemote.js';
 import { OverlayGASPStorage } from './GASP/OverlayGASPStorage.js';
 import { BASM_ZERO_HASH, computeBasmRoot, computeTac, extractMerkleProofMetadata } from './BASM.js';
 import { BASMRemote } from './BASMRemote.js';
+import { serializeErrorForLog, serializeLogValue } from './SafeLog.js';
 const DEFAULT_GASP_SYNC_LIMIT = 10000;
 const DEFAULT_BASM_RANGE_LIMIT = 1024;
+function findSpendingInputIndex(tx, output) {
+    return tx.inputs.findIndex(input => {
+        const realSource = input.sourceTXID || input.sourceTransaction?.id('hex');
+        return realSource === output.txid && input.sourceOutputIndex === output.outputIndex;
+    });
+}
 /**
  * An engine for running BSV Overlay Services (topic managers and lookup services).
  */
@@ -29,6 +36,7 @@ export class Engine {
     topicAnchorHeaderResolver;
     basmSyncEnabled;
     unprovenEvictionBlocks;
+    maxLookupResults;
     /**
      * Creates a new Overlay Services Engine
      * @param {[key: string]: TopicManager} managers - manages topic admittance
@@ -50,8 +58,9 @@ export class Engine {
      * @param {TopicAnchorHeaderResolver} topicAnchorHeaderResolver - Resolves block hashes for BASM anchors.
      * @param {boolean} basmSyncEnabled - Whether BASM sync should run automatically.
      * @param {number} unprovenEvictionBlocks - Default block age for opt-in unproven state eviction.
+     * @param {number} maxLookupResults - Maximum lookup formulas hydrated per request. Use -1 to opt out.
      */
-    constructor(managers, lookupServices, storage, chainTracker, hostingURL, shipTrackers, slapTrackers, broadcaster, advertiser, syncConfiguration, logTime = false, logPrefix = '[OVERLAY_ENGINE] ', throwOnBroadcastFailure = false, overlayBroadcastFacilitator = new HTTPSOverlayBroadcastFacilitator(), logger = console, suppressDefaultSyncAdvertisements = true, topicAnchorHeaderResolver, basmSyncEnabled = false, unprovenEvictionBlocks = 144) {
+    constructor(managers, lookupServices, storage, chainTracker, hostingURL, shipTrackers, slapTrackers, broadcaster, advertiser, syncConfiguration, logTime = false, logPrefix = '[OVERLAY_ENGINE] ', throwOnBroadcastFailure = false, overlayBroadcastFacilitator = new HTTPSOverlayBroadcastFacilitator(), logger = console, suppressDefaultSyncAdvertisements = true, topicAnchorHeaderResolver, basmSyncEnabled = false, unprovenEvictionBlocks = 144, maxLookupResults = 1000) {
         this.managers = managers;
         this.lookupServices = lookupServices;
         this.storage = storage;
@@ -71,14 +80,13 @@ export class Engine {
         this.topicAnchorHeaderResolver = topicAnchorHeaderResolver;
         this.basmSyncEnabled = basmSyncEnabled;
         this.unprovenEvictionBlocks = unprovenEvictionBlocks;
+        this.maxLookupResults = maxLookupResults;
+        if (maxLookupResults !== -1 && (!Number.isSafeInteger(maxLookupResults) || maxLookupResults < 1)) {
+            throw new TypeError('maxLookupResults must be -1 or a positive safe integer');
+        }
         // To encourage synchronization of overlay services, the SHIP sync strategy is used by default for all overlay topics, except for 'tm_ship' and 'tm_slap'.
         // For these two topics, any existing trackers are combined with the provided shipTrackers and slapTrackers omitting any duplicates.
-        if (syncConfiguration === undefined) {
-            this.syncConfiguration = {};
-        }
-        else {
-            this.syncConfiguration = syncConfiguration;
-        }
+        this.syncConfiguration ??= {};
         for (const managerName of Object.keys(managers)) {
             if (managerName === 'tm_ship' && this.shipTrackers !== undefined && this.syncConfiguration[managerName] !== false) {
                 // Combine tm_ship trackers with preexisting entries if any
@@ -137,7 +145,7 @@ export class Engine {
             return header.blockHash;
         }
         catch (error) {
-            this.logger.warn(`Unable to resolve BASM block hash for height ${blockHeight}: ${error instanceof Error ? error.message : String(error)}`);
+            this.logger.warn(`Unable to resolve BASM block hash: height=${serializeLogValue(blockHeight)} error=${serializeErrorForLog(error)}`);
             return undefined;
         }
     }
@@ -242,7 +250,7 @@ export class Engine {
         }
         if (toHeight - fromHeight + 1 > DEFAULT_BASM_RANGE_LIMIT) {
             // Bound the work per pass; the next trigger resumes from the new tip.
-            this.logger.warn(`[BASM] capping anchor chain extension for "${topic}" at ${DEFAULT_BASM_RANGE_LIMIT} blocks (requested ${fromHeight}..${toHeight}); will continue on the next pass`);
+            this.logger.warn(`[BASM] capping anchor chain extension: topic=${serializeLogValue(topic)} limit=${serializeLogValue(DEFAULT_BASM_RANGE_LIMIT)} requestedFrom=${serializeLogValue(fromHeight)} requestedTo=${serializeLogValue(toHeight)}; will continue on the next pass`);
             toHeight = fromHeight + DEFAULT_BASM_RANGE_LIMIT - 1;
         }
         const previousAnchor = fromHeight > 0
@@ -256,7 +264,7 @@ export class Engine {
             // canonical re-resolution from the header resolver instead of reusing it.
             const blockHash = blockHashHints.get(height) ?? (forceResolve ? undefined : existing?.blockHash) ?? await this.resolveBlockHash(height);
             if (blockHash === undefined) {
-                this.logger.warn(`[BASM] unable to resolve block hash for "${topic}" at height ${height}; halting chain extension`);
+                this.logger.warn(`[BASM] unable to resolve block hash: topic=${serializeLogValue(topic)} height=${serializeLogValue(height)}; halting chain extension`);
                 return;
             }
             const basmRoot = computeBasmRoot(admitted);
@@ -383,381 +391,325 @@ export class Engine {
             newTipHeight: tip
         });
     }
-    /**
-     * Submits a transaction for processing by Overlay Services.
-     * @param {TaggedBEEF} taggedBEEF - The transaction to process
-     * @param {function(STEAK): void} [onSTEAKReady] - Optional callback function invoked when the STEAK is ready.
-     * @param {string} mode — Indicates the submission behavior, whether historical or current. Historical transactions are not broadcast or propagated.
-     * @param {number[]} offChainValues — Values necessary to evaluate topical admittance that are not stored on-chain.
-     *
-     * The optional callback function should be used to get STEAK when ready, and avoid waiting for broadcast and transaction propagation to complete.
-     *
-     * @returns {Promise<STEAK>} The submitted transaction execution acknowledgement
-     */
-    async submit(taggedBEEF, onSteakReady, mode = 'current-tx', offChainValues) {
-        for (const t of taggedBEEF.topics) {
-            if (this.managers[t] === undefined || this.managers[t] === null) {
-                throw new Error(`This server does not support this topic: ${t}`);
-            }
-        }
-        // Validate the transaction SPV information
-        const tx = Transaction.fromBEEF(taggedBEEF.beef);
-        const txid = tx.id('hex');
-        this.startTime(`submit_${txid}`);
-        if (mode !== 'historical-tx-no-spv') {
-            this.startTime(`chainTracker_${txid.substring(0, 10)}`);
-            const txValid = await tx.verify(this.chainTracker);
-            if (!txValid)
-                throw new Error('Unable to verify SPV information.');
-            this.endTime(`chainTracker_${txid.substring(0, 10)}`);
-        }
-        const steak = {};
-        const dupeTopics = new Set();
-        const failedTopics = new Set();
-        const topicValidations = taggedBEEF.topics.map(async (topic) => {
-            try {
-                if (this.managers[topic] === undefined || this.managers[topic] === null) {
-                    throw new Error(`This server does not support this topic: ${topic}`);
-                }
-                // Check for duplicate transactions
-                this.startTime(`dupCheck_${txid.substring(0, 10)}`);
-                const dupeCheck = await this.storage.doesAppliedTransactionExist({ txid, topic });
-                this.endTime(`dupCheck_${txid.substring(0, 10)}`);
-                if (dupeCheck) {
-                    dupeTopics.add(topic);
-                    return {
-                        topic,
-                        isDupe: true,
-                        previousCoins: [],
-                        previousOutputs: [],
-                        admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
-                    };
-                }
-                // Identify previous coins admitted to this specific topic
-                const previousCoins = [];
-                const outputPromises = tx.inputs.map(async (input, i) => {
-                    const previousTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex');
-                    if (previousTXID !== undefined) {
-                        // Check if the previous output was admitted to this specific topic
-                        const output = await this.storage.findOutput(previousTXID, input.sourceOutputIndex, topic);
-                        if (output !== undefined && output !== null) {
-                            previousCoins.push(i);
-                            return output;
-                        }
-                    }
-                    return null;
-                });
-                this.startTime(`previousOutputQuery_${txid.substring(0, 10)}`);
-                const previousOutputs = await Promise.all(outputPromises);
-                this.endTime(`previousOutputQuery_${txid.substring(0, 10)}`);
-                // Determine which outputs are admissible for this topic (validation only)
-                this.startTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
-                const admissibleOutputs = await this.managers[topic].identifyAdmissibleOutputs(taggedBEEF.beef, previousCoins, offChainValues, mode);
-                this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
-                return {
-                    topic,
-                    isDupe: false,
-                    previousCoins,
-                    previousOutputs,
-                    admissibleOutputs
-                };
-            }
-            catch (error) {
-                this.logger.error('Error validating topic during submit:', error);
-                failedTopics.add(topic);
+    async validateTopicSubmission(topic, context) {
+        const { tx, txid, beef, offChainValues, mode, dupeTopics, failedTopics } = context;
+        try {
+            if (this.managers[topic] === undefined || this.managers[topic] === null) {
+                throw new Error(`This server does not support this topic: ${topic}`);
+            }
+            this.startTime(`dupCheck_${txid.substring(0, 10)}`);
+            const isDupe = await this.storage.doesAppliedTransactionExist({ txid, topic });
+            this.endTime(`dupCheck_${txid.substring(0, 10)}`);
+            if (isDupe) {
+                dupeTopics.add(topic);
                 return {
                     topic,
-                    isDupe: false,
+                    isDupe: true,
                     previousCoins: [],
                     previousOutputs: [],
                     admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
                 };
             }
-        });
-        const validations = await Promise.all(topicValidations);
-        // Build preliminary STEAK from validation results
-        for (const validation of validations) {
-            steak[validation.topic] = validation.admissibleOutputs;
+            const previousCoins = [];
+            const outputPromises = tx.inputs.map(async (input, inputIndex) => {
+                const previousTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex');
+                if (previousTXID === undefined)
+                    return null;
+                const output = await this.storage.findOutput(previousTXID, input.sourceOutputIndex, topic);
+                if (output !== undefined && output !== null)
+                    previousCoins.push(inputIndex);
+                return output ?? null;
+            });
+            this.startTime(`previousOutputQuery_${txid.substring(0, 10)}`);
+            const previousOutputs = await Promise.all(outputPromises);
+            this.endTime(`previousOutputQuery_${txid.substring(0, 10)}`);
+            this.startTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
+            const admissibleOutputs = await this.managers[topic].identifyAdmissibleOutputs(beef, previousCoins, offChainValues, mode);
+            this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
+            return {
+                topic,
+                isDupe: false,
+                previousCoins,
+                previousOutputs,
+                admissibleOutputs
+            };
         }
-        // ===================================================================
-        // PHASE 2: BROADCAST (before any mutations)
-        // ===================================================================
-        // Only broadcast when at least one topic actually accepted the
-        // transaction. For a non-failed topic, acceptance means: previously
-        // accepted (dupe / client retry), outputs admitted, coins retained, or
-        // previously-admitted coins consumed (e.g. a consume-only deletion such
-        // as a KVStore remove, even one that retains nothing). A topic manager
-        // REJECTS by throwing from identifyAdmissibleOutputs (tracked in
-        // failedTopics). A transaction every topic rejected must never reach the
-        // network: submitters treat an empty STEAK as a rejection and
-        // abort/release their held inputs, so broadcasting it anyway would
-        // desync their wallets from the chain.
-        const anyTopicAccepted = validations.some(v => !failedTopics.has(v.topic) && (v.isDupe ||
-            v.admissibleOutputs.outputsToAdmit.length > 0 ||
-            v.admissibleOutputs.coinsToRetain.length > 0 ||
-            v.previousCoins.length > 0));
+        catch (error) {
+            this.logger.error(`Error validating topic during submit: topic=${serializeLogValue(topic)} error=${serializeErrorForLog(error)}`);
+            failedTopics.add(topic);
+            return {
+                topic,
+                isDupe: false,
+                previousCoins: [],
+                previousOutputs: [],
+                admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
+            };
+        }
+    }
+    isTopicSubmissionAccepted(validation, failedTopics) {
+        return !failedTopics.has(validation.topic) && (validation.isDupe ||
+            validation.admissibleOutputs.outputsToAdmit.length > 0 ||
+            validation.admissibleOutputs.coinsToRetain.length > 0 ||
+            validation.previousCoins.length > 0);
+    }
+    async broadcastAcceptedSubmission(tx, txid, mode, anyTopicAccepted) {
         this.startTime(`broadcast_${txid.substring(0, 10)}`);
         if (mode !== 'historical-tx' && this.broadcaster !== undefined && anyTopicAccepted) {
             try {
                 let response;
                 if (tx.merklePath !== undefined) {
-                    // tx has been verified, thus if there is a merklePath, the transaction is already on-chain...skip broadcast.
-                    const txid = tx.id('hex');
                     const mp = tx.merklePath;
                     const leaf = mp.path[0].find(leaf => leaf.hash === txid);
-                    const r = {
+                    response = {
                         status: 'success',
-                        txid: tx.id('hex'),
-                        message: `In block at height ${mp.blockHeight} index ${leaf?.offset}`,
+                        txid,
+                        message: `In block at height ${mp.blockHeight} index ${leaf?.offset}`
                     };
-                    response = r;
                 }
                 else {
                     response = await this.broadcaster.broadcast(tx);
                 }
                 if (isBroadcastFailure(response) && this.throwOnBroadcastFailure) {
-                    const e = new Error(`Failed to broadcast transaction! Error: ${response.description}`);
-                    e.more = response.more;
-                    throw e;
+                    const error = new Error(`Failed to broadcast transaction! Error: ${response.description}`);
+                    error.more = response.more;
+                    throw error;
                 }
             }
             catch (error) {
-                if (this.throwOnBroadcastFailure) {
+                if (this.throwOnBroadcastFailure)
                     throw error;
-                }
                 this.logger.error('Error broadcasting transaction:', error);
             }
         }
         this.endTime(`broadcast_${txid.substring(0, 10)}`);
-        // Call the callback function with STEAK if it is provided (before storage mutations)
-        if (onSteakReady !== undefined) {
-            onSteakReady(steak);
+    }
+    async notifyOutputSpent(lookupService, tx, txid, output, topic, offChainValues) {
+        if (typeof lookupService.outputSpent !== 'function')
+            return;
+        if (lookupService.spendNotificationMode === 'txid') {
+            await lookupService.outputSpent({
+                mode: 'txid',
+                spendingTxid: txid,
+                txid: output.txid,
+                outputIndex: output.outputIndex,
+                topic
+            });
+            return;
         }
-        // ===================================================================
-        // PHASE 3: MUTATE STORAGE (only after broadcast succeeded)
-        // ===================================================================
-        // Mark previous outputs as spent and notify lookup services
+        if (lookupService.spendNotificationMode === 'script') {
+            const inputIndex = findSpendingInputIndex(tx, output);
+            if (inputIndex === -1)
+                throw new Error('Could not find input index');
+            await lookupService.outputSpent({
+                mode: 'script',
+                spendingTxid: txid,
+                inputIndex,
+                sequenceNumber: tx.inputs[inputIndex].sequence ?? 0xffffffff,
+                unlockingScript: tx.inputs[inputIndex].unlockingScript,
+                txid: output.txid,
+                outputIndex: output.outputIndex,
+                topic,
+                offChainValues
+            });
+            return;
+        }
+        if (lookupService.spendNotificationMode === 'whole-tx') {
+            await lookupService.outputSpent({
+                mode: 'whole-tx',
+                spendingAtomicBEEF: tx.toAtomicBEEF(),
+                txid: output.txid,
+                outputIndex: output.outputIndex,
+                topic,
+                offChainValues
+            });
+            return;
+        }
+        await lookupService.outputSpent({
+            mode: 'none',
+            txid: output.txid,
+            outputIndex: output.outputIndex,
+            topic
+        });
+    }
+    async markPreviousOutputSpent(output, topic, tx, txid, offChainValues) {
+        if (output === null)
+            return;
+        try {
+            await this.storage.markUTXOAsSpent(output.txid, output.outputIndex, topic);
+            await Promise.all(Object.values(this.lookupServices).map(async (lookupService) => {
+                try {
+                    await this.notifyOutputSpent(lookupService, tx, txid, output, topic, offChainValues);
+                }
+                catch (error) {
+                    this.logger.error('Error in lookup service for outputSpent:', error);
+                }
+            }));
+        }
+        catch (error) {
+            this.logger.error('Error marking UTXO as spent:', error);
+        }
+    }
+    async markPreviousOutputsSpent(validations, failedTopics, tx, txid, offChainValues) {
         await Promise.all(validations.map(async (validation) => {
-            if (validation.isDupe || failedTopics.has(validation.topic)) {
+            if (validation.isDupe || failedTopics.has(validation.topic))
                 return;
+            await Promise.all(validation.previousOutputs.map(async (output) => {
+                await this.markPreviousOutputSpent(output, validation.topic, tx, txid, offChainValues);
+            }));
+        }));
+    }
+    classifyPreviousCoins(tx, validation) {
+        const outputsConsumed = [];
+        const outputsToMarkStale = [];
+        for (const inputIndex of validation.previousCoins) {
+            const input = tx.inputs[inputIndex];
+            const previousTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex');
+            if (typeof previousTXID !== 'string')
+                continue;
+            if (validation.admissibleOutputs.coinsToRetain.includes(inputIndex)) {
+                outputsConsumed.push({
+                    txid: previousTXID,
+                    outputIndex: input.sourceOutputIndex
+                });
             }
-            const topic = validation.topic;
-            const previousOutputs = validation.previousOutputs;
-            // Mark all previous outputs as spent
-            const markSpentPromises = previousOutputs.map(async (output) => {
-                if (output !== undefined && output !== null) {
-                    try {
-                        await this.storage.markUTXOAsSpent(output.txid, output.outputIndex, topic);
-                        await Promise.all(Object.values(this.lookupServices).map(async (l) => {
-                            try {
-                                if (typeof l.outputSpent === 'function') {
-                                    if (l.spendNotificationMode === 'txid') {
-                                        await l.outputSpent({
-                                            mode: 'txid',
-                                            spendingTxid: txid,
-                                            txid: output.txid,
-                                            outputIndex: output.outputIndex,
-                                            topic
-                                        });
-                                    }
-                                    else if (l.spendNotificationMode === 'script') {
-                                        const inputIndex = tx.inputs.findIndex(i => {
-                                            let realSource = i.sourceTXID;
-                                            if (!realSource) {
-                                                realSource = i.sourceTransaction?.id('hex');
-                                            }
-                                            return realSource === output.txid && i.sourceOutputIndex === output.outputIndex;
-                                        });
-                                        if (inputIndex === -1) {
-                                            throw new Error('Could not find input index');
-                                        }
-                                        await l.outputSpent({
-                                            mode: 'script',
-                                            spendingTxid: txid,
-                                            inputIndex,
-                                            sequenceNumber: tx.inputs[inputIndex].sequence ?? 0xffffffff,
-                                            unlockingScript: tx.inputs[inputIndex].unlockingScript,
-                                            txid: output.txid,
-                                            outputIndex: output.outputIndex,
-                                            topic,
-                                            offChainValues
-                                        });
-                                    }
-                                    else if (l.spendNotificationMode === 'whole-tx') {
-                                        await l.outputSpent({
-                                            mode: 'whole-tx',
-                                            spendingAtomicBEEF: tx.toAtomicBEEF(),
-                                            txid: output.txid,
-                                            outputIndex: output.outputIndex,
-                                            topic,
-                                            offChainValues
-                                        });
-                                    }
-                                    else { // none
-                                        await l.outputSpent({
-                                            mode: 'none',
-                                            txid: output.txid,
-                                            outputIndex: output.outputIndex,
-                                            topic
-                                        });
-                                    }
-                                }
-                            }
-                            catch (error) {
-                                this.logger.error('Error in lookup service for outputSpent:', error);
-                            }
-                        }));
-                    }
-                    catch (error) {
-                        this.logger.error('Error marking UTXO as spent:', error);
-                    }
-                }
+            else {
+                outputsToMarkStale.push({
+                    txid: previousTXID,
+                    previousOutputIndex: input.sourceOutputIndex,
+                    inputIndex
+                });
+            }
+        }
+        return { outputsConsumed, outputsToMarkStale };
+    }
+    async removeStaleOutputs(outputs, topic, txid) {
+        this.startTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
+        await Promise.all(outputs.map(async (coin) => {
+            const output = await this.storage.findOutput(coin.txid, coin.previousOutputIndex, topic);
+            if (output !== undefined && output !== null)
+                await this.deleteUTXODeep(output);
+        }));
+        this.endTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
+    }
+    async notifyOutputAdmitted(lookupService, tx, txid, outputIndex, topic, offChainValues) {
+        if (lookupService.admissionMode === 'locking-script') {
+            if (typeof tx.outputs[outputIndex].lockingScript !== 'object' ||
+                typeof tx.outputs[outputIndex].satoshis !== 'number')
+                return;
+            await lookupService.outputAdmittedByTopic({
+                mode: 'locking-script',
+                txid,
+                outputIndex,
+                lockingScript: tx.outputs[outputIndex].lockingScript,
+                satoshis: tx.outputs[outputIndex].satoshis,
+                topic,
+                offChainValues
+            });
+            return;
+        }
+        await lookupService.outputAdmittedByTopic({
+            mode: 'whole-tx',
+            atomicBEEF: tx.toAtomicBEEF(),
+            outputIndex,
+            topic,
+            offChainValues
+        });
+    }
+    async admitOutput(outputIndex, context) {
+        const { tx, txid, beef, topic, outputsConsumed, newUTXOs, offChainValues } = context;
+        if (typeof tx.outputs[outputIndex].satoshis !== 'number')
+            return;
+        this.startTime(`insertNewOutput_${txid.substring(0, 10)}`);
+        await this.storage.insertOutput({
+            txid,
+            outputIndex,
+            outputScript: tx.outputs[outputIndex].lockingScript.toBinary(),
+            satoshis: tx.outputs[outputIndex].satoshis,
+            topic,
+            spent: false,
+            beef: this.compactBEEFForStorage(tx, beef),
+            consumedBy: [],
+            outputsConsumed,
+            score: Date.now(),
+            blockHeight: extractMerkleProofMetadata(txid, tx.merklePath)?.blockHeight
+        });
+        this.endTime(`insertNewOutput_${txid.substring(0, 10)}`);
+        newUTXOs.push({ txid, outputIndex });
+        this.startTime(`notifyLookupService${txid.substring(0, 10)}`);
+        await Promise.all(Object.values(this.lookupServices).map(async (lookupService) => {
+            try {
+                await this.notifyOutputAdmitted(lookupService, tx, txid, outputIndex, topic, offChainValues);
+            }
+            catch (error) {
+                this.logger.error('Error in lookup service for outputAdmittedByTopic:', error);
+            }
+        }));
+        this.endTime(`notifyLookupService${txid.substring(0, 10)}`);
+    }
+    async updateConsumedOutput(output, newUTXOs, topic) {
+        const storedOutput = await this.storage.findOutput(output.txid, output.outputIndex, topic);
+        if (storedOutput === undefined || storedOutput === null)
+            return;
+        const consumedBy = [...new Set([...newUTXOs, ...storedOutput.consumedBy])];
+        await this.storage.updateConsumedBy(output.txid, output.outputIndex, topic, consumedBy);
+    }
+    async applyTopicStorageMutation(validation, steak, tx, txid, beef, offChainValues) {
+        const topic = validation.topic;
+        const { outputsConsumed, outputsToMarkStale } = this.classifyPreviousCoins(tx, validation);
+        await this.removeStaleOutputs(outputsToMarkStale, topic, txid);
+        steak[topic].coinsRemoved = outputsToMarkStale.map(output => output.inputIndex);
+        const newUTXOs = [];
+        await Promise.all(validation.admissibleOutputs.outputsToAdmit.map(async (outputIndex) => {
+            await this.admitOutput(outputIndex, {
+                tx,
+                txid,
+                beef,
+                topic,
+                outputsConsumed,
+                newUTXOs,
+                offChainValues
             });
-            await Promise.all(markSpentPromises);
         }));
-        // Continue with storage updates and lookup service notifications
+        this.startTime(`outputConsumed_${txid.substring(0, 10)}`);
+        const appliedRecord = await this.buildAppliedTransactionRecord(tx);
+        await this.recordTransactionData(tx, beef, appliedRecord.blockHash);
+        await Promise.all([
+            ...outputsConsumed.map(async (output) => {
+                await this.updateConsumedOutput(output, newUTXOs, topic);
+            }),
+            this.storage.insertAppliedTransaction({ txid, topic, ...appliedRecord })
+        ]);
+        if (appliedRecord.blockHeight !== undefined && appliedRecord.blockHash !== undefined) {
+            await this.recomputeTopicBlockAnchor(topic, appliedRecord.blockHeight, appliedRecord.blockHash);
+        }
+        this.endTime(`outputConsumed_${txid.substring(0, 10)}`);
+    }
+    async applyStorageMutations(validations, context) {
+        const { dupeTopics, failedTopics, steak, tx, txid, beef, offChainValues } = context;
         for (const validation of validations) {
             const topic = validation.topic;
-            if (dupeTopics.has(topic)) {
-                continue;
-            }
-            if (failedTopics.has(topic)) {
+            if (dupeTopics.has(topic) || failedTopics.has(topic))
                 continue;
-            }
             try {
-                const admissibleOutputs = steak[topic];
-                const outputsToAdmit = admissibleOutputs.outputsToAdmit;
-                const outputsConsumed = [];
-                const outputsToMarkStale = [];
-                // Use previousCoins from validation
-                const previousCoins = validation.previousCoins;
-                // For each of the previous UTXOs for this topic, if the UTXO was not included in the list of UTXOs identified for retention, then it will be marked as stale.
-                for (const inputIndex of previousCoins) {
-                    const previousTXID = tx.inputs[inputIndex].sourceTXID ?? tx.inputs[inputIndex].sourceTransaction?.id('hex');
-                    if (typeof previousTXID !== 'string')
-                        continue;
-                    const previousOutputIndex = tx.inputs[inputIndex].sourceOutputIndex;
-                    if (admissibleOutputs.coinsToRetain.includes(inputIndex)) {
-                        outputsConsumed.push({
-                            txid: previousTXID,
-                            outputIndex: previousOutputIndex
-                        });
-                    }
-                    else {
-                        outputsToMarkStale.push({
-                            txid: previousTXID,
-                            previousOutputIndex,
-                            inputIndex
-                        });
-                    }
-                }
-                // Remove stale outputs recursively
-                this.startTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
-                await Promise.all(outputsToMarkStale.map(async (coin) => {
-                    const output = await this.storage.findOutput(coin.txid, coin.previousOutputIndex, topic);
-                    if (output !== undefined && output !== null) {
-                        await this.deleteUTXODeep(output);
-                    }
-                }));
-                this.endTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
-                // Update the STEAK to indicate which coins were removed
-                steak[topic].coinsRemoved = outputsToMarkStale.map(x => x.inputIndex);
-                // Handle admittance and notification of incoming UTXOs
-                const newUTXOs = [];
-                await Promise.all(outputsToAdmit.map(async (outputIndex) => {
-                    if (typeof tx.outputs[outputIndex].satoshis !== 'number')
-                        return;
-                    this.startTime(`insertNewOutput_${txid.substring(0, 10)}`);
-                    await this.storage.insertOutput({
-                        txid,
-                        outputIndex,
-                        outputScript: tx.outputs[outputIndex].lockingScript.toBinary(),
-                        satoshis: tx.outputs[outputIndex].satoshis,
-                        topic,
-                        spent: false,
-                        beef: this.compactBEEFForStorage(tx, taggedBEEF.beef),
-                        consumedBy: [],
-                        outputsConsumed,
-                        score: Date.now(),
-                        blockHeight: extractMerkleProofMetadata(txid, tx.merklePath)?.blockHeight
-                    });
-                    this.endTime(`insertNewOutput_${txid.substring(0, 10)}`);
-                    newUTXOs.push({ txid, outputIndex });
-                    this.startTime(`notifyLookupService${txid.substring(0, 10)}`);
-                    await Promise.all(Object.values(this.lookupServices).map(async (l) => {
-                        try {
-                            if (l.admissionMode === 'locking-script') {
-                                if (typeof tx.outputs[outputIndex].lockingScript !== 'object' ||
-                                    typeof tx.outputs[outputIndex].satoshis !== 'number') {
-                                    return;
-                                }
-                                await l.outputAdmittedByTopic({
-                                    mode: 'locking-script',
-                                    txid,
-                                    outputIndex,
-                                    lockingScript: tx.outputs[outputIndex].lockingScript,
-                                    satoshis: tx.outputs[outputIndex].satoshis,
-                                    topic,
-                                    offChainValues
-                                });
-                            }
-                            else {
-                                await l.outputAdmittedByTopic({
-                                    mode: 'whole-tx',
-                                    atomicBEEF: tx.toAtomicBEEF(),
-                                    outputIndex,
-                                    topic,
-                                    offChainValues
-                                });
-                            }
-                        }
-                        catch (error) {
-                            this.logger.error('Error in lookup service for outputAdmittedByTopic:', error);
-                        }
-                    }));
-                    this.endTime(`notifyLookupService${txid.substring(0, 10)}`);
-                }));
-                this.startTime(`outputConsumed_${txid.substring(0, 10)}`);
-                // Update each output consumed to know who consumed it and insert applied transaction in parallel
-                const appliedRecord = await this.buildAppliedTransactionRecord(tx);
-                await this.recordTransactionData(tx, taggedBEEF.beef, appliedRecord.blockHash);
-                await Promise.all([
-                    ...outputsConsumed.map(async (output) => {
-                        const outputToUpdate = await this.storage.findOutput(output.txid, output.outputIndex, topic);
-                        if (outputToUpdate !== undefined && outputToUpdate !== null) {
-                            const newConsumedBy = [...new Set([...newUTXOs, ...outputToUpdate.consumedBy])];
-                            await this.storage.updateConsumedBy(output.txid, output.outputIndex, topic, newConsumedBy);
-                        }
-                    }),
-                    this.storage.insertAppliedTransaction({
-                        txid,
-                        topic,
-                        ...appliedRecord
-                    })
-                ]);
-                if (appliedRecord.blockHeight !== undefined && appliedRecord.blockHash !== undefined) {
-                    await this.recomputeTopicBlockAnchor(topic, appliedRecord.blockHeight, appliedRecord.blockHash);
-                }
-                this.endTime(`outputConsumed_${txid.substring(0, 10)}`);
+                await this.applyTopicStorageMutation(validation, steak, tx, txid, beef, offChainValues);
             }
             catch (error) {
                 this.logger.error('Error updating storage and notifying lookup services for topic', topic, error);
             }
         }
-        // If we don't have an advertiser or we are dealing with historical transactions, just return the steak
-        if (this.advertiser === undefined || mode === 'historical-tx' || mode === 'historical-tx-no-spv') {
-            return steak;
-        }
+    }
+    async propagateSubmission(taggedBEEF, steak, dupeTopics, tx, txid) {
         this.startTime(`transactionPropagation_${txid.substring(0, 10)}`);
-        const relevantTopics = taggedBEEF.topics.filter(topic => steak[topic] !== undefined && !dupeTopics.has(topic) && (steak[topic].outputsToAdmit.length !== 0 || steak[topic].coinsRemoved?.length !== 0));
+        const relevantTopics = taggedBEEF.topics.filter(topic => steak[topic] !== undefined &&
+            !dupeTopics.has(topic) &&
+            (steak[topic].outputsToAdmit.length !== 0 ||
+                steak[topic].coinsRemoved?.length !== 0));
         if (relevantTopics.length === 0) {
             this.endTime(`transactionPropagation_${txid.substring(0, 10)}`);
-            return steak;
+            return;
         }
-        // Create a SHIPBroadcaster instance
         let customBroadcasterConfig;
         if (Array.isArray(this.slapTrackers)) {
-            // Custom SLAP trackers warrant a custom broadcaster config
             const resolverConfig = {
                 slapTrackers: this.slapTrackers
             };
@@ -773,6 +725,93 @@ export class Engine {
             this.logger.error('Error during propagation to other nodes:', error);
         }
         this.endTime(`transactionPropagation_${txid.substring(0, 10)}`);
+    }
+    /**
+     * Submits a transaction for processing by Overlay Services.
+     * @param {TaggedBEEF} taggedBEEF - The transaction to process
+     * @param {function(STEAK): void} [onSTEAKReady] - Optional callback function invoked when the STEAK is ready.
+     * @param {string} mode — Indicates the submission behavior, whether historical or current. Historical transactions are not broadcast or propagated.
+     * @param {number[]} offChainValues — Values necessary to evaluate topical admittance that are not stored on-chain.
+     *
+     * The optional callback function should be used to get STEAK when ready, and avoid waiting for broadcast and transaction propagation to complete.
+     *
+     * @returns {Promise<STEAK>} The submitted transaction execution acknowledgement
+     */
+    async submit(taggedBEEF, onSteakReady, mode = 'current-tx', offChainValues) {
+        for (const t of taggedBEEF.topics) {
+            if (this.managers[t] === undefined || this.managers[t] === null) {
+                throw new Error(`This server does not support this topic: ${t}`);
+            }
+        }
+        // Validate the transaction SPV information
+        const tx = Transaction.fromBEEF(taggedBEEF.beef);
+        const txid = tx.id('hex');
+        this.startTime(`submit_${txid}`);
+        if (mode !== 'historical-tx-no-spv') {
+            this.startTime(`chainTracker_${txid.substring(0, 10)}`);
+            const txValid = await tx.verify(this.chainTracker);
+            if (!txValid)
+                throw new Error('Unable to verify SPV information.');
+            this.endTime(`chainTracker_${txid.substring(0, 10)}`);
+        }
+        const steak = {};
+        const dupeTopics = new Set();
+        const failedTopics = new Set();
+        // ===================================================================
+        // PHASE 1: VALIDATE (read-only, no mutations)
+        // ===================================================================
+        const topicValidations = taggedBEEF.topics.map(async (topic) => await this.validateTopicSubmission(topic, {
+            tx,
+            txid,
+            beef: taggedBEEF.beef,
+            offChainValues,
+            mode,
+            dupeTopics,
+            failedTopics
+        }));
+        const validations = await Promise.all(topicValidations);
+        // Build preliminary STEAK from validation results
+        for (const validation of validations) {
+            steak[validation.topic] = validation.admissibleOutputs;
+        }
+        // ===================================================================
+        // PHASE 2: BROADCAST (before any mutations)
+        // ===================================================================
+        // Only broadcast when at least one topic actually accepted the
+        // transaction. For a non-failed topic, acceptance means: previously
+        // accepted (dupe / client retry), outputs admitted, coins retained, or
+        // previously-admitted coins consumed (e.g. a consume-only deletion such
+        // as a KVStore remove, even one that retains nothing). A topic manager
+        // REJECTS by throwing from identifyAdmissibleOutputs (tracked in
+        // failedTopics). A transaction every topic rejected must never reach the
+        // network: submitters treat an empty STEAK as a rejection and
+        // abort/release their held inputs, so broadcasting it anyway would
+        // desync their wallets from the chain.
+        const anyTopicAccepted = validations.some(validation => this.isTopicSubmissionAccepted(validation, failedTopics));
+        await this.broadcastAcceptedSubmission(tx, txid, mode, anyTopicAccepted);
+        // Call the callback function with STEAK if it is provided (before storage mutations)
+        if (onSteakReady !== undefined) {
+            onSteakReady(steak);
+        }
+        // ===================================================================
+        // PHASE 3: MUTATE STORAGE (only after broadcast succeeded)
+        // ===================================================================
+        // Mark previous outputs as spent and notify lookup services
+        await this.markPreviousOutputsSpent(validations, failedTopics, tx, txid, offChainValues);
+        await this.applyStorageMutations(validations, {
+            dupeTopics,
+            failedTopics,
+            steak,
+            tx,
+            txid,
+            beef: taggedBEEF.beef,
+            offChainValues
+        });
+        // If we don't have an advertiser or we are dealing with historical transactions, just return the steak
+        if (this.advertiser === undefined || mode === 'historical-tx' || mode === 'historical-tx-no-spv') {
+            return steak;
+        }
+        await this.propagateSubmission(taggedBEEF, steak, dupeTopics, tx, txid);
         // Immediately return from the function without waiting for the promises to resolve.
         return steak;
     }
@@ -787,6 +826,9 @@ export class Engine {
         if (lookupService === undefined || lookupService === null)
             throw new Error(`Lookup service not found for provider: ${lookupQuestion.service}`);
         const lookupResult = await lookupService.lookup(lookupQuestion);
+        if (this.maxLookupResults !== -1 && lookupResult.length > this.maxLookupResults) {
+            throw new RangeError(`Lookup returned ${lookupResult.length} results; maximum is ${this.maxLookupResults}`);
+        }
         const hydrationContext = this.createUTXOHistoryHydrationContext();
         await this.preloadOutputsWithBEEF(lookupResult.map(({ txid, outputIndex }) => ({ txid, outputIndex })), hydrationContext);
         const hydratedOutputs = (await Promise.all(lookupResult.map(async ({ txid, outputIndex, history, context }) => {
@@ -1004,71 +1046,37 @@ export class Engine {
             throw new Error('Overlay Service Engine not configured for topical synchronization!');
         }
         for (const topic of Object.keys(this.syncConfiguration)) {
-            // Make sure syncEndpoints is an array or SHIP
-            let syncEndpoints = this.syncConfiguration[topic];
-            // Check if this topic has been configured NOT to sync
-            if (syncEndpoints === false) {
+            const configuredEndpoints = this.syncConfiguration[topic];
+            if (configuredEndpoints === false)
                 continue;
+            if (!Array.isArray(configuredEndpoints) && configuredEndpoints !== 'SHIP')
+                continue;
+            const syncEndpoints = await this.resolveSyncEndpointsForTopic(topic, 'Failed to parse advertisement output:');
+            this.logger.info(`[GASP SYNC] Will attempt to sync with ${syncEndpoints.length} peer${syncEndpoints.length === 1 ? '' : 's'}`);
+            // Sync with each endpoint sequentially to avoid parallel locks while
+            // keeping peer failures isolated.
+            for (const endpoint of syncEndpoints) {
+                await this.syncGASPWithPeer(topic, endpoint);
             }
-            if (syncEndpoints === 'SHIP') {
-                // Perform lookup and find ship advertisements to set syncEndpoints for topic
-                const resolverConfig = this.slapTrackers
-                    ? { slapTrackers: this.slapTrackers }
-                    : {};
-                const resolver = new LookupResolver(resolverConfig);
-                const lookupAnswer = await resolver.query({
-                    service: 'ls_ship',
-                    query: {
-                        topics: [topic]
-                    }
-                });
-                // Lookup will currently always return type output-list
-                if (lookupAnswer.type === 'output-list') {
-                    const endpointSet = new Set();
-                    lookupAnswer.outputs.forEach(output => {
-                        try {
-                            // Parse out the advertisements using the provided parser
-                            const tx = Transaction.fromBEEF(output.beef);
-                            const advertisement = this.advertiser?.parseAdvertisement(tx.outputs[output.outputIndex].lockingScript);
-                            if (advertisement?.protocol === 'SHIP') {
-                                endpointSet.add(advertisement.domain);
-                            }
-                        }
-                        catch (error) {
-                            this.logger.error('Failed to parse advertisement output:', error);
-                        }
-                    });
-                    syncEndpoints = Array.from(endpointSet);
-                }
-            }
-            // Now syncEndpoints is guaranteed to be an array of strings without duplicates
-            if (Array.isArray(syncEndpoints)) {
-                // Remove our own hosting URL so we don't sync with ourselves
-                syncEndpoints = syncEndpoints.filter((endpoint) => endpoint !== this.hostingURL);
-                this.logger.info(`[GASP SYNC] Will attempt to sync with ${syncEndpoints.length} peer${syncEndpoints.length === 1 ? '' : 's'}`);
-                // Sync with each endpoint individually to avoid parallel locks and let failures be isolated
-                for (const endpoint of syncEndpoints) {
-                    this.logger.info(`[GASP SYNC] Starting sync for topic "${topic}" with peer "${endpoint}"`);
-                    try {
-                        // Read the last interaction score from storage
-                        const lastInteraction = await this.storage.getLastInteraction(endpoint, topic);
-                        const gasp = new GASP(new OverlayGASPStorage(topic, this), new OverlayGASPRemote(endpoint, topic), lastInteraction, `[GASP Sync of ${topic} with ${endpoint}]`, true, true);
-                        await gasp.sync(endpoint, DEFAULT_GASP_SYNC_LIMIT);
-                        // Save the updated last interaction score
-                        if (gasp.lastInteraction > lastInteraction) {
-                            await this.storage.updateLastInteraction(endpoint, topic, gasp.lastInteraction);
-                        }
-                        this.logger.info(`[GASP SYNC] Sync successful for topic "${topic}" with peer "${endpoint}"`);
-                    }
-                    catch (err) {
-                        this.logger.error(`[GASP SYNC] Sync failed for topic "${topic}" with peer "${endpoint}"`, err);
-                        // Continue on to the next endpoint without throwing
-                    }
-                }
+        }
+    }
+    async syncGASPWithPeer(topic, endpoint) {
+        this.logger.info(`[GASP SYNC] Starting sync for topic "${topic}" with peer "${endpoint}"`);
+        try {
+            const lastInteraction = await this.storage.getLastInteraction(endpoint, topic);
+            const gasp = new GASP(new OverlayGASPStorage(topic, this), new OverlayGASPRemote(endpoint, topic), lastInteraction, `[GASP Sync of ${topic} with ${endpoint}]`, true, true);
+            await gasp.sync(endpoint, DEFAULT_GASP_SYNC_LIMIT);
+            if (gasp.lastInteraction > lastInteraction) {
+                await this.storage.updateLastInteraction(endpoint, topic, gasp.lastInteraction);
             }
+            this.logger.info(`[GASP SYNC] Sync successful for topic "${topic}" with peer "${endpoint}"`);
+        }
+        catch (error) {
+            this.logger.error(`[GASP SYNC] Sync failed for topic "${topic}" with peer "${endpoint}"`, error);
+            // Continue on to the next endpoint without throwing.
         }
     }
-    async resolveSyncEndpointsForTopic(topic) {
+    async resolveSyncEndpointsForTopic(topic, advertisementErrorMessage = 'Failed to parse BASM advertisement output:') {
         if (this.syncConfiguration === undefined) {
             return [];
         }
@@ -1098,7 +1106,7 @@ export class Engine {
                         }
                     }
                     catch (error) {
-                        this.logger.error('Failed to parse BASM advertisement output:', error);
+                        this.logger.error(advertisementErrorMessage, error);
                     }
                 });
             }
@@ -1805,7 +1813,7 @@ export class Engine {
             // If none of the disallowed conditions matched, the URL is valid
             return true;
         }
-        catch (_e) {
+        catch {
             // URL constructor throws on malformed input — not a valid URL, return false
             return false;
         }