@bsv/overlay 0.1.7 → 0.1.9

package/dist/cjs/src/Engine.js

@@ -1,8 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.OverlayGASPStorage = exports.OverlayGASPRemote = exports.Engine = void 0;
+exports.Engine = void 0;
 const sdk_1 = require("@bsv/sdk");
 const gasp_1 = require("@bsv/gasp");
+const OverlayGASPRemote_js_1 = require("./GASP/OverlayGASPRemote.js");
+const OverlayGASPStorage_js_1 = require("./GASP/OverlayGASPStorage.js");
 /**
  * Am engine for running BSV Overlay Services (topic managers and lookup services).
  */
@@ -18,9 +20,11 @@ class Engine {
      * @param {Advertiser} [Advertiser] - handles SHIP and SLAP advertisements for peer-discovery
      * @param {string} shipTrackers - SHIP domains we know to bootstrap the system
      * @param {string} slapTrackers - SLAP domains we know to bootstrap the system
-     * @param {Record<string, string[] | 'SHIP'>} syncConfiguration — Configuration object describing historical synchronization of topics.
+     * @param {SyncConfiguration} syncConfiguration — Configuration object describing historical synchronization of topics.
+     * @param {boolean} logTime - Enables / disables the timing logs for various operations in the Overlay submit route.
+     * @param {string} logPrefix - Supports overriding the log prefix with a custom string.
      */
-    constructor(managers, lookupServices, storage, chainTracker, hostingURL, shipTrackers, slapTrackers, broadcaster, advertiser, syncConfiguration) {
+    constructor(managers, lookupServices, storage, chainTracker, hostingURL, shipTrackers, slapTrackers, broadcaster, advertiser, syncConfiguration, logTime = false, logPrefix = '[OVERLAY_ENGINE] ') {
         this.managers = managers;
         this.lookupServices = lookupServices;
         this.storage = storage;
@@ -31,14 +35,52 @@ class Engine {
         this.broadcaster = broadcaster;
         this.advertiser = advertiser;
         this.syncConfiguration = syncConfiguration;
-        // To encourage synchronization of overlay services, the SHIP sync strategy is used by default for all topics.
+        this.logTime = logTime;
+        this.logPrefix = logPrefix;
+        // To encourage synchronization of overlay services, the SHIP sync strategy is used by default for all overlay topics, except for 'tm_ship' and 'tm_slap'.
+        // For these two topics, any existing trackers are combined with the provided shipTrackers and slapTrackers omitting any duplicates.
         if (syncConfiguration === undefined) {
             this.syncConfiguration = {};
-            for (const managerName of Object.keys(managers)) {
-                this.syncConfiguration[managerName] = 'SHIP';
+        }
+        else {
+            this.syncConfiguration = syncConfiguration;
+        }
+        for (const managerName of Object.keys(managers)) {
+            if (managerName === 'tm_ship' && this.shipTrackers !== undefined && this.syncConfiguration[managerName] !== false) {
+                // Combine tm_ship trackers with preexisting entries if any
+                const combinedSet = new Set([
+                    ...(Array.isArray(this.syncConfiguration[managerName]) ? this.syncConfiguration[managerName] : []),
+                    ...this.shipTrackers
+                ]);
+                this.syncConfiguration[managerName] = Array.from(combinedSet);
+            }
+            else if (managerName === 'tm_slap' && this.slapTrackers !== undefined && this.syncConfiguration[managerName] !== false) {
+                // Combine tm_slap trackers with preexisting entries if any
+                const combinedSet = new Set([
+                    ...(Array.isArray(this.syncConfiguration[managerName]) ? this.syncConfiguration[managerName] : []),
+                    ...this.slapTrackers
+                ]);
+                this.syncConfiguration[managerName] = Array.from(combinedSet);
+            }
+            else {
+                // Set undefined managers to 'SHIP' by default
+                if (this.syncConfiguration[managerName] === undefined) {
+                    this.syncConfiguration[managerName] = 'SHIP';
+                }
             }
         }
     }
+    // Helper functions for logging timings
+    startTime(label) {
+        if (this.logTime) {
+            console.time(`${this.logPrefix} ${label}`);
+        }
+    }
+    endTime(label) {
+        if (this.logTime) {
+            console.timeEnd(`${this.logPrefix} ${label}`);
+        }
+    }
     /**
      * Submits a transaction for processing by Overlay Services.
      * @param {TaggedBEEF} taggedBEEF - The transaction to process
@@ -50,7 +92,7 @@ class Engine {
      * @returns {Promise<STEAK>} The submitted transaction execution acknowledgement
      */
     async submit(taggedBEEF, onSteakReady, mode = 'current-tx') {
-        var _a, _b;
+        var _a;
         for (const t of taggedBEEF.topics) {
             if (this.managers[t] === undefined || this.managers[t] === null) {
                 throw new Error(`This server does not support this topic: ${t}`);
@@ -59,70 +101,95 @@ class Engine {
         // Validate the transaction SPV information
         const tx = sdk_1.Transaction.fromBEEF(taggedBEEF.beef);
         const txid = tx.id('hex');
-        const txValid = await tx.verify(this.chainTracker); // Note: also verifying historical-tx with SPV. Needed?
+        this.startTime(`submit_${txid}`);
+        this.startTime(`chainTracker_${txid.substring(0, 10)}`);
+        const txValid = await tx.verify(this.chainTracker);
         if (!txValid)
             throw new Error('Unable to verify SPV information.');
-        // Broadcast the transaction if not historical and broadcaster is configured
-        if (mode !== 'historical-tx' && this.broadcaster !== undefined) {
-            const response = await this.broadcaster.broadcast(tx);
-            if ((0, sdk_1.isBroadcastFailure)(response)) {
-                throw new Error(`Failed to broadcast transaction! Error: ${response.description}`);
-            }
-        }
-        // Find UTXOs belonging to a particular topic
+        this.endTime(`chainTracker_${txid.substring(0, 10)}`);
         const steak = {};
-        for (const topic of taggedBEEF.topics) {
-            // Ensure transaction is not already applied to the topic
-            const dupeCheck = await this.storage.doesAppliedTransactionExist({
-                txid,
-                topic
-            });
+        let admissableOutputs = { outputsToAdmit: [], coinsToRetain: [] };
+        const previousCoins = [];
+        // Parallelize the topic processing
+        const topicPromises = taggedBEEF.topics.map(async (topic) => {
+            if (this.managers[topic] === undefined || this.managers[topic] === null) {
+                throw new Error(`This server does not support this topic: ${topic}`);
+            }
+            // Check for duplicate transactions
+            this.startTime(`dupCheck_${txid.substring(0, 10)}`);
+            const dupeCheck = await this.storage.doesAppliedTransactionExist({ txid, topic });
+            this.endTime(`dupCheck_${txid.substring(0, 10)}`);
             if (dupeCheck) {
-                // The transaction was already processed.
-                // Currently, NO OUTPUTS ARE ADMITTED FOR DUPLICATE TRANSACTIONS.
-                // An alternative decision, one that was decided against, would be to act as if the operation was successful: looking up and returning the list of admitted outputs from when the transaction was originally processed.
-                // This was decided against, because we don't want to encourage unnecessary flooding of duplicative transactions to overlay services.
-                steak[topic] = {
-                    outputsToAdmit: [],
-                    coinsToRetain: []
-                };
-                continue;
+                steak[topic] = { outputsToAdmit: [], coinsToRetain: [] };
+                return;
             }
-            // Check if any input of this transaction is a previous UTXO, adding previous UTXOs to the list
-            const previousCoins = [];
-            for (const [i, input] of tx.inputs.entries()) {
-                const previousTXID = input.sourceTXID || ((_a = input.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex'));
-                // Check if a previous UTXO exists in the storage medium
-                const output = await this.storage.findOutput(previousTXID, input.sourceOutputIndex, topic);
+            // Check if any input of this transaction is a previous UTXO
+            const outputPromises = tx.inputs.map(async (input, i) => {
+                var _a;
+                const previousTXID = input.sourceTXID !== undefined ? input.sourceTXID : (_a = input.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex');
+                if (previousTXID !== undefined) {
+                    const output = this.storage.findOutput(previousTXID, input.sourceOutputIndex, topic);
+                    if (output !== undefined && output !== null) {
+                        previousCoins.push(i);
+                        return await Promise.resolve(output);
+                    }
+                }
+                return await Promise.resolve(null);
+            });
+            this.startTime(`previousOutputQuery_${txid.substring(0, 10)}`);
+            const outputs = await Promise.all(outputPromises);
+            this.endTime(`previousOutputQuery_${txid.substring(0, 10)}`);
+            const markSpentPromises = outputs.map(async (output) => {
                 if (output !== undefined && output !== null) {
-                    previousCoins.push(i);
-                    // This output is now spent.
-                    await this.storage.markUTXOAsSpent(output.txid, output.outputIndex, topic);
-                    // Notify the lookup services about the spending of this output
-                    for (const l of Object.values(this.lookupServices)) {
-                        try {
-                            if (l.outputSpent !== undefined && l.outputSpent !== null) {
-                                await l.outputSpent(output.txid, output.outputIndex, topic);
+                    try {
+                        await this.storage.markUTXOAsSpent(output.txid, output.outputIndex, topic);
+                        await Promise.all(Object.values(this.lookupServices).map(async (l) => {
+                            var _a;
+                            try {
+                                await ((_a = l.outputSpent) === null || _a === void 0 ? void 0 : _a.call(l, output.txid, output.outputIndex, topic));
                             }
-                        }
-                        catch (_) { }
+                            catch (error) {
+                                console.error('Error in lookup service for outputSpent:', error);
+                            }
+                        }));
+                    }
+                    catch (error) {
+                        console.error('Error marking UTXO as spent:', error);
                     }
                 }
+            });
+            // Determine which outputs are admissible
+            const admissibleOutputsPromise = (async () => {
+                try {
+                    this.startTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
+                    admissableOutputs = await this.managers[topic].identifyAdmissibleOutputs(taggedBEEF.beef, previousCoins);
+                    this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
+                }
+                catch (_) {
+                    steak[topic] = { outputsToAdmit: [], coinsToRetain: [] };
+                }
+            })();
+            // Wait for both tasks to complete
+            await Promise.all([markSpentPromises, admissibleOutputsPromise]);
+            // Keep track of what outputs were admitted for what topic
+            steak[topic] = admissableOutputs;
+        });
+        await Promise.all(topicPromises);
+        // Broadcast the transaction if not historical and broadcaster is configured
+        this.startTime(`broadcast_${txid.substring(0, 10)}`);
+        if (mode !== 'historical-tx' && this.broadcaster !== undefined) {
+            const response = await this.broadcaster.broadcast(tx);
+            if ((0, sdk_1.isBroadcastFailure)(response)) {
+                throw new Error(`Failed to broadcast transaction! Error: ${response.description}`);
             }
-            // Use the manager to determine which outputs are admissable
-            let admissableOutputs;
-            try {
-                admissableOutputs = await this.managers[topic].identifyAdmissibleOutputs(taggedBEEF.beef, previousCoins);
-            }
-            catch (_) {
-                // If the topic manager throws an error, other topics may still succeed, so we continue to the next one.
-                // No outputs were admitted to this topic in this case. Note, however, that the transaction is still valid according to Bitcoin, so it may have spent some previous overlay members. This is unavoidable and good.
-                steak[topic] = {
-                    outputsToAdmit: [],
-                    coinsToRetain: []
-                };
-                continue;
-            }
+        }
+        this.endTime(`broadcast_${txid.substring(0, 10)}`);
+        // Call the callback function if it is provided
+        if (onSteakReady !== undefined) {
+            this.endTime(`submit_${txid}`);
+            onSteakReady(steak);
+        }
+        for (const topic of taggedBEEF.topics) {
             // Keep track of which outputs to admit, mark as stale, or retain
             const outputsToAdmit = admissableOutputs.outputsToAdmit;
             const staleCoins = [];
@@ -130,7 +197,7 @@ class Engine {
             // Find which outputs should not be retained and mark them as stale
             // For each of the previous UTXOs, if the the UTXO was not included in the list of UTXOs identified for retention, then it will be marked as stale.
             for (const inputIndex of previousCoins) {
-                const previousTXID = tx.inputs[inputIndex].sourceTXID || ((_b = tx.inputs[inputIndex].sourceTransaction) === null || _b === void 0 ? void 0 : _b.id('hex'));
+                const previousTXID = tx.inputs[inputIndex].sourceTXID || ((_a = tx.inputs[inputIndex].sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex'));
                 const previousOutputIndex = tx.inputs[inputIndex].sourceOutputIndex;
                 if (!admissableOutputs.coinsToRetain.includes(inputIndex)) {
                     staleCoins.push({
@@ -146,16 +213,18 @@ class Engine {
                 }
             }
             // Remove stale outputs recursively
-            for (const coin of staleCoins) {
+            this.startTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
+            await Promise.all(staleCoins.map(async (coin) => {
                 const output = await this.storage.findOutput(coin.txid, coin.outputIndex, topic);
                 if (output !== undefined && output !== null) {
                     await this.deleteUTXODeep(output);
                 }
-            }
+            }));
+            this.endTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
             // Handle admittance and notification of incoming UTXOs
             const newUTXOs = [];
-            for (const outputIndex of outputsToAdmit) {
-                // Store the output
+            await Promise.all(outputsToAdmit.map(async (outputIndex) => {
+                this.startTime(`insertNewOutput_${txid.substring(0, 10)}`);
                 await this.storage.insertOutput({
                     txid,
                     outputIndex,
@@ -167,51 +236,40 @@ class Engine {
                     consumedBy: [],
                     outputsConsumed
                 });
-                newUTXOs.push({
-                    txid,
-                    outputIndex
-                });
-                // Notify all the lookup services about the new UTXO
-                for (const l of Object.values(this.lookupServices)) {
-                    try {
-                        if (l.outputAdded !== undefined && l.outputAdded !== null) {
-                            await l.outputAdded(txid, outputIndex, tx.outputs[outputIndex].lockingScript, topic);
-                        }
+                this.endTime(`insertNewOutput_${txid.substring(0, 10)}`);
+                newUTXOs.push({ txid, outputIndex });
+                this.startTime(`notifyLookupService${txid.substring(0, 10)}`);
+                await Promise.all(Object.values(this.lookupServices).map(async (l) => { var _a; return await ((_a = l.outputAdded) === null || _a === void 0 ? void 0 : _a.call(l, txid, outputIndex, tx.outputs[outputIndex].lockingScript, topic)); }));
+                this.endTime(`notifyLookupService${txid.substring(0, 10)}`);
+            }));
+            this.startTime(`outputConsumed_${txid.substring(0, 10)}`);
+            // Update each output consumed to know who consumed it and insert applied transaction in parallel
+            await Promise.all([
+                ...outputsConsumed.map(async (output) => {
+                    const outputToUpdate = await this.storage.findOutput(output.txid, output.outputIndex, topic);
+                    if (outputToUpdate !== undefined && outputToUpdate !== null) {
+                        const newConsumedBy = [...new Set([...newUTXOs, ...outputToUpdate.consumedBy])];
+                        await this.storage.updateConsumedBy(output.txid, output.outputIndex, topic, newConsumedBy);
                     }
-                    catch (_) { }
-                }
-            }
-            // Update each output consumed to know who consumed it
-            for (const output of outputsConsumed) {
-                const outputToUpdate = await this.storage.findOutput(output.txid, output.outputIndex, topic);
-                if (outputToUpdate !== undefined && outputToUpdate !== null) {
-                    const newConsumedBy = [...new Set([...newUTXOs, ...outputToUpdate.consumedBy])];
-                    // Note: only update if newConsumedBy !== new Set(JSON.parse(outputToUpdate.consumedBy)) ?
-                    await this.storage.updateConsumedBy(output.txid, output.outputIndex, topic, newConsumedBy);
-                }
-            }
-            // Insert the applied transaction to prevent duplicate processing
-            await this.storage.insertAppliedTransaction({
-                txid,
-                topic
-            });
-            // Keep track of what outputs were admitted for what topic
-            steak[topic] = admissableOutputs;
-        }
-        // Call the callback function if it is provided
-        // TODO: To call `onSteakReady` sooner, we could have two loops. First we figure out topical admittance only, then we call `onSteakReeady` and do everything else after the first loop.
-        if (onSteakReady !== undefined) {
-            onSteakReady(steak);
+                }),
+                this.storage.insertAppliedTransaction({
+                    txid,
+                    topic
+                })
+            ]);
+            this.endTime(`outputConsumed_${txid.substring(0, 10)}`);
         }
         // If we don't have an advertiser or we are dealing with historical transactions, just return the steak
         if (this.advertiser === undefined || mode === 'historical-tx') {
             return steak;
         }
+        this.startTime(`transactionPropagation_${txid.substring(0, 10)}`);
         // Propagate transaction to other nodes according to synchronization agreements
         // 1. Find nodes that host the topics associated with admissable outputs
         // We want to figure out which topics we actually care about (because their associated outputs were admitted)
         // AND if the topic was not admitted we want to remove it from the list of topics we care about.
         const relevantTopics = taggedBEEF.topics.filter(topic => steak[topic] !== undefined && steak[topic].outputsToAdmit.length !== 0);
+        // TODO: Cache ship/slap lookup with expiry (every 5min)
         if (relevantTopics.length > 0) {
             // Find all SHIP advertisements for the topics we care about
             const domainToTopicsMap = new Map();
@@ -302,6 +360,7 @@ class Engine {
                 console.error('Error during broadcasting:', error);
             }
         }
+        this.endTime(`transactionPropgation_${txid.substring(0, 10)}`);
         // Immediately return from the function without waiting for the promises to resolve.
         return steak;
     }
@@ -314,7 +373,7 @@ class Engine {
         // Validate a lookup service for the provider is found
         const lookupService = this.lookupServices[lookupQuestion.service];
         if (lookupService === undefined || lookupService === null)
-            throw new Error(`Lookup service not found for provider: ${lookupQuestion.service}`);
+            throw new Error(`Lookup service not found for provider: ${lookupQuestion.service} `);
         let lookupResult = await lookupService.lookup(lookupQuestion);
         // Handle custom lookup service answers
         if (lookupResult.type === 'freeform' || lookupResult.type === 'output-list') {
@@ -324,12 +383,12 @@ class Engine {
         const hydratedOutputs = [];
         for (const { txid, outputIndex, history } of lookupResult) {
             // Make sure this is an unspent output (UTXO)
-            const UTXO = await this.storage.findOutput(txid, outputIndex, undefined, false);
+            const UTXO = await this.storage.findOutput(txid, outputIndex, undefined, undefined, true);
             if (UTXO === undefined || UTXO === null)
                 continue;
             // Get the history for this utxo and construct a BRC-8 Envelope
             const output = await this.getUTXOHistory(UTXO, history, 0);
-            if (output !== undefined && output !== null) {
+            if ((output === null || output === void 0 ? void 0 : output.beef) !== undefined) {
                 hydratedOutputs.push({
                     beef: output.beef,
                     outputIndex: output.outputIndex
@@ -425,6 +484,10 @@ class Engine {
         for (const topic of Object.keys(this.syncConfiguration)) {
             // Make sure syncEndpoints is an array or SHIP
             let syncEndpoints = this.syncConfiguration[topic];
+            // Check if this topic has been configured NOT to sync
+            if (syncEndpoints === false) {
+                continue;
+            }
             if (syncEndpoints === 'SHIP') {
                 // Perform lookup and find ship advertisements to set syncEndpoints for topic
                 const lookupAnswer = await this.lookup({
@@ -458,7 +521,7 @@ class Engine {
             if (Array.isArray(syncEndpoints)) {
                 await Promise.all(syncEndpoints.map(async (endpoint) => {
                     // Sync to each host that is associated with this topic
-                    const gasp = new gasp_1.GASP(new OverlayGASPStorage(topic, this), new OverlayGASPRemote(endpoint, topic), 0, `[GASP Sync of ${topic} with ${endpoint}] `, true);
+                    const gasp = new gasp_1.GASP(new OverlayGASPStorage_js_1.OverlayGASPStorage(topic, this), new OverlayGASPRemote_js_1.OverlayGASPRemote(endpoint, topic), 0, `[GASP Sync of ${topic} with ${endpoint}]`, true);
                     await gasp.sync();
                 }));
             }
@@ -498,7 +561,7 @@ class Engine {
      */
     async provideForeignGASPNode(graphID, txid, outputIndex) {
         const hydrator = async (output) => {
-            if (output === undefined || output === null) {
+            if ((output === null || output === void 0 ? void 0 : output.beef) === undefined) {
                 throw new Error('No matching output found!');
             }
             const rootTx = sdk_1.Transaction.fromBEEF(output.beef);
@@ -576,6 +639,9 @@ class Engine {
         if (typeof historySelector === 'undefined') {
             return output;
         }
+        if (output.beef === undefined) {
+            throw new Error('Output must have associated transaction BEEF!');
+        }
         // Determine if history traversal should continue for the current node
         let shouldTraverseHistory;
         if (typeof historySelector !== 'number') {
@@ -596,7 +662,7 @@ class Engine {
             const outputsConsumed = output.outputsConsumed;
             // Find the child outputs for each utxo consumed by the current output
             const childHistories = (await Promise.all(outputsConsumed.map(async (outputIdentifier) => {
-                const output = await this.storage.findOutput(outputIdentifier.txid, outputIdentifier.outputIndex);
+                const output = await this.storage.findOutput(outputIdentifier.txid, outputIdentifier.outputIndex, undefined, undefined, true);
                 // Make sure an output was found
                 if (output === undefined || output === null) {
                     return undefined;
@@ -615,6 +681,9 @@ class Engine {
                         : (_a = input.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex');
                     return sourceTXID === output.txid && input.sourceOutputIndex === output.outputIndex;
                 });
+                if (input.beef === undefined) {
+                    throw new Error('Input must have associated transaction BEEF!');
+                }
                 tx.inputs[inputIndex].sourceTransaction = sdk_1.Transaction.fromBEEF(input.beef);
             }
             const beef = tx.toBEEF();
@@ -626,9 +695,9 @@ class Engine {
         catch (e) {
             // Handle any errors that occurred
             // Note: Test this!
-            console.error(`Error retrieving UTXO history: ${e}`);
+            console.error(`Error retrieving UTXO history: ${e} `);
             // return []
-            throw new Error(`Error retrieving UTXO history: ${e}`);
+            throw new Error(`Error retrieving UTXO history: ${e} `);
         }
     }
     /**
@@ -672,7 +741,7 @@ class Engine {
             });
         }
         catch (error) {
-            throw new Error(`Failed to delete all stale outputs: ${error}`);
+            throw new Error(`Failed to delete all stale outputs: ${error} `);
         }
     }
     /**
@@ -709,6 +778,9 @@ class Engine {
      * @param proof - The merklePath proving txid is a mined transaction hash
      */
     async updateMerkleProof(output, txid, proof) {
+        if (output.beef === undefined) {
+            throw new Error('Output must have associated transaction BEEF!');
+        }
         const tx = sdk_1.Transaction.fromBEEF(output.beef);
         if (tx.merklePath !== undefined) {
             // Update the merkle path to handle potential reorgs
@@ -718,10 +790,10 @@ class Engine {
         // recursively update all sourceTransactions proven by (txid,proof)
         this.updateInputProofs(tx, txid, proof);
         // Update the output's BEEF in the storage DB
-        await this.storage.updateOutputBeef(output.txid, output.outputIndex, output.topic, tx.toBEEF());
+        await this.storage.updateTransactionBEEF(output.txid, tx.toBEEF());
         // Recursively update the consumedBy outputs
         for (const consumingOutput of output.consumedBy) {
-            const consumedOutputs = await this.storage.findOutputsForTransaction(consumingOutput.txid);
+            const consumedOutputs = await this.storage.findOutputsForTransaction(consumingOutput.txid, true);
             for (const consumedOutput of consumedOutputs) {
                 await this.updateMerkleProof(consumedOutput, txid, proof);
             }
@@ -736,7 +808,7 @@ class Engine {
      */
     async handleNewMerkleProof(txid, proof, blockHeight) {
         var _a, _b;
-        const outputs = await this.storage.findOutputsForTransaction(txid);
+        const outputs = await this.storage.findOutputsForTransaction(txid, true);
         if (outputs === undefined || outputs.length === 0) {
             throw new Error('Could not find matching transaction outputs for proof ingest!');
         }
@@ -795,388 +867,5 @@ There is currently a bug with the test runner that prevents importing and using
 Thus, all non-type exports have been moved to Engine.
 */
 // TODO: fix bug with imports that break tests. -----[GASP/OverlayGASPRemote.ts]-----
-class OverlayGASPRemote {
-    constructor(endpointURL, topic) {
-        this.endpointURL = endpointURL;
-        this.topic = topic;
-    }
-    /**
-     * Given an outgoing initial request, sends the request to the foreign instance and obtains their initial response.
-     * @param request
-     * @returns
-     */
-    async getInitialResponse(request) {
-        // Send out an HTTP request to the URL (current host for topic)
-        // Include the topic in the request
-        // Parse out response and return correct format
-        const url = `${this.endpointURL}/requestSyncResponse`;
-        const response = await fetch(url, {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json',
-                'X-BSV-Topic': this.topic
-            },
-            body: JSON.stringify(request)
-        });
-        if (!response.ok) {
-            throw new Error(`HTTP error! Status: ${response.status}`);
-        }
-        const result = await response.json();
-        // Validate and return the response in the correct format
-        if (!Array.isArray(result.UTXOList) || typeof result.since !== 'number') {
-            throw new Error('Invalid response format');
-        }
-        return {
-            UTXOList: result.UTXOList.map((utxo) => ({
-                txid: utxo.txid,
-                outputIndex: utxo.outputIndex
-            })),
-            since: result.since
-        };
-    }
-    /**
-     * Given an outgoing txid, outputIndex and optional metadata, request the associated GASP node from the foreign instance.
-     * @param graphID
-     * @param txid
-     * @param outputIndex
-     * @param metadata
-     * @returns
-     */
-    async requestNode(graphID, txid, outputIndex, metadata) {
-        // Send an HTTP request with the provided info and get back a gaspNode
-        const url = `${this.endpointURL}/requestForeignGASPNode`;
-        const body = {
-            graphID,
-            txid,
-            outputIndex,
-            metadata
-        };
-        const response = await fetch(url, {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json'
-            },
-            body: JSON.stringify(body)
-        });
-        if (!response.ok) {
-            throw new Error(`HTTP error! Status: ${response.status}`);
-        }
-        const result = await response.json();
-        // Validate and return the response in the correct format
-        if (typeof result.graphID !== 'string' || typeof result.rawTx !== 'string' || typeof result.outputIndex !== 'number') {
-            throw new Error('Invalid response format');
-        }
-        const gaspNode = {
-            graphID: result.graphID,
-            rawTx: result.rawTx,
-            outputIndex: result.outputIndex,
-            proof: result.proof,
-            txMetadata: result.txMetadata,
-            outputMetadata: result.outputMetadata,
-            inputs: result.inputs
-        };
-        return gaspNode;
-    }
-    // ---- Now optional methods ----
-    // When are only syncing to them
-    async getInitialReply(response) {
-        throw new Error('Function not supported!');
-    }
-    // Only used when supporting bidirectional sync.
-    // Overlay services does not support this.
-    async submitNode(node) {
-        throw new Error('Node submission not supported!');
-    }
-}
-exports.OverlayGASPRemote = OverlayGASPRemote;
-class OverlayGASPStorage {
-    constructor(topic, engine, maxNodesInGraph) {
-        this.topic = topic;
-        this.engine = engine;
-        this.maxNodesInGraph = maxNodesInGraph;
-        this.temporaryGraphNodeRefs = {};
-    }
-    /**
-     *
-     * @param since
-     * @returns
-     */
-    async findKnownUTXOs(since) {
-        const UTXOs = await this.engine.storage.findUTXOsForTopic(this.topic, since);
-        return UTXOs.map(output => ({
-            txid: output.txid,
-            outputIndex: output.outputIndex
-        }));
-    }
-    /**
-     * For a given txid and output index, returns the associated transaction, a merkle proof if the transaction is in a block, and metadata if if requested. If no metadata is requested, metadata hashes on inputs are not returned.
-     * @param graphID
-     * @param txid
-     * @param outputIndex
-     * @param metadata
-     * @returns
-     */
-    async hydrateGASPNode(graphID, txid, outputIndex, metadata) {
-        const output = await this.engine.storage.findOutput(txid, outputIndex);
-        if (output === undefined || output === null) {
-            throw new Error('No matching output found!');
-        }
-        const tx = sdk_1.Transaction.fromBEEF(output.beef);
-        const rawTx = tx.toHex();
-        const node = {
-            rawTx,
-            graphID,
-            outputIndex
-        };
-        if (tx.merklePath !== undefined) {
-            node.proof = tx.merklePath.toHex();
-        }
-        return node;
-    }
-    /**
-    * For a given node, returns the inputs needed to complete the graph, including whether updated metadata is requested for those inputs.
-    * @param tx The node for which needed inputs should be found.
-    * @returns A promise for a mapping of requested input transactions and whether metadata should be provided for each.
-    */
-    async findNeededInputs(tx) {
-        var _a, _b, _c;
-        // If there is no Merkle proof, we always need the inputs
-        const response = {
-            requestedInputs: {}
-        };
-        const parsedTx = sdk_1.Transaction.fromHex(tx.rawTx);
-        if (tx.proof === undefined) {
-            for (const input of parsedTx.inputs) {
-                response.requestedInputs[`${input.sourceTXID}.${input.sourceOutputIndex}`] = {
-                    metadata: false
-                };
-            }
-            return await this.stripAlreadyKnownInputs(response);
-        }
-        // Attempt to check if the current transaction is admissible
-        parsedTx.merklePath = sdk_1.MerklePath.fromHex(tx.proof);
-        const admittanceResult = await this.engine.managers[this.topic].identifyAdmissibleOutputs(parsedTx.toBEEF(), []);
-        if (admittanceResult.outputsToAdmit.includes(tx.outputIndex)) {
-            // The transaction is admissible, no further inputs are needed
-        }
-        else {
-            // The transaction is not admissible, get inputs needed for further verification
-            // TopicManagers should implement a function to identify which inputs are needed.
-            if (this.engine.managers[this.topic] !== undefined && typeof this.engine.managers[this.topic].identifyNeededInputs === 'function') {
-                try {
-                    const neededInputs = (_c = await ((_b = (_a = this.engine.managers[this.topic]).identifyNeededInputs) === null || _b === void 0 ? void 0 : _b.call(_a, parsedTx.toBEEF()))) !== null && _c !== void 0 ? _c : [];
-                    for (const input of neededInputs) {
-                        response.requestedInputs[`${input.txid}.${input.outputIndex}`] = {
-                            metadata: false
-                        };
-                    }
-                    return await this.stripAlreadyKnownInputs(response);
-                }
-                catch (e) {
-                    console.error(`An error occurred when identifying needed inputs for transaction: ${parsedTx.id('hex')}.${tx.outputIndex}!`);
-                    // Cut off the graph in case of an error here.
-                }
-            }
-            // By default, if the topic manager isn't able to stipulate needed inputs, only the inputs necessary for SPV are requested.
-        }
-        // Everything else falls through to returning undefined/void, which will terminate the synchronization at this point.
-    }
-    /**
-     * Ensures that no inputs are requested from foreign nodes before sending any GASP response
-     * Also terminates graphs if the response would be empty.
-     */
-    async stripAlreadyKnownInputs(response) {
-        if (typeof response === 'undefined') {
-            return response;
-        }
-        for (const inputNodeId of Object.keys(response.requestedInputs)) {
-            const [txid, outputIndex] = inputNodeId.split('.');
-            const found = await this.engine.storage.findOutput(txid, Number(outputIndex), this.topic);
-            if (found) {
-                delete response.requestedInputs[inputNodeId];
-            }
-        }
-        if (Object.keys(response.requestedInputs).length === 0) {
-            return undefined;
-        }
-        return response;
-    }
-    /**
-    * Appends a new node to a temporary graph.
-    * @param tx The node to append to this graph.
-    * @param spentBy Unless this is the same node identified by the graph ID, denotes the TXID and input index for the node which spent this one, in 36-byte format.
-    * @throws If the node cannot be appended to the graph, either because the graph ID is for a graph the recipient does not want or because the graph has grown to be too large before being finalized.
-    */
-    async appendToGraph(tx, spentBy) {
-        if (this.maxNodesInGraph !== undefined && Object.keys(this.temporaryGraphNodeRefs).length >= this.maxNodesInGraph) {
-            throw new Error('The max number of nodes in transaction graph has been reached!');
-        }
-        const parsedTx = sdk_1.Transaction.fromHex(tx.rawTx);
-        const txid = parsedTx.id('hex');
-        if (tx.proof !== undefined) {
-            parsedTx.merklePath = sdk_1.MerklePath.fromHex(tx.proof);
-        }
-        // Given the passed in node, append to the temp graph
-        // Use the spentBy param which should be a txid.inputIndex for the node which spent this one in 36-byte format
-        const newGraphNode = {
-            txid,
-            graphID: tx.graphID,
-            rawTx: tx.rawTx,
-            outputIndex: tx.outputIndex,
-            proof: tx.proof,
-            txMetadata: tx.txMetadata,
-            outputMetadata: tx.outputMetadata,
-            inputs: tx.inputs,
-            children: []
-        };
-        // If spentBy is undefined, then we know it's the root node.
-        if (spentBy === undefined) {
-            this.temporaryGraphNodeRefs[tx.graphID] = newGraphNode;
-        }
-        else {
-            // Find the parent node based on spentBy
-            const parentNode = this.temporaryGraphNodeRefs[spentBy];
-            if (parentNode !== undefined) {
-                // Set parent-child relationship
-                parentNode.children.push(newGraphNode);
-                newGraphNode.parent = parentNode;
-                this.temporaryGraphNodeRefs[`${newGraphNode.txid}.${newGraphNode.outputIndex}`] = newGraphNode;
-            }
-            else {
-                throw new Error(`Parent node with GraphID ${spentBy} not found`);
-            }
-        }
-    }
-    /**
-      * Checks whether the given graph, in its current state, makes reference only to transactions that are proven in the blockchain, or already known by the recipient to be valid.
-      * Additionally, in a breadth-first manner (ensuring that all inputs for any given node are processed before nodes that spend them), it ensures that the root node remains valid according to the rules of the overlay's topic manager,
-      * while considering any coins which the Manager had previously indicated were either valid or invalid.
-      * @param graphID The TXID and output index (in 36-byte format) for the UTXO at the tip of this graph.
-      * @throws If the graph is not well-anchored, according to the rules of Bitcoin or the rules of the Overlay Topic Manager.
-      */
-    async validateGraphAnchor(graphID) {
-        var _a;
-        const rootNode = this.temporaryGraphNodeRefs[graphID];
-        if (rootNode === undefined) {
-            throw new Error(`Graph node with ID ${graphID} not found`);
-        }
-        // Check that the root node is Bitcoin-valid.
-        const beef = this.getBEEFForNode(rootNode);
-        const spvTx = sdk_1.Transaction.fromBEEF(beef);
-        const isBitcoinValid = await spvTx.verify(this.engine.chainTracker);
-        if (!isBitcoinValid) {
-            throw new Error('The graph is not well-anchored according to the rules of Bitcoin.');
-        }
-        // Then, ensure the node is Overlay-valid.
-        const beefs = this.computeOrderedBEEFsForGraph(graphID);
-        // coins: a Set of all historical coins to retain (no need to remove them), used to emulate topical admittance of previous inputs over time.
-        const coins = new Set();
-        // Submit all historical BEEFs in order through the topic manager, tracking what would be retained until we submit the root node last.
-        // If, at the end, the root node is admitted, we have a valid overlay-specific graph.
-        for (const beef of beefs) {
-            // For any input to this transaction, see if it's a valid coin that's admitted. If so, it's a previous coin.
-            const previousCoins = [];
-            const tx = sdk_1.Transaction.fromBEEF(beef);
-            for (const inputIndex in tx.inputs) {
-                const input = tx.inputs[inputIndex];
-                const sourceTXID = input.sourceTXID || ((_a = input.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex'));
-                const coin = `${sourceTXID}.${input.sourceOutputIndex}`;
-                if (coins.has(coin)) {
-                    previousCoins.push(Number(inputIndex));
-                }
-            }
-            const admittanceInstructions = await this.engine.managers[this.topic].identifyAdmissibleOutputs(beef, previousCoins);
-            // Every admitted output is now a coin.
-            for (const outputIndex of admittanceInstructions.outputsToAdmit) {
-                coins.add(`${tx.id('hex')}.${outputIndex}`);
-            }
-        }
-        // After sending through all the graph's BEEFs...
-        // If the root node is now a coin, we have acceptance by the overlay.
-        // Otherwise, throw.
-        if (!coins.has(graphID)) {
-            throw new Error('This graph did not result in topical admittance of the root node. Rejecting.');
-        }
-    }
-    /**
-     * Deletes all data associated with a temporary graph that has failed to sync, if the graph exists.
-     * @param graphID The TXID and output index (in 36-byte format) for the UTXO at the tip of this graph.
-     */
-    async discardGraph(graphID) {
-        for (const [nodeId, graphRef] of Object.entries(this.temporaryGraphNodeRefs)) {
-            if (graphRef.graphID === graphID) {
-                // Delete child node
-                // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-                delete this.temporaryGraphNodeRefs[nodeId];
-            }
-        }
-    }
-    /**
-     * Finalizes a graph, solidifying the new UTXO and its ancestors so that it will appear in the list of known UTXOs.
-     * @param graphID The TXID and output index (in 36-byte format) for the UTXO at the root of this graph.
-     */
-    async finalizeGraph(graphID) {
-        const beefs = this.computeOrderedBEEFsForGraph(graphID);
-        // Submit all historical BEEFs in order, finalizing the graph for the current UTXO
-        for (const beef of beefs) {
-            await this.engine.submit({
-                beef,
-                topics: [this.topic]
-            }, () => { }, 'historical-tx');
-        }
-    }
-    /**
-     * Computes an ordered set of BEEFs for the graph with the given graph IDs
-     * @param {string} graphID — The ID of the graph for which BEEFs are required
-     * @returns Ordered BEEFs for the graph
-     */
-    computeOrderedBEEFsForGraph(graphID) {
-        const beefs = [];
-        const hydrator = (node) => {
-            const currentBEEF = this.getBEEFForNode(node);
-            if (beefs.indexOf(currentBEEF) === -1) {
-                beefs.unshift(currentBEEF);
-            }
-            for (const child of node.children) {
-                // Continue backwards to the earliest nodes, adding them onto the beginning
-                hydrator(child);
-            }
-        };
-        // Start the hydrator with the root node
-        const foundRoot = this.temporaryGraphNodeRefs[graphID];
-        if (!foundRoot) {
-            throw new Error('Unable to find root node in graph for finalization!');
-        }
-        hydrator(foundRoot);
-        return beefs;
-    }
-    /**
-     * Computes a full BEEF for a given graph node, based on the temporary graph store.
-     * @param node Graph node for which BEEF is needed.
-     * @returns BEEF array, including all proofs on inputs.
-     */
-    getBEEFForNode(node) {
-        // Given a node, hydrate its merkle proof or all inputs, returning a reference to the hydrated node's Transaction object
-        const hydrator = (node) => {
-            const tx = sdk_1.Transaction.fromHex(node.rawTx);
-            if (node.proof) {
-                tx.merklePath = sdk_1.MerklePath.fromHex(node.proof);
-                return tx; // Transaction with proof, end of the line.
-            }
-            // For each input, look it up and recurse.
-            for (const inputIndex in tx.inputs) {
-                const input = tx.inputs[inputIndex];
-                const foundNode = this.temporaryGraphNodeRefs[`${input.sourceTXID}.${input.sourceOutputIndex}`];
-                if (!foundNode) {
-                    throw new Error('Required input node for unproven parent not found in temporary graph store. Ensure, for every parent of any given already-proven node (kept for Overlay-specific historical reasons), that a proof is also provided on those inputs. While implicitly they are valid by virtue of their descendents being proven in the blockchain, BEEF serialization will still fail when winding forward the topical UTXO set histories during sync.');
-                }
-                tx.inputs[inputIndex].sourceTransaction = hydrator(foundNode);
-            }
-            return tx;
-        };
-        const finalTX = hydrator(node);
-        return finalTX.toBEEF();
-    }
-}
-exports.OverlayGASPStorage = OverlayGASPStorage;
+// TODO: fix bug with imports that break tests. -----[GASP/OverlayGASPStorage.ts]-----
 //# sourceMappingURL=Engine.js.map