@bsv/overlay 0.1.8 → 0.1.10

package/src/GASP/OverlayGASPRemote.ts

@@ -0,0 +1,107 @@
+import { GASPInitialReply, GASPInitialRequest, GASPInitialResponse, GASPNode, GASPNodeResponse, GASPRemote } from "@bsv/gasp"
+
+export class OverlayGASPRemote implements GASPRemote {
+  constructor(public endpointURL: string, public topic: string) { }
+
+  /**
+   * Given an outgoing initial request, sends the request to the foreign instance and obtains their initial response.
+   * @param request
+   * @returns
+   */
+  async getInitialResponse(request: GASPInitialRequest): Promise<GASPInitialResponse> {
+    // Send out an HTTP request to the URL (current host for topic)
+    // Include the topic in the request
+    // Parse out response and return correct format
+    const url = `${this.endpointURL}/requestSyncResponse`
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-BSV-Topic': this.topic
+      },
+      body: JSON.stringify(request)
+    })
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! Status: ${response.status}`)
+    }
+
+    const result: GASPInitialResponse = await response.json()
+
+    // Validate and return the response in the correct format
+    if (!Array.isArray(result.UTXOList) || typeof result.since !== 'number') {
+      throw new Error('Invalid response format')
+    }
+
+    return {
+      UTXOList: result.UTXOList.map((utxo: any) => ({
+        txid: utxo.txid,
+        outputIndex: utxo.outputIndex
+      })),
+      since: result.since
+    }
+  }
+
+  /**
+   * Given an outgoing txid, outputIndex and optional metadata, request the associated GASP node from the foreign instance.
+   * @param graphID
+   * @param txid
+   * @param outputIndex
+   * @param metadata
+   * @returns
+   */
+  async requestNode(graphID: string, txid: string, outputIndex: number, metadata: boolean): Promise<GASPNode> {
+    // Send an HTTP request with the provided info and get back a gaspNode
+    const url = `${this.endpointURL}/requestForeignGASPNode`
+    const body = {
+      graphID,
+      txid,
+      outputIndex,
+      metadata
+    }
+
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify(body)
+    })
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! Status: ${response.status}`)
+    }
+
+    const result = await response.json()
+
+    // Validate and return the response in the correct format
+    if (typeof result.graphID !== 'string' || typeof result.rawTx !== 'string' || typeof result.outputIndex !== 'number') {
+      throw new Error('Invalid response format')
+    }
+
+    const gaspNode: GASPNode = {
+      graphID: result.graphID,
+      rawTx: result.rawTx,
+      outputIndex: result.outputIndex,
+      proof: result.proof,
+      txMetadata: result.txMetadata,
+      outputMetadata: result.outputMetadata,
+      inputs: result.inputs
+    }
+
+    return gaspNode
+  }
+
+  // ---- Now optional methods ----
+
+  // When are only syncing to them
+  async getInitialReply(response: GASPInitialResponse): Promise<GASPInitialReply> {
+    throw new Error('Function not supported!')
+  }
+
+  // Only used when supporting bidirectional sync.
+  // Overlay services does not support this.
+  async submitNode(node: GASPNode): Promise<void | GASPNodeResponse> {
+    throw new Error('Node submission not supported!')
+  }
+}

package/src/GASP/OverlayGASPStorage.ts

@@ -0,0 +1,331 @@
+import { GASPNode, GASPNodeResponse, GASPStorage } from '@bsv/gasp'
+import { MerklePath, Transaction } from '@bsv/sdk'
+import { Engine } from '../Engine.js'
+
+/**
+ * Represents a node in the temporary graph.
+ */
+export interface GraphNode {
+  txid: string
+  graphID: string
+  rawTx: string
+  outputIndex: number
+  spentBy?: string
+  proof?: string
+  txMetadata?: string
+  outputMetadata?: string
+  inputs?: Record<string, { hash: string }> | undefined
+  children: GraphNode[]
+  parent?: GraphNode
+}
+
+export class OverlayGASPStorage implements GASPStorage {
+  readonly temporaryGraphNodeRefs: Record<string, GraphNode> = {}
+
+  constructor(public topic: string, public engine: Engine, public maxNodesInGraph?: number) { }
+
+  /**
+   *
+   * @param since
+   * @returns
+   */
+  async findKnownUTXOs(since: number): Promise<Array<{ txid: string, outputIndex: number }>> {
+    const UTXOs = await this.engine.storage.findUTXOsForTopic(this.topic, since)
+    return UTXOs.map(output => ({
+      txid: output.txid,
+      outputIndex: output.outputIndex
+    }))
+  }
+
+  /**
+   * For a given txid and output index, returns the associated transaction, a merkle proof if the transaction is in a block, and metadata if if requested. If no metadata is requested, metadata hashes on inputs are not returned.
+   * @param graphID 
+   * @param txid 
+   * @param outputIndex 
+   * @param metadata 
+   * @returns 
+   */
+  async hydrateGASPNode(graphID: string, txid: string, outputIndex: number, metadata: boolean): Promise<GASPNode> {
+    const output = await this.engine.storage.findOutput(txid, outputIndex, undefined, undefined, true)
+
+    if (output?.beef === undefined) {
+      throw new Error('No matching output found!')
+    }
+
+    const tx = Transaction.fromBEEF(output.beef)
+    const rawTx = tx.toHex()
+
+    const node: GASPNode = {
+      rawTx,
+      graphID,
+      outputIndex
+    }
+    if (tx.merklePath !== undefined) {
+      node.proof = tx.merklePath.toHex()
+    }
+
+    return node
+  }
+
+  /**
+  * For a given node, returns the inputs needed to complete the graph, including whether updated metadata is requested for those inputs.
+  * @param tx The node for which needed inputs should be found.
+  * @returns A promise for a mapping of requested input transactions and whether metadata should be provided for each.
+  */
+  async findNeededInputs(tx: GASPNode): Promise<GASPNodeResponse | undefined> {
+    // If there is no Merkle proof, we always need the inputs
+    const response: GASPNodeResponse = {
+      requestedInputs: {}
+    }
+    const parsedTx = Transaction.fromHex(tx.rawTx)
+    if (tx.proof === undefined) {
+      for (const input of parsedTx.inputs) {
+        response.requestedInputs[`${input.sourceTXID}.${input.sourceOutputIndex}`] = {
+          metadata: false
+        }
+      }
+
+      return await this.stripAlreadyKnownInputs(response)
+    }
+
+    // Attempt to check if the current transaction is admissible
+    parsedTx.merklePath = MerklePath.fromHex(tx.proof)
+    const admittanceResult = await this.engine.managers[this.topic].identifyAdmissibleOutputs(parsedTx.toBEEF(), [])
+
+    if (admittanceResult.outputsToAdmit.includes(tx.outputIndex)) {
+      // The transaction is admissible, no further inputs are needed
+    } else {
+      // The transaction is not admissible, get inputs needed for further verification
+      // TopicManagers should implement a function to identify which inputs are needed.
+      if (this.engine.managers[this.topic] !== undefined && typeof this.engine.managers[this.topic].identifyNeededInputs === 'function') {
+        try {
+          const neededInputs = await this.engine.managers[this.topic].identifyNeededInputs?.(parsedTx.toBEEF()) ?? []
+          for (const input of neededInputs) {
+            response.requestedInputs[`${input.txid}.${input.outputIndex}`] = {
+              metadata: false
+            }
+          }
+          return await this.stripAlreadyKnownInputs(response)
+        } catch (e) {
+          console.error(`An error occurred when identifying needed inputs for transaction: ${parsedTx.id('hex')}.${tx.outputIndex}!`)
+          // Cut off the graph in case of an error here.
+        }
+      }
+      // By default, if the topic manager isn't able to stipulate needed inputs, only the inputs necessary for SPV are requested.
+    }
+    // Everything else falls through to returning undefined/void, which will terminate the synchronization at this point.
+  }
+
+  /**
+   * Ensures that no inputs are requested from foreign nodes before sending any GASP response
+   * Also terminates graphs if the response would be empty.
+   */
+  private async stripAlreadyKnownInputs(response: GASPNodeResponse | undefined): Promise<GASPNodeResponse | undefined> {
+    if (typeof response === 'undefined') {
+      return response
+    }
+    for (const inputNodeId of Object.keys(response.requestedInputs)) {
+      const [txid, outputIndex] = inputNodeId.split('.')
+      const found = await this.engine.storage.findOutput(txid, Number(outputIndex), this.topic)
+      if (found !== null && found !== undefined) {
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+        delete response.requestedInputs[inputNodeId]
+      }
+    }
+    if (Object.keys(response.requestedInputs).length === 0) {
+      return undefined
+    }
+    return response
+  }
+
+  /**
+  * Appends a new node to a temporary graph.
+  * @param tx The node to append to this graph.
+  * @param spentBy Unless this is the same node identified by the graph ID, denotes the TXID and input index for the node which spent this one, in 36-byte format.
+  * @throws If the node cannot be appended to the graph, either because the graph ID is for a graph the recipient does not want or because the graph has grown to be too large before being finalized.
+  */
+  async appendToGraph(tx: GASPNode, spentBy?: string | undefined): Promise<void> {
+    if (this.maxNodesInGraph !== undefined && Object.keys(this.temporaryGraphNodeRefs).length >= this.maxNodesInGraph) {
+      throw new Error('The max number of nodes in transaction graph has been reached!')
+    }
+
+    const parsedTx = Transaction.fromHex(tx.rawTx)
+    const txid = parsedTx.id('hex')
+    if (tx.proof !== undefined) {
+      parsedTx.merklePath = MerklePath.fromHex(tx.proof)
+    }
+
+    // Given the passed in node, append to the temp graph
+    // Use the spentBy param which should be a txid.inputIndex for the node which spent this one in 36-byte format
+    const newGraphNode: GraphNode = {
+      txid,
+      graphID: tx.graphID,
+      rawTx: tx.rawTx,
+      outputIndex: tx.outputIndex,
+      proof: tx.proof,
+      txMetadata: tx.txMetadata,
+      outputMetadata: tx.outputMetadata,
+      inputs: tx.inputs,
+      children: []
+    }
+
+    // If spentBy is undefined, then we know it's the root node.
+    if (spentBy === undefined) {
+      this.temporaryGraphNodeRefs[tx.graphID] = newGraphNode
+    } else {
+      // Find the parent node based on spentBy
+      const parentNode = this.temporaryGraphNodeRefs[spentBy]
+
+      if (parentNode !== undefined) {
+        // Set parent-child relationship
+        parentNode.children.push(newGraphNode)
+        newGraphNode.parent = parentNode
+        this.temporaryGraphNodeRefs[`${newGraphNode.txid}.${newGraphNode.outputIndex}`] = newGraphNode
+      } else {
+        throw new Error(`Parent node with GraphID ${spentBy} not found`)
+      }
+    }
+  }
+
+  /**
+    * Checks whether the given graph, in its current state, makes reference only to transactions that are proven in the blockchain, or already known by the recipient to be valid.
+    * Additionally, in a breadth-first manner (ensuring that all inputs for any given node are processed before nodes that spend them), it ensures that the root node remains valid according to the rules of the overlay's topic manager,
+    * while considering any coins which the Manager had previously indicated were either valid or invalid.
+    * @param graphID The TXID and output index (in 36-byte format) for the UTXO at the tip of this graph.
+    * @throws If the graph is not well-anchored, according to the rules of Bitcoin or the rules of the Overlay Topic Manager.
+    */
+  async validateGraphAnchor(graphID: string): Promise<void> {
+    const rootNode = this.temporaryGraphNodeRefs[graphID]
+    if (rootNode === undefined) {
+      throw new Error(`Graph node with ID ${graphID} not found`)
+    }
+
+    // Check that the root node is Bitcoin-valid.
+    const beef = this.getBEEFForNode(rootNode)
+    const spvTx = Transaction.fromBEEF(beef)
+    const isBitcoinValid = await spvTx.verify(this.engine.chainTracker)
+    if (!isBitcoinValid) {
+      throw new Error('The graph is not well-anchored according to the rules of Bitcoin.')
+    }
+
+    // Then, ensure the node is Overlay-valid.
+    const beefs = this.computeOrderedBEEFsForGraph(graphID)
+
+    // coins: a Set of all historical coins to retain (no need to remove them), used to emulate topical admittance of previous inputs over time.
+    const coins = new Set<string>()
+
+    // Submit all historical BEEFs in order through the topic manager, tracking what would be retained until we submit the root node last.
+    // If, at the end, the root node is admitted, we have a valid overlay-specific graph.
+    for (const beef of beefs) {
+      // For any input to this transaction, see if it's a valid coin that's admitted. If so, it's a previous coin.
+      const previousCoins: number[] = []
+      const tx = Transaction.fromBEEF(beef)
+      for (const inputIndex in tx.inputs) {
+        const input = tx.inputs[inputIndex]
+        const sourceTXID = input.sourceTXID || input.sourceTransaction?.id('hex')
+        const coin = `${sourceTXID}.${input.sourceOutputIndex}`
+        if (coins.has(coin)) {
+          previousCoins.push(Number(inputIndex))
+        }
+      }
+      const admittanceInstructions = await this.engine.managers[this.topic].identifyAdmissibleOutputs(beef, previousCoins)
+      // Every admitted output is now a coin.
+      for (const outputIndex of admittanceInstructions.outputsToAdmit) {
+        coins.add(`${tx.id('hex')}.${outputIndex}`)
+      }
+    }
+    // After sending through all the graph's BEEFs...
+    // If the root node is now a coin, we have acceptance by the overlay.
+    // Otherwise, throw.
+    if (!coins.has(graphID)) {
+      throw new Error('This graph did not result in topical admittance of the root node. Rejecting.')
+    }
+  }
+
+  /**
+   * Deletes all data associated with a temporary graph that has failed to sync, if the graph exists.
+   * @param graphID The TXID and output index (in 36-byte format) for the UTXO at the tip of this graph.
+   */
+  async discardGraph(graphID: string): Promise<void> {
+    for (const [nodeId, graphRef] of Object.entries(this.temporaryGraphNodeRefs)) {
+      if (graphRef.graphID === graphID) {
+        // Delete child node
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+        delete this.temporaryGraphNodeRefs[nodeId]
+      }
+    }
+  }
+
+  /**
+   * Finalizes a graph, solidifying the new UTXO and its ancestors so that it will appear in the list of known UTXOs.
+   * @param graphID The TXID and output index (in 36-byte format) for the UTXO at the root of this graph.
+   */
+  async finalizeGraph(graphID: string): Promise<void> {
+    const beefs = this.computeOrderedBEEFsForGraph(graphID)
+
+    // Submit all historical BEEFs in order, finalizing the graph for the current UTXO
+    for (const beef of beefs) {
+      await this.engine.submit({
+        beef,
+        topics: [this.topic]
+      }, () => { }, 'historical-tx')
+    }
+  }
+
+  /**
+   * Computes an ordered set of BEEFs for the graph with the given graph IDs
+   * @param {string} graphID — The ID of the graph for which BEEFs are required
+   * @returns Ordered BEEFs for the graph
+   */
+  private computeOrderedBEEFsForGraph(graphID: string): number[][] {
+    const beefs: number[][] = []
+    const hydrator = (node: GraphNode): void => {
+      const currentBEEF = this.getBEEFForNode(node)
+      if (beefs.indexOf(currentBEEF) === -1) {
+        beefs.unshift(currentBEEF)
+      }
+
+      for (const child of node.children) {
+        // Continue backwards to the earliest nodes, adding them onto the beginning
+        hydrator(child)
+      }
+    }
+
+    // Start the hydrator with the root node
+    const foundRoot = this.temporaryGraphNodeRefs[graphID]
+    if (!foundRoot) {
+      throw new Error('Unable to find root node in graph for finalization!')
+    }
+    hydrator(foundRoot)
+    return beefs
+  }
+
+  /**
+   * Computes a full BEEF for a given graph node, based on the temporary graph store.
+   * @param node Graph node for which BEEF is needed.
+   * @returns BEEF array, including all proofs on inputs.
+   */
+  private getBEEFForNode(node: GraphNode): number[] {
+    // Given a node, hydrate its merkle proof or all inputs, returning a reference to the hydrated node's Transaction object
+    const hydrator = (node: GraphNode): Transaction => {
+      const tx = Transaction.fromHex(node.rawTx)
+      if (node.proof) {
+        tx.merklePath = MerklePath.fromHex(node.proof)
+        return tx // Transaction with proof, end of the line.
+      }
+      // For each input, look it up and recurse.
+      for (const inputIndex in tx.inputs) {
+        const input = tx.inputs[inputIndex]
+        const foundNode = this.temporaryGraphNodeRefs[`${input.sourceTXID}.${input.sourceOutputIndex}`]
+        if (!foundNode) {
+          throw new Error('Required input node for unproven parent not found in temporary graph store. Ensure, for every parent of any given already-proven node (kept for Overlay-specific historical reasons), that a proof is also provided on those inputs. While implicitly they are valid by virtue of their descendents being proven in the blockchain, BEEF serialization will still fail when winding forward the topical UTXO set histories during sync.')
+        }
+        tx.inputs[inputIndex].sourceTransaction = hydrator(foundNode)
+      }
+      return tx
+    }
+
+    const finalTX = hydrator(node)
+    return finalTX.toBEEF()
+  }
+}

package/src/Output.ts

@@ -14,8 +14,6 @@ export type Output = {
   topic: string
   /** Whether the output is spent */
   spent: boolean
-  /** The transaction data for the output */
-  beef: number[]
   /** Outputs consumed by the transaction associated with the output */
   outputsConsumed: Array<{
     txid: string
@@ -26,5 +24,7 @@ export type Output = {
     txid: string
     outputIndex: number
   }>
+  /** The transaction data for the output */
+  beef?: number[]
   blockHeight?: number
 }

package/src/__tests/Engine.test.ts

@@ -1,4 +1,5 @@
-import { Engine } from '../Engine'
+// Note: References Engine from dist due to issues with compiled code references in Engine.ts
+import { Engine } from '../../dist/cjs/src/Engine'
 import { LookupService } from '../LookupService'
 import { TopicManager } from '../TopicManager'
 import { AdmittanceInstructions } from '../AdmittanceInstructions'
@@ -60,7 +61,7 @@ describe('BSV Overlay Services Engine', () => {
       findOutputsForTransaction: jest.fn(async () => []),
       markUTXOAsSpent: jest.fn(),
       updateConsumedBy: jest.fn(),
-      updateOutputBeef: jest.fn(),
+      updateTransactionBEEF: jest.fn(),
       deleteOutput: jest.fn(),
       findUTXOsForTopic: jest.fn()
     }
@@ -179,7 +180,7 @@ describe('BSV Overlay Services Engine', () => {
       let newBEEF: number[] = []
       mockStorageEngine.findOutput = jest.fn(async () => output27c8f_0)
       mockStorageEngine.findOutputsForTransaction = jest.fn(async () => [output27c8f_0])
-      mockStorageEngine.updateOutputBeef = jest.fn(async (txid: string, outputIndex: number, topic: string, beef: number[]) => {
+      mockStorageEngine.updateTransactionBEEF = jest.fn(async (txid: string, beef: number[]) => {
         newBEEF = beef
       })
       const engine = new Engine(
@@ -201,7 +202,7 @@ describe('BSV Overlay Services Engine', () => {
 
     it('1 recurse proof', async () => {
       const outputs: Output[] = []
-      const findOutput = (txid: string, outputIndex: number): Output => {
+      const findOutput = (txid: string, outputIndex: number, includeBEEF?: boolean): Output => {
         const i = outputs.findIndex(o => o.txid === txid && o.outputIndex === outputIndex)
         if (i < 0) throw new Error(`missing output ${txid} ${outputIndex}`)
         return outputs[i]
@@ -230,15 +231,15 @@ describe('BSV Overlay Services Engine', () => {
 
       mockLookupService.lookup = jest.fn(async () => [{ txid: txid17d182, outputIndex: 0, history: 1 }])
       const newBEEF: Record<string, string> = {}
-      mockStorageEngine.findOutput = jest.fn(async (txid: string, outputIndex: number, topic?: string, spent?: boolean) => {
-        return findOutput(txid, outputIndex)
+      mockStorageEngine.findOutput = jest.fn(async (txid: string, outputIndex: number, topic?: string, spent?: boolean, includeBEEF?: boolean) => {
+        return findOutput(txid, outputIndex, true)
       })
-      mockStorageEngine.findOutputsForTransaction = jest.fn(async (txid: string) => {
+      mockStorageEngine.findOutputsForTransaction = jest.fn(async (txid: string, includeBEEF?: boolean) => {
         const os = outputs.filter(o => o.txid === txid)
         return os
       })
-      mockStorageEngine.updateOutputBeef = jest.fn(async (txid: string, outputIndex: number, topic: string, beef: number[]) => {
-        newBEEF[`${txid}_${outputIndex}`] = Utils.toHex(beef)
+      mockStorageEngine.updateTransactionBEEF = jest.fn(async (txid: string, beef: number[]) => {
+        newBEEF[txid] = Utils.toHex(beef)
       })
       const engine = new Engine({ Hello: mockTopicManager }, { Hello: mockLookupService }, mockStorageEngine, mockChainTracker)
 
@@ -270,34 +271,34 @@ describe('BSV Overlay Services Engine', () => {
 
       const mp877734 = Transaction.fromHexBEEF(beef877734_0).merklePath
       await engine.handleNewMerkleProof(txid877734, mp877734)
-      expect(newBEEF[`${txid877734}_0`]).toBe(beef877734_0)
-      expect(newBEEF[`${txid509f5e}_0`].length).toBeGreaterThan(beef509f5e_0.length)
-      expect(newBEEF[`${txid942620}_0`].length).toBeGreaterThan(beef942620_0.length)
-      expect(newBEEF[`${txid17d182}_0`].length).toBeGreaterThan(beef17d182_0.length)
+      expect(newBEEF[`${txid877734}`]).toBe(beef877734_0)
+      expect(newBEEF[`${txid509f5e}`].length).toBeGreaterThan(beef509f5e_0.length)
+      expect(newBEEF[`${txid942620}`].length).toBeGreaterThan(beef942620_0.length)
+      expect(newBEEF[`${txid17d182}`].length).toBeGreaterThan(beef17d182_0.length)
       expect(Object.keys(newBEEF).length).toBe(4)
 
       const mp509f5e = Transaction.fromHexBEEF(beef509f5e_0).merklePath
       await engine.handleNewMerkleProof(txid509f5e, mp509f5e)
-      expect(newBEEF[`${txid877734}_0`]).toBe(beef877734_0)
-      expect(newBEEF[`${txid509f5e}_0`]).toBe(beef509f5e_0)
-      expect(newBEEF[`${txid942620}_0`].length).toBeGreaterThan(beef942620_0.length)
-      expect(newBEEF[`${txid17d182}_0`].length).toBeGreaterThan(beef17d182_0.length)
+      expect(newBEEF[`${txid877734}`]).toBe(beef877734_0)
+      expect(newBEEF[`${txid509f5e}`]).toBe(beef509f5e_0)
+      expect(newBEEF[`${txid942620}`].length).toBeGreaterThan(beef942620_0.length)
+      expect(newBEEF[`${txid17d182}`].length).toBeGreaterThan(beef17d182_0.length)
       expect(Object.keys(newBEEF).length).toBe(4)
 
       const mp942620 = Transaction.fromHexBEEF(beef942620_0).merklePath
       await engine.handleNewMerkleProof(txid942620, mp942620)
-      expect(newBEEF[`${txid877734}_0`]).toBe(beef877734_0)
-      expect(newBEEF[`${txid509f5e}_0`]).toBe(beef509f5e_0)
-      expect(newBEEF[`${txid942620}_0`]).toBe(beef942620_0)
-      expect(newBEEF[`${txid17d182}_0`].length).toBeGreaterThan(beef17d182_0.length)
+      expect(newBEEF[`${txid877734}`]).toBe(beef877734_0)
+      expect(newBEEF[`${txid509f5e}`]).toBe(beef509f5e_0)
+      expect(newBEEF[`${txid942620}`]).toBe(beef942620_0)
+      expect(newBEEF[`${txid17d182}`].length).toBeGreaterThan(beef17d182_0.length)
       expect(Object.keys(newBEEF).length).toBe(4)
 
       const mp17d182 = Transaction.fromHexBEEF(beef17d182_0).merklePath
       await engine.handleNewMerkleProof(txid17d182, mp17d182)
-      expect(newBEEF[`${txid877734}_0`]).toBe(beef877734_0)
-      expect(newBEEF[`${txid509f5e}_0`]).toBe(beef509f5e_0)
-      expect(newBEEF[`${txid942620}_0`]).toBe(beef942620_0)
-      expect(newBEEF[`${txid17d182}_0`]).toBe(beef17d182_0)
+      expect(newBEEF[`${txid877734}`]).toBe(beef877734_0)
+      expect(newBEEF[`${txid509f5e}`]).toBe(beef509f5e_0)
+      expect(newBEEF[`${txid942620}`]).toBe(beef942620_0)
+      expect(newBEEF[`${txid17d182}`]).toBe(beef17d182_0)
       expect(Object.keys(newBEEF).length).toBe(4)
     })
   })
@@ -763,7 +764,7 @@ describe('BSV Overlay Services Engine', () => {
             query: { name: 'Bob' }
           })
           expect(mockStorageEngine.findOutput).toHaveBeenCalledWith(
-            'mockTXID', 0, undefined, false
+            'mockTXID', 0, undefined, undefined, true
           )
         })
         it('Calls getUTXOHistory with the correct UTXO and history parameters', async () => {

package/src/__tests/OverlayGASPRemote.test.ts

@@ -1,4 +1,4 @@
-import { OverlayGASPRemote } from '../Engine'
+import { OverlayGASPRemote } from '../GASP/OverlayGASPRemote'
 import { GASPInitialRequest, GASPInitialResponse, GASPNode } from '@bsv/gasp'
 
 global.fetch = jest.fn(async () => {

package/src/__tests/OverlayGASPStorage.test.ts

@@ -1,4 +1,4 @@
-import { GraphNode, OverlayGASPStorage } from '../Engine'
+import { GraphNode, OverlayGASPStorage } from '../GASP/OverlayGASPStorage'
 import { Transaction, MerklePath } from '@bsv/sdk'
 import { GASPNode } from '@bsv/gasp'
 
@@ -61,9 +61,10 @@ describe('OverlayGASPStorage', () => {
   })
 
   describe('hydrateGASPNode', () => {
-    it('should throw an error', async () => {
-      await expect(overlayStorage.hydrateGASPNode('graphID', 'txid', 0, false)).rejects.toThrow('GASP node hydration Not supported!')
+    it('should throw an error if no output is found', async () => {
+      await expect(overlayStorage.hydrateGASPNode('graphID', 'txid', 0, false)).rejects.toThrow('No matching output found!')
     })
+    // TODO: Further test coverage
   })
 
   describe('findNeededInputs', () => {

package/src/storage/Storage.ts

@@ -27,13 +27,13 @@ export interface Storage {
    * @param topic — The topic in which the output is stored
    * @param spent — Whether the output must be spent to be returned
    */
-  findOutput: (txid: string, outputIndex: number, topic?: string, spent?: boolean) => Promise<Output | null>
+  findOutput: (txid: string, outputIndex: number, topic?: string, spent?: boolean, includeBEEF?: boolean) => Promise<Output | null>
 
   /**
    * Finds outputs with a matching transaction ID from storage
    * @param txid — TXID of the outputs to find
    */
-  findOutputsForTransaction: (txid: string) => Promise<Output[]>
+  findOutputsForTransaction: (txid: string, includeBEEF?: boolean) => Promise<Output[]>
 
   /**
    * Finds current UTXOs that have been admitted into a given topic
@@ -41,7 +41,7 @@ export interface Storage {
    * @param since - Optional parameter indicating the minimum date (timestamp? block height?) to retrieve matching UTXOs from. TODO: We need to decide about which time format to use for synchronization.
    * @returns A promise that resolves to an array of matching UTXOs.
    */
-  findUTXOsForTopic: (topic: string, since?: number) => Promise<Output[]>
+  findUTXOsForTopic: (topic: string, since?: number, includeBEEF?: boolean) => Promise<Output[]>
 
   /**
    * Deletes an output from storage
@@ -72,13 +72,11 @@ export interface Storage {
   }>) => Promise<void>
 
   /**
-   * Updates the beef data on an output
-   * @param txid — TXID of the output to update
-   * @param outputIndex — Index of the output to update
-   * @param topic — Topic in which the output should be updated
+   * Updates the beef data for a transaction
+   * @param txid — TXID of the transaction to update
    * @param beef - BEEF data to update
    */
-  updateOutputBeef: (txid: string, outputIndex: number, topic: string, beef: number[]) => Promise<void>
+  updateTransactionBEEF: (txid: string, beef: number[]) => Promise<void>
 
 
   /**