@bsv/overlay 0.5.3 → 0.6.0

package/src/Engine.ts

@@ -16,7 +16,9 @@ import {
   HTTPSOverlayBroadcastFacilitator,
   LookupResolver,
   LookupResolverConfig,
-  OverlayBroadcastFacilitator
+  OverlayBroadcastFacilitator,
+  BroadcastResponse,
+  BroadcastFailure
 } from '@bsv/sdk'
 import { AdvertisementData, Advertiser } from './Advertiser.js'
 import { GASP, GASPInitialRequest, GASPInitialResponse, GASPNode } from '@bsv/gasp'
@@ -132,7 +134,7 @@ export class Engine {
    *
    * @returns {Promise<STEAK>} The submitted transaction execution acknowledgement
    */
-  async submit(taggedBEEF: TaggedBEEF, onSteakReady?: (steak: STEAK) => void, mode: 'historical-tx' | 'current-tx' = 'current-tx', offChainValues?: number[]): Promise<STEAK> {
+  async submit(taggedBEEF: TaggedBEEF, onSteakReady?: (steak: STEAK) => void, mode: 'historical-tx' | 'current-tx' | 'historical-tx-no-spv' = 'current-tx', offChainValues?: number[]): Promise<STEAK> {
     for (const t of taggedBEEF.topics) {
       if (this.managers[t] === undefined || this.managers[t] === null) {
         throw new Error(`This server does not support this topic: ${t}`)
@@ -144,10 +146,12 @@ export class Engine {
     const txid = tx.id('hex')
 
     this.startTime(`submit_${txid}`)
-    this.startTime(`chainTracker_${txid.substring(0, 10)}`)
-    const txValid = await tx.verify(this.chainTracker)
-    if (!txValid) throw new Error('Unable to verify SPV information.')
-    this.endTime(`chainTracker_${txid.substring(0, 10)}`)
+    if (mode !== 'historical-tx-no-spv') {
+      this.startTime(`chainTracker_${txid.substring(0, 10)}`)
+      const txValid = await tx.verify(this.chainTracker)
+      if (!txValid) throw new Error('Unable to verify SPV information.')
+      this.endTime(`chainTracker_${txid.substring(0, 10)}`)
+    }
 
     const steak: STEAK = {}
     const dupeTopics = new Set<string>()
@@ -210,7 +214,8 @@ export class Engine {
         const admissibleOutputs = await this.managers[topic].identifyAdmissibleOutputs(
           taggedBEEF.beef,
           previousCoins,
-          offChainValues
+          offChainValues,
+          mode
         )
         this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`)
 
@@ -247,7 +252,21 @@ export class Engine {
     this.startTime(`broadcast_${txid.substring(0, 10)}`)
     if (mode !== 'historical-tx' && this.broadcaster !== undefined) {
       try {
-        const response = await this.broadcaster.broadcast(tx)
+        let response: BroadcastResponse | BroadcastFailure
+        if (tx.merklePath !== undefined) {
+          // tx has been verified, thus if there is a merklePath, the transaction is already on-chain...skip broadcast.
+          const txid = tx.id('hex')
+          const mp = tx.merklePath
+          const leaf = mp.path[0].find(leaf => leaf.hash === txid)
+          const r: BroadcastResponse = {
+            status: 'success',
+            txid: tx.id('hex'),
+            message: `In block at height ${mp.blockHeight} index ${leaf?.offset}`,
+          }
+          response = r
+        } else {
+          response = await this.broadcaster.broadcast(tx)
+        }
         if (isBroadcastFailure(response) && this.throwOnBroadcastFailure) {
           const e = new Error(`Failed to broadcast transaction! Error: ${response.description}`)
             ; (e as any).more = response.more
@@ -483,7 +502,7 @@ export class Engine {
     }
 
     // If we don't have an advertiser or we are dealing with historical transactions, just return the steak
-    if (this.advertiser === undefined || mode === 'historical-tx') {
+    if (this.advertiser === undefined || mode === 'historical-tx' || mode === 'historical-tx-no-spv') {
       return steak
     }
 

package/src/GASP/OverlayGASPStorage.ts

@@ -21,9 +21,45 @@ export interface GraphNode {
 
 export class OverlayGASPStorage implements GASPStorage {
   readonly temporaryGraphNodeRefs: Record<string, GraphNode> = {}
+  private static activeAnchorValidations = 0
+  private static readonly anchorValidationQueue: Array<() => void> = []
+  private static activeFinalizations = 0
+  private static readonly finalizationQueue: Array<() => void> = []
+  private static readonly MAX_CONCURRENT_ANCHOR_VALIDATIONS = 4
+  private static readonly MAX_CONCURRENT_FINALIZATIONS = 2
 
   constructor (public topic: string, public engine: Engine, public maxNodesInGraph?: number) { }
 
+  private static async acquireAnchorValidationSlot (): Promise<void> {
+    if (OverlayGASPStorage.activeAnchorValidations >= OverlayGASPStorage.MAX_CONCURRENT_ANCHOR_VALIDATIONS) {
+      await new Promise<void>(resolve => { OverlayGASPStorage.anchorValidationQueue.push(resolve) })
+    }
+    OverlayGASPStorage.activeAnchorValidations++
+  }
+
+  private static releaseAnchorValidationSlot (): void {
+    OverlayGASPStorage.activeAnchorValidations--
+    const next = OverlayGASPStorage.anchorValidationQueue.shift()
+    if (next !== undefined) {
+      next()
+    }
+  }
+
+  private static async acquireFinalizationSlot (): Promise<void> {
+    if (OverlayGASPStorage.activeFinalizations >= OverlayGASPStorage.MAX_CONCURRENT_FINALIZATIONS) {
+      await new Promise<void>(resolve => { OverlayGASPStorage.finalizationQueue.push(resolve) })
+    }
+    OverlayGASPStorage.activeFinalizations++
+  }
+
+  private static releaseFinalizationSlot (): void {
+    OverlayGASPStorage.activeFinalizations--
+    const next = OverlayGASPStorage.finalizationQueue.shift()
+    if (next !== undefined) {
+      next()
+    }
+  }
+
   /**
    *
    * @param since
@@ -91,8 +127,12 @@ export class OverlayGASPStorage implements GASPStorage {
 
     // Attempt to check if the current transaction is admissible
     parsedTx.merklePath = MerklePath.fromHex(tx.proof)
-    const admittanceResult = await this.engine.managers[this.topic].identifyAdmissibleOutputs(parsedTx.toBEEF(), [], typeof tx.txMetadata === 'string' ? Utils.toArray(tx.txMetadata) : undefined)
-
+    const admittanceResult = await this.engine.managers[this.topic].identifyAdmissibleOutputs(
+      parsedTx.toBEEF(),
+      [],
+      typeof tx.txMetadata === 'string' ? Utils.toArray(tx.txMetadata) : undefined,
+      'historical-tx'
+    )
     if (admittanceResult.outputsToAdmit.includes(tx.outputIndex)) {
       // The transaction is admissible, no further inputs are needed
     } else {
@@ -196,51 +236,61 @@ export class OverlayGASPStorage implements GASPStorage {
     * @throws If the graph is not well-anchored, according to the rules of Bitcoin or the rules of the Overlay Topic Manager.
     */
   async validateGraphAnchor (graphID: string): Promise<void> {
-    const rootNode = this.temporaryGraphNodeRefs[graphID]
-    if (rootNode === undefined) {
-      throw new Error(`Graph node with ID ${graphID} not found`)
-    }
-
-    // Check that the root node is Bitcoin-valid.
-    const beef = this.getBEEFForNode(rootNode)
-    const spvTx = Transaction.fromBEEF(beef)
-    const isBitcoinValid = await spvTx.verify(this.engine.chainTracker)
-    if (!isBitcoinValid) {
-      throw new Error('The graph is not well-anchored according to the rules of Bitcoin.')
-    }
-
-    // Then, ensure the node is Overlay-valid.
-    const beefs = this.computeOrderedBEEFsForGraph(graphID)
+    await OverlayGASPStorage.acquireAnchorValidationSlot()
+    try {
+      const rootNode = this.temporaryGraphNodeRefs[graphID]
+      if (rootNode === undefined) {
+        throw new Error(`Graph node with ID ${graphID} not found`)
+      }
 
-    // coins: a Set of all historical coins to retain (no need to remove them), used to emulate topical admittance of previous inputs over time.
-    const coins = new Set<string>()
+      // Check that the root node is Bitcoin-valid.
+      const beef = this.getBEEFForNode(rootNode)
+      const spvTx = Transaction.fromBEEF(beef)
+      const isBitcoinValid = await spvTx.verify(this.engine.chainTracker)
+      if (!isBitcoinValid) {
+        throw new Error('The graph is not well-anchored according to the rules of Bitcoin.')
+      }
 
-    // Submit all historical BEEFs in order through the topic manager, tracking what would be retained until we submit the root node last.
-    // If, at the end, the root node is admitted, we have a valid overlay-specific graph.
-    for (const beef of beefs) {
-      // For any input to this transaction, see if it's a valid coin that's admitted. If so, it's a previous coin.
-      const previousCoins: number[] = []
-      const tx = Transaction.fromBEEF(beef)
-      for (const [inputIndex, input] of tx.inputs.entries()) {
-        const sourceTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex')
-        if (sourceTXID != null && sourceTXID !== '') {
-          const coin = `${sourceTXID}.${input.sourceOutputIndex}`
-          if (coins.has(coin)) {
-            previousCoins.push(Number(inputIndex))
+      // Then, ensure the node is Overlay-valid.
+      const beefs = this.computeOrderedBEEFsForGraph(graphID)
+
+      // coins: a Set of all historical coins to retain (no need to remove them), used to emulate topical admittance of previous inputs over time.
+      const coins = new Set<string>()
+
+      // Submit all historical BEEFs in order through the topic manager, tracking what would be retained until we submit the root node last.
+      // If, at the end, the root node is admitted, we have a valid overlay-specific graph.
+      for (const beef of beefs) {
+        // For any input to this transaction, see if it's a valid coin that's admitted. If so, it's a previous coin.
+        const previousCoins: number[] = []
+        const tx = Transaction.fromBEEF(beef)
+        for (const [inputIndex, input] of tx.inputs.entries()) {
+          const sourceTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex')
+          if (sourceTXID != null && sourceTXID !== '') {
+            const coin = `${sourceTXID}.${input.sourceOutputIndex}`
+            if (coins.has(coin)) {
+              previousCoins.push(Number(inputIndex))
+            }
           }
         }
+        const admittanceInstructions = await this.engine.managers[this.topic].identifyAdmissibleOutputs(
+          beef,
+          previousCoins,
+          undefined,
+          'historical-tx'
+        )
+        // Every admitted output is now a coin.
+        for (const outputIndex of admittanceInstructions.outputsToAdmit) {
+          coins.add(`${tx.id('hex')}.${outputIndex}`)
+        }
       }
-      const admittanceInstructions = await this.engine.managers[this.topic].identifyAdmissibleOutputs(beef, previousCoins)
-      // Every admitted output is now a coin.
-      for (const outputIndex of admittanceInstructions.outputsToAdmit) {
-        coins.add(`${tx.id('hex')}.${outputIndex}`)
+      // After sending through all the graph's BEEFs...
+      // If the root node is now a coin, we have acceptance by the overlay.
+      // Otherwise, throw.
+      if (!coins.has(graphID)) {
+        throw new Error('This graph did not result in topical admittance of the root node. Rejecting.')
       }
-    }
-    // After sending through all the graph's BEEFs...
-    // If the root node is now a coin, we have acceptance by the overlay.
-    // Otherwise, throw.
-    if (!coins.has(graphID)) {
-      throw new Error('This graph did not result in topical admittance of the root node. Rejecting.')
+    } finally {
+      OverlayGASPStorage.releaseAnchorValidationSlot()
     }
   }
 
@@ -263,14 +313,20 @@ export class OverlayGASPStorage implements GASPStorage {
    * @param graphID The TXID and output index (in 36-byte format) for the UTXO at the root of this graph.
    */
   async finalizeGraph (graphID: string): Promise<void> {
-    const beefs = this.computeOrderedBEEFsForGraph(graphID)
-
-    // Submit all historical BEEFs in order, finalizing the graph for the current UTXO
-    for (const beef of beefs) {
-      await this.engine.submit({
-        beef,
-        topics: [this.topic]
-      }, () => { }, 'historical-tx')
+    await OverlayGASPStorage.acquireFinalizationSlot()
+    try {
+      const beefs = this.computeOrderedBEEFsForGraph(graphID)
+
+      // Submit all historical BEEFs in order, finalizing the graph for the current UTXO.
+      // We skip SPV verification here because validateGraphAnchor has already done it.
+      for (const beef of beefs) {
+        await this.engine.submit({
+          beef,
+          topics: [this.topic]
+        }, () => { }, 'historical-tx-no-spv')
+      }
+    } finally {
+      OverlayGASPStorage.releaseFinalizationSlot()
     }
   }
 

package/src/TopicManager.ts

@@ -9,7 +9,12 @@ export interface TopicManager {
    * Accepts the transaction in BEEF format and an array of those input indices which spend previously-admitted outputs from the same topic.
    * The transaction's BEEF structure will always contain the transactions associated with previous coins for reference (if any), regardless of whether the current transaction was directly proven.
    */
-  identifyAdmissibleOutputs: (beef: number[], previousCoins: number[], offChainValues?: number[]) => Promise<AdmittanceInstructions>
+  identifyAdmissibleOutputs: (
+    beef: number[],
+    previousCoins: number[],
+    offChainValues?: number[],
+    mode?: 'historical-tx' | 'current-tx' | 'historical-tx-no-spv'
+  ) => Promise<AdmittanceInstructions>
 
   /**
    * Identifies and returns the inputs needed to anchor any topical outputs from this transaction to their associated previous history.

package/src/__tests/Engine.test.ts

@@ -528,12 +528,16 @@ describe('BSV Overlay Services Engine', () => {
           mockChainTracker
         )
 
-        // Submit the utxo
         await engine.submit({
           beef: exampleBeef,
           topics: ['Hello']
         })
-        expect(engine.managers.Hello.identifyAdmissibleOutputs).toHaveBeenCalledWith(exampleBeef, [0], undefined)
+        expect(engine.managers.Hello.identifyAdmissibleOutputs).toHaveBeenCalledWith(
+          exampleBeef,
+          [0],
+          undefined,
+          'current-tx'
+        )
       })
       describe('When previous UTXOs were retained by the topic manager', () => {
         it('Notifies all lookup services about the output being spent (not deleted, see the comment about this in deleteUTXODeep)', async () => {

package/src/storage/knex/KnexStorage.ts

@@ -170,28 +170,33 @@ export class KnexStorage implements Storage {
   }
 
   async insertOutput (output: Output): Promise<void> {
-    const insertPromises = [this.knex('outputs').insert({
-      txid: output.txid,
-      outputIndex: Number(output.outputIndex),
-      outputScript: Buffer.from(output.outputScript),
-      topic: output.topic,
-      satoshis: Number(output.satoshis),
-      outputsConsumed: JSON.stringify(output.outputsConsumed),
-      consumedBy: JSON.stringify(output.consumedBy),
-      spent: output.spent,
-      score: output.score
-    })]
-
-    if (output.beef !== undefined) {
-      const insertTransactionPromise = this.knex('transactions').insert({
+    await this.knex.transaction(async trx => {
+      const existing = await trx('outputs').where({
         txid: output.txid,
-        beef: Buffer.from(output.beef)
-      }).onConflict('txid').ignore()
-      insertPromises.push(insertTransactionPromise)
-    }
+        outputIndex: Number(output.outputIndex),
+        topic: output.topic
+      }).first()
+
+      if (existing === undefined || existing === null) {
+        await trx('outputs').insert({
+          txid: output.txid,
+          outputIndex: Number(output.outputIndex),
+          outputScript: Buffer.from(output.outputScript),
+          topic: output.topic,
+          satoshis: Number(output.satoshis),
+          outputsConsumed: JSON.stringify(output.outputsConsumed),
+          consumedBy: JSON.stringify(output.consumedBy),
+          spent: output.spent,
+          score: output.score
+        })
+      }
 
-    await this.knex.transaction(async trx => {
-      await Promise.all(insertPromises.map(promise => promise.transacting(trx)))
+      if (output.beef !== undefined) {
+        await trx('transactions').insert({
+          txid: output.txid,
+          beef: Buffer.from(output.beef)
+        }).onConflict('txid').ignore()
+      }
     })
   }