@bsv/overlay 0.5.2 → 0.5.4

package/src/Engine.ts

@@ -16,7 +16,9 @@ import {
   HTTPSOverlayBroadcastFacilitator,
   LookupResolver,
   LookupResolverConfig,
-  OverlayBroadcastFacilitator
+  OverlayBroadcastFacilitator,
+  BroadcastResponse,
+  BroadcastFailure
 } from '@bsv/sdk'
 import { AdvertisementData, Advertiser } from './Advertiser.js'
 import { GASP, GASPInitialRequest, GASPInitialResponse, GASPNode } from '@bsv/gasp'
@@ -153,8 +155,18 @@ export class Engine {
     const dupeTopics = new Set<string>()
     const failedTopics = new Set<string>()
 
-    // Parallelize the topic processing
-    const topicPromises = taggedBEEF.topics.map(async (topic) => {
+    // ===================================================================
+    // PHASE 1: VALIDATE (read-only, no mutations)
+    // ===================================================================
+    type TopicValidation = {
+      topic: string
+      isDupe: boolean
+      previousCoins: number[]
+      previousOutputs: Array<Output | null>
+      admissibleOutputs: AdmittanceInstructions
+    }
+
+    const topicValidations = taggedBEEF.topics.map(async (topic): Promise<TopicValidation> => {
       try {
         if (this.managers[topic] === undefined || this.managers[topic] === null) {
           throw new Error(`This server does not support this topic: ${topic}`)
@@ -167,8 +179,13 @@ export class Engine {
 
         if (dupeCheck) {
           dupeTopics.add(topic)
-          steak[topic] = { outputsToAdmit: [], coinsToRetain: [] }
-          return
+          return {
+            topic,
+            isDupe: true,
+            previousCoins: [],
+            previousOutputs: [],
+            admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
+          }
         }
 
         // Identify previous coins admitted to this specific topic
@@ -187,108 +204,69 @@ export class Engine {
         })
 
         this.startTime(`previousOutputQuery_${txid.substring(0, 10)}`)
-        const outputs = await Promise.all(outputPromises)
+        const previousOutputs = await Promise.all(outputPromises)
         this.endTime(`previousOutputQuery_${txid.substring(0, 10)}`)
 
-        const markSpentPromises = outputs.map(async (output) => {
-          if (output !== undefined && output !== null) {
-            try {
-              await this.storage.markUTXOAsSpent(output.txid, output.outputIndex, topic)
-              await Promise.all(Object.values(this.lookupServices).map(async l => {
-                try {
-                  if (typeof l.outputSpent === 'function') {
-                    if (l.spendNotificationMode === 'txid') {
-                      await l.outputSpent({
-                        mode: 'txid',
-                        spendingTxid: txid,
-                        txid: output.txid,
-                        outputIndex: output.outputIndex,
-                        topic
-                      })
-                    } else if (l.spendNotificationMode === 'script') {
-                      const inputIndex = tx.inputs.findIndex(i => {
-                        let realSource = i.sourceTXID
-                        if (!realSource) {
-                          realSource = i.sourceTransaction?.id('hex')
-                        }
-                        return realSource === output.txid && i.sourceOutputIndex === output.outputIndex
-                      })
-                      if (inputIndex === -1) {
-                        throw new Error('Could not find input index')
-                      }
-                      await l.outputSpent({
-                        mode: 'script',
-                        spendingTxid: txid,
-                        inputIndex,
-                        sequenceNumber: tx.inputs[inputIndex].sequence ?? 0xffffffff,
-                        unlockingScript: tx.inputs[inputIndex].unlockingScript!,
-                        txid: output.txid,
-                        outputIndex: output.outputIndex,
-                        topic,
-                        offChainValues
-                      })
-                    } else if (l.spendNotificationMode === 'whole-tx') {
-                      await l.outputSpent({
-                        mode: 'whole-tx',
-                        spendingAtomicBEEF: tx.toAtomicBEEF(),
-                        txid: output.txid,
-                        outputIndex: output.outputIndex,
-                        topic,
-                        offChainValues
-                      })
-                    } else { // none
-                      await l.outputSpent({
-                        mode: 'none',
-                        txid: output.txid,
-                        outputIndex: output.outputIndex,
-                        topic
-                      })
-                    }
-                  }
-                } catch (error) {
-                  this.logger.error('Error in lookup service for outputSpent:', error)
-                }
-              }))
-            } catch (error) {
-              this.logger.error('Error marking UTXO as spent:', error)
-            }
-          }
-        })
-
-        let admissibleOutputs: AdmittanceInstructions = { outputsToAdmit: [], coinsToRetain: [] }
-        // Determine which outputs are admissible for this topic
-        const admissibleOutputsPromise = (async () => {
-          try {
-            this.startTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`)
-            admissibleOutputs = await this.managers[topic].identifyAdmissibleOutputs(taggedBEEF.beef, previousCoins, offChainValues)
-            this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`)
-          } catch (_) {
-            failedTopics.add(topic)
-            steak[topic] = { outputsToAdmit: [], coinsToRetain: [] }
-          }
-        })()
+        // Determine which outputs are admissible for this topic (validation only)
+        this.startTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`)
+        const admissibleOutputs = await this.managers[topic].identifyAdmissibleOutputs(
+          taggedBEEF.beef,
+          previousCoins,
+          offChainValues
+        )
+        this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`)
 
-        // Wait for both tasks to complete
-        await Promise.all([...markSpentPromises, admissibleOutputsPromise])
-        // Keep track of what outputs were admitted for what topic
-        steak[topic] = admissibleOutputs
+        return {
+          topic,
+          isDupe: false,
+          previousCoins,
+          previousOutputs,
+          admissibleOutputs
+        }
       } catch (error) {
-        this.logger.error('Error processing topic during submit:', error)
+        this.logger.error('Error validating topic during submit:', error)
         failedTopics.add(topic)
-        steak[topic] = { outputsToAdmit: [], coinsToRetain: [] }
+        return {
+          topic,
+          isDupe: false,
+          previousCoins: [],
+          previousOutputs: [],
+          admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
+        }
       }
     })
 
-    await Promise.all(topicPromises)
+    const validations = await Promise.all(topicValidations)
+
+    // Build preliminary STEAK from validation results
+    for (const validation of validations) {
+      steak[validation.topic] = validation.admissibleOutputs
+    }
 
-    // Broadcast the transaction if not historical and broadcaster is configured
+    // ===================================================================
+    // PHASE 2: BROADCAST (before any mutations)
+    // ===================================================================
     this.startTime(`broadcast_${txid.substring(0, 10)}`)
     if (mode !== 'historical-tx' && this.broadcaster !== undefined) {
       try {
-        const response = await this.broadcaster.broadcast(tx)
+        let response: BroadcastResponse | BroadcastFailure
+        if (tx.merklePath !== undefined) {
+          // tx has been verified, thus if there is a merklePath, the transaction is already on-chain...skip broadcast.
+          const txid = tx.id('hex')
+          const mp = tx.merklePath
+          const leaf = mp.path[0].find(leaf => leaf.hash === txid)
+          const r: BroadcastResponse = {
+            status: 'success',
+            txid: tx.id('hex'),
+            message: `In block at height ${mp.blockHeight} index ${leaf?.offset}`,
+          }
+          response = r
+        } else {
+          response = await this.broadcaster.broadcast(tx)
+        }
         if (isBroadcastFailure(response) && this.throwOnBroadcastFailure) {
           const e = new Error(`Failed to broadcast transaction! Error: ${response.description}`)
-          ;(e as any).more = response.more
+            ; (e as any).more = response.more
           throw e
         }
       } catch (error) {
@@ -300,13 +278,95 @@ export class Engine {
     }
     this.endTime(`broadcast_${txid.substring(0, 10)}`)
 
-    // Call the callback function if it is provided (moved here to ensure topic processing is complete)
+    // Call the callback function with STEAK if it is provided (before storage mutations)
     if (onSteakReady !== undefined) {
       onSteakReady(steak)
     }
 
-    // Update storage and notify lookup services
-    for (const topic of taggedBEEF.topics) {
+    // ===================================================================
+    // PHASE 3: MUTATE STORAGE (only after broadcast succeeded)
+    // ===================================================================
+    // Mark previous outputs as spent and notify lookup services
+    await Promise.all(validations.map(async (validation) => {
+      if (validation.isDupe || failedTopics.has(validation.topic)) {
+        return
+      }
+
+      const topic = validation.topic
+      const previousOutputs = validation.previousOutputs
+
+      // Mark all previous outputs as spent
+      const markSpentPromises = previousOutputs.map(async (output) => {
+        if (output !== undefined && output !== null) {
+          try {
+            await this.storage.markUTXOAsSpent(output.txid, output.outputIndex, topic)
+            await Promise.all(Object.values(this.lookupServices).map(async l => {
+              try {
+                if (typeof l.outputSpent === 'function') {
+                  if (l.spendNotificationMode === 'txid') {
+                    await l.outputSpent({
+                      mode: 'txid',
+                      spendingTxid: txid,
+                      txid: output.txid,
+                      outputIndex: output.outputIndex,
+                      topic
+                    })
+                  } else if (l.spendNotificationMode === 'script') {
+                    const inputIndex = tx.inputs.findIndex(i => {
+                      let realSource = i.sourceTXID
+                      if (!realSource) {
+                        realSource = i.sourceTransaction?.id('hex')
+                      }
+                      return realSource === output.txid && i.sourceOutputIndex === output.outputIndex
+                    })
+                    if (inputIndex === -1) {
+                      throw new Error('Could not find input index')
+                    }
+                    await l.outputSpent({
+                      mode: 'script',
+                      spendingTxid: txid,
+                      inputIndex,
+                      sequenceNumber: tx.inputs[inputIndex].sequence ?? 0xffffffff,
+                      unlockingScript: tx.inputs[inputIndex].unlockingScript!,
+                      txid: output.txid,
+                      outputIndex: output.outputIndex,
+                      topic,
+                      offChainValues
+                    })
+                  } else if (l.spendNotificationMode === 'whole-tx') {
+                    await l.outputSpent({
+                      mode: 'whole-tx',
+                      spendingAtomicBEEF: tx.toAtomicBEEF(),
+                      txid: output.txid,
+                      outputIndex: output.outputIndex,
+                      topic,
+                      offChainValues
+                    })
+                  } else { // none
+                    await l.outputSpent({
+                      mode: 'none',
+                      txid: output.txid,
+                      outputIndex: output.outputIndex,
+                      topic
+                    })
+                  }
+                }
+              } catch (error) {
+                this.logger.error('Error in lookup service for outputSpent:', error)
+              }
+            }))
+          } catch (error) {
+            this.logger.error('Error marking UTXO as spent:', error)
+          }
+        }
+      })
+
+      await Promise.all(markSpentPromises)
+    }))
+
+    // Continue with storage updates and lookup service notifications
+    for (const validation of validations) {
+      const topic = validation.topic
       if (dupeTopics.has(topic)) {
         continue
       }
@@ -327,19 +387,10 @@ export class Engine {
           inputIndex: number
         }> = []
 
-      // Recompute previousCoins for this topic to use in the update logic
-        const previousCoins: number[] = []
-        await Promise.all(tx.inputs.map(async (input, i) => {
-          const previousTXID = input.sourceTXID !== undefined ? input.sourceTXID : input.sourceTransaction?.id('hex')
-          if (previousTXID !== undefined) {
-            const output = await this.storage.findOutput(previousTXID, input.sourceOutputIndex, topic)
-            if (output !== undefined && output !== null) {
-              previousCoins.push(i)
-            }
-          }
-        }))
+        // Use previousCoins from validation
+        const previousCoins = validation.previousCoins
 
-      // For each of the previous UTXOs for this topic, if the UTXO was not included in the list of UTXOs identified for retention, then it will be marked as stale.
+        // For each of the previous UTXOs for this topic, if the UTXO was not included in the list of UTXOs identified for retention, then it will be marked as stale.
         for (const inputIndex of previousCoins) {
           const previousTXID = tx.inputs[inputIndex].sourceTXID ?? tx.inputs[inputIndex].sourceTransaction?.id('hex')
           if (typeof previousTXID !== 'string') continue
@@ -358,7 +409,7 @@ export class Engine {
           }
         }
 
-      // Remove stale outputs recursively
+        // Remove stale outputs recursively
         this.startTime(`lookForStaleOutputs_${txid.substring(0, 10)}`)
         await Promise.all(outputsToMarkStale.map(async coin => {
           const output = await this.storage.findOutput(coin.txid, coin.previousOutputIndex, topic)
@@ -368,10 +419,10 @@ export class Engine {
         }))
         this.endTime(`lookForStaleOutputs_${txid.substring(0, 10)}`)
 
-      // Update the STEAK to indicate which coins were removed
+        // Update the STEAK to indicate which coins were removed
         steak[topic].coinsRemoved = outputsToMarkStale.map(x => x.inputIndex)
 
-      // Handle admittance and notification of incoming UTXOs
+        // Handle admittance and notification of incoming UTXOs
         const newUTXOs: Array<{ txid: string, outputIndex: number }> = []
         await Promise.all(outputsToAdmit.map(async outputIndex => {
           if (typeof tx.outputs[outputIndex].satoshis !== 'number') return
@@ -427,7 +478,7 @@ export class Engine {
         }))
 
         this.startTime(`outputConsumed_${txid.substring(0, 10)}`)
-      // Update each output consumed to know who consumed it and insert applied transaction in parallel
+        // Update each output consumed to know who consumed it and insert applied transaction in parallel
         await Promise.all([
           ...outputsConsumed.map(async output => {
             const outputToUpdate = await this.storage.findOutput(output.txid, output.outputIndex, topic)

package/src/storage/knex/all-migrations.ts

@@ -6,8 +6,7 @@ import { up as addedIndexesUp, down as addedIndexesDown } from './migrations/202
 import { up as enlargeUp, down as enlargeDown } from './migrations/2025-05-28-001-enlarge.js'
 import { up as gaspPaginationSupportUp, down as gaspPaginationSupportDown } from './migrations/2025-06-25-001-gasp-pagination-support.js'
 import { up as fixScoreColumnTypeUp, down as fixScoreColumnTypeDown } from './migrations/2025-07-22-001-fix-score-column-type.js'
-
-
+import { up as utxoLookupIndexUp, down as utxoLookupIndexDown } from './migrations/2025-11-11-001-utxo-lookup-index.js'
 
 /**
  * An array of all migrations, in order.
@@ -24,7 +23,8 @@ const allMigrations: Migration[] = [
   { up: addedIndexesUp, down: addedIndexesDown },
   { up: enlargeUp, down: enlargeDown },
   { up: gaspPaginationSupportUp, down: gaspPaginationSupportDown },
-  { up: fixScoreColumnTypeUp, down: fixScoreColumnTypeDown }
+  { up: fixScoreColumnTypeUp, down: fixScoreColumnTypeDown },
+  { up: utxoLookupIndexUp, down: utxoLookupIndexDown }
 ]
 
 export default allMigrations

package/src/storage/knex/migrations/2025-11-11-001-utxo-lookup-index.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex'
+
+/**
+ * Adds optimized index for findUTXOsForTopic queries.
+ * This query pattern is: WHERE topic = ? AND spent = false ORDER BY score
+ * The composite index (topic, spent, score) enables efficient range scans.
+ */
+export async function up (knex: Knex): Promise<void> {
+  await knex.schema.table('outputs', function (table) {
+    table.index(['topic', 'spent', 'score'], 'idx_outputs_topic_spent_score')
+  })
+}
+
+export async function down (knex: Knex): Promise<void> {
+  await knex.schema.table('outputs', function (table) {
+    table.dropIndex(['topic', 'spent', 'score'], 'idx_outputs_topic_spent_score')
+  })
+}