@bsv/overlay 0.1.4 → 0.1.6

package/src/Engine.ts

@@ -9,10 +9,10 @@ import { LookupQuestion } from './LookupQuestion.js'
 import { LookupAnswer } from './LookupAnswer.js'
 import { LookupFormula } from './LookupFormula.js'
 import { Transaction, ChainTracker, MerklePath, Broadcaster, isBroadcastFailure } from '@bsv/sdk'
-import { Advertiser } from './Advertiser.js'
-import { SHIPAdvertisement } from './SHIPAdvertisement.js'
+import { AdvertisementData, Advertiser } from './Advertiser.js'
 import { GASP, GASPInitialReply, GASPInitialRequest, GASPInitialResponse, GASPNode, GASPNodeResponse, GASPRemote, GASPStorage } from '@bsv/gasp'
 import { SyncConfiguration } from './SyncConfiguration.js'
+import { Advertisement } from './Advertisement.js'
 
 /**
  * Am engine for running BSV Overlay Services (topic managers and lookup services).
@@ -272,7 +272,7 @@ export class Engine {
 
           // Lookup will currently always return type output-list
           if (lookupAnswer.type === 'output-list') {
-            const shipAdvertisements: SHIPAdvertisement[] = []
+            const shipAdvertisements: Advertisement[] = []
             lookupAnswer.outputs.forEach(output => {
               try {
                 // Parse out the advertisements using the provided parser
@@ -286,7 +286,7 @@ export class Engine {
               }
             })
             if (shipAdvertisements.length > 0) {
-              shipAdvertisements.forEach((advertisement: SHIPAdvertisement) => {
+              shipAdvertisements.forEach((advertisement: Advertisement) => {
                 if (!domainToTopicsMap.has(advertisement.domain)) {
                   domainToTopicsMap.set(advertisement.domain, new Set<string>())
                 }
@@ -423,57 +423,49 @@ export class Engine {
     const configuredServices = Object.keys(this.lookupServices)
 
     // Step 2: Fetch Existing Advertisements
-    const currentSHIPAdvertisements = await advertiser.findAllSHIPAdvertisements()
-    const currentSLAPAdvertisements = await advertiser.findAllSLAPAdvertisements()
+    const currentSHIPAdvertisements = await advertiser.findAllAdvertisements('SHIP')
+    const currentSLAPAdvertisements = await advertiser.findAllAdvertisements('SLAP')
 
     // Step 3: Compare and Determine Actions
     const requiredSHIPAdvertisements = new Set(configuredTopics)
     const requiredSLAPAdvertisements = new Set(configuredServices)
 
-    const existingSHIPTopics = new Set(currentSHIPAdvertisements.map(ad => ad.topic))
-    const existingSLAPServices = new Set(currentSLAPAdvertisements.map(ad => ad.service))
+    const existingTopics = new Set(currentSHIPAdvertisements.map(ad => ad.topicOrService))
+    const existingServices = new Set(currentSLAPAdvertisements.map(ad => ad.topicOrService))
 
-    const shipToCreate = Array.from(requiredSHIPAdvertisements).filter(topic => !existingSHIPTopics.has(topic))
-    const shipToRevoke = currentSHIPAdvertisements.filter(ad => !requiredSHIPAdvertisements.has(ad.topic))
+    const shipsToCreate = Array.from(requiredSHIPAdvertisements).filter(topicOrService => !existingTopics.has(topicOrService))
+    const slapsToCreate = Array.from(requiredSLAPAdvertisements).filter(topicOrService => !existingServices.has(topicOrService))
+    const shipsToRevoke = currentSHIPAdvertisements.filter(ad => !requiredSHIPAdvertisements.has(ad.topicOrService))
+    const slapsToRevoke = currentSLAPAdvertisements.filter(ad => !requiredSLAPAdvertisements.has(ad.topicOrService))
 
-    const slapToCreate = Array.from(requiredSLAPAdvertisements).filter(service => !existingSLAPServices.has(service))
-    const slapToRevoke = currentSLAPAdvertisements.filter(ad => !requiredSLAPAdvertisements.has(ad.service))
-
-    // Step 4: Update Advertisements
-    for (const topic of shipToCreate) {
-      try {
-        const taggedBEEF = await advertiser.createSHIPAdvertisement(topic)
-        await this.submit(taggedBEEF)
-      } catch (error) {
-        console.error('Failed to create SHIP advertisement:', error)
-      }
-    }
-
-    for (const service of slapToCreate) {
-      try {
-        const taggedBEEF = await advertiser.createSLAPAdvertisement(service)
-        await this.submit(taggedBEEF)
-      } catch (error) {
-        console.error('Failed to create SLAP advertisement:', error)
-      }
-    }
-
-    for (const ad of shipToRevoke) {
-      try {
-        const taggedBEEF = await advertiser.revokeAdvertisement(ad)
+    // Create needed SHIP/SLAP advertisements
+    try {
+      if (shipsToCreate.length > 0 || slapsToCreate.length > 0) {
+        const advertisementData: AdvertisementData[] = [
+          ...shipsToCreate.map(topic => ({
+            protocol: 'SHIP' as 'SHIP',
+            topicOrServiceName: topic
+          })),
+          ...slapsToCreate.map(service => ({
+            protocol: 'SLAP' as 'SLAP',
+            topicOrServiceName: service
+          }))
+        ]
+        const taggedBEEF = await advertiser.createAdvertisements(advertisementData)
         await this.submit(taggedBEEF)
-      } catch (error) {
-        console.error('Failed to revoke SHIP advertisement:', error)
       }
+    } catch (error) {
+      console.error('Failed to create SHIP advertisement:', error)
     }
 
-    for (const ad of slapToRevoke) {
-      try {
-        const taggedBEEF = await advertiser.revokeAdvertisement(ad)
+    // Revoke all advertisements to revoke
+    try {
+      if (shipsToRevoke.length > 0 || slapsToRevoke.length > 0) {
+        const taggedBEEF = await advertiser.revokeAdvertisements([...shipsToRevoke, ...slapsToRevoke])
         await this.submit(taggedBEEF)
-      } catch (error) {
-        console.error('Failed to revoke SLAP advertisement:', error)
       }
+    } catch (error) {
+      console.error('Failed to revoke SHIP/SLAP advertisements:', error)
     }
   }
 
@@ -539,12 +531,12 @@ export class Engine {
    * Given a GASP request, create an initial response.
    *
    * This method processes an initial synchronization request by finding the relevant UTXOs for the given topic
-   * since the provided (TODO: timestamp or block height, we need to decide on sync timing semantics) in the request. It constructs a response that includes a list of these UTXOs
-   * and the (timestamp or block height, TODO...) from the initial request.
+   * since the provided block height in the request. It constructs a response that includes a list of these UTXOs
+   * and the min block height from the initial request.
    *
-   * @param initialRequest - The GASP initial request containing the version and the (timestamp or block height, TODO...) since the last sync.
+   * @param initialRequest - The GASP initial request containing the version and the block height since the last sync.
    * @param topic - The topic for which UTXOs are being requested.
-   * @returns A promise that resolves to a GASPInitialResponse containing the list of UTXOs and the provided timestamp.
+   * @returns A promise that resolves to a GASPInitialResponse containing the list of UTXOs and the provided min block height.
    */
   async provideForeignSyncResponse(initialRequest: GASPInitialRequest, topic: string): Promise<GASPInitialResponse> {
     const UTXOs = await this.storage.findUTXOsForTopic(topic, initialRequest.since)
@@ -570,25 +562,66 @@ export class Engine {
    * @throws An error if no output is found for the given transaction ID and output index.
    */
   async provideForeignGASPNode(graphID: string, txid: string, outputIndex: number): Promise<GASPNode> {
-    const output = await this.storage.findOutput(txid, outputIndex)
+    const hydrator = async (output: Output | null): Promise<GASPNode> => {
+      if (output === undefined || output === null) {
+        throw new Error('No matching output found!')
+      }
 
-    if (output === undefined || output === null) {
-      throw new Error('No matching output found!')
-    }
+      const rootTx = Transaction.fromBEEF(output.beef)
+      let correctTx: Transaction | undefined
 
-    const tx = Transaction.fromBEEF(output.beef)
-    const rawTx = tx.toHex()
+      const searchInput = (tx: Transaction): void => {
+        if (tx.id('hex') === txid) {
+          correctTx = tx
+        } else {
+          // For each input, look it up and recurse.
+          for (const input of tx.inputs) {
+            // We should always have a source transaction
+            if (input.sourceTransaction !== undefined) {
+              searchInput(input.sourceTransaction)
+            } else {
+              throw new Error('Incomplete SPV data!')
+            }
+          }
+        }
+      }
 
-    const node: GASPNode = {
-      rawTx,
-      graphID,
-      outputIndex
-    }
-    if (tx.merklePath !== undefined) {
-      node.proof = tx.merklePath.toHex()
+      searchInput(rootTx)
+
+      if (correctTx !== undefined) {
+        const rawTx = correctTx.toHex()
+        const node: GASPNode = {
+          rawTx,
+          graphID,
+          outputIndex
+        }
+        if (correctTx.merklePath !== undefined) {
+          node.proof = correctTx.merklePath.toHex()
+        }
+
+        return node
+      } else {
+        // Recursively try to find a matching output
+        let foundNode: GASPNode | undefined
+        for (const currentOutput of output.outputsConsumed) {
+          try {
+            const outputFound = await this.storage.findOutput(currentOutput.txid, currentOutput.outputIndex)
+            foundNode = await hydrator(outputFound)
+            break
+          } catch (error) {
+            continue
+          }
+        }
+        if (foundNode !== undefined) {
+          return foundNode
+        }
+      }
+      throw new Error('Unable to find output associated with your request!')
     }
 
-    return node
+    const [rootTxid, rootOutputIndex] = graphID.split('.')
+    const output = await this.storage.findOutput(rootTxid, Number(rootOutputIndex))
+    return await hydrator(output)
   }
 
   /**
@@ -729,20 +762,19 @@ export class Engine {
 
   /**
    * Given a new transaction proof (txid, proof),
-   * 
    * update tx.merklePath if appropriate,
-   * 
    * and if not, recurse through all input sourceTransactions.
-   * 
+   *
    * @param tx transaction which may benefit from new proof.
    * @param txid BE hex string double hash of transaction proven by proof.
    * @param proof for txid
    */
-  private updateInputProofs(tx: Transaction, txid: string, proof: MerklePath) {
-    if (tx.merklePath)
-      // transaction already has a proof
+  private updateInputProofs(tx: Transaction, txid: string, proof: MerklePath): void {
+    if (tx.merklePath !== undefined) {
+      // Update the merkle path to handle potential reorgs
+      tx.merklePath = proof
       return
-
+    }
     if (tx.id('hex') === txid) {
       tx.merklePath = proof
     } else {
@@ -762,12 +794,12 @@ export class Engine {
    * @param proof - The merklePath proving txid is a mined transaction hash
    */
   private async updateMerkleProof(output: Output, txid: string, proof: MerklePath): Promise<void> {
-
     const tx = Transaction.fromBEEF(output.beef)
-
-    if (tx.merklePath)
-      // Already have a proof for this output's transaction.
+    if (tx.merklePath !== undefined) {
+      // Update the merkle path to handle potential reorgs
+      tx.merklePath = proof
       return
+    }
 
     // recursively update all sourceTransactions proven by (txid,proof)
     this.updateInputProofs(tx, txid, proof)
@@ -789,16 +821,23 @@ export class Engine {
    *
    * @param txid - Transaction ID of the associated outputs to prune.
    * @param proof - Merkle proof containing the Merkle path and other relevant data to verify the transaction.
+   * @param blockHeight - The block height associated with the incoming merkle proof.
    */
-  async handleNewMerkleProof(txid: string, proof: MerklePath): Promise<void> {
+  async handleNewMerkleProof(txid: string, proof: MerklePath, blockHeight?: number): Promise<void> {
     const outputs = await this.storage.findOutputsForTransaction(txid)
 
-    if (outputs == undefined || outputs.length === 0) {
+    if (outputs === undefined || outputs.length === 0) {
       throw new Error('Could not find matching transaction outputs for proof ingest!')
     }
 
     for (const output of outputs) {
       await this.updateMerkleProof(output, txid, proof)
+
+      // Add the associated blockHeight
+      if (blockHeight !== undefined) {
+        output.blockHeight = blockHeight
+        await this.storage.updateOutputBlockHeight?.(output.txid, output.outputIndex, output.topic, blockHeight)
+      }
     }
   }
 
@@ -965,7 +1004,6 @@ export class OverlayGASPRemote implements GASPRemote {
  */
 export interface GraphNode {
   txid: string
-  time: number
   graphID: string
   rawTx: string
   outputIndex: number
@@ -1117,7 +1155,6 @@ export class OverlayGASPStorage implements GASPStorage {
     // Use the spentBy param which should be a txid.inputIndex for the node which spent this one in 36-byte format
     const newGraphNode: GraphNode = {
       txid,
-      time: 0, // TODO: Determine required format for Time (either block height or timestamp, undefined / Infinity for unconfirmed transactions
       graphID: tx.graphID,
       rawTx: tx.rawTx,
       outputIndex: tx.outputIndex,

package/src/Output.ts

@@ -26,4 +26,5 @@ export type Output = {
     txid: string
     outputIndex: number
   }>
+  blockHeight?: number
 }

package/src/storage/Storage.ts

@@ -80,6 +80,16 @@ export interface Storage {
    */
   updateOutputBeef: (txid: string, outputIndex: number, topic: string, beef: number[]) => Promise<void>
 
+
+  /**
+   * Updates the block height on an output
+   * @param txid — TXID of the output to update
+   * @param outputIndex — Index of the output to update
+   * @param topic— Topic in which the output should be updated
+   * @param blockHeight - height of the block the transaction associated with this output was included in
+   */
+  updateOutputBlockHeight?: (txid: string, outputIndex: number, topic: string, blockHeight: number) => Promise<void>
+
   /**
    * Inserts record of the applied transaction
    * @param tx — The transaction to insert

package/src/storage/knex/KnexStorage.ts

@@ -59,11 +59,10 @@ export class KnexStorage implements Storage {
   async findUTXOsForTopic(topic: string, since?: number): Promise<Output[]> {
     const query = this.knex('outputs').where({ topic, spent: false })
 
-    // TODO: Create new migration for outputs that adds either created_at or block height tagging for proven outpoints if necessary.
-    // if (since !== undefined) {
-    //   const sinceDate = new Date(since).toISOString()
-    //   await query.andWhere('created_at', '>=', sinceDate)
-    // }
+    // If provided, additionally filters UTXOs by block height
+    if (since !== undefined) {
+      await query.andWhere('blockHeight', '>=', since)
+    }
 
     const outputs = await query.select(
       'txid', 'outputIndex', 'outputScript', 'topic', 'satoshis', 'beef', 'outputsConsumed', 'spent', 'consumedBy'
@@ -127,6 +126,14 @@ export class KnexStorage implements Storage {
     }).update('beef', Buffer.from(beef))
   }
 
+  async updateOutputBlockHeight(txid: string, outputIndex: number, topic: string, blockHeight: number): Promise<void> {
+    await this.knex('outputs').where({
+      txid,
+      outputIndex,
+      topic
+    }).update('blockHeight', blockHeight)
+  }
+
   async insertAppliedTransaction(tx: { txid: string, topic: string }): Promise<void> {
     await this.knex('applied_transactions').insert({
       txid: tx.txid,

package/src/storage/knex/migrations/2024-07-10-001-block-height.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex'
+
+export async function up(knex: Knex): Promise<void> {
+  // Add new column for blockHeight associated with outputs
+  await knex.schema.table('outputs', table => {
+    table.integer('blockHeight').unsigned().nullable()
+  })
+}
+
+export async function down(knex: Knex): Promise<void> {
+  // Remove the blockHeight column (rollback operation)
+  await knex.schema.table('outputs', table => {
+    table.dropColumn('blockHeight')
+  })
+}