@bsv/overlay 2.0.1 → 2.0.2

package/src/Engine.ts

@@ -552,6 +552,10 @@ export class Engine {
 
     const lookupResult = await lookupService.lookup(lookupQuestion)
     const hydrationContext = this.createUTXOHistoryHydrationContext()
+    await this.preloadOutputsWithBEEF(
+      lookupResult.map(({ txid, outputIndex }) => ({ txid, outputIndex })),
+      hydrationContext
+    )
     const hydratedOutputs = (await Promise.all(
       lookupResult.map(async ({ txid, outputIndex, history, context }) => {
         const UTXO = await this.loadOutputWithBEEF(txid, outputIndex, hydrationContext)
@@ -590,12 +594,66 @@ export class Engine {
     }
   }
 
+  private toOutputCacheKey(txid: string, outputIndex: number): string {
+    return `${txid}:${outputIndex}`
+  }
+
+  private async preloadOutputsWithBEEF(
+    outpoints: Array<{ txid: string, outputIndex: number }>,
+    context: UTXOHistoryHydrationContext
+  ): Promise<void> {
+    if (outpoints.length === 0) {
+      return
+    }
+
+    const deduped: Array<{ txid: string, outputIndex: number }> = []
+    const seen = new Set<string>()
+
+    for (const outpoint of outpoints) {
+      const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex)
+      if (seen.has(cacheKey)) {
+        continue
+      }
+      seen.add(cacheKey)
+      if (!context.outputCache.has(cacheKey)) {
+        deduped.push(outpoint)
+      }
+    }
+
+    if (deduped.length === 0) {
+      return
+    }
+
+    const findOutputsByOutpoints = this.storage.findOutputsByOutpoints
+    if (typeof findOutputsByOutpoints === 'function') {
+      const outputs = await findOutputsByOutpoints.call(this.storage, deduped, true)
+      const outputsByKey = new Map<string, Output>()
+      for (const output of outputs) {
+        outputsByKey.set(this.toOutputCacheKey(output.txid, output.outputIndex), output)
+      }
+
+      for (const outpoint of deduped) {
+        const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex)
+        context.outputCache.set(cacheKey, Promise.resolve(outputsByKey.get(cacheKey) ?? null))
+      }
+      return
+    }
+
+    for (const outpoint of deduped) {
+      const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex)
+      context.outputCache.set(
+        cacheKey,
+        this.storage.findOutput(outpoint.txid, outpoint.outputIndex, undefined, undefined, true)
+      )
+    }
+  }
+
   private async loadOutputWithBEEF(
     txid: string,
     outputIndex: number,
     context: UTXOHistoryHydrationContext
   ): Promise<Output | null> {
-    const cacheKey = `${txid}:${outputIndex}`
+    const cacheKey = this.toOutputCacheKey(txid, outputIndex)
     let cached = context.outputCache.get(cacheKey)
     if (cached === undefined) {
       cached = this.storage.findOutput(txid, outputIndex, undefined, undefined, true)
@@ -626,6 +684,8 @@ export class Engine {
       return undefined
     }
 
+    await this.preloadOutputsWithBEEF(output.outputsConsumed, context)
+
     const childNodes = (await Promise.all(
       output.outputsConsumed.map(async (outputIdentifier) => {
         const childOutput = await this.loadOutputWithBEEF(outputIdentifier.txid, outputIdentifier.outputIndex, context)
@@ -638,15 +698,21 @@ export class Engine {
     )).filter((node): node is HydratedUTXOHistoryNode => node !== undefined)
 
     const tx = Transaction.fromBEEF(output.beef)
+    const inputIndexBySource = new Map<string, number>()
+    tx.inputs.forEach((candidateInput, index) => {
+      const sourceTXID = candidateInput.sourceTXID !== undefined && candidateInput.sourceTXID !== ''
+        ? candidateInput.sourceTXID
+        : candidateInput.sourceTransaction?.id('hex')
 
-    for (const child of childNodes) {
-      const inputIndex = tx.inputs.findIndex((candidateInput) => {
-        const sourceTXID = candidateInput.sourceTXID !== undefined && candidateInput.sourceTXID !== ''
-          ? candidateInput.sourceTXID
-          : candidateInput.sourceTransaction?.id('hex')
+      if (sourceTXID === undefined) {
+        return
+      }
 
-        return sourceTXID === child.output.txid && candidateInput.sourceOutputIndex === child.output.outputIndex
-      })
+      inputIndexBySource.set(`${sourceTXID}:${candidateInput.sourceOutputIndex}`, index)
+    })
+
+    for (const child of childNodes) {
+      const inputIndex = inputIndexBySource.get(`${child.output.txid}:${child.output.outputIndex}`)
 
       if (inputIndex === -1 || inputIndex == null) {
         continue

package/src/__tests/Engine.test.ts

@@ -845,6 +845,34 @@ describe('BSV Overlay Services Engine', () => {
           query: { name: 'Bob' }
         })
       })
+      it('Uses batched output loading when the storage engine supports it', async () => {
+        mockLookupService.lookup = jest.fn(async () => [{
+          txid: 'mockTXID',
+          outputIndex: 0,
+          history: undefined
+        }])
+        mockStorageEngine.findOutput = jest.fn(async () => mockOutput)
+        const findOutputsByOutpoints = jest.fn(async () => [mockOutput])
+        ;(mockStorageEngine).findOutputsByOutpoints = findOutputsByOutpoints
+        const engine = new Engine(
+          {
+            Hello: mockTopicManager
+          },
+          {
+            Hello: mockLookupService
+          },
+          mockStorageEngine,
+          mockChainTracker
+        )
+
+        await engine.lookup({
+          service: 'Hello',
+          query: { name: 'Bob' }
+        })
+
+        expect(findOutputsByOutpoints).toHaveBeenCalledWith([{ txid: 'mockTXID', outputIndex: 0 }], true)
+        expect(mockStorageEngine.findOutput).not.toHaveBeenCalled()
+      })
       describe('For each returned result', () => {
         it('Finds the identified UTXO by its txid and vout', async () => {
           // TODO: Make the default storage engine return something...?

package/src/storage/Storage.ts

@@ -30,6 +30,17 @@ export interface Storage {
    */
   findOutput: (txid: string, outputIndex: number, topic?: string, spent?: boolean, includeBEEF?: boolean) => Promise<Output | null>
 
+  /**
+   * Finds multiple outputs from storage by txid/output index pairs.
+   * Implementations can use this to collapse many point lookups into a single query.
+   * @param outpoints — txid/output index pairs to find
+   * @param includeBEEF — Whether to include the BEEF data for the outputs (optional)
+   */
+  findOutputsByOutpoints?: (
+    outpoints: Array<{ txid: string, outputIndex: number }>,
+    includeBEEF?: boolean
+  ) => Promise<Output[]>
+
   /**
    * Finds outputs with a matching transaction ID from storage
    * @param txid — TXID of the outputs to find

package/src/storage/knex/KnexStorage.ts

@@ -2,6 +2,18 @@ import { Storage } from '../Storage.js'
 import { Knex } from 'knex'
 import type { Output } from '../../Output.js'
 
+const OUTPUT_SELECT_FIELDS = [
+  'outputs.txid',
+  'outputs.outputIndex',
+  'outputs.outputScript',
+  'outputs.topic',
+  'outputs.satoshis',
+  'outputs.outputsConsumed',
+  'outputs.spent',
+  'outputs.consumedBy',
+  'outputs.score'
+] as const
+
 export class KnexStorage implements Storage {
   knex: Knex
 
@@ -9,6 +21,48 @@ export class KnexStorage implements Storage {
     this.knex = knex
   }
 
+  private parseOutputRelations(
+    value: string | Array<{ txid: string, outputIndex: number }>
+  ): Array<{ txid: string, outputIndex: number }> {
+    if (Array.isArray(value)) {
+      return value
+    }
+    return JSON.parse(value)
+  }
+
+  private parseOutputRecord(
+    row: any,
+    includeBEEF: boolean,
+    beefOverride?: number[]
+  ): Output {
+    return {
+      ...row,
+      outputScript: Array.from(row.outputScript),
+      beef: includeBEEF ? (beefOverride ?? (row.beef !== undefined ? Array.from(row.beef) : undefined)) : undefined,
+      spent: Boolean(row.spent),
+      outputsConsumed: this.parseOutputRelations(row.outputsConsumed),
+      consumedBy: this.parseOutputRelations(row.consumedBy)
+    }
+  }
+
+  private async fetchTransactionBeefMap(txids: string[]): Promise<Map<string, number[]>> {
+    if (txids.length === 0) {
+      return new Map<string, number[]>()
+    }
+
+    const rows = await this.knex('transactions')
+      .whereIn('txid', txids)
+      .select(['txid', 'beef'])
+
+    const beefByTxid = new Map<string, number[]>()
+    for (const row of rows) {
+      if (row.beef !== undefined) {
+        beefByTxid.set(row.txid, Array.from(row.beef))
+      }
+    }
+    return beefByTxid
+  }
+
   async findOutput (txid: string, outputIndex: number, topic?: string, spent?: boolean, includeBEEF: boolean = false): Promise<Output | null> {
     const search: {
       'outputs.txid': string
@@ -22,21 +76,8 @@ export class KnexStorage implements Storage {
     if (topic !== undefined) search['outputs.topic'] = topic
     if (spent !== undefined) search['outputs.spent'] = spent
 
-    // Base query to get the output
     const query = this.knex('outputs').where(search)
-
-    // Select necessary fields from outputs and conditionally include beef from transactions
-    const selectFields = [
-      'outputs.txid',
-      'outputs.outputIndex',
-      'outputs.outputScript',
-      'outputs.topic',
-      'outputs.satoshis',
-      'outputs.outputsConsumed',
-      'outputs.spent',
-      'outputs.consumedBy',
-      'outputs.score'
-    ]
+    const selectFields: string[] = [...OUTPUT_SELECT_FIELDS]
 
     if (includeBEEF) {
       // eslint-disable-next-line @typescript-eslint/no-floating-promises
@@ -50,53 +91,57 @@ export class KnexStorage implements Storage {
       return null
     }
 
-    return {
-      ...output,
-      outputScript: [...output.outputScript],
-      beef: includeBEEF ? (output.beef !== undefined ? [...output.beef] : undefined) : undefined,
-      spent: Boolean(output.spent),
-      outputsConsumed: JSON.parse(output.outputsConsumed),
-      consumedBy: JSON.parse(output.consumedBy)
-    }
+    return this.parseOutputRecord(output, includeBEEF)
   }
 
-  async findOutputsForTransaction (txid: string, includeBEEF: boolean = false): Promise<Output[]> {
-    // Base query to get outputs
-    const query = this.knex('outputs').where({ 'outputs.txid': txid })
-
-    // Select necessary fields from outputs and conditionally include beef from transactions
-    const selectFields = [
-      'outputs.txid',
-      'outputs.outputIndex',
-      'outputs.outputScript',
-      'outputs.topic',
-      'outputs.satoshis',
-      'outputs.outputsConsumed',
-      'outputs.spent',
-      'outputs.consumedBy',
-      'outputs.score'
-    ]
+  async findOutputsByOutpoints (
+    outpoints: Array<{ txid: string, outputIndex: number }>,
+    includeBEEF: boolean = false
+  ): Promise<Output[]> {
+    if (outpoints.length === 0) {
+      return []
+    }
 
-    if (includeBEEF) {
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      query.leftJoin('transactions', 'outputs.txid', 'transactions.txid')
-      selectFields.push('transactions.beef')
+    const deduped = new Map<string, { txid: string, outputIndex: number }>()
+    for (const outpoint of outpoints) {
+      deduped.set(`${outpoint.txid}:${outpoint.outputIndex}`, outpoint)
+    }
+
+    const rows = await this.knex('outputs')
+      .whereIn(
+        ['outputs.txid', 'outputs.outputIndex'],
+        Array.from(deduped.values()).map(outpoint => [outpoint.txid, outpoint.outputIndex])
+      )
+      .select([...OUTPUT_SELECT_FIELDS])
+
+    if (rows === undefined || rows.length === 0) {
+      return []
+    }
+
+    if (!includeBEEF) {
+      return rows.map(row => this.parseOutputRecord(row, false))
     }
 
-    const outputs = await query.select(selectFields)
+    const txids = Array.from(new Set(rows.map(row => row.txid)))
+    const beefByTxid = await this.fetchTransactionBeefMap(txids)
+    return rows.map(row => this.parseOutputRecord(row, true, beefByTxid.get(row.txid)))
+  }
+
+  async findOutputsForTransaction (txid: string, includeBEEF: boolean = false): Promise<Output[]> {
+    const outputs = await this.knex('outputs')
+      .where({ 'outputs.txid': txid })
+      .select([...OUTPUT_SELECT_FIELDS])
 
     if (outputs === undefined || outputs.length === 0) {
       return []
     }
 
-    return outputs.map(output => ({
-      ...output,
-      outputScript: [...output.outputScript],
-      beef: includeBEEF ? (output.beef !== undefined ? [...output.beef] : undefined) : undefined,
-      spent: Boolean(output.spent),
-      outputsConsumed: JSON.parse(output.outputsConsumed),
-      consumedBy: JSON.parse(output.consumedBy)
-    }))
+    if (!includeBEEF) {
+      return outputs.map(output => this.parseOutputRecord(output, false))
+    }
+
+    const beefByTxid = await this.fetchTransactionBeefMap([txid])
+    return outputs.map(output => this.parseOutputRecord(output, true, beefByTxid.get(output.txid)))
   }
 
   async findUTXOsForTopic (topic: string, since?: number, limit?: number, includeBEEF: boolean = false): Promise<Output[]> {
@@ -113,45 +158,25 @@ export class KnexStorage implements Storage {
     // eslint-disable-next-line @typescript-eslint/no-floating-promises
     query.orderBy('outputs.score', 'asc')
 
-    // Select necessary fields from outputs and conditionally include beef from transactions
-    const selectFields = [
-      'outputs.txid',
-      'outputs.outputIndex',
-      'outputs.outputScript',
-      'outputs.topic',
-      'outputs.satoshis',
-      'outputs.outputsConsumed',
-      'outputs.spent',
-      'outputs.consumedBy',
-      'outputs.score'
-    ]
-
-    if (includeBEEF) {
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      query.leftJoin('transactions', 'outputs.txid', 'transactions.txid')
-      selectFields.push('transactions.beef')
-    }
-
     // Apply limit if specified
     if (limit !== undefined && limit > 0) {
       // eslint-disable-next-line @typescript-eslint/no-floating-promises
       query.limit(limit)
     }
 
-    const outputs = await query.select(selectFields)
+    const outputs = await query.select([...OUTPUT_SELECT_FIELDS])
 
     if (outputs === undefined || outputs.length === 0) {
       return []
     }
 
-    return outputs.map(output => ({
-      ...output,
-      outputScript: [...output.outputScript],
-      beef: includeBEEF ? (output.beef !== undefined ? [...output.beef] : undefined) : undefined,
-      spent: Boolean(output.spent),
-      outputsConsumed: JSON.parse(output.outputsConsumed),
-      consumedBy: JSON.parse(output.consumedBy)
-    }))
+    if (!includeBEEF) {
+      return outputs.map(output => this.parseOutputRecord(output, false))
+    }
+
+    const txids = Array.from(new Set(outputs.map(output => output.txid)))
+    const beefByTxid = await this.fetchTransactionBeefMap(txids)
+    return outputs.map(output => this.parseOutputRecord(output, true, beefByTxid.get(output.txid)))
   }
 
   async deleteOutput (txid: string, outputIndex: number, _: string): Promise<void> {