@bsv/overlay 2.0.1 → 2.0.3

package/docs/examples/README.md

@@ -4,11 +4,11 @@
 
 ---
 
-Here, you will find documentation for common example usages of the Overlay Services Engine.
+Here, you will find documentation for common example usages of the Overlay Services Engine. These examples focus on direct `@bsv/overlay` integration. For a ready-to-run HTTP overlay node, use [`@bsv/overlay-express`](https://github.com/bsv-blockchain/overlay-express) and [`overlay-express-examples`](https://github.com/bsv-blockchain/overlay-express-examples).
 
 ## Available Examples
 
-- [Getting Started (WIP)](./gs-wip.md) — Introduction and getting started guide
+- [Getting Started](./gs-wip.md) — Introduction and low-level engine setup guide
 
 ---
 

package/docs/examples/gs-wip.md

@@ -6,106 +6,114 @@
 
 ## Introduction to BSV Overlay Services Engine
 
-The BSV Overlay Services Engine is designed to process transactions and manage data within a blockchain-based system, specifically targeting the Bitcoin SV (BSV) blockchain. It integrates various components such as Topic Managers, Lookup Services, Storage, and Chain Tracker to provide a robust environment for managing transaction data and overlay services.
+The BSV Overlay Services Engine is the low-level runtime for topic managers and lookup services. It validates candidate outputs, stores admitted UTXOs, tracks spends, answers lookup questions, and coordinates synchronization between overlay nodes.
+
+If you want to run an HTTP overlay node, start with [`@bsv/overlay-express`](https://github.com/bsv-blockchain/overlay-express). If you want a local or cloud application runtime, use LARS or CARS with the BRC-102 `deployment-info.json` structure. Use `@bsv/overlay` directly when you are building custom infrastructure around the engine.
 
 ### Components of the System
 
-1. **Topic Managers**: Responsible for managing the admittance of transactions related to specific topics.
-2. **Lookup Services**: Handle the lookup of UTXO (Unspent Transaction Output) data for transactions.
-3. **Storage**: Manages persistent data storage, tracking UTXOs and their states within the system.
-4. **Chain Tracker**: Verifies SPV (Simplified Payment Verification) data associated with transactions to ensure their validity.
+1. **Topic Managers** decide which transaction outputs are admitted for a topic.
+2. **Lookup Services** index admitted and spent outputs and answer domain-specific lookup questions.
+3. **Storage** persists admitted outputs, spend state, history, sync state, and interaction timestamps.
+4. **Chain Tracker** verifies SPV data for transactions unless the engine is configured for script-only validation.
+5. **Broadcaster** submits accepted transactions to the network.
+6. **Advertiser** publishes SHIP and SLAP availability records when peer discovery is enabled.
 
 ### Setting Up the Engine
 
-Before you can use the engine, you must initialize it with the required components:
-
-```ts
-import { Engine, KnexStorage } from "@bsv/overlay";
-import { HelloTopicManager, HelloLookupService } from 'hello-services';
-import { WoChain } from "@bsv/sdk";
-
-// Initialize components
-const managers = {
-    "exampleTopic": new HelloTopicManager()
-};
+Install the current packages:
 
-const lookupServices = {
-    "exampleLookup": new HelloLookupService()
-};
+```bash
+npm i @bsv/overlay @bsv/sdk knex
+```
 
-const storage = new KnexStorage();
-const chainTracker = new WoChain();
+Create the engine with your concrete implementations:
 
-// Create the engine instance
-const engine = new Engine(managers, lookupServices, storage, chainTracker);
+```ts
+import { Engine, KnexStorage } from '@bsv/overlay'
+import { WhatsOnChain, NodejsHttpClient, ARC } from '@bsv/sdk'
+import knexFactory from 'knex'
+
+import { ExampleLookupService } from './services/ExampleLookupService.js'
+import { ExampleTopicManager } from './services/ExampleTopicManager.js'
+
+const knex = knexFactory({
+  client: 'mysql2',
+  connection: process.env.KNEX_URL
+})
+
+const engine = new Engine(
+  {
+    tm_example: new ExampleTopicManager()
+  },
+  {
+    ls_example: new ExampleLookupService()
+  },
+  new KnexStorage(knex),
+  new WhatsOnChain('test', { httpClient: new NodejsHttpClient() }),
+  process.env.HOSTING_URL,
+  process.env.SHIP_TRACKERS?.split(',') ?? [],
+  process.env.SLAP_TRACKERS?.split(',') ?? [],
+  new ARC(process.env.ARC_URL ?? 'https://arc.taal.com', {
+    apiKey: process.env.ARC_API_KEY
+  }),
+  undefined,
+  {
+    tm_example: 'SHIP'
+  }
+)
 ```
 
+Use the string `'scripts only'` for the chain tracker only when the service intentionally skips SPV validation and relies only on script-level checks.
+
 ### Submitting a Transaction
 
-To submit a transaction for processing by the Overlay Services:
+Submit tagged BEEF bytes with the topics that should evaluate the transaction:
 
 ```ts
 import { Transaction } from '@bsv/sdk'
 
-const tx = new Transaction(/* ... */);
+const tx = new Transaction(/* ... */)
 
-const transaction = {
-    beef: tx.toBEEF(),
-    topics: ['exampleTopic']
-}
+const steak = await engine.submit({
+  beef: tx.toBEEF(),
+  topics: ['tm_example']
+})
 
-// Submit transaction
-engine.submit(transaction).then(steak => {
-    console.log("Transaction processed:", steak);
-}).catch(error => {
-    console.error("Error processing transaction:", error);
-});
+console.log('Transaction processed:', steak)
 ```
 
 ### Lookup Queries
 
-To perform a lookup query using the engine:
+Ask a lookup service a domain-specific question:
 
 ```ts
-const question = {
-    service: 'exampleLookup',
-    query: {
-        name: 'Bob'
-    }
-}
-
-// Perform a lookup
-engine.lookup(question).then(answer => {
-    console.log("Lookup result:", answer);
-}).catch(error => {
-    console.error("Error performing lookup:", error);
-});
+const answer = await engine.lookup({
+  service: 'ls_example',
+  query: {
+    identityKey: '03...'
+  }
+})
+
+console.log('Lookup result:', answer)
 ```
 
-### Managing UTXOs
+### Service Documentation
 
-The system's core functionality involves managing UTXOs:
+Overlay clients and dashboards can retrieve service documentation and metadata directly from the engine:
 
-1. **Inserting a New UTXO**: Store new UTXO data when transactions are processed.
-2. **Deleting a UTXO**: Remove UTXOs that are no longer needed or have been consumed by newer transactions.
-3. **Tracking UTXO Consumption**: Monitor which transactions consume which UTXOs.
+```ts
+const topicDocs = await engine.getDocumentationForTopicManger('tm_example')
+const lookupDocs = await engine.getDocumentationForLookupServiceProvider('ls_example')
 
-### Retrieving Documentation
+const topics = await engine.listTopicManagers()
+const lookupServices = await engine.listLookupServiceProviders()
+```
 
-To retrieve documentation for specific managers or services:
+### Deployment Path
 
-```ts
-// For a topic manager
-engine.getDocumentationForTopicManger("exampleTopic").then(doc => {
-    console.log("Documentation for Topic Manager:", doc);
-});
-
-// For a lookup service
-engine.getDocumentationForLookupServiceProvider("exampleLookup").then(doc => {
-    console.log("Documentation for Lookup Service:", doc);
-});
-```
+For a public overlay node, wire the engine into HTTP routes using `@bsv/overlay-express` instead of hand-rolling routes. Overlay Express already exposes the standard submit, lookup, sync, health, documentation, and admin surfaces expected by the rest of the BSV overlay ecosystem.
 
 ### Conclusion
 
-The BSV Overlay Services Engine provides a powerful toolset for managing transactions and data on the Bitcoin SV blockchain. It's designed to handle complex data structures and ensure the integrity and security of transactions through rigorous validation and management processes. By following this tutorial, developers can effectively integrate and utilize these capabilities within their blockchain applications.
+The BSV Overlay Services Engine is the shared core for overlay validation, indexing, lookup, and synchronization. Keep direct engine usage focused on infrastructure-level integrations, and use Overlay Express, LARS, and CARS for the standard application path.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bsv/overlay",
-  "version": "2.0.1",
+  "version": "2.0.3",
   "type": "module",
   "description": "BSV Blockchain Overlay Services Engine",
   "main": "dist/cjs/mod.js",
@@ -74,7 +74,7 @@
   },
   "dependencies": {
     "@bsv/gasp": "^1.2.2",
-    "@bsv/sdk": "^2.0.4",
+    "@bsv/sdk": "^2.0.14",
     "knex": "^3.1.0"
   }
 }

package/src/Engine.ts

@@ -552,6 +552,10 @@ export class Engine {
 
     const lookupResult = await lookupService.lookup(lookupQuestion)
     const hydrationContext = this.createUTXOHistoryHydrationContext()
+    await this.preloadOutputsWithBEEF(
+      lookupResult.map(({ txid, outputIndex }) => ({ txid, outputIndex })),
+      hydrationContext
+    )
     const hydratedOutputs = (await Promise.all(
       lookupResult.map(async ({ txid, outputIndex, history, context }) => {
         const UTXO = await this.loadOutputWithBEEF(txid, outputIndex, hydrationContext)
@@ -590,12 +594,66 @@ export class Engine {
     }
   }
 
+  private toOutputCacheKey(txid: string, outputIndex: number): string {
+    return `${txid}:${outputIndex}`
+  }
+
+  private async preloadOutputsWithBEEF(
+    outpoints: Array<{ txid: string, outputIndex: number }>,
+    context: UTXOHistoryHydrationContext
+  ): Promise<void> {
+    if (outpoints.length === 0) {
+      return
+    }
+
+    const deduped: Array<{ txid: string, outputIndex: number }> = []
+    const seen = new Set<string>()
+
+    for (const outpoint of outpoints) {
+      const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex)
+      if (seen.has(cacheKey)) {
+        continue
+      }
+      seen.add(cacheKey)
+      if (!context.outputCache.has(cacheKey)) {
+        deduped.push(outpoint)
+      }
+    }
+
+    if (deduped.length === 0) {
+      return
+    }
+
+    const findOutputsByOutpoints = this.storage.findOutputsByOutpoints
+    if (typeof findOutputsByOutpoints === 'function') {
+      const outputs = await findOutputsByOutpoints.call(this.storage, deduped, true)
+      const outputsByKey = new Map<string, Output>()
+      for (const output of outputs) {
+        outputsByKey.set(this.toOutputCacheKey(output.txid, output.outputIndex), output)
+      }
+
+      for (const outpoint of deduped) {
+        const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex)
+        context.outputCache.set(cacheKey, Promise.resolve(outputsByKey.get(cacheKey) ?? null))
+      }
+      return
+    }
+
+    for (const outpoint of deduped) {
+      const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex)
+      context.outputCache.set(
+        cacheKey,
+        this.storage.findOutput(outpoint.txid, outpoint.outputIndex, undefined, undefined, true)
+      )
+    }
+  }
+
   private async loadOutputWithBEEF(
     txid: string,
     outputIndex: number,
     context: UTXOHistoryHydrationContext
   ): Promise<Output | null> {
-    const cacheKey = `${txid}:${outputIndex}`
+    const cacheKey = this.toOutputCacheKey(txid, outputIndex)
     let cached = context.outputCache.get(cacheKey)
     if (cached === undefined) {
       cached = this.storage.findOutput(txid, outputIndex, undefined, undefined, true)
@@ -626,6 +684,8 @@ export class Engine {
       return undefined
     }
 
+    await this.preloadOutputsWithBEEF(output.outputsConsumed, context)
+
     const childNodes = (await Promise.all(
       output.outputsConsumed.map(async (outputIdentifier) => {
         const childOutput = await this.loadOutputWithBEEF(outputIdentifier.txid, outputIdentifier.outputIndex, context)
@@ -638,15 +698,21 @@ export class Engine {
     )).filter((node): node is HydratedUTXOHistoryNode => node !== undefined)
 
     const tx = Transaction.fromBEEF(output.beef)
+    const inputIndexBySource = new Map<string, number>()
+    tx.inputs.forEach((candidateInput, index) => {
+      const sourceTXID = candidateInput.sourceTXID !== undefined && candidateInput.sourceTXID !== ''
+        ? candidateInput.sourceTXID
+        : candidateInput.sourceTransaction?.id('hex')
 
-    for (const child of childNodes) {
-      const inputIndex = tx.inputs.findIndex((candidateInput) => {
-        const sourceTXID = candidateInput.sourceTXID !== undefined && candidateInput.sourceTXID !== ''
-          ? candidateInput.sourceTXID
-          : candidateInput.sourceTransaction?.id('hex')
+      if (sourceTXID === undefined) {
+        return
+      }
 
-        return sourceTXID === child.output.txid && candidateInput.sourceOutputIndex === child.output.outputIndex
-      })
+      inputIndexBySource.set(`${sourceTXID}:${candidateInput.sourceOutputIndex}`, index)
+    })
+
+    for (const child of childNodes) {
+      const inputIndex = inputIndexBySource.get(`${child.output.txid}:${child.output.outputIndex}`)
 
       if (inputIndex === -1 || inputIndex == null) {
         continue

package/src/__tests/Engine.test.ts

@@ -845,6 +845,34 @@ describe('BSV Overlay Services Engine', () => {
           query: { name: 'Bob' }
         })
       })
+      it('Uses batched output loading when the storage engine supports it', async () => {
+        mockLookupService.lookup = jest.fn(async () => [{
+          txid: 'mockTXID',
+          outputIndex: 0,
+          history: undefined
+        }])
+        mockStorageEngine.findOutput = jest.fn(async () => mockOutput)
+        const findOutputsByOutpoints = jest.fn(async () => [mockOutput])
+        ;(mockStorageEngine).findOutputsByOutpoints = findOutputsByOutpoints
+        const engine = new Engine(
+          {
+            Hello: mockTopicManager
+          },
+          {
+            Hello: mockLookupService
+          },
+          mockStorageEngine,
+          mockChainTracker
+        )
+
+        await engine.lookup({
+          service: 'Hello',
+          query: { name: 'Bob' }
+        })
+
+        expect(findOutputsByOutpoints).toHaveBeenCalledWith([{ txid: 'mockTXID', outputIndex: 0 }], true)
+        expect(mockStorageEngine.findOutput).not.toHaveBeenCalled()
+      })
       describe('For each returned result', () => {
         it('Finds the identified UTXO by its txid and vout', async () => {
           // TODO: Make the default storage engine return something...?

package/src/storage/Storage.ts

@@ -30,6 +30,17 @@ export interface Storage {
    */
   findOutput: (txid: string, outputIndex: number, topic?: string, spent?: boolean, includeBEEF?: boolean) => Promise<Output | null>
 
+  /**
+   * Finds multiple outputs from storage by txid/output index pairs.
+   * Implementations can use this to collapse many point lookups into a single query.
+   * @param outpoints — txid/output index pairs to find
+   * @param includeBEEF — Whether to include the BEEF data for the outputs (optional)
+   */
+  findOutputsByOutpoints?: (
+    outpoints: Array<{ txid: string, outputIndex: number }>,
+    includeBEEF?: boolean
+  ) => Promise<Output[]>
+
   /**
    * Finds outputs with a matching transaction ID from storage
    * @param txid — TXID of the outputs to find

package/src/storage/knex/KnexStorage.ts

@@ -2,6 +2,18 @@ import { Storage } from '../Storage.js'
 import { Knex } from 'knex'
 import type { Output } from '../../Output.js'
 
+const OUTPUT_SELECT_FIELDS = [
+  'outputs.txid',
+  'outputs.outputIndex',
+  'outputs.outputScript',
+  'outputs.topic',
+  'outputs.satoshis',
+  'outputs.outputsConsumed',
+  'outputs.spent',
+  'outputs.consumedBy',
+  'outputs.score'
+] as const
+
 export class KnexStorage implements Storage {
   knex: Knex
 
@@ -9,6 +21,48 @@ export class KnexStorage implements Storage {
     this.knex = knex
   }
 
+  private parseOutputRelations(
+    value: string | Array<{ txid: string, outputIndex: number }>
+  ): Array<{ txid: string, outputIndex: number }> {
+    if (Array.isArray(value)) {
+      return value
+    }
+    return JSON.parse(value)
+  }
+
+  private parseOutputRecord(
+    row: any,
+    includeBEEF: boolean,
+    beefOverride?: number[]
+  ): Output {
+    return {
+      ...row,
+      outputScript: Array.from(row.outputScript),
+      beef: includeBEEF ? (beefOverride ?? (row.beef !== undefined ? Array.from(row.beef) : undefined)) : undefined,
+      spent: Boolean(row.spent),
+      outputsConsumed: this.parseOutputRelations(row.outputsConsumed),
+      consumedBy: this.parseOutputRelations(row.consumedBy)
+    }
+  }
+
+  private async fetchTransactionBeefMap(txids: string[]): Promise<Map<string, number[]>> {
+    if (txids.length === 0) {
+      return new Map<string, number[]>()
+    }
+
+    const rows = await this.knex('transactions')
+      .whereIn('txid', txids)
+      .select(['txid', 'beef'])
+
+    const beefByTxid = new Map<string, number[]>()
+    for (const row of rows) {
+      if (row.beef !== undefined) {
+        beefByTxid.set(row.txid, Array.from(row.beef))
+      }
+    }
+    return beefByTxid
+  }
+
   async findOutput (txid: string, outputIndex: number, topic?: string, spent?: boolean, includeBEEF: boolean = false): Promise<Output | null> {
     const search: {
       'outputs.txid': string
@@ -22,21 +76,8 @@ export class KnexStorage implements Storage {
     if (topic !== undefined) search['outputs.topic'] = topic
     if (spent !== undefined) search['outputs.spent'] = spent
 
-    // Base query to get the output
     const query = this.knex('outputs').where(search)
-
-    // Select necessary fields from outputs and conditionally include beef from transactions
-    const selectFields = [
-      'outputs.txid',
-      'outputs.outputIndex',
-      'outputs.outputScript',
-      'outputs.topic',
-      'outputs.satoshis',
-      'outputs.outputsConsumed',
-      'outputs.spent',
-      'outputs.consumedBy',
-      'outputs.score'
-    ]
+    const selectFields: string[] = [...OUTPUT_SELECT_FIELDS]
 
     if (includeBEEF) {
       // eslint-disable-next-line @typescript-eslint/no-floating-promises
@@ -50,53 +91,57 @@ export class KnexStorage implements Storage {
       return null
     }
 
-    return {
-      ...output,
-      outputScript: [...output.outputScript],
-      beef: includeBEEF ? (output.beef !== undefined ? [...output.beef] : undefined) : undefined,
-      spent: Boolean(output.spent),
-      outputsConsumed: JSON.parse(output.outputsConsumed),
-      consumedBy: JSON.parse(output.consumedBy)
-    }
+    return this.parseOutputRecord(output, includeBEEF)
   }
 
-  async findOutputsForTransaction (txid: string, includeBEEF: boolean = false): Promise<Output[]> {
-    // Base query to get outputs
-    const query = this.knex('outputs').where({ 'outputs.txid': txid })
-
-    // Select necessary fields from outputs and conditionally include beef from transactions
-    const selectFields = [
-      'outputs.txid',
-      'outputs.outputIndex',
-      'outputs.outputScript',
-      'outputs.topic',
-      'outputs.satoshis',
-      'outputs.outputsConsumed',
-      'outputs.spent',
-      'outputs.consumedBy',
-      'outputs.score'
-    ]
+  async findOutputsByOutpoints (
+    outpoints: Array<{ txid: string, outputIndex: number }>,
+    includeBEEF: boolean = false
+  ): Promise<Output[]> {
+    if (outpoints.length === 0) {
+      return []
+    }
 
-    if (includeBEEF) {
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      query.leftJoin('transactions', 'outputs.txid', 'transactions.txid')
-      selectFields.push('transactions.beef')
+    const deduped = new Map<string, { txid: string, outputIndex: number }>()
+    for (const outpoint of outpoints) {
+      deduped.set(`${outpoint.txid}:${outpoint.outputIndex}`, outpoint)
+    }
+
+    const rows = await this.knex('outputs')
+      .whereIn(
+        ['outputs.txid', 'outputs.outputIndex'],
+        Array.from(deduped.values()).map(outpoint => [outpoint.txid, outpoint.outputIndex])
+      )
+      .select([...OUTPUT_SELECT_FIELDS])
+
+    if (rows === undefined || rows.length === 0) {
+      return []
+    }
+
+    if (!includeBEEF) {
+      return rows.map(row => this.parseOutputRecord(row, false))
     }
 
-    const outputs = await query.select(selectFields)
+    const txids = Array.from(new Set(rows.map(row => row.txid)))
+    const beefByTxid = await this.fetchTransactionBeefMap(txids)
+    return rows.map(row => this.parseOutputRecord(row, true, beefByTxid.get(row.txid)))
+  }
+
+  async findOutputsForTransaction (txid: string, includeBEEF: boolean = false): Promise<Output[]> {
+    const outputs = await this.knex('outputs')
+      .where({ 'outputs.txid': txid })
+      .select([...OUTPUT_SELECT_FIELDS])
 
     if (outputs === undefined || outputs.length === 0) {
       return []
     }
 
-    return outputs.map(output => ({
-      ...output,
-      outputScript: [...output.outputScript],
-      beef: includeBEEF ? (output.beef !== undefined ? [...output.beef] : undefined) : undefined,
-      spent: Boolean(output.spent),
-      outputsConsumed: JSON.parse(output.outputsConsumed),
-      consumedBy: JSON.parse(output.consumedBy)
-    }))
+    if (!includeBEEF) {
+      return outputs.map(output => this.parseOutputRecord(output, false))
+    }
+
+    const beefByTxid = await this.fetchTransactionBeefMap([txid])
+    return outputs.map(output => this.parseOutputRecord(output, true, beefByTxid.get(output.txid)))
   }
 
   async findUTXOsForTopic (topic: string, since?: number, limit?: number, includeBEEF: boolean = false): Promise<Output[]> {
@@ -113,45 +158,25 @@ export class KnexStorage implements Storage {
     // eslint-disable-next-line @typescript-eslint/no-floating-promises
     query.orderBy('outputs.score', 'asc')
 
-    // Select necessary fields from outputs and conditionally include beef from transactions
-    const selectFields = [
-      'outputs.txid',
-      'outputs.outputIndex',
-      'outputs.outputScript',
-      'outputs.topic',
-      'outputs.satoshis',
-      'outputs.outputsConsumed',
-      'outputs.spent',
-      'outputs.consumedBy',
-      'outputs.score'
-    ]
-
-    if (includeBEEF) {
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      query.leftJoin('transactions', 'outputs.txid', 'transactions.txid')
-      selectFields.push('transactions.beef')
-    }
-
     // Apply limit if specified
     if (limit !== undefined && limit > 0) {
       // eslint-disable-next-line @typescript-eslint/no-floating-promises
       query.limit(limit)
     }
 
-    const outputs = await query.select(selectFields)
+    const outputs = await query.select([...OUTPUT_SELECT_FIELDS])
 
     if (outputs === undefined || outputs.length === 0) {
       return []
     }
 
-    return outputs.map(output => ({
-      ...output,
-      outputScript: [...output.outputScript],
-      beef: includeBEEF ? (output.beef !== undefined ? [...output.beef] : undefined) : undefined,
-      spent: Boolean(output.spent),
-      outputsConsumed: JSON.parse(output.outputsConsumed),
-      consumedBy: JSON.parse(output.consumedBy)
-    }))
+    if (!includeBEEF) {
+      return outputs.map(output => this.parseOutputRecord(output, false))
+    }
+
+    const txids = Array.from(new Set(outputs.map(output => output.txid)))
+    const beefByTxid = await this.fetchTransactionBeefMap(txids)
+    return outputs.map(output => this.parseOutputRecord(output, true, beefByTxid.get(output.txid)))
   }
 
   async deleteOutput (txid: string, outputIndex: number, _: string): Promise<void> {