@bsv/overlay 0.1.27 → 0.2.0

package/src/LookupService.ts

@@ -1,68 +1,131 @@
 import { LookupFormula } from './LookupFormula.js'
-import { LookupQuestion, LookupAnswer, Script } from '@bsv/sdk'
+import { Script, LookupQuestion, LookupAnswer } from '@bsv/sdk'
 
-/**
- * Defines a Lookup Service interface to be implemented for specific use-cases
- */
+/* ---------------------------------------------------------------------------
+ *  Modes a Lookup Service may request from the Overlay Services Engine
+ * -------------------------------------------------------------------------- */
+export type AdmissionMode = 'locking-script' | 'whole-tx'
+export type SpendNotificationMode = 'none' | 'txid' | 'script' | 'whole-tx'
+
+/* ---------------------------------------------------------------------------
+ *  Admission-Notification payloads
+ * -------------------------------------------------------------------------- */
+export type OutputAdmittedByTopic =
+  | { // «locking-script» mode
+    mode: 'locking-script'
+    txid: string // ID of the *admitted* transaction
+    outputIndex: number // index of admitted output
+    topic: string // topic into which it was admitted
+    satoshis: number // value of the output
+    lockingScript: Script // script in this output
+  }
+  | { // «whole-tx» mode
+    mode: 'whole-tx'
+    atomicBEEF: number[] // whole transaction (Atomic BEEF)
+    outputIndex: number
+    topic: string
+  }
+
+/* ---------------------------------------------------------------------------
+ *  Spend-Notification payloads
+ * -------------------------------------------------------------------------- */
+export type OutputSpent =
+  | { // «none»      – “it was spent”
+    mode: 'none'
+    txid: string
+    outputIndex: number
+    topic: string
+  }
+  | { // «txid»      – pointer only
+    mode: 'txid'
+    txid: string
+    outputIndex: number
+    topic: string
+    spendingTxid: string
+  }
+  | { // «script»    – granular input data
+    mode: 'script'
+    txid: string
+    outputIndex: number
+    topic: string
+    spendingTxid: string
+    inputIndex: number
+    unlockingScript: Script
+    sequenceNumber: number
+  }
+  | { // «whole-tx»  – full spending TX
+    mode: 'whole-tx'
+    txid: string
+    outputIndex: number
+    topic: string
+    spendingAtomicBEEF: number[]
+  }
+
+/* ---------------------------------------------------------------------------
+ *  Metadata structure returned by getMetaData()
+ * -------------------------------------------------------------------------- */
+export interface LookupServiceMetaData {
+  name: string
+  shortDescription: string
+  iconURL?: string
+  version?: string
+  informationURL?: string
+}
+
+/* ---------------------------------------------------------------------------
+ *  Lookup Service Interface
+ * -------------------------------------------------------------------------- */
 export interface LookupService {
+  /* -------------------------------------------------------------------------
+   *  REQUIRED static declarations
+   * ----------------------------------------------------------------------- */
+  readonly admissionMode: AdmissionMode
+  readonly spendNotificationMode: SpendNotificationMode
 
+  /* -------------------------------------------------------------------------
+   *  REQUIRED lifecycle hooks
+   * ----------------------------------------------------------------------- */
   /**
-   * Process the event when a new UTXO is let into a topic
-   * @param txid - The transaction ID (TXID) of the transaction where the new UTXO resides.
-   * @param outputIndex - The index of the transaction output (UTXO) within the transaction.
-   * @param outputScript - The script associated with the transaction output.
-   * @param topic - The topic to which the new UTXO is being added.
-   * @returns A promise that resolves when the processing of the new UTXO is complete.
+   * Invoked when a Topic Manager admits a new UTXO.
+   * The payload shape depends on this.admissionMode.
    */
-  outputAdded?: (txid: string, outputIndex: number, outputScript: Script, topic: string) => Promise<void>
+  outputAdmittedByTopic: (payload: OutputAdmittedByTopic) => Promise<void> | void
 
   /**
-   * Processes the spend event for a UTXO.
-   *
-   * @param txid - The transaction ID of the spent UTXO.
-   * @param outputIndex - The index of the transaction output that was spent.
-   * @param topic - The topic from which the UTXO was spent.
-   *
-   * @returns A promise that resolves when processing is complete.
+   * Invoked when a previously-admitted UTXO is spent.
+   * The payload shape depends on this.spendNotificationMode.
    */
-  outputSpent?: (txid: string, outputIndex: number, topic: string) => Promise<void>
+  outputSpent?: (payload: OutputSpent) => Promise<void> | void
 
   /**
-   * Processes the deletion event for a UTXO.
-   *
-   * @param txid - The transaction ID of the deleted UTXO.
-   * @param outputIndex - The index of the transaction output that was deleted.
-   * @param topic - The topic from which the UTXO was deleted.
-   *
-   * @returns A promise that resolves when processing is complete.
+   * Called when a Topic Manager decides that **historical retention** of the
+   * specified UTXO is no longer required.
    */
-  outputDeleted?: (txid: string, outputIndex: number, topic: string) => Promise<void>
+  outputNoLongerRetainedInHistory?: (
+    txid: string,
+    outputIndex: number,
+    topic: string
+  ) => Promise<void> | void
 
   /**
-   * Queries the lookup service for information
-   * @param question — The question to be answered by the lookup service
-   * @returns — The Lookup Answer or Lookup Formula used to answer the question
+   * LEGAL EVICTION:
+   * Permanently remove the referenced UTXO from all indices maintained by the
+   * Lookup Service.  After eviction the service MUST NOT reference the output
+   * in any future lookup answer.
    */
+  outputEvicted: (
+    txid: string,
+    outputIndex: number
+  ) => Promise<void> | void
+
+  /* -------------------------------------------------------------------------
+   *  Query API
+   * ----------------------------------------------------------------------- */
   lookup: (question: LookupQuestion) => Promise<LookupAnswer | LookupFormula>
 
-  /**
-   * Returns a Markdown-formatted documentation string for the lookup service.
-   *
-   * @returns A promise that resolves to a documentation string.
-   */
+  /* -------------------------------------------------------------------------
+   *  Documentation helpers
+   * ----------------------------------------------------------------------- */
   getDocumentation: () => Promise<string>
-
-  /**
-   * Returns a metadata object that can be used to identify the lookup service.
-   *
-   * @returns A promise that resolves to a metadata object containing the name, short description,
-   *          and optional properties such as icon URL, version, and information URL.
-   */
-  getMetaData: () => Promise<{
-    name: string
-    shortDescription: string
-    iconURL?: string
-    version?: string
-    informationURL?: string
-  }>
+  getMetaData: () => Promise<LookupServiceMetaData>
 }

package/src/Output.ts

@@ -1,7 +1,7 @@
 /**
  * Represents an output to be tracked by the Overlay Services Engine
  */
-export type Output = {
+export interface Output {
   /** TXID of the output */
   txid: string
   /** index of the output */

package/src/__tests/Engine.test.ts

@@ -25,7 +25,7 @@ const mockOutput: Output = {
   outputIndex: 0,
   outputScript: exampleTX.outputs[0].lockingScript.toBinary(),
   topic: 'hello',
-  satoshis: exampleTX.outputs[0].satoshis as number,
+  satoshis: exampleTX.outputs[0].satoshis,
   beef: exampleBeef,
   spent: false,
   outputsConsumed: [],
@@ -33,20 +33,19 @@ const mockOutput: Output = {
 }
 
 const invalidHostingUrls = [
-  "http://example.com",               // Invalid: http
-  "https://localhost:3000",           // Invalid: localhost
-  "https://192.168.1.1",              // Invalid: internal private IP
-  "https://127.0.0.1",                // Invalid: loopback IP
-  "https://0.0.0.0",                  // Invalid: non-routable IP
-  "http://172.16.0.1",                // Invalid: private IP
-  "[::1]"
+  'http://example.com', // Invalid: http
+  'https://localhost:3000', // Invalid: localhost
+  'https://192.168.1.1', // Invalid: internal private IP
+  'https://127.0.0.1', // Invalid: loopback IP
+  'https://0.0.0.0', // Invalid: non-routable IP
+  'http://172.16.0.1', // Invalid: private IP
+  '[::1]'
 ]
 
-
 const validHostingUrls = [
-  "https://example.com",              // Valid: public URL
-  "https://8.8.8.8",                  // Valid: public routable IP
-  "https://255.255.255.255",          // Valid: public routable IP
+  'https://example.com', // Valid: public URL
+  'https://8.8.8.8', // Valid: public routable IP
+  'https://255.255.255.255' // Valid: public routable IP
 ]
 
 const mockAdvertiser: Advertiser = {
@@ -70,10 +69,13 @@ describe('BSV Overlay Services Engine', () => {
     }
 
     mockLookupService = {
-      outputAdded: jest.fn(),
+      outputAdmittedByTopic: jest.fn(),
       outputSpent: jest.fn(),
       lookup: jest.fn(),
-      outputDeleted: jest.fn(),
+      outputNoLongerRetainedInHistory: jest.fn(),
+      outputEvicted: jest.fn(),
+      admissionMode: 'locking-script',
+      spendNotificationMode: 'none',
       getDocumentation: async () => 'Service Documentation',
       getMetaData: async () => ({ name: 'Mock Service', shortDescription: 'Mock Short Service Description' })
     }
@@ -107,7 +109,7 @@ describe('BSV Overlay Services Engine', () => {
         undefined
       )
 
-      engine.submit = jest.fn(async (taggedBEEF: TaggedBEEF, onSteakReady: any, mode?: string) => ({ 'tm_helloworld': { outputsToAdmit: [], coinsToRetain: [], coinsRemoved: [] } } as STEAK))
+      engine.submit = jest.fn(async (taggedBEEF: TaggedBEEF, onSteakReady: any, mode?: string) => ({ tm_helloworld: { outputsToAdmit: [], coinsToRetain: [], coinsRemoved: [] } } as STEAK))
 
       // Call the method that would normally trigger syncAdvertisements
       const result = await engine.syncAdvertisements()
@@ -128,8 +130,8 @@ describe('BSV Overlay Services Engine', () => {
       parseAdvertisement: jest.fn().mockReturnValue({
         protocol: 'SHIP',
         topicOrServiceName: 'mock-topic',
-        timestamp: Date.now(),
-      }),
+        timestamp: Date.now()
+      })
     }
 
     for (const url of validHostingUrls) {
@@ -147,7 +149,7 @@ describe('BSV Overlay Services Engine', () => {
       )
 
       // Call the method that would normally trigger syncAdvertisements
-      engine.submit = jest.fn(async (taggedBEEF: TaggedBEEF, onSteakReady: any, mode?: string) => ({ 'tm_helloworld': { outputsToAdmit: [], coinsToRetain: [], coinsRemoved: [] } } as STEAK))
+      engine.submit = jest.fn(async (taggedBEEF: TaggedBEEF, onSteakReady: any, mode?: string) => ({ tm_helloworld: { outputsToAdmit: [], coinsToRetain: [], coinsRemoved: [] } } as STEAK))
       const result = await engine.syncAdvertisements()
       expect(engine.submit).toHaveBeenCalled()
 
@@ -240,7 +242,7 @@ describe('BSV Overlay Services Engine', () => {
       outputIndex: 0,
       outputScript: exampleTX.outputs[0].lockingScript.toBinary(),
       topic: 'hello',
-      satoshis: exampleTX.outputs[0].satoshis as number,
+      satoshis: exampleTX.outputs[0].satoshis,
       beef: exampleBeef,
       spent: false,
       outputsConsumed: [],
@@ -305,7 +307,7 @@ describe('BSV Overlay Services Engine', () => {
           outputIndex,
           outputScript: tx.outputs[outputIndex].lockingScript.toBinary(),
           topic: 'hello',
-          satoshis: tx.outputs[outputIndex].satoshis as number,
+          satoshis: tx.outputs[outputIndex].satoshis,
           beef: tx.toBEEF(),
           spent: false,
           outputsConsumed: [],
@@ -554,7 +556,12 @@ describe('BSV Overlay Services Engine', () => {
             beef: exampleBeef,
             topics: ['Hello']
           })
-          expect(engine.lookupServices.Hello.outputSpent).toHaveBeenCalledWith(exampleTXID, 0, 'Hello')
+          expect(engine.lookupServices.Hello.outputSpent).toHaveBeenCalledWith({
+            mode: 'none',
+            txid: exampleTXID,
+            outputIndex: 0,
+            topic: 'Hello'
+          })
         })
       })
       describe('When previous UTXOs were not retained by the topic manager', () => {
@@ -579,7 +586,7 @@ describe('BSV Overlay Services Engine', () => {
           })
           // Test that previous UTXOs are deleted
           expect(mockStorageEngine.deleteOutput).toHaveBeenCalledWith(exampleTXID, 0, 'hello')
-          expect(mockLookupService.outputDeleted).toHaveBeenCalledWith(exampleTXID, 0, 'hello')
+          expect(mockLookupService.outputNoLongerRetainedInHistory).toHaveBeenCalledWith(exampleTXID, 0, 'hello')
         })
         it('Notifies all lookup services about the output being spent (the notification about the actual deletion will come from deleteUTXODeep)', async () => {
           // Mock findUTXO to return a UTXO
@@ -601,7 +608,7 @@ describe('BSV Overlay Services Engine', () => {
             topics: ['Hello']
           })
           // Was the lookup service notified of the output deletion?
-          expect(mockLookupService.outputDeleted).toHaveBeenCalledWith(exampleTXID, 0, 'hello')
+          expect(mockLookupService.outputNoLongerRetainedInHistory).toHaveBeenCalledWith(exampleTXID, 0, 'hello')
         })
       })
       it('Adds admissible UTXOs to the storage engine', async () => {
@@ -646,12 +653,14 @@ describe('BSV Overlay Services Engine', () => {
           topics: ['Hello']
         })
         // Test the lookup service was notified of the new UTXO
-        expect(mockLookupService.outputAdded).toHaveBeenCalledWith(
-          exampleTXID,
-          0,
-          exampleTX.outputs[0].lockingScript,
-          'Hello'
-        )
+        expect(mockLookupService.outputAdmittedByTopic).toHaveBeenCalledWith({
+          mode: 'locking-script',
+          txid: exampleTXID,
+          satoshis: 26172,
+          outputIndex: 0,
+          lockingScript: exampleTX.outputs[0].lockingScript,
+          topic: 'Hello'
+        })
       })
       describe('For each consumed UTXO', () => {
         it('Finds the UTXO', async () => {

package/src/storage/Storage.ts

@@ -78,7 +78,6 @@ export interface Storage {
    */
   updateTransactionBEEF: (txid: string, beef: number[]) => Promise<void>
 
-
   /**
    * Updates the block height on an output
    * @param txid — TXID of the output to update

package/src/storage/knex/KnexStorage.ts

@@ -5,11 +5,11 @@ import type { Output } from '../../Output.js'
 export class KnexStorage implements Storage {
   knex: Knex
 
-  constructor(knex: Knex) {
+  constructor (knex: Knex) {
     this.knex = knex
   }
 
-  async findOutput(txid: string, outputIndex: number, topic?: string, spent?: boolean, includeBEEF: boolean = false): Promise<Output | null> {
+  async findOutput (txid: string, outputIndex: number, topic?: string, spent?: boolean, includeBEEF: boolean = false): Promise<Output | null> {
     const search: {
       'outputs.txid': string
       'outputs.outputIndex': number
@@ -59,7 +59,7 @@ export class KnexStorage implements Storage {
     }
   }
 
-  async findOutputsForTransaction(txid: string, includeBEEF: boolean = false): Promise<Output[]> {
+  async findOutputsForTransaction (txid: string, includeBEEF: boolean = false): Promise<Output[]> {
     // Base query to get outputs
     const query = this.knex('outputs').where({ 'outputs.txid': txid })
 
@@ -97,7 +97,7 @@ export class KnexStorage implements Storage {
     }))
   }
 
-  async findUTXOsForTopic(topic: string, since?: number, includeBEEF: boolean = false): Promise<Output[]> {
+  async findUTXOsForTopic (topic: string, since?: number, includeBEEF: boolean = false): Promise<Output[]> {
     // Base query to get outputs
     const query = this.knex('outputs').where({ 'outputs.topic': topic, 'outputs.spent': false })
 
@@ -141,7 +141,7 @@ export class KnexStorage implements Storage {
     }))
   }
 
-  async deleteOutput(txid: string, outputIndex: number, topic: string): Promise<void> {
+  async deleteOutput (txid: string, outputIndex: number, topic: string): Promise<void> {
     await this.knex.transaction(async trx => {
       // Delete the specific output
       await trx('outputs').where({ txid, outputIndex }).del()
@@ -156,7 +156,7 @@ export class KnexStorage implements Storage {
     })
   }
 
-  async insertOutput(output: Output): Promise<void> {
+  async insertOutput (output: Output): Promise<void> {
     const insertPromises = [this.knex('outputs').insert({
       txid: output.txid,
       outputIndex: Number(output.outputIndex),
@@ -181,7 +181,7 @@ export class KnexStorage implements Storage {
     })
   }
 
-  async markUTXOAsSpent(txid: string, outputIndex: number, topic?: string): Promise<void> {
+  async markUTXOAsSpent (txid: string, outputIndex: number, topic?: string): Promise<void> {
     await this.knex('outputs').where({
       txid,
       outputIndex,
@@ -189,7 +189,7 @@ export class KnexStorage implements Storage {
     }).update('spent', true)
   }
 
-  async updateConsumedBy(txid: string, outputIndex: number, topic: string, consumedBy: Array<{ txid: string, outputIndex: number }>): Promise<void> {
+  async updateConsumedBy (txid: string, outputIndex: number, topic: string, consumedBy: Array<{ txid: string, outputIndex: number }>): Promise<void> {
     await this.knex('outputs').where({
       txid,
       outputIndex,
@@ -197,13 +197,13 @@ export class KnexStorage implements Storage {
     }).update('consumedBy', JSON.stringify(consumedBy))
   }
 
-  async updateTransactionBEEF(txid: string, beef: number[]): Promise<void> {
+  async updateTransactionBEEF (txid: string, beef: number[]): Promise<void> {
     await this.knex('transactions').where({
       txid
     }).update('beef', Buffer.from(beef))
   }
 
-  async updateOutputBlockHeight(txid: string, outputIndex: number, topic: string, blockHeight: number): Promise<void> {
+  async updateOutputBlockHeight (txid: string, outputIndex: number, topic: string, blockHeight: number): Promise<void> {
     await this.knex('outputs').where({
       txid,
       outputIndex,
@@ -211,14 +211,14 @@ export class KnexStorage implements Storage {
     }).update('blockHeight', blockHeight)
   }
 
-  async insertAppliedTransaction(tx: { txid: string, topic: string }): Promise<void> {
+  async insertAppliedTransaction (tx: { txid: string, topic: string }): Promise<void> {
     await this.knex('applied_transactions').insert({
       txid: tx.txid,
       topic: tx.topic
     })
   }
 
-  async doesAppliedTransactionExist(tx: { txid: string, topic: string }): Promise<boolean> {
+  async doesAppliedTransactionExist (tx: { txid: string, topic: string }): Promise<boolean> {
     const result = await this.knex('applied_transactions')
       .where({ txid: tx.txid, topic: tx.topic })
       .select(this.knex.raw('1'))

package/src/storage/knex/all-migrations.ts

@@ -7,7 +7,7 @@ import { up as addedIndexesUp, down as addedIndexesDown } from './migrations/202
 /**
  * An array of all migrations, in order.
  */
-type Migration = {
+interface Migration {
   up: (knex: Knex) => Promise<void>
   down: (knex: Knex) => Promise<void>
 }

package/src/storage/knex/migrations/2024-05-18-001-initial.ts

@@ -1,6 +1,6 @@
 import type { Knex } from 'knex'
 
-export async function up(knex: Knex): Promise<void> {
+export async function up (knex: Knex): Promise<void> {
   await knex.schema.createTable('outputs', table => {
     table.increments()
     table.string('txid', 64)
@@ -24,7 +24,7 @@ export async function up(knex: Knex): Promise<void> {
   })
 }
 
-export async function down(knex: Knex): Promise<void> {
+export async function down (knex: Knex): Promise<void> {
   await knex.schema.dropTable('applied_transactions')
   await knex.schema.dropTable('outputs')
 }

package/src/storage/knex/migrations/2024-07-10-001-block-height.ts

@@ -1,13 +1,13 @@
 import type { Knex } from 'knex'
 
-export async function up(knex: Knex): Promise<void> {
+export async function up (knex: Knex): Promise<void> {
   // Add new column for blockHeight associated with outputs
   await knex.schema.table('outputs', table => {
     table.integer('blockHeight').unsigned().nullable()
   })
 }
 
-export async function down(knex: Knex): Promise<void> {
+export async function down (knex: Knex): Promise<void> {
   // Remove the blockHeight column (rollback operation)
   await knex.schema.table('outputs', table => {
     table.dropColumn('blockHeight')

package/src/storage/knex/migrations/2024-07-17-001-transactions.ts

@@ -1,6 +1,6 @@
 import type { Knex } from 'knex'
 
-export async function up(knex: Knex): Promise<void> {
+export async function up (knex: Knex): Promise<void> {
   // Create the new transactions table
   await knex.schema.createTable('transactions', table => {
     table.increments()
@@ -22,7 +22,7 @@ export async function up(knex: Knex): Promise<void> {
   })
 }
 
-export async function down(knex: Knex): Promise<void> {
+export async function down (knex: Knex): Promise<void> {
   // Add the beef column back to the outputs table
   await knex.schema.table('outputs', table => {
     table.binary('beef')

package/src/storage/knex/migrations/2024-07-18-001-indexes.ts

@@ -1,5 +1,5 @@
 import type { Knex } from 'knex'
-export async function up(knex: Knex): Promise<void> {
+export async function up (knex: Knex): Promise<void> {
   // Add index for applied_transactions table
   await knex.schema.table('applied_transactions', function (table) {
     table.index(['txid', 'topic'], 'idx_applied_transactions_txid_topic')
@@ -14,7 +14,7 @@ export async function up(knex: Knex): Promise<void> {
   })
 }
 
-export async function down(knex: Knex): Promise<void> {
+export async function down (knex: Knex): Promise<void> {
   // Drop index for applied_transactions table
   await knex.schema.table('applied_transactions', function (table) {
     table.dropIndex(['txid', 'topic'], 'idx_applied_transactions_txid_topic')