@bsv/overlay 2.2.0 → 2.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bsv/overlay",
-  "version": "2.2.0",
+  "version": "2.2.1",
   "type": "module",
   "description": "BSV Blockchain Overlay Services Engine",
   "main": "dist/cjs/mod.js",
@@ -55,18 +55,18 @@
   "homepage": "https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay#readme",
   "devDependencies": {
     "@types/jest": "^30.0.0",
-    "@types/node": "^26.0.0",
+    "@types/node": "^26.1.1",
     "jest": "^30.4.2",
-    "ts-jest": "^29.4.11",
-    "ts-standard": "^12.0.2",
-    "ts2md": "^0.2.0",
+    "ts-jest": "^29.4.12",
+    "ts2md": "^0.2.8",
     "tsconfig-to-dual-package": "^1.2.0",
     "typescript": "^6.0.3",
-    "@bsv/sdk": "^2.1.3"
+    "oxlint": "^1.75.0",
+    "@bsv/sdk": "^2.2.0"
   },
   "dependencies": {
-    "@bsv/gasp": "^1.3.0",
-    "knex": "^3.2.10"
+    "knex": "^3.3.0",
+    "@bsv/gasp": "^1.3.1"
   },
   "peerDependencies": {
     "@bsv/sdk": "^2.1.6"
@@ -80,8 +80,8 @@
     "test": "npm run build && jest",
     "test:watch": "npm run build && jest --watch",
     "test:coverage": "npm run build && jest --coverage",
-    "lint": "ts-standard --fix src/**/*.ts",
-    "lint:ci": "ts-standard src/**/*.ts",
+    "lint": "oxlint src",
+    "lint:ci": "oxlint src",
     "build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json",
     "dev": "tsc -b -w",
     "prepublish": "npm run build",

package/src/Engine.ts

@@ -919,7 +919,7 @@ export class Engine {
   async lookup(lookupQuestion: LookupQuestion): Promise<LookupAnswer> {
     // Validate a lookup service for the provider is found
     const lookupService = this.lookupServices[lookupQuestion.service]
-    if (lookupService === undefined || lookupService === null) throw new Error(`Lookup service not found for provider: ${lookupQuestion.service} `)
+    if (lookupService === undefined || lookupService === null) throw new Error(`Lookup service not found for provider: ${lookupQuestion.service}`)
 
     const lookupResult = await lookupService.lookup(lookupQuestion)
     const hydrationContext = this.createUTXOHistoryHydrationContext()
@@ -1954,20 +1954,20 @@ export class Engine {
    * @param proof for txid
    */
   private updateInputProofs(tx: Transaction, txid: string, proof: MerklePath): void {
-    if (tx.merklePath !== undefined) {
+    if (tx.id('hex') === txid) {
       // Update the merkle path to handle potential reorgs
       tx.merklePath = proof
       return
     }
-    if (tx.id('hex') === txid) {
-      tx.merklePath = proof
-    } else {
-      for (const input of tx.inputs) {
-        // All inputs must have sourceTransactions
-        const stx = input.sourceTransaction
-        if (typeof stx !== 'object') continue
-        this.updateInputProofs(stx, txid, proof)
-      }
+    // A mined transaction's source graph is no longer part of its BEEF. Do not
+    // replace an unrelated transaction's proof with the proof for an ancestor.
+    if (tx.merklePath !== undefined) return
+
+    for (const input of tx.inputs) {
+      // All inputs must have sourceTransactions
+      const stx = input.sourceTransaction
+      if (typeof stx !== 'object') continue
+      this.updateInputProofs(stx, txid, proof)
     }
   }
 
@@ -1984,13 +1984,8 @@ export class Engine {
     }
 
     const tx = Transaction.fromBEEF(output.beef)
-    if (tx.merklePath !== undefined) {
-      // Update the merkle path to handle potential reorgs
-      tx.merklePath = proof
-      return
-    }
-
-    // recursively update all sourceTransactions proven by (txid,proof)
+    // Update this transaction, or recursively update the matching source
+    // transaction. This also persists replacement proofs after a reorg.
     this.updateInputProofs(tx, txid, proof)
 
     // Update the output's BEEF in the storage DB

package/src/GASP/OverlayGASPStorage.ts

@@ -145,7 +145,8 @@ export class OverlayGASPStorage implements GASPStorage {
         }
         return await this.stripAlreadyKnownInputs(response)
       } catch (e) {
-        console.error(`An error occurred when identifying needed inputs for transaction: ${parsedTx.id('hex')}.${tx.outputIndex}: ${e}`)
+        const message = e instanceof Error ? e.message : String(e)
+        console.error(`An error occurred when identifying needed inputs for transaction: ${parsedTx.id('hex')}.${tx.outputIndex}: ${message}`)
         // Cut off the graph in case of an error here.
       }
       // By default, if the topic manager isn't able to stipulate needed inputs, only the inputs necessary for SPV are requested.

package/src/__tests/BASMChain.test.ts

@@ -28,6 +28,11 @@ interface FakeStore {
   applied?: AppliedRow[]
 }
 
+function must<T> (value: T | undefined, message: string): T {
+  if (value === undefined) throw new Error(message)
+  return value
+}
+
 function makeStorage (store: FakeStore): any {
   const key = (topic: string, height: number): string => `${topic}:${height}`
   return {
@@ -111,11 +116,11 @@ describe('BRC-136 BASM anchor chain continuity', () => {
     const store: FakeStore = { anchors: new Map(), admitted: new Map() }
 
     // Genesis: topic admits two txs at height 100.
-    store.admitted!.set(100, [
+    must(store.admitted, 'admitted transaction map').set(100, [
       { txid: TXID_1, blockIndex: 0 },
       { txid: TXID_2, blockIndex: 1 }
     ])
-    const genesisRoot = computeBasmRoot(store.admitted!.get(100)!)
+    const genesisRoot = computeBasmRoot(must(must(store.admitted, 'admitted transaction map').get(100), 'genesis admissions'))
     const genesisTac = computeTac(ZERO, blockHashFor(100), genesisRoot)
     store.anchors.set('tm_test:100', {
       topic: 'tm_test',
@@ -137,7 +142,7 @@ describe('BRC-136 BASM anchor chain continuity', () => {
     // 101..105 are empty anchors (zero root, zero count) chained off genesis.
     let expectedTac = genesisTac
     for (let h = 101; h <= 105; h++) {
-      const anchor = store.anchors.get(`tm_test:${h}`)!
+      const anchor = must(store.anchors.get(`tm_test:${h}`), `anchor ${h}`)
       expect(anchor.basmRoot).toBe(ZERO)
       expect(anchor.admittedCount).toBe(0)
       expectedTac = computeTac(expectedTac, blockHashFor(h), ZERO)
@@ -146,7 +151,7 @@ describe('BRC-136 BASM anchor chain continuity', () => {
 
     // The tip TAC is a cumulative hash that still depends on the genesis block —
     // i.e. it was NOT reset to a per-block value.
-    const tipTac = store.anchors.get('tm_test:105')!.tac
+    const tipTac = must(store.anchors.get('tm_test:105'), 'tip anchor').tac
     const resetTac = computeTac(ZERO, blockHashFor(105), ZERO)
     expect(tipTac).not.toBe(resetTac)
   })
@@ -192,29 +197,29 @@ describe('BRC-136 BASM reorg handling', () => {
     })
 
     // TXID_2 is demoted: no longer proven, block metadata cleared.
-    const row = store.applied!.find(r => r.txid === TXID_2)!
+    const row = must(store.applied?.find(r => r.txid === TXID_2), 'orphaned applied row')
     expect(row.proven).toBe(false)
     expect(row.blockHeight).toBeUndefined()
     // It survives as a receipt record (firstSeenHeight retained).
     expect(row.firstSeenHeight).toBe(101)
 
     // Anchor at 101 rebuilt: empty admitted set, canonical hash, recomputed TAC.
-    const anchor101 = store.anchors.get('tm_test:101')!
+    const anchor101 = must(store.anchors.get('tm_test:101'), 'anchor 101')
     expect(anchor101.admittedCount).toBe(0)
     expect(anchor101.basmRoot).toBe(ZERO)
     expect(anchor101.blockHash).toBe(H101_NEW)
-    const tac100 = store.anchors.get('tm_test:100')!.tac
+    const tac100 = must(store.anchors.get('tm_test:100'), 'anchor 100').tac
     expect(anchor101.tac).toBe(computeTac(tac100, H101_NEW, ZERO))
 
     // Height 100 is untouched.
     const root100 = computeBasmRoot([{ txid: TXID_1, blockIndex: 0 }])
-    expect(store.anchors.get('tm_test:100')!.basmRoot).toBe(root100)
+    expect(must(store.anchors.get('tm_test:100'), 'anchor 100').basmRoot).toBe(root100)
   })
 
   it('is idempotent: orphaned hashes matching no proven rows leave the chain unchanged', async () => {
     const store = seedTwoBlockChain()
     const engine = makeEngine(store, { currentHeight: 101 })
-    const before = store.anchors.get('tm_test:101')!.tac
+    const before = must(store.anchors.get('tm_test:101'), 'anchor 101').tac
 
     await (engine as any).handleReorg({
       orphanedBlockHashes: ['ffff000000000000000000000000000000000000000000000000000000000000'],
@@ -222,8 +227,8 @@ describe('BRC-136 BASM reorg handling', () => {
       newTipHeight: 101
     })
 
-    expect(store.applied!.find(r => r.txid === TXID_2)!.proven).toBe(true)
-    expect(store.anchors.get('tm_test:101')!.tac).toBe(before)
+    expect(must(store.applied?.find(r => r.txid === TXID_2), 'applied row 2').proven).toBe(true)
+    expect(must(store.anchors.get('tm_test:101'), 'anchor 101').tac).toBe(before)
   })
 
   it('re-proving a demoted tx at a new height restores it to the admitted set', async () => {
@@ -235,10 +240,10 @@ describe('BRC-136 BASM reorg handling', () => {
       rebuildFromHeight: 101,
       newTipHeight: 101
     })
-    expect(store.anchors.get('tm_test:101')!.basmRoot).toBe(ZERO)
+    expect(must(store.anchors.get('tm_test:101'), 'anchor 101').basmRoot).toBe(ZERO)
 
     // TXID_2 re-mined into the new canonical block at 101.
-    const row = store.applied!.find(r => r.txid === TXID_2)!
+    const row = must(store.applied?.find(r => r.txid === TXID_2), 'applied row 2')
     row.proven = true
     row.blockHeight = 101
     row.blockHash = H101_NEW
@@ -251,7 +256,7 @@ describe('BRC-136 BASM reorg handling', () => {
       newTipHeight: 101
     })
 
-    const anchor101 = store.anchors.get('tm_test:101')!
+    const anchor101 = must(store.anchors.get('tm_test:101'), 'anchor 101')
     expect(anchor101.admittedCount).toBe(1)
     expect(anchor101.basmRoot).toBe(computeBasmRoot([{ txid: TXID_2, blockIndex: 0 }]))
     expect(anchor101.blockHash).toBe(H101_NEW)
@@ -268,23 +273,23 @@ describe('BRC-136 BASM reorg revalidation sweep', () => {
 
     await (engine as any).revalidateRecentAnchors(3)
 
-    expect(store.applied!.find(r => r.txid === TXID_2)!.proven).toBe(false)
-    const anchor101 = store.anchors.get('tm_test:101')!
+    expect(must(store.applied?.find(r => r.txid === TXID_2), 'applied row 2').proven).toBe(false)
+    const anchor101 = must(store.anchors.get('tm_test:101'), 'anchor 101')
     expect(anchor101.basmRoot).toBe(ZERO)
     expect(anchor101.blockHash).toBe(H101_NEW)
     // Height 100 still valid and untouched.
-    expect(store.applied!.find(r => r.txid === TXID_1)!.proven).toBe(true)
-    expect(store.anchors.get('tm_test:100')!.basmRoot).toBe(computeBasmRoot([{ txid: TXID_1, blockIndex: 0 }]))
+    expect(must(store.applied?.find(r => r.txid === TXID_1), 'applied row 1').proven).toBe(true)
+    expect(must(store.anchors.get('tm_test:100'), 'anchor 100').basmRoot).toBe(computeBasmRoot([{ txid: TXID_1, blockIndex: 0 }]))
   })
 
   it('leaves the chain unchanged when every proof in the window still validates', async () => {
     const store = seedTwoBlockChain()
     const engine = makeEngine(store, { currentHeight: 101, isValidRootForHeight: async () => true })
-    const before = store.anchors.get('tm_test:101')!.tac
+    const before = must(store.anchors.get('tm_test:101'), 'anchor 101').tac
 
     await (engine as any).revalidateRecentAnchors(3)
 
-    expect(store.applied!.find(r => r.txid === TXID_2)!.proven).toBe(true)
-    expect(store.anchors.get('tm_test:101')!.tac).toBe(before)
+    expect(must(store.applied?.find(r => r.txid === TXID_2), 'applied row 2').proven).toBe(true)
+    expect(must(store.anchors.get('tm_test:101'), 'anchor 101').tac).toBe(before)
   })
 })

package/src/__tests/Engine.test.ts

@@ -309,6 +309,37 @@ describe('BSV Overlay Services Engine', () => {
     })
   })
   describe('handleNewMerkleProof tests', () => {
+    it('persists a replacement proof when the stored BEEF is already mined', async () => {
+      const tx = Transaction.fromHexBEEF(beef37abad0)
+      const txid = tx.id('hex')
+      const output: Output = {
+        txid,
+        outputIndex: 0,
+        outputScript: tx.outputs[0].lockingScript.toBinary(),
+        topic: 'hello',
+        satoshis: tx.outputs[0].satoshis,
+        beef: tx.toBEEF(),
+        spent: false,
+        outputsConsumed: [],
+        consumedBy: [],
+        score: Date.now()
+      }
+      const updateTransactionBEEF = jest.fn(async () => undefined)
+      mockStorageEngine.findOutputsForTransaction = jest.fn(async () => [output])
+      mockStorageEngine.updateTransactionBEEF = updateTransactionBEEF
+      const engine = new Engine(
+        { Hello: mockTopicManager },
+        { Hello: mockLookupService },
+        mockStorageEngine,
+        mockChainTracker
+      )
+
+      if (tx.merklePath === undefined) throw new Error('improper test setup')
+      await engine.handleNewMerkleProof(txid, tx.merklePath)
+
+      expect(updateTransactionBEEF).toHaveBeenCalledWith(txid, tx.toBEEF())
+    })
+
     it('0 simple proof', async () => {
       const beef = beef27c8f1
       const txid = txid27c8f
@@ -423,7 +454,12 @@ describe('BSV Overlay Services Engine', () => {
 
       const mp37abad = Transaction.fromHexBEEF(beef37abad0).merklePath
       await engine.handleNewMerkleProof(txid37abad, mp37abad)
-      expect(Object.keys(newBEEF).length).toBe(0)
+      expect(newBEEF[`${txid37abad}`]).toBe(beef37abad0)
+      expect(newBEEF[`${txid877734}`].length).toBeGreaterThan(beef8777340.length)
+      expect(newBEEF[`${txid509f5e}`].length).toBeGreaterThan(beef509f5e0.length)
+      expect(newBEEF[`${txid942620}`].length).toBeGreaterThan(beef9426200.length)
+      expect(newBEEF[`${txid17d182}`].length).toBeGreaterThan(beef17d1820.length)
+      expect(Object.keys(newBEEF)).toHaveLength(5)
 
       const mp877734 = Transaction.fromHexBEEF(beef8777340).merklePath
       await engine.handleNewMerkleProof(txid877734, mp877734)
@@ -431,7 +467,7 @@ describe('BSV Overlay Services Engine', () => {
       expect(newBEEF[`${txid509f5e}`].length).toBeGreaterThan(beef509f5e0.length)
       expect(newBEEF[`${txid942620}`].length).toBeGreaterThan(beef9426200.length)
       expect(newBEEF[`${txid17d182}`].length).toBeGreaterThan(beef17d1820.length)
-      expect(Object.keys(newBEEF).length).toBe(4)
+      expect(Object.keys(newBEEF)).toHaveLength(5)
 
       const mp509f5e = Transaction.fromHexBEEF(beef509f5e0).merklePath
       await engine.handleNewMerkleProof(txid509f5e, mp509f5e)
@@ -439,7 +475,7 @@ describe('BSV Overlay Services Engine', () => {
       expect(newBEEF[`${txid509f5e}`]).toBe(beef509f5e0)
       expect(newBEEF[`${txid942620}`].length).toBeGreaterThan(beef9426200.length)
       expect(newBEEF[`${txid17d182}`].length).toBeGreaterThan(beef17d1820.length)
-      expect(Object.keys(newBEEF).length).toBe(4)
+      expect(Object.keys(newBEEF)).toHaveLength(5)
 
       const mp942620 = Transaction.fromHexBEEF(beef9426200).merklePath
       await engine.handleNewMerkleProof(txid942620, mp942620)
@@ -447,7 +483,7 @@ describe('BSV Overlay Services Engine', () => {
       expect(newBEEF[`${txid509f5e}`]).toBe(beef509f5e0)
       expect(newBEEF[`${txid942620}`]).toBe(beef9426200)
       expect(newBEEF[`${txid17d182}`].length).toBeGreaterThan(beef17d1820.length)
-      expect(Object.keys(newBEEF).length).toBe(4)
+      expect(Object.keys(newBEEF)).toHaveLength(5)
 
       const mp17d182 = Transaction.fromHexBEEF(beef17d1820).merklePath
       await engine.handleNewMerkleProof(txid17d182, mp17d182)
@@ -455,7 +491,7 @@ describe('BSV Overlay Services Engine', () => {
       expect(newBEEF[`${txid509f5e}`]).toBe(beef509f5e0)
       expect(newBEEF[`${txid942620}`]).toBe(beef9426200)
       expect(newBEEF[`${txid17d182}`]).toBe(beef17d1820)
-      expect(Object.keys(newBEEF).length).toBe(4)
+      expect(Object.keys(newBEEF)).toHaveLength(5)
     })
   })
 
@@ -963,7 +999,7 @@ describe('BSV Overlay Services Engine', () => {
         await expect(engine.lookup({
           service: 'HelloWorld',
           query: { name: 'Bob' }
-        })).rejects.toThrow()
+        })).rejects.toThrow('Lookup service not found for provider: HelloWorld')
       })
       it('Calls the lookup function from the lookup service', async () => {
         // TODO: Make the default storage engine return something...?

package/src/__tests/OverlayGASPStorage.test.ts

@@ -125,6 +125,40 @@ describe('OverlayGASPStorage', () => {
         requestedInputs: { 'neededTxid.1': { metadata: false } }
       })
     })
+
+    it.each([
+      [new Error('manager failed'), 'manager failed'],
+      ['manager failed as a string', 'manager failed as a string']
+    ])('terminates the graph and logs normalized identifyNeededInputs errors', async (failure, message) => {
+      const mockTx: GASPNode = {
+        rawTx: '001122',
+        proof: 'someproof',
+        graphID: 'txid123.0',
+        outputIndex: 0
+      }
+      const parsedTx = {
+        inputs: [],
+        toBEEF: jest.fn().mockReturnValue([1, 2, 3]),
+        id: jest.fn().mockReturnValue('txid123'),
+        merklePath: {}
+      }
+      Transaction.fromHex = jest.fn().mockReturnValue(parsedTx)
+      MerklePath.fromHex = jest.fn().mockReturnValue(parsedTx.merklePath)
+      mockEngine.managers['test-topic'] = {
+        identifyAdmissibleOutputs: jest.fn().mockResolvedValue({
+          outputsToAdmit: []
+        }),
+        identifyNeededInputs: jest.fn().mockRejectedValue(failure)
+      }
+      const consoleError = jest.spyOn(console, 'error').mockImplementation(() => {})
+
+      await expect(overlayStorage.findNeededInputs(mockTx)).resolves.toBeUndefined()
+
+      expect(consoleError).toHaveBeenCalledWith(
+        `An error occurred when identifying needed inputs for transaction: txid123.0: ${message}`
+      )
+      consoleError.mockRestore()
+    })
   })
 
   describe('discardGraph', () => {

package/src/storage/knex/migrations/2024-07-17-001-transactions.ts

@@ -1,6 +1,8 @@
 import type { Knex } from 'knex'
 
 export async function up (knex: Knex): Promise<void> {
+  const client = (knex.client.config.client || '').toLowerCase()
+
   // Create the new transactions table
   await knex.schema.createTable('transactions', table => {
     table.increments()
@@ -9,12 +11,23 @@ export async function up (knex: Knex): Promise<void> {
   })
 
   // Move data from outputs table to the new transactions table and ensure deduplication
-  await knex.raw(`
-    INSERT IGNORE INTO transactions (txid, beef)
-    SELECT txid, beef
-    FROM outputs
-    WHERE beef IS NOT NULL
-  `)
+  if (client.startsWith('mysql')) {
+    await knex.raw(`
+      INSERT IGNORE INTO transactions (txid, beef)
+      SELECT txid, beef
+      FROM outputs
+      WHERE beef IS NOT NULL
+    `)
+  } else {
+    // SQLite / PostgreSQL: INSERT IGNORE is MySQL-only syntax
+    await knex.raw(`
+      INSERT INTO transactions (txid, beef)
+      SELECT txid, beef
+      FROM outputs
+      WHERE beef IS NOT NULL
+      ON CONFLICT (txid) DO NOTHING
+    `)
+  }
 
   // Drop the beef column from the outputs table
   await knex.schema.table('outputs', table => {
@@ -23,17 +36,28 @@ export async function up (knex: Knex): Promise<void> {
 }
 
 export async function down (knex: Knex): Promise<void> {
+  const client = (knex.client.config.client || '').toLowerCase()
+
   // Add the beef column back to the outputs table
   await knex.schema.table('outputs', table => {
     table.binary('beef')
   })
 
-  // Move data back from the transactions table to the outputs table using a JOIN
-  await knex.raw(`
-    UPDATE outputs
-    JOIN transactions ON outputs.txid = transactions.txid
-    SET outputs.beef = transactions.beef
-  `)
+  // Move data back from the transactions table to the outputs table
+  if (client.startsWith('mysql')) {
+    await knex.raw(`
+      UPDATE outputs
+      JOIN transactions ON outputs.txid = transactions.txid
+      SET outputs.beef = transactions.beef
+    `)
+  } else {
+    // SQLite / PostgreSQL: UPDATE ... JOIN is MySQL-only syntax
+    await knex.raw(`
+      UPDATE outputs
+      SET beef = (SELECT beef FROM transactions WHERE transactions.txid = outputs.txid)
+      WHERE txid IN (SELECT txid FROM transactions)
+    `)
+  }
 
   // Drop the transactions table
   await knex.schema.dropTable('transactions')