@bsv/overlay 0.1.0-alpha.9 → 0.1.1

package/src/SyncConfiguration.ts

@@ -0,0 +1,19 @@
+/**
+ * Configuration for synchronizing supported topic managers.
+ *
+ * This configuration determines which topics should support synchronization and specifies the mode of synchronization.
+ *
+ * There are two synchronization modes:
+ * 1. Sync to predefined hardcoded peers for the specified topic, including associated hosting URLs.
+ * 2. Use SHIP (Service Host Interconnect Protocol) to sync with all known peers that support the specified topic.
+ *
+ * Each entry in the configuration object maps a topic to either an array of overlay service peers (hardcoded URLs) or the string 'SHIP' (for dynamic syncing using SHIP).
+ *
+ * @example
+ * // Example usage of SyncConfiguration
+ * const config: SyncConfiguration = {
+ *   "topicManager1": ["http://peer1.com", "http://peer2.com"],
+ *   "topicManager2": "SHIP"
+ * }
+ */
+export type SyncConfiguration = Record<string, string[] | 'SHIP'>

package/src/TopicManager.ts

@@ -11,6 +11,12 @@ export interface TopicManager {
    */
   identifyAdmissibleOutputs: (beef: number[], previousCoins: number[]) => Promise<AdmittanceInstructions>
 
+  /**
+   * Identifies and returns the inputs needed to anchor any topical outputs from this transaction to their associated previous history.
+   * @throws - if there are no potentially valid topical outputs in this transaction
+   */
+  identifyNeededInputs?: (beef: number[]) => Promise<Array<{ txid: string, outputIndex: number }>>
+
   /**
   * Returns a Markdown-formatted documentation string for the topic manager.
   */

package/src/__tests/Engine.test.ts

@@ -3,7 +3,7 @@ import { LookupService } from '../LookupService'
 import { TopicManager } from '../TopicManager'
 import { AdmittanceInstructions } from '../AdmittanceInstructions'
 import { Storage } from '../storage/Storage'
-import { Transaction } from '@bsv/sdk'
+import { Transaction, Utils } from '@bsv/sdk'
 import { Output } from '../Output'
 
 const mockChainTracker = {
@@ -14,7 +14,7 @@ const BRC62Hex = '0100beef01fe636d0c0007021400fe507c0c7aa754cef1f7889d5fd395cf1f
 const exampleTX = Transaction.fromHexBEEF(BRC62Hex)
 
 const exampleBeef = exampleTX.toBEEF()
-const exampleTXID = exampleTX.id('hex') as string
+const exampleTXID = exampleTX.id('hex')
 const examplePreviousTXID = '3ecead27a44d013ad1aae40038acbb1883ac9242406808bb4667c15b4f164eac'
 let mockTopicManager: TopicManager, mockLookupService: LookupService, mockStorageEngine: Storage
 const mockOutput: Output = {
@@ -22,7 +22,7 @@ const mockOutput: Output = {
   outputIndex: 0,
   outputScript: exampleTX.outputs[0].lockingScript.toBinary(),
   topic: 'hello',
-  satoshis: exampleTX.outputs[0].satoshis as number,
+  satoshis: exampleTX.outputs[0].satoshis,
   beef: exampleBeef,
   spent: false,
   outputsConsumed: [],
@@ -60,32 +60,33 @@ describe('BSV Overlay Services Engine', () => {
       markUTXOAsSpent: jest.fn(),
       updateConsumedBy: jest.fn(),
       updateOutputBeef: jest.fn(),
-      deleteOutput: jest.fn()
+      deleteOutput: jest.fn(),
+      findUTXOsForTopic: jest.fn()
     }
   })
   describe('handleNewMerkleProof tests', () => {
-const mockOutput: Output = {
-  txid: exampleTXID,
-  outputIndex: 0,
-  outputScript: exampleTX.outputs[0].lockingScript.toBinary(),
-  topic: 'hello',
-  satoshis: exampleTX.outputs[0].satoshis as number,
-  beef: exampleBeef,
-  spent: false,
-  outputsConsumed: [],
-  consumedBy: []
-}
+    const mockOutput: Output = {
+      txid: exampleTXID,
+      outputIndex: 0,
+      outputScript: exampleTX.outputs[0].lockingScript.toBinary(),
+      topic: 'hello',
+      satoshis: exampleTX.outputs[0].satoshis,
+      beef: exampleBeef,
+      spent: false,
+      outputsConsumed: [],
+      consumedBy: []
+    }
     it('0 simple proof', async () => {
       const beef = beef27c8f_1
       const txid = txid27c8f
       const tx = Transaction.fromHexBEEF(beef)
 
       const output27c8f_0: Output = {
-        txid: txid,
+        txid,
         outputIndex: 0,
         outputScript: tx.outputs[0].lockingScript.toBinary(),
         topic: 'hello',
-        satoshis: tx.outputs[0].satoshis as number,
+        satoshis: tx.outputs[0].satoshis,
         beef: tx.toBEEF(),
         spent: false,
         outputsConsumed: [],
@@ -93,7 +94,7 @@ const mockOutput: Output = {
       }
 
       mockLookupService.lookup = jest.fn(async () => [{
-        txid: txid,
+        txid,
         outputIndex: 0,
         history: 1
       }])
@@ -115,10 +116,112 @@ const mockOutput: Output = {
       )
 
       const merklePath = Transaction.fromHexBEEF(beef27c8f_0).merklePath
-      if (!merklePath) throw('improper test setup')
+      if (!merklePath) throw ('improper test setup')
       await engine.handleNewMerkleProof(txid, merklePath)
       expect(newBEEF.length > 0 && newBEEF.length < beef.length / 2).toBe(true)
     })
+
+    it('1 recurse proof', async () => {
+      const outputs: Output[] = []
+      const findOutput = (txid: string, outputIndex: number): Output => {
+        const i = outputs.findIndex(o => o.txid === txid && o.outputIndex === outputIndex)
+        if (i < 0) throw new Error(`missing output ${txid} ${outputIndex}`)
+        return outputs[i]
+      }
+
+      const addConsumingOutput = (tx: Transaction, outputIndex: number, consumes?: Output): Output => {
+        const o: Output = {
+          txid: tx.id('hex'),
+          outputIndex,
+          outputScript: tx.outputs[outputIndex].lockingScript.toBinary(),
+          topic: 'hello',
+          satoshis: tx.outputs[outputIndex].satoshis,
+          beef: tx.toBEEF(),
+          spent: false,
+          outputsConsumed: [],
+          consumedBy: []
+        }
+        if (consumes) {
+          const c = findOutput(consumes.txid, consumes.outputIndex)
+          c.consumedBy.push(o)
+          o.outputsConsumed.push(c)
+        }
+        outputs.push(o)
+        return o
+      }
+
+      mockLookupService.lookup = jest.fn(async () => [{ txid: txid17d182, outputIndex: 0, history: 1 }])
+      const newBEEF: Record<string, string> = {}
+      mockStorageEngine.findOutput = jest.fn(async (txid: string, outputIndex: number, topic?: string, spent?: boolean) => {
+        return findOutput(txid, outputIndex)
+      })
+      mockStorageEngine.findOutputsForTransaction = jest.fn(async (txid: string) => {
+        const os = outputs.filter(o => o.txid === txid)
+        return os
+      })
+      mockStorageEngine.updateOutputBeef = jest.fn(async (txid: string, outputIndex: number, topic: string, beef: number[]) => {
+        newBEEF[`${txid}_${outputIndex}`] = Utils.toHex(beef)
+      })
+      const engine = new Engine({ Hello: mockTopicManager }, { Hello: mockLookupService }, mockStorageEngine, mockChainTracker)
+
+      /*
+        txid 17d1829ba8424b97369ee8b528ee8b65d9d4b9c08d037d224a7be7c025f78f78 output 0 1196 sats
+          txid 9426201003b01e5bb66e9240a1cc337174238e816a25a06ba532fa67af4457d7 output 0 1197 sats
+            txid 509f5ef79c18504fdfe21e67608f56bacb8d8a200634e46f60a9dc8430dcd6e9 output 0 1198 sats
+              txid 877734db8eb917d0e2174a4bdddc085b34f67cedd6b94628c4feb1b979a2b2c5 output 0 1199 sats
+                txid 37abad7168d47d4e107d0f9f96813a4feef6ea346482fce554c3fade85d45409 output 0 1200 sats
+                  txid d786db393ec7f1315bee2cbd3aa47634394f57c861feed8c075563308f14c237 output 0 1568 sats
+                  txid a7d4cac391f5b3d8dedaacb3cd8b5afac54df2d8e043ae0452abcfe190686494 output 17 1132 sats
+      */
+
+      const tx17d182 = Transaction.fromHexBEEF(beef17d182_4)
+      const tx942620 = tx17d182.inputs[0].sourceTransaction
+      const tx509f5e = tx942620.inputs[0].sourceTransaction
+      const tx877734 = tx509f5e.inputs[0].sourceTransaction
+      const tx37abad = tx877734.inputs[0].sourceTransaction
+
+      const output37abad_0 = addConsumingOutput(tx37abad, 0)
+      const output877734_0 = addConsumingOutput(tx877734, 0, output37abad_0)
+      const output509f5e_0 = addConsumingOutput(tx509f5e, 0, output877734_0)
+      const output942620_0 = addConsumingOutput(tx942620, 0, output509f5e_0)
+      const output17d182_0 = addConsumingOutput(tx17d182, 0, output942620_0)
+
+      const mp37abad = Transaction.fromHexBEEF(beef37abad_0).merklePath
+      await engine.handleNewMerkleProof(txid37abad, mp37abad)
+      expect(Object.keys(newBEEF).length).toBe(0)
+
+      const mp877734 = Transaction.fromHexBEEF(beef877734_0).merklePath
+      await engine.handleNewMerkleProof(txid877734, mp877734)
+      expect(newBEEF[`${txid877734}_0`]).toBe(beef877734_0)
+      expect(newBEEF[`${txid509f5e}_0`].length).toBeGreaterThan(beef509f5e_0.length)
+      expect(newBEEF[`${txid942620}_0`].length).toBeGreaterThan(beef942620_0.length)
+      expect(newBEEF[`${txid17d182}_0`].length).toBeGreaterThan(beef17d182_0.length)
+      expect(Object.keys(newBEEF).length).toBe(4)
+
+      const mp509f5e = Transaction.fromHexBEEF(beef509f5e_0).merklePath
+      await engine.handleNewMerkleProof(txid509f5e, mp509f5e)
+      expect(newBEEF[`${txid877734}_0`]).toBe(beef877734_0)
+      expect(newBEEF[`${txid509f5e}_0`]).toBe(beef509f5e_0)
+      expect(newBEEF[`${txid942620}_0`].length).toBeGreaterThan(beef942620_0.length)
+      expect(newBEEF[`${txid17d182}_0`].length).toBeGreaterThan(beef17d182_0.length)
+      expect(Object.keys(newBEEF).length).toBe(4)
+
+      const mp942620 = Transaction.fromHexBEEF(beef942620_0).merklePath
+      await engine.handleNewMerkleProof(txid942620, mp942620)
+      expect(newBEEF[`${txid877734}_0`]).toBe(beef877734_0)
+      expect(newBEEF[`${txid509f5e}_0`]).toBe(beef509f5e_0)
+      expect(newBEEF[`${txid942620}_0`]).toBe(beef942620_0)
+      expect(newBEEF[`${txid17d182}_0`].length).toBeGreaterThan(beef17d182_0.length)
+      expect(Object.keys(newBEEF).length).toBe(4)
+
+      const mp17d182 = Transaction.fromHexBEEF(beef17d182_0).merklePath
+      await engine.handleNewMerkleProof(txid17d182, mp17d182)
+      expect(newBEEF[`${txid877734}_0`]).toBe(beef877734_0)
+      expect(newBEEF[`${txid509f5e}_0`]).toBe(beef509f5e_0)
+      expect(newBEEF[`${txid942620}_0`]).toBe(beef942620_0)
+      expect(newBEEF[`${txid17d182}_0`]).toBe(beef17d182_0)
+      expect(Object.keys(newBEEF).length).toBe(4)
+    })
   })
 
   describe('submit', () => {
@@ -582,7 +685,7 @@ const mockOutput: Output = {
             query: { name: 'Bob' }
           })
           expect(mockStorageEngine.findOutput).toHaveBeenCalledWith(
-            "mockTXID", 0, undefined, false
+            'mockTXID', 0, undefined, false
           )
         })
         it('Calls getUTXOHistory with the correct UTXO and history parameters', async () => {
@@ -648,7 +751,8 @@ const mockOutput: Output = {
           outputs: [{
             beef: mockOutput.beef,
             outputIndex: mockOutput.outputIndex
-          }], type: 'output-list'
+          }],
+          type: 'output-list'
         })
       })
     })
@@ -726,7 +830,7 @@ const mockOutput: Output = {
         )
         expect(results).toEqual({
           outputs: [],
-          type: "output-list"
+          type: 'output-list'
         })
       })
       it('Returns undefined if the history selector is a number, and less than the current depth', async () => {
@@ -754,7 +858,7 @@ const mockOutput: Output = {
         })
         expect(results).toEqual({
           outputs: [],
-          type: "output-list"
+          type: 'output-list'
         })
       })
       it('Returns the current output even if history should be traversed, if the current output is part of a transaction that does not consume any previous topical UTXOs', async () => {
@@ -785,7 +889,8 @@ const mockOutput: Output = {
           outputs: [{
             beef: mockOutput.beef,
             outputIndex: mockOutput.outputIndex
-          }], type: 'output-list'
+          }],
+          type: 'output-list'
         })
       })
       // it('Traversing history, calls findOutput with any output consumed by this UTXO', async () => {
@@ -856,4 +961,25 @@ const beef27c8f_1 = '0100beef01feafcf0c000802020044ef36fc6c79af360b13f2164c6e772
 // beef27c8f_0 is 0 levels deep, i.e. tx has merklePath
 const beef27c8f_0 = '0100beef01fef4f10c000902fd020100ab2d9b3bbfc2ecf5c834f7719bf10dcabef31cfa3b78fa714bd3a2b3958fc6b5fd0301029515629d935d81e704ca97b8dc02a698c39d78f06bbb3d8d46bcaa5178f3c827018000fc94b1da08b5d0850afcc4948a9e129a2c2c0bb629090ca84fef5d326ceadbc2014100fa78356192a22189dd3c2a03bfb2e043667a55a85386a8d0f0853bbaa54ba748012100e37888006acc37cbfc6a459117c3e957b00e8ff3a90d93bc9aaa3fc9d972453701110052564e444cbe4b1f24015d91f5cfd7eff63e5f403613a7b48e77efa32761f950010900acef65445ef232f1fcb527bab67fccc90ae9d0c610a4f752d84c5e10a3415d74010500e1fadd176551150808ae96ff2b4dff487c432788d23db25fd3fb5e8b3e16b96f0103003e530f146a992ed9378807fa43bc1e6033de0711d728ee3a0ea1507b5ffff2940100006c8eca8a9d680ab2ae4dc1f0e247f282de3f32b99175e6f9ddb4f8e76ae0bf1b010100000001792c8c563b17721744d8f101d79e873aae62816d0fd47dd9d90ada219c1b252d070000006b48304502210090368b5925795abb0415a3cd6265d7abae7d3541fb7f2fdb1fdffdd916b3fce3022068136da44df8ee367807c7c59c950ea8d428adfe142268cd6371cf4c1b4c3208412102858ea1804708d1e16e77b49ba1559b3742797f5f43efd3922fa3d21557521d7cffffffff03f401000000000000fde10121026afd108ddbc05e093207ebdf06557a3d66e15b96accbf1766c2e7d6efa3d1981ac22313852713463583334356b6a7461734467764b41455a435135564c5a72516f76786d2081d846bcae65ce281829b6d81eff3862a68b8eaf4e99ad97c0c4ad2e149631c020f936ace2d2b4034710dec819e343f69ddbb1a16d721b88413b697c2475f75f504c4e8470062e6966cb7ec80864e5e1217ecf2b3f113bb5307e1ec1d338678e54b426a11a267e990f7dcd85072c7b3870fabb54812d7f8a7aaa672ce01e4f8a2c6a1d532849877329bd23be0b3383fcb437ffd1ff05aad824343bfe5bae52d5775ed99ba9d5a72038dd50f81153e674a8b1c5d659aac361e660cc5657b291eda149de2505efb0bbac0a353331393633353238334c797f3040f01934bb25733d4700a0309194a7defd8ceea7a8a59ad279c6237e2917264a115773777c5746546288b15ec1cf6788b801dd3952fec804cadbba2ff4e05464735e2db83c3509b59119efc843716c34becd2884f011f2e7f61f382237891875115b6ea5bac1c3e9ca6c575b44547d589c38c1b5a8cadb463044022016e1fdb226b465f3151aaa1e033b122dd58f3b495d2846d532f784ec8f6fc9ae022031dc12155c02e7f588c9f6d975a79c9c26cf341c1c88578cd6a87a3c01f6a73d6d6d6d6dc8000000000000001976a914010b7bfa8ff585b6d95f8a18a998cbd87508bd4188aca9010000000000001976a914254c6f1ce67d27ae4dafb03d2ba08318df2883c588ac000000000100'
 // full txid
-const txid27c8f = '27c8f37851aabc468d3dbb6bf0789dc398a602dcb897ca04e7815d939d621595'
+const txid27c8f = '27c8f37851aabc468d3dbb6bf0789dc398a602dcb897ca04e7815d939d621595'
+
+/*
+  txid 17d1829ba8424b97369ee8b528ee8b65d9d4b9c08d037d224a7be7c025f78f78 output 0 1196 sats
+    txid 9426201003b01e5bb66e9240a1cc337174238e816a25a06ba532fa67af4457d7 output 0 1197 sats
+      txid 509f5ef79c18504fdfe21e67608f56bacb8d8a200634e46f60a9dc8430dcd6e9 output 0 1198 sats
+        txid 877734db8eb917d0e2174a4bdddc085b34f67cedd6b94628c4feb1b979a2b2c5 output 0 1199 sats
+          txid 37abad7168d47d4e107d0f9f96813a4feef6ea346482fce554c3fade85d45409 output 0 1200 sats
+            txid d786db393ec7f1315bee2cbd3aa47634394f57c861feed8c075563308f14c237 output 0 1568 sats
+            txid a7d4cac391f5b3d8dedaacb3cd8b5afac54df2d8e043ae0452abcfe190686494 output 17 1132 sats
+*/
+const txid17d182 = '17d1829ba8424b97369ee8b528ee8b65d9d4b9c08d037d224a7be7c025f78f78'
+const beef17d182_0 = '0100beef01feaff60c000c02fd660302788ff725c0e77b4a227d038dc0b9d4d9658bee28b5e89e36974b42a89b82d117fd670300fe2b9c31ae10a15803f91f63df0cf6a5b6771db9dd831e5bc37a36460a1ffa0701fdb201009bf0e3a5854c4b4c8d62c0ade2754ae1bbc0f16a3e49251ed964486b8dc387f101d80003ae2ec810744139e9989e4f36d8f5ebb160981481e68647890d11e3dda971fa016d0081efcddc14858ead1f427f235d2b21c8de43e33bfeb155ed79f666dd4dd490c2013700a6a847f6e264cfa64c0b6aad6b74912c63c6fc89728be813d104abd2724b98a5011a0062c7d393b9518b05d2d0891ae8c2cd840c93e051d2944bb8e651c976301c4e50010c008d8ee79c1a042cf43f4dcd880aeb4ff7f54552cd3a4c9a40c72cc16ed6847cee01070050194d11d12eddf5e56926b28778f0f64ad54e79eb4df4ce7834ddfc57bf882d010200bd66a0cdbb0af933166931b1050dd108e02d76cff19afd614e681fcbdf1be0740100008b36364159630b47b4c27f6e1e056e820b0aa350ab5dde2520fc1753129b628c010100cab620ac48bb55de40c8c5ddb62bbc44c907fbf5d882326d427f4cd4b34334800101001163b76c2c6824c07a53af2ea86da1b1d946340d9377d3b9b84560e074aea3a6010100000001d75744af67fa32a56ba0256a818e23747133cca140926eb65b1eb00310202694000000006a473044022027137acdeb7583e966a9e8825c9f5edb909b2fd46d40e6ee83846d164529630e0220644e82a46d813fac2836650ce832cdba7eb46b6c134982b36e84f5f08a921dbb412102b963cd9b5a1419c041a4c4ac46f5205a2b8cb691cd5633e5d498ddcf3d37460b0000000001ac040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88ac000000000100'
+const beef17d182_4 = '0100beef01fedcee0c000d02f8020954d485defac354e5fc826434eaf6ee4f3a81969f0f7d104e7dd46871adab37f900fa225908f222f8f7f291731a10e93cdf410189ca01adc688edf5577552aa8ff2017d006cfea8ad6ace7ecce9bbbe729eb9dca25f9be064b150d290588c3364867ee112013f00bfbacc8bfdda7f734c160fcfee972dd9182aede73290367aab62b5ce242e5655011e007a0eb94f2bcf98489247a01551a843d9cf1e091522c32f50ce347ba5057b439d010e00beb857ba543d2bd199e68c63a480c73b83660eca27d2ce105355741f27017876010600d5fb1ea8440e41b4ffde093dd53d6ed741c4fe4fe66ff43f6a4eb9af3c6ddebb010200169db3be130a9e8aff1dbff39abee477d3a224cb3e118c39794bb7ba427b707501000083cd0ce2d2734835c45331cd06f38556cf4e07634d599f59692ad9980175a7f201010022f4a629e99ca5838f56f87b050bdefa9b9f621c08745cb27b941d9c6750d99a01010032b90a4148ff984b0efee309a578cb87d278142c014f1a7fd5351b3e3cd4c7da01010082dc3ea570250357fca88652ee1628928136c29eb4910b7e5627e0ffef1d958a010100f5194f41c28960f0dba669f4715229f59f59afb5d55f50a2a7556111b8da67130101009cd7f9a65014cc6fd5c0b71391d0b70eb983ea3b03a2fbd7a13150775855953405010000000237c2148f306355078cedfe61c8574f393476a43abd2cee5b31f1c73e39db86d7000000006a47304402207ee089c64fa810ad88e03f7bad4ae57c9b7de215d222a5bf52f9259d171b717a0220643d3cb4423b5b7ffcb7b2b59eb81dede0a25319aaa0228d27e59d5e238c35a2412102641b5a033ce332a9fc0f81dee7948c099b194758d8352bd5572988ac760ae2faffffffff94646890e1cfab5204ae43e0d8f24dc5fa5a8bcdb3acdaded8b3f591c3cad4a7110000006a47304402205b92eaba4a6c029385e9cf291631b98d0c7aa6851755fb44279e2b1ebd12759702206e96f3bac252f2c5cf5edf7f8828d6358baaa9fe34aa163faeb9b31aac830e47412102577e2a3877a253d9e7587c3d77a4038b4af5e51113393c2713c6d50c9da5de6effffffff03b0040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88acc8000000000000001976a914292e61672fc24a9a70cfa6f73a9af5655652629d88ac13050000000000001976a914ad2ab55d09559074e41e2a2c1c89742e13bee03088ac00000000010001000000010954d485defac354e5fc826434eaf6ee4f3a81969f0f7d104e7dd46871adab37000000006b4830450221009c67deb38aa6b5ed8bfa140e9464e1b1ee2908a296bb70e3d164725254491c6a02207b595dcced1136c276edfc1e5e5901279cf6f0a6432e4c1a908468262e7d9543412102b963cd9b5a1419c041a4c4ac46f5205a2b8cb691cd5633e5d498ddcf3d37460b0000000001af040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88ac00000000000100000001c5b2a279b9b1fec42846b9d6ed7cf6345b08dcdd4b4a17e2d017b98edb347787000000006b483045022100c0992a684be99a07bc87e03bf88701acefc033c1aa57cf4a3b15253e64c8962a022031d5b3af1684a1a1a201eb16fc1fbca8522ae16b9c0d340e8bec48eeea1fe458412102b963cd9b5a1419c041a4c4ac46f5205a2b8cb691cd5633e5d498ddcf3d37460b0000000001ae040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88ac00000000000100000001e9d6dc3084dca9606fe43406208a8dcbba568f60671ee2df4f50189cf75e9f50000000006b483045022100c9c8baa777fe09c1310788614d4eba88b6027d39eada18a5a79c6a33e55919c302205222961b1f63dddd425cd418571797cb3a68aa59ac37331c0fc61fc9de66c34d412102b963cd9b5a1419c041a4c4ac46f5205a2b8cb691cd5633e5d498ddcf3d37460b0000000001ad040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88ac00000000000100000001d75744af67fa32a56ba0256a818e23747133cca140926eb65b1eb00310202694000000006a473044022027137acdeb7583e966a9e8825c9f5edb909b2fd46d40e6ee83846d164529630e0220644e82a46d813fac2836650ce832cdba7eb46b6c134982b36e84f5f08a921dbb412102b963cd9b5a1419c041a4c4ac46f5205a2b8cb691cd5633e5d498ddcf3d37460b0000000001ac040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88ac0000000000'
+const txid942620 = '9426201003b01e5bb66e9240a1cc337174238e816a25a06ba532fa67af4457d7'
+const beef942620_0 = '0100beef01fe07ef0c0008022802d75744af67fa32a56ba0256a818e23747133cca140926eb65b1eb00310202694290089598873694014f9706b6074a1b349fcf59df875e58a0cf3a8900c6e5830f33601150036fffb4b882171c594a27d37c5d7827cc706811c3e0acd152e0f3ee1c7ac98f9010b002b3b5c611cbea3bbdf9e99646cb23e80b62fc2dc2e619656157296878d46d8b30104002877b6b97c95fd55c0584d0ccf683c9ad6c60be146dd73605b93ef042da648a0010300d1d01ac14f19d974f8131b9cdb38ae2abcf5403d5c3f0d2772a1c995c136356b01000017ece09932fa7c6d464c633df200003c2328beb26290c7b9b1333eb84c90c544010100cd4147f92f226bc39354a78209db1f521ffd8720e3fcc9c942ee1dd535513d0f0101005daeb3f84528a58700c58c8479f29cd627b07c198602b6c08033340262c0c607010100000001e9d6dc3084dca9606fe43406208a8dcbba568f60671ee2df4f50189cf75e9f50000000006b483045022100c9c8baa777fe09c1310788614d4eba88b6027d39eada18a5a79c6a33e55919c302205222961b1f63dddd425cd418571797cb3a68aa59ac37331c0fc61fc9de66c34d412102b963cd9b5a1419c041a4c4ac46f5205a2b8cb691cd5633e5d498ddcf3d37460b0000000001ad040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88ac000000000100'
+const txid509f5e = '509f5ef79c18504fdfe21e67608f56bacb8d8a200634e46f60a9dc8430dcd6e9'
+const beef509f5e_0 = '0100beef01fe05ef0c0009029800fb09f12f5632f7af6409432fc914d1e68fc499270fa29f3b3c43fb00ebeb8ed09902e9d6dc3084dca9606fe43406208a8dcbba568f60671ee2df4f50189cf75e9f50014d00ccb4f6ea2fabdfaf86b0e3837cc702ddde19cb5112e680f00832330b03be1d290127000480365544cd4dca6262cc25567f241d650339773f3f9ef1b9a25b9a034945c30112004ab6bdfecf74af1da1e0574671d06aef63dcde80133d1a6c46beda2ad08abc15010800ce75abf88ee3de3e2f89be0462e1cb0b9470b67947b9880e941bb6565a44bc86010500940d69edc69fc8694485452b961b8df8a2e0c1ff7bdbfeee1e6855542a7e7cd10103002217abcbd7e5431fffc0f52624ee3cf659e39734ae5335d5d196193f7f6e39ab01000082d3a59ca910a1c6fc1f06ab2a5ea80ffee7a39d9a987c85701cfaaf604219f80101008bed85ed818e9d69147000e8687b2efed77e5ee0073e5a2640033715747bd0c2010100000001c5b2a279b9b1fec42846b9d6ed7cf6345b08dcdd4b4a17e2d017b98edb347787000000006b483045022100c0992a684be99a07bc87e03bf88701acefc033c1aa57cf4a3b15253e64c8962a022031d5b3af1684a1a1a201eb16fc1fbca8522ae16b9c0d340e8bec48eeea1fe458412102b963cd9b5a1419c041a4c4ac46f5205a2b8cb691cd5633e5d498ddcf3d37460b0000000001ae040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88ac000000000100'
+const txid877734 = '877734db8eb917d0e2174a4bdddc085b34f67cedd6b94628c4feb1b979a2b2c5'
+const beef877734_0 = '0100beef01fedfee0c000e02fd202900eeebd665d8893249c90dabbd50266fb7e0f125dd8ee087a3c8089debf09499d4fd212902c5b2a279b9b1fec42846b9d6ed7cf6345b08dcdd4b4a17e2d017b98edb34778701fd911400b32e903076f6869d4e0a9fa2d00f5c8bad598d3ede5b4bafef88debdb1d3de6901fd490a000a79573c8eb909d30c4e9653f90ea2869e44220d551da81cc96913b2fcd734aa01fd2505003d6db6b39d889be1535a478da0c3428e8107872ac55f0fcfdd2225b6a402d1dc01fd9302008ae31e7e4d4e766814ea11b337b5b51f5d807737f802c44c8d0d75a583a61bde01fd480100e962a171fcc0e79bf55b1433d79564f8d3b2622ecbcb84c9ede792d353e0afb901a5007d0a8d8a3a2f0eb209e82f5e80ef6c4835b7d07c85d6ac8c613cc8754a47bd79015300b070576d92310f1e08a12581ecea4d1d4de48beb15ff7e42eddc1b957fa2909d012800544a58cac6b4dd78cebd84178dc1572693552fec5777a055dad0068bd67aec6101150025674a887fff026640fe9851cbffb37bd8e02780c4f6c0476ed3cee4e4884c2f010b0023aa2b01e8a59c3b79976918d95ae6a044ace2264e3690f9846f452499c0cbad01040078033c2619a3b115bfc71d09addf13e288660be1841cf6769ffb9924aca246970103004d2d6ae3e2c02703b773c49af789b7372b5b71bf92b49e2dc4b921880df984540100003b86537a85e5047596a482ac2dba30d1798b26d200c08a2fb19d8d75b2c2a1b50101000000010954d485defac354e5fc826434eaf6ee4f3a81969f0f7d104e7dd46871adab37000000006b4830450221009c67deb38aa6b5ed8bfa140e9464e1b1ee2908a296bb70e3d164725254491c6a02207b595dcced1136c276edfc1e5e5901279cf6f0a6432e4c1a908468262e7d9543412102b963cd9b5a1419c041a4c4ac46f5205a2b8cb691cd5633e5d498ddcf3d37460b0000000001af040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88ac000000000100'
+const txid37abad = '37abad7168d47d4e107d0f9f96813a4feef6ea346482fce554c3fade85d45409'
+const beef37abad_0 = '0100beef01fedcee0c000d02f8020954d485defac354e5fc826434eaf6ee4f3a81969f0f7d104e7dd46871adab37f900fa225908f222f8f7f291731a10e93cdf410189ca01adc688edf5577552aa8ff2017d006cfea8ad6ace7ecce9bbbe729eb9dca25f9be064b150d290588c3364867ee112013f00bfbacc8bfdda7f734c160fcfee972dd9182aede73290367aab62b5ce242e5655011e007a0eb94f2bcf98489247a01551a843d9cf1e091522c32f50ce347ba5057b439d010e00beb857ba543d2bd199e68c63a480c73b83660eca27d2ce105355741f27017876010600d5fb1ea8440e41b4ffde093dd53d6ed741c4fe4fe66ff43f6a4eb9af3c6ddebb010200169db3be130a9e8aff1dbff39abee477d3a224cb3e118c39794bb7ba427b707501000083cd0ce2d2734835c45331cd06f38556cf4e07634d599f59692ad9980175a7f201010022f4a629e99ca5838f56f87b050bdefa9b9f621c08745cb27b941d9c6750d99a01010032b90a4148ff984b0efee309a578cb87d278142c014f1a7fd5351b3e3cd4c7da01010082dc3ea570250357fca88652ee1628928136c29eb4910b7e5627e0ffef1d958a010100f5194f41c28960f0dba669f4715229f59f59afb5d55f50a2a7556111b8da67130101009cd7f9a65014cc6fd5c0b71391d0b70eb983ea3b03a2fbd7a13150775855953401010000000237c2148f306355078cedfe61c8574f393476a43abd2cee5b31f1c73e39db86d7000000006a47304402207ee089c64fa810ad88e03f7bad4ae57c9b7de215d222a5bf52f9259d171b717a0220643d3cb4423b5b7ffcb7b2b59eb81dede0a25319aaa0228d27e59d5e238c35a2412102641b5a033ce332a9fc0f81dee7948c099b194758d8352bd5572988ac760ae2faffffffff94646890e1cfab5204ae43e0d8f24dc5fa5a8bcdb3acdaded8b3f591c3cad4a7110000006a47304402205b92eaba4a6c029385e9cf291631b98d0c7aa6851755fb44279e2b1ebd12759702206e96f3bac252f2c5cf5edf7f8828d6358baaa9fe34aa163faeb9b31aac830e47412102577e2a3877a253d9e7587c3d77a4038b4af5e51113393c2713c6d50c9da5de6effffffff03b0040000000000001976a914ae0545a670c6f739f9a8963aee37be95df0d9b2a88acc8000000000000001976a914292e61672fc24a9a70cfa6f73a9af5655652629d88ac13050000000000001976a914ad2ab55d09559074e41e2a2c1c89742e13bee03088ac000000000100'

package/src/__tests/OverlayGASPRemote.test.ts

@@ -0,0 +1,132 @@
+import { OverlayGASPRemote } from '../Engine'
+import { GASPInitialRequest, GASPInitialResponse, GASPNode } from '@bsv/gasp'
+
+global.fetch = jest.fn(async () => {
+  await Promise.resolve({
+    ok: true,
+    status: 200,
+    json: async () => await Promise.resolve({})
+  })
+}
+) as jest.Mock
+
+describe('OverlayGASPRemote', () => {
+  let overlayRemote: OverlayGASPRemote
+
+  beforeEach(() => {
+    overlayRemote = new OverlayGASPRemote('http://example.com');
+    (fetch as jest.Mock).mockClear()
+  })
+
+  describe('getInitialResponse', () => {
+    it('should send a request and return a valid response', async () => {
+      const mockRequest: GASPInitialRequest = { version: 1, since: 0 }
+      const mockResponse: GASPInitialResponse = {
+        UTXOList: [{ txid: 'txid1', outputIndex: 0 }],
+        since: 1234567890
+      };
+
+      (fetch as jest.Mock).mockResolvedValue({
+        ok: true,
+        status: 200,
+        json: jest.fn().mockResolvedValue(mockResponse)
+      })
+
+      const response = await overlayRemote.getInitialResponse(mockRequest)
+
+      expect(fetch).toHaveBeenCalledWith('http://example.com/requestSyncResponse', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(mockRequest)
+      })
+      expect(response).toEqual(mockResponse)
+    })
+
+    it('should throw an error if the response status is not OK', async () => {
+      const mockRequest: GASPInitialRequest = { version: 1, since: 0 };
+
+      (fetch as jest.Mock).mockResolvedValue({
+        ok: false,
+        status: 500
+      })
+
+      await expect(overlayRemote.getInitialResponse(mockRequest)).rejects.toThrow('HTTP error! Status: 500')
+    })
+
+    it('should throw an error if the response format is invalid', async () => {
+      const mockRequest: GASPInitialRequest = { version: 1, since: 0 }
+      const invalidResponse = { invalid: 'data' };
+
+      (fetch as jest.Mock).mockResolvedValue({
+        ok: true,
+        status: 200,
+        json: jest.fn().mockResolvedValue(invalidResponse)
+      })
+
+      await expect(overlayRemote.getInitialResponse(mockRequest)).rejects.toThrow('Invalid response format')
+    })
+  })
+
+  describe('requestNode', () => {
+    it('should send a request and return a valid GASPNode', async () => {
+      const graphID = 'graphID1'
+      const txid = 'txid1'
+      const outputIndex = 0
+      const metadata = true
+      const mockResponse: GASPNode = {
+        graphID,
+        rawTx: 'rawTxData',
+        outputIndex,
+        proof: 'proofData',
+        txMetadata: 'txMetadata',
+        outputMetadata: 'outputMetadata',
+        inputs: {}
+      };
+
+      (fetch as jest.Mock).mockResolvedValue({
+        ok: true,
+        status: 200,
+        json: jest.fn().mockResolvedValue(mockResponse)
+      })
+
+      const response = await overlayRemote.requestNode(graphID, txid, outputIndex, metadata)
+
+      expect(fetch).toHaveBeenCalledWith('http://example.com/requestForeignGASPNode', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ graphID, txid, outputIndex, metadata })
+      })
+      expect(response).toEqual(mockResponse)
+    })
+
+    it('should throw an error if the response status is not OK', async () => {
+      const graphID = 'graphID1'
+      const txid = 'txid1'
+      const outputIndex = 0
+      const metadata = true;
+
+      (fetch as jest.Mock).mockResolvedValue({
+        ok: false,
+        status: 500
+      })
+
+      await expect(overlayRemote.requestNode(graphID, txid, outputIndex, metadata)).rejects.toThrow('HTTP error! Status: 500')
+    })
+
+    it('should throw an error if the response format is invalid', async () => {
+      const graphID = 'graphID1'
+      const txid = 'txid1'
+      const outputIndex = 0
+      const metadata = true
+      const invalidResponse = { invalid: 'data' };
+
+      (fetch as jest.Mock).mockResolvedValue({
+        ok: true,
+        status: 200,
+        json: jest.fn().mockResolvedValue(invalidResponse)
+      })
+
+      await expect(overlayRemote.requestNode(graphID, txid, outputIndex, metadata)).rejects.toThrow('Invalid response format')
+    })
+  })
+})

package/src/__tests/OverlayGASPStorage.test.ts

@@ -0,0 +1,166 @@
+import { GraphNode, OverlayGASPStorage } from '../Engine'
+import { Transaction, MerklePath } from '@bsv/sdk'
+import { GASPNode } from '@bsv/gasp'
+
+describe('OverlayGASPStorage', () => {
+  let overlayStorage: OverlayGASPStorage
+  let mockEngine: any
+
+  beforeEach(() => {
+    mockEngine = { storage: { findUTXOsForTopic: jest.fn() }, managers: {} }
+    overlayStorage = new OverlayGASPStorage('test-topic', mockEngine)
+  })
+
+  describe('appendToGraph', () => {
+    it('should append a new node to an empty graph', async () => {
+      const mockTx = {
+        rawTx: '001122',
+        outputIndex: 0,
+        graphID: 'txid123.0'
+      }
+
+      // Use actual Transaction implementation
+      const parsedTx = Transaction.fromHex(mockTx.rawTx)
+
+      await overlayStorage.appendToGraph(mockTx)
+
+      expect(Object.keys(overlayStorage.temporaryGraphNodeRefs).length).toBe(1)
+      expect(overlayStorage.temporaryGraphNodeRefs['txid123.0'].txid).toBe(parsedTx.id('hex'))
+    })
+
+    it('throws error when max nodes are exceeded', async () => {
+      overlayStorage.maxNodesInGraph = 1
+      overlayStorage.temporaryGraphNodeRefs['txid123.0'] = { txid: 'txid123', children: [] } as GraphNode
+
+      const mockTx = {
+        rawTx: '334455',
+        outputIndex: 1,
+        graphID: 'txid234.1'
+      }
+
+      await expect(overlayStorage.appendToGraph(mockTx)).rejects.toThrow('The max number of nodes in transaction graph has been reached!')
+    })
+  })
+
+  describe('findKnownUTXOs', () => {
+    it('should return known UTXOs since a given timestamp', async () => {
+      const mockUTXOs = [{ txid: 'txid1', outputIndex: 0 }, { txid: 'txid2', outputIndex: 1 }]
+      mockEngine.storage.findUTXOsForTopic.mockResolvedValue(mockUTXOs)
+
+      const result = await overlayStorage.findKnownUTXOs(1234567890)
+
+      expect(result).toEqual(mockUTXOs)
+      expect(mockEngine.storage.findUTXOsForTopic).toHaveBeenCalledWith('test-topic', 1234567890)
+    })
+
+    it('should handle errors correctly', async () => {
+      mockEngine.storage.findUTXOsForTopic.mockRejectedValue(new Error('Database error'))
+
+      await expect(overlayStorage.findKnownUTXOs(1234567890)).rejects.toThrow('Database error')
+    })
+  })
+
+  describe('hydrateGASPNode', () => {
+    it('should throw an error', async () => {
+      await expect(overlayStorage.hydrateGASPNode('graphID', 'txid', 0, false)).rejects.toThrow('GASP node hydration Not supported!')
+    })
+  })
+
+  describe('findNeededInputs', () => {
+    // TODO: Write more complicated test cases
+    it('should return inputs needed for further verification when no proof is present', async () => {
+      const mockTx: GASPNode = {
+        rawTx: '001122',
+        proof: undefined,
+        graphID: 'txid123.0',
+        outputIndex: 0
+      }
+
+      const parsedTx = {
+        inputs: [{ sourceTXID: 'inputTxid1', sourceOutputIndex: 0 }],
+        toBEEF: jest.fn(),
+        id: jest.fn().mockReturnValue('txid123')
+      }
+      Transaction.fromHex = jest.fn().mockReturnValue(parsedTx)
+
+      const result = await overlayStorage.findNeededInputs(mockTx)
+
+      expect(result).toEqual({
+        requestedInputs: { 'inputTxid1.0': { metadata: false } }
+      })
+    })
+
+    it('should return inputs needed for further verification when proof is present', async () => {
+      const mockTx: GASPNode = {
+        rawTx: '001122',
+        proof: 'someproof',
+        graphID: 'txid123.0',
+        outputIndex: 0
+      }
+
+      const parsedTx = {
+        inputs: [{ sourceTXID: 'neededTxid', sourceOutputIndex: 1 }],
+        toBEEF: jest.fn(),
+        id: jest.fn().mockReturnValue('txid123'),
+        merklePath: {}
+      }
+      Transaction.fromHex = jest.fn().mockReturnValue(parsedTx)
+      MerklePath.fromHex = jest.fn().mockReturnValue(parsedTx.merklePath)
+
+      mockEngine.managers['test-topic'] = {
+        identifyAdmissibleOutputs: jest.fn().mockResolvedValue({
+          outputsToAdmit: []
+        }),
+        identifyNeededInputs: jest.fn().mockResolvedValue([{ txid: 'neededTxid', outputIndex: 1 }])
+      }
+
+      const result = await overlayStorage.findNeededInputs(mockTx)
+
+      expect(result).toEqual({
+        requestedInputs: { 'neededTxid.1': { metadata: false } }
+      })
+    })
+  })
+
+  describe('discardGraph', () => {
+    it('should discard the graph and its nodes', async () => {
+      overlayStorage.temporaryGraphNodeRefs['txid123.0'] = {
+        txid: 'txid123',
+        time: Date.now(),
+        graphID: 'txid123.0',
+        rawTx: 'rawTxData',
+        outputIndex: 0,
+        children: [],
+        parent: undefined
+      } as GraphNode
+
+      overlayStorage.temporaryGraphNodeRefs['txid124.0'] = {
+        txid: 'txid124',
+        time: Date.now(),
+        graphID: 'txid123.0',
+        rawTx: 'rawTxData',
+        outputIndex: 1,
+        children: [],
+        parent: {
+          txid: 'txid123',
+          time: Date.now(),
+          graphID: 'txid123.0',
+          rawTx: 'rawTxData',
+          outputIndex: 0,
+          children: []
+        } as GraphNode
+      } as GraphNode
+
+      await overlayStorage.discardGraph('txid123.0')
+
+      expect(overlayStorage.temporaryGraphNodeRefs['txid123.0']).toBeUndefined()
+      expect(overlayStorage.temporaryGraphNodeRefs['txid124.0']).toBeUndefined()
+    })
+  })
+
+  it('should handle non-existent graphID', async () => {
+    await overlayStorage.discardGraph('nonexistent.0')
+
+    expect(Object.keys(overlayStorage.temporaryGraphNodeRefs).length).toBe(0)
+  })
+})

package/src/storage/Storage.ts

@@ -35,6 +35,14 @@ export interface Storage {
    */
   findOutputsForTransaction: (txid: string) => Promise<Output[]>
 
+  /**
+   * Finds current UTXOs that have been admitted into a given topic
+   * @param topic - The topic for which we want to find Unspent Transaction Outputs (UTXOs).
+   * @param since - Optional parameter indicating the minimum date (timestamp? block height?) to retrieve matching UTXOs from. TODO: We need to decide about which time format to use for synchronization.
+   * @returns A promise that resolves to an array of matching UTXOs.
+   */
+  findUTXOsForTopic: (topic: string, since?: number) => Promise<Output[]>
+
   /**
    * Deletes an output from storage
    * @param txid — The TXID of the output to delete

package/src/storage/knex/KnexStorage.ts

@@ -56,6 +56,32 @@ export class KnexStorage implements Storage {
     }))
   }
 
+  async findUTXOsForTopic(topic: string, since?: number): Promise<Output[]> {
+    const query = this.knex('outputs').where({ topic, spent: false })
+
+    // TODO: Create new migration for outputs that adds either created_at or block height tagging for proven outpoints if necessary.
+    // if (since !== undefined) {
+    //   const sinceDate = new Date(since).toISOString()
+    //   await query.andWhere('created_at', '>=', sinceDate)
+    // }
+
+    const outputs = await query.select(
+      'txid', 'outputIndex', 'outputScript', 'topic', 'satoshis', 'beef', 'outputsConsumed', 'spent', 'consumedBy'
+    )
+
+    if (outputs === undefined || outputs.length === 0) {
+      return []
+    }
+
+    return outputs.map(output => ({
+      ...output,
+      outputScript: [...output.outputScript],
+      beef: [...output.beef],
+      spent: Boolean(output.spent),
+      outputsConsumed: JSON.parse(output.outputsConsumed),
+      consumedBy: JSON.parse(output.consumedBy)
+    }))
+  }
 
   async deleteOutput(txid: string, outputIndex: number, topic: string): Promise<void> {
     await this.knex('outputs').where({