@bsv/overlay 0.1.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/LICENSE.txt +28 -0
  2. package/README.md +220 -0
  3. package/dist/cjs/mod.js +43 -0
  4. package/dist/cjs/mod.js.map +1 -0
  5. package/dist/cjs/package.json +53 -0
  6. package/dist/cjs/src/AdmittanceInstructions.js +3 -0
  7. package/dist/cjs/src/AdmittanceInstructions.js.map +1 -0
  8. package/dist/cjs/src/Engine.js +361 -0
  9. package/dist/cjs/src/Engine.js.map +1 -0
  10. package/dist/cjs/src/LookupAnswer.js +3 -0
  11. package/dist/cjs/src/LookupAnswer.js.map +1 -0
  12. package/dist/cjs/src/LookupFormula.js +3 -0
  13. package/dist/cjs/src/LookupFormula.js.map +1 -0
  14. package/dist/cjs/src/LookupQuestion.js +3 -0
  15. package/dist/cjs/src/LookupQuestion.js.map +1 -0
  16. package/dist/cjs/src/LookupService.js +3 -0
  17. package/dist/cjs/src/LookupService.js.map +1 -0
  18. package/dist/cjs/src/Output.js +3 -0
  19. package/dist/cjs/src/Output.js.map +1 -0
  20. package/dist/cjs/src/STEAK.js +3 -0
  21. package/dist/cjs/src/STEAK.js.map +1 -0
  22. package/dist/cjs/src/TaggedBEEF.js +3 -0
  23. package/dist/cjs/src/TaggedBEEF.js.map +1 -0
  24. package/dist/cjs/src/TopicManager.js +3 -0
  25. package/dist/cjs/src/TopicManager.js.map +1 -0
  26. package/dist/cjs/src/storage/Storage.js +3 -0
  27. package/dist/cjs/src/storage/Storage.js.map +1 -0
  28. package/dist/cjs/src/storage/knex/KnexStorage.js +76 -0
  29. package/dist/cjs/src/storage/knex/KnexStorage.js.map +1 -0
  30. package/dist/cjs/src/storage/knex/all-migrations.js +11 -0
  31. package/dist/cjs/src/storage/knex/all-migrations.js.map +1 -0
  32. package/dist/cjs/src/storage/knex/migrations/2024-05-18-001-initial.js +33 -0
  33. package/dist/cjs/src/storage/knex/migrations/2024-05-18-001-initial.js.map +1 -0
  34. package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -0
  35. package/dist/esm/mod.js +17 -0
  36. package/dist/esm/mod.js.map +1 -0
  37. package/dist/esm/src/AdmittanceInstructions.js +2 -0
  38. package/dist/esm/src/AdmittanceInstructions.js.map +1 -0
  39. package/dist/esm/src/Engine.js +357 -0
  40. package/dist/esm/src/Engine.js.map +1 -0
  41. package/dist/esm/src/LookupAnswer.js +2 -0
  42. package/dist/esm/src/LookupAnswer.js.map +1 -0
  43. package/dist/esm/src/LookupFormula.js +2 -0
  44. package/dist/esm/src/LookupFormula.js.map +1 -0
  45. package/dist/esm/src/LookupQuestion.js +2 -0
  46. package/dist/esm/src/LookupQuestion.js.map +1 -0
  47. package/dist/esm/src/LookupService.js +2 -0
  48. package/dist/esm/src/LookupService.js.map +1 -0
  49. package/dist/esm/src/Output.js +2 -0
  50. package/dist/esm/src/Output.js.map +1 -0
  51. package/dist/esm/src/STEAK.js +2 -0
  52. package/dist/esm/src/STEAK.js.map +1 -0
  53. package/dist/esm/src/TaggedBEEF.js +2 -0
  54. package/dist/esm/src/TaggedBEEF.js.map +1 -0
  55. package/dist/esm/src/TopicManager.js +2 -0
  56. package/dist/esm/src/TopicManager.js.map +1 -0
  57. package/dist/esm/src/storage/Storage.js +2 -0
  58. package/dist/esm/src/storage/Storage.js.map +1 -0
  59. package/dist/esm/src/storage/knex/KnexStorage.js +74 -0
  60. package/dist/esm/src/storage/knex/KnexStorage.js.map +1 -0
  61. package/dist/esm/src/storage/knex/all-migrations.js +9 -0
  62. package/dist/esm/src/storage/knex/all-migrations.js.map +1 -0
  63. package/dist/esm/src/storage/knex/migrations/2024-05-18-001-initial.js +28 -0
  64. package/dist/esm/src/storage/knex/migrations/2024-05-18-001-initial.js.map +1 -0
  65. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -0
  66. package/dist/types/mod.d.ts +14 -0
  67. package/dist/types/mod.d.ts.map +1 -0
  68. package/dist/types/src/AdmittanceInstructions.d.ts +14 -0
  69. package/dist/types/src/AdmittanceInstructions.d.ts.map +1 -0
  70. package/dist/types/src/Engine.d.ts +92 -0
  71. package/dist/types/src/Engine.d.ts.map +1 -0
  72. package/dist/types/src/LookupAnswer.d.ts +15 -0
  73. package/dist/types/src/LookupAnswer.d.ts.map +1 -0
  74. package/dist/types/src/LookupFormula.d.ts +25 -0
  75. package/dist/types/src/LookupFormula.d.ts.map +1 -0
  76. package/dist/types/src/LookupQuestion.d.ts +15 -0
  77. package/dist/types/src/LookupQuestion.d.ts.map +1 -0
  78. package/dist/types/src/LookupService.d.ts +52 -0
  79. package/dist/types/src/LookupService.d.ts.map +1 -0
  80. package/dist/types/src/Output.d.ts +30 -0
  81. package/dist/types/src/Output.d.ts.map +1 -0
  82. package/dist/types/src/STEAK.d.ts +10 -0
  83. package/dist/types/src/STEAK.d.ts.map +1 -0
  84. package/dist/types/src/TaggedBEEF.d.ts +11 -0
  85. package/dist/types/src/TaggedBEEF.d.ts.map +1 -0
  86. package/dist/types/src/TopicManager.d.ts +27 -0
  87. package/dist/types/src/TopicManager.d.ts.map +1 -0
  88. package/dist/types/src/storage/Storage.d.ts +66 -0
  89. package/dist/types/src/storage/Storage.d.ts.map +1 -0
  90. package/dist/types/src/storage/knex/KnexStorage.d.ts +24 -0
  91. package/dist/types/src/storage/knex/KnexStorage.d.ts.map +1 -0
  92. package/dist/types/src/storage/knex/all-migrations.d.ts +10 -0
  93. package/dist/types/src/storage/knex/all-migrations.d.ts.map +1 -0
  94. package/dist/types/src/storage/knex/migrations/2024-05-18-001-initial.d.ts +4 -0
  95. package/dist/types/src/storage/knex/migrations/2024-05-18-001-initial.d.ts.map +1 -0
  96. package/dist/types/tsconfig.types.tsbuildinfo +1 -0
  97. package/docs/API.md +621 -0
  98. package/docs/README.md +8 -0
  99. package/docs/concepts/README.md +4 -0
  100. package/docs/examples/README.md +5 -0
  101. package/docs/examples/gs-wip.md +105 -0
  102. package/docs/internal/README.md +4 -0
  103. package/mod.ts +18 -0
  104. package/package.json +78 -0
  105. package/src/AdmittanceInstructions.ts +14 -0
  106. package/src/Engine.ts +419 -0
  107. package/src/LookupAnswer.ts +14 -0
  108. package/src/LookupFormula.ts +26 -0
  109. package/src/LookupQuestion.ts +15 -0
  110. package/src/LookupService.ts +52 -0
  111. package/src/Output.ts +29 -0
  112. package/src/STEAK.ts +10 -0
  113. package/src/TaggedBEEF.ts +10 -0
  114. package/src/TopicManager.ts +23 -0
  115. package/src/__tests/Engine.test.ts +792 -0
  116. package/src/storage/Storage.ts +72 -0
  117. package/src/storage/knex/KnexStorage.ts +90 -0
  118. package/src/storage/knex/all-migrations.ts +14 -0
  119. package/src/storage/knex/migrations/2024-05-18-001-initial.ts +30 -0
@@ -0,0 +1,105 @@
1
+ ### Introduction to BSV Overlay Services Engine
2
+
3
+ The BSV Overlay Services Engine is designed to process transactions and manage data within a blockchain-based system, specifically targeting the Bitcoin SV (BSV) blockchain. It integrates various components such as Topic Managers, Lookup Services, Storage, and Chain Tracker to provide a robust environment for managing transaction data and overlay services.
4
+
5
+ ### Components of the System
6
+
7
+ 1. **Topic Managers**: Responsible for managing the admittance of transactions related to specific topics.
8
+ 2. **Lookup Services**: Handle the lookup of UTXO (Unspent Transaction Output) data for transactions.
9
+ 3. **Storage**: Manages persistent data storage, tracking UTXOs and their states within the system.
10
+ 4. **Chain Tracker**: Verifies SPV (Simplified Payment Verification) data associated with transactions to ensure their validity.
11
+
12
+ ### Setting Up the Engine
13
+
14
+ Before you can use the engine, you must initialize it with the required components:
15
+
16
+ ```ts
17
+ import { Engine, KnexStorage } from "@bsv/overlay";
18
+ import { HelloTopicManager, HelloLookupService } from 'hello-services';
19
+ import { WoChain } from "@bsv/sdk";
20
+
21
+ // Initialize components
22
+ const managers = {
23
+ "exampleTopic": new HelloTopicManager()
24
+ };
25
+
26
+ const lookupServices = {
27
+ "exampleLookup": new HelloLookupService()
28
+ };
29
+
30
+ const storage = new KnexStorage();
31
+ const chainTracker = new WoChain();
32
+
33
+ // Create the engine instance
34
+ const engine = new Engine(managers, lookupServices, storage, chainTracker);
35
+ ```
36
+
37
+ ### Submitting a Transaction
38
+
39
+ To submit a transaction for processing by the Overlay Services:
40
+
41
+ ```ts
42
+ import { Transaction } from '@bsv/sdk'
43
+
44
+ const tx = new Transaction(/* ... */);
45
+
46
+ const transaction = {
47
+ beef: tx.toBEEF(),
48
+ topics: ['exampleTopic']
49
+ }
50
+
51
+ // Submit transaction
52
+ engine.submit(transaction).then(steak => {
53
+ console.log("Transaction processed:", steak);
54
+ }).catch(error => {
55
+ console.error("Error processing transaction:", error);
56
+ });
57
+ ```
58
+
59
+ ### Lookup Queries
60
+
61
+ To perform a lookup query using the engine:
62
+
63
+ ```ts
64
+ const question = {
65
+ service: 'exampleLookup',
66
+ query: {
67
+ name: 'Bob'
68
+ }
69
+ }
70
+
71
+ // Perform a lookup
72
+ engine.lookup(question).then(answer => {
73
+ console.log("Lookup result:", answer);
74
+ }).catch(error => {
75
+ console.error("Error performing lookup:", error);
76
+ });
77
+ ```
78
+
79
+ ### Managing UTXOs
80
+
81
+ The system's core functionality involves managing UTXOs:
82
+
83
+ 1. **Inserting a New UTXO**: Store new UTXO data when transactions are processed.
84
+ 2. **Deleting a UTXO**: Remove UTXOs that are no longer needed or have been consumed by newer transactions.
85
+ 3. **Tracking UTXO Consumption**: Monitor which transactions consume which UTXOs.
86
+
87
+ ### Retrieving Documentation
88
+
89
+ To retrieve documentation for specific managers or services:
90
+
91
+ ```ts
92
+ // For a topic manager
93
+ engine.getDocumentationForTopicManger("exampleTopic").then(doc => {
94
+ console.log("Documentation for Topic Manager:", doc);
95
+ });
96
+
97
+ // For a lookup service
98
+ engine.getDocumentationForLookupServiceProvider("exampleLookup").then(doc => {
99
+ console.log("Documentation for Lookup Service:", doc);
100
+ });
101
+ ```
102
+
103
+ ### Conclusion
104
+
105
+ The BSV Overlay Services Engine provides a powerful toolset for managing transactions and data on the Bitcoin SV blockchain. It's designed to handle complex data structures and ensure the integrity and security of transactions through rigorous validation and management processes. By following this tutorial, developers can effectively integrate and utilize these capabilities within their blockchain applications.
@@ -0,0 +1,4 @@
1
+ # Internals
2
+
3
+ These documents cover the internal components of the Overlay Services engine. Generally, these are only useful for creating customized deployments:
4
+
package/mod.ts ADDED
@@ -0,0 +1,18 @@
1
+ // Fundamentals
2
+ export * as Engine from "./src/Engine.js"
3
+ export * as LookupService from "./src/LookupService.js"
4
+ export * as TopicManager from "./src/TopicManager.js"
5
+
6
+ // Interfaces and structures
7
+ export * as Storage from "./src/storage/Storage.js"
8
+ export * as Output from './src/Output.js'
9
+ export * as AdmittanceInstructions from './src/AdmittanceInstructions.js'
10
+ export * as TaggedBEEF from './src/TaggedBEEF.js'
11
+ export * as STEAK from './src/STEAK.js'
12
+ export * as LookupQuestion from './src/LookupQuestion.js'
13
+ export * as LookupFormula from './src/LookupFormula.js'
14
+ export * as LookupAnswer from './src/LookupAnswer.js'
15
+
16
+ // The Knex storage system
17
+ export * as KnexStorage from './src/storage/knex/KnexStorage.js'
18
+ export * as KnexStorageMigrations from './src/storage/knex/all-migrations.js'
package/package.json ADDED
@@ -0,0 +1,78 @@
1
+ {
2
+ "name": "@bsv/overlay",
3
+ "version": "0.1.0-alpha.1",
4
+ "type": "module",
5
+ "description": "BSV Blockchain Overlay Services Engine",
6
+ "main": "dist/cjs/mod.js",
7
+ "module": "dist/esm/mod.js",
8
+ "types": "dist/types/mod.d.ts",
9
+ "files": [
10
+ "dist",
11
+ "src",
12
+ "docs",
13
+ "mod.ts",
14
+ "LICENSE.txt"
15
+ ],
16
+ "exports": {
17
+ ".": {
18
+ "types": "./dist/types/mod.d.ts",
19
+ "import": "./dist/esm/mod.js",
20
+ "require": "./dist/cjs/mod.js"
21
+ },
22
+ "./*.ts": {
23
+ "types": "./dist/types/src/*.d.ts",
24
+ "import": "./dist/esm/src/*.js",
25
+ "require": "./dist/cjs/src/*.js"
26
+ },
27
+ "./storage": {
28
+ "import": "./dist/esm/src/storage/index.js",
29
+ "require": "./dist/cjs/src/storage/index.js",
30
+ "types": "./dist/types/src/storage/index.d.ts"
31
+ },
32
+ "./storage/*": {
33
+ "import": "./dist/esm/src/storage/*.js",
34
+ "require": "./dist/cjs/src/storage/*.js",
35
+ "types": "./dist/types/src/storage/*.d.ts"
36
+ }
37
+ },
38
+ "scripts": {
39
+ "test": "npm run build && jest",
40
+ "test:watch": "npm run build && jest --watch",
41
+ "test:coverage": "npm run build && jest --coverage",
42
+ "lint": "ts-standard --fix src/**/*.ts",
43
+ "build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json",
44
+ "dev": "tsc -b -w",
45
+ "prepublish": "npm run build",
46
+ "doc": "ts2md --inputFilename=mod.ts --outputFilename=docs/API.md --filenameSubstring=API --firstHeadingLevel=1"
47
+ },
48
+ "repository": {
49
+ "type": "git",
50
+ "url": "git+https://github.com/bitcoin-sv/overlay-services.git"
51
+ },
52
+ "keywords": [
53
+ "BSV",
54
+ "Blockchain",
55
+ "Overlay",
56
+ "Bitcoin",
57
+ "SV"
58
+ ],
59
+ "author": "BSV Association",
60
+ "license": "SEE LICENSE IN LICENSE.txt",
61
+ "bugs": {
62
+ "url": "https://github.com/bitcoin-sv/overlay-services/issues"
63
+ },
64
+ "homepage": "https://github.com/bitcoin-sv/overlay-services#readme",
65
+ "devDependencies": {
66
+ "@types/jest": "^29.5.12",
67
+ "jest": "^29.7.0",
68
+ "ts-jest": "^29.1.1",
69
+ "ts-standard": "^12.0.2",
70
+ "ts2md": "^0.2.0",
71
+ "tsconfig-to-dual-package": "^1.2.0",
72
+ "typescript": "^5.2.2"
73
+ },
74
+ "dependencies": {
75
+ "@bsv/sdk": "^1.0.20",
76
+ "knex": "^3.1.0"
77
+ }
78
+ }
@@ -0,0 +1,14 @@
1
+ /**
2
+ * Instructs the Overlay Services Engine about which outputs to admit and which previous outputs to retain. Returned by a Topic Manager.
3
+ */
4
+ export type AdmittanceInstructions = {
5
+ /**
6
+ * The indicies of all admissable outputs into the managed topic from the provided transaction.
7
+ */
8
+ outputsToAdmit: number[]
9
+
10
+ /**
11
+ * The indicies of all inputs from the provided transaction which spend previously-admitted outputs that should be retained for historical record-keeping.
12
+ */
13
+ coinsToRetain: number[]
14
+ }
package/src/Engine.ts ADDED
@@ -0,0 +1,419 @@
1
+ import TopicManager from "./TopicManager.js"
2
+ import LookupService from "./LookupService.js"
3
+ import Storage from './storage/Storage.js'
4
+ import type { AdmittanceInstructions } from "./AdmittanceInstructions.js"
5
+ import type { Output } from './Output.js'
6
+ import { TaggedBEEF } from "./TaggedBEEF.js"
7
+ import { STEAK } from './STEAK.js'
8
+ import { LookupQuestion } from "./LookupQuestion.js"
9
+ import { LookupAnswer } from "./LookupAnswer.js"
10
+ import { LookupFormula } from "./LookupFormula.js"
11
+ import { Transaction, ChainTracker } from '@bsv/sdk'
12
+
13
+ /**
14
+ * Am engine for running BSV Overlay Services (topic managers and lookup services).
15
+ */
16
+ export default class Engine {
17
+ /**
18
+ * Creates a new Overlay Services Engine
19
+ * @param {[key: string]: TopicManager} managers - manages topic admittance
20
+ * @param {[key: string]: LookupService} lookupServices - manages UTXO lookups
21
+ * @param {Storage} storage - for interacting with internally-managed persistent data
22
+ * @param {ChainTracker} chainTracker - Verifies SPV data associated with transactions
23
+ * @param {ProofNotifier[]} [proofNotifiers] - proof notifier services coming soon!
24
+ */
25
+ constructor(
26
+ public managers: { [key: string]: TopicManager },
27
+ public lookupServices: { [key: string]: LookupService },
28
+ public storage: Storage,
29
+ public chainTracker: ChainTracker
30
+ // public proofNotifiers: ProofNotifier[]
31
+ ) { }
32
+
33
+ /**
34
+ * Submits a transaction for processing by Overlay Services.
35
+ * @param taggedBEEF — The transaction to process
36
+ * @returns The submitted transaction execution acknowledgement
37
+ */
38
+ async submit(taggedBEEF: TaggedBEEF): Promise<STEAK> {
39
+ for (const t of taggedBEEF.topics) {
40
+ if (!this.managers[t]) throw new Error(`This server does not support this topic: ${t}`)
41
+ }
42
+ // Validate the transaction SPV information
43
+ const tx = Transaction.fromBEEF(taggedBEEF.beef)
44
+ const txid = tx.id('hex') as string
45
+ const txValid = await tx.verify(this.chainTracker)
46
+ if (!txValid) throw new Error('Unable to verify SPV information.')
47
+
48
+ // Find UTXOs belonging to a particular topic
49
+ const steak: STEAK = {}
50
+ for (const topic of taggedBEEF.topics) {
51
+ // Ensure transaction is not already applied to the topic
52
+ const dupeCheck = await this.storage.doesAppliedTransactionExist({
53
+ txid,
54
+ topic
55
+ })
56
+ if (dupeCheck) {
57
+ // The transaction was already processed.
58
+ // Currently, NO OUTPUTS ARE ADMITTED FOR DUPLICATE TRANSACTIONS.
59
+ // An alternative decision, one that was decided against, would be to act as if the operation was successful: looking up and returning the list of admitted outputs from when the transaction was originally processed.
60
+ // This was decided against, because we don't want to encourage unnecessary flooding of duplicative transactions to overlay services.
61
+ steak[topic] = {
62
+ outputsToAdmit: [],
63
+ coinsToRetain: []
64
+ }
65
+ continue
66
+ }
67
+
68
+ // Check if any input of this transaction is a previous UTXO, adding previous UTXOs to the list
69
+ const previousCoins: number[] = []
70
+ for (const [i, input] of tx.inputs.entries()) {
71
+ const previousTXID = input.sourceTXID || input.sourceTransaction?.id('hex') as string
72
+ // Check if a previous UTXO exists in the storage medium
73
+ const output = await this.storage.findOutput(
74
+ previousTXID,
75
+ input.sourceOutputIndex,
76
+ topic
77
+ )
78
+ if (output) {
79
+ previousCoins.push(i)
80
+
81
+ // This output is now spent.
82
+ await this.storage.markUTXOAsSpent(
83
+ output.txid,
84
+ output.outputIndex,
85
+ topic
86
+ )
87
+
88
+ // Notify the lookup services about the spending of this output
89
+ for (const l of Object.values(this.lookupServices)) {
90
+ try {
91
+ if (l.outputSpent) {
92
+ await l.outputSpent(
93
+ output.txid,
94
+ output.outputIndex,
95
+ topic
96
+ )
97
+ }
98
+ } catch (_) { }
99
+ }
100
+ }
101
+ }
102
+
103
+ // Use the manager to determine which outputs are admissable
104
+ let admissableOutputs: AdmittanceInstructions
105
+ try {
106
+ admissableOutputs = await this.managers[topic].identifyAdmissibleOutputs(taggedBEEF.beef, previousCoins)
107
+ } catch (_) {
108
+ // If the topic manager throws an error, other topics may still succeed, so we continue to the next one.
109
+ // No outputs were admitted to this topic in this case. Note, however, that the transaction is still valid according to Bitcoin, so it may have spent some previous overlay members. This is unavoidable and good.
110
+ steak[topic] = {
111
+ outputsToAdmit: [],
112
+ coinsToRetain: []
113
+ }
114
+ continue
115
+ }
116
+
117
+ // Keep track of which outputs to admit, mark as stale, or retain
118
+ let outputsToAdmit: number[] = admissableOutputs.outputsToAdmit
119
+ let staleCoins: {
120
+ txid: string
121
+ outputIndex: number
122
+ }[] = []
123
+ let outputsConsumed: {
124
+ txid: string
125
+ outputIndex: number
126
+ }[] = []
127
+
128
+ // Find which outputs should not be retained and mark them as stale
129
+ // For each of the previous UTXOs, if the the UTXO was not included in the list of UTXOs identified for retention, then it will be marked as stale.
130
+ for (const inputIndex of previousCoins) {
131
+ const previousTXID = tx.inputs[inputIndex].sourceTXID || tx.inputs[inputIndex].sourceTransaction?.id('hex') as string
132
+ const previousOutputIndex = tx.inputs[inputIndex].sourceOutputIndex
133
+ if (!admissableOutputs.coinsToRetain.includes(inputIndex)) {
134
+ staleCoins.push({
135
+ txid: previousTXID,
136
+ outputIndex: previousOutputIndex
137
+ })
138
+ } else {
139
+ outputsConsumed.push({
140
+ txid: previousTXID,
141
+ outputIndex: previousOutputIndex
142
+ })
143
+ }
144
+ }
145
+
146
+ // Remove stale outputs recursively
147
+ for (const coin of staleCoins) {
148
+ const output = await this.storage.findOutput(coin.txid, coin.outputIndex, topic)
149
+ if (output) {
150
+ await this.deleteUTXODeep(output)
151
+ }
152
+ }
153
+
154
+ // Handle admittance and notification of incoming UTXOs
155
+ const newUTXOs: { txid: string, outputIndex: number }[] = []
156
+ for (const outputIndex of outputsToAdmit) {
157
+ // Store the output
158
+ await this.storage.insertOutput({
159
+ txid,
160
+ outputIndex,
161
+ outputScript: tx.outputs[outputIndex].lockingScript.toBinary(),
162
+ satoshis: tx.outputs[outputIndex].satoshis as number,
163
+ topic,
164
+ spent: false,
165
+ beef: taggedBEEF.beef,
166
+ consumedBy: [],
167
+ outputsConsumed
168
+ })
169
+ newUTXOs.push({
170
+ txid,
171
+ outputIndex
172
+ })
173
+
174
+ // Notify all the lookup services about the new UTXO
175
+ for (const l of Object.values(this.lookupServices)) {
176
+ try {
177
+ if (l.outputAdded) {
178
+ await l.outputAdded(txid, outputIndex, tx.outputs[outputIndex].lockingScript, topic)
179
+ }
180
+ } catch (_) { }
181
+ }
182
+ }
183
+
184
+ // Update each output consumed to know who consumed it
185
+ for (const output of outputsConsumed) {
186
+ const outputToUpdate = await this.storage.findOutput(output.txid, output.outputIndex, topic)
187
+ if (outputToUpdate) {
188
+ const newConsumedBy = [...new Set([...newUTXOs, ...outputToUpdate.consumedBy])]
189
+ // Note: only update if newConsumedBy !== new Set(JSON.parse(outputToUpdate.consumedBy)) ?
190
+ await this.storage.updateConsumedBy(output.txid, output.outputIndex, topic, newConsumedBy)
191
+ }
192
+ }
193
+
194
+ // Insert the applied transaction to prevent duplicate processing
195
+ await this.storage.insertAppliedTransaction({
196
+ txid,
197
+ topic
198
+ })
199
+
200
+ // Keep track of what outputs were admitted for what topic
201
+ steak[topic] = admissableOutputs
202
+ }
203
+
204
+ return steak
205
+ // TODO subscribe to get notified by proof notifiers when proof is found for TX if not already present, so the tree can be chopped down
206
+ // TODO propagate transaction to other nodes according to synchronization agreements
207
+ }
208
+
209
+ /**
210
+ * Submit a lookup question to the Overlay Services Engine, and receive bakc a Lookup Answer
211
+ * @param LookupQuestion — The question to ask the Overlay Services Engine
212
+ * @returns The answer to the question
213
+ */
214
+ async lookup(lookupQuestion: LookupQuestion): Promise<LookupAnswer> {
215
+ // Validate a lookup service for the provider is found
216
+ const lookupService = this.lookupServices[lookupQuestion.service]
217
+ if (!lookupService) throw new Error(`Lookup service not found for provider: ${lookupQuestion.service}`)
218
+
219
+ let lookupResult = await lookupService.lookup(lookupQuestion)
220
+ // Handle custom lookup service answers
221
+ if ((lookupResult as LookupAnswer).type === 'freeform' || (lookupResult as LookupAnswer).type === 'output-list') {
222
+ return lookupResult as LookupAnswer
223
+ }
224
+ lookupResult = lookupResult as LookupFormula
225
+
226
+ const hydratedOutputs: { beef: number[], outputIndex: number }[] = []
227
+
228
+ for (const { txid, outputIndex, history } of lookupResult) {
229
+ // Make sure this is an unspent output (UTXO)
230
+ const UTXO = await this.storage.findOutput(
231
+ txid,
232
+ outputIndex,
233
+ undefined,
234
+ false
235
+ )
236
+ if (!UTXO) continue
237
+
238
+ // Get the history for this utxo and construct a BRC-8 Envelope
239
+ const output = await this.getUTXOHistory(UTXO, history, 0)
240
+ if (output) {
241
+ hydratedOutputs.push({
242
+ beef: output.beef,
243
+ outputIndex: output.outputIndex
244
+ })
245
+ }
246
+ }
247
+ return {
248
+ type: 'output-list',
249
+ outputs: hydratedOutputs
250
+ }
251
+ }
252
+
253
+ /**
254
+ * Traverse and return the history of a UTXO
255
+ * @param historySelector
256
+ * @param currentDepth
257
+ * @param output
258
+ * @param txid
259
+ * @param outputIndex
260
+ * @param id
261
+ * @returns {Promise<EnvelopeEvidenceApi>}
262
+ */
263
+ async getUTXOHistory(
264
+ output: Output,
265
+ historySelector?: ((beef: number[], outputIndex: number, currentDepth: number) => Promise<boolean>) | number, currentDepth = 0,
266
+ ): Promise<Output | undefined> {
267
+ // If we have an output but no history selector, jsut return the output.
268
+ if (typeof historySelector === 'undefined') {
269
+ return output
270
+ }
271
+
272
+ // Determine if history traversal should continue for the current node
273
+ let shouldTraverseHistory
274
+ if (typeof historySelector !== 'number') {
275
+ shouldTraverseHistory = await historySelector(output.beef, output.outputIndex, currentDepth)
276
+ } else {
277
+ shouldTraverseHistory = currentDepth <= historySelector
278
+ }
279
+
280
+ if (shouldTraverseHistory === false) {
281
+ return undefined
282
+ } else if (output && output.outputsConsumed.length === 0) {
283
+ return output
284
+ }
285
+
286
+ try {
287
+ // Query the storage engine for UTXOs consumed by this UTXO
288
+ // Only retrieve unique values in case outputs are doubly referenced
289
+ const outputsConsumed: { txid: string, outputIndex: number }[] = output.outputsConsumed
290
+
291
+ // Find the child outputs for each utxo consumed by the current output
292
+ const childHistories = await (await Promise.all(
293
+ outputsConsumed.map(async (outputIdentifier) => {
294
+ const output = await this.storage.findOutput(outputIdentifier.txid, outputIdentifier.outputIndex)
295
+
296
+ // Make sure an output was found
297
+ if (!output) {
298
+ return undefined
299
+ }
300
+
301
+ // Find previousUTXO history
302
+ return this.getUTXOHistory(output, historySelector, currentDepth + 1)
303
+ })
304
+ )).filter(x => x !== undefined)
305
+
306
+ const tx = Transaction.fromBEEF(output.beef)
307
+ for (const input of childHistories) {
308
+ if (!input) continue
309
+ const inputIndex = tx.inputs.findIndex((input) => {
310
+ const sourceTXID = input.sourceTXID || input.sourceTransaction?.id('hex') as string
311
+ return sourceTXID === output.txid && input.sourceOutputIndex === output.outputIndex
312
+ })
313
+ tx.inputs[inputIndex].sourceTransaction = Transaction.fromBEEF(input.beef)
314
+ }
315
+ const beef = tx.toBEEF()
316
+ return {
317
+ ...output,
318
+ beef
319
+ }
320
+ } catch (e) {
321
+ // Handle any errors that occurred
322
+ // Note: Test this!
323
+ console.error(`Error retrieving UTXO history: ${e}`)
324
+ // return []
325
+ throw new Error(`Error retrieving UTXO history: ${e}`)
326
+ }
327
+ }
328
+
329
+ /**
330
+ * Delete a UTXO and all stale consumed inputs
331
+ * @param output
332
+ * @param id
333
+ * @param txid
334
+ * @param outputIndex
335
+ * @returns {Promise<void>}
336
+ */
337
+ private async deleteUTXODeep(output: Output): Promise<void> {
338
+ try {
339
+ // Delete the current output IFF there are no references to it
340
+ if (output.consumedBy.length === 0) {
341
+ await this.storage.deleteOutput(output.txid, output.outputIndex, output.topic)
342
+
343
+ // Notify the lookup services of the UTXO being deleted
344
+ for (const l of Object.values(this.lookupServices)) {
345
+ try {
346
+ await l.outputDeleted?.(
347
+ output.txid,
348
+ output.outputIndex,
349
+ output.topic!
350
+ )
351
+ } catch (_) { }
352
+ }
353
+ }
354
+
355
+ // If there are no more consumed utxos, return
356
+ if (output.outputsConsumed.length === 0) {
357
+ return
358
+ }
359
+
360
+ // Delete any stale outputs that were consumed as inputs
361
+ output.outputsConsumed.map(async (outputIdentifier) => {
362
+ const staleOutput = await this.storage.findOutput(outputIdentifier.txid, outputIdentifier.outputIndex, output.topic)
363
+
364
+ // Make sure an output was found
365
+ if (!staleOutput) {
366
+ return undefined
367
+ }
368
+
369
+ // Parse out the existing data, then concat the new outputs with no duplicates
370
+ if (staleOutput.consumedBy.length !== 0) {
371
+ staleOutput.consumedBy = staleOutput.consumedBy.filter(x => x.txid !== output.txid && x.outputIndex !== output.outputIndex)
372
+ // Update with the new consumedBy data
373
+ await this.storage.updateConsumedBy(outputIdentifier.txid, outputIdentifier.outputIndex, output.topic, staleOutput.consumedBy)
374
+ }
375
+
376
+ // Find previousUTXO history
377
+ return await this.deleteUTXODeep(staleOutput)
378
+ })
379
+ } catch (error) {
380
+ throw new Error(`Failed to delete all stale outputs: ${error}`)
381
+ }
382
+ }
383
+
384
+ /**
385
+ * Find a list of supported topic managers
386
+ * @public
387
+ * @returns {Promise<string[]>} - array of supported topic managers
388
+ */
389
+ async listTopicManagers(): Promise<string[]> {
390
+ return Object.keys(this.managers)
391
+ }
392
+
393
+ /**
394
+ * Find a list of supported lookup services
395
+ * @public
396
+ * @returns {Promise<string[]>} - array of supported lookup services
397
+ */
398
+ async listLookupServiceProviders(): Promise<string[]> {
399
+ return Object.keys(this.lookupServices)
400
+ }
401
+
402
+ /**
403
+ * Run a query to get the documentation for a particular topic manager
404
+ * @public
405
+ * @returns {Promise<string>} - the documentation for the topic manager
406
+ */
407
+ async getDocumentationForTopicManger(manager: any): Promise<string> {
408
+ return this.managers[manager].getDocumentation?.() || 'No documentation found!'
409
+ }
410
+
411
+ /**
412
+ * Run a query to get the documentation for a particular lookup service
413
+ * @public
414
+ * @returns {Promise<string>} - the documentation for the lookup service
415
+ */
416
+ async getDocumentationForLookupServiceProvider(provider: any): Promise<string> {
417
+ return this.lookupServices[provider].getDocumentation?.() || 'No documentation found!'
418
+ }
419
+ }
@@ -0,0 +1,14 @@
1
+ /**
2
+ * How the Overlay Services Engine responds to a Lookup Question.
3
+ * It may comprise either an output list or a freeform response from the Lookup Service.
4
+ */
5
+ export type LookupAnswer = {
6
+ type: 'output-list'
7
+ outputs: Array<{
8
+ beef: number[]
9
+ outputIndex: number
10
+ }>
11
+ } | {
12
+ type: 'freeform',
13
+ result: unknown
14
+ }
@@ -0,0 +1,26 @@
1
+ /**
2
+ * The formula that will be used by the Overlay Services Engine to compute the Lookup Answer. Can be returned by Lookup Services in response to a Lookup Question.
3
+ */
4
+ export type LookupFormula = {
5
+ /**
6
+ * TXID of the transaction where an output responsive to the Lookup Question resides.
7
+ */
8
+ txid: string,
9
+
10
+ /**
11
+ * Index of the transaction output responsive to the Lookup Question.
12
+ */
13
+ outputIndex: number,
14
+
15
+ /**
16
+ * Decides what history to incorporate into the Lookup Answer.
17
+ *
18
+ * Optionally directs the Overlay Services Engine as to the historical context (preceeding outputs) to include as part of the Lookup Answer.
19
+ * - If a number, denotes how many previous spends (in terms of chain depth) the Engine should include with the Answer.
20
+ * - If a decider function, accepts a BEEF-formatted transaction, an output index and the current depth (relative to the top-level respnosive UTXO) as parameters.
21
+ * The function returns a promise for a boolean indicating whether the output should be incorporated into the Lookup Answer.
22
+ * If so, the function will be called again for transactions that preceeded the one provided, ultimately allowing the complete shape of the responsive spend history to be described.
23
+ * If not provided, no historical information will be included with the Lookup Answer, except that which may incidentally be required to fully anchor the Lookup Answer to the blockchain.
24
+ */
25
+ history?: ((beef: number[], outputIndex: number, currentDepth: number) => Promise<boolean>) | number
26
+ }[]