@bsv/overlay 2.0.3 → 2.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +226 -106
  2. package/dist/cjs/mod.js +6 -1
  3. package/dist/cjs/mod.js.map +1 -1
  4. package/dist/cjs/package.json +13 -11
  5. package/dist/cjs/src/BASM.js +92 -0
  6. package/dist/cjs/src/BASM.js.map +1 -0
  7. package/dist/cjs/src/BASMRemote.js +43 -0
  8. package/dist/cjs/src/BASMRemote.js.map +1 -0
  9. package/dist/cjs/src/Engine.js +649 -47
  10. package/dist/cjs/src/Engine.js.map +1 -1
  11. package/dist/cjs/src/GASP/OverlayGASPRemote.js +2 -2
  12. package/dist/cjs/src/GASP/OverlayGASPRemote.js.map +1 -1
  13. package/dist/cjs/src/GASP/OverlayGASPStorage.js +18 -25
  14. package/dist/cjs/src/GASP/OverlayGASPStorage.js.map +1 -1
  15. package/dist/cjs/src/storage/knex/KnexStorage.js +329 -12
  16. package/dist/cjs/src/storage/knex/KnexStorage.js.map +1 -1
  17. package/dist/cjs/src/storage/knex/all-migrations.js +3 -1
  18. package/dist/cjs/src/storage/knex/all-migrations.js.map +1 -1
  19. package/dist/cjs/src/storage/knex/migrations/2025-05-28-001-enlarge.js +2 -24
  20. package/dist/cjs/src/storage/knex/migrations/2025-05-28-001-enlarge.js.map +1 -1
  21. package/dist/cjs/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js +114 -0
  22. package/dist/cjs/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js.map +1 -0
  23. package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
  24. package/dist/esm/mod.js +1 -0
  25. package/dist/esm/mod.js.map +1 -1
  26. package/dist/esm/src/BASM.js +85 -0
  27. package/dist/esm/src/BASM.js.map +1 -0
  28. package/dist/esm/src/BASMRemote.js +42 -0
  29. package/dist/esm/src/BASMRemote.js.map +1 -0
  30. package/dist/esm/src/Engine.js +637 -46
  31. package/dist/esm/src/Engine.js.map +1 -1
  32. package/dist/esm/src/GASP/OverlayGASPRemote.js +2 -2
  33. package/dist/esm/src/GASP/OverlayGASPRemote.js.map +1 -1
  34. package/dist/esm/src/GASP/OverlayGASPStorage.js +18 -25
  35. package/dist/esm/src/GASP/OverlayGASPStorage.js.map +1 -1
  36. package/dist/esm/src/storage/knex/KnexStorage.js +323 -12
  37. package/dist/esm/src/storage/knex/KnexStorage.js.map +1 -1
  38. package/dist/esm/src/storage/knex/all-migrations.js +3 -1
  39. package/dist/esm/src/storage/knex/all-migrations.js.map +1 -1
  40. package/dist/esm/src/storage/knex/migrations/2025-05-28-001-enlarge.js +2 -24
  41. package/dist/esm/src/storage/knex/migrations/2025-05-28-001-enlarge.js.map +1 -1
  42. package/dist/esm/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js +109 -0
  43. package/dist/esm/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js.map +1 -0
  44. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  45. package/dist/types/mod.d.ts +2 -0
  46. package/dist/types/mod.d.ts.map +1 -1
  47. package/dist/types/src/BASM.d.ts +110 -0
  48. package/dist/types/src/BASM.d.ts.map +1 -0
  49. package/dist/types/src/BASMRemote.d.ts +14 -0
  50. package/dist/types/src/BASMRemote.d.ts.map +1 -0
  51. package/dist/types/src/Engine.d.ts +78 -1
  52. package/dist/types/src/Engine.d.ts.map +1 -1
  53. package/dist/types/src/GASP/OverlayGASPStorage.d.ts +1 -1
  54. package/dist/types/src/GASP/OverlayGASPStorage.d.ts.map +1 -1
  55. package/dist/types/src/LookupService.d.ts.map +1 -1
  56. package/dist/types/src/storage/Storage.d.ts +118 -0
  57. package/dist/types/src/storage/Storage.d.ts.map +1 -1
  58. package/dist/types/src/storage/knex/KnexStorage.d.ts +38 -6
  59. package/dist/types/src/storage/knex/KnexStorage.d.ts.map +1 -1
  60. package/dist/types/src/storage/knex/all-migrations.d.ts.map +1 -1
  61. package/dist/types/src/storage/knex/migrations/2025-05-28-001-enlarge.d.ts.map +1 -1
  62. package/dist/types/src/storage/knex/migrations/2026-05-29-001-brc136-basm.d.ts +4 -0
  63. package/dist/types/src/storage/knex/migrations/2026-05-29-001-brc136-basm.d.ts.map +1 -0
  64. package/dist/types/tsconfig.types.tsbuildinfo +1 -1
  65. package/docs/API.md +72 -580
  66. package/docs/BRC-136-BASM.md +285 -0
  67. package/docs/README.md +2 -0
  68. package/docs/Synchronization.md +10 -1
  69. package/docs/examples/README.md +2 -2
  70. package/docs/examples/gs-wip.md +68 -76
  71. package/mod.ts +25 -1
  72. package/package.json +25 -23
  73. package/src/BASM.ts +208 -0
  74. package/src/BASMRemote.ts +54 -0
  75. package/src/Engine.ts +762 -47
  76. package/src/GASP/OverlayGASPRemote.ts +2 -2
  77. package/src/GASP/OverlayGASPStorage.ts +17 -22
  78. package/src/LookupService.ts +2 -1
  79. package/src/__tests/BASM.test.ts +44 -0
  80. package/src/__tests/BASMChain.test.ts +290 -0
  81. package/src/__tests/Engine.test.ts +3 -4
  82. package/src/storage/Storage.ts +127 -0
  83. package/src/storage/knex/KnexStorage.ts +376 -14
  84. package/src/storage/knex/all-migrations.ts +3 -1
  85. package/src/storage/knex/migrations/2025-05-28-001-enlarge.ts +2 -22
  86. package/src/storage/knex/migrations/2026-05-29-001-brc136-basm.ts +123 -0
@@ -0,0 +1,285 @@
1
+ # BRC-136 BASM Synchronization & Admin/Janitor Removal
2
+
3
+ This document explains how the BRC-136 Block-Anchored synchronization layer
4
+ ([spec](https://bsv.brc.dev/overlays/0136)) is implemented in this engine, and —
5
+ critically — **how it interacts with admin and janitor removal of outputs from a
6
+ topic**. The two systems are deliberately decoupled. Read the final section
7
+ ("How BASM and removal interact") before operating a node that runs the janitor
8
+ or removes tokens via the admin API.
9
+
10
+ ## What BRC-136 gives you
11
+
12
+ GASP ([Synchronization.md](./Synchronization.md)) replicates transaction *data*
13
+ between peers. BRC-136 sits alongside it and answers a different question:
14
+
15
+ > Do two overlays **agree about which transactions were admitted to a topic** at
16
+ > every block height, from the topic's genesis to the chain tip?
17
+
18
+ It does this with a per-topic **Topic Anchor Chain (TAC)** — a cumulative hash
19
+ that commits to every block's admitted-transaction set. If two nodes report the
20
+ same `TAC(topic, H)`, they provably agree on every admitted transaction for that
21
+ topic from genesis through height `H`. A single round trip detects agreement; a
22
+ binary search over heights localizes any divergence in `O(log H)` rounds.
23
+
24
+ ### Core data structures
25
+
26
+ | Concept | Type | Source of truth |
27
+ |---|---|---|
28
+ | Admitted set at a block | `AdmittedTxRef[]` | `applied_transactions` table |
29
+ | Block anchor | `TopicBlockAnchor` | `topic_block_anchors` table |
30
+ | Cumulative chain hash | `tac` field on the anchor | derived |
31
+
32
+ - **`basmRoot`** — Merkle root over the admitted txids at one block height, in
33
+ canonical block order. Empty set → 32 zero bytes; single txid → the txid
34
+ itself (unhashed); multiple → `SHA256d` binary tree with Bitcoin odd-leaf
35
+ duplication. See `computeBasmRoot` in `src/BASM.ts`.
36
+ - **`tac`** — `SHA256d(prevTac ‖ blockHash ‖ basmRoot)`, all inputs in internal
37
+ byte order. See `computeTac` in `src/BASM.ts`.
38
+
39
+ ### Gap-free chains
40
+
41
+ The TAC must advance on **every** block — including blocks with no admitted
42
+ transactions — or peers could not distinguish "no transactions here" from "I
43
+ haven't synced this height." The engine keeps chains contiguous:
44
+
45
+ - `advanceTopicAnchorChains(toHeight)` extends each topic forward with empty
46
+ anchors (zero root, zero count) up to `toHeight`.
47
+ - `rebuildTopicAnchorChain(...)` / `recomputeTopicBlockAnchor(...)` rebuild a
48
+ contiguous slice after an admission (including out-of-order proofs), filling
49
+ any missing heights with empty anchors rather than leaving a hole.
50
+
51
+ A chain only *starts* at the topic's first admitted height (its genesis).
52
+ Covered by `src/__tests/BASMChain.test.ts`.
53
+
54
+ ## The admitted set is a historical fact, not current UTXO state
55
+
56
+ This is the key design decision, and the source of the assurance this layer
57
+ provides.
58
+
59
+ The admitted set for `(topic, height)` is read by
60
+ `Storage.findAdmittedTransactionsForBlock`, which queries:
61
+
62
+ ```
63
+ applied_transactions WHERE topic = ? AND blockHeight = ? AND proven = true AND blockIndex IS NOT NULL
64
+ ```
65
+
66
+ It is **append-only for proven transactions**. A row lands here when a
67
+ transaction is admitted to the topic and has a merkle proof
68
+ (`buildAppliedTransactionRecord` sets `proven: true` only when proof metadata is
69
+ present). From that point the row — and therefore the `basmRoot` and `tac` that
70
+ commit to it — is an immutable record of "this overlay received and admitted
71
+ this txid at this block height."
72
+
73
+ It is **not** derived from current unspent outputs. Spending an output, pruning
74
+ its history, or removing it from lookup does not change the admitted set.
75
+
76
+ ## The lookup set is separate, mutable, and policy-driven
77
+
78
+ What an overlay *serves* in lookup responses is a different set, in different
79
+ storage:
80
+
81
+ - The engine `outputs` table, filtered to `spent: false`.
82
+ - The lookup service's own index (e.g. the Mongo `shipRecords` / `slapRecords`
83
+ collections for the discovery services).
84
+
85
+ This set is mutable by design. Outputs get marked spent, pruned from history,
86
+ removed by an admin, or deleted by the janitor.
87
+
88
+ ## Removal paths — exactly what each one touches
89
+
90
+ | Path | Code | Touches lookup set | Touches admitted set (`applied_transactions`) |
91
+ |---|---|---|---|
92
+ | Spend / history prune | `Engine.deleteUTXODeep` → `deleteOutput` | yes | **no** |
93
+ | Admin remove-token | `/admin/remove-token` → `evictFromServices` → `LookupService.outputEvicted` | yes (lookup index) | **no** |
94
+ | Janitor (dead SHIP/SLAP host) | `JanitorService.handleUnhealthyOutput` → `deleteOne` + ban | yes (lookup index) | **no** |
95
+ | Ban enforcement | `BanAwareLookupWrapper.outputAdmittedByTopic` | blocks re-entry into lookup index on GASP re-sync | **no** |
96
+ | Unproven eviction | `Engine.evictUnprovenTransactions` → `deleteAppliedTransaction` | yes | yes — **but only `proven: false` rows** |
97
+
98
+ Notes:
99
+
100
+ - **`deleteUTXODeep` never deletes the applied-transaction record.** It calls
101
+ `deleteOutput` only. The admission proof survives a normal spend or prune.
102
+ - **Admin and janitor removal operate on the lookup layer only.** Admin
103
+ `evictFromServices` calls `LookupService.outputEvicted`; the janitor deletes
104
+ from its own Mongo collection and writes to the ban list. Neither calls
105
+ `deleteAppliedTransaction`, `deleteOutput` on proven engine state, or anything
106
+ that mutates `topic_block_anchors`.
107
+ - **The ban list is a lookup-layer filter.** `BanAwareLookupWrapper` blocks
108
+ banned outpoints/domains at `outputAdmittedByTopic`, preventing a removed token
109
+ from re-entering the *lookup index* when GASP re-syncs it from a peer. It does
110
+ not — and must not — block the BRC-136 admitted record.
111
+ - **`evictUnprovenTransactions` is the only path that deletes an
112
+ applied-transaction row**, and it filters to `proven: false`. Such rows never
113
+ appear in any anchor (the anchor query requires `proven: true`), so evicting
114
+ them cannot change any `basmRoot` or `tac`.
115
+
116
+ ## How BASM and removal interact
117
+
118
+ This is the question operators most need answered.
119
+
120
+ ### You can always prove you received and processed a token
121
+
122
+ Even after the janitor deletes a dead host's advertisement, or an admin removes a
123
+ malicious output via `/admin/remove-token` (with or without a ban), the overlay
124
+ **retains full proof that it received and admitted that token**:
125
+
126
+ - the `applied_transactions` row (`proven: true`, with `blockHeight`,
127
+ `blockIndex`, `blockHash`, `merkleRoot`),
128
+ - the `topic_block_anchors` row whose `basmRoot` commits to that txid, and
129
+ - the ability to serve a compound merkle path for it via the BRC-136 endpoints.
130
+
131
+ Removal is a **lookup-serving policy decision**. It changes what the node returns
132
+ to lookup queries. It does not, and cannot, rewrite the historical admitted set
133
+ or the TAC.
134
+
135
+ ### TAC agreement means agreement on *admission*, not on *lookup results*
136
+
137
+ This is the most important operational caveat:
138
+
139
+ > **A matching TAC proves two overlays agree on which transactions were admitted.
140
+ > It does NOT prove they will return identical lookup results.**
141
+
142
+ Bans and removals are **per-node policy and are not synchronized between peers**.
143
+ Two honest overlays running the same topic manager can therefore:
144
+
145
+ - report the **same** `TAC(topic, H)` (identical admitted sets), while
146
+ - returning **different** lookup results — because one node banned a malicious
147
+ output that the other still serves.
148
+
149
+ This is correct and intended. BRC-136 anchors *receipt and admission*, which is
150
+ an objective, deterministic function of chain state and the topic manager. It
151
+ intentionally does not anchor each node's local moderation policy.
152
+
153
+ ### Consequence: divergence diagnosis
154
+
155
+ - **TACs match** → the two nodes admitted exactly the same transactions. Any
156
+ difference in lookup output is explained by local removal/ban policy, not by a
157
+ sync problem.
158
+ - **TACs diverge** → the nodes genuinely disagree about admission at some height
159
+ (e.g. different topic-manager versions, a missed transaction, or a reorg
160
+ handled differently). This is a real sync issue; the binary-search localizes
161
+ the height.
162
+
163
+ Removing a token from lookup will **not** cause TAC divergence. If you ever see
164
+ TAC divergence after only running the janitor/admin removal, that is a bug —
165
+ something is incorrectly deleting proven `applied_transactions` rows or mutating
166
+ anchors.
167
+
168
+ ### Caveat for the GASP validation guidance
169
+
170
+ [Synchronization.md](./Synchronization.md) suggests validating sync by comparing
171
+ per-topic SQL row counts across nodes. Once the janitor or admin removal is
172
+ active, **lookup-set counts can legitimately differ between nodes even when their
173
+ TACs match**. To validate BRC-136 agreement, compare `TAC(topic, tip)` between
174
+ peers — not lookup row counts.
175
+
176
+ ## Reorg handling
177
+
178
+ A blockchain reorganization can orphan a block whose transactions were already
179
+ admitted and anchored. Since anchors and the admitted set are keyed by
180
+ `(topic, blockHeight, blockHash)`, an orphaned block would otherwise leave stale
181
+ `applied_transactions` rows and an anchor whose `basmRoot`/`tac` commit to a
182
+ block hash that is no longer canonical — permanently diverging the TAC from peers
183
+ that observed the reorg. The engine reconciles this automatically.
184
+
185
+ ### Chaintracks is the reorg authority
186
+
187
+ Reorg detection is **not** reinvented here. The production chain tracker is
188
+ Arcade, which wraps go-chaintracks; its reorg SSE is the source of truth:
189
+
190
+ - **`GET /v2/reorg/stream`** emits `data: <JSON>\n\n` frames (no `event:`/`id:`
191
+ lines; `: keepalive` comments between events). Each frame is a `ReorgEvent`:
192
+
193
+ ```jsonc
194
+ {
195
+ "orphanedHashes": ["<blockHash>", ...], // blocks removed from the active chain
196
+ "commonAncestor": { "height": 100, "hash": "..." },
197
+ "newTip": { "height": 103, "hash": "..." },
198
+ "depth": 3
199
+ }
200
+ ```
201
+
202
+ - The same go-chaintracks service answers the merkle-root checks
203
+ (`isValidRootForHeight`) the engine already trusts for all SPV verification.
204
+
205
+ `packages/overlays/overlay-express/src/ReorgStream.ts` consumes this stream and
206
+ maps each event to the engine: `orphanedHashes` → blocks to reconcile,
207
+ `commonAncestor.height + 1` → rebuild floor, `newTip.height` → rebuild ceiling.
208
+ Block hashes are lower-cased to match stored display-hex hashes (go-sdk
209
+ `chainhash.Hash` marshals as reversed/display hex).
210
+
211
+ > **No replay on this stream.** It carries no event ids, so a reorg that fires
212
+ > while the SSE client is disconnected is lost. Every (re)connect therefore runs a
213
+ > catch-up revalidation sweep (below), and the periodic block poll runs the same
214
+ > sweep as a standing fallback for chain trackers that expose no reorg stream.
215
+
216
+ ### What the engine does on a reorg — `Engine.handleReorg`
217
+
218
+ 1. **Demote orphaned admissions.** For each orphaned block hash, every **proven**
219
+ `applied_transactions` row anchored to it is demoted to unproven
220
+ (`demoteAppliedTransactionToUnproven`): block height/hash/index/merkleRoot are
221
+ cleared and `proven` is set to `false`, **but `firstSeenHeight` is kept**. The
222
+ transaction immediately leaves the admitted set.
223
+ 2. **Rebuild affected anchor chains.** Every topic whose chain intersects the
224
+ reorged height range is rebuilt over the **canonical** block hashes (the rebuild
225
+ forces header re-resolution rather than reusing the stale stored hash). Topics
226
+ with no demoted transaction are rebuilt too, because the canonical block hash
227
+ for those heights changed and the `tac` must re-chain over it.
228
+
229
+ `handleReorg` is **idempotent**: a clean window demotes nothing and reproduces an
230
+ identical TAC, so it is safe to run on every reorg event, SSE reconnect, and poll.
231
+
232
+ ### Fallback / catch-up — `Engine.revalidateRecentAnchors(depth = 3)`
233
+
234
+ Scans proven `applied_transactions` in `[tip - depth + 1, tip]`. Any row whose
235
+ proof root no longer validates (`isValidRootForHeight`) or whose block hash
236
+ diverges from the canonical header is treated as orphaned and fed into
237
+ `handleReorg`. This is the reorg path for chain trackers without a reorg stream,
238
+ and the catch-up step on every SSE reconnect. Default depth is 3 blocks
239
+ (`reorgScanDepth`, configurable via `configureReorgStream`).
240
+
241
+ ### Interaction with admin/janitor removal
242
+
243
+ Reorg recovery and lookup-layer removal are independent and do not interfere:
244
+
245
+ - Removal/bans touch only the lookup layer; they never demote `applied_transactions`.
246
+ - Reorg demotion touches only the admitted set (proven → unproven) via the chain
247
+ tracker; it never consults the ban list or lookup index.
248
+
249
+ A demoted transaction follows the engine's existing unproven lifecycle: if it is
250
+ re-mined, Arcade/ARC re-notifies `/arc-ingest` → `handleNewMerkleProof` re-proves
251
+ it at its new height and the anchor includes it again; if it is never re-mined,
252
+ `evictUnprovenTransactions` removes it after the threshold. The "we received it"
253
+ record survives until one of those resolves.
254
+
255
+ ### Configuration
256
+
257
+ ```ts
258
+ server.configureReorgStream('https://arcade.example/v2/reorg/stream', 3)
259
+ ```
260
+
261
+ - `reorgStreamUrl` — when set, the SSE adapter reconciles reorgs in real time.
262
+ - `reorgScanDepth` — sweep depth from tip (default `3`).
263
+ - The block poll (`basmBlockPollIntervalMs`) runs the sweep as a fallback even
264
+ when no stream is configured.
265
+
266
+ ## Verification checklist for operators
267
+
268
+ 1. After removing a token, confirm its `applied_transactions` row still exists
269
+ (`proven: true`) and the `topic_block_anchors` row for its height is unchanged.
270
+ 2. Confirm `TAC(topic, tip)` is unchanged by the removal.
271
+ 3. Confirm the token no longer appears in lookup responses (and, if banned, does
272
+ not reappear after a GASP re-sync).
273
+
274
+ If all three hold, removal is behaving correctly: the node no longer serves the
275
+ token, yet can still prove it was received and admitted.
276
+
277
+ ### Reorg verification
278
+
279
+ 4. After a reorg orphans an admitted transaction's block, confirm its
280
+ `applied_transactions` row is demoted (`proven: false`, block fields cleared,
281
+ `firstSeenHeight` retained) and the affected `topic_block_anchors` rows now
282
+ carry the canonical block hashes with a recomputed `tac`.
283
+ 5. Confirm `TAC(topic, tip)` reconverges with peers that observed the same reorg.
284
+ 6. If the orphaned transaction is re-mined, confirm `/arc-ingest` re-proves it and
285
+ the anchor re-includes it; otherwise confirm it is eventually evicted as unproven.
package/docs/README.md CHANGED
@@ -5,6 +5,8 @@ The documentation is split into various pages, each covering a set of related fu
5
5
  ## Main Documentation
6
6
 
7
7
  - [API](./API.md) — How to interact with the Overlay Services Engine
8
+ - [Synchronization](./Synchronization.md) — GASP peer synchronization
9
+ - [BRC-136 BASM](./BRC-136-BASM.md) — Block-anchored sync, and how it interacts with admin/janitor removal
8
10
  - [Concepts](./concepts/README.md) — High-level concepts
9
11
  - [Examples](./examples/README.md) — Guides, tutorials and examples of using the Engine
10
12
  - [Internal](./internal/README.md) — References for working with internal system components
@@ -200,4 +200,13 @@ Example (adapt the table/query to your schema):
200
200
  +-------------+------+
201
201
  ```
202
202
 
203
- **What you’re looking for:** the topic(s) you care about (e.g. tm_plite, tm_blockbeta) should move toward the same counts/data across nodes after sync, and your lookup results for identical queries should converge as well.
203
+ **What you’re looking for:** the topic(s) you care about (e.g. tm_plite, tm_blockbeta) should move toward the same counts/data across nodes after sync, and your lookup results for identical queries should converge as well.
204
+
205
+ > **Caveat (BRC-136 / janitor / admin removal):** Row-count comparison is only a
206
+ > valid convergence check on nodes that do **not** remove outputs locally. Once
207
+ > the janitor or `/admin/remove-token` is active, lookup-set counts can
208
+ > legitimately differ between peers even when they fully agree on what was
209
+ > admitted, because bans/removals are per-node policy and are not synchronized.
210
+ > To verify that two nodes agree on which transactions were admitted to a topic,
211
+ > compare their per-topic Topic Anchor Chain (`TAC`) at the tip rather than
212
+ > lookup row counts. See [BRC-136 BASM](./BRC-136-BASM.md).
@@ -4,11 +4,11 @@
4
4
 
5
5
  ---
6
6
 
7
- Here, you will find documentation for common example usages of the Overlay Services Engine. These examples focus on direct `@bsv/overlay` integration. For a ready-to-run HTTP overlay node, use [`@bsv/overlay-express`](https://github.com/bsv-blockchain/overlay-express) and [`overlay-express-examples`](https://github.com/bsv-blockchain/overlay-express-examples).
7
+ Here, you will find documentation for common example usages of the Overlay Services Engine.
8
8
 
9
9
  ## Available Examples
10
10
 
11
- - [Getting Started](./gs-wip.md) — Introduction and low-level engine setup guide
11
+ - [Getting Started (WIP)](./gs-wip.md) — Introduction and getting started guide
12
12
 
13
13
  ---
14
14
 
@@ -6,114 +6,106 @@
6
6
 
7
7
  ## Introduction to BSV Overlay Services Engine
8
8
 
9
- The BSV Overlay Services Engine is the low-level runtime for topic managers and lookup services. It validates candidate outputs, stores admitted UTXOs, tracks spends, answers lookup questions, and coordinates synchronization between overlay nodes.
10
-
11
- If you want to run an HTTP overlay node, start with [`@bsv/overlay-express`](https://github.com/bsv-blockchain/overlay-express). If you want a local or cloud application runtime, use LARS or CARS with the BRC-102 `deployment-info.json` structure. Use `@bsv/overlay` directly when you are building custom infrastructure around the engine.
9
+ The BSV Overlay Services Engine is designed to process transactions and manage data within a blockchain-based system, specifically targeting the Bitcoin SV (BSV) blockchain. It integrates various components such as Topic Managers, Lookup Services, Storage, and Chain Tracker to provide a robust environment for managing transaction data and overlay services.
12
10
 
13
11
  ### Components of the System
14
12
 
15
- 1. **Topic Managers** decide which transaction outputs are admitted for a topic.
16
- 2. **Lookup Services** index admitted and spent outputs and answer domain-specific lookup questions.
17
- 3. **Storage** persists admitted outputs, spend state, history, sync state, and interaction timestamps.
18
- 4. **Chain Tracker** verifies SPV data for transactions unless the engine is configured for script-only validation.
19
- 5. **Broadcaster** submits accepted transactions to the network.
20
- 6. **Advertiser** publishes SHIP and SLAP availability records when peer discovery is enabled.
13
+ 1. **Topic Managers**: Responsible for managing the admittance of transactions related to specific topics.
14
+ 2. **Lookup Services**: Handle the lookup of UTXO (Unspent Transaction Output) data for transactions.
15
+ 3. **Storage**: Manages persistent data storage, tracking UTXOs and their states within the system.
16
+ 4. **Chain Tracker**: Verifies SPV (Simplified Payment Verification) data associated with transactions to ensure their validity.
21
17
 
22
18
  ### Setting Up the Engine
23
19
 
24
- Install the current packages:
20
+ Before you can use the engine, you must initialize it with the required components:
25
21
 
26
- ```bash
27
- npm i @bsv/overlay @bsv/sdk knex
28
- ```
22
+ ```ts
23
+ import { Engine, KnexStorage } from "@bsv/overlay";
24
+ import { HelloTopicManager, HelloLookupService } from 'hello-services';
25
+ import { WoChain } from "@bsv/sdk";
29
26
 
30
- Create the engine with your concrete implementations:
27
+ // Initialize components
28
+ const managers = {
29
+ "exampleTopic": new HelloTopicManager()
30
+ };
31
31
 
32
- ```ts
33
- import { Engine, KnexStorage } from '@bsv/overlay'
34
- import { WhatsOnChain, NodejsHttpClient, ARC } from '@bsv/sdk'
35
- import knexFactory from 'knex'
36
-
37
- import { ExampleLookupService } from './services/ExampleLookupService.js'
38
- import { ExampleTopicManager } from './services/ExampleTopicManager.js'
39
-
40
- const knex = knexFactory({
41
- client: 'mysql2',
42
- connection: process.env.KNEX_URL
43
- })
44
-
45
- const engine = new Engine(
46
- {
47
- tm_example: new ExampleTopicManager()
48
- },
49
- {
50
- ls_example: new ExampleLookupService()
51
- },
52
- new KnexStorage(knex),
53
- new WhatsOnChain('test', { httpClient: new NodejsHttpClient() }),
54
- process.env.HOSTING_URL,
55
- process.env.SHIP_TRACKERS?.split(',') ?? [],
56
- process.env.SLAP_TRACKERS?.split(',') ?? [],
57
- new ARC(process.env.ARC_URL ?? 'https://arc.taal.com', {
58
- apiKey: process.env.ARC_API_KEY
59
- }),
60
- undefined,
61
- {
62
- tm_example: 'SHIP'
63
- }
64
- )
65
- ```
32
+ const lookupServices = {
33
+ "exampleLookup": new HelloLookupService()
34
+ };
66
35
 
67
- Use the string `'scripts only'` for the chain tracker only when the service intentionally skips SPV validation and relies only on script-level checks.
36
+ const storage = new KnexStorage();
37
+ const chainTracker = new WoChain();
38
+
39
+ // Create the engine instance
40
+ const engine = new Engine(managers, lookupServices, storage, chainTracker);
41
+ ```
68
42
 
69
43
  ### Submitting a Transaction
70
44
 
71
- Submit tagged BEEF bytes with the topics that should evaluate the transaction:
45
+ To submit a transaction for processing by the Overlay Services:
72
46
 
73
47
  ```ts
74
48
  import { Transaction } from '@bsv/sdk'
75
49
 
76
- const tx = new Transaction(/* ... */)
50
+ const tx = new Transaction(/* ... */);
77
51
 
78
- const steak = await engine.submit({
79
- beef: tx.toBEEF(),
80
- topics: ['tm_example']
81
- })
52
+ const transaction = {
53
+ beef: tx.toBEEF(),
54
+ topics: ['exampleTopic']
55
+ }
82
56
 
83
- console.log('Transaction processed:', steak)
57
+ // Submit transaction
58
+ engine.submit(transaction).then(steak => {
59
+ console.log("Transaction processed:", steak);
60
+ }).catch(error => {
61
+ console.error("Error processing transaction:", error);
62
+ });
84
63
  ```
85
64
 
86
65
  ### Lookup Queries
87
66
 
88
- Ask a lookup service a domain-specific question:
67
+ To perform a lookup query using the engine:
89
68
 
90
69
  ```ts
91
- const answer = await engine.lookup({
92
- service: 'ls_example',
93
- query: {
94
- identityKey: '03...'
95
- }
96
- })
97
-
98
- console.log('Lookup result:', answer)
70
+ const question = {
71
+ service: 'exampleLookup',
72
+ query: {
73
+ name: 'Bob'
74
+ }
75
+ }
76
+
77
+ // Perform a lookup
78
+ engine.lookup(question).then(answer => {
79
+ console.log("Lookup result:", answer);
80
+ }).catch(error => {
81
+ console.error("Error performing lookup:", error);
82
+ });
99
83
  ```
100
84
 
101
- ### Service Documentation
85
+ ### Managing UTXOs
102
86
 
103
- Overlay clients and dashboards can retrieve service documentation and metadata directly from the engine:
87
+ The system's core functionality involves managing UTXOs:
104
88
 
105
- ```ts
106
- const topicDocs = await engine.getDocumentationForTopicManger('tm_example')
107
- const lookupDocs = await engine.getDocumentationForLookupServiceProvider('ls_example')
89
+ 1. **Inserting a New UTXO**: Store new UTXO data when transactions are processed.
90
+ 2. **Deleting a UTXO**: Remove UTXOs that are no longer needed or have been consumed by newer transactions.
91
+ 3. **Tracking UTXO Consumption**: Monitor which transactions consume which UTXOs.
108
92
 
109
- const topics = await engine.listTopicManagers()
110
- const lookupServices = await engine.listLookupServiceProviders()
111
- ```
93
+ ### Retrieving Documentation
112
94
 
113
- ### Deployment Path
95
+ To retrieve documentation for specific managers or services:
114
96
 
115
- For a public overlay node, wire the engine into HTTP routes using `@bsv/overlay-express` instead of hand-rolling routes. Overlay Express already exposes the standard submit, lookup, sync, health, documentation, and admin surfaces expected by the rest of the BSV overlay ecosystem.
97
+ ```ts
98
+ // For a topic manager
99
+ engine.getDocumentationForTopicManger("exampleTopic").then(doc => {
100
+ console.log("Documentation for Topic Manager:", doc);
101
+ });
102
+
103
+ // For a lookup service
104
+ engine.getDocumentationForLookupServiceProvider("exampleLookup").then(doc => {
105
+ console.log("Documentation for Lookup Service:", doc);
106
+ });
107
+ ```
116
108
 
117
109
  ### Conclusion
118
110
 
119
- The BSV Overlay Services Engine is the shared core for overlay validation, indexing, lookup, and synchronization. Keep direct engine usage focused on infrastructure-level integrations, and use Overlay Express, LARS, and CARS for the standard application path.
111
+ The BSV Overlay Services Engine provides a powerful toolset for managing transactions and data on the Bitcoin SV blockchain. It's designed to handle complex data structures and ensure the integrity and security of transactions through rigorous validation and management processes. By following this tutorial, developers can effectively integrate and utilize these capabilities within their blockchain applications.
package/mod.ts CHANGED
@@ -10,7 +10,31 @@ export type { TaggedBEEF, STEAK, LookupQuestion, LookupAnswer, AdmittanceInstruc
10
10
  export type { LookupFormula } from './src/LookupFormula.js'
11
11
  export type { Advertisement } from './src/Advertisement.js'
12
12
  export type { AdvertisementData, Advertiser } from './src/Advertiser.js'
13
+ export {
14
+ BASM_ZERO_HASH,
15
+ computeBasmRoot,
16
+ computeTac,
17
+ extractMerkleProofMetadata
18
+ } from './src/BASM.js'
19
+ export type {
20
+ AdmittedListRequest,
21
+ AdmittedListResponse,
22
+ AdmittedTxRef,
23
+ BASMPeerSyncReport,
24
+ CompoundMerklePathRequest,
25
+ CompoundMerklePathResponse,
26
+ MerkleProofMetadata,
27
+ RawTransactionRecord,
28
+ RawTransactionRequest,
29
+ RawTransactionResponse,
30
+ TopicAnchorHeader,
31
+ TopicAnchorHeaderResolver,
32
+ TopicAnchorRangeRequest,
33
+ TopicAnchorRangeResponse,
34
+ TopicAnchorTip,
35
+ TopicBlockAnchor
36
+ } from './src/BASM.js'
13
37
 
14
38
  // The Knex storage system
15
39
  export { KnexStorage } from './src/storage/knex/KnexStorage.js'
16
- export * as KnexStorageMigrations from './src/storage/knex/all-migrations.js'
40
+ export * as KnexStorageMigrations from './src/storage/knex/all-migrations.js'
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bsv/overlay",
3
- "version": "2.0.3",
3
+ "version": "2.0.4",
4
4
  "type": "module",
5
5
  "description": "BSV Blockchain Overlay Services Engine",
6
6
  "main": "dist/cjs/mod.js",
@@ -35,20 +35,10 @@
35
35
  "types": "./dist/types/src/storage/*.d.ts"
36
36
  }
37
37
  },
38
- "scripts": {
39
- "test": "npm run build && jest",
40
- "test:watch": "npm run build && jest --watch",
41
- "test:coverage": "npm run build && jest --coverage",
42
- "lint": "ts-standard --fix src/**/*.ts",
43
- "lint:ci": "ts-standard src/**/*.ts",
44
- "build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json",
45
- "dev": "tsc -b -w",
46
- "prepublish": "npm run build",
47
- "doc": "ts2md --inputFilename=mod.ts --outputFilename=docs/API.md --filenameSubstring=API --firstHeadingLevel=1"
48
- },
49
38
  "repository": {
50
39
  "type": "git",
51
- "url": "git+https://github.com/bsv-blockchain/overlay-services.git"
40
+ "url": "git+https://github.com/bsv-blockchain/ts-stack.git",
41
+ "directory": "packages/overlays/overlay"
52
42
  },
53
43
  "keywords": [
54
44
  "BSV",
@@ -60,21 +50,33 @@
60
50
  "author": "BSV Association",
61
51
  "license": "SEE LICENSE IN LICENSE.txt",
62
52
  "bugs": {
63
- "url": "https://github.com/bitcoin-sv/overlay-services/issues"
53
+ "url": "https://github.com/bsv-blockchain/ts-stack/issues"
64
54
  },
65
- "homepage": "https://github.com/bitcoin-sv/overlay-services#readme",
55
+ "homepage": "https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay#readme",
66
56
  "devDependencies": {
67
- "@types/jest": "^29.5.12",
68
- "jest": "^29.7.0",
69
- "ts-jest": "^29.1.1",
57
+ "@types/jest": "^30.0.0",
58
+ "@types/node": "^25.9.3",
59
+ "jest": "^30.4.2",
60
+ "ts-jest": "^29.4.11",
70
61
  "ts-standard": "^12.0.2",
71
62
  "ts2md": "^0.2.0",
72
63
  "tsconfig-to-dual-package": "^1.2.0",
73
- "typescript": "^5.2.2"
64
+ "typescript": "^6.0.3"
74
65
  },
75
66
  "dependencies": {
76
- "@bsv/gasp": "^1.2.2",
77
- "@bsv/sdk": "^2.0.14",
78
- "knex": "^3.1.0"
67
+ "@bsv/gasp": "^1.2.3",
68
+ "@bsv/sdk": "^2.1.4",
69
+ "knex": "^3.2.10"
70
+ },
71
+ "scripts": {
72
+ "test": "npm run build && jest",
73
+ "test:watch": "npm run build && jest --watch",
74
+ "test:coverage": "npm run build && jest --coverage",
75
+ "lint": "ts-standard --fix src/**/*.ts",
76
+ "lint:ci": "ts-standard src/**/*.ts",
77
+ "build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json",
78
+ "dev": "tsc -b -w",
79
+ "prepublish": "npm run build",
80
+ "doc": "ts2md --inputFilename=mod.ts --outputFilename=docs/API.md --filenameSubstring=API --firstHeadingLevel=1"
79
81
  }
80
- }
82
+ }