@bsv/overlay 2.0.2 → 2.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/dist/cjs/mod.js +6 -1
  2. package/dist/cjs/mod.js.map +1 -1
  3. package/dist/cjs/package.json +13 -11
  4. package/dist/cjs/src/BASM.js +92 -0
  5. package/dist/cjs/src/BASM.js.map +1 -0
  6. package/dist/cjs/src/BASMRemote.js +43 -0
  7. package/dist/cjs/src/BASMRemote.js.map +1 -0
  8. package/dist/cjs/src/Engine.js +649 -47
  9. package/dist/cjs/src/Engine.js.map +1 -1
  10. package/dist/cjs/src/GASP/OverlayGASPRemote.js +2 -2
  11. package/dist/cjs/src/GASP/OverlayGASPRemote.js.map +1 -1
  12. package/dist/cjs/src/GASP/OverlayGASPStorage.js +18 -25
  13. package/dist/cjs/src/GASP/OverlayGASPStorage.js.map +1 -1
  14. package/dist/cjs/src/storage/knex/KnexStorage.js +329 -12
  15. package/dist/cjs/src/storage/knex/KnexStorage.js.map +1 -1
  16. package/dist/cjs/src/storage/knex/all-migrations.js +3 -1
  17. package/dist/cjs/src/storage/knex/all-migrations.js.map +1 -1
  18. package/dist/cjs/src/storage/knex/migrations/2025-05-28-001-enlarge.js +2 -24
  19. package/dist/cjs/src/storage/knex/migrations/2025-05-28-001-enlarge.js.map +1 -1
  20. package/dist/cjs/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js +114 -0
  21. package/dist/cjs/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js.map +1 -0
  22. package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
  23. package/dist/esm/mod.js +1 -0
  24. package/dist/esm/mod.js.map +1 -1
  25. package/dist/esm/src/BASM.js +85 -0
  26. package/dist/esm/src/BASM.js.map +1 -0
  27. package/dist/esm/src/BASMRemote.js +42 -0
  28. package/dist/esm/src/BASMRemote.js.map +1 -0
  29. package/dist/esm/src/Engine.js +637 -46
  30. package/dist/esm/src/Engine.js.map +1 -1
  31. package/dist/esm/src/GASP/OverlayGASPRemote.js +2 -2
  32. package/dist/esm/src/GASP/OverlayGASPRemote.js.map +1 -1
  33. package/dist/esm/src/GASP/OverlayGASPStorage.js +18 -25
  34. package/dist/esm/src/GASP/OverlayGASPStorage.js.map +1 -1
  35. package/dist/esm/src/storage/knex/KnexStorage.js +323 -12
  36. package/dist/esm/src/storage/knex/KnexStorage.js.map +1 -1
  37. package/dist/esm/src/storage/knex/all-migrations.js +3 -1
  38. package/dist/esm/src/storage/knex/all-migrations.js.map +1 -1
  39. package/dist/esm/src/storage/knex/migrations/2025-05-28-001-enlarge.js +2 -24
  40. package/dist/esm/src/storage/knex/migrations/2025-05-28-001-enlarge.js.map +1 -1
  41. package/dist/esm/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js +109 -0
  42. package/dist/esm/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js.map +1 -0
  43. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  44. package/dist/types/mod.d.ts +2 -0
  45. package/dist/types/mod.d.ts.map +1 -1
  46. package/dist/types/src/BASM.d.ts +110 -0
  47. package/dist/types/src/BASM.d.ts.map +1 -0
  48. package/dist/types/src/BASMRemote.d.ts +14 -0
  49. package/dist/types/src/BASMRemote.d.ts.map +1 -0
  50. package/dist/types/src/Engine.d.ts +78 -1
  51. package/dist/types/src/Engine.d.ts.map +1 -1
  52. package/dist/types/src/GASP/OverlayGASPStorage.d.ts +1 -1
  53. package/dist/types/src/GASP/OverlayGASPStorage.d.ts.map +1 -1
  54. package/dist/types/src/LookupService.d.ts.map +1 -1
  55. package/dist/types/src/storage/Storage.d.ts +118 -0
  56. package/dist/types/src/storage/Storage.d.ts.map +1 -1
  57. package/dist/types/src/storage/knex/KnexStorage.d.ts +38 -6
  58. package/dist/types/src/storage/knex/KnexStorage.d.ts.map +1 -1
  59. package/dist/types/src/storage/knex/all-migrations.d.ts.map +1 -1
  60. package/dist/types/src/storage/knex/migrations/2025-05-28-001-enlarge.d.ts.map +1 -1
  61. package/dist/types/src/storage/knex/migrations/2026-05-29-001-brc136-basm.d.ts +4 -0
  62. package/dist/types/src/storage/knex/migrations/2026-05-29-001-brc136-basm.d.ts.map +1 -0
  63. package/dist/types/tsconfig.types.tsbuildinfo +1 -1
  64. package/docs/BRC-136-BASM.md +285 -0
  65. package/docs/README.md +2 -0
  66. package/docs/Synchronization.md +10 -1
  67. package/mod.ts +25 -1
  68. package/package.json +25 -23
  69. package/src/BASM.ts +208 -0
  70. package/src/BASMRemote.ts +54 -0
  71. package/src/Engine.ts +762 -47
  72. package/src/GASP/OverlayGASPRemote.ts +2 -2
  73. package/src/GASP/OverlayGASPStorage.ts +17 -22
  74. package/src/LookupService.ts +2 -1
  75. package/src/__tests/BASM.test.ts +44 -0
  76. package/src/__tests/BASMChain.test.ts +290 -0
  77. package/src/__tests/Engine.test.ts +3 -4
  78. package/src/storage/Storage.ts +127 -0
  79. package/src/storage/knex/KnexStorage.ts +376 -14
  80. package/src/storage/knex/all-migrations.ts +3 -1
  81. package/src/storage/knex/migrations/2025-05-28-001-enlarge.ts +2 -22
  82. package/src/storage/knex/migrations/2026-05-29-001-brc136-basm.ts +123 -0
@@ -0,0 +1,285 @@
1
+ # BRC-136 BASM Synchronization & Admin/Janitor Removal
2
+
3
+ This document explains how the BRC-136 Block-Anchored synchronization layer
4
+ ([spec](https://bsv.brc.dev/overlays/0136)) is implemented in this engine, and —
5
+ critically — **how it interacts with admin and janitor removal of outputs from a
6
+ topic**. The two systems are deliberately decoupled. Read the final section
7
+ ("How BASM and removal interact") before operating a node that runs the janitor
8
+ or removes tokens via the admin API.
9
+
10
+ ## What BRC-136 gives you
11
+
12
+ GASP ([Synchronization.md](./Synchronization.md)) replicates transaction *data*
13
+ between peers. BRC-136 sits alongside it and answers a different question:
14
+
15
+ > Do two overlays **agree about which transactions were admitted to a topic** at
16
+ > every block height, from the topic's genesis to the chain tip?
17
+
18
+ It does this with a per-topic **Topic Anchor Chain (TAC)** — a cumulative hash
19
+ that commits to every block's admitted-transaction set. If two nodes report the
20
+ same `TAC(topic, H)`, they provably agree on every admitted transaction for that
21
+ topic from genesis through height `H`. A single round trip detects agreement; a
22
+ binary search over heights localizes any divergence in `O(log H)` rounds.
23
+
24
+ ### Core data structures
25
+
26
+ | Concept | Type | Source of truth |
27
+ |---|---|---|
28
+ | Admitted set at a block | `AdmittedTxRef[]` | `applied_transactions` table |
29
+ | Block anchor | `TopicBlockAnchor` | `topic_block_anchors` table |
30
+ | Cumulative chain hash | `tac` field on the anchor | derived |
31
+
32
+ - **`basmRoot`** — Merkle root over the admitted txids at one block height, in
33
+ canonical block order. Empty set → 32 zero bytes; single txid → the txid
34
+ itself (unhashed); multiple → `SHA256d` binary tree with Bitcoin odd-leaf
35
+ duplication. See `computeBasmRoot` in `src/BASM.ts`.
36
+ - **`tac`** — `SHA256d(prevTac ‖ blockHash ‖ basmRoot)`, all inputs in internal
37
+ byte order. See `computeTac` in `src/BASM.ts`.
38
+
39
+ ### Gap-free chains
40
+
41
+ The TAC must advance on **every** block — including blocks with no admitted
42
+ transactions — or peers could not distinguish "no transactions here" from "I
43
+ haven't synced this height." The engine keeps chains contiguous:
44
+
45
+ - `advanceTopicAnchorChains(toHeight)` extends each topic forward with empty
46
+ anchors (zero root, zero count) up to `toHeight`.
47
+ - `rebuildTopicAnchorChain(...)` / `recomputeTopicBlockAnchor(...)` rebuild a
48
+ contiguous slice after an admission (including out-of-order proofs), filling
49
+ any missing heights with empty anchors rather than leaving a hole.
50
+
51
+ A chain only *starts* at the topic's first admitted height (its genesis).
52
+ Covered by `src/__tests/BASMChain.test.ts`.
53
+
54
+ ## The admitted set is a historical fact, not current UTXO state
55
+
56
+ This is the key design decision, and the source of the assurance this layer
57
+ provides.
58
+
59
+ The admitted set for `(topic, height)` is read by
60
+ `Storage.findAdmittedTransactionsForBlock`, which queries:
61
+
62
+ ```
63
+ applied_transactions WHERE topic = ? AND blockHeight = ? AND proven = true AND blockIndex IS NOT NULL
64
+ ```
65
+
66
+ It is **append-only for proven transactions**. A row lands here when a
67
+ transaction is admitted to the topic and has a merkle proof
68
+ (`buildAppliedTransactionRecord` sets `proven: true` only when proof metadata is
69
+ present). From that point the row — and therefore the `basmRoot` and `tac` that
70
+ commit to it — is an immutable record of "this overlay received and admitted
71
+ this txid at this block height."
72
+
73
+ It is **not** derived from current unspent outputs. Spending an output, pruning
74
+ its history, or removing it from lookup does not change the admitted set.
75
+
76
+ ## The lookup set is separate, mutable, and policy-driven
77
+
78
+ What an overlay *serves* in lookup responses is a different set, in different
79
+ storage:
80
+
81
+ - The engine `outputs` table, filtered to `spent: false`.
82
+ - The lookup service's own index (e.g. the Mongo `shipRecords` / `slapRecords`
83
+ collections for the discovery services).
84
+
85
+ This set is mutable by design. Outputs get marked spent, pruned from history,
86
+ removed by an admin, or deleted by the janitor.
87
+
88
+ ## Removal paths — exactly what each one touches
89
+
90
+ | Path | Code | Touches lookup set | Touches admitted set (`applied_transactions`) |
91
+ |---|---|---|---|
92
+ | Spend / history prune | `Engine.deleteUTXODeep` → `deleteOutput` | yes | **no** |
93
+ | Admin remove-token | `/admin/remove-token` → `evictFromServices` → `LookupService.outputEvicted` | yes (lookup index) | **no** |
94
+ | Janitor (dead SHIP/SLAP host) | `JanitorService.handleUnhealthyOutput` → `deleteOne` + ban | yes (lookup index) | **no** |
95
+ | Ban enforcement | `BanAwareLookupWrapper.outputAdmittedByTopic` | blocks re-entry into lookup index on GASP re-sync | **no** |
96
+ | Unproven eviction | `Engine.evictUnprovenTransactions` → `deleteAppliedTransaction` | yes | yes — **but only `proven: false` rows** |
97
+
98
+ Notes:
99
+
100
+ - **`deleteUTXODeep` never deletes the applied-transaction record.** It calls
101
+ `deleteOutput` only. The admission proof survives a normal spend or prune.
102
+ - **Admin and janitor removal operate on the lookup layer only.** Admin
103
+ `evictFromServices` calls `LookupService.outputEvicted`; the janitor deletes
104
+ from its own Mongo collection and writes to the ban list. Neither calls
105
+ `deleteAppliedTransaction`, `deleteOutput` on proven engine state, or anything
106
+ that mutates `topic_block_anchors`.
107
+ - **The ban list is a lookup-layer filter.** `BanAwareLookupWrapper` blocks
108
+ banned outpoints/domains at `outputAdmittedByTopic`, preventing a removed token
109
+ from re-entering the *lookup index* when GASP re-syncs it from a peer. It does
110
+ not — and must not — block the BRC-136 admitted record.
111
+ - **`evictUnprovenTransactions` is the only path that deletes an
112
+ applied-transaction row**, and it filters to `proven: false`. Such rows never
113
+ appear in any anchor (the anchor query requires `proven: true`), so evicting
114
+ them cannot change any `basmRoot` or `tac`.
115
+
116
+ ## How BASM and removal interact
117
+
118
+ This is the question operators most need answered.
119
+
120
+ ### You can always prove you received and processed a token
121
+
122
+ Even after the janitor deletes a dead host's advertisement, or an admin removes a
123
+ malicious output via `/admin/remove-token` (with or without a ban), the overlay
124
+ **retains full proof that it received and admitted that token**:
125
+
126
+ - the `applied_transactions` row (`proven: true`, with `blockHeight`,
127
+ `blockIndex`, `blockHash`, `merkleRoot`),
128
+ - the `topic_block_anchors` row whose `basmRoot` commits to that txid, and
129
+ - the ability to serve a compound merkle path for it via the BRC-136 endpoints.
130
+
131
+ Removal is a **lookup-serving policy decision**. It changes what the node returns
132
+ to lookup queries. It does not, and cannot, rewrite the historical admitted set
133
+ or the TAC.
134
+
135
+ ### TAC agreement means agreement on *admission*, not on *lookup results*
136
+
137
+ This is the most important operational caveat:
138
+
139
+ > **A matching TAC proves two overlays agree on which transactions were admitted.
140
+ > It does NOT prove they will return identical lookup results.**
141
+
142
+ Bans and removals are **per-node policy and are not synchronized between peers**.
143
+ Two honest overlays running the same topic manager can therefore:
144
+
145
+ - report the **same** `TAC(topic, H)` (identical admitted sets), while
146
+ - returning **different** lookup results — because one node banned a malicious
147
+ output that the other still serves.
148
+
149
+ This is correct and intended. BRC-136 anchors *receipt and admission*, which is
150
+ an objective, deterministic function of chain state and the topic manager. It
151
+ intentionally does not anchor each node's local moderation policy.
152
+
153
+ ### Consequence: divergence diagnosis
154
+
155
+ - **TACs match** → the two nodes admitted exactly the same transactions. Any
156
+ difference in lookup output is explained by local removal/ban policy, not by a
157
+ sync problem.
158
+ - **TACs diverge** → the nodes genuinely disagree about admission at some height
159
+ (e.g. different topic-manager versions, a missed transaction, or a reorg
160
+ handled differently). This is a real sync issue; the binary-search localizes
161
+ the height.
162
+
163
+ Removing a token from lookup will **not** cause TAC divergence. If you ever see
164
+ TAC divergence after only running the janitor/admin removal, that is a bug —
165
+ something is incorrectly deleting proven `applied_transactions` rows or mutating
166
+ anchors.
167
+
168
+ ### Caveat for the GASP validation guidance
169
+
170
+ [Synchronization.md](./Synchronization.md) suggests validating sync by comparing
171
+ per-topic SQL row counts across nodes. Once the janitor or admin removal is
172
+ active, **lookup-set counts can legitimately differ between nodes even when their
173
+ TACs match**. To validate BRC-136 agreement, compare `TAC(topic, tip)` between
174
+ peers — not lookup row counts.
175
+
176
+ ## Reorg handling
177
+
178
+ A blockchain reorganization can orphan a block whose transactions were already
179
+ admitted and anchored. Since anchors and the admitted set are keyed by
180
+ `(topic, blockHeight, blockHash)`, an orphaned block would otherwise leave stale
181
+ `applied_transactions` rows and an anchor whose `basmRoot`/`tac` commit to a
182
+ block hash that is no longer canonical — permanently diverging the TAC from peers
183
+ that observed the reorg. The engine reconciles this automatically.
184
+
185
+ ### Chaintracks is the reorg authority
186
+
187
+ Reorg detection is **not** reinvented here. The production chain tracker is
188
+ Arcade, which wraps go-chaintracks; its reorg SSE is the source of truth:
189
+
190
+ - **`GET /v2/reorg/stream`** emits `data: <JSON>\n\n` frames (no `event:`/`id:`
191
+ lines; `: keepalive` comments between events). Each frame is a `ReorgEvent`:
192
+
193
+ ```jsonc
194
+ {
195
+ "orphanedHashes": ["<blockHash>", ...], // blocks removed from the active chain
196
+ "commonAncestor": { "height": 100, "hash": "..." },
197
+ "newTip": { "height": 103, "hash": "..." },
198
+ "depth": 3
199
+ }
200
+ ```
201
+
202
+ - The same go-chaintracks service answers the merkle-root checks
203
+ (`isValidRootForHeight`) the engine already trusts for all SPV verification.
204
+
205
+ `packages/overlays/overlay-express/src/ReorgStream.ts` consumes this stream and
206
+ maps each event to the engine: `orphanedHashes` → blocks to reconcile,
207
+ `commonAncestor.height + 1` → rebuild floor, `newTip.height` → rebuild ceiling.
208
+ Block hashes are lower-cased to match stored display-hex hashes (go-sdk
209
+ `chainhash.Hash` marshals as reversed/display hex).
210
+
211
+ > **No replay on this stream.** It carries no event ids, so a reorg that fires
212
+ > while the SSE client is disconnected is lost. Every (re)connect therefore runs a
213
+ > catch-up revalidation sweep (below), and the periodic block poll runs the same
214
+ > sweep as a standing fallback for chain trackers that expose no reorg stream.
215
+
216
+ ### What the engine does on a reorg — `Engine.handleReorg`
217
+
218
+ 1. **Demote orphaned admissions.** For each orphaned block hash, every **proven**
219
+ `applied_transactions` row anchored to it is demoted to unproven
220
+ (`demoteAppliedTransactionToUnproven`): block height/hash/index/merkleRoot are
221
+ cleared and `proven` is set to `false`, **but `firstSeenHeight` is kept**. The
222
+ transaction immediately leaves the admitted set.
223
+ 2. **Rebuild affected anchor chains.** Every topic whose chain intersects the
224
+ reorged height range is rebuilt over the **canonical** block hashes (the rebuild
225
+ forces header re-resolution rather than reusing the stale stored hash). Topics
226
+ with no demoted transaction are rebuilt too, because the canonical block hash
227
+ for those heights changed and the `tac` must re-chain over it.
228
+
229
+ `handleReorg` is **idempotent**: a clean window demotes nothing and reproduces an
230
+ identical TAC, so it is safe to run on every reorg event, SSE reconnect, and poll.
231
+
232
+ ### Fallback / catch-up — `Engine.revalidateRecentAnchors(depth = 3)`
233
+
234
+ Scans proven `applied_transactions` in `[tip - depth + 1, tip]`. Any row whose
235
+ proof root no longer validates (`isValidRootForHeight`) or whose block hash
236
+ diverges from the canonical header is treated as orphaned and fed into
237
+ `handleReorg`. This is the reorg path for chain trackers without a reorg stream,
238
+ and the catch-up step on every SSE reconnect. Default depth is 3 blocks
239
+ (`reorgScanDepth`, configurable via `configureReorgStream`).
240
+
241
+ ### Interaction with admin/janitor removal
242
+
243
+ Reorg recovery and lookup-layer removal are independent and do not interfere:
244
+
245
+ - Removal/bans touch only the lookup layer; they never demote `applied_transactions`.
246
+ - Reorg demotion touches only the admitted set (proven → unproven) via the chain
247
+ tracker; it never consults the ban list or lookup index.
248
+
249
+ A demoted transaction follows the engine's existing unproven lifecycle: if it is
250
+ re-mined, Arcade/ARC re-notifies `/arc-ingest` → `handleNewMerkleProof` re-proves
251
+ it at its new height and the anchor includes it again; if it is never re-mined,
252
+ `evictUnprovenTransactions` removes it after the threshold. The "we received it"
253
+ record survives until one of those resolves.
254
+
255
+ ### Configuration
256
+
257
+ ```ts
258
+ server.configureReorgStream('https://arcade.example/v2/reorg/stream', 3)
259
+ ```
260
+
261
+ - `reorgStreamUrl` — when set, the SSE adapter reconciles reorgs in real time.
262
+ - `reorgScanDepth` — sweep depth from tip (default `3`).
263
+ - The block poll (`basmBlockPollIntervalMs`) runs the sweep as a fallback even
264
+ when no stream is configured.
265
+
266
+ ## Verification checklist for operators
267
+
268
+ 1. After removing a token, confirm its `applied_transactions` row still exists
269
+ (`proven: true`) and the `topic_block_anchors` row for its height is unchanged.
270
+ 2. Confirm `TAC(topic, tip)` is unchanged by the removal.
271
+ 3. Confirm the token no longer appears in lookup responses (and, if banned, does
272
+ not reappear after a GASP re-sync).
273
+
274
+ If all three hold, removal is behaving correctly: the node no longer serves the
275
+ token, yet can still prove it was received and admitted.
276
+
277
+ ### Reorg verification
278
+
279
+ 4. After a reorg orphans an admitted transaction's block, confirm its
280
+ `applied_transactions` row is demoted (`proven: false`, block fields cleared,
281
+ `firstSeenHeight` retained) and the affected `topic_block_anchors` rows now
282
+ carry the canonical block hashes with a recomputed `tac`.
283
+ 5. Confirm `TAC(topic, tip)` reconverges with peers that observed the same reorg.
284
+ 6. If the orphaned transaction is re-mined, confirm `/arc-ingest` re-proves it and
285
+ the anchor re-includes it; otherwise confirm it is eventually evicted as unproven.
package/docs/README.md CHANGED
@@ -5,6 +5,8 @@ The documentation is split into various pages, each covering a set of related fu
5
5
  ## Main Documentation
6
6
 
7
7
  - [API](./API.md) — How to interact with the Overlay Services Engine
8
+ - [Synchronization](./Synchronization.md) — GASP peer synchronization
9
+ - [BRC-136 BASM](./BRC-136-BASM.md) — Block-anchored sync, and how it interacts with admin/janitor removal
8
10
  - [Concepts](./concepts/README.md) — High-level concepts
9
11
  - [Examples](./examples/README.md) — Guides, tutorials and examples of using the Engine
10
12
  - [Internal](./internal/README.md) — References for working with internal system components
@@ -200,4 +200,13 @@ Example (adapt the table/query to your schema):
200
200
  +-------------+------+
201
201
  ```
202
202
 
203
- **What you’re looking for:** the topic(s) you care about (e.g. tm_plite, tm_blockbeta) should move toward the same counts/data across nodes after sync, and your lookup results for identical queries should converge as well.
203
+ **What you’re looking for:** the topic(s) you care about (e.g. tm_plite, tm_blockbeta) should move toward the same counts/data across nodes after sync, and your lookup results for identical queries should converge as well.
204
+
205
+ > **Caveat (BRC-136 / janitor / admin removal):** Row-count comparison is only a
206
+ > valid convergence check on nodes that do **not** remove outputs locally. Once
207
+ > the janitor or `/admin/remove-token` is active, lookup-set counts can
208
+ > legitimately differ between peers even when they fully agree on what was
209
+ > admitted, because bans/removals are per-node policy and are not synchronized.
210
+ > To verify that two nodes agree on which transactions were admitted to a topic,
211
+ > compare their per-topic Topic Anchor Chain (`TAC`) at the tip rather than
212
+ > lookup row counts. See [BRC-136 BASM](./BRC-136-BASM.md).
package/mod.ts CHANGED
@@ -10,7 +10,31 @@ export type { TaggedBEEF, STEAK, LookupQuestion, LookupAnswer, AdmittanceInstruc
10
10
  export type { LookupFormula } from './src/LookupFormula.js'
11
11
  export type { Advertisement } from './src/Advertisement.js'
12
12
  export type { AdvertisementData, Advertiser } from './src/Advertiser.js'
13
+ export {
14
+ BASM_ZERO_HASH,
15
+ computeBasmRoot,
16
+ computeTac,
17
+ extractMerkleProofMetadata
18
+ } from './src/BASM.js'
19
+ export type {
20
+ AdmittedListRequest,
21
+ AdmittedListResponse,
22
+ AdmittedTxRef,
23
+ BASMPeerSyncReport,
24
+ CompoundMerklePathRequest,
25
+ CompoundMerklePathResponse,
26
+ MerkleProofMetadata,
27
+ RawTransactionRecord,
28
+ RawTransactionRequest,
29
+ RawTransactionResponse,
30
+ TopicAnchorHeader,
31
+ TopicAnchorHeaderResolver,
32
+ TopicAnchorRangeRequest,
33
+ TopicAnchorRangeResponse,
34
+ TopicAnchorTip,
35
+ TopicBlockAnchor
36
+ } from './src/BASM.js'
13
37
 
14
38
  // The Knex storage system
15
39
  export { KnexStorage } from './src/storage/knex/KnexStorage.js'
16
- export * as KnexStorageMigrations from './src/storage/knex/all-migrations.js'
40
+ export * as KnexStorageMigrations from './src/storage/knex/all-migrations.js'
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bsv/overlay",
3
- "version": "2.0.2",
3
+ "version": "2.0.4",
4
4
  "type": "module",
5
5
  "description": "BSV Blockchain Overlay Services Engine",
6
6
  "main": "dist/cjs/mod.js",
@@ -35,20 +35,10 @@
35
35
  "types": "./dist/types/src/storage/*.d.ts"
36
36
  }
37
37
  },
38
- "scripts": {
39
- "test": "npm run build && jest",
40
- "test:watch": "npm run build && jest --watch",
41
- "test:coverage": "npm run build && jest --coverage",
42
- "lint": "ts-standard --fix src/**/*.ts",
43
- "lint:ci": "ts-standard src/**/*.ts",
44
- "build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json",
45
- "dev": "tsc -b -w",
46
- "prepublish": "npm run build",
47
- "doc": "ts2md --inputFilename=mod.ts --outputFilename=docs/API.md --filenameSubstring=API --firstHeadingLevel=1"
48
- },
49
38
  "repository": {
50
39
  "type": "git",
51
- "url": "git+https://github.com/bsv-blockchain/overlay-services.git"
40
+ "url": "git+https://github.com/bsv-blockchain/ts-stack.git",
41
+ "directory": "packages/overlays/overlay"
52
42
  },
53
43
  "keywords": [
54
44
  "BSV",
@@ -60,21 +50,33 @@
60
50
  "author": "BSV Association",
61
51
  "license": "SEE LICENSE IN LICENSE.txt",
62
52
  "bugs": {
63
- "url": "https://github.com/bitcoin-sv/overlay-services/issues"
53
+ "url": "https://github.com/bsv-blockchain/ts-stack/issues"
64
54
  },
65
- "homepage": "https://github.com/bitcoin-sv/overlay-services#readme",
55
+ "homepage": "https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay#readme",
66
56
  "devDependencies": {
67
- "@types/jest": "^29.5.12",
68
- "jest": "^29.7.0",
69
- "ts-jest": "^29.1.1",
57
+ "@types/jest": "^30.0.0",
58
+ "@types/node": "^25.9.3",
59
+ "jest": "^30.4.2",
60
+ "ts-jest": "^29.4.11",
70
61
  "ts-standard": "^12.0.2",
71
62
  "ts2md": "^0.2.0",
72
63
  "tsconfig-to-dual-package": "^1.2.0",
73
- "typescript": "^5.2.2"
64
+ "typescript": "^6.0.3"
74
65
  },
75
66
  "dependencies": {
76
- "@bsv/gasp": "^1.2.2",
77
- "@bsv/sdk": "^2.0.4",
78
- "knex": "^3.1.0"
67
+ "@bsv/gasp": "^1.2.3",
68
+ "@bsv/sdk": "^2.1.4",
69
+ "knex": "^3.2.10"
70
+ },
71
+ "scripts": {
72
+ "test": "npm run build && jest",
73
+ "test:watch": "npm run build && jest --watch",
74
+ "test:coverage": "npm run build && jest --coverage",
75
+ "lint": "ts-standard --fix src/**/*.ts",
76
+ "lint:ci": "ts-standard src/**/*.ts",
77
+ "build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json",
78
+ "dev": "tsc -b -w",
79
+ "prepublish": "npm run build",
80
+ "doc": "ts2md --inputFilename=mod.ts --outputFilename=docs/API.md --filenameSubstring=API --firstHeadingLevel=1"
79
81
  }
80
- }
82
+ }
package/src/BASM.ts ADDED
@@ -0,0 +1,208 @@
1
+ import { createHash } from 'node:crypto'
2
+ import { MerklePath } from '@bsv/sdk'
3
+
4
+ export const BASM_ZERO_HASH = '0000000000000000000000000000000000000000000000000000000000000000'
5
+
6
+ export interface TopicBlockAnchor {
7
+ topic: string
8
+ blockHeight: number
9
+ blockHash: string
10
+ basmRoot: string
11
+ admittedCount: number
12
+ tac: string
13
+ }
14
+
15
+ export interface TopicAnchorTip {
16
+ topic: string
17
+ blockHeight: number
18
+ blockHash?: string
19
+ basmRoot?: string
20
+ admittedCount?: number
21
+ tac: string
22
+ }
23
+
24
+ export interface AdmittedTxRef {
25
+ txid: string
26
+ blockIndex: number
27
+ }
28
+
29
+ export interface RawTransactionRecord {
30
+ txid: string
31
+ rawTx: string
32
+ }
33
+
34
+ export interface RawTransactionRequest {
35
+ txids: string[]
36
+ }
37
+
38
+ export interface RawTransactionResponse {
39
+ transactions: RawTransactionRecord[]
40
+ missing: string[]
41
+ }
42
+
43
+ export interface TopicAnchorHeader {
44
+ blockHeight: number
45
+ blockHash: string
46
+ merkleRoot?: string
47
+ }
48
+
49
+ export type TopicAnchorHeaderResolver = (blockHeight: number) => Promise<TopicAnchorHeader | undefined>
50
+
51
+ export interface TopicAnchorRangeRequest {
52
+ fromHeight: number
53
+ toHeight: number
54
+ }
55
+
56
+ export interface TopicAnchorRangeResponse {
57
+ topic: string
58
+ anchors: TopicBlockAnchor[]
59
+ }
60
+
61
+ export interface AdmittedListRequest {
62
+ blockHeight: number
63
+ blockHash?: string
64
+ }
65
+
66
+ export interface AdmittedListResponse {
67
+ topic: string
68
+ blockHeight: number
69
+ blockHash?: string
70
+ admitted: AdmittedTxRef[]
71
+ }
72
+
73
+ export interface CompoundMerklePathRequest {
74
+ blockHeight: number
75
+ txids: string[]
76
+ }
77
+
78
+ export interface CompoundMerklePathResponse {
79
+ topic: string
80
+ blockHeight: number
81
+ txids: string[]
82
+ merklePath: string
83
+ }
84
+
85
+ export interface MerkleProofMetadata {
86
+ blockHeight: number
87
+ blockIndex: number
88
+ merkleRoot: string
89
+ }
90
+
91
+ export interface ReorgReport {
92
+ perTopic: Array<{
93
+ topic: string
94
+ demotedTxids: string[]
95
+ rebuiltFrom: number
96
+ rebuiltTo: number
97
+ }>
98
+ }
99
+
100
+ export interface BASMPeerSyncReport {
101
+ topic: string
102
+ endpoint: string
103
+ status: 'matched' | 'advanced' | 'diverged' | 'skipped' | 'error'
104
+ localTip?: TopicAnchorTip
105
+ remoteTip?: TopicAnchorTip
106
+ checkedHeights: number[]
107
+ missingTxids: string[]
108
+ fetchedTxCount: number
109
+ message?: string
110
+ }
111
+
112
+ type AdmittedTxLike = string | AdmittedTxRef
113
+
114
+ function sha256d(buffer: Buffer): Buffer {
115
+ const first = createHash('sha256').update(buffer).digest()
116
+ return createHash('sha256').update(first).digest()
117
+ }
118
+
119
+ function assertHashHex(hash: string, label: string): void {
120
+ if (!/^[0-9a-fA-F]{64}$/.test(hash)) {
121
+ throw new Error(`${label} must be 32 bytes of hex`)
122
+ }
123
+ }
124
+
125
+ function displayHexToInternal(hash: string): Buffer {
126
+ assertHashHex(hash, 'hash')
127
+ return Buffer.from(hash, 'hex').reverse()
128
+ }
129
+
130
+ function internalToDisplayHex(hash: Buffer): string {
131
+ return Buffer.from(hash).reverse().toString('hex')
132
+ }
133
+
134
+ function normalizeAdmittedTxids(admitted: AdmittedTxLike[]): string[] {
135
+ return admitted
136
+ .map((item, originalIndex) => {
137
+ if (typeof item === 'string') {
138
+ return { txid: item, blockIndex: originalIndex }
139
+ }
140
+ return item
141
+ })
142
+ .sort((a, b) => a.blockIndex - b.blockIndex)
143
+ .map(item => item.txid.toLowerCase())
144
+ }
145
+
146
+ /**
147
+ * Computes a BRC-136 BASM root from admitted topic txids.
148
+ *
149
+ * The API accepts normal display-order txid hex because that is what the BSV
150
+ * TypeScript stack exposes at its public boundaries. Hashing is performed on
151
+ * internal byte order as required by the BRC, and the returned root is display
152
+ * order for JSON/wire compatibility.
153
+ */
154
+ export function computeBasmRoot(admitted: AdmittedTxLike[]): string {
155
+ const txids = normalizeAdmittedTxids(admitted)
156
+
157
+ if (txids.length === 0) {
158
+ return BASM_ZERO_HASH
159
+ }
160
+
161
+ let layer = txids.map(txid => displayHexToInternal(txid))
162
+ if (layer.length === 1) {
163
+ return internalToDisplayHex(layer[0])
164
+ }
165
+
166
+ while (layer.length > 1) {
167
+ const next: Buffer[] = []
168
+ for (let i = 0; i < layer.length; i += 2) {
169
+ const left = layer[i]
170
+ const right = i + 1 < layer.length ? layer[i + 1] : left
171
+ next.push(sha256d(Buffer.concat([left, right])))
172
+ }
173
+ layer = next
174
+ }
175
+
176
+ return internalToDisplayHex(layer[0])
177
+ }
178
+
179
+ /**
180
+ * Computes the BRC-136 cumulative Topic Anchor Chain hash:
181
+ * SHA256d(prevTac || blockHash || basmRoot), with all inputs reversed to
182
+ * internal byte order before hashing and the output returned as display hex.
183
+ */
184
+ export function computeTac(prevTac: string, blockHash: string, basmRoot: string): string {
185
+ const input = Buffer.concat([
186
+ displayHexToInternal(prevTac.toLowerCase()),
187
+ displayHexToInternal(blockHash.toLowerCase()),
188
+ displayHexToInternal(basmRoot.toLowerCase())
189
+ ])
190
+ return internalToDisplayHex(sha256d(input))
191
+ }
192
+
193
+ export function extractMerkleProofMetadata(txid: string, proof?: MerklePath): MerkleProofMetadata | undefined {
194
+ if (proof === undefined) {
195
+ return undefined
196
+ }
197
+
198
+ const leaf = proof.path[0]?.find(candidate => candidate.hash === txid)
199
+ if (leaf === undefined) {
200
+ return undefined
201
+ }
202
+
203
+ return {
204
+ blockHeight: proof.blockHeight,
205
+ blockIndex: leaf.offset,
206
+ merkleRoot: proof.computeRoot(txid)
207
+ }
208
+ }
@@ -0,0 +1,54 @@
1
+ import type {
2
+ AdmittedListResponse,
3
+ CompoundMerklePathResponse,
4
+ RawTransactionResponse,
5
+ TopicAnchorRangeResponse,
6
+ TopicAnchorTip
7
+ } from './BASM.js'
8
+
9
+ export class BASMRemote {
10
+ constructor(
11
+ private readonly endpoint: string,
12
+ private readonly topic: string,
13
+ private readonly fetchImpl: typeof fetch = fetch.bind(globalThis)
14
+ ) { }
15
+
16
+ async requestTopicAnchorTip(): Promise<TopicAnchorTip> {
17
+ return await this.post<TopicAnchorTip>('/requestTopicAnchorTip', {})
18
+ }
19
+
20
+ async requestTopicAnchorRange(fromHeight: number, toHeight: number): Promise<TopicAnchorRangeResponse> {
21
+ return await this.post<TopicAnchorRangeResponse>('/requestTopicAnchorRange', { fromHeight, toHeight })
22
+ }
23
+
24
+ async requestAdmittedList(blockHeight: number, blockHash?: string): Promise<AdmittedListResponse> {
25
+ return await this.post<AdmittedListResponse>('/requestAdmittedList', { blockHeight, blockHash })
26
+ }
27
+
28
+ async requestCompoundMerklePath(blockHeight: number, txids: string[]): Promise<CompoundMerklePathResponse> {
29
+ return await this.post<CompoundMerklePathResponse>('/requestCompoundMerklePath', { blockHeight, txids })
30
+ }
31
+
32
+ async requestRawTransactions(txids: string[]): Promise<RawTransactionResponse> {
33
+ return await this.post<RawTransactionResponse>('/requestRawTransactions', { txids })
34
+ }
35
+
36
+ private async post<T>(path: string, body: unknown): Promise<T> {
37
+ const response = await this.fetchImpl(new URL(path, this.endpoint).toString(), {
38
+ method: 'POST',
39
+ headers: {
40
+ Accept: 'application/json',
41
+ 'Content-Type': 'application/json',
42
+ 'x-bsv-topic': this.topic
43
+ },
44
+ body: JSON.stringify(body)
45
+ })
46
+
47
+ const text = await response.text()
48
+ if (!response.ok) {
49
+ throw new Error(`BASM peer ${this.endpoint} returned ${response.status}: ${text}`)
50
+ }
51
+
52
+ return (text.length === 0 ? {} : JSON.parse(text)) as T
53
+ }
54
+ }