@bsv/overlay 2.0.3 → 2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +226 -106
- package/dist/cjs/mod.js +6 -1
- package/dist/cjs/mod.js.map +1 -1
- package/dist/cjs/package.json +13 -11
- package/dist/cjs/src/BASM.js +92 -0
- package/dist/cjs/src/BASM.js.map +1 -0
- package/dist/cjs/src/BASMRemote.js +43 -0
- package/dist/cjs/src/BASMRemote.js.map +1 -0
- package/dist/cjs/src/Engine.js +649 -47
- package/dist/cjs/src/Engine.js.map +1 -1
- package/dist/cjs/src/GASP/OverlayGASPRemote.js +2 -2
- package/dist/cjs/src/GASP/OverlayGASPRemote.js.map +1 -1
- package/dist/cjs/src/GASP/OverlayGASPStorage.js +18 -25
- package/dist/cjs/src/GASP/OverlayGASPStorage.js.map +1 -1
- package/dist/cjs/src/storage/knex/KnexStorage.js +329 -12
- package/dist/cjs/src/storage/knex/KnexStorage.js.map +1 -1
- package/dist/cjs/src/storage/knex/all-migrations.js +3 -1
- package/dist/cjs/src/storage/knex/all-migrations.js.map +1 -1
- package/dist/cjs/src/storage/knex/migrations/2025-05-28-001-enlarge.js +2 -24
- package/dist/cjs/src/storage/knex/migrations/2025-05-28-001-enlarge.js.map +1 -1
- package/dist/cjs/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js +114 -0
- package/dist/cjs/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js.map +1 -0
- package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
- package/dist/esm/mod.js +1 -0
- package/dist/esm/mod.js.map +1 -1
- package/dist/esm/src/BASM.js +85 -0
- package/dist/esm/src/BASM.js.map +1 -0
- package/dist/esm/src/BASMRemote.js +42 -0
- package/dist/esm/src/BASMRemote.js.map +1 -0
- package/dist/esm/src/Engine.js +637 -46
- package/dist/esm/src/Engine.js.map +1 -1
- package/dist/esm/src/GASP/OverlayGASPRemote.js +2 -2
- package/dist/esm/src/GASP/OverlayGASPRemote.js.map +1 -1
- package/dist/esm/src/GASP/OverlayGASPStorage.js +18 -25
- package/dist/esm/src/GASP/OverlayGASPStorage.js.map +1 -1
- package/dist/esm/src/storage/knex/KnexStorage.js +323 -12
- package/dist/esm/src/storage/knex/KnexStorage.js.map +1 -1
- package/dist/esm/src/storage/knex/all-migrations.js +3 -1
- package/dist/esm/src/storage/knex/all-migrations.js.map +1 -1
- package/dist/esm/src/storage/knex/migrations/2025-05-28-001-enlarge.js +2 -24
- package/dist/esm/src/storage/knex/migrations/2025-05-28-001-enlarge.js.map +1 -1
- package/dist/esm/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js +109 -0
- package/dist/esm/src/storage/knex/migrations/2026-05-29-001-brc136-basm.js.map +1 -0
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/types/mod.d.ts +2 -0
- package/dist/types/mod.d.ts.map +1 -1
- package/dist/types/src/BASM.d.ts +110 -0
- package/dist/types/src/BASM.d.ts.map +1 -0
- package/dist/types/src/BASMRemote.d.ts +14 -0
- package/dist/types/src/BASMRemote.d.ts.map +1 -0
- package/dist/types/src/Engine.d.ts +78 -1
- package/dist/types/src/Engine.d.ts.map +1 -1
- package/dist/types/src/GASP/OverlayGASPStorage.d.ts +1 -1
- package/dist/types/src/GASP/OverlayGASPStorage.d.ts.map +1 -1
- package/dist/types/src/LookupService.d.ts.map +1 -1
- package/dist/types/src/storage/Storage.d.ts +118 -0
- package/dist/types/src/storage/Storage.d.ts.map +1 -1
- package/dist/types/src/storage/knex/KnexStorage.d.ts +38 -6
- package/dist/types/src/storage/knex/KnexStorage.d.ts.map +1 -1
- package/dist/types/src/storage/knex/all-migrations.d.ts.map +1 -1
- package/dist/types/src/storage/knex/migrations/2025-05-28-001-enlarge.d.ts.map +1 -1
- package/dist/types/src/storage/knex/migrations/2026-05-29-001-brc136-basm.d.ts +4 -0
- package/dist/types/src/storage/knex/migrations/2026-05-29-001-brc136-basm.d.ts.map +1 -0
- package/dist/types/tsconfig.types.tsbuildinfo +1 -1
- package/docs/API.md +72 -580
- package/docs/BRC-136-BASM.md +285 -0
- package/docs/README.md +2 -0
- package/docs/Synchronization.md +10 -1
- package/docs/examples/README.md +2 -2
- package/docs/examples/gs-wip.md +68 -76
- package/mod.ts +25 -1
- package/package.json +25 -23
- package/src/BASM.ts +208 -0
- package/src/BASMRemote.ts +54 -0
- package/src/Engine.ts +762 -47
- package/src/GASP/OverlayGASPRemote.ts +2 -2
- package/src/GASP/OverlayGASPStorage.ts +17 -22
- package/src/LookupService.ts +2 -1
- package/src/__tests/BASM.test.ts +44 -0
- package/src/__tests/BASMChain.test.ts +290 -0
- package/src/__tests/Engine.test.ts +3 -4
- package/src/storage/Storage.ts +127 -0
- package/src/storage/knex/KnexStorage.ts +376 -14
- package/src/storage/knex/all-migrations.ts +3 -1
- package/src/storage/knex/migrations/2025-05-28-001-enlarge.ts +2 -22
- package/src/storage/knex/migrations/2026-05-29-001-brc136-basm.ts +123 -0
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
# BRC-136 BASM Synchronization & Admin/Janitor Removal
|
|
2
|
+
|
|
3
|
+
This document explains how the BRC-136 Block-Anchored synchronization layer
|
|
4
|
+
([spec](https://bsv.brc.dev/overlays/0136)) is implemented in this engine, and —
|
|
5
|
+
critically — **how it interacts with admin and janitor removal of outputs from a
|
|
6
|
+
topic**. The two systems are deliberately decoupled. Read the final section
|
|
7
|
+
("How BASM and removal interact") before operating a node that runs the janitor
|
|
8
|
+
or removes tokens via the admin API.
|
|
9
|
+
|
|
10
|
+
## What BRC-136 gives you
|
|
11
|
+
|
|
12
|
+
GASP ([Synchronization.md](./Synchronization.md)) replicates transaction *data*
|
|
13
|
+
between peers. BRC-136 sits alongside it and answers a different question:
|
|
14
|
+
|
|
15
|
+
> Do two overlays **agree about which transactions were admitted to a topic** at
|
|
16
|
+
> every block height, from the topic's genesis to the chain tip?
|
|
17
|
+
|
|
18
|
+
It does this with a per-topic **Topic Anchor Chain (TAC)** — a cumulative hash
|
|
19
|
+
that commits to every block's admitted-transaction set. If two nodes report the
|
|
20
|
+
same `TAC(topic, H)`, they provably agree on every admitted transaction for that
|
|
21
|
+
topic from genesis through height `H`. A single round trip detects agreement; a
|
|
22
|
+
binary search over heights localizes any divergence in `O(log H)` rounds.
|
|
23
|
+
|
|
24
|
+
### Core data structures
|
|
25
|
+
|
|
26
|
+
| Concept | Type | Source of truth |
|
|
27
|
+
|---|---|---|
|
|
28
|
+
| Admitted set at a block | `AdmittedTxRef[]` | `applied_transactions` table |
|
|
29
|
+
| Block anchor | `TopicBlockAnchor` | `topic_block_anchors` table |
|
|
30
|
+
| Cumulative chain hash | `tac` field on the anchor | derived |
|
|
31
|
+
|
|
32
|
+
- **`basmRoot`** — Merkle root over the admitted txids at one block height, in
|
|
33
|
+
canonical block order. Empty set → 32 zero bytes; single txid → the txid
|
|
34
|
+
itself (unhashed); multiple → `SHA256d` binary tree with Bitcoin odd-leaf
|
|
35
|
+
duplication. See `computeBasmRoot` in `src/BASM.ts`.
|
|
36
|
+
- **`tac`** — `SHA256d(prevTac ‖ blockHash ‖ basmRoot)`, all inputs in internal
|
|
37
|
+
byte order. See `computeTac` in `src/BASM.ts`.
|
|
38
|
+
|
|
39
|
+
### Gap-free chains
|
|
40
|
+
|
|
41
|
+
The TAC must advance on **every** block — including blocks with no admitted
|
|
42
|
+
transactions — or peers could not distinguish "no transactions here" from "I
|
|
43
|
+
haven't synced this height." The engine keeps chains contiguous:
|
|
44
|
+
|
|
45
|
+
- `advanceTopicAnchorChains(toHeight)` extends each topic forward with empty
|
|
46
|
+
anchors (zero root, zero count) up to `toHeight`.
|
|
47
|
+
- `rebuildTopicAnchorChain(...)` / `recomputeTopicBlockAnchor(...)` rebuild a
|
|
48
|
+
contiguous slice after an admission (including out-of-order proofs), filling
|
|
49
|
+
any missing heights with empty anchors rather than leaving a hole.
|
|
50
|
+
|
|
51
|
+
A chain only *starts* at the topic's first admitted height (its genesis).
|
|
52
|
+
Covered by `src/__tests/BASMChain.test.ts`.
|
|
53
|
+
|
|
54
|
+
## The admitted set is a historical fact, not current UTXO state
|
|
55
|
+
|
|
56
|
+
This is the key design decision, and the source of the assurance this layer
|
|
57
|
+
provides.
|
|
58
|
+
|
|
59
|
+
The admitted set for `(topic, height)` is read by
|
|
60
|
+
`Storage.findAdmittedTransactionsForBlock`, which queries:
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
applied_transactions WHERE topic = ? AND blockHeight = ? AND proven = true AND blockIndex IS NOT NULL
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
It is **append-only for proven transactions**. A row lands here when a
|
|
67
|
+
transaction is admitted to the topic and has a merkle proof
|
|
68
|
+
(`buildAppliedTransactionRecord` sets `proven: true` only when proof metadata is
|
|
69
|
+
present). From that point the row — and therefore the `basmRoot` and `tac` that
|
|
70
|
+
commit to it — is an immutable record of "this overlay received and admitted
|
|
71
|
+
this txid at this block height."
|
|
72
|
+
|
|
73
|
+
It is **not** derived from current unspent outputs. Spending an output, pruning
|
|
74
|
+
its history, or removing it from lookup does not change the admitted set.
|
|
75
|
+
|
|
76
|
+
## The lookup set is separate, mutable, and policy-driven
|
|
77
|
+
|
|
78
|
+
What an overlay *serves* in lookup responses is a different set, in different
|
|
79
|
+
storage:
|
|
80
|
+
|
|
81
|
+
- The engine `outputs` table, filtered to `spent: false`.
|
|
82
|
+
- The lookup service's own index (e.g. the Mongo `shipRecords` / `slapRecords`
|
|
83
|
+
collections for the discovery services).
|
|
84
|
+
|
|
85
|
+
This set is mutable by design. Outputs get marked spent, pruned from history,
|
|
86
|
+
removed by an admin, or deleted by the janitor.
|
|
87
|
+
|
|
88
|
+
## Removal paths — exactly what each one touches
|
|
89
|
+
|
|
90
|
+
| Path | Code | Touches lookup set | Touches admitted set (`applied_transactions`) |
|
|
91
|
+
|---|---|---|---|
|
|
92
|
+
| Spend / history prune | `Engine.deleteUTXODeep` → `deleteOutput` | yes | **no** |
|
|
93
|
+
| Admin remove-token | `/admin/remove-token` → `evictFromServices` → `LookupService.outputEvicted` | yes (lookup index) | **no** |
|
|
94
|
+
| Janitor (dead SHIP/SLAP host) | `JanitorService.handleUnhealthyOutput` → `deleteOne` + ban | yes (lookup index) | **no** |
|
|
95
|
+
| Ban enforcement | `BanAwareLookupWrapper.outputAdmittedByTopic` | blocks re-entry into lookup index on GASP re-sync | **no** |
|
|
96
|
+
| Unproven eviction | `Engine.evictUnprovenTransactions` → `deleteAppliedTransaction` | yes | yes — **but only `proven: false` rows** |
|
|
97
|
+
|
|
98
|
+
Notes:
|
|
99
|
+
|
|
100
|
+
- **`deleteUTXODeep` never deletes the applied-transaction record.** It calls
|
|
101
|
+
`deleteOutput` only. The admission proof survives a normal spend or prune.
|
|
102
|
+
- **Admin and janitor removal operate on the lookup layer only.** Admin
|
|
103
|
+
`evictFromServices` calls `LookupService.outputEvicted`; the janitor deletes
|
|
104
|
+
from its own Mongo collection and writes to the ban list. Neither calls
|
|
105
|
+
`deleteAppliedTransaction`, `deleteOutput` on proven engine state, or anything
|
|
106
|
+
that mutates `topic_block_anchors`.
|
|
107
|
+
- **The ban list is a lookup-layer filter.** `BanAwareLookupWrapper` blocks
|
|
108
|
+
banned outpoints/domains at `outputAdmittedByTopic`, preventing a removed token
|
|
109
|
+
from re-entering the *lookup index* when GASP re-syncs it from a peer. It does
|
|
110
|
+
not — and must not — block the BRC-136 admitted record.
|
|
111
|
+
- **`evictUnprovenTransactions` is the only path that deletes an
|
|
112
|
+
applied-transaction row**, and it filters to `proven: false`. Such rows never
|
|
113
|
+
appear in any anchor (the anchor query requires `proven: true`), so evicting
|
|
114
|
+
them cannot change any `basmRoot` or `tac`.
|
|
115
|
+
|
|
116
|
+
## How BASM and removal interact
|
|
117
|
+
|
|
118
|
+
This is the question operators most need answered.
|
|
119
|
+
|
|
120
|
+
### You can always prove you received and processed a token
|
|
121
|
+
|
|
122
|
+
Even after the janitor deletes a dead host's advertisement, or an admin removes a
|
|
123
|
+
malicious output via `/admin/remove-token` (with or without a ban), the overlay
|
|
124
|
+
**retains full proof that it received and admitted that token**:
|
|
125
|
+
|
|
126
|
+
- the `applied_transactions` row (`proven: true`, with `blockHeight`,
|
|
127
|
+
`blockIndex`, `blockHash`, `merkleRoot`),
|
|
128
|
+
- the `topic_block_anchors` row whose `basmRoot` commits to that txid, and
|
|
129
|
+
- the ability to serve a compound merkle path for it via the BRC-136 endpoints.
|
|
130
|
+
|
|
131
|
+
Removal is a **lookup-serving policy decision**. It changes what the node returns
|
|
132
|
+
to lookup queries. It does not, and cannot, rewrite the historical admitted set
|
|
133
|
+
or the TAC.
|
|
134
|
+
|
|
135
|
+
### TAC agreement means agreement on *admission*, not on *lookup results*
|
|
136
|
+
|
|
137
|
+
This is the most important operational caveat:
|
|
138
|
+
|
|
139
|
+
> **A matching TAC proves two overlays agree on which transactions were admitted.
|
|
140
|
+
> It does NOT prove they will return identical lookup results.**
|
|
141
|
+
|
|
142
|
+
Bans and removals are **per-node policy and are not synchronized between peers**.
|
|
143
|
+
Two honest overlays running the same topic manager can therefore:
|
|
144
|
+
|
|
145
|
+
- report the **same** `TAC(topic, H)` (identical admitted sets), while
|
|
146
|
+
- returning **different** lookup results — because one node banned a malicious
|
|
147
|
+
output that the other still serves.
|
|
148
|
+
|
|
149
|
+
This is correct and intended. BRC-136 anchors *receipt and admission*, which is
|
|
150
|
+
an objective, deterministic function of chain state and the topic manager. It
|
|
151
|
+
intentionally does not anchor each node's local moderation policy.
|
|
152
|
+
|
|
153
|
+
### Consequence: divergence diagnosis
|
|
154
|
+
|
|
155
|
+
- **TACs match** → the two nodes admitted exactly the same transactions. Any
|
|
156
|
+
difference in lookup output is explained by local removal/ban policy, not by a
|
|
157
|
+
sync problem.
|
|
158
|
+
- **TACs diverge** → the nodes genuinely disagree about admission at some height
|
|
159
|
+
(e.g. different topic-manager versions, a missed transaction, or a reorg
|
|
160
|
+
handled differently). This is a real sync issue; the binary-search localizes
|
|
161
|
+
the height.
|
|
162
|
+
|
|
163
|
+
Removing a token from lookup will **not** cause TAC divergence. If you ever see
|
|
164
|
+
TAC divergence after only running the janitor/admin removal, that is a bug —
|
|
165
|
+
something is incorrectly deleting proven `applied_transactions` rows or mutating
|
|
166
|
+
anchors.
|
|
167
|
+
|
|
168
|
+
### Caveat for the GASP validation guidance
|
|
169
|
+
|
|
170
|
+
[Synchronization.md](./Synchronization.md) suggests validating sync by comparing
|
|
171
|
+
per-topic SQL row counts across nodes. Once the janitor or admin removal is
|
|
172
|
+
active, **lookup-set counts can legitimately differ between nodes even when their
|
|
173
|
+
TACs match**. To validate BRC-136 agreement, compare `TAC(topic, tip)` between
|
|
174
|
+
peers — not lookup row counts.
|
|
175
|
+
|
|
176
|
+
## Reorg handling
|
|
177
|
+
|
|
178
|
+
A blockchain reorganization can orphan a block whose transactions were already
|
|
179
|
+
admitted and anchored. Since anchors and the admitted set are keyed by
|
|
180
|
+
`(topic, blockHeight, blockHash)`, an orphaned block would otherwise leave stale
|
|
181
|
+
`applied_transactions` rows and an anchor whose `basmRoot`/`tac` commit to a
|
|
182
|
+
block hash that is no longer canonical — permanently diverging the TAC from peers
|
|
183
|
+
that observed the reorg. The engine reconciles this automatically.
|
|
184
|
+
|
|
185
|
+
### Chaintracks is the reorg authority
|
|
186
|
+
|
|
187
|
+
Reorg detection is **not** reinvented here. The production chain tracker is
|
|
188
|
+
Arcade, which wraps go-chaintracks; its reorg SSE is the source of truth:
|
|
189
|
+
|
|
190
|
+
- **`GET /v2/reorg/stream`** emits `data: <JSON>\n\n` frames (no `event:`/`id:`
|
|
191
|
+
lines; `: keepalive` comments between events). Each frame is a `ReorgEvent`:
|
|
192
|
+
|
|
193
|
+
```jsonc
|
|
194
|
+
{
|
|
195
|
+
"orphanedHashes": ["<blockHash>", ...], // blocks removed from the active chain
|
|
196
|
+
"commonAncestor": { "height": 100, "hash": "..." },
|
|
197
|
+
"newTip": { "height": 103, "hash": "..." },
|
|
198
|
+
"depth": 3
|
|
199
|
+
}
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
- The same go-chaintracks service answers the merkle-root checks
|
|
203
|
+
(`isValidRootForHeight`) the engine already trusts for all SPV verification.
|
|
204
|
+
|
|
205
|
+
`packages/overlays/overlay-express/src/ReorgStream.ts` consumes this stream and
|
|
206
|
+
maps each event to the engine: `orphanedHashes` → blocks to reconcile,
|
|
207
|
+
`commonAncestor.height + 1` → rebuild floor, `newTip.height` → rebuild ceiling.
|
|
208
|
+
Block hashes are lower-cased to match stored display-hex hashes (go-sdk
|
|
209
|
+
`chainhash.Hash` marshals as reversed/display hex).
|
|
210
|
+
|
|
211
|
+
> **No replay on this stream.** It carries no event ids, so a reorg that fires
|
|
212
|
+
> while the SSE client is disconnected is lost. Every (re)connect therefore runs a
|
|
213
|
+
> catch-up revalidation sweep (below), and the periodic block poll runs the same
|
|
214
|
+
> sweep as a standing fallback for chain trackers that expose no reorg stream.
|
|
215
|
+
|
|
216
|
+
### What the engine does on a reorg — `Engine.handleReorg`
|
|
217
|
+
|
|
218
|
+
1. **Demote orphaned admissions.** For each orphaned block hash, every **proven**
|
|
219
|
+
`applied_transactions` row anchored to it is demoted to unproven
|
|
220
|
+
(`demoteAppliedTransactionToUnproven`): block height/hash/index/merkleRoot are
|
|
221
|
+
cleared and `proven` is set to `false`, **but `firstSeenHeight` is kept**. The
|
|
222
|
+
transaction immediately leaves the admitted set.
|
|
223
|
+
2. **Rebuild affected anchor chains.** Every topic whose chain intersects the
|
|
224
|
+
reorged height range is rebuilt over the **canonical** block hashes (the rebuild
|
|
225
|
+
forces header re-resolution rather than reusing the stale stored hash). Topics
|
|
226
|
+
with no demoted transaction are rebuilt too, because the canonical block hash
|
|
227
|
+
for those heights changed and the `tac` must re-chain over it.
|
|
228
|
+
|
|
229
|
+
`handleReorg` is **idempotent**: a clean window demotes nothing and reproduces an
|
|
230
|
+
identical TAC, so it is safe to run on every reorg event, SSE reconnect, and poll.
|
|
231
|
+
|
|
232
|
+
### Fallback / catch-up — `Engine.revalidateRecentAnchors(depth = 3)`
|
|
233
|
+
|
|
234
|
+
Scans proven `applied_transactions` in `[tip - depth + 1, tip]`. Any row whose
|
|
235
|
+
proof root no longer validates (`isValidRootForHeight`) or whose block hash
|
|
236
|
+
diverges from the canonical header is treated as orphaned and fed into
|
|
237
|
+
`handleReorg`. This is the reorg path for chain trackers without a reorg stream,
|
|
238
|
+
and the catch-up step on every SSE reconnect. Default depth is 3 blocks
|
|
239
|
+
(`reorgScanDepth`, configurable via `configureReorgStream`).
|
|
240
|
+
|
|
241
|
+
### Interaction with admin/janitor removal
|
|
242
|
+
|
|
243
|
+
Reorg recovery and lookup-layer removal are independent and do not interfere:
|
|
244
|
+
|
|
245
|
+
- Removal/bans touch only the lookup layer; they never demote `applied_transactions`.
|
|
246
|
+
- Reorg demotion touches only the admitted set (proven → unproven) via the chain
|
|
247
|
+
tracker; it never consults the ban list or lookup index.
|
|
248
|
+
|
|
249
|
+
A demoted transaction follows the engine's existing unproven lifecycle: if it is
|
|
250
|
+
re-mined, Arcade/ARC re-notifies `/arc-ingest` → `handleNewMerkleProof` re-proves
|
|
251
|
+
it at its new height and the anchor includes it again; if it is never re-mined,
|
|
252
|
+
`evictUnprovenTransactions` removes it after the threshold. The "we received it"
|
|
253
|
+
record survives until one of those resolves.
|
|
254
|
+
|
|
255
|
+
### Configuration
|
|
256
|
+
|
|
257
|
+
```ts
|
|
258
|
+
server.configureReorgStream('https://arcade.example/v2/reorg/stream', 3)
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
- `reorgStreamUrl` — when set, the SSE adapter reconciles reorgs in real time.
|
|
262
|
+
- `reorgScanDepth` — sweep depth from tip (default `3`).
|
|
263
|
+
- The block poll (`basmBlockPollIntervalMs`) runs the sweep as a fallback even
|
|
264
|
+
when no stream is configured.
|
|
265
|
+
|
|
266
|
+
## Verification checklist for operators
|
|
267
|
+
|
|
268
|
+
1. After removing a token, confirm its `applied_transactions` row still exists
|
|
269
|
+
(`proven: true`) and the `topic_block_anchors` row for its height is unchanged.
|
|
270
|
+
2. Confirm `TAC(topic, tip)` is unchanged by the removal.
|
|
271
|
+
3. Confirm the token no longer appears in lookup responses (and, if banned, does
|
|
272
|
+
not reappear after a GASP re-sync).
|
|
273
|
+
|
|
274
|
+
If all three hold, removal is behaving correctly: the node no longer serves the
|
|
275
|
+
token, yet can still prove it was received and admitted.
|
|
276
|
+
|
|
277
|
+
### Reorg verification
|
|
278
|
+
|
|
279
|
+
4. After a reorg orphans an admitted transaction's block, confirm its
|
|
280
|
+
`applied_transactions` row is demoted (`proven: false`, block fields cleared,
|
|
281
|
+
`firstSeenHeight` retained) and the affected `topic_block_anchors` rows now
|
|
282
|
+
carry the canonical block hashes with a recomputed `tac`.
|
|
283
|
+
5. Confirm `TAC(topic, tip)` reconverges with peers that observed the same reorg.
|
|
284
|
+
6. If the orphaned transaction is re-mined, confirm `/arc-ingest` re-proves it and
|
|
285
|
+
the anchor re-includes it; otherwise confirm it is eventually evicted as unproven.
|
package/docs/README.md
CHANGED
|
@@ -5,6 +5,8 @@ The documentation is split into various pages, each covering a set of related fu
|
|
|
5
5
|
## Main Documentation
|
|
6
6
|
|
|
7
7
|
- [API](./API.md) — How to interact with the Overlay Services Engine
|
|
8
|
+
- [Synchronization](./Synchronization.md) — GASP peer synchronization
|
|
9
|
+
- [BRC-136 BASM](./BRC-136-BASM.md) — Block-anchored sync, and how it interacts with admin/janitor removal
|
|
8
10
|
- [Concepts](./concepts/README.md) — High-level concepts
|
|
9
11
|
- [Examples](./examples/README.md) — Guides, tutorials and examples of using the Engine
|
|
10
12
|
- [Internal](./internal/README.md) — References for working with internal system components
|
package/docs/Synchronization.md
CHANGED
|
@@ -200,4 +200,13 @@ Example (adapt the table/query to your schema):
|
|
|
200
200
|
+-------------+------+
|
|
201
201
|
```
|
|
202
202
|
|
|
203
|
-
**What you’re looking for:** the topic(s) you care about (e.g. tm_plite, tm_blockbeta) should move toward the same counts/data across nodes after sync, and your lookup results for identical queries should converge as well.
|
|
203
|
+
**What you’re looking for:** the topic(s) you care about (e.g. tm_plite, tm_blockbeta) should move toward the same counts/data across nodes after sync, and your lookup results for identical queries should converge as well.
|
|
204
|
+
|
|
205
|
+
> **Caveat (BRC-136 / janitor / admin removal):** Row-count comparison is only a
|
|
206
|
+
> valid convergence check on nodes that do **not** remove outputs locally. Once
|
|
207
|
+
> the janitor or `/admin/remove-token` is active, lookup-set counts can
|
|
208
|
+
> legitimately differ between peers even when they fully agree on what was
|
|
209
|
+
> admitted, because bans/removals are per-node policy and are not synchronized.
|
|
210
|
+
> To verify that two nodes agree on which transactions were admitted to a topic,
|
|
211
|
+
> compare their per-topic Topic Anchor Chain (`TAC`) at the tip rather than
|
|
212
|
+
> lookup row counts. See [BRC-136 BASM](./BRC-136-BASM.md).
|
package/docs/examples/README.md
CHANGED
|
@@ -4,11 +4,11 @@
|
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
Here, you will find documentation for common example usages of the Overlay Services Engine.
|
|
7
|
+
Here, you will find documentation for common example usages of the Overlay Services Engine.
|
|
8
8
|
|
|
9
9
|
## Available Examples
|
|
10
10
|
|
|
11
|
-
- [Getting Started](./gs-wip.md) — Introduction and
|
|
11
|
+
- [Getting Started (WIP)](./gs-wip.md) — Introduction and getting started guide
|
|
12
12
|
|
|
13
13
|
---
|
|
14
14
|
|
package/docs/examples/gs-wip.md
CHANGED
|
@@ -6,114 +6,106 @@
|
|
|
6
6
|
|
|
7
7
|
## Introduction to BSV Overlay Services Engine
|
|
8
8
|
|
|
9
|
-
The BSV Overlay Services Engine is
|
|
10
|
-
|
|
11
|
-
If you want to run an HTTP overlay node, start with [`@bsv/overlay-express`](https://github.com/bsv-blockchain/overlay-express). If you want a local or cloud application runtime, use LARS or CARS with the BRC-102 `deployment-info.json` structure. Use `@bsv/overlay` directly when you are building custom infrastructure around the engine.
|
|
9
|
+
The BSV Overlay Services Engine is designed to process transactions and manage data within a blockchain-based system, specifically targeting the Bitcoin SV (BSV) blockchain. It integrates various components such as Topic Managers, Lookup Services, Storage, and Chain Tracker to provide a robust environment for managing transaction data and overlay services.
|
|
12
10
|
|
|
13
11
|
### Components of the System
|
|
14
12
|
|
|
15
|
-
1. **Topic Managers
|
|
16
|
-
2. **Lookup Services
|
|
17
|
-
3. **Storage
|
|
18
|
-
4. **Chain Tracker
|
|
19
|
-
5. **Broadcaster** submits accepted transactions to the network.
|
|
20
|
-
6. **Advertiser** publishes SHIP and SLAP availability records when peer discovery is enabled.
|
|
13
|
+
1. **Topic Managers**: Responsible for managing the admittance of transactions related to specific topics.
|
|
14
|
+
2. **Lookup Services**: Handle the lookup of UTXO (Unspent Transaction Output) data for transactions.
|
|
15
|
+
3. **Storage**: Manages persistent data storage, tracking UTXOs and their states within the system.
|
|
16
|
+
4. **Chain Tracker**: Verifies SPV (Simplified Payment Verification) data associated with transactions to ensure their validity.
|
|
21
17
|
|
|
22
18
|
### Setting Up the Engine
|
|
23
19
|
|
|
24
|
-
|
|
20
|
+
Before you can use the engine, you must initialize it with the required components:
|
|
25
21
|
|
|
26
|
-
```
|
|
27
|
-
|
|
28
|
-
|
|
22
|
+
```ts
|
|
23
|
+
import { Engine, KnexStorage } from "@bsv/overlay";
|
|
24
|
+
import { HelloTopicManager, HelloLookupService } from 'hello-services';
|
|
25
|
+
import { WoChain } from "@bsv/sdk";
|
|
29
26
|
|
|
30
|
-
|
|
27
|
+
// Initialize components
|
|
28
|
+
const managers = {
|
|
29
|
+
"exampleTopic": new HelloTopicManager()
|
|
30
|
+
};
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
import knexFactory from 'knex'
|
|
36
|
-
|
|
37
|
-
import { ExampleLookupService } from './services/ExampleLookupService.js'
|
|
38
|
-
import { ExampleTopicManager } from './services/ExampleTopicManager.js'
|
|
39
|
-
|
|
40
|
-
const knex = knexFactory({
|
|
41
|
-
client: 'mysql2',
|
|
42
|
-
connection: process.env.KNEX_URL
|
|
43
|
-
})
|
|
44
|
-
|
|
45
|
-
const engine = new Engine(
|
|
46
|
-
{
|
|
47
|
-
tm_example: new ExampleTopicManager()
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
ls_example: new ExampleLookupService()
|
|
51
|
-
},
|
|
52
|
-
new KnexStorage(knex),
|
|
53
|
-
new WhatsOnChain('test', { httpClient: new NodejsHttpClient() }),
|
|
54
|
-
process.env.HOSTING_URL,
|
|
55
|
-
process.env.SHIP_TRACKERS?.split(',') ?? [],
|
|
56
|
-
process.env.SLAP_TRACKERS?.split(',') ?? [],
|
|
57
|
-
new ARC(process.env.ARC_URL ?? 'https://arc.taal.com', {
|
|
58
|
-
apiKey: process.env.ARC_API_KEY
|
|
59
|
-
}),
|
|
60
|
-
undefined,
|
|
61
|
-
{
|
|
62
|
-
tm_example: 'SHIP'
|
|
63
|
-
}
|
|
64
|
-
)
|
|
65
|
-
```
|
|
32
|
+
const lookupServices = {
|
|
33
|
+
"exampleLookup": new HelloLookupService()
|
|
34
|
+
};
|
|
66
35
|
|
|
67
|
-
|
|
36
|
+
const storage = new KnexStorage();
|
|
37
|
+
const chainTracker = new WoChain();
|
|
38
|
+
|
|
39
|
+
// Create the engine instance
|
|
40
|
+
const engine = new Engine(managers, lookupServices, storage, chainTracker);
|
|
41
|
+
```
|
|
68
42
|
|
|
69
43
|
### Submitting a Transaction
|
|
70
44
|
|
|
71
|
-
|
|
45
|
+
To submit a transaction for processing by the Overlay Services:
|
|
72
46
|
|
|
73
47
|
```ts
|
|
74
48
|
import { Transaction } from '@bsv/sdk'
|
|
75
49
|
|
|
76
|
-
const tx = new Transaction(/* ... */)
|
|
50
|
+
const tx = new Transaction(/* ... */);
|
|
77
51
|
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
52
|
+
const transaction = {
|
|
53
|
+
beef: tx.toBEEF(),
|
|
54
|
+
topics: ['exampleTopic']
|
|
55
|
+
}
|
|
82
56
|
|
|
83
|
-
|
|
57
|
+
// Submit transaction
|
|
58
|
+
engine.submit(transaction).then(steak => {
|
|
59
|
+
console.log("Transaction processed:", steak);
|
|
60
|
+
}).catch(error => {
|
|
61
|
+
console.error("Error processing transaction:", error);
|
|
62
|
+
});
|
|
84
63
|
```
|
|
85
64
|
|
|
86
65
|
### Lookup Queries
|
|
87
66
|
|
|
88
|
-
|
|
67
|
+
To perform a lookup query using the engine:
|
|
89
68
|
|
|
90
69
|
```ts
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
|
|
70
|
+
const question = {
|
|
71
|
+
service: 'exampleLookup',
|
|
72
|
+
query: {
|
|
73
|
+
name: 'Bob'
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Perform a lookup
|
|
78
|
+
engine.lookup(question).then(answer => {
|
|
79
|
+
console.log("Lookup result:", answer);
|
|
80
|
+
}).catch(error => {
|
|
81
|
+
console.error("Error performing lookup:", error);
|
|
82
|
+
});
|
|
99
83
|
```
|
|
100
84
|
|
|
101
|
-
###
|
|
85
|
+
### Managing UTXOs
|
|
102
86
|
|
|
103
|
-
|
|
87
|
+
The system's core functionality involves managing UTXOs:
|
|
104
88
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
89
|
+
1. **Inserting a New UTXO**: Store new UTXO data when transactions are processed.
|
|
90
|
+
2. **Deleting a UTXO**: Remove UTXOs that are no longer needed or have been consumed by newer transactions.
|
|
91
|
+
3. **Tracking UTXO Consumption**: Monitor which transactions consume which UTXOs.
|
|
108
92
|
|
|
109
|
-
|
|
110
|
-
const lookupServices = await engine.listLookupServiceProviders()
|
|
111
|
-
```
|
|
93
|
+
### Retrieving Documentation
|
|
112
94
|
|
|
113
|
-
|
|
95
|
+
To retrieve documentation for specific managers or services:
|
|
114
96
|
|
|
115
|
-
|
|
97
|
+
```ts
|
|
98
|
+
// For a topic manager
|
|
99
|
+
engine.getDocumentationForTopicManger("exampleTopic").then(doc => {
|
|
100
|
+
console.log("Documentation for Topic Manager:", doc);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
// For a lookup service
|
|
104
|
+
engine.getDocumentationForLookupServiceProvider("exampleLookup").then(doc => {
|
|
105
|
+
console.log("Documentation for Lookup Service:", doc);
|
|
106
|
+
});
|
|
107
|
+
```
|
|
116
108
|
|
|
117
109
|
### Conclusion
|
|
118
110
|
|
|
119
|
-
The BSV Overlay Services Engine
|
|
111
|
+
The BSV Overlay Services Engine provides a powerful toolset for managing transactions and data on the Bitcoin SV blockchain. It's designed to handle complex data structures and ensure the integrity and security of transactions through rigorous validation and management processes. By following this tutorial, developers can effectively integrate and utilize these capabilities within their blockchain applications.
|
package/mod.ts
CHANGED
|
@@ -10,7 +10,31 @@ export type { TaggedBEEF, STEAK, LookupQuestion, LookupAnswer, AdmittanceInstruc
|
|
|
10
10
|
export type { LookupFormula } from './src/LookupFormula.js'
|
|
11
11
|
export type { Advertisement } from './src/Advertisement.js'
|
|
12
12
|
export type { AdvertisementData, Advertiser } from './src/Advertiser.js'
|
|
13
|
+
export {
|
|
14
|
+
BASM_ZERO_HASH,
|
|
15
|
+
computeBasmRoot,
|
|
16
|
+
computeTac,
|
|
17
|
+
extractMerkleProofMetadata
|
|
18
|
+
} from './src/BASM.js'
|
|
19
|
+
export type {
|
|
20
|
+
AdmittedListRequest,
|
|
21
|
+
AdmittedListResponse,
|
|
22
|
+
AdmittedTxRef,
|
|
23
|
+
BASMPeerSyncReport,
|
|
24
|
+
CompoundMerklePathRequest,
|
|
25
|
+
CompoundMerklePathResponse,
|
|
26
|
+
MerkleProofMetadata,
|
|
27
|
+
RawTransactionRecord,
|
|
28
|
+
RawTransactionRequest,
|
|
29
|
+
RawTransactionResponse,
|
|
30
|
+
TopicAnchorHeader,
|
|
31
|
+
TopicAnchorHeaderResolver,
|
|
32
|
+
TopicAnchorRangeRequest,
|
|
33
|
+
TopicAnchorRangeResponse,
|
|
34
|
+
TopicAnchorTip,
|
|
35
|
+
TopicBlockAnchor
|
|
36
|
+
} from './src/BASM.js'
|
|
13
37
|
|
|
14
38
|
// The Knex storage system
|
|
15
39
|
export { KnexStorage } from './src/storage/knex/KnexStorage.js'
|
|
16
|
-
export * as KnexStorageMigrations from './src/storage/knex/all-migrations.js'
|
|
40
|
+
export * as KnexStorageMigrations from './src/storage/knex/all-migrations.js'
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bsv/overlay",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.4",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "BSV Blockchain Overlay Services Engine",
|
|
6
6
|
"main": "dist/cjs/mod.js",
|
|
@@ -35,20 +35,10 @@
|
|
|
35
35
|
"types": "./dist/types/src/storage/*.d.ts"
|
|
36
36
|
}
|
|
37
37
|
},
|
|
38
|
-
"scripts": {
|
|
39
|
-
"test": "npm run build && jest",
|
|
40
|
-
"test:watch": "npm run build && jest --watch",
|
|
41
|
-
"test:coverage": "npm run build && jest --coverage",
|
|
42
|
-
"lint": "ts-standard --fix src/**/*.ts",
|
|
43
|
-
"lint:ci": "ts-standard src/**/*.ts",
|
|
44
|
-
"build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json",
|
|
45
|
-
"dev": "tsc -b -w",
|
|
46
|
-
"prepublish": "npm run build",
|
|
47
|
-
"doc": "ts2md --inputFilename=mod.ts --outputFilename=docs/API.md --filenameSubstring=API --firstHeadingLevel=1"
|
|
48
|
-
},
|
|
49
38
|
"repository": {
|
|
50
39
|
"type": "git",
|
|
51
|
-
"url": "git+https://github.com/bsv-blockchain/
|
|
40
|
+
"url": "git+https://github.com/bsv-blockchain/ts-stack.git",
|
|
41
|
+
"directory": "packages/overlays/overlay"
|
|
52
42
|
},
|
|
53
43
|
"keywords": [
|
|
54
44
|
"BSV",
|
|
@@ -60,21 +50,33 @@
|
|
|
60
50
|
"author": "BSV Association",
|
|
61
51
|
"license": "SEE LICENSE IN LICENSE.txt",
|
|
62
52
|
"bugs": {
|
|
63
|
-
"url": "https://github.com/
|
|
53
|
+
"url": "https://github.com/bsv-blockchain/ts-stack/issues"
|
|
64
54
|
},
|
|
65
|
-
"homepage": "https://github.com/
|
|
55
|
+
"homepage": "https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay#readme",
|
|
66
56
|
"devDependencies": {
|
|
67
|
-
"@types/jest": "^
|
|
68
|
-
"
|
|
69
|
-
"
|
|
57
|
+
"@types/jest": "^30.0.0",
|
|
58
|
+
"@types/node": "^25.9.3",
|
|
59
|
+
"jest": "^30.4.2",
|
|
60
|
+
"ts-jest": "^29.4.11",
|
|
70
61
|
"ts-standard": "^12.0.2",
|
|
71
62
|
"ts2md": "^0.2.0",
|
|
72
63
|
"tsconfig-to-dual-package": "^1.2.0",
|
|
73
|
-
"typescript": "^
|
|
64
|
+
"typescript": "^6.0.3"
|
|
74
65
|
},
|
|
75
66
|
"dependencies": {
|
|
76
|
-
"@bsv/gasp": "^1.2.
|
|
77
|
-
"@bsv/sdk": "^2.
|
|
78
|
-
"knex": "^3.
|
|
67
|
+
"@bsv/gasp": "^1.2.3",
|
|
68
|
+
"@bsv/sdk": "^2.1.4",
|
|
69
|
+
"knex": "^3.2.10"
|
|
70
|
+
},
|
|
71
|
+
"scripts": {
|
|
72
|
+
"test": "npm run build && jest",
|
|
73
|
+
"test:watch": "npm run build && jest --watch",
|
|
74
|
+
"test:coverage": "npm run build && jest --coverage",
|
|
75
|
+
"lint": "ts-standard --fix src/**/*.ts",
|
|
76
|
+
"lint:ci": "ts-standard src/**/*.ts",
|
|
77
|
+
"build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json",
|
|
78
|
+
"dev": "tsc -b -w",
|
|
79
|
+
"prepublish": "npm run build",
|
|
80
|
+
"doc": "ts2md --inputFilename=mod.ts --outputFilename=docs/API.md --filenameSubstring=API --firstHeadingLevel=1"
|
|
79
81
|
}
|
|
80
|
-
}
|
|
82
|
+
}
|