@serfab/cadre-core 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/announce-addrs.d.ts +49 -0
- package/dist/announce-addrs.js +78 -0
- package/dist/announce-addrs.js.map +1 -0
- package/dist/cadre-node.d.ts +607 -53
- package/dist/cadre-node.js +1268 -157
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-database.d.ts +174 -23
- package/dist/control-database.js +317 -107
- package/dist/control-database.js.map +1 -1
- package/dist/control-read-retry.d.ts +103 -0
- package/dist/control-read-retry.js +145 -0
- package/dist/control-read-retry.js.map +1 -0
- package/dist/control-retry.d.ts +101 -0
- package/dist/control-retry.js +108 -0
- package/dist/control-retry.js.map +1 -0
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +50 -6
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +59 -35
- package/dist/control-write-retry.js +21 -92
- package/dist/control-write-retry.js.map +1 -1
- package/dist/delegate-admission.d.ts +17 -15
- package/dist/delegate-admission.js +17 -15
- package/dist/delegate-admission.js.map +1 -1
- package/dist/enrolled-machine-store-file.d.ts +13 -0
- package/dist/enrolled-machine-store-file.js +41 -0
- package/dist/enrolled-machine-store-file.js.map +1 -0
- package/dist/enrolled-machine-store.d.ts +81 -0
- package/dist/enrolled-machine-store.js +225 -0
- package/dist/enrolled-machine-store.js.map +1 -0
- package/dist/file-durable-slot.js +5 -3
- package/dist/file-durable-slot.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.js +39 -12
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +226 -26
- package/dist/membership-connection-gater.js +316 -30
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/node-local-snapshot.d.ts +6 -0
- package/dist/node-local-snapshot.js +7 -1
- package/dist/node-local-snapshot.js.map +1 -1
- package/dist/{strand-backfill.d.ts → peer-join-backfill.d.ts} +49 -20
- package/dist/{strand-backfill.js → peer-join-backfill.js} +146 -56
- package/dist/peer-join-backfill.js.map +1 -0
- package/dist/peer-record.d.ts +26 -0
- package/dist/peer-record.js +32 -0
- package/dist/peer-record.js.map +1 -1
- package/dist/relay-addrs.d.ts +145 -22
- package/dist/relay-addrs.js +314 -31
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +20 -15
- package/dist/relay-reservation.js +18 -14
- package/dist/relay-reservation.js.map +1 -1
- package/dist/strand-addr-protocol.js +16 -0
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-database.d.ts +12 -0
- package/dist/strand-database.js +16 -0
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +22 -9
- package/dist/strand-formation-manager.js +20 -12
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +83 -9
- package/dist/strand-formation-protocol.js +86 -10
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +151 -17
- package/dist/strand-instance-manager.js +237 -32
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +118 -14
- package/dist/strand-membership-writer.js +203 -15
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +84 -0
- package/dist/strand-network-config.js +136 -0
- package/dist/strand-network-config.js.map +1 -0
- package/dist/strand-revocation-enforcer.d.ts +392 -0
- package/dist/strand-revocation-enforcer.js +533 -0
- package/dist/strand-revocation-enforcer.js.map +1 -0
- package/dist/strand-solicitation.d.ts +10 -3
- package/dist/strand-solicitation.js +12 -7
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +47 -2
- package/dist/strand-wake-protocol.js +76 -6
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/strand-watcher.js +6 -1
- package/dist/strand-watcher.js.map +1 -1
- package/dist/types.d.ts +282 -43
- package/dist/types.js +7 -1
- package/dist/types.js.map +1 -1
- package/package.json +16 -8
- package/src/announce-addrs.ts +93 -0
- package/src/cadre-node.ts +1366 -171
- package/src/control-database.ts +2628 -2406
- package/src/control-read-retry.ts +167 -0
- package/src/control-retry.ts +190 -0
- package/src/control-schema.ts +50 -6
- package/src/control-write-retry.ts +30 -130
- package/src/delegate-admission.ts +17 -15
- package/src/enrolled-machine-store-file.ts +43 -0
- package/src/enrolled-machine-store.ts +280 -0
- package/src/file-durable-slot.ts +5 -3
- package/src/index.ts +477 -407
- package/src/membership-connection-gater.ts +539 -204
- package/src/node-local-snapshot.ts +13 -1
- package/src/peer-addr-book.ts +251 -250
- package/src/{strand-backfill.ts → peer-join-backfill.ts} +187 -75
- package/src/peer-record.ts +33 -0
- package/src/relay-addrs.ts +436 -111
- package/src/relay-reservation.ts +828 -823
- package/src/strand-addr-protocol.ts +17 -0
- package/src/strand-database.ts +17 -0
- package/src/strand-formation-manager.ts +511 -491
- package/src/strand-formation-protocol.ts +808 -692
- package/src/strand-instance-manager.ts +899 -611
- package/src/strand-membership-writer.ts +232 -15
- package/src/strand-network-config.ts +172 -0
- package/src/strand-revocation-enforcer.ts +653 -0
- package/src/strand-solicitation.ts +484 -472
- package/src/strand-wake-protocol.ts +94 -8
- package/src/strand-watcher.ts +319 -314
- package/src/types.ts +1554 -1309
- package/dist/strand-backfill.js.map +0 -1
|
@@ -1,40 +1,63 @@
|
|
|
1
1
|
import debug from 'debug';
|
|
2
2
|
import type { Connection, Libp2p, PeerId } from '@libp2p/interface';
|
|
3
3
|
import type { ActionId, IPeerNetwork } from '@optimystic/db-core';
|
|
4
|
-
import { BlockTransferClient, type IRawStorage } from '@optimystic/db-p2p';
|
|
4
|
+
import { BlockTransferClient, type BlockCommitProof, type IRawStorage } from '@optimystic/db-p2p';
|
|
5
5
|
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
6
|
+
// Peer-join whole-store block catch-up, shared by the STRAND networks and the CONTROL
|
|
7
|
+
// network. On `connection:open` (debounced) it pushes every committed, materialized block
|
|
8
|
+
// in the local raw store to the new peer, so a machine that joined after blocks were
|
|
9
|
+
// committed still ends up physically holding them. Optimystic has no cohort-join catch-up
|
|
10
|
+
// of its own; a block committed while its writer was alone has exactly one holder forever
|
|
11
|
+
// without this (named collection-header blocks are written once, at collection creation,
|
|
12
|
+
// and their revision never moves again — so no later commit ever carries them anywhere).
|
|
13
|
+
//
|
|
14
|
+
// NOTE: this copies the WHOLE local store to every newly connected peer, which is right
|
|
15
|
+
// while a network is one party's handful of machines (see docs/architecture.md →
|
|
16
|
+
// Replication cluster size — the control network replicates to the whole party by intent,
|
|
17
|
+
// and "On a strand, the machine count is not a party count"). If these meshes ever get
|
|
18
|
+
// large, filter the pushed set by FRET cohort responsibility for each block id instead of
|
|
19
|
+
// pushing everything.
|
|
11
20
|
//
|
|
12
21
|
// NOTE: this lives in cadre-core only because `../optimystic` is read-only from this repo.
|
|
13
22
|
// If Optimystic ever grows a cohort-join catch-up of its own, delete this module rather
|
|
14
23
|
// than running two.
|
|
15
24
|
//
|
|
16
|
-
// Membership
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
25
|
+
// Membership — the one place the two networks differ, expressed as the optional
|
|
26
|
+
// `authorizePeer` dep:
|
|
27
|
+
//
|
|
28
|
+
// - STRAND networks pass none. Anything connected on a strand's own libp2p network already
|
|
29
|
+
// receives cohort replicas of new commits, so pushing older blocks to it exposes nothing
|
|
30
|
+
// new; adding a gate would diverge from what ordinary replication already does there. A
|
|
31
|
+
// peer that does NOT speak the strand's block-transfer protocol (a bare circuit relay or
|
|
32
|
+
// bootstrap node the strand node also connects to) cannot receive anything: the protocol
|
|
33
|
+
// id is namespaced per strand, so the dial fails and the push is dropped.
|
|
34
|
+
// - The CONTROL network MUST pass one, because that argument does not carry over: its
|
|
35
|
+
// inbound connection gate deliberately admits non-members in several states (an
|
|
36
|
+
// un-enrolled node taking its seed, an open enrollment window, an outstanding
|
|
37
|
+
// invitation, configured bootstrap/relay peers — see docs/architecture.md, the
|
|
38
|
+
// control-network inbound connection gate). Pushing the whole control store to such a
|
|
39
|
+
// peer would hand a stranger the party's entire membership, addresses and strand list.
|
|
40
|
+
// `CadreNode` passes `isAuthorizedMember`; the gate is consulted at PUSH time, not at
|
|
41
|
+
// schedule time, and fails closed on a thrown check. A denied run is not memoized, so
|
|
42
|
+
// the peer is retried on its next `connection:open` — or sooner, via
|
|
43
|
+
// `scheduleConnectedPeers()` on a membership change (the production join order is
|
|
44
|
+
// connect-then-authorize, so the denial at dial time is the expected first pass).
|
|
23
45
|
//
|
|
24
|
-
// NOTE:
|
|
25
|
-
// is never memoized (only clean runs are). Bounded today by the
|
|
26
|
-
// unreachable-peer bail in `runCatchUp` — one dial per event, not one
|
|
27
|
-
// node ever holds many
|
|
28
|
-
// gets expensive, pre-check `libp2p.peerStore` for this
|
|
29
|
-
// before enumerating.
|
|
46
|
+
// NOTE: a failing dial to a non-speaking peer costs one dial timeout per connection:open
|
|
47
|
+
// from such a peer, and is never memoized (only clean runs are). Bounded today by the
|
|
48
|
+
// debounce plus the unreachable-peer bail in `runCatchUp` — one dial per event, not one
|
|
49
|
+
// per chunk. If a node ever holds many such connections, or the enumeration ahead of the
|
|
50
|
+
// first chunk gets expensive, pre-check `libp2p.peerStore` for this network's
|
|
51
|
+
// block-transfer protocol before enumerating.
|
|
30
52
|
//
|
|
31
|
-
// NOTE: a run that fails is retried only when that peer next opens a connection
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
//
|
|
53
|
+
// NOTE: a run that fails is retried only when that peer next opens a connection (or, with
|
|
54
|
+
// an `authorizePeer` gate wired, when `scheduleConnectedPeers()` is driven). Over a stable
|
|
55
|
+
// connection a transient push failure therefore leaves the peer partially copied until the
|
|
56
|
+
// next reconnect (read repair still covers reads meanwhile). If these meshes ever hold
|
|
57
|
+
// long-lived connections where that matters, re-arm the debounce timer with a backoff on a
|
|
58
|
+
// non-clean run instead of waiting for connection:open.
|
|
36
59
|
|
|
37
|
-
const log = debug('sereus:cadre:
|
|
60
|
+
const log = debug('sereus:cadre:peer-join-backfill');
|
|
38
61
|
|
|
39
62
|
/**
|
|
40
63
|
* The block-transfer protocol's hard cap on one length-prefixed message. Mirrors
|
|
@@ -44,8 +67,8 @@ const log = debug('sereus:cadre:strand-backfill');
|
|
|
44
67
|
*/
|
|
45
68
|
export const MAX_BLOCK_MESSAGE_BYTES = 8 * 1024 * 1024;
|
|
46
69
|
|
|
47
|
-
/** Tuning for the per-peer
|
|
48
|
-
export interface
|
|
70
|
+
/** Tuning for the per-peer catch-up. Every field optional; defaults in {@link DEFAULT_PEER_JOIN_BACKFILL}. */
|
|
71
|
+
export interface PeerJoinBackfillConfig {
|
|
49
72
|
/** Default true. False disables the catch-up entirely (the pre-existing behaviour). */
|
|
50
73
|
enabled?: boolean;
|
|
51
74
|
/** Settle time after a connection opens before catching that peer up, ms. Default 1000. */
|
|
@@ -62,8 +85,8 @@ export interface StrandBackfillConfig {
|
|
|
62
85
|
responseTimeoutMs?: number;
|
|
63
86
|
}
|
|
64
87
|
|
|
65
|
-
/** The resolved defaults every {@link
|
|
66
|
-
export const
|
|
88
|
+
/** The resolved defaults every {@link PeerJoinBackfill} starts from. */
|
|
89
|
+
export const DEFAULT_PEER_JOIN_BACKFILL: Required<PeerJoinBackfillConfig> = {
|
|
67
90
|
enabled: true,
|
|
68
91
|
debounceMs: 1000,
|
|
69
92
|
maxBlocks: 10_000,
|
|
@@ -76,29 +99,36 @@ export const DEFAULT_STRAND_BACKFILL: Required<StrandBackfillConfig> = {
|
|
|
76
99
|
/**
|
|
77
100
|
* The one capability the catch-up needs from a transfer client. Structural (rather than
|
|
78
101
|
* the concrete `BlockTransferClient`) so unit tests can capture pushes without dialing
|
|
79
|
-
* libp2p — see {@link
|
|
102
|
+
* libp2p — see {@link PeerJoinBackfillDeps.createPushClient}.
|
|
80
103
|
*/
|
|
81
|
-
export type
|
|
104
|
+
export type PeerJoinBackfillPushClient = Pick<BlockTransferClient, 'pushBlocks'>;
|
|
82
105
|
|
|
83
|
-
export interface
|
|
84
|
-
|
|
85
|
-
|
|
106
|
+
export interface PeerJoinBackfillDeps {
|
|
107
|
+
/** Log tag naming which network this catch-up serves (a strand id, or `control-<partyId>`). */
|
|
108
|
+
label: string;
|
|
109
|
+
/** The libp2p node of the network being caught up — source of connection events and peer ids. */
|
|
86
110
|
libp2p: Libp2p;
|
|
87
111
|
/** `node.keyNetwork`, the IPeerNetwork BlockTransferClient dials through. */
|
|
88
112
|
peerNetwork: IPeerNetwork;
|
|
89
|
-
/** This
|
|
113
|
+
/** This network's own raw block store — the same instance handed to the libp2p node. */
|
|
90
114
|
storage: IRawStorage;
|
|
91
|
-
/** Must equal the prefix the receiver registered its handler under: `/optimystic
|
|
115
|
+
/** Must equal the prefix the receiver registered its handler under: `/optimystic/<networkName>`. */
|
|
92
116
|
protocolPrefix: string;
|
|
117
|
+
/**
|
|
118
|
+
* Membership gate, consulted at PUSH time (fails closed on a throw). Absent = push to
|
|
119
|
+
* every connected peer — correct for strand networks, NEVER for the control network;
|
|
120
|
+
* see the module comment for why the strand no-gate argument does not carry over.
|
|
121
|
+
*/
|
|
122
|
+
authorizePeer?: (peerId: string) => Promise<boolean>;
|
|
93
123
|
/**
|
|
94
124
|
* Test seam: build the per-peer push client. Defaults to a real
|
|
95
125
|
* `BlockTransferClient` over {@link peerNetwork} + {@link protocolPrefix}.
|
|
96
126
|
*/
|
|
97
|
-
createPushClient?: (peerId: PeerId) =>
|
|
127
|
+
createPushClient?: (peerId: PeerId) => PeerJoinBackfillPushClient;
|
|
98
128
|
}
|
|
99
129
|
|
|
100
130
|
/** What one peer's catch-up actually did. Returned for tests and logged at the end of each run. */
|
|
101
|
-
export interface
|
|
131
|
+
export interface PeerJoinBackfillResult {
|
|
102
132
|
/** Blocks offered: had a committed `latest` AND materialized content locally. */
|
|
103
133
|
offered: number;
|
|
104
134
|
/** Blocks the remote reported it persisted. */
|
|
@@ -113,10 +143,12 @@ export interface StrandBackfillResult {
|
|
|
113
143
|
capped: number;
|
|
114
144
|
/** Skipped: a single block whose wire size alone exceeds {@link MAX_BLOCK_MESSAGE_BYTES}. */
|
|
115
145
|
oversized: string[];
|
|
146
|
+
/** True when the `authorizePeer` gate refused (or threw) — nothing was pushed, peer not memoized. */
|
|
147
|
+
denied: boolean;
|
|
116
148
|
}
|
|
117
149
|
|
|
118
|
-
function emptyResult():
|
|
119
|
-
return { offered: 0, accepted: 0, rejected: [], uncommitted: 0, unmaterialized: 0, capped: 0, oversized: [] };
|
|
150
|
+
function emptyResult(): PeerJoinBackfillResult {
|
|
151
|
+
return { offered: 0, accepted: 0, rejected: [], uncommitted: 0, unmaterialized: 0, capped: 0, oversized: [], denied: false };
|
|
120
152
|
}
|
|
121
153
|
|
|
122
154
|
/** One in-flight push message being accumulated. */
|
|
@@ -124,6 +156,19 @@ interface Chunk {
|
|
|
124
156
|
ids: string[];
|
|
125
157
|
buffers: Uint8Array[];
|
|
126
158
|
meta: Record<string, { rev: number; actionId: ActionId }>;
|
|
159
|
+
/**
|
|
160
|
+
* The cohort commit proof retained for each block's pushed revision, where one exists.
|
|
161
|
+
*
|
|
162
|
+
* A receiver running the default `requirePushCertificate: true` REJECTS a block pushed
|
|
163
|
+
* without one, so this is what makes peer-join catch-up land at all — a push carrying only
|
|
164
|
+
* `meta` is the legacy shape and is refused. Sparse deliberately: a block whose proof was
|
|
165
|
+
* never retained is still offered with its meta, which is the only thing that lets a
|
|
166
|
+
* receiver migrating with the flag off land the replica at this node's revision rather than
|
|
167
|
+
* a fabricated rev 1. Upstream's rule is the other direction and holds here by construction:
|
|
168
|
+
* a proof is never attached WITHOUT its meta, because both are written from the same
|
|
169
|
+
* `latest` in one place.
|
|
170
|
+
*/
|
|
171
|
+
proofs: Record<string, BlockCommitProof>;
|
|
127
172
|
bytes: number;
|
|
128
173
|
}
|
|
129
174
|
|
|
@@ -136,32 +181,41 @@ function base64WireBytes(rawBytes: number): number {
|
|
|
136
181
|
}
|
|
137
182
|
|
|
138
183
|
/**
|
|
139
|
-
* Copies every block in one
|
|
140
|
-
* to, so a peer that joined the
|
|
184
|
+
* Copies every block in one network's own raw store to each peer its libp2p node connects
|
|
185
|
+
* to, so a peer that joined the network after blocks were committed still ends up holding
|
|
141
186
|
* them physically (the receiver persists each push via `saveReplicatedBlock`, which is
|
|
142
187
|
* monotonic and idempotent — crossing pushes from both ends cannot regress a revision).
|
|
143
188
|
*
|
|
144
|
-
* Best-effort throughout: nothing here throws into a libp2p event handler or into
|
|
145
|
-
*
|
|
189
|
+
* Best-effort throughout: nothing here throws into a libp2p event handler or into a
|
|
190
|
+
* runtime bring-up; a failed chunk is logged and the run continues. A peer is marked
|
|
146
191
|
* fully caught up ONLY after a run with no thrown chunk and an empty `missing` list, so
|
|
147
192
|
* the next `connection:open` from a peer whose catch-up failed retries it.
|
|
148
193
|
*/
|
|
149
|
-
export class
|
|
150
|
-
private readonly config: Required<
|
|
151
|
-
private readonly createPushClient: (peerId: PeerId) =>
|
|
194
|
+
export class PeerJoinBackfill {
|
|
195
|
+
private readonly config: Required<PeerJoinBackfillConfig>;
|
|
196
|
+
private readonly createPushClient: (peerId: PeerId) => PeerJoinBackfillPushClient;
|
|
152
197
|
private started = false;
|
|
153
198
|
private stopped = false;
|
|
154
199
|
/** Peers fully caught up this runtime (never retried until the runtime is rebuilt). */
|
|
155
200
|
private readonly done = new Set<string>();
|
|
156
201
|
/** Peers with a catch-up currently running (suppresses concurrent duplicates). */
|
|
157
202
|
private readonly inFlight = new Set<string>();
|
|
203
|
+
/**
|
|
204
|
+
* Peers whose (re-)schedule arrived while their OWN run was in flight, replayed once
|
|
205
|
+
* that run finishes. Load-bearing for the gated path: the gate's authorization check is
|
|
206
|
+
* a control-database read, so the window between it and the run's end is wide enough for
|
|
207
|
+
* the very membership commit that would have authorized the peer to land inside it —
|
|
208
|
+
* dropping that schedule (rather than deferring it) leaves the denied peer waiting for a
|
|
209
|
+
* reconnect, which is exactly what the re-arm exists to avoid.
|
|
210
|
+
*/
|
|
211
|
+
private readonly rearmAfterFlight = new Set<string>();
|
|
158
212
|
/** Pending per-peer debounce timers, cleared on stop. */
|
|
159
213
|
private readonly timers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
160
214
|
private readonly onConnectionOpen: (evt: CustomEvent<Connection>) => void;
|
|
161
215
|
private loggedNoListBlockIds = false;
|
|
162
216
|
|
|
163
|
-
constructor(private readonly deps:
|
|
164
|
-
this.config = { ...
|
|
217
|
+
constructor(private readonly deps: PeerJoinBackfillDeps, config?: PeerJoinBackfillConfig) {
|
|
218
|
+
this.config = { ...DEFAULT_PEER_JOIN_BACKFILL, ...config };
|
|
165
219
|
this.createPushClient = deps.createPushClient
|
|
166
220
|
?? ((peerId) => new BlockTransferClient(peerId, deps.peerNetwork, deps.protocolPrefix));
|
|
167
221
|
this.onConnectionOpen = (evt) => this.schedulePeer(evt.detail.remotePeer);
|
|
@@ -174,11 +228,8 @@ export class StrandBackfill {
|
|
|
174
228
|
this.deps.libp2p.addEventListener('connection:open', this.onConnectionOpen);
|
|
175
229
|
// A runtime rebuilt over live connections (resumeStrand) never sees their
|
|
176
230
|
// connection:open, so walk what is already connected once.
|
|
177
|
-
const
|
|
178
|
-
|
|
179
|
-
this.schedulePeer(connection.remotePeer);
|
|
180
|
-
}
|
|
181
|
-
log('[%s] started (%d peer(s) already connected)', this.deps.strandId, existing.length);
|
|
231
|
+
const scheduled = this.scheduleConnectedPeers();
|
|
232
|
+
log('[%s] started (%d peer(s) already connected)', this.deps.label, scheduled);
|
|
182
233
|
}
|
|
183
234
|
|
|
184
235
|
/** Unsubscribe, clear timers; in-flight runs observe the stopped flag and bail. */
|
|
@@ -192,13 +243,42 @@ export class StrandBackfill {
|
|
|
192
243
|
clearTimeout(timer);
|
|
193
244
|
}
|
|
194
245
|
this.timers.clear();
|
|
195
|
-
|
|
246
|
+
this.rearmAfterFlight.clear();
|
|
247
|
+
log('[%s] stopped', this.deps.label);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* (Re-)schedule a debounced catch-up for every currently-connected peer not yet caught
|
|
252
|
+
* up. Idempotent and cheap (caught-up peers are skipped before any timer is set, and a
|
|
253
|
+
* peer whose run is in flight is deferred to the end of that run rather than dropped).
|
|
254
|
+
* Driven by {@link start}, and — on a gated network — by the embedder whenever
|
|
255
|
+
* membership changes, so a peer whose first pass was denied (connected before it was
|
|
256
|
+
* authorized: the production join order) is retried without waiting for a reconnect.
|
|
257
|
+
* Returns how many distinct peers were considered (for the start() log) — a peer holding
|
|
258
|
+
* several connections is one peer, and is scheduled once.
|
|
259
|
+
*/
|
|
260
|
+
scheduleConnectedPeers(): number {
|
|
261
|
+
if (this.stopped) return 0;
|
|
262
|
+
const peers = new Set<string>();
|
|
263
|
+
for (const connection of this.deps.libp2p.getConnections()) {
|
|
264
|
+
const key = connection.remotePeer.toString();
|
|
265
|
+
if (peers.has(key)) continue;
|
|
266
|
+
peers.add(key);
|
|
267
|
+
this.schedulePeer(connection.remotePeer);
|
|
268
|
+
}
|
|
269
|
+
return peers.size;
|
|
196
270
|
}
|
|
197
271
|
|
|
198
272
|
/** Debounced entry point for connection churn: one run per peer per settle window. */
|
|
199
273
|
private schedulePeer(peerId: PeerId): void {
|
|
200
274
|
const key = peerId.toString();
|
|
201
|
-
if (this.stopped || this.done.has(key)
|
|
275
|
+
if (this.stopped || this.done.has(key)) return;
|
|
276
|
+
if (this.inFlight.has(key)) {
|
|
277
|
+
// Defer rather than drop: the running pass may already be past its gate check (or
|
|
278
|
+
// past the block this schedule was meant to carry), so replay it when that pass ends.
|
|
279
|
+
this.rearmAfterFlight.add(key);
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
202
282
|
const existing = this.timers.get(key);
|
|
203
283
|
if (existing) clearTimeout(existing);
|
|
204
284
|
this.timers.set(key, setTimeout(() => {
|
|
@@ -212,7 +292,7 @@ export class StrandBackfill {
|
|
|
212
292
|
* all-zero result without pushing when the peer is already caught up, already being
|
|
213
293
|
* caught up, or this backfill is stopped.
|
|
214
294
|
*/
|
|
215
|
-
async catchUpPeer(peerId: PeerId): Promise<
|
|
295
|
+
async catchUpPeer(peerId: PeerId): Promise<PeerJoinBackfillResult> {
|
|
216
296
|
const key = peerId.toString();
|
|
217
297
|
if (this.stopped || this.done.has(key) || this.inFlight.has(key)) {
|
|
218
298
|
return emptyResult();
|
|
@@ -223,41 +303,63 @@ export class StrandBackfill {
|
|
|
223
303
|
// NOTE: a run that hit `maxBlocks` is still "clean" and still memoizes the peer, so
|
|
224
304
|
// the tail past the ceiling never reaches it. Deliberate: enumeration is not
|
|
225
305
|
// resumable, so not memoizing would re-push the same prefix on every reconnect
|
|
226
|
-
// without ever advancing. Loud in the log below (capped > 0). If a
|
|
306
|
+
// without ever advancing. Loud in the log below (capped > 0). If a store can
|
|
227
307
|
// realistically exceed maxBlocks, the fix is a resumable cursor, not either policy.
|
|
228
308
|
if (clean && !this.stopped) {
|
|
229
309
|
this.done.add(key);
|
|
230
310
|
}
|
|
231
|
-
log('[%s] catch-up peer=%s offered=%d accepted=%d rejected=%d uncommitted=%d unmaterialized=%d capped=%d oversized=%d done=%s',
|
|
232
|
-
this.deps.
|
|
233
|
-
result.uncommitted, result.unmaterialized, result.capped, result.oversized.length, clean);
|
|
311
|
+
log('[%s] catch-up peer=%s offered=%d accepted=%d rejected=%d uncommitted=%d unmaterialized=%d capped=%d oversized=%d denied=%s done=%s',
|
|
312
|
+
this.deps.label, key, result.offered, result.accepted, result.rejected.length,
|
|
313
|
+
result.uncommitted, result.unmaterialized, result.capped, result.oversized.length, result.denied, clean);
|
|
234
314
|
return result;
|
|
235
315
|
} catch (error) {
|
|
236
316
|
// runCatchUp already contains a per-chunk catch; this guards the enumeration and
|
|
237
317
|
// metadata reads too — a backfill fault must never surface through a libp2p event.
|
|
238
|
-
log('[%s] catch-up peer=%s failed: %o', this.deps.
|
|
318
|
+
log('[%s] catch-up peer=%s failed: %o', this.deps.label, key, error);
|
|
239
319
|
return emptyResult();
|
|
240
320
|
} finally {
|
|
241
321
|
this.inFlight.delete(key);
|
|
322
|
+
// Replay a schedule that arrived mid-run. `schedulePeer` re-checks `done`, so a run
|
|
323
|
+
// that finished clean re-arms nothing; a denied or failed one gets its retry.
|
|
324
|
+
if (this.rearmAfterFlight.delete(key)) {
|
|
325
|
+
this.schedulePeer(peerId);
|
|
326
|
+
}
|
|
242
327
|
}
|
|
243
328
|
}
|
|
244
329
|
|
|
245
330
|
/** The actual copy. `clean` = every chunk pushed and the remote persisted every block. */
|
|
246
|
-
private async runCatchUp(peerId: PeerId): Promise<{ result:
|
|
331
|
+
private async runCatchUp(peerId: PeerId): Promise<{ result: PeerJoinBackfillResult; clean: boolean }> {
|
|
247
332
|
const result = emptyResult();
|
|
248
333
|
const { storage } = this.deps;
|
|
249
334
|
|
|
335
|
+
// The membership gate, judged at push time (never at schedule time — authorization can
|
|
336
|
+
// change during the debounce window in either direction). Fails CLOSED on a throw: a
|
|
337
|
+
// gate that cannot answer must not leak the store. A denied run is not clean, so the
|
|
338
|
+
// peer is retried later rather than memoized as caught up.
|
|
339
|
+
if (this.deps.authorizePeer) {
|
|
340
|
+
let authorized = false;
|
|
341
|
+
try {
|
|
342
|
+
authorized = await this.deps.authorizePeer(peerId.toString());
|
|
343
|
+
} catch (error) {
|
|
344
|
+
log('[%s] authorizePeer(%s) threw — treating as denied: %o', this.deps.label, peerId.toString(), error);
|
|
345
|
+
}
|
|
346
|
+
if (!authorized) {
|
|
347
|
+
result.denied = true;
|
|
348
|
+
return { result, clean: false };
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
250
352
|
if (!storage.listBlockIds) {
|
|
251
353
|
if (!this.loggedNoListBlockIds) {
|
|
252
354
|
this.loggedNoListBlockIds = true;
|
|
253
|
-
log('[%s] raw storage does not implement listBlockIds(); backfill is inert', this.deps.
|
|
355
|
+
log('[%s] raw storage does not implement listBlockIds(); backfill is inert', this.deps.label);
|
|
254
356
|
}
|
|
255
357
|
return { result, clean: false };
|
|
256
358
|
}
|
|
257
359
|
|
|
258
360
|
const client = this.createPushClient(peerId);
|
|
259
361
|
const encoder = new TextEncoder();
|
|
260
|
-
let chunk: Chunk = { ids: [], buffers: [], meta: {}, bytes: 0 };
|
|
362
|
+
let chunk: Chunk = { ids: [], buffers: [], meta: {}, proofs: {}, bytes: 0 };
|
|
261
363
|
let chunkFailed = false;
|
|
262
364
|
let anyChunkDelivered = false;
|
|
263
365
|
/**
|
|
@@ -270,10 +372,13 @@ export class StrandBackfill {
|
|
|
270
372
|
|
|
271
373
|
const flush = async (): Promise<void> => {
|
|
272
374
|
if (chunk.ids.length === 0) return;
|
|
273
|
-
const { ids, buffers, meta } = chunk;
|
|
274
|
-
chunk = { ids: [], buffers: [], meta: {}, bytes: 0 };
|
|
375
|
+
const { ids, buffers, meta, proofs } = chunk;
|
|
376
|
+
chunk = { ids: [], buffers: [], meta: {}, proofs: {}, bytes: 0 };
|
|
275
377
|
try {
|
|
276
|
-
|
|
378
|
+
// ONE certification value rather than two arguments, so a caller cannot pair a proof
|
|
379
|
+
// for one revision with meta for another — both were written from the same `latest`.
|
|
380
|
+
const response = await client.pushBlocks(ids, buffers, 'replication',
|
|
381
|
+
{ blockMeta: meta, blockProofs: proofs }, {
|
|
277
382
|
dialTimeoutMs: this.config.dialTimeoutMs,
|
|
278
383
|
responseTimeoutMs: this.config.responseTimeoutMs
|
|
279
384
|
});
|
|
@@ -287,11 +392,11 @@ export class StrandBackfill {
|
|
|
287
392
|
}
|
|
288
393
|
}
|
|
289
394
|
if (response.missing.length > 0) {
|
|
290
|
-
log('[%s] peer=%s rejected %d block(s): %o', this.deps.
|
|
395
|
+
log('[%s] peer=%s rejected %d block(s): %o', this.deps.label, peerId.toString(), response.missing.length, response.missing);
|
|
291
396
|
}
|
|
292
397
|
} catch (error) {
|
|
293
398
|
chunkFailed = true;
|
|
294
|
-
log('[%s] push to peer=%s failed for %d block(s): %s', this.deps.
|
|
399
|
+
log('[%s] push to peer=%s failed for %d block(s): %s', this.deps.label, peerId.toString(), ids.length, (error as Error).message);
|
|
295
400
|
}
|
|
296
401
|
};
|
|
297
402
|
|
|
@@ -319,7 +424,7 @@ export class StrandBackfill {
|
|
|
319
424
|
const block = await storage.getMaterializedBlock(blockId, latest.actionId);
|
|
320
425
|
if (!block) {
|
|
321
426
|
result.unmaterialized += 1;
|
|
322
|
-
log('[%s] block=%s has latest rev %d but no materialized content; skipped', this.deps.
|
|
427
|
+
log('[%s] block=%s has latest rev %d but no materialized content; skipped', this.deps.label, blockId, latest.rev);
|
|
323
428
|
continue;
|
|
324
429
|
}
|
|
325
430
|
|
|
@@ -328,12 +433,12 @@ export class StrandBackfill {
|
|
|
328
433
|
// this test ignores the request envelope (ids array, blockMeta, JSON punctuation).
|
|
329
434
|
// A lone block within a few KiB of the cap therefore still ships and is rejected by
|
|
330
435
|
// the receiver's length-prefix decoder — a permanent chunk failure for that peer. No
|
|
331
|
-
// block
|
|
332
|
-
//
|
|
436
|
+
// block comes close today; if one ever can, subtract a measured envelope allowance
|
|
437
|
+
// here rather than raising the cap.
|
|
333
438
|
if (base64WireBytes(buffer.length) >= MAX_BLOCK_MESSAGE_BYTES) {
|
|
334
439
|
result.oversized.push(blockId);
|
|
335
440
|
log('[%s] block=%s is %d bytes (%d on the wire) — exceeds the %d-byte protocol cap alone; skipped',
|
|
336
|
-
this.deps.
|
|
441
|
+
this.deps.label, blockId, buffer.length, base64WireBytes(buffer.length), MAX_BLOCK_MESSAGE_BYTES);
|
|
337
442
|
continue;
|
|
338
443
|
}
|
|
339
444
|
|
|
@@ -347,6 +452,13 @@ export class StrandBackfill {
|
|
|
347
452
|
chunk.ids.push(blockId);
|
|
348
453
|
chunk.buffers.push(buffer);
|
|
349
454
|
chunk.meta[blockId] = { rev: latest.rev, actionId: latest.actionId };
|
|
455
|
+
// Read the proof for EXACTLY the revision being pushed, from the same `latest` the meta
|
|
456
|
+
// above was written from. Absent is normal (nothing retained a proof for that revision);
|
|
457
|
+
// the block still ships with its meta and the receiver decides.
|
|
458
|
+
const proof = await storage.getBlockProof(blockId, latest.rev);
|
|
459
|
+
if (proof) {
|
|
460
|
+
chunk.proofs[blockId] = proof;
|
|
461
|
+
}
|
|
350
462
|
chunk.bytes += buffer.length;
|
|
351
463
|
result.offered += 1;
|
|
352
464
|
}
|
|
@@ -357,7 +469,7 @@ export class StrandBackfill {
|
|
|
357
469
|
|
|
358
470
|
if (result.capped > 0) {
|
|
359
471
|
log('[%s] peer=%s catch-up CAPPED at %d blocks; %d block id(s) not attempted',
|
|
360
|
-
this.deps.
|
|
472
|
+
this.deps.label, peerId.toString(), this.config.maxBlocks, result.capped);
|
|
361
473
|
}
|
|
362
474
|
|
|
363
475
|
const clean = !chunkFailed && result.rejected.length === 0;
|
package/src/peer-record.ts
CHANGED
|
@@ -135,6 +135,39 @@ export function trailingPeerId(addr: Multiaddr): string | null {
|
|
|
135
135
|
return p2p.length > 0 ? (p2p[p2p.length - 1].value ?? null) : null;
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
+
/**
|
|
139
|
+
* `addr` guaranteed to terminate in `/p2p/<peerId>` — the shape every dial path
|
|
140
|
+
* needs — or `null` when it cannot, because the address already terminates in a
|
|
141
|
+
* DIFFERENT peer id and therefore does not reach `peerId` at all.
|
|
142
|
+
*
|
|
143
|
+
* Three input shapes, matching what a `CadrePeer` record can legitimately carry:
|
|
144
|
+
*
|
|
145
|
+
* - no `/p2p/` component (`/ip4/1.2.3.4/tcp/4001`) — the suffix is appended;
|
|
146
|
+
* - a relay hop with the destination missing (`…/p2p/<relay>/p2p-circuit`) — the
|
|
147
|
+
* last component is the circuit, not a peer id, so the suffix is appended and
|
|
148
|
+
* the address becomes the full `…/p2p/<relay>/p2p-circuit/p2p/<peerId>`;
|
|
149
|
+
* - already terminating in `/p2p/<X>` — returned untouched when `X` is `peerId`,
|
|
150
|
+
* dropped otherwise.
|
|
151
|
+
*
|
|
152
|
+
* This is deliberately the same rule libp2p applies internally in
|
|
153
|
+
* `calculateMultiaddrs` (append when the LAST component is not `p2p`) followed by
|
|
154
|
+
* its wrong-peer-id filter, so a normalized list matches what libp2p would have
|
|
155
|
+
* built anyway. Normalizing before the dial is what keeps a list from mixing
|
|
156
|
+
* suffixed and unsuffixed entries, which `getPeerAddress` rejects outright
|
|
157
|
+
* ("Multiaddrs must all have the same peer id or have no peer id") — taking the
|
|
158
|
+
* whole peer down with it, not just the odd address.
|
|
159
|
+
*
|
|
160
|
+
* Never use before signature verification: verification runs against the
|
|
161
|
+
* original on-record strings.
|
|
162
|
+
*/
|
|
163
|
+
export function withTrailingPeerId(addr: Multiaddr, peerId: string): Multiaddr | null {
|
|
164
|
+
const components = addr.getComponents();
|
|
165
|
+
if (components[components.length - 1]?.name !== 'p2p') {
|
|
166
|
+
return addr.encapsulate(`/p2p/${peerId}`);
|
|
167
|
+
}
|
|
168
|
+
return trailingPeerId(addr) === peerId ? addr : null;
|
|
169
|
+
}
|
|
170
|
+
|
|
138
171
|
/**
|
|
139
172
|
* Return `addrs` with signaling (`/p2p-circuit`) addrs first, otherwise stable.
|
|
140
173
|
* Used to present the WebRTC dial input ahead of direct addrs. Does NOT mutate
|