@serfab/cadre-core 1.17.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/cadre-invite-protocol.d.ts +35 -13
- package/dist/cadre-invite-protocol.js +90 -25
- package/dist/cadre-invite-protocol.js.map +1 -1
- package/dist/cadre-node.d.ts +586 -255
- package/dist/cadre-node.js +1182 -566
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +66 -9
- package/dist/control-cohort.js +67 -9
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +451 -193
- package/dist/control-database.js +701 -359
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +46 -43
- package/dist/control-formation-recorder.js +77 -78
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-protocol-guard.d.ts +6 -0
- package/dist/control-protocol-guard.js +14 -2
- package/dist/control-protocol-guard.js.map +1 -1
- package/dist/control-schema-version.d.ts +48 -31
- package/dist/control-schema-version.js +67 -45
- package/dist/control-schema-version.js.map +1 -1
- package/dist/control-schema.d.ts +8 -8
- package/dist/control-schema.js +16 -16
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +42 -7
- package/dist/control-write-retry.js +87 -19
- package/dist/control-write-retry.js.map +1 -1
- package/dist/enrolled-machine-store.d.ts +3 -3
- package/dist/enrolled-machine-store.js +3 -3
- package/dist/formation-token.d.ts +48 -0
- package/dist/formation-token.js +26 -0
- package/dist/formation-token.js.map +1 -0
- package/dist/index.d.ts +15 -14
- package/dist/index.js +17 -14
- package/dist/index.js.map +1 -1
- package/dist/joined-strand-store.d.ts +26 -9
- package/dist/joined-strand-store.js +63 -24
- package/dist/joined-strand-store.js.map +1 -1
- package/dist/link-budget.d.ts +9 -7
- package/dist/link-budget.js +9 -7
- package/dist/link-budget.js.map +1 -1
- package/dist/member-redemption.d.ts +15 -9
- package/dist/member-redemption.js +13 -8
- package/dist/member-redemption.js.map +1 -1
- package/dist/membership-view.d.ts +91 -0
- package/dist/membership-view.js +110 -0
- package/dist/membership-view.js.map +1 -0
- package/dist/owner-grant-protocol.d.ts +11 -7
- package/dist/owner-grant-protocol.js +31 -17
- package/dist/owner-grant-protocol.js.map +1 -1
- package/dist/peer-addr-book.d.ts +10 -1
- package/dist/peer-addr-book.js +4 -3
- package/dist/peer-addr-book.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +96 -30
- package/dist/peer-join-backfill.js +272 -128
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/pending-join-runner.d.ts +48 -18
- package/dist/pending-join-runner.js +74 -56
- package/dist/pending-join-runner.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +47 -40
- package/dist/seed-bootstrap.js +66 -61
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/self-address-update.d.ts +13 -0
- package/dist/self-address-update.js +19 -0
- package/dist/self-address-update.js.map +1 -0
- package/dist/strand-database.d.ts +23 -2
- package/dist/strand-database.js +26 -6
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +176 -89
- package/dist/strand-first-sync-gate.js +288 -122
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-deadlines.d.ts +9 -9
- package/dist/strand-formation-deadlines.js +9 -9
- package/dist/strand-formation-manager.d.ts +23 -17
- package/dist/strand-formation-manager.js +46 -44
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +5 -8
- package/dist/strand-formation-protocol.js +101 -16
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +26 -17
- package/dist/strand-instance-manager.js +119 -50
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +16 -13
- package/dist/strand-membership-reconciler.js +28 -22
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +40 -28
- package/dist/strand-membership-writer.js +212 -153
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-move.js +5 -2
- package/dist/strand-move.js.map +1 -1
- package/dist/strand-revocation-enforcer.d.ts +134 -60
- package/dist/strand-revocation-enforcer.js +144 -71
- package/dist/strand-revocation-enforcer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +50 -39
- package/dist/strand-solicitation.js +10 -3
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +80 -13
- package/dist/strand-watcher.js +152 -48
- package/dist/strand-watcher.js.map +1 -1
- package/dist/table-change-watch.d.ts +63 -0
- package/dist/table-change-watch.js +209 -0
- package/dist/table-change-watch.js.map +1 -0
- package/dist/timeout-scheduler.d.ts +4 -3
- package/dist/timeout-scheduler.js +4 -3
- package/dist/timeout-scheduler.js.map +1 -1
- package/dist/types.d.ts +90 -49
- package/dist/types.js +4 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-invite-protocol.ts +119 -31
- package/src/cadre-node.ts +1289 -588
- package/src/control-cohort.ts +86 -9
- package/src/control-database.ts +848 -403
- package/src/control-formation-recorder.ts +359 -353
- package/src/control-protocol-guard.ts +16 -2
- package/src/control-schema-version.ts +101 -51
- package/src/control-schema.ts +16 -16
- package/src/control-write-retry.ts +91 -19
- package/src/delegate-admission.ts +351 -351
- package/src/enrolled-machine-store.ts +3 -3
- package/src/formation-token.ts +63 -0
- package/src/index.ts +30 -18
- package/src/joined-strand-store.ts +69 -24
- package/src/link-budget.ts +9 -7
- package/src/member-redemption.ts +20 -13
- package/src/membership-view.ts +144 -0
- package/src/owner-grant-protocol.ts +33 -18
- package/src/peer-addr-book.ts +4 -3
- package/src/peer-join-backfill.ts +961 -761
- package/src/pending-join-runner.ts +76 -53
- package/src/seed-bootstrap.ts +72 -67
- package/src/self-address-update.ts +19 -0
- package/src/strand-database.ts +352 -322
- package/src/strand-first-sync-gate.ts +334 -134
- package/src/strand-formation-deadlines.ts +227 -227
- package/src/strand-formation-manager.ts +731 -725
- package/src/strand-formation-protocol.ts +122 -24
- package/src/strand-instance-manager.ts +2092 -2026
- package/src/strand-membership-reconciler.ts +31 -25
- package/src/strand-membership-writer.ts +2093 -2019
- package/src/strand-move.ts +5 -2
- package/src/strand-revocation-enforcer.ts +740 -654
- package/src/strand-solicitation.ts +61 -34
- package/src/strand-watcher.ts +175 -56
- package/src/table-change-watch.ts +285 -0
- package/src/timeout-scheduler.ts +29 -28
- package/src/types.ts +89 -51
|
@@ -6,10 +6,11 @@
|
|
|
6
6
|
* is described in `docs/strands.md` → "Joining while the inviter is offline".
|
|
7
7
|
*
|
|
8
8
|
* `CadreNode` owns every side effect and hands this module plain functions
|
|
9
|
-
* ({@link PendingJoinRunnerDeps}): read the joins,
|
|
10
|
-
* an outcome, say whether this machine is an
|
|
11
|
-
*
|
|
12
|
-
*
|
|
9
|
+
* ({@link PendingJoinRunnerDeps}): read the joins, hear a change to the tables they are read
|
|
10
|
+
* from, run one attempt (its own `formStrand`), write an outcome, say whether this machine is an
|
|
11
|
+
* owner, and the clock. This module owns the policy: when to read the joins, when to attempt,
|
|
12
|
+
* how to read an attempt's failure, and how to settle an outcome write that another machine's
|
|
13
|
+
* write beat.
|
|
13
14
|
*
|
|
14
15
|
* Only owner machines run it, because every pending-join write is owner-signed. Letting a
|
|
15
16
|
* machine without an owner key finish a join is `blocked/decide-non-owner-machine-completes-a-pending-join`;
|
|
@@ -30,12 +31,28 @@ import { PendingJoinChangedError, type JoinRequestFields, type PendingJoinFields
|
|
|
30
31
|
import { FormationPostApprovalError, FormationRejectedError, FormationUnreachableError } from './strand-formation-rejection.js';
|
|
31
32
|
import { formationDeadlines } from './strand-formation-deadlines.js';
|
|
32
33
|
import { MEMBERSHIP_INVITE_TTL_MS } from './strand-formation-manager.js';
|
|
34
|
+
import { scheduleRuns, type TableChangeWatch, type TableChangeWatchOptions } from './table-change-watch.js';
|
|
33
35
|
import { defaultTimeoutScheduler, type TimeoutScheduler } from './timeout-scheduler.js';
|
|
34
36
|
|
|
35
37
|
const log = debug('sereus:cadre:pending-join');
|
|
36
38
|
|
|
37
|
-
/**
|
|
38
|
-
|
|
39
|
+
/**
|
|
40
|
+
* The longest an owner machine goes without re-reading the party's pending joins when no change
|
|
41
|
+
* to their tables is heard. It covers a change that arrives without a storage event, and the
|
|
42
|
+
* time-driven removal of a finished join, which lands up to this much late.
|
|
43
|
+
*
|
|
44
|
+
* NOTE: on a cadre larger than `CONTROL_REPLICATION_BREADTH` (16), a machine outside a join
|
|
45
|
+
* table's cohort hears no change event, so a dismissal or another machine's outcome reaches it
|
|
46
|
+
* up to this much late. The worst effect is one more attempt on a dismissed join, whose outcome
|
|
47
|
+
* write then finds the request gone, or a `token-spent` answer whose re-read adopts the other
|
|
48
|
+
* machine's outcome. A 30 s pass while any join is pending was declined: a join can stay pending
|
|
49
|
+
* for {@link MAX_PENDING_JOIN_MS}, and those reads are what this backstop removes. Revisit if a
|
|
50
|
+
* cadre above 16 machines is seen attempting dismissed joins.
|
|
51
|
+
*/
|
|
52
|
+
export const PENDING_JOIN_BACKSTOP_MS = 10 * 60_000;
|
|
53
|
+
|
|
54
|
+
/** How soon a pass runs again after one failed, or after an outcome write gave up to another machine's writes. */
|
|
55
|
+
const PENDING_JOIN_PASS_RETRY_MS = 30_000;
|
|
39
56
|
|
|
40
57
|
/** Ceiling on the wait between two attempts of one join on one machine, before jitter. */
|
|
41
58
|
export const PENDING_JOIN_MAX_BACKOFF_MS = 10 * 60_000;
|
|
@@ -52,7 +69,7 @@ const MAX_BACKGROUND_ATTEMPTS = 2;
|
|
|
52
69
|
/** Each retry delay is moved by up to this fraction either way, so machines that failed together do not retry together. */
|
|
53
70
|
const RETRY_JITTER = 0.2;
|
|
54
71
|
|
|
55
|
-
/** Re-read-and-rewrite rounds after another machine's write won, before
|
|
72
|
+
/** Re-read-and-rewrite rounds after another machine's write won, before a pass {@link PENDING_JOIN_PASS_RETRY_MS} later decides instead. */
|
|
56
73
|
const MAX_OUTCOME_WRITE_ROUNDS = 3;
|
|
57
74
|
|
|
58
75
|
type AttemptError = NonNullable<PendingJoinStatus['lastError']>;
|
|
@@ -68,6 +85,10 @@ export interface PendingJoinRunnerDeps {
|
|
|
68
85
|
/** Whether this machine may write pending joins now. Asked every pass. */
|
|
69
86
|
isOwner(): Promise<boolean>;
|
|
70
87
|
readRows(): Promise<PendingJoin[]>;
|
|
88
|
+
/** Subscribe `onChange` to the tables a pass reads. Absent: passes come from `start`, `kick`, retries and the backstop only (tests). */
|
|
89
|
+
watchChanges?(onChange: () => Promise<void>, options: TableChangeWatchOptions): TableChangeWatch;
|
|
90
|
+
/** Run a pass at least this long after the last one settled; default {@link PENDING_JOIN_BACKSTOP_MS}. */
|
|
91
|
+
backstopMs?: number;
|
|
71
92
|
readRow(id: string): Promise<PendingJoin | null>;
|
|
72
93
|
/** One formation attempt for the request's invitation and disclosure (`CadreNode.formStrand`). */
|
|
73
94
|
attempt(join: PendingJoin): Promise<FormStrandResult>;
|
|
@@ -77,7 +98,7 @@ export interface PendingJoinRunnerDeps {
|
|
|
77
98
|
rewrite(expectedStampId: string, next: PendingJoinFields): Promise<PendingJoin>;
|
|
78
99
|
/** `ControlDatabase.deletePendingJoin`, owner-signed. */
|
|
79
100
|
remove(id: string): Promise<boolean>;
|
|
80
|
-
/** Whether this machine
|
|
101
|
+
/** Whether this machine reaches no sibling now (`CadreNode.hasReachableControlSibling`), so a write it commits reaches no other machine. */
|
|
81
102
|
isAlone(): boolean;
|
|
82
103
|
/** The sApp the request's invitation names. */
|
|
83
104
|
sAppIdOf(join: PendingJoin): string;
|
|
@@ -231,7 +252,7 @@ function stableFraction(key: string): number {
|
|
|
231
252
|
|
|
232
253
|
export class PendingJoinRunner {
|
|
233
254
|
private readonly tracked = new Map<string, TrackedJoin>();
|
|
234
|
-
/** Joins whose latest write by this machine committed with no
|
|
255
|
+
/** Joins whose latest write by this machine committed with no sibling reachable: id → the request stamp written. */
|
|
235
256
|
private readonly writtenAlone = new Map<string, string>();
|
|
236
257
|
private readonly dueQueue: TrackedJoin[] = [];
|
|
237
258
|
private readonly scheduler: TimeoutScheduler;
|
|
@@ -242,9 +263,8 @@ export class PendingJoinRunner {
|
|
|
242
263
|
/** How long a `token-spent` answer waits before its confirming attempt: long enough for a sibling's `joined` write to replicate. */
|
|
243
264
|
private readonly spentConfirmDelayMs: number;
|
|
244
265
|
private running = false;
|
|
245
|
-
|
|
246
|
-
private
|
|
247
|
-
private passAgain = false;
|
|
266
|
+
/** Every pass runs on it, one at a time; present between {@link start} and {@link stop}. */
|
|
267
|
+
private watch: TableChangeWatch | null = null;
|
|
248
268
|
private backgroundAttempts = 0;
|
|
249
269
|
|
|
250
270
|
constructor(private readonly deps: PendingJoinRunnerDeps) {
|
|
@@ -256,11 +276,21 @@ export class PendingJoinRunner {
|
|
|
256
276
|
this.spentConfirmDelayMs = 2 * deadlines.sessionMs;
|
|
257
277
|
}
|
|
258
278
|
|
|
259
|
-
/**
|
|
279
|
+
/**
|
|
280
|
+
* Start passes: one at once, then one whenever a table a pass reads changes, and at the latest
|
|
281
|
+
* {@link PENDING_JOIN_BACKSTOP_MS} after the last. Throws when the change subscription cannot
|
|
282
|
+
* be made.
|
|
283
|
+
*
|
|
284
|
+
* NOTE: a pass from before a {@link stop} that is still in flight runs alongside the new
|
|
285
|
+
* watch's passes, since each watch keeps only its own runs to one at a time. `CadreNode` builds
|
|
286
|
+
* a fresh runner per start, so nothing restarts one; if something comes to, make that pass
|
|
287
|
+
* check which start it belongs to.
|
|
288
|
+
*/
|
|
260
289
|
start(): void {
|
|
261
290
|
if (this.running) return;
|
|
291
|
+
this.watch = this.subscribe();
|
|
262
292
|
this.running = true;
|
|
263
|
-
this.
|
|
293
|
+
this.watch.trigger(0);
|
|
264
294
|
}
|
|
265
295
|
|
|
266
296
|
/**
|
|
@@ -274,18 +304,21 @@ export class PendingJoinRunner {
|
|
|
274
304
|
*/
|
|
275
305
|
stop(): void {
|
|
276
306
|
this.running = false;
|
|
277
|
-
|
|
307
|
+
// A pass already in flight finishes; `running` keeps it from scheduling anything further.
|
|
308
|
+
this.watch?.dispose();
|
|
309
|
+
this.watch = null;
|
|
278
310
|
for (const entry of this.tracked.values()) this.cancelTimer(entry);
|
|
279
311
|
this.tracked.clear();
|
|
280
312
|
this.dueQueue.length = 0;
|
|
281
313
|
this.writtenAlone.clear();
|
|
282
314
|
}
|
|
283
315
|
|
|
284
|
-
/**
|
|
316
|
+
/**
|
|
317
|
+
* Run a pass now, for a change the watched tables do not show (this machine's owner signer was
|
|
318
|
+
* just wired). A pass in flight is followed by one more.
|
|
319
|
+
*/
|
|
285
320
|
kick(): void {
|
|
286
|
-
|
|
287
|
-
this.clearPollTimer();
|
|
288
|
-
void this.pass();
|
|
321
|
+
this.watch?.trigger(0);
|
|
289
322
|
}
|
|
290
323
|
|
|
291
324
|
/**
|
|
@@ -327,8 +360,8 @@ export class PendingJoinRunner {
|
|
|
327
360
|
|
|
328
361
|
/**
|
|
329
362
|
* Re-write, with identical content under a fresh stamp, each join this machine last wrote while
|
|
330
|
-
*
|
|
331
|
-
*
|
|
363
|
+
* no sibling was reachable, so the other machines receive it. Called on the control cohort's
|
|
364
|
+
* growth edge, when a sibling becomes reachable. A join someone wrote since is left alone.
|
|
332
365
|
*
|
|
333
366
|
* NOTE: in memory only, so a join written alone by a process that stopped before it reconnected
|
|
334
367
|
* reaches the party only with that join's next write. A sweep of every join this machine wrote,
|
|
@@ -359,41 +392,29 @@ export class PendingJoinRunner {
|
|
|
359
392
|
}
|
|
360
393
|
}
|
|
361
394
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
this.
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
395
|
+
/** The watch every pass runs on: the deps' change subscription, or a timer-only one. */
|
|
396
|
+
private subscribe(): TableChangeWatch {
|
|
397
|
+
const onChange = (): Promise<void> => this.pass();
|
|
398
|
+
const options: TableChangeWatchOptions = {
|
|
399
|
+
label: 'pending-join',
|
|
400
|
+
backstopMs: this.deps.backstopMs ?? PENDING_JOIN_BACKSTOP_MS,
|
|
401
|
+
scheduler: this.scheduler,
|
|
402
|
+
};
|
|
403
|
+
return this.deps.watchChanges?.(onChange, options) ?? scheduleRuns(onChange, options);
|
|
369
404
|
}
|
|
370
405
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
this.
|
|
406
|
+
/** One pass; a failed one runs again {@link PENDING_JOIN_PASS_RETRY_MS} later. */
|
|
407
|
+
private async pass(): Promise<void> {
|
|
408
|
+
try {
|
|
409
|
+
await this.runPass();
|
|
410
|
+
} catch (error) {
|
|
411
|
+
log('pending-join pass failed; retried in 30 s: %s', errorText(error));
|
|
412
|
+
this.retryPass();
|
|
375
413
|
}
|
|
376
414
|
}
|
|
377
415
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
if (this.passInFlight) {
|
|
381
|
-
this.passAgain = true;
|
|
382
|
-
return this.passInFlight;
|
|
383
|
-
}
|
|
384
|
-
const run = this.runPass()
|
|
385
|
-
.catch((error: unknown) => log('pending-join pass failed; the next one retries: %s', errorText(error)))
|
|
386
|
-
.finally(() => {
|
|
387
|
-
this.passInFlight = null;
|
|
388
|
-
if (this.passAgain && this.running) {
|
|
389
|
-
this.passAgain = false;
|
|
390
|
-
void this.pass();
|
|
391
|
-
return;
|
|
392
|
-
}
|
|
393
|
-
this.schedulePass(PENDING_JOIN_POLL_MS);
|
|
394
|
-
});
|
|
395
|
-
this.passInFlight = run;
|
|
396
|
-
return run;
|
|
416
|
+
private retryPass(): void {
|
|
417
|
+
this.watch?.trigger(PENDING_JOIN_PASS_RETRY_MS);
|
|
397
418
|
}
|
|
398
419
|
|
|
399
420
|
private async runPass(): Promise<void> {
|
|
@@ -415,7 +436,8 @@ export class PendingJoinRunner {
|
|
|
415
436
|
try {
|
|
416
437
|
await this.consider(join);
|
|
417
438
|
} catch (error) {
|
|
418
|
-
log('pending join %s: this pass could not handle it;
|
|
439
|
+
log('pending join %s: this pass could not handle it; retried in 30 s: %s', join.Id, errorText(error));
|
|
440
|
+
this.retryPass();
|
|
419
441
|
}
|
|
420
442
|
}
|
|
421
443
|
}
|
|
@@ -663,7 +685,8 @@ export class PendingJoinRunner {
|
|
|
663
685
|
}
|
|
664
686
|
expected = live!;
|
|
665
687
|
}
|
|
666
|
-
log('pending join %s: another machine kept changing the join;
|
|
688
|
+
log('pending join %s: another machine kept changing the join; a pass in 30 s decides', entry.row.Id);
|
|
689
|
+
this.retryPass();
|
|
667
690
|
}
|
|
668
691
|
|
|
669
692
|
private settle(join: PendingJoin): void {
|
package/src/seed-bootstrap.ts
CHANGED
|
@@ -17,6 +17,7 @@ import { relayedRequestBudgetMs } from './link-budget.js';
|
|
|
17
17
|
import type {
|
|
18
18
|
ControlNetworkSeed,
|
|
19
19
|
SeedPeer,
|
|
20
|
+
SeedDialTarget,
|
|
20
21
|
SeedMessage,
|
|
21
22
|
SeedAckMessage,
|
|
22
23
|
SeedRefusalCode,
|
|
@@ -31,10 +32,9 @@ import type {
|
|
|
31
32
|
PeerAddressRecord,
|
|
32
33
|
DeviceTokenRecord,
|
|
33
34
|
RevocationRow,
|
|
34
|
-
RevocationLedgerOpenResult,
|
|
35
35
|
MachineRemovalResult
|
|
36
36
|
} from './types.js';
|
|
37
|
-
import type { ControlDatabase } from './control-database.js';
|
|
37
|
+
import type { CadrePeerRetouchMode, CadrePeerRetouchOutcome, ControlDatabase } from './control-database.js';
|
|
38
38
|
import { generateStampId, ownerKeyLabel } from './control-database.js';
|
|
39
39
|
import { canonicalJson } from './canonical-json.js';
|
|
40
40
|
import { deviceTokenAddDigest } from './peer-authorization.js';
|
|
@@ -117,6 +117,20 @@ export function seedDigest(seed: Pick<ControlNetworkSeed, 'partyId' | 'peers'>):
|
|
|
117
117
|
return digest([canonicalSeedPayload(seed)], 'sha256', 'base64url') as string;
|
|
118
118
|
}
|
|
119
119
|
|
|
120
|
+
/**
|
|
121
|
+
* The peers `seed` nominates for dialing: its preferred peers (`SeedPeer.preferred`: the machine
|
|
122
|
+
* that minted it and every always-on machine) that have an address and are not `selfPeerId`.
|
|
123
|
+
*
|
|
124
|
+
* Self is dropped because `createSeed` projects every non-revoked CadrePeer row, so a preferred
|
|
125
|
+
* machine applying a seed minted after it joined finds itself in the list, and a dial of self
|
|
126
|
+
* always throws. An undefined `selfPeerId` matches nothing.
|
|
127
|
+
*/
|
|
128
|
+
export function seedDialTargets(seed: ControlNetworkSeed, selfPeerId: string | undefined): SeedDialTarget[] {
|
|
129
|
+
return seed.peers
|
|
130
|
+
.filter((peer) => peer.preferred && peer.multiaddrs.length > 0 && peer.peerId !== selfPeerId)
|
|
131
|
+
.map(({ peerId, multiaddrs }) => ({ peerId, multiaddrs }));
|
|
132
|
+
}
|
|
133
|
+
|
|
120
134
|
/** Add `relayAddr` to the entry for `selfPeerId` in `peers`, when that entry exists and lacks it. */
|
|
121
135
|
function addSelfRelayAddr(peers: SeedPeer[], selfPeerId: string, relayAddr: string): void {
|
|
122
136
|
const self = peers.find((peer) => peer.peerId === selfPeerId);
|
|
@@ -307,7 +321,8 @@ export interface SeedEventCallbacks {
|
|
|
307
321
|
/** Called when a seed is received via the protocol */
|
|
308
322
|
onSeedReceived?: (partyId: string, peerId: string) => void;
|
|
309
323
|
/**
|
|
310
|
-
* Called when a seed
|
|
324
|
+
* Called when a seed delivered over the protocol has been verified, merged and
|
|
325
|
+
* acknowledged; the dials of its preferred peers follow, so this waits on no connection.
|
|
311
326
|
*
|
|
312
327
|
* `seed` is the applied seed itself: the inbound protocol handler applies it
|
|
313
328
|
* INSIDE the service, so this callback is the only seam through which a
|
|
@@ -553,10 +568,10 @@ export class SeedBootstrapService {
|
|
|
553
568
|
/**
|
|
554
569
|
* Return the configured owner signer, or throw if none is set. The single precondition
|
|
555
570
|
* gate for every owner-signed write. {@link removePeer} / {@link reauthorizePeer} read the
|
|
556
|
-
* row
|
|
557
|
-
*
|
|
558
|
-
*
|
|
559
|
-
*
|
|
571
|
+
* row from the DB BEFORE they sign, so they call this up front — otherwise a signerless
|
|
572
|
+
* service would either surface the wrong "Control database not initialized" error or,
|
|
573
|
+
* worse, silently no-op when the target row is absent (their early absent return) instead
|
|
574
|
+
* of rejecting.
|
|
560
575
|
*/
|
|
561
576
|
private requireOwnerSigner(): OwnerSigner {
|
|
562
577
|
if (!this.config.ownerSigner) {
|
|
@@ -697,35 +712,47 @@ export class SeedBootstrapService {
|
|
|
697
712
|
}
|
|
698
713
|
|
|
699
714
|
/**
|
|
700
|
-
* Owner "re-touch" of an existing `CadrePeer` membership row
|
|
701
|
-
*
|
|
702
|
-
* the write-while-alone re-replication
|
|
703
|
-
* a membership row that committed local-only
|
|
704
|
-
* pushed to the cohort once it grows
|
|
715
|
+
* Owner "re-touch" of an existing `CadrePeer` membership row, in one of two modes
|
|
716
|
+
* ({@link CadrePeerRetouchMode}): `'rebroadcast'` bumps `UpdatedAt` and re-vouches the row so
|
|
717
|
+
* it is re-emitted as a fresh, broadcasting transaction — the write-while-alone re-replication
|
|
718
|
+
* primitive (`control-write-ensure-replicated`): a membership row that committed local-only
|
|
719
|
+
* (its block's cluster ≤1 at insert) is pushed to the cohort once it grows — and writes only
|
|
720
|
+
* while the machine has not self-published; `'revouch'` rebinds the voucher to this node's
|
|
721
|
+
* owner key, leaving `UpdatedAt` alone, whether or not the machine has self-published
|
|
722
|
+
* (`CadreNode.removeOwner`).
|
|
705
723
|
*
|
|
706
|
-
* The
|
|
724
|
+
* The row read, the mode's decision, the stamp bump, the voucher signature, the write lock,
|
|
725
|
+
* the lost-update retry and the membership notify are
|
|
707
726
|
* {@link ControlDatabase.reauthorizeCadrePeer}'s — including the caveat that it rebinds
|
|
708
727
|
* `VouchOwner` to this node's owner key. What stays here is the owner-signer precondition.
|
|
709
728
|
*
|
|
710
|
-
* A no-op (no throw, no notify) when the row does not exist.
|
|
711
|
-
*
|
|
712
729
|
* @param peerId - the membership row to re-touch.
|
|
713
|
-
* @param
|
|
730
|
+
* @param mode - re-broadcast or re-vouch.
|
|
731
|
+
* @returns what was found and done ({@link CadrePeerRetouchOutcome}); an absent row, or a
|
|
732
|
+
* self-published one in `'rebroadcast'` mode, writes nothing and does not throw.
|
|
714
733
|
* @throws if no owner signer is configured (a non-owner cannot
|
|
715
734
|
* re-sign another peer's row) or the control database is not initialized.
|
|
716
735
|
*/
|
|
717
|
-
async reauthorizePeer(peerId: string,
|
|
736
|
+
async reauthorizePeer(peerId: string, mode: CadrePeerRetouchMode, sign?: OwnerSign): Promise<CadrePeerRetouchOutcome> {
|
|
718
737
|
// Fail fast on a signerless service before any DB read (see removePeer): a non-owner
|
|
719
738
|
// cannot re-sign the voucher, and must not silently no-op on an absent row.
|
|
720
739
|
const ownerKey = this.requireOwnerSigner().publicKey;
|
|
721
740
|
const controlDatabase = this.requireControlDatabase();
|
|
722
|
-
const
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
741
|
+
const action = mode === 'revouch' ? `Re-vouch the machine ${peerId}` : `Re-issue the membership of ${peerId}`;
|
|
742
|
+
const outcome = await this.withOwnerSign(action, sign, (approved) =>
|
|
743
|
+
controlDatabase.reauthorizeCadrePeer(peerId, mode, ownerKey, approved));
|
|
744
|
+
switch (outcome) {
|
|
745
|
+
case 'absent':
|
|
746
|
+
log('reauthorizePeer(%s): no CadrePeer row for %s (nothing to re-touch)', mode, peerId);
|
|
747
|
+
break;
|
|
748
|
+
case 'self-published':
|
|
749
|
+
log('reauthorizePeer(%s): %s has self-published; its row is its own to refresh', mode, peerId);
|
|
750
|
+
break;
|
|
751
|
+
case 'retouched':
|
|
752
|
+
log('Peer %s re-authorized (%s)', peerId, mode);
|
|
753
|
+
break;
|
|
727
754
|
}
|
|
728
|
-
|
|
755
|
+
return outcome;
|
|
729
756
|
}
|
|
730
757
|
|
|
731
758
|
/**
|
|
@@ -753,26 +780,6 @@ export class SeedBootstrapService {
|
|
|
753
780
|
controlDatabase.reissueRevocations(rows, reissuedAt, ownerKey, approved));
|
|
754
781
|
}
|
|
755
782
|
|
|
756
|
-
/**
|
|
757
|
-
* Owner filing of the singleton `Revocation` ledger marker, so the table is never a
|
|
758
|
-
* never-written block that the storage layer re-consults on every read.
|
|
759
|
-
*
|
|
760
|
-
* The signature, the insert-if-absent guard and the `'already-open'` mapping are
|
|
761
|
-
* {@link ControlDatabase.openRevocationLedger}'s. What stays here is the owner-signer
|
|
762
|
-
* precondition.
|
|
763
|
-
*
|
|
764
|
-
* @throws if no owner signer is configured or the control database is not
|
|
765
|
-
* initialized.
|
|
766
|
-
*/
|
|
767
|
-
async openRevocationLedger(sign?: OwnerSign): Promise<RevocationLedgerOpenResult> {
|
|
768
|
-
// Fail fast on a signerless service before any DB work (see removePeer): a
|
|
769
|
-
// non-owner cannot sign the marker's append digest.
|
|
770
|
-
const ownerKey = this.requireOwnerSigner().publicKey;
|
|
771
|
-
const controlDatabase = this.requireControlDatabase();
|
|
772
|
-
return await this.withOwnerSign('Open the cadre\'s removal ledger', sign, (approved) =>
|
|
773
|
-
controlDatabase.openRevocationLedger(ownerKey, approved));
|
|
774
|
-
}
|
|
775
|
-
|
|
776
783
|
/**
|
|
777
784
|
* Owner-signed INSERT of a `CadreInvite` row — an invitation any member machine can redeem
|
|
778
785
|
* on this owner's behalf ({@link ControlDatabase.insertCadreInvite}). The keypair mint, the
|
|
@@ -997,8 +1004,11 @@ export class SeedBootstrapService {
|
|
|
997
1004
|
* `CadreNode.dialColdStartBootstrap` retries these same addresses on every control-cohort
|
|
998
1005
|
* reconcile pass until the control database has siblings.
|
|
999
1006
|
*
|
|
1000
|
-
* The inbound handler runs this AFTER it has acked and
|
|
1001
|
-
* ({@link handleSeedStream}), because an unreachable peer can take `dialBudget.totalMs`.
|
|
1007
|
+
* The inbound handler runs this AFTER it has acked, closed the stream and reported the seed
|
|
1008
|
+
* applied ({@link handleSeedStream}), because an unreachable peer can take `dialBudget.totalMs`.
|
|
1009
|
+
* The node may stop while these dials are in flight (cadre-cli restarts a claimed node on
|
|
1010
|
+
* `seed:applied`): the node's stop aborts the dial in flight and every later one is refused
|
|
1011
|
+
* at once (`dialAddr`), and each is counted as failed here.
|
|
1002
1012
|
*/
|
|
1003
1013
|
private async dialPreferredPeers(seed: ControlNetworkSeed): Promise<PreferredDialCounts> {
|
|
1004
1014
|
const node = this.libp2pNode;
|
|
@@ -1006,23 +1016,16 @@ export class SeedBootstrapService {
|
|
|
1006
1016
|
log('Seed service shut down before its dials; none attempted');
|
|
1007
1017
|
return { preferredDialsAttempted: 0, preferredDialsFailed: 0 };
|
|
1008
1018
|
}
|
|
1009
|
-
//
|
|
1010
|
-
//
|
|
1011
|
-
//
|
|
1012
|
-
// Optional-chained: partial libp2p handles (unit-test doubles) omit `peerId`,
|
|
1013
|
-
// and an undefined self simply matches nothing.
|
|
1019
|
+
// Self is not dialed: dialing self always throws, which would report a healthy node as
|
|
1020
|
+
// "seeded but stranded". Optional-chained: partial libp2p handles (unit-test doubles)
|
|
1021
|
+
// omit `peerId`.
|
|
1014
1022
|
// Every one of a peer's addresses is a candidate, each on its own time limit
|
|
1015
1023
|
// (`dialPeerAddrs`), so a peer whose first address never answers neither
|
|
1016
1024
|
// stalls seed application nor goes undialed at its other addresses. The seed's
|
|
1017
1025
|
// addresses carry no peer id; the dial binds each to the peer's.
|
|
1018
|
-
const
|
|
1019
|
-
let preferredDialsAttempted = 0;
|
|
1026
|
+
const targets = seedDialTargets(seed, node.peerId?.toString());
|
|
1020
1027
|
let preferredDialsFailed = 0;
|
|
1021
|
-
for (const peer of
|
|
1022
|
-
if (peer.multiaddrs.length === 0 || peer.peerId === selfPeerId) {
|
|
1023
|
-
continue;
|
|
1024
|
-
}
|
|
1025
|
-
preferredDialsAttempted++;
|
|
1028
|
+
for (const peer of targets) {
|
|
1026
1029
|
try {
|
|
1027
1030
|
const addrs = parseDialAddrs(peer.multiaddrs);
|
|
1028
1031
|
|
|
@@ -1041,8 +1044,8 @@ export class SeedBootstrapService {
|
|
|
1041
1044
|
}
|
|
1042
1045
|
}
|
|
1043
1046
|
|
|
1044
|
-
log('Dialed preferred seed peers: %d/%d dial(s) failed', preferredDialsFailed,
|
|
1045
|
-
return { preferredDialsAttempted, preferredDialsFailed };
|
|
1047
|
+
log('Dialed preferred seed peers: %d/%d dial(s) failed', preferredDialsFailed, targets.length);
|
|
1048
|
+
return { preferredDialsAttempted: targets.length, preferredDialsFailed };
|
|
1046
1049
|
}
|
|
1047
1050
|
|
|
1048
1051
|
/**
|
|
@@ -1317,14 +1320,16 @@ export class SeedBootstrapService {
|
|
|
1317
1320
|
|
|
1318
1321
|
/**
|
|
1319
1322
|
* Read one inbound seed frame, verify and merge it, write the ack and close the
|
|
1320
|
-
* stream, then dial the seed's preferred peers.
|
|
1323
|
+
* stream, report the seed applied, then dial the seed's preferred peers.
|
|
1321
1324
|
*
|
|
1322
|
-
* The ack
|
|
1323
|
-
* a dial for `dialBudget.totalMs` (86 s at the default declared link), which is
|
|
1324
|
-
* link time and does not belong inside the sender's delivery deadline
|
|
1325
|
-
*
|
|
1326
|
-
*
|
|
1327
|
-
*
|
|
1325
|
+
* The ack, the close and `onSeedApplied` come BEFORE those dials: an unreachable peer
|
|
1326
|
+
* can hold a dial for `dialBudget.totalMs` (86 s at the default declared link), which is
|
|
1327
|
+
* not link time and does not belong inside the sender's delivery deadline, nor ahead of
|
|
1328
|
+
* a claimed node's restart (which stops this node, so its dials would be wasted). The
|
|
1329
|
+
* close is what releases the sender, which reads the ack to end-of-stream, and it is
|
|
1330
|
+
* awaited before `onSeedApplied`, so the ack is flushed before anything the callback
|
|
1331
|
+
* starts. The stream stays counted in {@link activeStreams} through the dials, so
|
|
1332
|
+
* {@link maxConcurrentSeeds} still bounds concurrent dial phases.
|
|
1328
1333
|
*
|
|
1329
1334
|
* Hardened against a buggy/compromised own-cadre node: a concurrency cap (over
|
|
1330
1335
|
* {@link maxConcurrentSeeds}, reply without applying), a read timeout (a peer
|
|
@@ -1361,8 +1366,8 @@ export class SeedBootstrapService {
|
|
|
1361
1366
|
);
|
|
1362
1367
|
|
|
1363
1368
|
if (merged.success) {
|
|
1364
|
-
await this.dialPreferredPeers(seed);
|
|
1365
1369
|
this.eventCallbacks.onSeedApplied?.(seed.partyId, merged.peersAdded, seed);
|
|
1370
|
+
await this.dialPreferredPeers(seed);
|
|
1366
1371
|
} else {
|
|
1367
1372
|
this.eventCallbacks.onSeedError?.(seed.partyId, merged.error ?? 'Unknown error');
|
|
1368
1373
|
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { PeerUpdate } from '@libp2p/interface';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Whether a libp2p `self:peer:update` changed the node's own addresses — the only part of
|
|
5
|
+
* the update the self record (`CadrePeer.Multiaddr`) carries.
|
|
6
|
+
*
|
|
7
|
+
* libp2p emits the event for any change to the node's own peer-store entry, and registering
|
|
8
|
+
* or unregistering a protocol handler is one (the registrar merges the node's protocol list).
|
|
9
|
+
* Cadre registers handlers on a running control node (member redemption, strand formation,
|
|
10
|
+
* the seed listener), so reacting to every event would make each of those a consensus write
|
|
11
|
+
* of an unchanged record, racing whatever write the cohort is making to `CadrePeer` at the
|
|
12
|
+
* time. An update with no `previous` is treated as a change: there is nothing to compare.
|
|
13
|
+
*/
|
|
14
|
+
export function selfAddressesChanged(update: PeerUpdate): boolean {
|
|
15
|
+
if (update.previous === undefined) return true;
|
|
16
|
+
const before = new Set(update.previous.addresses.map((address) => address.multiaddr.toString()));
|
|
17
|
+
const after = update.peer.addresses.map((address) => address.multiaddr.toString());
|
|
18
|
+
return after.length !== before.size || after.some((address) => !before.has(address));
|
|
19
|
+
}
|