@serfab/cadre-core 1.14.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +293 -274
- package/dist/cadre-invite-protocol.d.ts +64 -29
- package/dist/cadre-invite-protocol.js +207 -38
- package/dist/cadre-invite-protocol.js.map +1 -1
- package/dist/cadre-join-request.d.ts +42 -0
- package/dist/cadre-join-request.js +144 -0
- package/dist/cadre-join-request.js.map +1 -0
- package/dist/cadre-node.d.ts +375 -79
- package/dist/cadre-node.js +758 -186
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-database.d.ts +92 -12
- package/dist/control-database.js +323 -142
- package/dist/control-database.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +19 -3
- package/dist/control-schema.js.map +1 -1
- package/dist/ed25519-key.d.ts +7 -9
- package/dist/ed25519-key.js +20 -10
- package/dist/ed25519-key.js.map +1 -1
- package/dist/fetch-impl.d.ts +12 -0
- package/dist/fetch-impl.js +23 -0
- package/dist/fetch-impl.js.map +1 -0
- package/dist/file-durable-slot.js +2 -1
- package/dist/file-durable-slot.js.map +1 -1
- package/dist/file-safe-name.d.ts +25 -0
- package/dist/file-safe-name.js +29 -0
- package/dist/file-safe-name.js.map +1 -0
- package/dist/formation-approval.js +5 -11
- package/dist/formation-approval.js.map +1 -1
- package/dist/fs-atomic.d.ts +0 -19
- package/dist/fs-atomic.js +3 -24
- package/dist/fs-atomic.js.map +1 -1
- package/dist/index.d.ts +12 -5
- package/dist/index.js +31 -6
- package/dist/index.js.map +1 -1
- package/dist/invitation-bootstrap.d.ts +50 -12
- package/dist/invitation-bootstrap.js +76 -13
- package/dist/invitation-bootstrap.js.map +1 -1
- package/dist/joined-strand-store.d.ts +7 -0
- package/dist/joined-strand-store.js +9 -0
- package/dist/joined-strand-store.js.map +1 -1
- package/dist/key-store-file.js +2 -1
- package/dist/key-store-file.js.map +1 -1
- package/dist/link-budget.d.ts +9 -4
- package/dist/link-budget.js +9 -4
- package/dist/link-budget.js.map +1 -1
- package/dist/machine-removal.d.ts +78 -0
- package/dist/machine-removal.js +166 -0
- package/dist/machine-removal.js.map +1 -0
- package/dist/membership-connection-gater.d.ts +6 -5
- package/dist/membership-connection-gater.js +6 -5
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/node-claim-payload.d.ts +55 -0
- package/dist/node-claim-payload.js +108 -4
- package/dist/node-claim-payload.js.map +1 -1
- package/dist/node-default-transports.d.ts +27 -0
- package/dist/node-default-transports.js +35 -0
- package/dist/node-default-transports.js.map +1 -0
- package/dist/node-transports.d.ts +30 -0
- package/dist/node-transports.js +40 -0
- package/dist/node-transports.js.map +1 -0
- package/dist/peer-dial.d.ts +124 -27
- package/dist/peer-dial.js +254 -30
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.js +6 -0
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/peer-record.d.ts +6 -45
- package/dist/peer-record.js +8 -50
- package/dist/peer-record.js.map +1 -1
- package/dist/primary-lan-address.d.ts +8 -0
- package/dist/primary-lan-address.js +46 -0
- package/dist/primary-lan-address.js.map +1 -0
- package/dist/provider/client.d.ts +131 -0
- package/dist/provider/client.js +226 -0
- package/dist/provider/client.js.map +1 -0
- package/dist/provider/descriptor.d.ts +41 -0
- package/dist/provider/descriptor.js +103 -0
- package/dist/provider/descriptor.js.map +1 -0
- package/dist/provider/index.d.ts +9 -0
- package/dist/provider/index.js +10 -0
- package/dist/provider/index.js.map +1 -0
- package/dist/provider/oauth.d.ts +125 -0
- package/dist/provider/oauth.js +199 -0
- package/dist/provider/oauth.js.map +1 -0
- package/dist/provider/rent-node.d.ts +101 -0
- package/dist/provider/rent-node.js +220 -0
- package/dist/provider/rent-node.js.map +1 -0
- package/dist/relay-addrs.d.ts +9 -4
- package/dist/relay-addrs.js +9 -4
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +51 -16
- package/dist/relay-reservation.js +107 -24
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +22 -5
- package/dist/seed-bootstrap.js +48 -25
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-commit-durability.d.ts +54 -0
- package/dist/strand-commit-durability.js +74 -0
- package/dist/strand-commit-durability.js.map +1 -0
- package/dist/strand-database.d.ts +23 -8
- package/dist/strand-database.js +41 -13
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +7 -1
- package/dist/strand-first-sync-gate.js +7 -1
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-protocol.js +4 -1
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-fret-addrs.d.ts +19 -1
- package/dist/strand-fret-addrs.js +27 -9
- package/dist/strand-fret-addrs.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +81 -19
- package/dist/strand-instance-manager.js +216 -105
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-move.d.ts +124 -0
- package/dist/strand-move.js +438 -0
- package/dist/strand-move.js.map +1 -0
- package/dist/strand-network-config.js +7 -6
- package/dist/strand-network-config.js.map +1 -1
- package/dist/strand-peer-redial.d.ts +113 -0
- package/dist/strand-peer-redial.js +236 -0
- package/dist/strand-peer-redial.js.map +1 -0
- package/dist/strand-wake-protocol.js +10 -7
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/types.d.ts +212 -4
- package/dist/types.js +10 -0
- package/dist/types.js.map +1 -1
- package/package.json +18 -10
- package/src/cadre-invite-protocol.ts +1052 -864
- package/src/cadre-join-request.ts +167 -0
- package/src/cadre-node.ts +10003 -9381
- package/src/control-database.ts +4663 -4451
- package/src/control-schema.ts +19 -3
- package/src/ed25519-key.ts +20 -10
- package/src/fetch-impl.ts +23 -0
- package/src/file-durable-slot.ts +64 -63
- package/src/file-safe-name.ts +32 -0
- package/src/formation-approval.ts +604 -610
- package/src/fs-atomic.ts +92 -117
- package/src/index.ts +71 -4
- package/src/invitation-bootstrap.ts +91 -13
- package/src/joined-strand-store.ts +508 -498
- package/src/key-store-file.ts +142 -141
- package/src/link-budget.ts +9 -4
- package/src/machine-removal.ts +216 -0
- package/src/membership-connection-gater.ts +638 -637
- package/src/node-claim-payload.ts +138 -4
- package/src/node-default-transports.ts +38 -0
- package/src/node-transports.ts +49 -0
- package/src/peer-dial.ts +604 -336
- package/src/peer-join-backfill.ts +6 -0
- package/src/peer-record.ts +159 -202
- package/src/primary-lan-address.ts +47 -0
- package/src/provider/client.ts +326 -0
- package/src/provider/descriptor.ts +129 -0
- package/src/provider/index.ts +59 -0
- package/src/provider/oauth.ts +285 -0
- package/src/provider/rent-node.ts +275 -0
- package/src/relay-addrs.ts +9 -4
- package/src/relay-reservation.ts +1211 -1121
- package/src/seed-bootstrap.ts +1502 -1471
- package/src/strand-commit-durability.ts +116 -0
- package/src/strand-database.ts +46 -14
- package/src/strand-first-sync-gate.ts +7 -1
- package/src/strand-formation-protocol.ts +1039 -1036
- package/src/strand-fret-addrs.ts +179 -145
- package/src/strand-instance-manager.ts +2026 -1883
- package/src/strand-move.ts +585 -0
- package/src/strand-network-config.ts +7 -6
- package/src/strand-peer-redial.ts +307 -0
- package/src/strand-wake-protocol.ts +10 -7
- package/src/types.ts +220 -4
|
@@ -13,17 +13,25 @@ import { assertStrandScopeKey } from './storage-scope.js';
|
|
|
13
13
|
import { StrandFirstSyncGate, StrandAwaitingFirstSyncError, strandFirstSyncComplete, DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS } from './strand-first-sync-gate.js';
|
|
14
14
|
import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
|
|
15
15
|
import { strandNodeAddrs } from './strand-network-config.js';
|
|
16
|
+
import { nodeTransportsOption } from './node-transports.js';
|
|
16
17
|
import { resolveRelayServer } from './relay-server.js';
|
|
17
18
|
import { strandCohortTopicOption } from './strand-reactivity.js';
|
|
18
19
|
import { superviseRelayReservation } from './relay-reservation.js';
|
|
20
|
+
import { strandRedialTimings, superviseStrandPeerRedial } from './strand-peer-redial.js';
|
|
21
|
+
import { groupAddrsByPeerId } from './peer-addr-book.js';
|
|
19
22
|
import { isCohortUnreachableRead } from './control-read-retry.js';
|
|
20
23
|
import { declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
21
24
|
const log = debug('sereus:cadre:strand-manager');
|
|
22
25
|
/**
|
|
23
26
|
* How often a joining launch that could not build its database, because no seed peer had
|
|
24
|
-
* answered,
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
+
* answered, retries the build on its running libp2p node
|
|
28
|
+
* (`StrandInstanceManager.deferUnreachableJoin`). Each attempt composes a fresh database and
|
|
29
|
+
* re-reads the schema catalog over the network, so it is spaced well above the first-sync
|
|
30
|
+
* gate's probe cadence.
|
|
31
|
+
*
|
|
32
|
+
* NOTE: the 5 s dates from when each attempt also rebuilt the libp2p node. A joiner whose seed
|
|
33
|
+
* peer answered just after a failed attempt waits up to this long to become writable; if that
|
|
34
|
+
* wait shows up in a measured join, try a shorter first retry.
|
|
27
35
|
*/
|
|
28
36
|
export const UNREACHABLE_JOIN_RETRY_MS = 5000;
|
|
29
37
|
const timing = debug('sereus:cadre:timing');
|
|
@@ -194,6 +202,13 @@ export class StrandInstanceManager {
|
|
|
194
202
|
* `CadreNode.reserveRelays`.
|
|
195
203
|
*/
|
|
196
204
|
this.relaySupervisors = new Map();
|
|
205
|
+
/**
|
|
206
|
+
* The per-strand redial of the strand's other members (`strand-peer-redial.ts`), keyed by
|
|
207
|
+
* strand id. Same lifecycle as {@link relaySupervisors}: started in `buildStrandRuntime` once
|
|
208
|
+
* the libp2p node exists, stopped first in `releaseRuntime`, so a hibernating strand dials
|
|
209
|
+
* nobody and a resume redials over the new node.
|
|
210
|
+
*/
|
|
211
|
+
this.peerRedials = new Map();
|
|
197
212
|
/**
|
|
198
213
|
* The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
|
|
199
214
|
* store, resolved once in `startStrand` and held until `stopStrand` disposes it.
|
|
@@ -216,10 +231,11 @@ export class StrandInstanceManager {
|
|
|
216
231
|
this.strandStorages = new Map();
|
|
217
232
|
/**
|
|
218
233
|
* The runtime build in flight per strand id (`buildStrandRuntime`, from `startStrand` or
|
|
219
|
-
* `resumeStrand
|
|
220
|
-
* with its database not yet constructed or not yet
|
|
221
|
-
* on the FINISHED runtime ({@link attachSApp},
|
|
222
|
-
* rejection belongs to the
|
|
234
|
+
* `resumeStrand`, or a deferred join's database retry), present until it settles. The
|
|
235
|
+
* instance is tracked for the whole build, with its database not yet constructed or not yet
|
|
236
|
+
* initialized, so a caller that must act on the FINISHED runtime ({@link attachSApp},
|
|
237
|
+
* {@link whenRuntimeBuilt}) waits on this instead. A failed build's rejection belongs to the
|
|
238
|
+
* call that started it.
|
|
223
239
|
*/
|
|
224
240
|
this.runtimeBuilds = new Map();
|
|
225
241
|
/**
|
|
@@ -232,9 +248,9 @@ export class StrandInstanceManager {
|
|
|
232
248
|
*/
|
|
233
249
|
this.transitions = new Map();
|
|
234
250
|
/**
|
|
235
|
-
* The pending
|
|
251
|
+
* The pending database retry per strand id of a launch whose database could not be built
|
|
236
252
|
* because no seed peer had answered yet ({@link deferUnreachableJoin}). Cleared by
|
|
237
|
-
* {@link releaseRuntime}, so a stop
|
|
253
|
+
* {@link releaseRuntime}, so a stop or a quiesce cancels it.
|
|
238
254
|
*/
|
|
239
255
|
this.unreachableJoinRetries = new Map();
|
|
240
256
|
this.stopping = false;
|
|
@@ -353,7 +369,7 @@ export class StrandInstanceManager {
|
|
|
353
369
|
async trackRuntimeBuild(strandId, build) {
|
|
354
370
|
this.runtimeBuilds.set(strandId, build);
|
|
355
371
|
try {
|
|
356
|
-
await build;
|
|
372
|
+
return await build;
|
|
357
373
|
}
|
|
358
374
|
finally {
|
|
359
375
|
if (this.runtimeBuilds.get(strandId) === build) {
|
|
@@ -386,10 +402,13 @@ export class StrandInstanceManager {
|
|
|
386
402
|
* cohort-derived values. Storage is the
|
|
387
403
|
* one input it does NOT re-read from `config`: that belongs to the instance and
|
|
388
404
|
* comes from `strandStorages`.
|
|
405
|
+
*
|
|
406
|
+
* A database that cannot be built because no seed peer has answered yet does not fail the
|
|
407
|
+
* build: the runtime comes up `'syncing'` without it and the database phase alone is
|
|
408
|
+
* retried on the same node ({@link deferUnreachableJoin}).
|
|
389
409
|
*/
|
|
390
410
|
async buildStrandRuntime(instance, config) {
|
|
391
411
|
const strandId = instance.strandId;
|
|
392
|
-
const { sAppConfig } = config;
|
|
393
412
|
// Refuse a bad `linkRoundTripMs` before any strand bring-up, for the reason
|
|
394
413
|
// `CadreNode.start()` does: every budget derived from it here is behind a condition
|
|
395
414
|
// (backfill disabled, no relay addrs), so a zero or NaN declaration would otherwise
|
|
@@ -403,9 +422,6 @@ export class StrandInstanceManager {
|
|
|
403
422
|
// services must derive its prefix from the SAME string the node was built
|
|
404
423
|
// with — hence one binding for both, not two literals that can drift.
|
|
405
424
|
const networkName = `strand-${strandId}`;
|
|
406
|
-
// Set when the database build failed only because no seed peer has answered yet; see
|
|
407
|
-
// {@link deferUnreachableJoin}.
|
|
408
|
-
let joinWaitsForSeed = false;
|
|
409
425
|
const protocolPrefix = `/optimystic/${networkName}`;
|
|
410
426
|
// Whether this node runs the circuit-relay server, and its init — the same
|
|
411
427
|
// resolution the control node takes from the same config (`relay-server.ts`).
|
|
@@ -521,7 +537,12 @@ export class StrandInstanceManager {
|
|
|
521
537
|
// remembered-serving fallback); dropping `persistence` brings back the split
|
|
522
538
|
// restart.
|
|
523
539
|
...(config.networkState && { persistence: strandNetworkStatePersistence(config.networkState, strandId) }),
|
|
524
|
-
|
|
540
|
+
// With relays configured, built with libp2p's relay discovery off (given a transport
|
|
541
|
+
// list to wrap — `node-transports.ts`), so the supervisors below are the only thing
|
|
542
|
+
// that fills the bare `/p2p-circuit` slots: otherwise a cohort seed running the relay
|
|
543
|
+
// server takes one as soon as it connects (`relay-reservation.ts`, "WHY LIBP2P'S
|
|
544
|
+
// RELAY DISCOVERY IS SWITCHED OFF").
|
|
545
|
+
...nodeTransportsOption(config.network, addrOptions, relayDialAddrs.length > 0),
|
|
525
546
|
...(config.network?.noiseCrypto && { noiseCrypto: config.network.noiseCrypto }),
|
|
526
547
|
// Unconditional, and the same default the control node takes: every node of
|
|
527
548
|
// every party runs the monitor over its connection to a slow phone, so the
|
|
@@ -537,13 +558,12 @@ export class StrandInstanceManager {
|
|
|
537
558
|
...optimysticDialLimits(config.network?.linkRoundTripMs),
|
|
538
559
|
// Listen entries plus the WebSocket transport switch they imply — a strand node
|
|
539
560
|
// announces nothing the operator configured (`strand-network-config.ts`), and
|
|
540
|
-
// spreads AFTER `transports` above because the switch is a no-op whenever
|
|
541
|
-
//
|
|
542
|
-
//
|
|
543
|
-
//
|
|
544
|
-
//
|
|
545
|
-
//
|
|
546
|
-
// see cadre-node.ts).
|
|
561
|
+
// spreads AFTER `transports` above because the switch is a no-op whenever a
|
|
562
|
+
// transport list was passed. The bare `/p2p-circuit` entries (one per relay) open
|
|
563
|
+
// no connection here; each registers a pending reservation that the matching
|
|
564
|
+
// supervisor below fills. That is what gives a NAT'd strand node a reachable relay
|
|
565
|
+
// slot — and works against a party-run relay because the launch path announced
|
|
566
|
+
// this strand's derived peerId to it first (delegate admission; see cadre-node.ts).
|
|
547
567
|
...addrOptions,
|
|
548
568
|
// The raw configured gater (open strands), or the revocation-composed
|
|
549
569
|
// gater plus the fail-closed per-stream revoked-peer gate (closed
|
|
@@ -562,72 +582,37 @@ export class StrandInstanceManager {
|
|
|
562
582
|
// map immediately so the failure rollback (releaseRuntime) stops them like every
|
|
563
583
|
// other runtime component.
|
|
564
584
|
this.relaySupervisors.set(strandId, this.startRelaySupervisors(strandId, node, relayDialAddrs, config));
|
|
565
|
-
//
|
|
566
|
-
//
|
|
567
|
-
//
|
|
568
|
-
//
|
|
569
|
-
|
|
570
|
-
// sees `instance.database` once this machine is allowed to write to the strand.
|
|
571
|
-
t0 = performance.now();
|
|
572
|
-
const strandDb = new StrandDatabase({
|
|
573
|
-
strandId,
|
|
574
|
-
sAppConfig,
|
|
575
|
-
libp2pNode: node,
|
|
576
|
-
coordinatedRepo: node.coordinatedRepo,
|
|
577
|
-
// Founder bootstrap inputs: the strand's type drives which membership rows
|
|
578
|
-
// are written, and the PARTY's own key (partyMemberPrivateKey, resolved by the
|
|
579
|
-
// caller from the control-layer StrandPartyKey row) derives the founding
|
|
580
|
-
// Member/Manager key. The row's shared MemberPrivateKey rides along as the
|
|
581
|
-
// read-gating secret only — it derives nobody's identity.
|
|
582
|
-
strandType: config.strandRow.Type,
|
|
583
|
-
memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
|
|
584
|
-
partyMemberPrivateKey: config.partyMemberPrivateKey,
|
|
585
|
-
founder: config.founder
|
|
586
|
-
});
|
|
587
|
-
const gate = new StrandFirstSyncGate({
|
|
585
|
+
// Redials the strand's other members whenever none is connected, on the budget the
|
|
586
|
+
// declared link gives a relayed dial — what reconnects two restarted phones on a slow
|
|
587
|
+
// link, where FRET's own redial cannot (`strand-peer-redial.ts`). The relays are
|
|
588
|
+
// excluded by peer id: the node is connected to them throughout, and they are no member.
|
|
589
|
+
this.peerRedials.set(strandId, superviseStrandPeerRedial({
|
|
588
590
|
label: strandId,
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
|
|
601
|
-
// The first-sync write gate. A founder just wrote the Header in its bootstrap; any
|
|
602
|
-
// other machine may commit only once it holds the Header — which it can only have
|
|
603
|
-
// received from a peer (now, or on an earlier run over the same store) — and has read
|
|
604
|
-
// each App table once, so their collections are fetched too. Publishing the database
|
|
605
|
-
// is what makes the strand writable to the app; until then the instance stays
|
|
606
|
-
// `'syncing'` and the gate re-probes on its cadence. Everything armed below reads
|
|
607
|
-
// `instance.database` lazily, so the reconciler and enforcer wait with it — and
|
|
608
|
-
// `publishDatabase` kicks the reconciler when the wait ends, so a joiner's membership
|
|
609
|
-
// rows land with the strand rather than one retry later.
|
|
610
|
-
t0 = performance.now();
|
|
611
|
-
if (config.founder === true || await strandFirstSyncComplete(strandDb.getDatabase(), strandId)) {
|
|
612
|
-
this.publishDatabase(instance, strandDb);
|
|
613
|
-
}
|
|
614
|
-
else {
|
|
615
|
-
gate.start();
|
|
591
|
+
node,
|
|
592
|
+
seed: config.bootstrapNodes ?? [],
|
|
593
|
+
excluded: new Set(groupAddrsByPeerId([...relayDialAddrs]).keys()),
|
|
594
|
+
...(revocationEnforcer && { isRevoked: (peerId) => revocationEnforcer.isRevoked(peerId) })
|
|
595
|
+
}, strandRedialTimings(config.network?.linkRoundTripMs)));
|
|
596
|
+
const databaseBuild = await this.buildStrandDatabase(instance, config, node);
|
|
597
|
+
if (!this.isCurrentRuntime(instance, node)) {
|
|
598
|
+
// A stop released this runtime during the database phase (`stopStrand` does not
|
|
599
|
+
// wait for builds); arming the loops below would restart them on a stopped node.
|
|
600
|
+
log('Strand %s was stopped while its runtime was being built', strandId);
|
|
601
|
+
return;
|
|
616
602
|
}
|
|
617
|
-
timing('[buildStrandRuntime:%s] first-sync probe: %dms', strandId, Math.round(performance.now() - t0));
|
|
618
603
|
// The membership reconciler: finish this party's join on every machine —
|
|
619
604
|
// redeem a staged formation invitation if one is pending, then write the
|
|
620
605
|
// durable machine→party binding (Strand.MemberPeer). Armed here, the one
|
|
621
|
-
// seam where the strand's transport peer id and
|
|
622
|
-
// for launch AND hibernation resume, for closed strands that carry the
|
|
606
|
+
// seam where the strand's transport peer id exists and its database has been
|
|
607
|
+
// attempted, for launch AND hibernation resume, for closed strands that carry the
|
|
623
608
|
// party's own membership key. start() kicks an immediate pass WITHOUT
|
|
624
609
|
// awaiting it — a joiner at this instant has not synced the founder's rows
|
|
625
610
|
// and may lack write quorum, so bring-up is never blocked; an unfinished join
|
|
626
611
|
// then retries on a short doubling ladder capped at the configured
|
|
627
612
|
// pollIntervalMs (which is also the flat cadence while nobody has admitted this
|
|
628
613
|
// party yet), and the loop stops once the member row and this machine's own
|
|
629
|
-
// binding are both in place. Armed AFTER the
|
|
630
|
-
// always has a reconciler registered by the time
|
|
614
|
+
// binding are both in place. Armed AFTER the database phase above, so a gated or
|
|
615
|
+
// deferred launch always has a reconciler registered by the time it can publish.
|
|
631
616
|
if (config.strandRow.Type === 'c' && config.partyMemberPrivateKey
|
|
632
617
|
&& config.membershipReconciliation?.enabled !== false) {
|
|
633
618
|
const reconciler = new StrandMembershipReconciler({
|
|
@@ -721,7 +706,12 @@ export class StrandInstanceManager {
|
|
|
721
706
|
// No `lastActivity` stamp: bringing a runtime up is not activity. A check-in marks
|
|
722
707
|
// activity before its resume and must see only what landed while the strand was up.
|
|
723
708
|
instance.status = liveStrandStatus(instance);
|
|
724
|
-
if (
|
|
709
|
+
if (databaseBuild === 'awaiting-seed') {
|
|
710
|
+
// Armed only now that the build is over: a retry firing mid-build would find the
|
|
711
|
+
// build in flight.
|
|
712
|
+
this.deferUnreachableJoin(instance);
|
|
713
|
+
}
|
|
714
|
+
else if (instance.status === 'syncing') {
|
|
725
715
|
log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
|
|
726
716
|
'a member of the strand is reached', strandId);
|
|
727
717
|
}
|
|
@@ -734,16 +724,105 @@ export class StrandInstanceManager {
|
|
|
734
724
|
await this.releaseRuntime(instance).catch((cleanupErr) => {
|
|
735
725
|
log('buildStrandRuntime cleanup for strand %s also failed: %o', strandId, cleanupErr);
|
|
736
726
|
});
|
|
737
|
-
if (joinWaitsForSeed) {
|
|
738
|
-
this.deferUnreachableJoin(instance, error);
|
|
739
|
-
return;
|
|
740
|
-
}
|
|
741
727
|
throw error;
|
|
742
728
|
}
|
|
743
729
|
}
|
|
730
|
+
/**
|
|
731
|
+
* The database phase of a runtime build, on the libp2p node already attached: create the
|
|
732
|
+
* StrandDatabase, initialize it, then publish it or arm its first-sync gate. Run by
|
|
733
|
+
* {@link buildStrandRuntime} and again by each {@link retryUnreachableJoin}, with a new
|
|
734
|
+
* StrandDatabase per attempt.
|
|
735
|
+
*
|
|
736
|
+
* Returns `'awaiting-seed'` when initialize failed only because no seed peer has answered
|
|
737
|
+
* (`cohort-unreachable`): that attempt's database is closed and the node keeps running. Any
|
|
738
|
+
* other failure throws, with the attempt's database left with its gate for
|
|
739
|
+
* {@link releaseRuntime} to close.
|
|
740
|
+
*/
|
|
741
|
+
async buildStrandDatabase(instance, config, node) {
|
|
742
|
+
const strandId = instance.strandId;
|
|
743
|
+
// Held by the first-sync gate (registered before initialize) rather than attached
|
|
744
|
+
// to the instance: a failed init is then cleaned up through the gate (close() is safe on
|
|
745
|
+
// a partially-initialized db), and the app only ever sees `instance.database` once this
|
|
746
|
+
// machine is allowed to write to the strand.
|
|
747
|
+
let t0 = performance.now();
|
|
748
|
+
const strandDb = new StrandDatabase({
|
|
749
|
+
strandId,
|
|
750
|
+
sAppConfig: config.sAppConfig,
|
|
751
|
+
libp2pNode: node,
|
|
752
|
+
coordinatedRepo: node.coordinatedRepo,
|
|
753
|
+
// Founder bootstrap inputs: the strand's type drives which membership rows
|
|
754
|
+
// are written, and the PARTY's own key (partyMemberPrivateKey, resolved by the
|
|
755
|
+
// caller from the control-layer StrandPartyKey row) derives the founding
|
|
756
|
+
// Member/Manager key. The row's shared MemberPrivateKey rides along as the
|
|
757
|
+
// read-gating secret only — it derives nobody's identity.
|
|
758
|
+
strandType: config.strandRow.Type,
|
|
759
|
+
memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
|
|
760
|
+
partyMemberPrivateKey: config.partyMemberPrivateKey,
|
|
761
|
+
founder: config.founder,
|
|
762
|
+
onCommitDurability: config.onCommitDurability,
|
|
763
|
+
onCommitHeld: config.onCommitHeld
|
|
764
|
+
});
|
|
765
|
+
const gate = new StrandFirstSyncGate({
|
|
766
|
+
label: strandId,
|
|
767
|
+
database: strandDb,
|
|
768
|
+
onHeaderHeld: () => this.publishDatabase(instance, strandDb)
|
|
769
|
+
}, config.firstSync);
|
|
770
|
+
this.firstSyncGates.set(strandId, gate);
|
|
771
|
+
try {
|
|
772
|
+
await strandDb.initialize();
|
|
773
|
+
}
|
|
774
|
+
catch (error) {
|
|
775
|
+
if (!isCohortUnreachableRead(error)) {
|
|
776
|
+
throw error;
|
|
777
|
+
}
|
|
778
|
+
await this.discardDatabaseAttempt(strandId, gate);
|
|
779
|
+
log('Strand %s: no seed peer has answered yet, so this joiner cannot build its database (%s); ' +
|
|
780
|
+
'its runtime stays up and the build is retried every %dms', strandId, error instanceof Error ? error.message : String(error), UNREACHABLE_JOIN_RETRY_MS);
|
|
781
|
+
return 'awaiting-seed';
|
|
782
|
+
}
|
|
783
|
+
timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
|
|
784
|
+
if (this.firstSyncGates.get(strandId) !== gate) {
|
|
785
|
+
// A stop released the runtime while initialize ran, closing this database through its
|
|
786
|
+
// gate part-way; close what initialize finished after that.
|
|
787
|
+
await strandDb.close();
|
|
788
|
+
return 'released';
|
|
789
|
+
}
|
|
790
|
+
// The first-sync write gate. A founder just wrote the Header in its bootstrap; any
|
|
791
|
+
// other machine may commit only once it holds the Header — which it can only have
|
|
792
|
+
// received from a peer (now, or on an earlier run over the same store) — and has read
|
|
793
|
+
// each App table once, so their collections are fetched too. Publishing the database
|
|
794
|
+
// is what makes the strand writable to the app; until then the instance stays
|
|
795
|
+
// `'syncing'` and the gate re-probes on its cadence. Everything armed after the database
|
|
796
|
+
// phase reads `instance.database` lazily, so the reconciler and enforcer wait with it —
|
|
797
|
+
// and `publishDatabase` kicks the reconciler when the wait ends, so a joiner's membership
|
|
798
|
+
// rows land with the strand rather than one retry later.
|
|
799
|
+
t0 = performance.now();
|
|
800
|
+
const synced = config.founder === true || await strandFirstSyncComplete(strandDb.getDatabase(), strandId);
|
|
801
|
+
timing('[buildStrandRuntime:%s] first-sync probe: %dms', strandId, Math.round(performance.now() - t0));
|
|
802
|
+
if (this.firstSyncGates.get(strandId) !== gate) {
|
|
803
|
+
// A stop released the runtime during the probe and closed this database through its gate.
|
|
804
|
+
return 'released';
|
|
805
|
+
}
|
|
806
|
+
if (synced) {
|
|
807
|
+
this.publishDatabase(instance, strandDb);
|
|
808
|
+
}
|
|
809
|
+
else {
|
|
810
|
+
gate.start();
|
|
811
|
+
}
|
|
812
|
+
return 'built';
|
|
813
|
+
}
|
|
814
|
+
/** Close a database attempt that failed to initialize, and drop its gate if it is still the registered one. */
|
|
815
|
+
async discardDatabaseAttempt(strandId, gate) {
|
|
816
|
+
gate.stop();
|
|
817
|
+
if (this.firstSyncGates.get(strandId) === gate) {
|
|
818
|
+
this.firstSyncGates.delete(strandId);
|
|
819
|
+
}
|
|
820
|
+
await gate.database.close();
|
|
821
|
+
}
|
|
744
822
|
/**
|
|
745
823
|
* Keep a launch `'syncing'` when its database could not be built because no seed peer has
|
|
746
|
-
* answered yet, and
|
|
824
|
+
* answered yet, and retry the database build every {@link UNREACHABLE_JOIN_RETRY_MS} on the
|
|
825
|
+
* runtime already up.
|
|
747
826
|
*
|
|
748
827
|
* Optimystic (since 1.11.0) will not rule a block never-created while any of a node's
|
|
749
828
|
* `bootstrapNodes` is unheard, so a joiner alone with an unreachable seed cannot build even
|
|
@@ -753,14 +832,15 @@ export class StrandInstanceManager {
|
|
|
753
832
|
* strand until the watcher relaunched it, and `addStrand` would reject with a raw Optimystic
|
|
754
833
|
* error. Deferring gives a joiner the same contract the first-sync gate gives one that did
|
|
755
834
|
* build: the strand stays tracked, {@link whenWritable} waits and rejects with the retryable
|
|
756
|
-
* `StrandAwaitingFirstSyncError`, and the launch proceeds once a seed peer answers.
|
|
757
|
-
*
|
|
835
|
+
* `StrandAwaitingFirstSyncError`, and the launch proceeds once a seed peer answers.
|
|
836
|
+
*
|
|
837
|
+
* The libp2p node, its connections and the background loops stay up meanwhile. A seed peer
|
|
838
|
+
* can answer just after the schema read gave up on it — on a slow link it connects and its
|
|
839
|
+
* peer-join backfill delivers the whole store — and the retry then builds the database, and
|
|
840
|
+
* runs the first-sync probe, while that peer is still connected.
|
|
758
841
|
*/
|
|
759
|
-
deferUnreachableJoin(instance
|
|
842
|
+
deferUnreachableJoin(instance) {
|
|
760
843
|
const strandId = instance.strandId;
|
|
761
|
-
instance.status = 'syncing';
|
|
762
|
-
log('Strand %s: no seed peer has answered yet, so this joiner cannot build its database (%s); ' +
|
|
763
|
-
'retrying in %dms', strandId, error instanceof Error ? error.message : String(error), UNREACHABLE_JOIN_RETRY_MS);
|
|
764
844
|
const timer = setTimeout(() => {
|
|
765
845
|
this.unreachableJoinRetries.delete(strandId);
|
|
766
846
|
void this.retryUnreachableJoin(instance);
|
|
@@ -768,22 +848,48 @@ export class StrandInstanceManager {
|
|
|
768
848
|
timer.unref?.();
|
|
769
849
|
this.unreachableJoinRetries.set(strandId, timer);
|
|
770
850
|
}
|
|
771
|
-
/**
|
|
851
|
+
/**
|
|
852
|
+
* One retry scheduled by {@link deferUnreachableJoin}: the database phase again, on the
|
|
853
|
+
* runtime still attached. Skipped when the strand was stopped or released since — a quiesce
|
|
854
|
+
* releases it and the resume after rebuilds the whole runtime — or already has a database.
|
|
855
|
+
* A failure other than an unanswered seed releases the runtime and marks the strand
|
|
856
|
+
* `'error'`, as a failed resume does.
|
|
857
|
+
*/
|
|
772
858
|
async retryUnreachableJoin(instance) {
|
|
773
859
|
const strandId = instance.strandId;
|
|
774
860
|
const config = this.launchConfigs.get(strandId);
|
|
775
|
-
|
|
776
|
-
|
|
861
|
+
// Every strand node is built by `buildStrandRuntime`, as a node with its coordinated repo.
|
|
862
|
+
const node = instance.libp2pNode;
|
|
863
|
+
if (!config || !node || !this.isCurrentRuntime(instance, node) || instance.database
|
|
864
|
+
|| this.firstSyncGates.has(strandId) || this.isQuiescing(strandId)) {
|
|
777
865
|
return;
|
|
778
866
|
}
|
|
867
|
+
let build;
|
|
779
868
|
try {
|
|
780
|
-
|
|
869
|
+
// Tracked like any runtime build, so a wake, a quiesce or an sApp attach waits for it
|
|
870
|
+
// rather than acting on a database that is still initializing.
|
|
871
|
+
build = await this.trackRuntimeBuild(strandId, this.buildStrandDatabase(instance, config, node));
|
|
781
872
|
}
|
|
782
873
|
catch (error) {
|
|
874
|
+
if (!this.isCurrentRuntime(instance, node)) {
|
|
875
|
+
log('Strand %s: a database retry failed after its runtime was released: %o', strandId, error);
|
|
876
|
+
return;
|
|
877
|
+
}
|
|
878
|
+
await this.releaseRuntime(instance).catch((cleanupErr) => {
|
|
879
|
+
log('Strand %s: releasing the runtime after a failed database retry also failed: %o', strandId, cleanupErr);
|
|
880
|
+
});
|
|
783
881
|
instance.status = 'error';
|
|
784
882
|
instance.error = error instanceof Error ? error.message : String(error);
|
|
785
|
-
log('Strand %s:
|
|
883
|
+
log('Strand %s: building the database of a joiner that was waiting for a seed peer failed: %s', strandId, instance.error);
|
|
884
|
+
return;
|
|
786
885
|
}
|
|
886
|
+
if (build === 'awaiting-seed' && this.isCurrentRuntime(instance, node) && !this.isQuiescing(strandId)) {
|
|
887
|
+
this.deferUnreachableJoin(instance);
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
/** Whether `instance` is still tracked, with `node` still its runtime's libp2p node. */
|
|
891
|
+
isCurrentRuntime(instance, node) {
|
|
892
|
+
return !this.stopping && this.instances.get(instance.strandId) === instance && instance.libp2pNode === node;
|
|
787
893
|
}
|
|
788
894
|
/**
|
|
789
895
|
* Release an instance's strand-network runtime: stop the relay supervisors and
|
|
@@ -813,6 +919,8 @@ export class StrandInstanceManager {
|
|
|
813
919
|
relaySupervisors.forEach((supervisor) => supervisor.stop());
|
|
814
920
|
this.relaySupervisors.delete(instance.strandId);
|
|
815
921
|
}
|
|
922
|
+
this.peerRedials.get(instance.strandId)?.stop();
|
|
923
|
+
this.peerRedials.delete(instance.strandId);
|
|
816
924
|
// Backfill next — before the database closes and the libp2p node stops — so
|
|
817
925
|
// no NEW catch-up push is issued against a torn-down transport. A push already
|
|
818
926
|
// in flight is not awaited; it fails into the module's own per-chunk catch.
|
|
@@ -857,7 +965,9 @@ export class StrandInstanceManager {
|
|
|
857
965
|
/**
|
|
858
966
|
* One {@link superviseRelayReservation} per relay dial addr, each over exactly
|
|
859
967
|
* that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
|
|
860
|
-
* one relay re-drives only that one.
|
|
968
|
+
* one relay re-drives only that one. They are the only thing that fills the node's
|
|
969
|
+
* search slots once the node is built with libp2p's relay discovery off
|
|
970
|
+
* (`nodeTransportsOption`). Default retry timings — the control node's (2 s
|
|
861
971
|
* doubling to 60 s between failed attempts, a 5 s liveness check while held). Each DRIVE's
|
|
862
972
|
* own deadline is counted in link round trips at this host's declared
|
|
863
973
|
* `network.linkRoundTripMs` (`link-budget.ts`), so the same declaration that lengthens the
|
|
@@ -1244,8 +1354,9 @@ export class StrandInstanceManager {
|
|
|
1244
1354
|
* schema signature is checked before anything changes; the retained launch config takes
|
|
1245
1355
|
* the sApp, so every later rebuild applies it through `composeStrand`; the live database —
|
|
1246
1356
|
* published, or still held by the first-sync gate — gets the schema; and only then does
|
|
1247
|
-
* the instance record `sAppInfo` and the sApp's latency hint.
|
|
1248
|
-
* config alone
|
|
1357
|
+
* the instance record `sAppInfo` and the sApp's latency hint. An instance with no live
|
|
1358
|
+
* database gets the config alone: a quiesced one applies the schema at its next resume, a
|
|
1359
|
+
* launch waiting for a seed peer at its next database retry.
|
|
1249
1360
|
*
|
|
1250
1361
|
* @returns `'already-attached'` when the instance already runs an sApp — a DIFFERENT one
|
|
1251
1362
|
* is logged and left as it is, as a second claim of a claimed strand always has been —
|
|
@@ -1283,7 +1394,7 @@ export class StrandInstanceManager {
|
|
|
1283
1394
|
await database.attachAppSchema(sAppConfig);
|
|
1284
1395
|
}
|
|
1285
1396
|
else {
|
|
1286
|
-
log('attachSApp: strand %s
|
|
1397
|
+
log('attachSApp: strand %s has no live database — its next build applies the sApp schema', strandId);
|
|
1287
1398
|
}
|
|
1288
1399
|
instance.sAppInfo = sAppInfo;
|
|
1289
1400
|
if (sAppConfig.latencyHint) {
|
|
@@ -1298,14 +1409,13 @@ export class StrandInstanceManager {
|
|
|
1298
1409
|
/**
|
|
1299
1410
|
* Whether `strandId` is launched but still waiting for its first sync — a joiner whose
|
|
1300
1411
|
* runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
|
|
1301
|
-
* the hibernation manager's idle timer fired on it),
|
|
1302
|
-
* seed peer
|
|
1303
|
-
* or untracked strand. The predicate {@link whenWritable}'s callers gate on.
|
|
1412
|
+
* the hibernation manager's idle timer fired on it), including one whose database build
|
|
1413
|
+
* waits for a seed peer to answer ({@link deferUnreachableJoin}). `false` for a writable,
|
|
1414
|
+
* quiesced, or untracked strand. The predicate {@link whenWritable}'s callers gate on.
|
|
1304
1415
|
*/
|
|
1305
1416
|
isAwaitingFirstSync(strandId) {
|
|
1306
1417
|
const instance = this.instances.get(strandId);
|
|
1307
|
-
return instance !== undefined
|
|
1308
|
-
&& (isAwaitingFirstSync(instance) || this.unreachableJoinRetries.has(strandId));
|
|
1418
|
+
return instance !== undefined && isAwaitingFirstSync(instance);
|
|
1309
1419
|
}
|
|
1310
1420
|
/**
|
|
1311
1421
|
* Resolve once the strand's database is published to the app — immediately for a
|
|
@@ -1459,8 +1569,9 @@ export class StrandInstanceManager {
|
|
|
1459
1569
|
const gate = this.firstSyncGates.get(strandId);
|
|
1460
1570
|
const database = instance.database ?? gate?.database;
|
|
1461
1571
|
if (!database) {
|
|
1462
|
-
throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: it
|
|
1463
|
-
'is
|
|
1572
|
+
throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: it has no live database to ` +
|
|
1573
|
+
'write to — it is quiesced (resume it first), or its database build is waiting for a ' +
|
|
1574
|
+
'seed peer to answer.');
|
|
1464
1575
|
}
|
|
1465
1576
|
// Forward the RETAINED config's party key: a foundExistingStrand that flipped a
|
|
1466
1577
|
// joiner launch to founder may have resolved a key the live database's captured
|