@serfab/cadre-core 0.9.0 → 0.10.0

package/src/strand-formation-protocol.ts

@@ -23,12 +23,27 @@ import { multiaddr } from '@multiformats/multiaddr';
 import type { Libp2p } from '@libp2p/interface';
 import type { StrandFormationDisclosure } from './types.js';
 import { type ControlStream, writeFrame, withTimeout } from './control-stream.js';
+import { canonicalJson } from './canonical-json.js';
+import { requireEd25519PublicKeyB64 } from './ed25519-key.js';
+import { verifyFormationConsent } from './peer-authorization.js';
+// seed-bootstrap imports neither this protocol nor the manager/solicitation layers,
+// so this import introduces no cycle.
+import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
 
 const log = debug('sereus:cadre:formation-proto');
 
 /** Protocol id for the native formation transport (parallel to `/sereus/seed/1.0.0`). */
 export const FORMATION_PROTOCOL = '/sereus/formation/1.0.0';
 
+/**
+ * Rejection reason for an invitation that cannot be redeemed at all — unknown, expired, or
+ * fully spent. Shared with `StrandFormationManager`, which reports an
+ * `InvitationExhaustedError` (an invite spent out from under a redemption already in flight)
+ * with this SAME wording on purpose: a joiner that lost the race must be indistinguishable
+ * from a non-racing latecomer, so the two sites must never drift apart.
+ */
+export const INVALID_TOKEN_REASON = 'Invalid token';
+
 /** Maximum formation message size (1MB). */
 const MAX_FORMATION_MSG_SIZE = 1024 * 1024;
 
@@ -36,7 +51,63 @@ const MAX_FORMATION_MSG_SIZE = 1024 * 1024;
 const DEFAULT_SESSION_TIMEOUT_MS = 30_000;
 /** Default per-step (single read/write) timeout (ms). */
 const DEFAULT_STEP_TIMEOUT_MS = 5_000;
-/** Default cap on concurrent inbound formation sessions. */
+/**
+ * Default provisioning budget (ms) — distinct from {@link DEFAULT_STEP_TIMEOUT_MS} because
+ * `provisionStrand` is real work (DB writes, and an outbound approval-hook HTTP call when
+ * the invite carries a `ValidationUrl`), not a bare wire read/write.
+ *
+ * Ordering is deliberate — each layer must be able to fail and report before the layer above
+ * it gives up:
+ *
+ *   approval hook (10 s, `formation-approval.ts`)
+ *     < responder provisioning (12 s)
+ *     < initiator await-response (15 s)
+ *     < session (30 s)
+ *
+ * The 3 s margin between responder provisioning and initiator await-response is the wire
+ * latency budget for the result frame to travel back.
+ *
+ * The last {@link PROVISION_SETTLE_GRACE_MS} of this budget is a settle grace, so the default
+ * WORK budget (12 s − 2 s = 10 s) EQUALS the hook's own 10 s: a dead hook races 'Formation
+ * approval unavailable, retry' against 'Formation provisioning timed out'. Both are retryable
+ * and both leave the invite unspent, so the race is benign.
+ */
+const DEFAULT_PROVISION_TIMEOUT_MS = 12_000;
+/**
+ * Settle grace (ms), carved OUT of `provisionTimeoutMs` — never added on top, so the budget
+ * ladder above is untouched. The listener aborts the provisioning hook when the work budget
+ * ({@link splitProvisionBudget}) expires, then waits up to this grace for the work to settle
+ * anyway: once the `FormationUsage` insert has been issued nothing can un-spend it (the table
+ * is append-only), so a provisioning that lands late is ADOPTED and reported as a real
+ * approval rather than lying "timed out" over a spent invite. A hook that observes the
+ * abort before writing leaves the invite unspent.
+ */
+const PROVISION_SETTLE_GRACE_MS = 2_000;
+/**
+ * Wire-latency margin (ms) added to the responder's provisioning budget to get the
+ * initiator's await-response budget — the travel time for the result frame to come back
+ * after the responder finishes (see {@link DEFAULT_PROVISION_TIMEOUT_MS}).
+ * `StrandFormationManager` reuses this constant so a CONFIGURED budget preserves the same
+ * margin the defaults do, instead of applying the same number to both sides and racing the
+ * initiator's own timeout against the responder's clean rejection reply. The responder's
+ * clamp ({@link provisionCeilingMs}) holds the same margin back, so the ordering survives a
+ * budget large enough that BOTH roles get clamped.
+ *
+ * NOTE: 3 s is a fixed assumption about how long the result frame takes to travel back. If
+ * formation ever runs over paths where that frame can take longer (a slow circuit relay, a
+ * congested mobile link), the margin is too thin and the initiator's timeout races the
+ * responder's reply again — raise it, or derive it from an observed round-trip.
+ */
+export const PROVISION_RESPONSE_TRAVEL_MARGIN_MS = 3_000;
+/** Default initiator await-response budget (ms); see {@link DEFAULT_PROVISION_TIMEOUT_MS}. */
+const DEFAULT_INITIATOR_PROVISION_TIMEOUT_MS = DEFAULT_PROVISION_TIMEOUT_MS + PROVISION_RESPONSE_TRAVEL_MARGIN_MS;
+/**
+ * Default cap on concurrent inbound formation sessions.
+ * NOTE: a longer provisioning budget holds a session's slot longer, so a slow hook can
+ * exhaust this cap and make the listener reject fresh joiners with 'Too many concurrent
+ * formation sessions'. Fine at the current default; revisit if `provisionTimeoutMs` is
+ * raised well above its default or this cap is lowered.
+ */
 const DEFAULT_MAX_CONCURRENT_SESSIONS = 100;
 
 // ── Roles ────────────────────────────────────────────────────────────────────
@@ -85,8 +156,20 @@ export type ResponderProvisionOutcome =
 export interface FormationContactMessage {
   /** The real invitation token. */
   token: string;
-  /** Initiator's member key (peer id). */
+  /** Initiator's member key (peer id) — human-facing identity in logs and `FormStrandResult.memberKey`. */
   partyId: string;
+  /**
+   * Initiator's base64url ed25519 public key — the key behind `partyId`, written to
+   * `FormationUsage.PeerKey` and covered by BOTH signed digests.
+   */
+  peerKey: string;
+  /** Joiner-minted single-use nonce for THIS redemption; both signed digests cover it. */
+  usageStampId: string;
+  /**
+   * The joiner's signature over the `'consent'` digest (see `formationConsentMessage`
+   * in control-database.ts) — proof the named peer agreed to this redemption.
+   */
+  peerSignature: string;
   /** The real disclosure, carried verbatim. */
   disclosure: StrandFormationDisclosure;
   /** Initiator's real multiaddrs. */
@@ -193,6 +276,62 @@ export function isValidResponderCreatesResult(response: FormationResultMessage):
   return true;
 }
 
+/**
+ * Ceiling for a resolved provisioning budget.
+ *
+ * The session budget also has to cover the wire step that PRECEDES provisioning (the
+ * responder's contact read, the initiator's dial-connect), so a whole step is held back
+ * rather than triggering only on a literal overrun.
+ *
+ * `reserveMs` is the EXTRA room only the responder holds back, so its own budget still
+ * lands strictly before the initiator's larger await-response budget even when both are
+ * clamped — without it, any configured budget at or above `sessionTimeoutMs -
+ * stepTimeoutMs` clamps both roles onto the same number and the responder's clean
+ * rejection races the initiator's timeout again. Capped at half the remaining room (like
+ * {@link splitProvisionBudget}) so a small session config still spends most of it working.
+ */
+function provisionCeilingMs(sessionTimeoutMs: number, stepTimeoutMs: number, reserveMs: number): number {
+  const roomMs = sessionTimeoutMs - stepTimeoutMs;
+  return Math.max(1, roomMs - Math.min(reserveMs, Math.floor(roomMs / 2)));
+}
+
+/**
+ * Resolve a caller-supplied provisioning budget: `0`/negative means "unset" (use
+ * `defaultMs`). If the result would let provisioning outlive the session — no result
+ * frame is ever sent, exactly the failure this budget exists to prevent — clamp it to
+ * {@link provisionCeilingMs} and log a warning.
+ */
+function resolveProvisionTimeoutMs(
+  configured: number | undefined,
+  defaultMs: number,
+  sessionTimeoutMs: number,
+  stepTimeoutMs: number,
+  role: string,
+  reserveMs = 0
+): number {
+  const requested = configured && configured > 0 ? configured : defaultMs;
+  const ceilingMs = provisionCeilingMs(sessionTimeoutMs, stepTimeoutMs, reserveMs);
+  if (requested >= ceilingMs) {
+    log(
+      '%s provisionTimeoutMs %dms leaves no room under sessionTimeoutMs %dms (step %dms, reserve %dms); clamping to %dms',
+      role, requested, sessionTimeoutMs, stepTimeoutMs, reserveMs, ceilingMs
+    );
+    return ceilingMs;
+  }
+  return requested;
+}
+
+/**
+ * Split a resolved provisioning budget into the WORK budget and the trailing settle grace.
+ *
+ * The grace is carved OUT of the budget (see {@link PROVISION_SETTLE_GRACE_MS}), and capped at
+ * half of it so a small configured budget still spends at least half its time doing work.
+ */
+function splitProvisionBudget(provisionTimeoutMs: number): { workMs: number; graceMs: number } {
+  const graceMs = Math.min(PROVISION_SETTLE_GRACE_MS, Math.floor(provisionTimeoutMs / 2));
+  return { workMs: provisionTimeoutMs - graceMs, graceMs };
+}
+
 // ── Responder (listener) ─────────────────────────────────────────────────────
 
 export interface FormationListenerOptions {
@@ -202,18 +341,31 @@ export interface FormationListenerOptions {
   validateDisclosure(token: string, disclosure: StrandFormationDisclosure): Promise<boolean>;
   /**
    * Provision (or, for provision-then-record, resolve + record consent against) the
-   * strand (`responderCreates`) for the given initiator. The REAL token is threaded
-   * in so the hook can map it to the bound host strand and its membership key.
+   * strand (`responderCreates`) for the given initiator. The WHOLE contact message is
+   * threaded in: the hook needs the REAL token to map the bound host strand, and the
+   * joiner's `peerKey`/`usageStampId`/`peerSignature` to write the consent columns.
    *
    * Returns a {@link ResponderProvisionOutcome}: the hook may REJECT post-validation
    * (e.g. an unconverged host strand) and the listener turns that into a clean,
    * non-disclosing `approved: false` reply rather than dropping the result frame.
+   *
+   * `signal` is aborted when the listener's work budget expires: a hook that observes it
+   * BEFORE writing abandons the redemption and leaves the invite unspent. Optional so
+   * signal-unaware hooks (tests, mocks) stay assignable.
    */
-  provisionStrand(token: string, initiatorPartyId: string, disclosure: StrandFormationDisclosure): Promise<ResponderProvisionOutcome>;
+  provisionStrand(contact: FormationContactMessage, signal?: AbortSignal): Promise<ResponderProvisionOutcome>;
   /** Responder identity, disclosed only AFTER token + disclosure validation passes. */
   getResponderIdentity(): { partyId: string; cadrePeerAddrs: string[] };
   sessionTimeoutMs?: number;
   stepTimeoutMs?: number;
+  /**
+   * Budget for the `provisionStrand` hook call — distinct from `stepTimeoutMs` because
+   * provisioning is real work, not a bare wire read/write. Default 12 s; `0`/unset uses
+   * the default; a value that would outlive the session is clamped ({@link
+   * provisionCeilingMs}). See {@link DEFAULT_PROVISION_TIMEOUT_MS} for the full ordering
+   * rationale.
+   */
+  provisionTimeoutMs?: number;
   maxConcurrentSessions?: number;
 }
 
@@ -227,6 +379,8 @@ export class FormationListener {
   private readonly options: FormationListenerOptions;
   private readonly sessionTimeoutMs: number;
   private readonly stepTimeoutMs: number;
+  private readonly provisionWorkMs: number;
+  private readonly provisionGraceMs: number;
   private readonly maxConcurrentSessions: number;
   private readonly registered = new Set<Libp2p>();
   private activeSessions = 0;
@@ -236,6 +390,13 @@ export class FormationListener {
     this.options = options;
     this.sessionTimeoutMs = options.sessionTimeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS;
     this.stepTimeoutMs = options.stepTimeoutMs ?? DEFAULT_STEP_TIMEOUT_MS;
+    const split = splitProvisionBudget(resolveProvisionTimeoutMs(
+      options.provisionTimeoutMs, DEFAULT_PROVISION_TIMEOUT_MS,
+      this.sessionTimeoutMs, this.stepTimeoutMs, 'FormationListener',
+      PROVISION_RESPONSE_TRAVEL_MARGIN_MS
+    ));
+    this.provisionWorkMs = split.workMs;
+    this.provisionGraceMs = split.graceMs;
     this.maxConcurrentSessions = options.maxConcurrentSessions ?? DEFAULT_MAX_CONCURRENT_SESSIONS;
   }
 
@@ -282,6 +443,125 @@ export class FormationListener {
     }
   }
 
+  /**
+   * Run the provisioning hook under its WORK budget — the configured provisioning budget minus
+   * the trailing settle grace ({@link splitProvisionBudget}, {@link PROVISION_SETTLE_GRACE_MS}).
+   *
+   * When the work budget expires the hook's signal is aborted, then
+   * {@link settleWithinGrace} waits out the grace for the aborted work to settle anyway.
+   * Returns `undefined` for "timed out" — which the caller turns into a retryable rejection
+   * frame — and rethrows every other failure so it reaches the internal-error path.
+   *
+   * Deliberately not `withDeadline`: that exposes neither the timed-out flag nor the signal
+   * outside its `op`, and both are needed here.
+   */
+  private async provision(id: number, contact: FormationContactMessage): Promise<ResponderProvisionOutcome | undefined> {
+    const controller = new AbortController();
+    let timedOut = false;
+    const pending = this.options.provisionStrand(contact, controller.signal);
+    try {
+      return await withTimeout(
+        this.provisionWorkMs,
+        `Formation provisioning#${id}`,
+        () => pending,
+        () => { timedOut = true; controller.abort(); }
+      );
+    } catch (err) {
+      if (!timedOut) throw err;
+      log(
+        'formation session #%d provisioning work budget expired after %dms; aborted, settling up to %dms',
+        id, this.provisionWorkMs, this.provisionGraceMs
+      );
+      return await this.settleWithinGrace(id, pending);
+    }
+  }
+
+  /**
+   * Wait out the settle grace for an ABORTED provisioning, adopting a late success.
+   *
+   * A hook that observed the abort before issuing its `FormationUsage` insert rejects (with
+   * `FormationAbortedError`) and leaves the invite unspent → reported as a timeout. A hook
+   * that had already written cannot be un-written (the table is append-only), so if it lands
+   * inside the grace its outcome is ADOPTED — the joiner is told the truth (approved, or the
+   * hook's own rejection reason) rather than "timed out" over a spent invite.
+   *
+   * NOTE: one window stays open — a work budget that expires after the `FormationUsage`
+   * insert was issued AND a commit that outlasts the grace. The joiner is told 'timed out'
+   * while its one-time invite is in fact spent, and since every retry mints a fresh keypair
+   * no recovery path can match it. Nothing can un-spend an append-only row, so the late-settle
+   * logging below is the observability for it; widen the grace only if it is seen in the wild.
+   */
+  private async settleWithinGrace(
+    id: number,
+    pending: Promise<ResponderProvisionOutcome>
+  ): Promise<ResponderProvisionOutcome | undefined> {
+    let stillPending = false;
+    try {
+      const outcome = await withTimeout(
+        this.provisionGraceMs,
+        `Formation settle#${id}`,
+        () => pending,
+        () => { stillPending = true; }
+      );
+      log('formation session #%d provisioning settled within its grace: approved=%s — adopting', id, outcome.approved);
+      return outcome;
+    } catch (err) {
+      if (!stillPending) {
+        log('formation session #%d provisioning failed after its abort: %o', id, err);
+        return undefined;
+      }
+      log('formation session #%d provisioning still pending after its %dms grace', id, this.provisionGraceMs);
+      // Log how it eventually settled so a late failure is not swallowed by the two
+      // abandoned `withTimeout` calls.
+      void pending.then(
+        (late) => log('formation session #%d provisioning settled after its grace: approved=%s', id, late.approved),
+        (err2) => log('formation session #%d provisioning failed after its grace: %o', id, err2)
+      );
+      return undefined;
+    }
+  }
+
+  /**
+   * Joiner-consent pre-check — the legibility twin of the schema's
+   * `FormationUsage.PeerConsented` CHECK, exactly as `verifyFormationApproval` is for the
+   * approver's vouch. The database constraint stays the authority; this turns a bad consent
+   * into a clean 'Invalid joiner consent' rejection instead of an opaque CHECK failure at
+   * commit. Runs BEFORE token/disclosure validation: it is the cheapest check (pure local
+   * crypto, no DB read) and a rejection discloses nothing.
+   *
+   * Also pins `partyId` to `peerKey`: an ed25519 peer id is an identity multihash of the
+   * very key, and this is the ONLY layer that can check the match — SQL cannot unwrap a
+   * multihash, and `partyId` itself is never stored.
+   */
+  private isJoinerConsentValid(id: number, contact: FormationContactMessage): boolean {
+    try {
+      requireEd25519PublicKeyB64(contact.peerKey, 'joiner peer key');
+    } catch {
+      // Deliberately not rethrown or echoed: the helper's error message embeds the whole
+      // rejected value, and this one arrived from a remote peer (the cap-the-echo case its
+      // own NOTE warns about). Log without the raw value instead.
+      log('formation session #%d: joiner peer key is not a base64url ed25519 public key', id);
+      return false;
+    }
+    const embedded = ed25519PublicKeyB64FromPeerId(contact.partyId);
+    if (!embedded || embedded !== contact.peerKey) {
+      log('formation session #%d: partyId does not embed the presented peer key', id);
+      return false;
+    }
+    const consentOk = verifyFormationConsent({
+      token: contact.token,
+      usageStampId: contact.usageStampId,
+      peerKey: contact.peerKey,
+      disclosure: canonicalJson(contact.disclosure),
+      peerSig: contact.peerSignature
+    });
+    if (!consentOk) {
+      log('formation session #%d: joiner consent signature failed verification', id);
+      return false;
+    }
+    return true;
+  }
+
   private async runSession(id: number, stream: ControlStream): Promise<void> {
     // Track whether ANY frame has been written so the catch below can convert an
     // unexpected internal error into a non-disclosing rejection ONLY when nothing has
@@ -297,9 +577,14 @@ export class FormationListener {
       const contact = await withTimeout(this.stepTimeoutMs, `Formation await-contact#${id}`, () => reader.read<FormationContactMessage>());
       log('formation session #%d contact: token=%s party=%s', id, contact.token, contact.partyId);
 
+      if (!this.isJoinerConsentValid(id, contact)) {
+        send({ approved: false, reason: 'Invalid joiner consent' });
+        return;
+      }
+
       const tokenResult = await this.options.validateToken(contact.token);
       if (!tokenResult.valid) {
-        send({ approved: false, reason: 'Invalid token' });
+        send({ approved: false, reason: INVALID_TOKEN_REASON });
         return;
       }
 
@@ -309,7 +594,13 @@ export class FormationListener {
         return;
       }
 
-      const outcome = await this.options.provisionStrand(contact.token, contact.partyId, contact.disclosure);
+      const outcome = await this.provision(id, contact);
+      if (!outcome) {
+        // Reported as its own retryable reason rather than falling through to the generic
+        // catch below, which would report the misleading 'Internal formation error'.
+        send({ approved: false, reason: 'Formation provisioning timed out' });
+        return;
+      }
       if (!outcome.approved) {
         // A post-validation rejection still discloses NEITHER identity NOR cadre,
         // exactly like the token/disclosure rejections above.
@@ -351,6 +642,12 @@ export interface FormationDialOptions {
   validateResponse(response: FormationResultMessage): Promise<boolean>;
   sessionTimeoutMs?: number;
   stepTimeoutMs?: number;
+  /**
+   * Budget for the `await-response` read only (the responder's provisioning + reply travel
+   * time) — `dial-connect` keeps using `stepTimeoutMs`. Default 15 s; `0`/unset uses the
+   * default. See {@link DEFAULT_PROVISION_TIMEOUT_MS} for the full ordering rationale.
+   */
+  provisionTimeoutMs?: number;
   protocolId?: string;
 }
 
@@ -366,6 +663,10 @@ export async function dialFormation(node: Libp2p, options: FormationDialOptions)
   const protocolId = options.protocolId ?? FORMATION_PROTOCOL;
   const sessionTimeoutMs = options.sessionTimeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS;
   const stepTimeoutMs = options.stepTimeoutMs ?? DEFAULT_STEP_TIMEOUT_MS;
+  const provisionTimeoutMs = resolveProvisionTimeoutMs(
+    options.provisionTimeoutMs, DEFAULT_INITIATOR_PROVISION_TIMEOUT_MS,
+    sessionTimeoutMs, stepTimeoutMs, 'dialFormation'
+  );
   const addr = multiaddr(options.responderAddrs[0]);
 
   return withTimeout(sessionTimeoutMs, 'Formation dial', async () => {
@@ -375,7 +676,7 @@ export async function dialFormation(node: Libp2p, options: FormationDialOptions)
       const reader = new FrameReader(stream);
       writeFrame(stream, options.contact);
 
-      const response = await withTimeout(stepTimeoutMs, 'Formation await-response', () => reader.read<FormationResultMessage>());
+      const response = await withTimeout(provisionTimeoutMs, 'Formation await-response', () => reader.read<FormationResultMessage>());
       if (!response.approved) {
         throw new Error(`Formation rejected: ${response.reason ?? 'no reason provided'}`);
       }

package/src/strand-instance-manager.ts

@@ -2,6 +2,7 @@ import debug from 'debug';
 import type { PrivateKey } from '@libp2p/interface';
 import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
 import { StrandDatabase } from './strand-database.js';
+import { StrandBackfill, type StrandBackfillConfig } from './strand-backfill.js';
 import { assertSchemaSignature } from './schema-verification.js';
 import type {
   StrandInstance,
@@ -16,6 +17,8 @@ import type {
   StrandMode,
   Libp2pNodeWithRepo
 } from './types.js';
+import { resolveStrandClusterSize, STRAND_CLUSTER_POLICY } from './types.js';
+import { resolveListenAddrs } from './relay-addrs.js';
 
 const log = debug('sereus:cadre:strand-manager');
 const timing = debug('sereus:cadre:timing');
@@ -53,6 +56,25 @@ export interface StartStrandConfig {
    * {@link StrandConfig.founder}.
    */
   founder?: boolean;
+  /**
+   * Number of nodes Optimystic is told this strand's replication cluster should
+   * have. Same rule as {@link CadreNodeConfig.strandClusterSize}, which CadreNode
+   * forwards here: every node on the strand should use the same value, and it is
+   * frozen when the strand's libp2p node is created. Defaults to
+   * `DEFAULT_STRAND_CLUSTER_SIZE` (4); values below `MIN_CLUSTER_SIZE` (2) are
+   * rejected by `resolveStrandClusterSize`. Strand-only — the
+   * control network's breadth is the fixed `CONTROL_REPLICATION_BREADTH`.
+   */
+  clusterSize?: number;
+  /**
+   * Tuning for the strand peer-join block catch-up ({@link StrandBackfill}),
+   * forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
+   * libp2p node connects to a peer this runtime has not yet caught up, every
+   * block in the strand's own raw store is pushed to it. Runs only on
+   * `networked` strands with per-strand storage; `{ enabled: false }` restores
+   * the pre-existing no-backfill behaviour.
+   */
+  backfill?: StrandBackfillConfig;
 }
 
 /**
@@ -132,6 +154,15 @@ export class StrandInstanceManager {
    * without the caller re-threading storage/network/profile/key/sApp config.
    */
   private launchConfigs: Map<string, StartStrandConfig> = new Map();
+  /**
+   * The per-strand peer-join block catch-up, keyed by strand id. Private (not on
+   * the public {@link StrandInstance}) because it is runtime plumbing with the
+   * same lifetime as the strand's libp2p node: created in `buildStrandRuntime`,
+   * stopped and dropped in `releaseRuntime` — so quiesce → resume rebuilds it
+   * with a fresh caught-up-peer memo, which is intended (a resumed node may have
+   * missed writes).
+   */
+  private backfills: Map<string, StrandBackfill> = new Map();
   private stopping = false;
 
   constructor() {
@@ -160,7 +191,16 @@ export class StrandInstanceManager {
   }
 
   /**
-   * Start a new strand instance
+   * Start a new strand instance.
+   *
+   * A failed launch leaves NOTHING tracked: the instance and its retained launch
+   * config are both dropped before the error is rethrown, so the strand id is
+   * free for a genuine retry. This matches the pre-registration failure path
+   * (a rejected schema signature, which throws before anything is recorded) —
+   * both failure modes of this call leave the same residue: none. Callers learn
+   * of the failure from the rejected promise (and, on the control-discovered
+   * path, from CadreNode's `strand:error` event), not from an error record left
+   * behind in `instances`.
    */
   async startStrand(config: StartStrandConfig): Promise<StrandInstance> {
     const { strandRow, sAppConfig } = config;
@@ -201,7 +241,10 @@ export class StrandInstanceManager {
       memberPrivateKey: strandRow.MemberPrivateKey ?? undefined,
       connectedPeers: 0,
       lastActivity: new Date(),
-      latencyHint
+      latencyHint,
+      // Seed only — buildStrandRuntime overwrites this with the same resolution,
+      // so an instance that errors mid-build still reads sensibly.
+      mode: config.mode ?? 'networked'
     };
 
     this.instances.set(strandId, instance);
@@ -213,9 +256,15 @@ export class StrandInstanceManager {
       log('Strand %s started successfully with sApp %s', strandId, sAppConfig.id);
       return instance;
     } catch (error) {
+      // Status/error first — the (now discarded) record is still what `log` reports on.
       instance.status = 'error';
       instance.error = error instanceof Error ? error.message : String(error);
       log('Failed to start strand %s: %s', strandId, instance.error);
+      // Drop the dead record so this strand id can be launched again. Keep the
+      // `launchConfigs` has an entry iff `instances` does invariant — resumeStrand
+      // reads both, and a config without an instance would strand the config.
+      this.instances.delete(strandId);
+      this.launchConfigs.delete(strandId);
       throw error;
     }
   }
@@ -231,6 +280,7 @@ export class StrandInstanceManager {
   private async buildStrandRuntime(instance: StrandInstance, config: StartStrandConfig): Promise<void> {
     const strandId = instance.strandId;
     const { sAppConfig } = config;
+    const mode = config.mode ?? 'networked';
 
     // Resolve storage for this strand. If a factory function is provided, it is
     // called with the strandId to create strand-specific storage (e.g.,
@@ -240,43 +290,60 @@ export class StrandInstanceManager {
       log('Strand %s using provided storage provider', strandId);
     }
 
-    // Note: protocolPrefix would be used by libp2p services, but createLibp2pNode
-    // may not support it yet - tracked for future implementation.
-    const _protocolPrefix = `/sereus/strand/${strandId}`;
+    // db-p2p namespaces every one of the node's protocol ids by network name
+    // (`/optimystic/<networkName>/...`), so anything dialing this node's own
+    // services must derive its prefix from the SAME string the node was built
+    // with — hence one binding for both, not two literals that can drift.
+    const networkName = `strand-${strandId}`;
+    const protocolPrefix = `/optimystic/${networkName}`;
 
     // Determine relay mode: if explicitly set in config, use that;
     // otherwise default to true for storage profile nodes.
     const enableRelay = config.network?.enableRelay ?? (config.profile === 'storage');
+    const listenAddrs = resolveListenAddrs(config.network);
 
     try {
       let t0 = performance.now();
       const node = await createLibp2pNode({
         port: 0, // Random port
         bootstrapNodes: config.bootstrapNodes ?? [],
-        networkName: `strand-${strandId}`,
+        networkName,
         storage: strandStorage,
         fretProfile: config.profile === 'storage' ? 'core' : 'edge',
         relay: enableRelay,
-        clusterSize: 3,
-        clusterPolicy: {
-          allowDownsize: true,
-          sizeTolerance: 0.5
-        },
+        clusterSize: resolveStrandClusterSize(config.clusterSize),
+        // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
+        // breadth reasoning, and the shape match with the control policy is a coincidence.
+        clusterPolicy: STRAND_CLUSTER_POLICY,
         arachnode: {
           enableRingZulu: config.profile === 'storage'
         },
         ...(config.privateKey && { privateKey: config.privateKey }),
         ...(config.network?.transports && { transports: config.network.transports }),
-        ...(config.network?.listenAddrs && { listenAddrs: config.network.listenAddrs }),
+        // NOTE: strand nodes receive the same RESOLVED listen addrs as the
+        // control node. A fixed-port listen addr (e.g. cadre-cli's example
+        // `/ip4/0.0.0.0/tcp/4001`) would have control + strand nodes racing to
+        // bind one port — EADDRINUSE. Unverified; if a deployment configures a
+        // fixed port and strands fail to start, rewrite the port per node here.
+        // An inherited `/p2p-circuit` addr, by contrast, is deliberate — it is
+        // what gives a NAT'd strand node a reachable relay slot — and works
+        // because the launch path announces this strand's derived peerId to
+        // the relay first (delegate admission; see cadre-node.ts). Those circuit
+        // entries come either from a hand-written `network.listenAddrs` or from
+        // `network.relayAddrs`, which `resolveListenAddrs` folds into the same list.
+        ...(listenAddrs && { listenAddrs }),
         ...(config.network?.connectionGater && { connectionGater: config.network.connectionGater })
       }) as Libp2pNodeWithRepo;
       timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
 
       instance.libp2pNode = node;
+      // Assigned on every runtime (re)build — resume re-enters here with
+      // resumeConfig, so a bootstrap → networked shift refreshes the field.
+      instance.mode = mode;
 
       // Create and initialize the StrandDatabase. In bootstrap mode the same
       // strandStorage instance also backs the optimystic local transactor so
-      // DML lands on the host's persistent storage (e.g. MMKV on RN). Sharing
+      // DML lands on the host's persistent storage (e.g. LevelDB on RN). Sharing
       // the instance — not creating a second one over the same id+prefix —
       // avoids cache divergence between the libp2p and database paths.
       //
@@ -288,7 +355,7 @@ export class StrandInstanceManager {
         sAppConfig,
         libp2pNode: node,
         coordinatedRepo: node.coordinatedRepo,
-        mode: config.mode ?? 'networked',
+        mode,
         rawStorage: strandStorage,
         // Founder bootstrap inputs: the strand's type drives which membership rows
         // are written, and the closed-strand MemberPrivateKey derives the founding
@@ -301,6 +368,28 @@ export class StrandInstanceManager {
       await strandDb.initialize();
       timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
 
+      // Peer-join block catch-up: push this strand's own blocks to each newly
+      // connected peer, so a machine that joined after blocks were committed
+      // still ends up physically holding them (a bootstrap strand has no mesh
+      // to catch up, and without per-strand storage there is nothing to copy).
+      if (mode === 'networked' && strandStorage && config.backfill?.enabled !== false) {
+        if (node.keyNetwork) {
+          const backfill = new StrandBackfill({
+            strandId,
+            libp2p: node,
+            peerNetwork: node.keyNetwork,
+            storage: strandStorage,
+            // The same prefix the receiver registered its block-transfer handler
+            // under — derived from networkName above, never re-spelled here.
+            protocolPrefix
+          }, config.backfill);
+          backfill.start();
+          this.backfills.set(strandId, backfill);
+        } else {
+          log('Strand %s: libp2p node exposes no keyNetwork; peer-join block catch-up is inert', strandId);
+        }
+      }
+
       instance.status = 'active';
       instance.lastActivity = new Date();
     } catch (error) {
@@ -323,6 +412,14 @@ export class StrandInstanceManager {
    * Shared by `quiesceStrand`, `stopStrand`, and that rollback path.
    */
   private async releaseRuntime(instance: StrandInstance): Promise<void> {
+    // Backfill first — before the database closes and the libp2p node stops — so
+    // no NEW catch-up push is issued against a torn-down transport. A push already
+    // in flight is not awaited; it fails into the module's own per-chunk catch.
+    const backfill = this.backfills.get(instance.strandId);
+    if (backfill) {
+      backfill.stop();
+      this.backfills.delete(instance.strandId);
+    }
     if (instance.database) {
       await instance.database.close();
       instance.database = undefined;