@serfab/cadre-core 0.13.0 → 1.1.0

package/src/enrolled-machine-store.ts

@@ -41,8 +41,8 @@
  * `Memory*` / `Persistent*` / `File*` split, same party scoping. It does NOT share
  * their `NodeLocalSnapshot` machinery, for two reasons:
  *
- *  - **Shape.** `NodeLocalSnapshot` is a `key -> entry` map with no delete. This
- *    record is one integer that must be able to go DOWN after a machine is removed.
+ *  - **Shape.** `NodeLocalSnapshot` is a `key -> entry` map. This record is one
+ *    integer that must be able to go DOWN after a machine is removed.
  *  - **Load policy — the important one.** `NodeLocalSnapshot.open` THROWS when the
  *    slot is present but unreadable. That is right for the anchor (a record that
  *    cannot be read in full is not a trust anchor) and right for the peer store (a

package/src/index.ts

@@ -8,7 +8,7 @@ export { canonicalJson } from './canonical-json.js';
 export { CadreNode } from './cadre-node.js';
 
 // Control database
-export { ControlDatabase, MissingHostStrandError, buildAuthorizationMessage, formationVouchMessage, formationConsentMessage, REAPABLE_TABLES, type ReapableTable, type ControlDatabaseConfig, type ControlTable, type RevocableTable, type FormationUsageResult, type MembershipChangeListener, type GuardedDeleteListener, type RevokedRowRef } from './control-database.js';
+export { ControlDatabase, MissingHostStrandError, buildAuthorizationMessage, formationVouchMessage, formationConsentMessage, REAPABLE_TABLES, type ReapableTable, type ControlDatabaseConfig, type ControlTable, type RevocableTable, type FormationUsageResult, type MembershipChangeListener, type GuardedDeleteListener, type ControlWriteAbandonedListener, type RevokedRowRef } from './control-database.js';
 
 // Bounded retry for transient control-write failures (classifier + loop behind
 // ControlDatabase.lockedWithRetry; exported so the integration package can drive the
@@ -35,6 +35,15 @@ export {
   type ControlReadRetryOptions
 } from './control-read-retry.js';
 
+// The bounded retry loop both control policies share. Only the abandonment record leaves
+// this package: it is the payload of the `control:write-abandoned` node event, so an
+// embedding app can type a handler for it.
+export type {
+  ControlRetryAbandonment,
+  ControlRetryAbandonReason,
+  ControlRetryAbandonListener
+} from './control-retry.js';
+
 // Control-plane authorization field vector (the domain/action tagging every signer shares)
 export { controlAuthorizationFields, type ControlDomain, type ControlAction } from './control-authorization.js';
 
@@ -108,6 +117,11 @@ export {
   type EnrolledMachineStore
 } from './enrolled-machine-store.js';
 
+// Storage scope keys — what `CadreNodeConfig.storage.provider` is called with. A
+// strand's key is its strand id; the control database's key carries the party id,
+// so two parties on one device never share a control store.
+export { controlStorageScope, isControlStorageScope } from './storage-scope.js';
+
 // Strand database
 export { StrandDatabase, type StrandDatabaseConfig } from './strand-database.js';
 
@@ -123,9 +137,24 @@ export {
   StrandInstanceManager,
   type StartStrandConfig,
   type ResumeStrandOverrides,
-  getStrandStoragePath
+  getStrandStoragePath,
+  isAwaitingFirstSync,
+  liveStrandStatus
 } from './strand-instance-manager.js';
 
+// The joining machine's first-sync write gate: a non-founder launch withholds its
+// database (status 'syncing') until the strand's Header has arrived from a peer, so a
+// write before the first sync can never fork the strand's tables.
+export {
+  StrandAwaitingFirstSyncError,
+  DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS,
+  DEFAULT_STRAND_FIRST_SYNC_POLL_MS,
+  strandHeaderHeld,
+  appTablesReadable,
+  strandFirstSyncComplete,
+  type StrandFirstSyncConfig
+} from './strand-first-sync-gate.js';
+
 // Peer-join block catch-up (push this network's own blocks to each newly
 // connected peer, so a late joiner physically holds pre-join blocks). Shared
 // by the strand networks (ungated) and the control network (membership-gated).
@@ -157,6 +186,23 @@ export {
   type RevocationRefreshScheduler
 } from './strand-revocation-enforcer.js';
 
+// Closed-strand membership reconciliation: the bring-up loop that finishes a
+// party's join on every machine — redeem the staged formation invitation, then
+// write the durable machine→party MemberPeer binding — without ever blocking
+// bring-up (see the module doc for the pass ladder and terminal states).
+export {
+  StrandMembershipReconciler,
+  IDLE_PASSES_BEFORE_ESCALATION,
+  INITIAL_JOIN_RETRY_INTERVAL_MS,
+  type StrandMembershipReconcilerDeps,
+  type StrandMembershipReconciliationConfig,
+  type PendingMembershipInviteSource
+} from './strand-membership-reconciler.js';
+
+// The timer seam the self-rescheduling strand bring-up loops share (first-sync gate
+// probe loop, membership retry ladder) — injected by tests as a hand-cranked clock.
+export { defaultTimeoutScheduler, type TimeoutScheduler } from './timeout-scheduler.js';
+
 // Hibernation
 export {
   HibernationManager,
@@ -220,8 +266,11 @@ export {
   signStrandApproval,
   generateStrandStampId,
   bootstrapFounderMembership,
+  PreSplitStrandIdentityError,
   issueInvite,
   consumeInvite,
+  burnInvite,
+  isStrandMember,
   cancelInvite,
   listOutstandingInvites,
   addMemberByManager,
@@ -235,6 +284,7 @@ export {
   removeManager,
   sealStrand,
   isStrandSealed,
+  StrandTransactionBusyError,
   STRAND_ENGINE,
   STRAND_ENGINE_VERSION,
   type FounderBootstrapParams,
@@ -253,7 +303,8 @@ export {
   type AddManagerParams,
   type AdmitManagerParams,
   type RemoveManagerParams,
-  type SealStrandParams
+  type SealStrandParams,
+  type StrandWriteOptions
 } from './strand-membership-writer.js';
 
 // Engine-canonical datetime helper (shared by control + strand signed-write flows
@@ -367,12 +418,25 @@ export {
 // the cadence/degree defaults the proactive reconcile routine reads.
 export {
   selectControlCohortDials,
-  DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
   DEFAULT_CONTROL_COHORT_RECONCILE_MS,
   DEFAULT_CONTROL_COHORT_TARGET_DEGREE,
-  type ControlCohortSelection
+  type ControlCohortSelection,
+  type ControlCohortReconcileResult
 } from './control-cohort.js';
 
+// Dialing one peer from several candidate addresses, each under its own time
+// limit, so an address that never answers cannot starve the rest.
+export {
+  dialPeerAddrs,
+  tryAddrsInTurn,
+  directBeforeRelayed,
+  DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
+  DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS,
+  DEFAULT_PEER_DIAL_BUDGET,
+  type PeerDialBudget,
+  type AddrDialer
+} from './peer-dial.js';
+
 // Seed trust policy (trust anchor for incoming seeds)
 export {
   anchoredTrustPolicy,
@@ -396,6 +460,7 @@ export {
   FormationListener,
   dialFormation,
   isValidResponderCreatesResult,
+  isWellFormedMembershipInvite,
   sanitizeStrandAddrs,
   FORMATION_PROTOCOL,
   type FormationParty,
@@ -414,6 +479,9 @@ export {
 export {
   StrandFormationManager,
   createStrandFormationManager,
+  MEMBERSHIP_INVITE_TTL_MS,
+  MEMBERSHIP_INVITE_UNAVAILABLE_REASON,
+  HOST_STRAND_MUST_BE_RECREATED_REASON,
   type StrandFormationManagerConfig,
   type StrandFormationManagerOptions
 } from './strand-formation-manager.js';
@@ -438,11 +506,20 @@ export {
 // `UnbindableListenAddressError` is the same posture for the OTHER half of the listen
 // config: a `network.listenAddrs` entry whose transport the node will not have refuses
 // start instead of being silently dropped by libp2p's transport manager.
+//
+// `resolveListenAddrs` and `strandNodeAddrs` are the derivations themselves — what a
+// `NetworkConfig` actually turns into for the control node and for each strand node.
+// Exported so an embedder that hand-writes a `network` block (the React Native and web
+// reference apps) can assert on the resulting SHAPE rather than on its own field names,
+// which is the difference between a test that proves reachability and one that proves a
+// spelling.
 export {
   RelayReservationFailedError,
   UnbindableListenAddressError,
-  RELAY_SEARCH_LISTEN_ADDR
+  RELAY_SEARCH_LISTEN_ADDR,
+  resolveListenAddrs
 } from './relay-addrs.js';
+export { strandNodeAddrs, type StrandNodeAddrs } from './strand-network-config.js';
 
 // Relay reservation via the bare `/p2p-circuit` search listener — the one route
 // every control node takes; `network.relayAddrs` is its fail-fast posture

package/src/node-local-snapshot.ts

@@ -104,9 +104,9 @@ export interface NodeLocalSnapshotSpec<E> {
  */
 export class NodeLocalSnapshot<E> {
 	/**
-	 * Serialises persists: every {@link put} snapshot-writes the full entry set,
-	 * so chaining writes keeps them ordered and the last landed snapshot
-	 * complete.
+	 * Serialises persists: every write ({@link put}, {@link remove}) snapshot-writes
+	 * the full entry set, so chaining writes keeps them ordered and the last landed
+	 * snapshot complete.
 	 */
 	private writeChain: Promise<void> = Promise.resolve();
 
@@ -145,15 +145,15 @@ export class NodeLocalSnapshot<E> {
 		return this.entries.has(key);
 	}
 
-	/** Fresh copy of the keys — a snapshot decoupled from later {@link put} calls. */
+	/** Fresh copy of the keys — a snapshot decoupled from later {@link put} / {@link remove} calls. */
 	keySnapshot(): Set<string> {
 		return new Set(this.entries.keys());
 	}
 
 	/**
 	 * Fresh copy of the `key -> entry` map — a snapshot decoupled from later
-	 * {@link put} calls. The entries themselves need no copy: {@link put}
-	 * REPLACES an entry rather than mutating it in place.
+	 * {@link put} / {@link remove} calls. The entries themselves need no copy:
+	 * {@link put} REPLACES an entry rather than mutating it in place.
 	 */
 	entrySnapshot(): Map<string, E> {
 		return new Map(this.entries);
@@ -164,12 +164,29 @@ export class NodeLocalSnapshot<E> {
 	 * synchronous caller may consult the store the moment this returns), then
 	 * the full snapshot is persisted. A persist failure rejects the returned
 	 * promise but leaves the entry in memory — this session's decision stands,
-	 * and any later successful {@link put} re-lands the complete set.
+	 * and any later successful write re-lands the complete set.
 	 */
 	put(key: string, entry: E): Promise<void> {
 		this.entries.set(key, entry);
+		return this.queuePersist();
+	}
+
+	/**
+	 * Remove an entry, with {@link put}'s contract: the in-memory map updates
+	 * SYNCHRONOUSLY, then the full snapshot is persisted. Removing an absent key
+	 * changes nothing, so it writes nothing.
+	 */
+	remove(key: string): Promise<void> {
+		if (!this.entries.delete(key)) {
+			return Promise.resolve();
+		}
+		return this.queuePersist();
+	}
+
+	/** Chain a full-snapshot write behind every earlier one (see {@link put}). */
+	private queuePersist(): Promise<void> {
 		const persist = this.writeChain.then(() => this.persistSnapshot());
-		// The chain itself must survive a failed persist (the next put retries the
+		// The chain itself must survive a failed persist (the next write retries the
 		// full snapshot); the caller still observes the rejection via `persist`.
 		this.writeChain = persist.catch((error) => {
 			log('%s: persist failed for party %s: %o', this.spec.label, this.partyId, error);

package/src/peer-dial.ts

@@ -0,0 +1,215 @@
+/**
+ * Dialing one peer from a list of candidate addresses, so that an address that
+ * never answers cannot use up the time the others needed.
+ *
+ * libp2p's own multi-address dial (`libp2p.dial(addrs)`, libp2p 3.1.3's
+ * `DialQueue.dialPeer`) tries the addresses ONE AT A TIME under ONE signal, with
+ * no time limit per address. An address whose connection attempt is silently
+ * dropped — a firewall, a subnet the dialer cannot route to, a virtual adapter's
+ * address a Windows host reports alongside its real ones — holds that dial until
+ * the whole signal expires, and every address sorted after it is never tried. It
+ * also sorts the list itself (`defaultAddressSorter`), putting loopback addresses
+ * last, behind exactly the LAN addresses most likely to be dropped. A phone
+ * borrowing a node from a cadre-host whose only reachable address was the
+ * forwarded loopback one never connected for this reason.
+ *
+ * So the addresses are dialed here, each as its own `dial()` with its own time
+ * limit ({@link PeerDialBudget.perAddressMs}), inside a limit for the whole peer
+ * ({@link PeerDialBudget.totalMs}). One address per call also means libp2p's
+ * sorter has nothing to reorder, so the order is chosen here
+ * ({@link directBeforeRelayed}).
+ *
+ * Dependency-free apart from `control-stream.ts`, so any module may import it
+ * without creating an import cycle.
+ */
+
+import debug from 'debug';
+import type { Connection } from '@libp2p/interface';
+import type { Multiaddr } from '@multiformats/multiaddr';
+import { withDeadline } from './control-stream.js';
+
+const log = debug('sereus:cadre:peer-dial');
+
+/**
+ * Default limit on ONE address's dial attempt, in ms.
+ *
+ * It has to cover the slowest address that should succeed, because an attempt
+ * that needs longer fails the same way on every retry. That is a relayed dial on
+ * a mobile link: connect to the relay (transport, encryption and multiplexer
+ * handshakes, when no relay connection is open yet), open the circuit, then run
+ * both handshakes again end to end through the relay at roughly twice the
+ * round-trip time. At a few hundred ms per round trip that is several seconds.
+ * A direct dial that works is far quicker — a phone's WebSocket dial to a node
+ * forwarded over `adb reverse` measured 1.6 s end to end. 8 s covers the relayed
+ * case with some margin while still abandoning a silently dropped address
+ * sooner than libp2p's own 10 s dial timeout would.
+ *
+ * NOTE: a `/p2p-circuit/webrtc` address also negotiates a WebRTC session after
+ * the relay leg. If such dials are ever seen failing at exactly this limit, raise
+ * it (or `network.controlCohort.perAddressDialTimeoutMs` on the affected nodes)
+ * rather than the per-peer limit, which cannot help an address that is itself
+ * cut off.
+ *
+ * Override per node with `network.controlCohort.perAddressDialTimeoutMs`.
+ */
+export const DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS = 8_000;
+
+/**
+ * Default limit on dialing ONE peer — every candidate address together — in ms.
+ *
+ * Without it, an offline peer costs (address count × up to
+ * {@link DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS}), a number that
+ * depends on how many addresses the peer happens to report. Callers dial peers
+ * one after another (the control-cohort reconcile pass dials each sibling in
+ * turn), so that cost also delays every later peer. With the limit, a reconcile
+ * pass costs at most (dialed siblings) × this.
+ *
+ * Sized to fit a typical list with dead addresses ahead of the one that works:
+ * a lent cadre-host node reports two to four addresses a phone can dial
+ * (loopback and LAN WebSocket addresses, plus any virtual adapters). Three
+ * silently dropped addresses cost 3 × 8 s = 24 s, leaving 6 s for the working
+ * address after them — well over the 1.6 s the phone measured. A peer whose
+ * working address sits behind four or more dropped ones does not fit, and fails
+ * each pass the same way; if that shows up, the remedy is ordering (try the
+ * address that last connected first), not a larger number here.
+ *
+ * Override per node with `network.controlCohort.dialTimeoutMs` — tests that
+ * drive dead addresses on purpose set it low so a pass's duration is a chosen
+ * number rather than a transitive libp2p default stretched by machine load.
+ */
+export const DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS = 30_000;
+
+/** Time limits for dialing one peer from a list of candidate addresses. */
+export interface PeerDialBudget {
+	/** Limit on one address's attempt, in ms. */
+	perAddressMs: number;
+	/** Limit on the whole peer — every address together — in ms. */
+	totalMs: number;
+}
+
+/** {@link PeerDialBudget} built from the two defaults above. */
+export const DEFAULT_PEER_DIAL_BUDGET: Readonly<PeerDialBudget> = {
+	perAddressMs: DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS,
+	totalMs: DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
+};
+
+/** The slice of a libp2p node {@link dialPeerAddrs} uses. */
+export interface AddrDialer {
+	dial(addr: Multiaddr, options: { signal: AbortSignal }): Promise<Connection>;
+}
+
+/** One candidate's outcome, kept so the thrown error can name every attempt. */
+interface AddrAttemptFailure {
+	addr: Multiaddr;
+	error: Error;
+}
+
+/**
+ * Dial one peer from `addrs`, one address per `dial()` call, and return the
+ * first connection that forms. Direct addresses are tried before relayed ones
+ * ({@link directBeforeRelayed}); otherwise the given order is kept.
+ *
+ * Addresses naming a transport this node lacks are not filtered out first: a
+ * one-address `dial()` rejects those at once (`NoValidAddressesError`), so they
+ * cost a log line, not time.
+ *
+ * Throws when no address connects — see {@link tryAddrsInTurn} for the error.
+ *
+ * @param label Names the dial in timeout messages and logs, e.g.
+ *   `reconcileControlCohort dial of sibling <peerId>`; each attempt reads
+ *   `<label> via <addr>`.
+ */
+export function dialPeerAddrs(
+	dialer: AddrDialer,
+	addrs: readonly Multiaddr[],
+	budget: PeerDialBudget,
+	label: string,
+): Promise<Connection> {
+	return tryAddrsInTurn(directBeforeRelayed(addrs), budget, label, (addr, signal) => dialer.dial(addr, { signal }));
+}
+
+/**
+ * Run `attempt` against each address in the given order until one succeeds, and
+ * return its result.
+ *
+ * Each attempt is limited to `budget.perAddressMs`, or to whatever remains of
+ * `budget.totalMs` when that is less. The attempt's signal is aborted when its
+ * limit passes, so a dial in progress is cancelled rather than left running.
+ * Addresses the total leaves no time for are reported as not tried.
+ *
+ * Throws when every address fails. With one address the error is that attempt's
+ * own, unchanged. With several it is one error naming each address and why it
+ * failed, in order, with the first failure as `cause` — so the report shows the
+ * address that mattered, not only whichever happened to be tried last.
+ *
+ * Between attempts this waits for one macrotask (`setTimeout(0)`). libp2p's dial
+ * queue lets a new `dial()` join any queued dial job for the same peer id
+ * WITHOUT checking whether that job has already finished, and an aborted job
+ * leaves the queue a few microtasks after its callers are rejected. Timing out
+ * one address and immediately dialing the next for the same peer therefore
+ * joins the dying job, which rejects the new dial with `AbortError` before it
+ * touches the network — the next address is never tried. Seen against libp2p
+ * 3.1.3; `test/peer-dial.spec.ts` dials real nodes so an upgrade that changes it
+ * shows up there. All of the queue's cleanup runs as microtasks, so it has
+ * finished by the time a macrotask runs.
+ */
+export async function tryAddrsInTurn<T>(
+	addrs: readonly Multiaddr[],
+	budget: PeerDialBudget,
+	label: string,
+	attempt: (addr: Multiaddr, signal: AbortSignal, attemptMs: number) => Promise<T>,
+): Promise<T> {
+	const deadline = Date.now() + budget.totalMs;
+	const failures: AddrAttemptFailure[] = [];
+	for (const addr of addrs) {
+		const remaining = deadline - Date.now();
+		if (remaining <= 0) {
+			failures.push({ addr, error: new Error(`not tried — the ${budget.totalMs}ms budget for ${label} was spent on earlier addresses`) });
+			continue;
+		}
+		if (failures.length > 0) {
+			await nextMacrotask();
+		}
+		const attemptMs = Math.min(budget.perAddressMs, remaining);
+		try {
+			return await withDeadline(attemptMs, `${label} via ${addr.toString()}`, (signal) => attempt(addr, signal, attemptMs));
+		} catch (err) {
+			const error = err instanceof Error ? err : new Error(String(err));
+			failures.push({ addr, error });
+			log('%s via %s failed: %o', label, addr.toString(), error);
+		}
+	}
+	throw allAttemptsFailed(label, failures);
+}
+
+/**
+ * `addrs` with every circuit-relay address (one containing `/p2p-circuit`) moved
+ * after every direct one, each group keeping its given order.
+ *
+ * The one part of libp2p's own ordering worth keeping: a relayed connection is
+ * slower and limited in how much data and time it may carry, so a direct
+ * address is always worth trying first. libp2p's other rules are deliberately
+ * dropped — it puts loopback addresses last and public ones before private ones,
+ * which is how a forwarded loopback address ended up behind addresses that
+ * never answer. With a per-address limit, trying a dead address first costs a
+ * bounded amount of time, so the caller's order is the better guide.
+ */
+export function directBeforeRelayed(addrs: readonly Multiaddr[]): Multiaddr[] {
+	const relayed = (addr: Multiaddr): boolean => addr.getComponents().some((c) => c.name === 'p2p-circuit');
+	return [...addrs.filter((addr) => !relayed(addr)), ...addrs.filter(relayed)];
+}
+
+function allAttemptsFailed(label: string, failures: AddrAttemptFailure[]): Error {
+	if (failures.length === 0) {
+		return new Error(`${label}: no candidate addresses`);
+	}
+	if (failures.length === 1) {
+		return failures[0].error;
+	}
+	const detail = failures.map((f) => `${f.addr.toString()} — ${f.error.message}`).join('; ');
+	return new Error(`${label} failed for all ${failures.length} candidate addresses: ${detail}`, { cause: failures[0].error });
+}
+
+function nextMacrotask(): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, 0));
+}