@humwick/server 0.0.0-stage → 0.130.0

package/src/services/restart-epoch-reconciler.ts

@@ -0,0 +1,326 @@
+// #2189 P7 — the PRODUCTION owner that carries an epoch past `quiesced`. Without this the epoch is a
+// one-way door: cd-watcher opens and seals it, and nothing ever releases it, so after a single deploy
+// every restart-sensitive create parks forever and every claim refuses forever.
+//
+// WHY IT LIVES ON THE RELAY, NOT IN cd-watcher. The deploy restarts the relay AND kills cd-watcher —
+// scripts/cd-watcher.ts says so directly of its own post-deploy code ("in production these lines are
+// unreachable"). So the process that must observe "the deploy finished" cannot be the one the deploy
+// kills. The relay is the natural owner precisely BECAUSE it is restarted: it comes back up on the new
+// binary, sees an epoch in `deploying` whose ledger row has gone terminal, and finishes the job. That
+// inversion — the restarted process reconciling its own restart — is the whole design.
+//
+// TWO RESPONSIBILITIES, both bounded and both idempotent:
+//
+//  1. FINISH a deploy that already happened. Adopt the epoch (fenced), settle it against the deploy
+//     ledger's terminal outcome, drain the parked backlog in bounded pages, close the epoch.
+//  2. ABANDON an epoch nobody is driving. An owner that died in `requested`/`quiesced` strands the
+//     fleet permanently, because nothing else expires an epoch. Once its lease is provably gone, hand
+//     the epoch to this reconciler and let it drive the ordinary release.
+//
+// It cancels nothing and reaps nothing — see src/restart-epoch-drain-capability.test.ts.
+
+import { randomUUID } from "node:crypto";
+import { errMessage } from "@humwick/sdk";
+import { DEPLOY_LEASE_MS, deployEventLeaseAt } from "../db/deploy-events.ts";
+import { forceFailDeployEventAsync, listDeployEventsAsync } from "../db/deploy-events-async.ts";
+import { getMaintenanceJobPause } from "../db/maintenance-pauses.ts";
+import {
+  abandonOpenRestartEpochAsync,
+  claimRestartEpochReleasePageAsync,
+  completeRestartEpochReleaseAsync,
+  getRestartEpochAsync,
+  openRestartEpochsAsync,
+  restartEpochDeployEventAsync,
+  settleRestartEpochDeployAsync,
+  settleRestartEpochReleasePageAsync,
+} from "../db/restart-epoch-async.ts";
+import { RESTART_EPOCH_OWNER_LEASE_MS, type RestartEpochRow } from "../db/restart-epoch.ts";
+import { RESTART_EPOCH_RELEASE_PAGE_SIZE } from "../db/restart-epoch-release.ts";
+
+/**
+ * How long an epoch may go WITHOUT PROGRESS before the reconciler abandons it.
+ *
+ * Keyed on progress, not on the lease alone, and that distinction is the whole of #2189 round-3
+ * BLOCKER 5. The CD owner is not a process that exists continuously — it is a tick that runs, polls,
+ * and exits — so "the lease expired" does not mean "the owner is gone", it usually just means the
+ * tick ended. Abandoning on lease expiry alone tore down LIVE epochs about 9.5 minutes after they
+ * opened, destroying the hold on healthy gates that routinely run longer than that: precisely the
+ * "a healthy gate is never cut short" property this phase exists to guarantee, broken by the sweeper
+ * meant to protect it.
+ *
+ * `updated_at` is a real progress signal: every blocked seal attempt stamps it, so an epoch a live CD
+ * tick is polling is continuously fresh, while one whose owner died goes silent immediately.
+ */
+export const RESTART_EPOCH_ABANDON_GRACE_MS = 5 * RESTART_EPOCH_OWNER_LEASE_MS;
+
+/**
+ * #2369 — the one case where the silence grace above is not just slow but WRONG.
+ *
+ * The grace exists because a tick-shaped owner going quiet is ambiguous: it usually means the tick
+ * ended, not that the owner died. But when the owner is a maintenance job that an operator has
+ * PAUSED, the ambiguity is gone. A paused job runs no further ticks, by construction — so its epoch
+ * has no driver at all, and waiting out a grace built for "probably still alive" strands every
+ * parked spawn in the fleet for as long as the pause holds.
+ *
+ * That is not hypothetical: pausing cd-watcher to stop deploy churn left epoch 65c997f5 open with
+ * three spawns pending across wbox and maclinux and no path to drain; resuming cd-watcher released
+ * it within seconds. The invariant this restores is "pausing a scheduler must never strand work that
+ * scheduler already owns".
+ *
+ * Note what is NOT skipped: `requireLeaseExpired` still holds. A tick already in flight when the
+ * pause landed keeps renewing its lease every 30s, so it is never cut off mid-drive; the fast path
+ * opens only once that lease has lapsed with nobody left to renew it. And the sweeper still refuses
+ * to touch a `deploying` epoch: a pause during an actual deploy never abandons anything. #2407 adds
+ * one thing there, and only one — such an epoch is REPORTED as stalled immediately rather than after
+ * the 30-minute bound, because the paused job is the only writer of the terminal ledger status it is
+ * waiting on. Reporting, never a write; see `finishDeploying`.
+ */
+export function ownerCannotResume(ownerId: string, now: number): { paused: true; reason: string } | null {
+  const pause = getMaintenanceJobPause(ownerId, now);
+  if (!pause) return null;
+  const until = pause.expiresAt ? `until ${new Date(pause.expiresAt).toISOString()}` : "indefinitely";
+  return { paused: true, reason: `owning maintenance job ${ownerId} is paused ${until} by ${pause.holder} (${pause.reason})` };
+}
+
+/**
+ * How long a `deploying` epoch may sit against a never-terminal deploy ledger before the reconciler
+ * reports it as stalled. It does NOT auto-abandon at this point — a slow deploy is not a dead one, and
+ * only an operator can tell them apart — but it must stop looking like healthy progress, so the
+ * critical-job watchdog and the operator surface both show it.
+ */
+export const RESTART_EPOCH_DEPLOY_STALL_MS = 30 * 60_000;
+
+export type ReconcileOutcome =
+  | "idle"
+  | "released"
+  | "abandoned"
+  | "deploy-in-flight"
+  | "owner-alive"
+  | "release-incomplete"
+  | "deploy-stalled"
+  | "aborted"
+  | "error";
+
+export interface ReconcileResult {
+  readonly outcome: ReconcileOutcome;
+  readonly epochId?: string;
+  readonly unparked: number;
+  readonly pages: number;
+  readonly detail?: string;
+}
+
+export interface ReconcileOptions {
+  readonly ownerId?: string;
+  /** Clock for THIS module's own decisions only. It deliberately does not reach the writer bodies —
+   *  the gateway stamps `ctx.now` at submission — so a test that needs an aged epoch must age the ROW,
+   *  not inject a clock here. */
+  readonly now?: () => number;
+  readonly pageSize?: number;
+  readonly abandonGraceMs?: number;
+  readonly deployStallMs?: number;
+  /** Published per unparked command after its page commits — never inside the writer transaction. */
+  readonly wake?: (commandId: string) => Promise<void>;
+  /** #1819 cooperative cancellation. Checked BETWEEN release pages — the one place a partial pass is
+   *  safe to stop, because each page is already durably claimed and settled, so the next pass resumes
+   *  from the cursor. Never checked mid-page: bailing there would leave a claimed page unsettled. */
+  readonly signal?: AbortSignal;
+}
+
+const TERMINAL_DEPLOY_STATUSES = new Set(["succeeded", "failed", "rolled-back"]);
+
+/**
+ * One bounded reconcile pass over every open epoch (#2988: one per Host, or one fleet epoch). Safe to
+ * run on boot and on a timer; every step is idempotent and fenced, so two concurrent passes converge
+ * rather than conflict. A failing epoch is reported and never stops its siblings.
+ */
+export async function reconcileRestartEpochs(options: ReconcileOptions = {}): Promise<ReconcileResult[]> {
+  const results: ReconcileResult[] = [];
+  for (const epoch of await openRestartEpochsAsync()) {
+    if (options.signal?.aborted) return [...results, { outcome: "aborted", epochId: epoch.id, unparked: 0, pages: 0 }];
+    try {
+      results.push(await reconcileOpenEpoch(epoch, options));
+    } catch (error) {
+      results.push({ outcome: "error", epochId: epoch.id, unparked: 0, pages: 0, detail: errMessage(error) });
+    }
+  }
+  return results;
+}
+
+/** One pass over a single open epoch: `epochId`, or the oldest open one. */
+export async function reconcileRestartEpoch(options: ReconcileOptions & { epochId?: string } = {}): Promise<ReconcileResult> {
+  if (options.signal?.aborted) return { outcome: "aborted", unparked: 0, pages: 0 };
+  const open = await openRestartEpochsAsync();
+  const epoch = options.epochId ? open.find((candidate) => candidate.id === options.epochId) : open[0];
+  if (!epoch) return { outcome: "idle", unparked: 0, pages: 0 };
+  return reconcileOpenEpoch(epoch, options);
+}
+
+async function reconcileOpenEpoch(epoch: RestartEpochRow, options: ReconcileOptions): Promise<ReconcileResult> {
+  const now = options.now ?? Date.now;
+  const ownerId = options.ownerId ?? "relay-restart-epoch-reconciler";
+  if (epoch.state === "requested" || epoch.state === "quiesced") {
+    return abandonStranded(epoch, ownerId, now, options);
+  }
+  // `deploying`: either the deploy is still running, or it finished and nobody closed the epoch —
+  // which, after a deploy that restarted this very process, is the expected state on boot.
+  return finishDeploying(epoch, ownerId, now, options);
+}
+
+async function abandonStranded(
+  epoch: RestartEpochRow, ownerId: string, now: () => number, options: ReconcileOptions,
+): Promise<ReconcileResult> {
+  const graceMs = options.abandonGraceMs ?? RESTART_EPOCH_ABANDON_GRACE_MS;
+  // Silence, measured from the LATER of the lease and the last write to the row. A live CD tick
+  // stamps `updated_at` on every seal attempt, so an epoch holding a two-hour gate never looks
+  // stranded; an owner that died stops stamping and is abandoned one grace window later.
+  const lastSignOfLife = Math.max(epoch.owner_lease_expires_at, epoch.updated_at);
+  let silentFor = now() - lastSignOfLife;
+  // Positive evidence the owner cannot come back beats the "probably still alive" grace. The lease
+  // check inside `abandonOpenEpoch` still gates it, so a tick mid-drive is untouched either way.
+  const halted = ownerCannotResume(epoch.owner_id, now());
+  // #2988 — a drain cannot deploy while another batch holds the one fleet deploy lease, and the tick
+  // that drives it is busy running that batch. The deploy's run is its progress until it ends.
+  if (!halted && silentFor >= graceMs) silentFor = Math.min(silentFor, await fleetDeploySilenceMs(epoch.id, now()));
+  if (!halted && silentFor < graceMs) {
+    return { outcome: "owner-alive", epochId: epoch.id, unparked: 0, pages: 0,
+      detail: `last progress ${Math.max(0, silentFor)}ms ago (grace ${graceMs}ms)` };
+  }
+  const ownerToken: string = randomUUID();
+  const abandoned = await abandonOpenRestartEpochAsync({
+    epochId: epoch.id, actor: ownerId,
+    reason: halted
+      ? `${halted.reason}; epoch left open in state ${epoch.state} with no driver`
+      : `owner ${epoch.owner_id} made no progress for ${silentFor}ms in state ${epoch.state}`,
+    requestId: `${epoch.id}:abandon:${epoch.owner_version}`, requireLeaseExpired: true,
+    newOwnerId: ownerId, newOwnerToken: ownerToken, newLeaseUntil: now() + RESTART_EPOCH_OWNER_LEASE_MS,
+  });
+  if (!abandoned.abandoned) {
+    return { outcome: "owner-alive", epochId: epoch.id, unparked: 0, pages: 0, detail: abandoned.reason };
+  }
+  // Release the FLEET LEASE too. A legacy epoch (pre-#2988) opened its deploy-ledger row up front, so
+  // an owner that died before launching leaves that row `pending` forever — and a pending row is the
+  // fleet mutex, so the next deploy would be refused for a deploy that never happened. An epoch that
+  // never bound a deploy has no row, and nothing to release.
+  const ledger = await restartEpochDeployEventAsync(epoch.id);
+  if (ledger?.status === "pending") {
+    await forceFailDeployEventAsync({
+      id: ledger.id,
+      reason: `restart epoch ${epoch.id} abandoned: owner ${epoch.owner_id} went silent in ${epoch.state} before the deploy launched`,
+    });
+  }
+  const current = await getRestartEpochAsync(epoch.id);
+  if (!current) return { outcome: "idle", epochId: epoch.id, unparked: 0, pages: 0 };
+  const drained = await drainAndClose(current, ownerToken, now, options);
+  return { ...drained, outcome: drained.outcome === "released" ? "abandoned" : drained.outcome };
+}
+
+/** How long the fleet has had no deploy running: 0 while a live one holds the lease. A legacy epoch's
+ *  own unlaunched ledger row is not a deploy running. */
+async function fleetDeploySilenceMs(epochId: string, at: number): Promise<number> {
+  const [latest] = await listDeployEventsAsync(1, 0);
+  if (latest?.status === "pending") {
+    return latest.restartEpochId !== epochId && at - deployEventLeaseAt(latest) < DEPLOY_LEASE_MS ? 0 : Infinity;
+  }
+  return latest?.finishedAt === undefined ? Infinity : at - latest.finishedAt;
+}
+
+async function finishDeploying(
+  epoch: RestartEpochRow, ownerId: string, now: () => number, options: ReconcileOptions,
+): Promise<ReconcileResult> {
+  let current = epoch;
+  if (current.release_state !== "pending") {
+    // Not settled yet. The deploy LEDGER is the terminal authority here, not a process probe: this
+    // reconciler may be running on a brand-new process that never saw the deploy child at all.
+    const ledger = await restartEpochDeployEventAsync(current.id);
+    if (!ledger || !TERMINAL_DEPLOY_STATUSES.has(ledger.status)) {
+      // A deploy whose ledger row never goes terminal would otherwise wait here forever, reported as
+      // healthy in-flight work. It is NOT auto-abandoned — a slow deploy and a dead one look identical
+      // from here — but past the stall bound it is reported as such, so the critical-job watchdog and
+      // `GET /api/restart-epoch` show an operator the state that needs the force-abandon button.
+      const stalledFor = now() - (current.deploying_at ?? current.updated_at);
+      const stallMs = options.deployStallMs ?? RESTART_EPOCH_DEPLOY_STALL_MS;
+      const detail = `deploy ${ledger?.id ?? current.execution_id ?? "unbound"} is ${ledger?.status ?? "unreadable"} after ${stalledFor}ms`;
+      // #2407 — the deploy LEDGER only ever goes terminal from the CD owner's own tick (cd-watcher's
+      // reconcileStalePendingDeploy, which reads live fleet attestation). So if that owner is PAUSED,
+      // this wait is not "slow", it is unbounded: nothing left in the system will ever write the
+      // terminal status this epoch is blocking on, and every restart-sensitive create stays parked.
+      // Still not auto-abandoned — only an operator can tell a shipped deploy from a dead one, and
+      // guessing here would mark a live fleet's deploy failed — but it must stop looking like healthy
+      // in-flight work the moment the wait becomes unbounded, not 30 minutes later.
+      const halted = ownerCannotResume(current.owner_id, now());
+      if (halted) {
+        return { outcome: "deploy-stalled", epochId: current.id, unparked: 0, pages: 0,
+          detail: `${detail} — ${halted.reason}, so nothing will finalize the deploy ledger; force-abandon via POST /api/restart-epoch/abandon or resume that job` };
+      }
+      if (stalledFor >= stallMs) {
+        return { outcome: "deploy-stalled", epochId: current.id, unparked: 0, pages: 0,
+          detail: `${detail} — force-abandon via POST /api/restart-epoch/abandon if the deploy is dead` };
+      }
+      return { outcome: "deploy-in-flight", epochId: current.id, unparked: 0, pages: 0, detail };
+    }
+    if (!current.execution_id) {
+      return { outcome: "deploy-in-flight", epochId: current.id, unparked: 0, pages: 0,
+        detail: "epoch is deploying with no bound execution id" };
+    }
+    const settled = await settleRestartEpochDeployAsync({
+      epochId: current.id, expectedVersion: current.owner_version, ownerToken: current.owner_token,
+      deployExecutionId: current.execution_id,
+      // `completed` is the honest proof: the deploy ledger reached a terminal status. Whether that
+      // status was success or failure is the LEDGER's business — the epoch only needs to know the
+      // execution is over, and a failed deploy must release parked work exactly as a successful one does.
+      terminalProof: "completed",
+      terminalResultId: `${current.id}:${current.execution_id}:terminal`,
+      outcome: { deployStatus: ledger.status, deployEventId: ledger.id },
+    });
+    if (!settled.settled) {
+      return { outcome: "deploy-in-flight", epochId: current.id, unparked: 0, pages: 0, detail: settled.reason };
+    }
+    const refreshed = await getRestartEpochAsync(current.id);
+    if (!refreshed) return { outcome: "idle", epochId: current.id, unparked: 0, pages: 0 };
+    current = refreshed;
+  }
+  return drainAndClose(current, current.owner_token, now, options);
+}
+
+/**
+ * The bounded release loop: claim a page, publish its wakes outside the transaction, settle, yield.
+ * Page tokens are derived from the epoch id and page ordinal, so a reconciler that dies mid-drain
+ * re-presents the same token on its next pass and recovers the same page instead of skipping it.
+ */
+async function drainAndClose(
+  epoch: RestartEpochRow, ownerToken: string, now: () => number, options: ReconcileOptions,
+): Promise<ReconcileResult> {
+  const limit = options.pageSize ?? RESTART_EPOCH_RELEASE_PAGE_SIZE;
+  let cursor = epoch.release_cursor ?? undefined;
+  let unparked = 0;
+  let pages = 0;
+  for (;;) {
+    const page = await claimRestartEpochReleasePageAsync({
+      epochId: epoch.id, ...(cursor ? { afterCommandId: cursor } : {}), limit,
+      claimToken: `${epoch.id}:release:${pages}`,
+    });
+    pages += 1;
+    if (page.commandIds.length === 0) {
+      await settleRestartEpochReleasePageAsync({ epochId: epoch.id, claimToken: page.claimToken, commandIds: [] });
+      break;
+    }
+    for (const commandId of page.commandIds) await options.wake?.(commandId);
+    const settled = await settleRestartEpochReleasePageAsync({
+      epochId: epoch.id, claimToken: page.claimToken, commandIds: page.commandIds,
+    });
+    unparked += settled.unparked;
+    cursor = page.lastCommandId ?? cursor;
+    // Between pages, never within one: the page just settled, so stopping here is a clean seam and the
+    // next pass picks up at `release_cursor`. The epoch stays open, which is the safe direction.
+    if (options.signal?.aborted) return { outcome: "aborted", epochId: epoch.id, unparked, pages };
+  }
+
+  const version = (await getRestartEpochAsync(epoch.id))?.owner_version ?? epoch.owner_version;
+  const done = await completeRestartEpochReleaseAsync({
+    epochId: epoch.id, expectedVersion: version, ownerToken, requestId: `${epoch.id}:release-complete`,
+  });
+  if (!done.released) {
+    return { outcome: "release-incomplete", epochId: epoch.id, unparked, pages, detail: done.reason };
+  }
+  return { outcome: "released", epochId: epoch.id, unparked, pages };
+}

package/src/services/restart-epoch-stale-command-recovery.ts

@@ -0,0 +1,133 @@
+// #2349 — stale-command recovery for a BLOCKED restart epoch.
+//
+// THE GAP THIS CLOSES. Quiescence (src/db/restart-epoch.ts) is deliberately fail-closed: an
+// accepted/running restart-sensitive command with no positive terminal proof keeps the epoch in
+// `requested` forever. That is exactly right for a healthy gate — and exactly wrong for a corpse. A
+// `workspace.merge` whose remote execution is genuinely gone (host reboot, executor process
+// replaced) is excluded from the TTL sweep by design (`expireCommands` skips cgroup-fenced merges;
+// only positive execution proof may terminalize them), and the ONLY production path that gathers
+// that proof was the workspace-auto-merge watchdog — which visits a command solely through a
+// workspace still in a land-in-flight status, and only its LATEST command. A stale `running` row
+// that fell out of that coverage blocked CD and all fleet spawning until an operator force-abandoned
+// the epoch (the Aug 21 → Aug 23 incident, issue #2349).
+//
+// WHAT THIS DOES. When the single open epoch has been sitting in `requested` on
+// `active_execution_unproven` past a short grace, sweep the active restart-sensitive merge commands
+// through {@link reconcileLegacyLandingExecution} — the EXISTING evidence-fenced recovery seam, not a
+// second implementation. That seam never acts on lease expiry alone: an accepted/running fenced
+// merge terminalizes only when its claiming orchestrator PROCESS is positively confirmed replaced
+// (reconnect-stable claim-token fingerprint, `commandExecutorConfirmedGone`) AND the host returns
+// identity-bound `observedAbsent` execution-tree evidence. The terminal write records the proof row,
+// the `terminal_process_proof` stamp, and a `reconciled-terminal` landing audit event in one
+// transaction — the auditable terminal result. A healthy gate fails every one of those fences and is
+// left untouched; the epoch simply stays blocked, which is the intended behaviour.
+//
+// WHAT THIS DELIBERATELY IS NOT. It is not part of the drain path (it is not imported by any module
+// in src/restart-epoch-drain-capability.test.ts's entry-point list) and it cancels nothing: the one
+// writer it drives demands positive host evidence, and the epoch itself is never touched — once the
+// corpse is terminal, the CD tick's own next seal attempt observes quiescence through the ordinary
+// fail-closed checks.
+
+import { activeRestartSensitiveCommands } from "../commands-db";
+import { createActivityEventAsync, listOrchestrators } from "../db";
+import { openRestartEpochsAsync } from "../db/restart-epoch-async";
+import type { Orchestrator } from "../types";
+import { reconcileLegacyLandingExecution } from "./legacy-landing-recovery";
+import { settleRecoveredLanding } from "./orchestrator-landing-recovery";
+
+/**
+ * How long the epoch must have been blocked on `active_execution_unproven` before this sweep probes
+ * at all. NOT a safety mechanism — the reconcile seam's evidence fences are — just politeness: most
+ * epochs seal within seconds, and there is no reason to touch the landing recovery machinery during
+ * an ordinary drain window. `blocker_started_at` survives detail churn while the blocker class is
+ * unchanged, so a healthy multi-hour gate does not reset the clock by merely being polled.
+ */
+export const STALE_RESTART_COMMAND_BLOCKER_GRACE_MS = 5 * 60_000;
+
+/** Per-pass bound. The active-merge set is single digits by construction; this is a backstop. */
+export const STALE_RESTART_COMMAND_RECOVERY_LIMIT = 5;
+
+export interface StaleRestartCommandRecoveryResult {
+  readonly outcome: "idle" | "not-blocked" | "blocker-grace" | "swept";
+  readonly epochId?: string;
+  /** Commands the evidence-fenced reconcile positively terminalized this pass. */
+  readonly recovered: string[];
+  /** Commands probed but NOT provable — held exactly as before, by design. */
+  readonly unproven: string[];
+  readonly aborted?: boolean;
+}
+
+export interface StaleRestartCommandRecoveryOptions {
+  /** Clock for the grace decisions AND the reconcile seam's own executor-orphan grace. */
+  readonly now?: () => number;
+  readonly signal?: AbortSignal;
+  readonly blockerGraceMs?: number;
+  readonly limit?: number;
+  /** Owner-liveness facts; defaults to the live orchestrator table. */
+  readonly orchestrators?: Orchestrator[];
+}
+
+/**
+ * One bounded recovery pass. Safe on a timer: every step is idempotent (the terminal writer is a
+ * fenced CAS; a lost race reads as "unproven" and the next pass converges) and every held command is
+ * genuinely held — nothing here can turn a live execution into a terminal one.
+ */
+export async function recoverStaleRestartEpochCommands(
+  options: StaleRestartCommandRecoveryOptions = {},
+): Promise<StaleRestartCommandRecoveryResult> {
+  const now = options.now ?? Date.now;
+  if (options.signal?.aborted) return { outcome: "idle", recovered: [], unproven: [], aborted: true };
+  const open = await openRestartEpochsAsync();
+  if (open.length === 0) return { outcome: "idle", recovered: [], unproven: [] };
+  const blocked = open.filter((epoch) => epoch.state === "requested" && epoch.blocker_code === "active_execution_unproven");
+  if (blocked.length === 0) return { outcome: "not-blocked", epochId: open[0]!.id, recovered: [], unproven: [] };
+  // A null blocker_started_at means no seal attempt has stamped the column yet — too early to call
+  // this a wedge; the next failed seal stamps it and starts the grace clock.
+  const graceMs = options.blockerGraceMs ?? STALE_RESTART_COMMAND_BLOCKER_GRACE_MS;
+  const due = blocked.filter((epoch) => epoch.blocker_started_at !== null && now() - epoch.blocker_started_at >= graceMs);
+  if (due.length === 0) return { outcome: "blocker-grace", epochId: blocked[0]!.id, recovered: [], unproven: [] };
+
+  const orchestrators = options.orchestrators ?? listOrchestrators();
+  // #2988 — a Host epoch is held only by work aimed at its Host; a fleet epoch by any.
+  const hostOf = new Map(orchestrators.filter((orch) => orch.hostId).map((orch) => [orch.agentId, orch.hostId]));
+  const holdingEpoch = (target: string) => due.find((epoch) => epoch.host_id === null || epoch.host_id === hostOf.get(target));
+  // The same population the seal's `active_execution_unproven` check scans, narrowed to the one type
+  // this recovery seam can prove anything about. Kernel `landing.*` rows have their own supervisor;
+  // a stale `agent.spawn` is lease-swept by `expireCommands` and never reaches this wedge.
+  const candidates = activeRestartSensitiveCommands(now())
+    .filter((command) => command.type === "workspace.merge" && holdingEpoch(command.target))
+    .slice(0, options.limit ?? STALE_RESTART_COMMAND_RECOVERY_LIMIT);
+  const epoch = due[0]!;
+  const recovered: string[] = [];
+  const unproven: string[] = [];
+  for (const command of candidates) {
+    if (options.signal?.aborted) {
+      return { outcome: "swept", epochId: epoch.id, recovered, unproven, aborted: true };
+    }
+    const holder = holdingEpoch(command.target)!;
+    const terminal = await reconcileLegacyLandingExecution(command, orchestrators, now());
+    if (!terminal) {
+      unproven.push(command.id);
+      continue;
+    }
+    recovered.push(command.id);
+    await settleRecoveredLanding(terminal);
+    await createActivityEventAsync({
+      // Keyed WITHOUT a timestamp: one recovery event per command, however many passes observe it.
+      clientId: `restart-epoch-stale-command-recovery-${command.id}`,
+      kind: "state",
+      title: "Recovered stale merge blocking restart epoch",
+      body: `workspace.merge ${command.id} was still ${command.status} with a confirmed-dead execution; terminalized (${terminal.status}, proof ${terminal.terminalProcessProof ?? "recorded"}) so restart epoch ${holder.id} can quiesce`,
+      meta: command.id,
+      icon: "ti-refresh-alert",
+      view: "orchestrators",
+      metadata: {
+        source: "server", maintenanceJobId: "restart-epoch-stale-command-recovery",
+        epochId: holder.id, commandId: command.id,
+        executionGeneration: command.executionGeneration ?? 0,
+        terminalProcessProof: terminal.terminalProcessProof ?? null,
+      },
+    });
+  }
+  return { outcome: "swept", epochId: epoch.id, recovered, unproven };
+}

package/src/services/restart-instructions.ts

@@ -0,0 +1,86 @@
+import { resolveCanonicalHost } from "../host-identity";
+import { memorySeamForAgent, memorySeamParams } from "../memory-seam";
+import { composeProfileSpawnInstructions, composeProfileSpawnInstructionsAsync } from "../project-instructions";
+import { buildWhoamiDefaultForAgent } from "../mcp/identity-projection";
+import { resolveAgentSpawnInstructionInputs, resolveAgentSpawnInstructionInputsAsync } from "./spawn-instruction-inputs";
+import type { AgentCard, AgentProfile } from "../types";
+
+/**
+ * #2171 — the machine a relaunched agent RUNS on, which is whose `.agent-relay/land-gates.json`
+ * the validation preamble must come from. The launch-recorded seam host is authoritative (it is
+ * the canonical value the original spawn resolved and cached under); `agent.machine` covers agents
+ * whose launch predates a recorded seam. This never feeds the memory identity — the seam owns that.
+ */
+function landGatesHostParam(agent: AgentCard, seam: ReturnType<typeof memorySeamForAgent>): { host?: string } {
+  // Canonicalized, never raw: `agent.machine` is what the runner reported (`WBOX`) while the spawn
+  // composed under the registered hostname (`wbox-1`), and no token normalization folds one into
+  // the other — the #1625 F3 divergence. A raw fallback would miss the entry the spawn wrote.
+  const host = seam.identity.host?.trim() || resolveCanonicalHost(agent.machine);
+  return host ? { host } : {};
+}
+
+export function restartInstructionParamsForAgent(
+  agent: AgentCard,
+  opts: { spawnRequestId?: string; cwd?: string } = {},
+): Record<string, unknown> | undefined {
+  const inputs = resolveAgentSpawnInstructionInputs(agent, opts);
+  if (!inputs) return undefined;
+  const agentRelayIdentity = buildWhoamiDefaultForAgent(agent.id);
+  // #1625 — replay the memory identity this agent's launch recorded rather than re-deriving it.
+  // #2134 derives Project memory inclusion from that identity's current profile. #1608 canonicalized
+  // `agent.machine` here so a restart
+  // would hash identically to its spawn, but `machine` is a value the spawn seam never consulted:
+  // for any host whose reported name is not a case/dot-hyphen variant of a REGISTERED orchestrator
+  // (`wbox-1.local`, a foreign `MacMiniNG.local`, `ip-10-0-0-5`, empty) it resolves to a different
+  // host cell — or none — and the relaunched agent silently loses its host memory. The recorded
+  // seam has no such failure mode; the derivation survives only as the fallback inside
+  // `memorySeamForAgent`, for agents that never had a composing launch.
+  const seam = memorySeamForAgent(agent);
+  const composed = composeProfileSpawnInstructions({
+    ...(inputs.cwd ? { cwd: inputs.cwd } : {}),
+    ...landGatesHostParam(agent, seam),
+    memorySeam: seam,
+    ...(inputs.profileName ? { profileName: inputs.profileName } : {}),
+    ...(inputs.profile ? { profile: inputs.profile } : {}),
+    // #1951 — forward the lineage so a restart re-composes the spawned-child report-up block
+    // (block 2) instead of dropping it. This is the restart half of the #1951 fix; the
+    // clear/compact half lives in `startup-context-injection.ts`.
+    ...(inputs.spawnedBy ? { spawnedBy: inputs.spawnedBy } : {}),
+    ...(inputs.workflowRunId ? { workflowRunId: inputs.workflowRunId } : {}),
+    ...(inputs.selfEffort ? { selfEffort: true } : {}),
+    ...(agentRelayIdentity ? { agentRelayIdentity } : {}),
+    ...(inputs.callerSystemPromptAppend ? { callerSystemPromptAppend: inputs.callerSystemPromptAppend } : {}),
+  });
+  return {
+    ...(composed.systemPromptAppend ? { systemPromptAppend: composed.systemPromptAppend } : {}),
+    ...(inputs.callerSystemPromptAppend ? { callerSystemPromptAppend: inputs.callerSystemPromptAppend } : {}),
+    ...(composed.relayInjectionEvents.length ? { relayInjectionEvents: composed.relayInjectionEvents } : {}),
+    // …and carry the identity forward onto the successor's params, so the next generation replays
+    // the same identity instead of falling back to a fresh derivation.
+    ...(composed.memorySeam ? { memorySeam: memorySeamParams(composed.memorySeam) } : {}),
+  };
+}
+
+/** Runtime restart projection.  Keep profile resolution and filesystem composition off the main
+ * DB connection so a worker-backed gateway can restart an agent after the main handle is closed. */
+export async function restartInstructionParamsForAgentAsync(
+  agent: AgentCard,
+  opts: { spawnRequestId?: string; cwd?: string; profile?: AgentProfile } = {},
+): Promise<Record<string, unknown> | undefined> {
+  const inputs = await resolveAgentSpawnInstructionInputsAsync(agent, opts);
+  if (!inputs) return undefined;
+  const agentRelayIdentity = buildWhoamiDefaultForAgent(agent.id);
+  const seam = memorySeamForAgent(agent);
+  const composed = await composeProfileSpawnInstructionsAsync({
+    ...(inputs.cwd ? { cwd: inputs.cwd } : {}), ...landGatesHostParam(agent, seam), memorySeam: seam, ...(inputs.profileName ? { profileName: inputs.profileName } : {}),
+    ...(inputs.profile ? { profile: inputs.profile } : {}),
+    ...(inputs.spawnedBy ? { spawnedBy: inputs.spawnedBy } : {}), ...(inputs.workflowRunId ? { workflowRunId: inputs.workflowRunId } : {}), ...(agentRelayIdentity ? { agentRelayIdentity } : {}), ...(inputs.selfEffort ? { selfEffort: true } : {}),
+    ...(inputs.callerSystemPromptAppend ? { callerSystemPromptAppend: inputs.callerSystemPromptAppend } : {}),
+  });
+  return {
+    ...(composed.systemPromptAppend ? { systemPromptAppend: composed.systemPromptAppend } : {}),
+    ...(inputs.callerSystemPromptAppend ? { callerSystemPromptAppend: inputs.callerSystemPromptAppend } : {}),
+    ...(composed.relayInjectionEvents.length ? { relayInjectionEvents: composed.relayInjectionEvents } : {}),
+    ...(composed.memorySeam ? { memorySeam: memorySeamParams(composed.memorySeam) } : {}),
+  };
+}

package/src/services/restart-project-role.ts

@@ -0,0 +1,74 @@
+// #3043 T3 — a role agent restarts as the same role at the commit its launch recorded. The role is
+// re-read at that sha on the agent's own host and recompiled the way the spawn compiled it, so a
+// role edit landed since never reaches the successor, and an unreadable sha fails the restart.
+import { errMessage, projectRoleProvenanceFrom, stringValue, type AgentLifecycle, type ProjectRoleProvenance, type SpawnProvider } from "@humwick/sdk";
+import { builtInProfileSpawnDefaults } from "../agent-profile-catalog";
+import { getAgentProfile } from "../config-store";
+import { readSpawnRegistrationMeta, ValidationError, withResolvedProvisioningAsync } from "../db";
+import { resolveSpawnLandBlocked } from "../land-block";
+import { assertProjectRoleMcpCwd, assertProjectRoleMcpTrusted, projectRoleProfile, projectRoleProvenance, projectRoleSpawnDefaults, readProjectRole, withRoleProfile } from "../project-roles";
+import { applyProjectCapabilityGates } from "../project-trust";
+import { applyDefaultSharedMcp } from "../shared-mcp";
+import { isOrchestratorLocalToRelay } from "../utils";
+import { resolveProjectRoleForProvider } from "./spawn-project-role";
+import type { AgentCard, AgentProfile, Orchestrator } from "../types";
+
+/** The role the agent's current generation launched as. Its launch record decides; meta, which
+ *  registration keeps per generation, covers a record retention has pruned. */
+export function launchedProjectRole(agent: AgentCard): ProjectRoleProvenance | undefined {
+  return readSpawnRegistrationMeta(stringValue(agent.meta?.spawnRequestId), agent.id).projectRole ?? projectRoleProvenanceFrom(agent.meta?.projectRole);
+}
+
+export function projectRoleRestartError(agentId: string, role: ProjectRoleProvenance, reason: string): ValidationError {
+  return new ValidationError(`cannot restart agent ${agentId} as project role "${role.name}" at ${role.sha}: ${reason}`);
+}
+
+/** The synchronous restart builder cannot read another host's checkout. */
+export function refuseSyncProjectRoleRestart(agent: AgentCard): void {
+  const role = launchedProjectRole(agent);
+  if (role) throw projectRoleRestartError(agent.id, role, "the role is re-read on its host, which only the async restart path (restartSpawnParamsForAgentAsync) can do");
+}
+
+export interface ProjectRoleRestart {
+  profileName: string;
+  agentProfile: AgentProfile;
+  /** The successor's spawn-command fields: the role's provenance and its spawn defaults. */
+  spawn: { projectRole: ProjectRoleProvenance; lifecycle?: AgentLifecycle; landBlocked?: true; landBlockedReason?: string };
+}
+
+export async function restartProjectRole(input: {
+  agentId: string;
+  recorded: ProjectRoleProvenance;
+  host: Orchestrator;
+  provider: SpawnProvider;
+  /** The checkout the role was spawned from: it is read there and its MCP trust is decided there. */
+  sourceCwd: string;
+  /** Where the successor runs (its isolated worktree, or the shared checkout). */
+  runtimeCwd: string;
+  workspaceMode?: string;
+  providerArgs: readonly string[];
+}): Promise<ProjectRoleRestart> {
+  try {
+    const role = withRoleProfile(await readProjectRole(input.host, input.sourceCwd, input.recorded.name, input.recorded.sha), input.recorded.profile);
+    const base = getAgentProfile(role.extends)?.value;
+    if (!base) throw new Error(`it extends unknown profile "${role.extends}"`);
+    resolveProjectRoleForProvider(role, input.provider, { providerArgs: input.providerArgs }, {});
+    let profile = projectRoleProfile((await withResolvedProvisioningAsync(base, input.provider))!, role);
+    if (profile.sources?.project) profile = applyProjectCapabilityGates(profile, input.sourceCwd, undefined, { orchestratorLocal: isOrchestratorLocalToRelay(input.host), pinnedMcpCommit: role.source.sha });
+    assertProjectRoleMcpTrusted(role, profile);
+    assertProjectRoleMcpCwd(role, input.runtimeCwd);
+    const defaults = projectRoleSpawnDefaults(builtInProfileSpawnDefaults(base.name), role);
+    const block = resolveSpawnLandBlocked({ explicit: defaults.landBlocked, isolated: input.workspaceMode === "isolated" });
+    return {
+      profileName: base.name,
+      agentProfile: applyDefaultSharedMcp(profile, false),
+      spawn: {
+        projectRole: projectRoleProvenance(role),
+        ...(defaults.lifecycle ? { lifecycle: defaults.lifecycle } : {}),
+        ...(block.landBlocked ? { landBlocked: true as const, landBlockedReason: block.reason } : {}),
+      },
+    };
+  } catch (error) {
+    throw projectRoleRestartError(input.agentId, input.recorded, errMessage(error));
+  }
+}

package/src/services/restart-secret-refs.ts

@@ -0,0 +1,14 @@
+import { writeChildEnv } from "@humwick/sdk/env";
+import { vaultEnabled } from "../secrets-vault/config";
+import { isRecord } from "@humwick/sdk";
+import { getGateway } from "../db/gateway";
+
+/** Restarts carry references from the immutable launch command, never an old resolved value. */
+export async function carryRestartSecretRefs(params: Record<string, unknown> | undefined, predecessor?: string): Promise<Record<string, unknown> | undefined> {
+  if (!vaultEnabled() || !params || !predecessor) return params;
+  const refs = await getGateway().read("secretsVault.launchRefs", { spawnRequestId: predecessor });
+  if (!Object.keys(refs).length) return params;
+  const env = isRecord(params.env) ? { ...params.env } : {};
+  for (const [key, ref] of Object.entries(refs)) if (typeof ref === "string") writeChildEnv(env, { [key]: `secret:${ref.replace(/@\d+$/, "")}` });
+  return { ...params, env };
+}