@humwick/server 0.0.0-stage → 0.130.0

package/src/workspace-rekey.ts

@@ -0,0 +1,106 @@
+import { randomUUID } from "node:crypto";
+import { getGateway } from "./db/gateway";
+import { emitWorkspaceChange, getAgent, getTaskDetail, getTeam, getWorkspace, listOrchestrators } from "./db";
+import { pendingStewardRekeyNotices, type StewardRekeyNoticeRow } from "./db/workspace-rekey";
+import { resolveCoordinatorAgentId } from "./coordinator-authority";
+import { taskOwnerTarget } from "./task-owner";
+import { taskOwnerRoleTeamId } from "./db/task-owner";
+import { parseJson } from "./utils";
+import { routeNotification } from "./notification-router";
+import { emitNewMessages } from "./sse";
+import { workspaceOwnerOrchestrator } from "./workspace-host";
+import { isWorkspaceSteward, workspaceStewardRefusal } from "./workspace-steward-authority";
+import { settleWorkspaceMergeCommand } from "./workspace-command-settle";
+import { handBackLandConflict } from "./workspace-conflict-resolver";
+import { stewardRekeyRefusedForArm } from "./workspace-steward-wake";
+import type { Command, WorkspaceRecord } from "./types";
+
+export async function requestWorkspaceRekey(workspace: WorkspaceRecord, actor: string | undefined, metadata: Record<string, unknown>) {
+  const refusal = !isWorkspaceSteward(actor, workspace) ? "rekey requires the repository steward" : workspaceStewardRefusal(workspace, actor);
+  if (refusal) return { ok: false as const, httpStatus: 403, error: refusal };
+  const { oldHeadSha, newHeadSha } = metadata;
+  if (typeof oldHeadSha !== "string" || typeof newHeadSha !== "string"
+    || !/^[0-9a-f]{40}$/i.test(oldHeadSha) || !/^[0-9a-f]{40}$/i.test(newHeadSha)) {
+    return { ok: false as const, httpStatus: 400, error: "oldHeadSha and newHeadSha must be full commit SHAs" };
+  }
+  const owner = workspaceOwnerOrchestrator(workspace, listOrchestrators(), { onlineOnly: false });
+  if (!owner) return { ok: false as const, httpStatus: 409, error: "no orchestrator owns this workspace" };
+  const result = await getGateway().write("workspaceRekey.prepare", {
+    commandId: randomUUID(), workspaceId: workspace.id, target: owner.agentId, actor,
+    oldHeadSha: oldHeadSha.toLowerCase(), newHeadSha: newHeadSha.toLowerCase(),
+  }, { critical: true }) as { command?: Command; superseded?: Command[]; error?: string };
+  if (!result.command) return { ok: false as const, httpStatus: 409, error: result.error ?? "rekey refused" };
+  for (const command of result.superseded ?? []) await settleWorkspaceMergeCommand(command);
+  emitWorkspaceChange(getWorkspace(workspace.id));
+  return { ok: true as const, httpStatus: 202, workspace: getWorkspace(workspace.id)!, command: result.command };
+}
+
+export async function settleWorkspaceRekeyCommand(command: Command) {
+  if (!["succeeded", "failed", "rejected", "canceled", "timed_out"].includes(command.status)) return { settled: false };
+  await getGateway().write("workspaceRekey.settle", { command }, { critical: true });
+  await drainStewardRekeyNotices();
+  const ws = command.correlationId ? getWorkspace(command.correlationId) : null;
+  // #3702 — a refused rekey goes back to its owner now; the hand-off watchdog re-drives a crash here.
+  if (ws && stewardRekeyRefusedForArm(ws.metadata)) await handBackLandConflict(ws, Date.now(), `steward rekey refused: ${String(ws.metadata.rekeyError)}`);
+  emitWorkspaceChange(command.correlationId ? getWorkspace(command.correlationId) : null);
+  return { settled: true };
+}
+
+export function fenceWorkspaceRekeyCommand(commandId: string): Promise<{ fenced: boolean }> {
+  return getGateway().write("workspaceRekey.fence", { commandId }, { critical: true }) as unknown as Promise<{ fenced: boolean }>;
+}
+
+// #2805 — the coordinator reviewed the old head, never the steward's resolution of it. Runs after
+// each settle and at boot; a notice stays pending until routed (idempotent per command and Task).
+export async function drainStewardRekeyNotices(): Promise<void> {
+  for (const row of pendingStewardRekeyNotices()) {
+    try {
+      const routed = notifyTaskCoordinatorOfRekey(row);
+      if (routed) await getGateway().write("workspaceRekey.noticeSent", { eventId: row.id });
+    } catch (error) {
+      console.warn(`[steward-rekey] notice for task event ${row.id} failed; retrying later: ${error}`);
+    }
+  }
+}
+
+function notifyTaskCoordinatorOfRekey(row: StewardRekeyNoticeRow): boolean {
+  const task = getTaskDetail(row.task_id);
+  if (!task) return false;
+  const m = parseJson<Record<string, unknown>>(row.metadata, {});
+  const owner = typeof m.ownerAgentId === "string" ? m.ownerAgentId : undefined;
+  const rewritten = Array.isArray(m.rewritten) ? m.rewritten : [], merges = Array.isArray(m.merges) ? m.merges : [];
+  const workspaceId = String(m.workspaceId), oldHeadSha = String(m.oldHeadSha), newHeadSha = String(m.newHeadSha);
+  // A coordinator seat survives takeover; concrete ids are only the no-team fallback.
+  const teamId = [task.teamId, owner && getAgent(owner)?.teamId, task.createdBy && getAgent(task.createdBy)?.teamId]
+    .find((id): id is string => Boolean(id) && getTeam(id as string)?.status === "active");
+  const ownerRole = taskOwnerRoleTeamId(task.owner) && taskOwnerTarget(task) === task.owner ? task.owner : undefined; // #2862
+  const recipients = ownerRole ? [ownerRole] : teamId ? [`coordinator:${teamId}`]
+    : [...new Set([resolveCoordinatorAgentId(owner), task.createdBy?.trim()].filter((r): r is string => Boolean(r)))];
+  if (!recipients.length) return false;
+  const reasons = [
+    ...(m.reviewCleared ? [`your review cleared ${oldHeadSha}, not this resolution`] : []),
+    ...(rewritten.length ? [`${rewritten.length} commit(s) changed patch identity`] : []),
+    ...(merges.length ? [`${merges.length} merge commit(s) were re-resolved and cannot be verified by patch-id`] : []),
+  ];
+  const delivered = routeNotification({
+    type: "task.steward_rekey",
+    scope: {
+      taskId: String(task.id), transition: "steward-rekey", workspaceId,
+      ...(task.createdBy ? { parent: task.createdBy } : {}), ...(teamId ? { teamId } : {}),
+    },
+    recipients,
+    ttlMs: null,
+    message: {
+      subject: `Task T${task.id} land authorization re-keyed by steward`,
+      body: `Steward ${String(m.steward)} moved T${task.id}'s land authorization ${oldHeadSha} → ${newHeadSha}: ${reasons.join("; ")}. Review the new head (\`git range-diff ${oldHeadSha}...${newHeadSha}\`); withdraw the land if it is wrong.`,
+      payload: {
+        kind: "task.steward-rekey", notification: "task-steward-rekey", taskId: task.id, workspaceId,
+        commandId: m.commandId, oldHeadSha, newHeadSha, reviewCleared: m.reviewCleared === true, rewritten, merges,
+      },
+      idempotencyKey: `task:${task.id}:steward-rekey:${String(m.commandId)}`,
+      replyExpected: false,
+    },
+  });
+  if (delivered.length > 0) emitNewMessages(delivered);
+  return delivered.length > 0;
+}

package/src/workspace-steward-authority.ts

@@ -0,0 +1,16 @@
+import { getRepoSteward } from "./db/workspaces";
+import { getStewardConfig } from "./config-store";
+import { isAttachedManagedPolicySteward } from "./steward-provenance";
+import { workspaceActiveClaim } from "./workspace-claim";
+import type { WorkspaceRecord } from "./types";
+
+export function isWorkspaceSteward(actor: string | undefined, ws: WorkspaceRecord): boolean {
+  return Boolean(actor && (getRepoSteward(ws.repoRoot)?.stewardAgentId === actor
+    || isAttachedManagedPolicySteward(actor, ws.repoRoot)));
+}
+
+export function workspaceStewardRefusal(ws: WorkspaceRecord, actor: string | undefined, now = Date.now()): string | undefined {
+  if (!getStewardConfig().enabled) return "automatic stewardship is disabled";
+  if (workspaceActiveClaim(ws, now)?.by !== actor || !actor) return "an active steward claim is required";
+  return undefined;
+}

package/src/workspace-steward-failure.ts

@@ -0,0 +1,42 @@
+import { randomUUID } from "node:crypto";
+import { STEWARD_HANDOFF_WATCHDOG_MS, stewardFallbackTarget } from "./config";
+import { getWorkspace, patchWorkspaceMetadataAsync, transitionWorkspaceLandStateAsync } from "./db";
+import { routeNotification } from "./notification-router";
+import { raiseOperatorAlarm } from "./operator-alarm";
+import { resolveStewardIdentity } from "./steward-identity";
+import { workspaceStewardEligible } from "./steward-queue";
+import { withWorkspaceTaskOwnerRoles } from "./task-owner-notices";
+import { stewardRecoveryAdvice } from "./steward-slot-recovery";
+import { recordLandStopped } from "./workspace-land-stopped";
+import type { WorkspaceRecord } from "./types";
+
+/** A failed first wake needs the same durable watchdog anchor as a successful handoff. #2889 */
+export async function recordStewardWakeFailure(workspace: WorkspaceRecord, error: string, tip?: string): Promise<void> {
+  const ws = getWorkspace(workspace.id);
+  if (!ws || !workspaceStewardEligible(ws)) return;
+  const now = Date.now();
+  const capExhausted = /cap \d/.test(error);
+  if (capExhausted) await recordLandStopped(ws.id, "steward-cap-exhausted", error);
+  const firstAt = typeof ws.metadata.stewardHandoffFirstAt === "number" ? ws.metadata.stewardHandoffFirstAt : now;
+  const failureId = typeof ws.metadata.stewardWakeFailureId === "string" ? ws.metadata.stewardWakeFailureId : randomUUID();
+  await transitionWorkspaceLandStateAsync(ws.id, "merge-error", {
+    stewardWakeError: error, stewardWakeErrorAt: now, stewardHandoffFirstAt: firstAt, stewardWakeFailureId: failureId,
+    ...(tip ? { stewardHandoffTip: tip } : {}),
+  });
+  if (typeof ws.metadata.stewardWakeFailureNotifiedAt === "number") return;
+  const { ownerAgentId, coordinatorAgentId } = resolveStewardIdentity(ws);
+  const recipients = withWorkspaceTaskOwnerRoles([ownerAgentId, coordinatorAgentId, stewardFallbackTarget()], ws);
+  const subject = "Steward could not start — branch has not landed";
+  const body = `Workspace \`${ws.id}\` (\`${ws.branch ?? ws.id}\` → \`${ws.baseRef ?? "base"}\`): ${error}. Relay will retry after ${Math.ceil(STEWARD_HANDOFF_WATCHDOG_MS / 60_000)} minutes. ${stewardRecoveryAdvice(ws.baseRef ?? "main", capExhausted)}`;
+  const payload = { kind: "workspace.steward-spawn-failed", workspaceId: ws.id, branch: ws.branch, reason: error };
+  if (recipients.length) {
+    routeNotification({
+      type: "workspace.stranded_escalation", forceDelivery: true, ttlMs: null,
+      scope: { workspaceId: ws.id, repoRoot: ws.repoRoot, author: ownerAgentId, parent: coordinatorAgentId ?? undefined },
+      recipients, message: { subject, body, payload, idempotencyKey: `steward-spawn-failed:${ws.id}:${failureId}` },
+    });
+  } else {
+    raiseOperatorAlarm({ clientId: `steward-spawn-failed:${ws.id}:${failureId}`, source: "workspace-steward-failure", subject, detail: body, dedupeEvent: true, metadata: payload });
+  }
+  await patchWorkspaceMetadataAsync(ws.id, { stewardWakeFailureNotifiedAt: now });
+}

package/src/workspace-steward-wake.ts

@@ -0,0 +1,150 @@
+import { getWorkspace, listOrchestrators, patchWorkspaceMetadataAsync, transitionWorkspaceLandStateAsync } from "./db";
+import { notifyStewardHandoff } from "./branch-landed";
+import { resolveStewardIdentity } from "./steward-identity";
+import { getLifecycleManager } from "./lifecycle-manager";
+import { getManagedAgentState } from "./config-store";
+import { stewardHandoffIdentity } from "./steward-provenance";
+import { ensureRepoSteward } from "./steward";
+import { recordStewardWakeFailure } from "./workspace-steward-failure";
+import { workspaceStewardEligible } from "./steward-queue";
+import { workspaceOwnerOrchestrator } from "./workspace-host";
+import { workspaceShowsDetectedLand } from "./workspace-phase";
+import { routeNotification } from "./notification-router";
+import { renderTemplate } from "./prompt-resolver";
+import { STEWARD_HANDOFF_WATCHDOG_MS, STEWARD_WAKE_COOLDOWN_MS } from "./config";
+import { isPathWithinBase } from "./utils";
+import type { Orchestrator, WorkspaceRecord } from "./types";
+
+function stewardOwnerForWorkspace(ws: WorkspaceRecord): { owner: Orchestrator } | { error: string } {
+  const orchestrators = listOrchestrators();
+  if (ws.orchestratorId) {
+    const owner = orchestrators.find((orch) => orch.id === ws.orchestratorId);
+    if (!owner) return { error: `cannot spawn steward: owning host ${ws.orchestratorId} is not registered` };
+    if (owner.status !== "online") return { error: `cannot spawn steward: owning host ${owner.id} is ${owner.status}` };
+    if (!isPathWithinBase(ws.worktreePath, owner.baseDir)) {
+      return { error: `cannot spawn steward: worktree is not on owning host ${owner.id}` };
+    }
+    return { owner };
+  }
+  const owner = workspaceOwnerOrchestrator(ws, orchestrators, { requireApiUrl: true });
+  return owner ? { owner } : { error: "cannot spawn steward: no online owner for workspace host path" };
+}
+
+/** #2704 — the conflict went back to its owner before the land armed now: that re-delivery starts a fresh steward cycle. */
+export function stewardHandoffPredatesLandArm(meta: Record<string, unknown>): boolean {
+  return typeof meta.stewardHandoffEscalatedAt === "number" && typeof meta.landRequestAttemptAt === "number"
+    && meta.stewardHandoffEscalatedAt < meta.landRequestAttemptAt;
+}
+
+/** #3702 — the steward's rekey for the land armed now was refused. Another steward would replay the
+ *  same refusal, so the land goes back to its owner and only a new arm (a re-delivery or a coordinator
+ *  re-arm) starts a fresh steward cycle. */
+export function stewardRekeyRefusedForArm(meta: Record<string, unknown>): boolean {
+  return typeof meta.rekeyRefusedAt === "number"
+    && (typeof meta.landRequestAttemptAt !== "number" || meta.rekeyRefusedAt >= meta.landRequestAttemptAt);
+}
+
+/** Unrelated metadata writes can lose the stamp's CAS; past this many, stamp unconditionally. */
+const STEWARD_WAKE_STAMP_ATTEMPTS = 3;
+
+export type StewardWakeOutcome =
+  | { ok: true; policy: string }
+  | { ok: false; cooldown: boolean; error?: string };
+
+export async function wakeRepoSteward(ws: WorkspaceRecord, reason: string, opts: { tip?: string } = {}): Promise<StewardWakeOutcome> {
+  ws = getWorkspace(ws.id) ?? ws;
+  if (!workspaceStewardEligible(ws) || stewardRekeyRefusedForArm(ws.metadata)) return { ok: false, cooldown: true };
+  const meta = ws.metadata;
+  const failedAt = typeof meta.stewardWakeErrorAt === "number" ? meta.stewardWakeErrorAt : undefined;
+  const tipMovedSinceFailure = failedAt !== undefined && opts.tip && meta.stewardHandoffTip && opts.tip !== meta.stewardHandoffTip;
+  if (tipMovedSinceFailure || stewardHandoffPredatesLandArm(meta)) {
+    ws = await patchWorkspaceMetadataAsync(ws.id, { stewardWakeError: undefined, stewardWakeErrorAt: undefined, stewardWakeFailureNotifiedAt: undefined, stewardWakeFailureId: undefined, stewardHandoffFirstAt: undefined, stewardHandoffEscalatedAt: undefined, stewardRewakeCount: undefined }) ?? ws;
+  } else if (failedAt !== undefined && Date.now() - failedAt < STEWARD_HANDOFF_WATCHDOG_MS) {
+    return { ok: false, cooldown: true };
+  }
+  if (workspaceShowsDetectedLand(ws)) return { ok: false, cooldown: true };
+  const lastWoke = typeof meta.stewardWokenAt === "number" ? meta.stewardWokenAt : 0;
+  if (lastWoke && Date.now() - lastWoke < STEWARD_WAKE_COOLDOWN_MS) return { ok: false, cooldown: true };
+  const ownerResult = stewardOwnerForWorkspace(ws);
+  if ("error" in ownerResult) {
+    await recordStewardWakeFailure(ws, ownerResult.error, opts.tip);
+    return { ok: false, cooldown: false, error: ownerResult.error };
+  }
+  try {
+    const policyName = ensureRepoSteward(ws.repoRoot, ownerResult.owner);
+    if (!policyName) throw new Error("cannot spawn steward: steward policy could not be provisioned");
+    const { ownerAgentId, coordinatorAgentId } = resolveStewardIdentity(ws);
+    const coordinatorLine = coordinatorAgentId
+      ? ` Escalate to the branch agent's coordinator \`${coordinatorAgentId}\` (not the system fallback) if you can't land it.`
+      : "";
+    const ownerLine = ownerAgentId ? ` Branch agent (owner): \`${ownerAgentId}\`.` : "";
+    // Publish the obligation before its message can be consumed or trigger a spawn. #3288
+    const persisted = getWorkspace(ws.id)?.metadata ?? meta;
+    const priorFirstAt = typeof persisted.stewardHandoffFirstAt === "number" ? persisted.stewardHandoffFirstAt : undefined;
+    const priorTip = typeof persisted.stewardHandoffTip === "string" ? persisted.stewardHandoffTip : undefined;
+    const freshHandoff = priorFirstAt === undefined || Boolean(opts.tip && opts.tip !== priorTip);
+    await patchWorkspaceMetadataAsync(ws.id, {
+      stewardPolicy: policyName,
+      stewardHandoffEndedAt: undefined,
+      ...(freshHandoff ? {
+        stewardHandoffFirstAt: Date.now(),
+        ...(opts.tip ? { stewardHandoffTip: opts.tip } : {}),
+        stewardHandoffEscalatedAt: undefined, stewardRewakeCount: undefined,
+      } : {}),
+    });
+    routeNotification({
+      type: "workspace.steward_task",
+      scope: { workspaceId: ws.id, repoRoot: ws.repoRoot, author: ownerAgentId, parent: coordinatorAgentId ?? undefined },
+      recipients: [`policy:${policyName}`],
+      message: {
+        // Failed retries share a task; a successful wake re-arms same-tip replacement delivery.
+        idempotencyKey: `steward-task:${ws.id}:${opts.tip ?? meta.stewardHandoffTip ?? "unknown"}:${lastWoke}`,
+        subject: `Steward: ${ws.status} workspace needs attention`,
+        body: renderTemplate("steward.wake", {
+          branch: ws.branch ?? ws.id,
+          wsId: ws.id,
+          repoRoot: ws.repoRoot,
+          status: ws.status,
+          reason,
+          ownerLine,
+          worktreePath: ws.worktreePath ?? ws.sourceCwd,
+          baseRef: ws.baseRef ?? "base",
+          coordinatorLine,
+        }),
+        payload: { kind: "workspace.steward-task", workspaceId: ws.id, repoRoot: ws.repoRoot, worktreePath: ws.worktreePath, branch: ws.branch, baseRef: ws.baseRef, status: ws.status, reason, ownerAgentId, coordinatorAgentId },
+      },
+    });
+    const spawn = await getLifecycleManager().onMessageForPolicy(policyName);
+    // #1870 S3 — the wake stamp is a compare-and-set against the row this decision read: concurrent
+    // wakes interleave across the writer hop, and only the one that lands the stamp announces.
+    for (let attempt = 0; ; attempt++) {
+      const current = getWorkspace(ws.id);
+      if (!current || current.status !== ws.status || current.ownerAgentId !== ws.ownerAgentId || !workspaceStewardEligible(current)) return { ok: false, cooldown: true };
+      const currentWokeAt = current.metadata.stewardWokenAt;
+      if (typeof currentWokeAt === "number" && Date.now() - currentWokeAt < STEWARD_WAKE_COOLDOWN_MS) return { ok: true, policy: policyName };
+      if (!spawn.woke) {
+        if (getManagedAgentState(policyName)?.status === "stopping") return { ok: false, cooldown: true };
+        await recordStewardWakeFailure(current, spawn.reason, opts.tip);
+        return { ok: false, cooldown: false, error: spawn.reason };
+      }
+      const stamped = await patchWorkspaceMetadataAsync(ws.id, {
+        stewardWokenAt: Date.now(),
+        ...stewardHandoffIdentity(policyName),
+        stewardPolicy: policyName,
+        coordinatorAgentId: coordinatorAgentId ?? undefined,
+        stewardWakeError: undefined,
+        stewardWakeErrorAt: undefined,
+        stewardWakeFailureNotifiedAt: undefined,
+        stewardWakeFailureId: undefined,
+      }, attempt < STEWARD_WAKE_STAMP_ATTEMPTS ? current : undefined);
+      if (!stamped) continue;
+      if (current.landState === "merge-error") await transitionWorkspaceLandStateAsync(ws.id, "steward-requested");
+      notifyStewardHandoff({ workspace: ws, stewardId: policyName, coordinatorAgentId, reason });
+      return { ok: true, policy: policyName };
+    }
+  } catch (e) {
+    const error = e instanceof Error ? e.message : "failed to wake steward";
+    await recordStewardWakeFailure(ws, error, opts.tip);
+    return { ok: false, cooldown: false, error };
+  }
+}

package/src/workspace-steward-watchdog.ts

@@ -0,0 +1,240 @@
+// Steward hand-off watchdog (#1440).
+//
+// A land-conflict routes an unlandable workspace to a repo steward and fires the reassuring
+// "🛟 …now with a steward, will land or escalate" hand-off notice (branch-landed.ts →
+// notifyStewardHandoff). But that notice rides on `wakeRepoSteward().ok`, which reports success
+// the instant a spawn *command* is emitted (`via:"spawned"`) or a managed state merely LOOKS live
+// (`via:"already-active"` — a stale "running" record, or a steward stuck "starting" on a provider-
+// approval hold). Nothing read `stewardWokenAt` back to confirm the steward actually landed or
+// escalated, so a steward that never came up / crashed / was reaped / held on approval silently
+// stranded the work: no land, no escalation, only a false "with a steward" board.
+//
+// This module is the single closer-of-loop that makes the #1440 invariant hold regardless of WHY a
+// steward isn't progressing: a land-conflict hand-off ALWAYS ends in (1) a landed branch, (2) a
+// hand-back to the branch owner with its coordinator told (#2704, workspace-conflict-resolver.ts), or
+// (3) a durable, surfaced "needs a steward, none available" state — never a silent strand. #2704 —
+// it also gives a `conflict` row that never got a steward wake its first one. It complements the
+// coarser 60-min workspace-GC custody backstop with a faster, steward-liveness-aware check.
+//
+// #1543 — a steward correctly verifying "this workspace's work is already on the base" and
+// stopping WITHOUT landing (never attempt a duplicate merge) is a fourth, equally-terminal outcome
+// that must NOT be confused with "not progressing": that stop looks identical, at the process
+// level, to a crashed/backed-off steward, so `classifyStewardHealth` alone can't tell them apart.
+// `checkAlreadyLanded` runs before every rewake/escalate decision to close this outcome out
+// immediately, so an already-landed hand-off can never be endlessly re-woken. #1631 — "close out"
+// means terminal `merged` only when the owning agent is GONE; a live owner keeps its workspace and
+// only loses the hand-off bookkeeping (see reconcileLandedWorkspace).
+
+import { errMessage, stringValue } from "@humwick/sdk";
+import { STEWARD_HANDOFF_DEADLINE_MS, STEWARD_HANDOFF_REWAKE_MAX, STEWARD_HANDOFF_WATCHDOG_MS } from "./config";
+import { getManagedAgentState, getStewardConfig } from "./config-store";
+import { getAgent, getWorkspace, listOrchestratorsAsync, listWorkspaces, patchWorkspaceMetadataAsync } from "./db";
+import { reconcileLandedWorkspace } from "./branch-landed";
+import { previewShowsNothingToLand } from "./maintenance/workspaces";
+import { STEWARD_QUEUE_STATUSES, workspaceStewardEligible } from "./steward-queue";
+import { stewardHandoffPredatesLandArm, stewardRekeyRefusedForArm, wakeRepoSteward } from "./workspace-steward-wake";
+import { conflictHandedBackForArm, handBackLandConflict } from "./workspace-conflict-resolver";
+import { workspaceOwnerOrchestrator } from "./workspace-host";
+import { TERMINAL_WORKSPACE_STATUSES, workspaceShowsDetectedLand } from "./workspace-phase";
+import { workspaceActiveClaim } from "./workspace-claim";
+import { stewardGenerationCanceled } from "./db/workspace-steward-fence";
+import { fetchHostMergePreview } from "./workspace-probe";
+import type { WorkspaceRecord } from "./types";
+
+type StewardHealth = { healthy: boolean; reason: string; retry?: false };
+
+// A live process only owns this handoff while its workspace claim remains active. #3288
+function classifyStewardHealth(ws: WorkspaceRecord, policyName: string | undefined, now: number, windowMs: number): StewardHealth {
+  if (typeof ws.metadata.stewardHandoffEndedAt === "number") return { healthy: false, retry: false, reason: "steward handed the workspace back" };
+  if (!policyName) return { healthy: false, reason: "no steward policy recorded on the hand-off" };
+  const state = getManagedAgentState(policyName);
+  if (!state) return { healthy: false, reason: "no managed steward state (never spawned or reaped)" };
+  switch (state.status) {
+    case "running": {
+      const agent = state.agentId ? getAgent(state.agentId) : null;
+      // #1440 — dead only when missing/`offline`, matching the shared managedRunningAgentDead
+      // predicate. `stale` is a transient reconnect-grace state (the heartbeat reaper flips a
+      // genuinely-gone agent to `offline`), so a `stale` steward is given grace (treated as live)
+      // here too — never spuriously disrupted or escalated during a reconnect blip.
+      if (!agent || agent.status === "offline") {
+        return { healthy: false, reason: `steward marked running but its process is ${agent?.status ?? "gone"} (dead-running)` };
+      }
+      if (workspaceActiveClaim(ws, now)?.by !== agent.id || stewardGenerationCanceled(agent.id, stringValue(agent.meta?.spawnRequestId))) {
+        return { healthy: false, retry: false, reason: "steward process is live but does not hold an eligible claim for this workspace" };
+      }
+      return { healthy: true, reason: "steward holds this workspace (live)" };
+    }
+    case "starting": {
+      const since = state.lastSpawnAt ?? state.updatedAt ?? 0;
+      if (!since || now - since >= windowMs) {
+        return { healthy: false, reason: "steward stuck starting past the window (provider-approval hold or wedged boot)" };
+      }
+      return { healthy: true, reason: "steward starting (within window)" };
+    }
+    case "backoff": return { healthy: false, reason: `steward in backoff (${state.lastError ?? "unknown"})` };
+    case "failed": return { healthy: false, reason: `steward policy failed (${state.lastError ?? "unknown"})` };
+    case "stopping":
+    case "stopped": return { healthy: false, reason: `steward ${state.status}` };
+    default: return { healthy: false, reason: `steward status ${state.status}` };
+  }
+}
+
+// #1543 — a steward that verifies the workspace's work is ALREADY on the base (landed out-of-band,
+// or a redundant/empty-diff workspace left over from a duplicate hand-off) is not stuck — it's
+// DONE. Correctly stopping without landing (never attempt a duplicate merge) must close the loop
+// as a TERMINAL outcome, not read as "not progressing" and get re-woken. This mirrors the same
+// `previewShowsNothingToLand` reconciliation the slower workspace-GC/auto-abandon sweeps already run
+// (maintenance/workspaces.ts), just on THIS watchdog's faster hand-off-scoped cadence, so a landed
+// hand-off doesn't have to wait up to an hour for those sweeps to close it out. Detection can only
+// under-report (never falsely claim "landed"), so any transport/host failure or ambiguous preview
+// is treated as NOT proven — the caller falls through to the normal rewake/escalate path.
+//
+// #1631 — this closes the hand-off out; it does NOT decide whether the workspace is retired.
+// `previewShowsNothingToLand` is true both for a branch whose work reached the base and for the fresh
+// empty `<branch>-N` a land-and-continue recycle cuts (nothing outstanding, vacuously), and the
+// candidate filter below deliberately still includes LIVE agents — a steward hand-off normally
+// belongs to a live branch agent parked waiting for its branch to land, so excluding them would
+// blind the watchdog to the exact strand it exists to catch. What must not follow from a vacuous
+// proof is retiring a running agent's checkout, and that call is made once, inside
+// `reconcileLandedWorkspace`, on owner liveness (mirroring the host's #327 no-op guard). This
+// returns which of the two outcomes happened purely so the sweep can report it honestly.
+async function checkAlreadyLanded(ws: WorkspaceRecord): Promise<"merged" | "kept-live" | null> {
+  // #2276 — this runs once PER CANDIDATE inside the sweep loop, so a raw `listOrchestrators()` here
+  // was a repeated synchronous read on the event loop. `orchestrators.list` is already a registered
+  // read op, so routing through the gateway moves it onto the (worker-backed) reader pool for free.
+  const orchestrators = (await listOrchestratorsAsync()).filter((o) => o.status === "online" && o.apiUrl);
+  const orch = workspaceOwnerOrchestrator(ws, orchestrators, { requireApiUrl: true });
+  if (!orch?.apiUrl) return null;
+  const preview = await fetchHostMergePreview(orch.apiUrl, ws, { checkPr: true, requireRemoteBase: true });
+  if (!preview || "available" in preview || preview.error || preview.missing) return null;
+  if (!previewShowsNothingToLand(preview)) return null;
+  return await reconcileLandedWorkspace(ws, preview);
+}
+
+// The steward-handoff watchdog sweep (#1440), the periodic half of the #2704 conflict resolver. Every
+// non-terminal `conflict` row, and every row handed to a steward, gets one live resolver: a conflict no
+// steward attempt served yet gets one now; once the bounded window has elapsed a healthy-running steward
+// is left alone until the deadline; otherwise re-wake up to STEWARD_HANDOFF_REWAKE_MAX times, then hand
+// the land back to its owner (outcome 2) or raise a durable alarm when nobody can take it (outcome 3).
+// A re-wake never disrupts a live steward — `onMessageForPolicy` just re-queues onto an already-active
+// one — so a false "unhealthy" read costs at most a harmless re-queue.
+export async function runStewardHandoffWatchdog(
+  opts: { now?: number; windowMs?: number; deadlineMs?: number; signal?: AbortSignal } = {},
+): Promise<Record<string, unknown>> {
+  if (opts.signal?.aborted) return { aborted: true };
+  const stewardEnabled = getStewardConfig().enabled;
+  const now = opts.now ?? Date.now();
+  const windowMs = opts.windowMs ?? STEWARD_HANDOFF_WATCHDOG_MS;
+  const deadlineMs = opts.deadlineMs ?? STEWARD_HANDOFF_DEADLINE_MS;
+  const woken: string[] = [], waiting: string[] = [], healthy: string[] = [], rewaked: string[] = [], escalated: string[] = [], alarmed: string[] = [], pending: string[] = [], dormant: string[] = [], failed: string[] = [], landed: string[] = [], landedOwnerLive: string[] = [];
+  const handBack = async (ws: WorkspaceRecord, why: string): Promise<void> => {
+    const outcome = await handBackLandConflict(ws, now, why);
+    if (outcome === "handed-back") escalated.push(ws.id);
+    else if (outcome === "alarmed") alarmed.push(ws.id);
+    else if (outcome === "pending") pending.push(ws.id);
+  };
+
+  const candidates = listWorkspaces().filter((ws) =>
+    ws.mode === "isolated"
+    && Boolean(ws.worktreePath)
+    && !TERMINAL_WORKSPACE_STATUSES.has(ws.status)
+    && (STEWARD_QUEUE_STATUSES.has(ws.status) || typeof ws.metadata.stewardHandoffFirstAt === "number" || typeof ws.metadata.stewardWokenAt === "number" || typeof ws.metadata.stewardWakeErrorAt === "number")
+    && !workspaceShowsDetectedLand(ws)
+    && (workspaceStewardEligible(ws) || typeof ws.metadata.conflictHandbackPending === "string"),
+  );
+
+  for (const ws of candidates) {
+    // #1819 — genuinely cancellable: a candidate awaits a host merge preview (the landed check) and
+    // steward health probes, so an abort lands there and the sweep stops at the next candidate. One
+    // candidate is the unit — its re-wake count, escalation flag and the wake/page it authorizes are
+    // written together — so this is the only safe checkpoint.
+    if (opts.signal?.aborted) break;
+    // #1440-4 — isolate each candidate: a single throwing/poison row (e.g. a bad fallback target)
+    // must not abort the sweep or skip the rows after it. Record it and keep going.
+    try {
+      const meta = ws.metadata as Record<string, unknown>;
+
+      // #2704 — a hand-back waiting on its steward's shutdown completes here once that settles.
+      if (typeof meta.conflictHandbackPending === "string") { await handBack(ws, meta.conflictHandbackPending); continue; }
+
+      // #1440-2 — a conflict handed back for the land armed now is DORMANT: its owner (or the
+      // needs-a-steward alarm) holds it. A re-delivery arms a new land and starts a new cycle.
+      if (conflictHandedBackForArm(meta)) { dormant.push(ws.id); continue; }
+
+      // #3702 — a refused rekey hands the land back at settle; this re-drives one a crash interrupted.
+      if (stewardRekeyRefusedForArm(meta)) { await handBack(ws, `steward rekey refused: ${String(meta.rekeyError ?? "unknown")}`); continue; }
+
+      // #2704 — with stewards off, the owner is the only resolver.
+      if (!stewardEnabled) { await handBack(ws, "no steward is enabled for this repo"); continue; }
+
+      // #2704 case A — a conflict that entered without a steward wake (a merge settled as a conflict),
+      // or whose steward served an earlier land arm, gets its steward now instead of waiting forever.
+      const attempted = !stewardHandoffPredatesLandArm(meta) && [meta.stewardHandoffFirstAt, meta.stewardWokenAt, meta.stewardWakeErrorAt].some((at) => typeof at === "number");
+      if (!attempted) {
+        const tip = typeof meta.landRequestHeadSha === "string" ? meta.landRequestHeadSha : undefined;
+        const wake = await wakeRepoSteward(ws, "conflict with no live steward", tip ? { tip } : {});
+        (wake.ok ? woken : waiting).push(ws.id);
+        continue;
+      }
+
+      // #1440-1 — measure the window from the IMMUTABLE first-hand-off anchor, NOT `stewardWokenAt`
+      // (which the 10-min auto-merge cooldown loop re-stamps every pass — that would reset the
+      // window forever and the watchdog would never fire). Fall back to stewardWokenAt only for
+      // rows handed off before this anchor existed.
+      const firstAt = typeof meta.stewardHandoffFirstAt === "number" ? meta.stewardHandoffFirstAt : (typeof meta.stewardWokenAt === "number" ? meta.stewardWokenAt : meta.stewardWakeErrorAt as number);
+      if (now - firstAt < windowMs) { waiting.push(ws.id); continue; }
+      // #2704 — a steward that looks alive but has not landed by the deadline is not resolving it either.
+      const pastDeadline = now - firstAt >= deadlineMs;
+
+      const policyName = typeof meta.stewardPolicy === "string" ? meta.stewardPolicy : undefined;
+      const health = classifyStewardHealth(ws, policyName, now, windowMs);
+      if (health.healthy && !pastDeadline) {
+        if (typeof meta.stewardWakeErrorAt === "number") await wakeRepoSteward(ws, "steward recovered after failed spawn");
+        healthy.push(ws.id); continue;
+      }
+
+      // #1543 — before treating "not progressing" as a genuine stall, check whether the branch's
+      // work is ALREADY on the base. A steward that stopped here because it correctly verified
+      // "nothing to land" is DONE, not stuck: close the loop now (terminal `merged`) rather than
+      // re-waking or handing back a workspace that has nothing left to land.
+      // #1631 — a live owner's row is closed out WITHOUT being retired (see checkAlreadyLanded);
+      // either way the hand-off is over and this sweep is done with the row.
+      const alreadyLanded = await checkAlreadyLanded(getWorkspace(ws.id) ?? ws);
+      if (alreadyLanded) { (alreadyLanded === "kept-live" ? landedOwnerLive : landed).push(ws.id); continue; }
+      const afterProbe = getWorkspace(ws.id);
+      if (!afterProbe || !workspaceStewardEligible(afterProbe)) continue;
+      if (!pastDeadline && classifyStewardHealth(afterProbe, policyName, now, windowMs).healthy) { healthy.push(ws.id); continue; }
+
+      const rewakeCount = typeof meta.stewardRewakeCount === "number" ? meta.stewardRewakeCount : 0;
+      if (!pastDeadline && health.retry !== false && rewakeCount < STEWARD_HANDOFF_REWAKE_MAX) {
+        const fresh = getWorkspace(ws.id) ?? ws;
+        // The window (> STEWARD_WAKE_COOLDOWN_MS) guarantees the wake cooldown has lapsed, so this
+        // re-wake attempts a spawn. #1440-3 — hardened onMessageForPolicy no longer returns
+        // "already-active" for a dead/wedged steward, so this re-wake forces an ACTUAL respawn
+        // (not just a re-queued message). No tip is passed → treated as the SAME strand, so the
+        // immutable anchor and escalation flag are preserved (findings #1440-1/-2).
+        const wake = await wakeRepoSteward(fresh, `steward-handoff watchdog re-wake: prior steward not progressing (${health.reason})`);
+        // Write the incremented counter AFTER the wake so the bound is honored (a same-strand wake
+        // does not reset it).
+        if (!wake.ok && wake.cooldown) { waiting.push(ws.id); continue; }
+        await patchWorkspaceMetadataAsync(ws.id, { stewardRewakeCount: rewakeCount + 1 });
+        // #2704 — a respawn the lifecycle is still backing off (or a host that is briefly away) spends
+        // one attempt and retries next sweep; the bound and the deadline decide the hand-back.
+        (wake.ok ? rewaked : waiting).push(ws.id);
+        continue;
+      }
+
+      const current = getWorkspace(ws.id);
+      if (!current || !workspaceStewardEligible(current)) continue;
+      const failure = pastDeadline
+        ? `no steward landed it within ${Math.round(deadlineMs / 60_000)} minutes`
+        : typeof current.metadata.stewardWakeError === "string" ? current.metadata.stewardWakeError : health.reason;
+      await handBack(current, failure);
+    } catch (e) {
+      failed.push(ws.id);
+      console.warn(`[workspace-steward-handoff-watchdog] failed to process ${ws.id}: ${errMessage(e)}`);
+    }
+  }
+
+  return { candidates: candidates.length, woken, waiting, healthy, rewaked, escalated, alarmed, pending, dormant, failed, landed, landedOwnerLive, ...(opts.signal?.aborted ? { aborted: true } : {}) };
+}