@humwick/server 0.0.0-stage → 0.130.0

package/src/steward-slot-recovery.ts

@@ -0,0 +1,34 @@
+import { stringValue } from "@humwick/sdk";
+import { getManagedAgentState, getStewardConfig } from "./config-store";
+import { getWorkspace, listWorkspaces, patchWorkspaceMetadataAsync } from "./db";
+import { landStoppedFact } from "./workspace-land-stopped";
+import { workspaceStewardEligible } from "./steward-queue";
+
+/** A confirmed exit frees capacity for handoffs stopped by the steward cap. */
+export async function retryStewardLandsAfterSlotFreed(): Promise<void> {
+  if (!getStewardConfig().enabled) return;
+  for (const snapshot of listWorkspaces()) {
+    const ws = getWorkspace(snapshot.id);
+    if (!ws || !workspaceStewardEligible(ws)) continue;
+    const stop = landStoppedFact(ws.metadata);
+    if (stop?.reason !== "steward-cap-exhausted" || stop.handoff !== ws.metadata.landRequestAttemptAt || typeof ws.metadata.stewardHandoffEndedAt === "number") continue;
+    const policyName = stringValue(ws.metadata.stewardPolicy);
+    const state = policyName ? getManagedAgentState(policyName) : undefined;
+    if (typeof ws.metadata.landRequestAttemptAt !== "number" || (ws.landedAt ?? 0) >= ws.metadata.landRequestAttemptAt) continue;
+    if (state?.status === "stopping") continue;
+    const fresh = await patchWorkspaceMetadataAsync(ws.id, {
+      landStopped: undefined, stewardHandoffEscalatedAt: undefined, stewardRewakeCount: undefined,
+      stewardHandoffFirstAt: undefined, stewardWokenAt: undefined,
+      stewardWakeError: undefined, stewardWakeErrorAt: undefined,
+      stewardWakeFailureNotifiedAt: undefined, stewardWakeFailureId: undefined,
+    });
+    if (fresh) {
+      const { wakeRepoSteward } = await import("./workspace-steward-wake");
+      await wakeRepoSteward(fresh, "steward slot freed");
+    }
+  }
+}
+
+export function stewardRecoveryAdvice(base: string, capExhausted = false): string {
+  return `${capExhausted ? "Wait for a steward slot; Relay retries automatically when one frees. " : ""}To recover yourself, rebase onto ${base}, resolve conflicts, run the project gates, push, and re-deliver complete with a new idempotencyKey. If a land is still armed, withdraw it with relay_task_withdraw_land first; withdrawn:false means no land is armed, so proceed. Workflow resume is not needed.`;
+}

package/src/steward-spawn-guard.ts

@@ -0,0 +1,344 @@
+// The single authoritative gate on spawning a landing-steward agent (#1745).
+//
+// Before this, four independent facts each let a steward (re)spawn with nothing that could hold it
+// down, which is what produced the runaway spiral (8+ codex stewards for one repo, un-killable from
+// the dashboard):
+//   1. Every spawn decision (`onMessageForPolicy`, `reconcilePolicy`) keyed off the SPAWN-POLICY's
+//      own `enabled` flag and `hasQueuedPolicyWork`, and NEVER `getStewardConfig().enabled` — so the
+//      operator's global "disable steward spawning" toggle changed nothing.
+//   2. `ensureRepoSteward` rewrote a manually-stopped steward policy back to `enabled:true` on the
+//      next sweep, reverting a UI stop.
+//   3. Durable `policy:<steward>` obligations (ttl null) and any actionable workspace kept
+//      `hasQueuedPolicyWork` true forever, so the reconcile tick respawned the steward every pass.
+//   4. Nothing capped concurrent stewards per repo or the respawn RATE of a repeatedly-dying one.
+//
+// This module centralises all of that into one predicate the lifecycle manager consults at every
+// spawn point, plus the reap the disable-toggle now performs. A NON-steward policy is never affected
+// — `stewardSpawnDenyReason` returns null for it — so this can be called unconditionally.
+
+import { basename } from "node:path";
+import { MANAGED_START_TIMEOUT_MS, MAX_LIVE_STEWARDS_FLEET, MAX_LIVE_STEWARDS_PER_REPO, MAX_LIVE_STEWARDS_PER_REPO_HOST, STEWARD_SPAWN_RATE_MAX, STEWARD_SPAWN_RATE_WINDOW_MS } from "./config";
+import { getManagedAgentState, getSpawnPolicy, getStewardConfig, listSpawnPolicies } from "./config-store";
+import { getAgent, getOrchestrator, listAgents, listWorkspaces } from "./db";
+import { resolveCanonicalHost } from "./host-identity";
+import { allActionableStewardWorkspacesClaimed, hasQueuedStewardWorkspace } from "./steward-queue";
+import { repoStewardPolicyName } from "./steward-policy-name";
+import { isStewardPolicy, isGenuineStewardPolicy, isProvenStewardAgent, repoRootForStewardPolicy, stewardHandoffReleased } from "./steward-provenance";
+export { isStewardPolicy, isGenuineStewardPolicy, isProvenStewardAgent, isAttachedManagedPolicySteward, repoRootForStewardPolicy } from "./steward-provenance";
+import { sameHostIdentity } from "./utils";
+import { AGENT_SYSTEM_TAGS, hasAgentSystemTag } from "./agent-system-tags";
+import type { AgentCard, SpawnPolicy } from "./types";
+
+/** The label buildManagedSpawnParams stamps on every steward agent for `repoRoot`
+ *  (`steward:<basename>`, see desiredStewardPolicy in steward.ts). This is how an ORPHANED or WEDGED
+ *  steward agent — one with no live managed policy state — is still attributed to its (repo, host).
+ *  Basename-collision caveat: two different repos sharing a directory basename produce the same label;
+ *  the (repo, host) cap below fails CLOSED on that (counts them together → denies), which is the safe
+ *  direction for a containment cap. */
+export function stewardAgentLabelForRepo(repoRoot: string): string {
+  return `steward:${basename(repoRoot)}`;
+}
+
+/**
+ * The extra token `cwdPrefix` a PROVEN steward needs beyond its launch cwd (#1623). A steward now
+ * launches at its project repo root (so its project memory/instructions/MCP-tool cwd resolve to the
+ * repo it serves), but the managed worktrees it lands live at `<baseDir>/.agent-relay/workspaces/…`,
+ * OUTSIDE that repo root. The `workspace.merge` command it dispatches to land a branch carries a
+ * `worktreePath` there, gated against this token's `cwdPrefixes` (services/dispatch-command.ts). So a
+ * proven steward's token must ALSO authorize the orchestrator baseDir — exactly as it did when baseDir
+ * WAS the launch cwd. baseDir ⊇ repoRoot, so this restores the steward's prior path authority byte-for-
+ * byte; only the launch/identity cwd moved to the repo.
+ *
+ * `proven` is the caller's UNFORGEABLE steward verdict — `isGenuineStewardPolicy` for the config-driven
+ * initial mint, or `isProvenStewardAgent` for a restart. It is NEVER derived from a tag here: passing a
+ * tag-only verdict would reopen the #1623 P1 privilege escalation (an ordinary agent minting sibling-
+ * repo authority by spawning with a `steward` tag). Non-stewards get nothing extra.
+ */
+export function stewardWorktreeCwdPrefixes(
+  proven: boolean,
+  launchCwd: string,
+  orchestratorBaseDir: string | undefined,
+): string[] {
+  if (!proven || !orchestratorBaseDir || launchCwd === orchestratorBaseDir) return [];
+  return [orchestratorBaseDir];
+}
+
+/** The operator's global kill-switch: a steward policy must not (re)spawn — and a live one must be
+ *  reaped — whenever steward spawning is globally disabled. This is the fact every spawn gate was
+ *  missing. Non-steward policies are never gated here. */
+export function stewardKillSwitchEngaged(policy: Pick<SpawnPolicy, "tags" | "name">): boolean {
+  return isStewardPolicy(policy) && !getStewardConfig().enabled;
+}
+
+/** #1953 RC3 — dashboard restart/resume re-launches a fresh steward process, so it must honor the
+ *  operator's kill-switch: with stewards disabled, the dashboard must not resurrect one. Returns a
+ *  denial reason, or null to allow. Only a GENUINE relay-provisioned steward policy qualifies (a
+ *  caller-set `steward` tag never does); the per-repo/host caps are intentionally NOT applied — a
+ *  restart/resume REPLACES this exact agent, not an addition, so counting it would false-deny. */
+export function stewardRestartResumeDenyReason(policyName: string | undefined): string | null {
+  if (!policyName) return null;
+  const policy = getSpawnPolicy(policyName)?.value;
+  if (!policy || !isGenuineStewardPolicy(policy) || !stewardKillSwitchEngaged(policy)) return null;
+  return "steward restart/resume refused: steward spawning is disabled by the operator";
+}
+
+/** A steward wake is meaningful only while the server-side queue still contains work for it.
+ * Durable policy messages can outlive a clean land; they are delivery records, not proof that a
+ * workspace still needs stewardship. */
+export function stewardQueueEmptyReason(
+  policy: Pick<SpawnPolicy, "tags" | "name" | "orchestratorId">,
+): string | null {
+  if (!isStewardPolicy(policy)) return null;
+  return hasQueuedStewardWorkspace(policy.name, policy.orchestratorId)
+    ? null
+    : "no workspace is awaiting stewardship for this policy";
+}
+
+/** #1789 broaden — even when the queue is NON-empty, a fresh steward process is redundant if EVERY
+ * actionable row is already actively claimed by a live steward (another process is mid-validate/land
+ * on all of them). The `6ec88e16` fix only skipped the empty-queue case; this closes the
+ * all-claimed case, the other way a land eager-spawns a steward that immediately finds nothing to do.
+ * Suppression requires a LIVE claim holder, so an EXITED steward's stale claim (part 2) never blocks
+ * the replacement it needs. */
+export function stewardWorkAllClaimedReason(
+  policy: Pick<SpawnPolicy, "tags" | "name" | "orchestratorId">,
+  now: number,
+): string | null {
+  if (!isStewardPolicy(policy)) return null;
+  return allActionableStewardWorkspacesClaimed(policy.name, policy.orchestratorId, now, (agentId) => {
+    const agent = getAgent(agentId);
+    // #1789 F3 — `tags` are ordinary caller-controlled registration data. A claimant only suppresses
+    // a replacement when it is still live AND it is the relay-bound steward for THIS exact policy.
+    // `isProvenStewardAgent` rests on the signed registration/spawn constraints hardened in #1797.
+    return Boolean(agent && agent.status !== "offline" && isProvenStewardAgent(agentId, policy.name));
+  })
+    ? "every workspace awaiting stewardship for this policy is already actively claimed by a live steward"
+    : null;
+}
+
+/** Is the managed process for `policyName` genuinely live (running-with-a-live-agent, or still
+ *  coming up)? Mirrors LifecycleManager.managedProcessActive so the cap counts the same processes the
+ *  spawn path would treat as already-active. */
+function stewardPolicyIsLive(policyName: string): boolean {
+  const state = getManagedAgentState(policyName);
+  if (!state) return false;
+  switch (state.status) {
+    case "running": {
+      const agent = state.agentId ? getAgent(state.agentId) : null;
+      return Boolean(agent && agent.status !== "offline" && !stewardHandoffReleased(agent));
+    }
+    case "starting":
+      // Coming up counts toward the cap so a burst of wakes can't fan out N processes before any of
+      // them registers.
+      return true;
+    default:
+      return false;
+  }
+}
+
+/** The repo-scoped prefix shared by every steward policy for `repoRoot` across all owning hosts:
+ *  `steward-<slug>-<hash>` (repoStewardPolicyName with no orchestrator). A per-host policy is that
+ *  prefix plus `-<owner>`. The 8-char sha1 hash is repo-unique, so prefix matching never conflates
+ *  two different repos. */
+function repoStewardPrefix(repoRoot: string): string {
+  return repoStewardPolicyName(repoRoot);
+}
+
+function policyBelongsToRepo(policyName: string, repoRoot: string): boolean {
+  const prefix = repoStewardPrefix(repoRoot);
+  return policyName === prefix || policyName.startsWith(`${prefix}-`);
+}
+
+/** Is this a live (non-offline) steward AGENT? Keyed on the same `steward` tag every steward policy
+ *  carries (and buildManagedSpawnParams forwards to the spawned agent), NOT on managed-state — so an
+ *  ORPHANED steward (a live agent no managed policy state still references, left behind when a respawn
+ *  overwrote the state's agentId) is still counted. That orphan is exactly the population the
+ *  policy-state-based per-repo cap cannot see, and the accumulation #1761 observed (12 idle stewards). */
+function isLiveStewardAgent(agent: Pick<AgentCard, "systemTags" | "status" | "epoch" | "meta">): boolean {
+  return agent.status !== "offline" && hasAgentSystemTag(agent, AGENT_SYSTEM_TAGS.mergeSteward) && !stewardHandoffReleased(agent);
+}
+
+/** #1761 — the fleet-wide aggregate: total LIVE steward agents across every repo and host. Counts
+ *  agents (tag-keyed), so it bounds orphans the per-repo policy cap misses. */
+export function liveStewardAgentCount(): number {
+  let live = 0;
+  for (const agent of listAgents()) if (isLiveStewardAgent(agent)) live++;
+  return live;
+}
+
+/** Managed bindings identify the repo and execution host even when registration fields drift. */
+function stewardAgentPlacementByState(): Map<string, { repoRoot?: string; host?: string }> {
+  const map = new Map<string, { repoRoot?: string; host?: string }>();
+  for (const entry of listSpawnPolicies()) {
+    if (!isStewardPolicy(entry.value)) continue;
+    const state = getManagedAgentState(entry.value.name);
+    if (!state?.agentId) continue;
+    const repoRoot = repoRootForStewardPolicy(entry.value.name);
+    const orchestratorId = state.orchestratorId ?? entry.value.orchestratorId;
+    const host = orchestratorId ? getOrchestrator(orchestratorId)?.hostname : undefined;
+    map.set(state.agentId, { repoRoot, host });
+  }
+  return map;
+}
+
+/** #1761 — the operator's "one steward per (repo, host)" bound, counting live steward AGENTS
+ *  attributable to `(repoRoot, host)`. Counting agents (not managed policy state) is the crux: a
+ *  WEDGED steward stuck `stopping` (alive, but state-based `stewardPolicyIsLive` reports false) or an
+ *  ORPHAN is invisible to the policy-state cap yet must still block a duplicate spawn on the same host.
+ *
+ *  Host match uses `resolveCanonicalHost` (NOT `===`): `host` is the orchestrator's registered
+ *  hostname (e.g. "wbox-1") while `agent.machine` is the runner's raw `os.hostname()` (e.g. "WBOX") —
+ *  the repo documents (host-identity.ts, utils.ts) these routinely differ and must never be compared
+ *  raw, or the cap silently fails OPEN on the real fleet. Repo attribution is by the stamped label OR
+ *  the managed-state placement map (fallback for a drifted label). Managed host bindings take
+ *  precedence over machine (#2673). When neither host source can be resolved,
+ *  the host filter is skipped (counts it → fails CLOSED, the safe direction). */
+export function liveStewardAgentCountForRepoHost(repoRoot: string, host: string | undefined): number {
+  const label = stewardAgentLabelForRepo(repoRoot);
+  const placementByState = stewardAgentPlacementByState();
+  let live = 0;
+  for (const agent of listAgents()) {
+    if (!isLiveStewardAgent(agent)) continue;
+    // Repo attribution: stamped label, or the managed-state fallback for a drifted label.
+    const placement = placementByState.get(agent.id);
+    if (agent.label !== label && placement?.repoRoot !== repoRoot) continue;
+    // Host scope: skip ONLY when we can positively prove a DIFFERENT host (canonicalized both sides).
+    const canonical = placement?.host ?? resolveCanonicalHost(agent.machine);
+    if (host && canonical && !sameHostIdentity(canonical, host)) continue;
+    live++;
+  }
+  return live;
+}
+
+/** #1953 RC2 — count steward policies for `(repoRoot, host)` currently `starting` (spawn emitted,
+ *  agent not yet registered), optionally excluding one policy. These are INVISIBLE to the
+ *  agent-counting per-host cap (`liveStewardAgentCountForRepoHost` keys on registered agents), which
+ *  is the TOCTOU window a burst of wakes drove N processes through before any registered. Counting
+ *  them here closes it. Host attribution is the policy's orchestrator hostname, canonicalized both
+ *  sides; an unresolvable host is COUNTED (fails closed, the safe direction for a containment cap).
+ *
+ *  #1955 (#1953-review nit 3) — `starting` counts only while FRESH, mirroring the lifecycle manager's
+ *  `managedProcessActive`. A spawn that never registered leaves the policy pinned `starting` until
+ *  reconcile fails it; counting that wedged state as a live process transiently denied a genuine-
+ *  conflict spawn against a cap of 1. Past MANAGED_START_TIMEOUT_MS (the manager's own threshold, so
+ *  the two verdicts cannot drift) it is not counted. A `starting` with NO lastSpawnAt is counted —
+ *  it has no age to judge, and over-counting is the safe direction. */
+export function startingStewardPolicyCountForRepoHost(repoRoot: string, host: string | undefined, now: number, excludePolicy?: string): number {
+  let starting = 0;
+  for (const entry of listSpawnPolicies()) {
+    const policy = entry.value;
+    if (!isStewardPolicy(policy)) continue;
+    if (policy.name === excludePolicy) continue;
+    if (!policyBelongsToRepo(policy.name, repoRoot)) continue;
+    const state = getManagedAgentState(policy.name);
+    if (state?.status !== "starting") continue;
+    if (state.lastSpawnAt && now - state.lastSpawnAt > MANAGED_START_TIMEOUT_MS) continue;
+    if (host) {
+      const canonical = policy.orchestratorId ? resolveCanonicalHost(getOrchestrator(policy.orchestratorId)?.hostname) : undefined;
+      if (canonical && !sameHostIdentity(canonical, host)) continue;
+    }
+    starting++;
+  }
+  return starting;
+}
+
+/** Count steward policies with a genuinely-live process for `repoRoot`, optionally excluding one
+ *  policy (the one about to spawn — it is not yet live and must not count against itself). */
+export function liveStewardCountForRepo(repoRoot: string, excludePolicy?: string): number {
+  let live = 0;
+  for (const entry of listSpawnPolicies()) {
+    const policy = entry.value;
+    if (!isStewardPolicy(policy)) continue;
+    if (policy.name === excludePolicy) continue;
+    if (!policyBelongsToRepo(policy.name, repoRoot)) continue;
+    if (stewardPolicyIsLive(policy.name)) live++;
+  }
+  return live;
+}
+
+// #1745 respawn-rate backstop. In-memory ring of recent ACTUAL spawn timestamps per policy. In-memory
+// is sufficient: the lifecycle manager is a process singleton, and a relay restart legitimately
+// resets the window. A repeatedly-dying steward (registers, then exits on quota) is not a spawn
+// failure, so the policy backoff never fires — this is the only thing that bounds its churn.
+const recentSpawns = new Map<string, number[]>();
+
+/** Record that a steward policy actually spawned a process, for the rate backstop. */
+export function noteStewardSpawn(policyName: string, now: number): void {
+  const cutoff = now - STEWARD_SPAWN_RATE_WINDOW_MS;
+  const kept = (recentSpawns.get(policyName) ?? []).filter((t) => t > cutoff);
+  kept.push(now);
+  recentSpawns.set(policyName, kept);
+}
+
+/** True when `policyName` has already spawned STEWARD_SPAWN_RATE_MAX times inside the window. */
+export function stewardSpawnRateExceeded(policyName: string, now: number): boolean {
+  const cutoff = now - STEWARD_SPAWN_RATE_WINDOW_MS;
+  const recent = (recentSpawns.get(policyName) ?? []).filter((t) => t > cutoff);
+  return recent.length >= STEWARD_SPAWN_RATE_MAX;
+}
+
+/** Test-only: forget all recorded spawn timestamps. */
+export function resetStewardSpawnRate(): void {
+  recentSpawns.clear();
+}
+
+/**
+ * The gate applied at EVERY steward spawn point (explicit wake and reconcile respawn alike). Returns
+ * a human-readable reason the spawn must be DENIED, or null to allow. Non-steward policies always
+ * return null. Ordered cheapest-first:
+ *   1. kill-switch (steward globally disabled) — the operator's authoritative off;
+ *   2. resolvable repo ownership (#1725) — fail closed when the policy has no workspace;
+ *   3. non-empty server-side steward queue (#1789) — stale messages never justify a process;
+ *   3b. all-actionable-rows-already-claimed (#1789 broaden) — a live steward is handling every row;
+ *   4. fleet-wide aggregate live-steward ceiling (#1761) — bounds the total population incl. orphans;
+ *   5. per-(repo,host) live-steward AGENT ceiling (#1761) — the operator's one-per-repo/host bound,
+ *      counting agents so a wedged/orphan steward still blocks a same-host duplicate spawn;
+ *   6. per-repo live-steward ceiling across hosts (policy-state dedup/cap).
+ * The respawn-RATE backstop is intentionally NOT here: it must bound only the unbounded reconcile-
+ * tick respawn of a repeatedly-dying steward, and must NOT throttle an explicit watchdog/auto-merge
+ * re-wake (those are already bounded by STEWARD_HANDOFF_REWAKE_MAX and the wake cooldown). Callers on
+ * the reconcile path additionally consult stewardSpawnRateExceeded.
+ */
+export function stewardSpawnDenyReason(policy: Pick<SpawnPolicy, "tags" | "name" | "orchestratorId">, now: number, opts: { ignoreFailureBackoff?: boolean } = {}): string | null {
+  if (!isStewardPolicy(policy)) return null;
+  if (!getStewardConfig().enabled) return "steward spawning is disabled by the operator";
+  const state = getManagedAgentState(policy.name);
+  if (!opts.ignoreFailureBackoff && (state?.status === "backoff" || state?.status === "failed") && (state.backoffUntil ?? 0) > now) return `steward spawn backing off until ${new Date(state.backoffUntil!).toISOString()}: ${state.lastError ?? "spawn failed"}`;
+  const repoRoot = repoRootForStewardPolicy(policy.name);
+  // #1725 fix-forward — an unresolvable steward policy has no live workspace to serve, so deny
+  // before the queue/cap checks while preserving the actionable containment diagnosis.
+  if (!repoRoot) return "steward repoRoot unresolvable; refusing to spawn without an enforceable per-repo cap";
+  const emptyQueue = stewardQueueEmptyReason(policy);
+  if (emptyQueue) return emptyQueue;
+  // #1789 broaden — the queue is non-empty, but every actionable row is already actively claimed by a
+  // live steward, so a new process would immediately find nothing unclaimed to do. Suppress it.
+  const allClaimed = stewardWorkAllClaimedReason(policy, now);
+  if (allClaimed) return allClaimed;
+  // #1761 aggregate ceiling. The per-repo cap below bounds one repo, but a burst that stranded lands
+  // across many repos (each under the per-repo cap) piled up fleet-wide. This counts live steward
+  // AGENTS, so an orphan the per-repo policy cap cannot see still counts against the total.
+  const fleet = liveStewardAgentCount();
+  if (fleet >= MAX_LIVE_STEWARDS_FLEET) {
+    return `fleet already has ${fleet} live steward(s) (cap ${MAX_LIVE_STEWARDS_FLEET}); not spawning another`;
+  }
+  // #1761 — the operator's one-per-(repo,host) bound, counting AGENTS. This is what actually stops the
+  // same-host pile-up: a wedged steward stuck `stopping` (alive, but reported not-live by the policy-
+  // state cap) or an orphan still counts here, so onMessageForPolicy/spawnAgent deny a duplicate while
+  // it lives. Only enforceable when the repoRoot resolves; the unresolvable case fails closed below.
+  if (repoRoot) {
+    const host = policy.orchestratorId ? getOrchestrator(policy.orchestratorId)?.hostname : undefined;
+    // #1953 RC2 — count live steward AGENTS *plus* policies still `starting` (spawn emitted, not yet
+    // registered) for this (repo, host). Counting only registered agents left a TOCTOU window a burst
+    // of cross-policy-name wakes fanned N processes through before the first registered.
+    const liveHere = liveStewardAgentCountForRepoHost(repoRoot, host)
+      + startingStewardPolicyCountForRepoHost(repoRoot, host, now, policy.name);
+    if (liveHere >= MAX_LIVE_STEWARDS_PER_REPO_HOST) {
+      return `repo/host already has ${liveHere} live/starting steward(s) (cap ${MAX_LIVE_STEWARDS_PER_REPO_HOST}); not spawning another`;
+    }
+  }
+  const live = liveStewardCountForRepo(repoRoot, policy.name);
+  if (live >= MAX_LIVE_STEWARDS_PER_REPO) {
+    return `repo already has ${live} live steward(s) (cap ${MAX_LIVE_STEWARDS_PER_REPO}); not spawning another`;
+  }
+  return null;
+}

package/src/steward.test-fixture.ts

@@ -0,0 +1,11 @@
+import { createDeliverableTask, getWorkspace, recordTaskDeliverable } from "./db";
+
+export function authorizeStewardWorkspace(id: string): number {
+  const workspace = getWorkspace(id)!;
+  const task = createDeliverableTask({ title: id, intent: "land", createdBy: "coord", branch: workspace.branch, doneWhen: "merged", dispatch: { type: "build" } });
+  recordTaskDeliverable(task.id, {
+    disposition: "complete", summary: "ready", actor: "worker", workspaceId: id,
+    land: { requested: true, triggered: false, reason: "pending", workspaceId: id, workspaceStatus: workspace.status },
+  });
+  return task.id;
+}

package/src/steward.ts

@@ -0,0 +1,199 @@
+import { basename } from "node:path";
+import { stewardFallbackTarget } from "./config";
+import { deleteConfig, getManagedAgentState, getSpawnPolicy, getStewardConfig, setConfig } from "./config-store";
+import { listOrchestrators } from "./db";
+import { getLifecycleManager } from "./lifecycle-manager";
+import { renderTemplate } from "./prompt-resolver";
+import { resolveInternalSpawnTarget } from "./auto-spawn-dispatch";
+import type { DispatchPin } from "./task-dispatcher";
+import { findProviderCatalogModel } from "@humwick/sdk/provider-catalog";
+import { ValidationError } from "./db/connection";
+import { effectiveProviderCatalogList } from "./provider-catalog-store";
+import type { Orchestrator, SpawnPolicy, StewardConfig } from "./types";
+import { isPathWithinBase } from "./utils";
+export { repoStewardPolicyName } from "./steward-identity";
+import { canonicalRepoRoot, legacyStewardPolicyNames, repoStewardPolicyName } from "./steward-identity";
+
+/**
+ * The steward's system prompt — provider-agnostic (works for any provider/model).
+ * It tells the agent the workflow via the `humwick workspace`/`steward` CLI
+ * toolkit (#208): see the queue, CLAIM a workspace so auto-merge yields, inspect
+ * its diagnostics, rebase/resolve/check, land green ones, release on escalation.
+ */
+export function buildStewardPrompt(repoRoot: string): string {
+  // #642 — escalation routes to the branch agent's COORDINATOR (named per-workspace in the
+  // wake/task message via #640), not a static `system` dead-end. The fallback target below is
+  // used ONLY when a wake message names no coordinator.
+  const fallback = stewardFallbackTarget();
+  return renderTemplate("steward.prompt", {
+    repoRoot,
+    escalationTarget: fallback ? `\`${fallback}\`` : "the human operator who configured this relay",
+  });
+}
+
+export function desiredStewardPolicy(rawRepoRoot: string, owner: Orchestrator, config: StewardConfig): SpawnPolicy {
+  // #1953 RC2 — one physical repo = one policy = one cap bucket = one cwd. Canonicalize once here so
+  // the policy NAME, the launch/identity `cwd`, the prompt, and the description all agree on a single
+  // form. This is what stops "1-2 correct-cwd + 2 wrong-cwd (base-dir) stewards for one repo".
+  const repoRoot = canonicalRepoRoot(rawRepoRoot);
+  // #1623 — LAUNCH at the project REPO ROOT, not the orchestrator baseDir. `cwd` is the
+  // project-resolution KEY (#1565): the project memory cell, `projectInstructions`, the
+  // memory identity, and cwd-defaulting MCP tools (`tl_*`) all resolve from the spawn cwd
+  // (composeProfileSpawnInstructions → resolveProjectForCanonicalCwd). A steward rooted at
+  // baseDir resolved the WRONG project cell (or none), so the one agent making irreversible
+  // landing decisions got no project memory, instructions, or repo rules. DECISION
+  // (operator-approved, #1623): stewards SHOULD receive project memory/instructions — bind
+  // their identity to the project repo root, which carries that identity without breaking landing.
+  //
+  // Landing is UNAFFECTED: the steward operates on explicit managed-worktree paths (git ops take
+  // them as arguments, not from process cwd), and its runtime-token AUTHORITY is preserved
+  // separately. Isolated worktrees live at `<baseDir>/.agent-relay/workspaces/...`, OUTSIDE the
+  // repo root, and the `workspace.merge` command the steward dispatches to land carries a
+  // `worktreePath` there that is gated against the token's cwdPrefixes (dispatch-command.ts). So the
+  // steward token STILL includes the orchestrator baseDir as an extra cwdPrefix — see
+  // stewardTokenExtraCwdPrefixes (steward-spawn-guard.ts), applied at the mint in managed-policy.ts
+  // and restart-spawn.ts. baseDir ⊇ repoRoot, so path authority is byte-identical to before; only
+  // the launch/identity cwd moved to the repo. The prompt scopes it to the specific repo.
+
+  // #901 — resolve via the auto-spawn dispatcher so routing policy + quota bands pick
+  // the model. StewardConfig.provider/model/effort are OPTIONAL EXPLICIT PINS: when set
+  // by the operator they fold into the DispatchPin and win over the router's choice.
+  // #1761 (4b) — the dispatcher applies its enabled-model self-heal (ensureEnabledModel) ONLY to an
+  // UNPINNED model, so a pinned model that the operator DISABLED in the catalog would be re-applied
+  // verbatim every sweep (the "I disabled gpt-5.6-terra and stewards kept spawning on it" report). If
+  // the pinned model is not usable in the effective catalog, drop the pin so the router resolves an
+  // enabled model instead of re-pinning the disabled one.
+  const modelPin = config.model && stewardModelUsable(config.provider, config.model) ? config.model : undefined;
+  const pin: DispatchPin = {
+    provider: config.provider,
+    ...(modelPin ? { model: modelPin } : {}),
+    ...(config.effort ? { effort: config.effort } : {}),
+  };
+  // This synchronous policy-refresh path must never fall through to the synchronous quota reader;
+  // live quota-aware resolution runs through resolveInternalSpawnTargetAsync at async entrypoints.
+  const resolved = resolveInternalSpawnTarget("landing-steward", pin, { bands: [] });
+
+  return {
+    name: repoStewardPolicyName(repoRoot, owner.id),
+    description: `Autonomous repo steward for ${repoRoot} (issue #167).`,
+    enabled: true,
+    orchestratorId: owner.id,
+    cwd: repoRoot,
+    provider: resolved.target.provider,
+    workspaceMode: "inherit",
+    ...(resolved.target.model ? { model: resolved.target.model } : {}),
+    ...(resolved.target.effort ? { effort: resolved.target.effort } : {}),
+    providerArgs: [],
+    prompt: buildStewardPrompt(repoRoot),
+    tags: ["steward"],
+    capabilities: ["merge", "review"],
+    label: `steward:${basename(repoRoot)}`,
+    mode: "on-demand",
+    permissionMode: config.permissionMode,
+    restartOnUpdate: false,
+    scheduledDailyRestart: false,
+    onDemand: { keepaliveSeconds: config.keepaliveSeconds, idleDefinition: "no-activity" },
+    backoff: { schedule: [30, 60, 120, 240, 480, 960, 1920, 3600], resetAfterSeconds: 300 },
+  };
+}
+
+// Explicit disabled pins may self-heal; missing or ambiguous ids must fail loudly.
+function stewardModelUsable(provider: StewardConfig["provider"], model: string): boolean {
+  const entry = effectiveProviderCatalogList().find((c) => c.provider === provider);
+  const m = entry && findProviderCatalogModel(entry, model, { requireUnambiguous: true });
+  if (!m) throw new ValidationError(`unknown ${provider} steward model: ${model}`);
+  return !m.disabled && !m.unavailable;
+}
+
+// Provider pins and catalog defaults resolve deterministically, so model changes must propagate.
+function stewardPolicyDiffers(existing: SpawnPolicy, desired: SpawnPolicy): boolean {
+  if (
+    existing.enabled !== desired.enabled ||
+    existing.provider !== desired.provider ||
+    existing.effort !== desired.effort ||
+    existing.permissionMode !== desired.permissionMode ||
+    existing.cwd !== desired.cwd ||
+    existing.orchestratorId !== desired.orchestratorId ||
+    existing.prompt !== desired.prompt ||
+    existing.onDemand?.keepaliveSeconds !== desired.onDemand?.keepaliveSeconds
+  ) return true;
+
+  if (existing.model !== desired.model) return true;
+
+  return false;
+}
+
+/**
+ * #1955 mechanism 1 — collapse any LEGACY-named policy for this exact (repo, host) into the canonical
+ * name, so provisioning MERGES the pair instead of adding a second policy beside it.
+ *
+ * Returns the `enabled` value the canonical policy must inherit, or undefined when nothing is
+ * inherited. An operator's stop (`enabled:false`) recorded only on the legacy row would otherwise be
+ * silently lifted by the collapse — the one piece of operator intent the legacy row still holds.
+ *
+ * Deliberately conservative — a legacy row is left in place (and collapsed on a later provision) when:
+ *   - it is not a steward policy (a same-named unrelated policy is never ours to delete);
+ *   - it names a DIFFERENT orchestrator (another host's steward for this repo, not this (repo,host));
+ *   - it still has a live/transitioning managed process. Deleting the policy under a running process
+ *     would orphan it: `onConfigChanged` finds no policy for a deleted key, so nothing would stop it.
+ */
+function collapseLegacyStewardPolicies(canonicalName: string, repoRoot: string, owner: Orchestrator): boolean | undefined {
+  let inheritedEnabled: boolean | undefined;
+  for (const legacyName of legacyStewardPolicyNames(repoRoot, owner.id)) {
+    if (legacyName === canonicalName) continue;
+    const legacy = getSpawnPolicy(legacyName)?.value;
+    if (!legacy) continue;
+    if (!legacy.tags?.includes("steward")) continue;
+    if (legacy.orchestratorId && legacy.orchestratorId !== owner.id) continue;
+    const state = getManagedAgentState(legacyName)?.status;
+    if (state === "running" || state === "starting" || state === "stopping") {
+      console.warn(`[steward] legacy policy '${legacyName}' still has a ${state} process; deferring its collapse into '${canonicalName}'`);
+      continue;
+    }
+    if (legacy.enabled === false) inheritedEnabled = false;
+    deleteConfig("spawn-policy", legacyName, "steward-legacy-collapse");
+    getLifecycleManager().onConfigChanged("spawn-policy", legacyName);
+    console.warn(`[steward] collapsed legacy steward policy '${legacyName}' into '${canonicalName}' (#1955)`);
+  }
+  return inheritedEnabled;
+}
+
+/**
+ * Ensure an on-demand steward spawn policy exists for `repoRoot`, built from the
+ * global steward config (issue #167). Returns the policy name (durable `policy:`
+ * target the caller wakes), or null when stewards are disabled or no orchestrator
+ * owns the repo. Idempotent — refreshes the policy only when config-derived fields
+ * change, so it can be called on every sweep without churn.
+ */
+export function ensureRepoSteward(repoRoot: string, owningOrchestrator?: Orchestrator): string | null {
+  const config = getStewardConfig();
+  if (!config.enabled) return null;
+  const owner = owningOrchestrator ?? listOrchestrators().find((orch) =>
+    orch.status === "online" && isPathWithinBase(repoRoot, orch.baseDir)
+  );
+  if (owner && (owner.status !== "online" || !isPathWithinBase(repoRoot, owner.baseDir))) return null;
+  if (!owner) return null;
+
+  const name = repoStewardPolicyName(repoRoot, owner.id);
+  // #1955 mechanism 1 — merge a legacy suffix-less twin for this (repo, host) BEFORE reading the
+  // canonical row, so the pair converges on one policy instead of coexisting as two.
+  const inheritedEnabled = collapseLegacyStewardPolicies(name, repoRoot, owner);
+  const baseDesired = desiredStewardPolicy(repoRoot, owner, config);
+  const existing = getSpawnPolicy(name);
+  // #1761 (9) — respect an operator's per-policy stop. desiredStewardPolicy hardcodes `enabled:true`;
+  // rewriting that back over an existing policy the operator manually stopped (enabled:false) reverted
+  // the UI stop every sweep (#1745 cause 2, never fixed at source). A NEW policy is created enabled; an
+  // EXISTING one keeps the operator's own `enabled` value. The global StewardConfig.enabled kill-switch
+  // (checked above, and folded into the lifecycle reconcile by #1745) remains the authoritative on/off.
+  // #1955 — a collapsed legacy row's operator STOP carries onto the canonical policy, and wins: a
+  // merge must never LIFT a stop the operator applied to either half of the pair (the safe direction
+  // — re-enabling is one dashboard click, an unwanted steward is a live process).
+  const desired = existing
+    ? { ...baseDesired, enabled: inheritedEnabled === false ? false : existing.value.enabled }
+    : { ...baseDesired, ...(inheritedEnabled === undefined ? {} : { enabled: inheritedEnabled }) };
+  if (existing && !stewardPolicyDiffers(existing.value, desired)) return name;
+
+  setConfig("spawn-policy", name, desired, "steward-autoprovision");
+  getLifecycleManager().onConfigChanged("spawn-policy", name);
+  return name;
+}