agent-relay-server 0.85.0 → 0.87.0

package/src/services/transient-agent-reaper.ts

@@ -1,6 +1,6 @@
 import { createCommand } from "../commands-db";
 import { emitRelayEvent } from "../events";
-import { createActivityEvent, getAgent, listAgents, listWorkspaces } from "../db";
+import { createActivityEvent, getAgent, listAgents, listWorkspaces, recordTerminalEvent } from "../db";
 import { routeNotification } from "../notification-router";
 import { isProviderBlocked } from "../provider-state";
 import { TERMINAL_WORKSPACE_STATUSES } from "../workspace-phase";
@@ -135,6 +135,9 @@ export function reapTransientAgents(): Record<string, unknown> {
     });
     emitRelayEvent({ type: "command.requested", source: command.source, subject: command.id, data: { command } });
     emitRelayEvent({ type: "agent.exited", source: "server", subject: agent.id, data: { agentId: agent.id, parent: agent.spawnedBy, reason: "transient-complete", workspaceId: land.workspaceId, workspaceStatus: land.status } });
+    // #645 — persist a clean terminal marker so the #634 heartbeat-gap watchdog reads this benign
+    // completion as a terminus (not a silent death) once the reaped worker stops heartbeating.
+    recordTerminalEvent({ agentId: agent.id, reason: "transient-complete", kind: "agent.exited", createdAt: now });
     createActivityEvent({
       clientId: `transient-agent-reaped-${agent.id}-${now}`,
       kind: "state",

package/src/spawn-targets.ts

@@ -48,7 +48,7 @@ export function buildSpawnTargets(input: { callerId?: string; quota: number }):
 interface SpawnCatalogish {
   provider: SpawnProvider;
   defaultModel?: string;
-  models: Array<{ alias: string; providerModel: string; efforts: readonly string[]; defaultEffort?: string; disabled?: boolean }>;
+  models: Array<{ alias: string; providerModel: string; efforts: readonly string[]; defaultEffort?: string; disabled?: boolean; unavailable?: string }>;
 }
 
 function providerMatrixFor(o: Orchestrator): Array<Record<string, unknown>> {
@@ -60,12 +60,17 @@ function providerMatrixFor(o: Orchestrator): Array<Record<string, unknown>> {
   // REPORTED catalog does not, so cross-reference the effective catalog by alias regardless of
   // which source supplies the matrix entry — otherwise the reported path keeps advertising a model
   // the server then rejects with "model is disabled".
+  // #368 — also drop models marked unavailable in the static catalog (provider-side suspension,
+  // e.g. export-control; semantically distinct from the operator-toggle disabled flag).
   const disabledAliasesFor = (provider: SpawnProvider): Set<string> =>
     new Set(global.find((e) => e.provider === provider)?.models.filter((m) => m.disabled).map((m) => m.alias) ?? []);
+  const unavailableAliasesFor = (provider: SpawnProvider): Set<string> =>
+    new Set(global.find((e) => e.provider === provider)?.models.filter((m) => m.unavailable).map((m) => m.alias) ?? []);
   return o.providers.map((provider) => {
     const cat: SpawnCatalogish | undefined =
       reported.find((s) => s.provider === provider) ?? global.find((e) => e.provider === provider);
     const disabled = disabledAliasesFor(provider);
+    const unavailable = unavailableAliasesFor(provider);
     return {
       provider,
       available: true,
@@ -73,7 +78,7 @@ function providerMatrixFor(o: Orchestrator): Array<Record<string, unknown>> {
         ? {
             defaultModel: cat.defaultModel,
             models: cat.models
-              .filter((m) => !m.disabled && !disabled.has(m.alias))
+              .filter((m) => !m.disabled && !disabled.has(m.alias) && !m.unavailable && !unavailable.has(m.alias))
               .map((m) => ({
                 id: m.alias,
                 providerModel: m.providerModel,
@@ -122,6 +127,7 @@ export function spawnCapablePrimer(input: { canSpawn: boolean; quota?: number })
     "- Default to your OWN host. Only set orchestratorId to spawn onto another host when it runs a provider/model yours doesn't, or the work must run there.",
     `- Prefer a named profile (profile: "<name>") over hand-assembling provider+model+effort+approvalMode. Omitting profile applies ${DEFAULT_SPAWN_PROFILE_NAME}; omitting model/effort uses the provider default. Lean to the cheapest provider/model/effort that fits; escalate only for complex work.`,
     "- Set spawnRequestId for idempotency so a retried spawn doesn't double-create.",
+    "- After spawning, use relay_my_agents for your self-scoped child roster + quota and copy a returned spawnRequestId into relay_shutdown_agent when you need to reap.",
     "- Spawn is fire-and-forget: pass waitForRegistrationMs:0 and keep working — you'll be PUSHED agent.ready when the child is genuinely ready (past onboarding), or agent.spawn_failed / agent.exited otherwise. Don't block-poll.",
     "- Spawned children cannot themselves spawn (no grandchildren).",
   ].join("\n");

package/src/steward-identity.ts

@@ -0,0 +1,50 @@
+// The steward ↔ branch-agent ↔ coordinator identity triangle (#639/#640).
+//
+// When the relay hands a workspace to a steward, the steward used to know only the
+// `repoRoot`. To make a handoff legible end-to-end it needs the people-graph:
+//   - the workspace's BRANCH AGENT = `ws.ownerAgentId` (already stored on the row), and
+//   - that agent's COORDINATOR = the owner's authoritative `spawnedBy` parent (the agent
+//     that spawned the branch worker), resolved to a LIVE instance via the shared
+//     agent-ref matchers so a coordinator that restarted under a new session id still
+//     resolves to its live twin.
+//
+// One home for this resolution so the wake path, the diagnostics/inspect surface, and the
+// escalation path can never drift on "who is the coordinator for this workspace?".
+
+import { getAgent, listAgents } from "./db";
+import { resolveAgentRef } from "./agent-ref";
+import type { WorkspaceRecord } from "./types";
+
+export interface StewardIdentityTriangle {
+  /** The workspace's branch agent (owner), or undefined when the row has no owner. */
+  ownerAgentId?: string;
+  /** The branch agent's coordinator (owner's `spawnedBy` lineage) resolved to a live id, or null when none resolves. */
+  coordinatorAgentId: string | null;
+}
+
+/**
+ * Resolve the branch agent's coordinator: the owner's authoritative `spawnedBy` parent,
+ * preferring a live instance. Returns the recorded parent id when no live match exists
+ * (store-ahead still delivers to a momentarily-offline coordinator), or null when the
+ * owner is unknown or wasn't spawned by anyone (e.g. a human-launched agent).
+ */
+export function resolveCoordinatorAgentId(ownerAgentId: string | undefined): string | null {
+  if (!ownerAgentId) return null;
+  const spawnedBy = getAgent(ownerAgentId)?.spawnedBy?.trim();
+  if (!spawnedBy) return null;
+  const resolved = resolveAgentRef(spawnedBy, listAgents());
+  if (resolved.status === "resolved") return resolved.agent.id;
+  if (resolved.status === "ambiguous") {
+    // Prefer the exact recorded id among live twins; else the most-recently-seen.
+    return (resolved.candidates.find((a) => a.id === spawnedBy) ?? resolved.candidates[0])!.id;
+  }
+  // No live match — keep the authoritative recorded parent id so escalation/handoff
+  // notices still address the coordinator (store-ahead delivers when it reconnects).
+  return spawnedBy;
+}
+
+/** Resolve the full identity triangle for a workspace handoff (#640). */
+export function resolveStewardIdentity(ws: Pick<WorkspaceRecord, "ownerAgentId">): StewardIdentityTriangle {
+  const ownerAgentId = ws.ownerAgentId ?? undefined;
+  return { ownerAgentId, coordinatorAgentId: resolveCoordinatorAgentId(ownerAgentId) };
+}

package/src/steward.ts

@@ -1,5 +1,6 @@
 import { createHash } from "node:crypto";
 import { basename, resolve } from "node:path";
+import { stewardFallbackTarget } from "./config";
 import { getSpawnPolicy, getStewardConfig, setConfig } from "./config-store";
 import { listOrchestrators } from "./db";
 import { getLifecycleManager } from "./lifecycle-manager";
@@ -20,23 +21,30 @@ export function repoStewardPolicyName(repoRoot: string): string {
  * its diagnostics, rebase/resolve/check, land green ones, release on escalation.
  */
 export function buildStewardPrompt(repoRoot: string): string {
+  // #642 — escalation routes to the branch agent's COORDINATOR (named per-workspace in the
+  // wake/task message via #640), not a static `system` dead-end. The fallback target below is
+  // used ONLY when a wake message names no coordinator.
+  const fallback = stewardFallbackTarget();
+  const fallbackTarget = fallback ? `\`${fallback}\`` : "the human operator who configured this relay";
   return [
     `You are the autonomous repository steward for ${repoRoot}.`,
     "",
     "Multiple agents work this repo in isolated git worktrees. The relay auto-merges any branch that rebases cleanly — including ones whose base moved on; it hands YOU only what it can't land deterministically: real merge conflicts, or an unknown/ambiguous merge state. Your job is to land that work with no human in the loop whenever you safely can.",
     "",
+    "Each workspace belongs to a BRANCH AGENT (the owner who did the work) who in turn has a COORDINATOR (the agent that spawned it and is responsible for the merge landing). Your wake/task message names both — the `ownerAgentId` and the `coordinatorAgentId` — and `agent-relay steward inspect <id>` shows them. When you can't land work yourself, the coordinator is who you escalate to.",
+    "",
     "Use the `agent-relay` CLI for EVERY relay interaction — it wraps the relay API and is the supported toolkit, and it already attaches your session token with the right scopes. Do NOT hand-curl the HTTP API (`/api/workspaces`, …): the CLI exists so you never have to. If you ever truly must call the API directly, authenticate with the `X-Agent-Relay-Token: <your session token>` header — NOT `Authorization: Bearer $AGENT_RELAY_TOKEN`; the admin env token is not present in your environment, so Bearer will 401.",
     "Handle ONE workspace at a time — the relay's per-repo merge lease serializes you with any other merger.",
     "",
     "1. See the queue: `agent-relay steward queue` (workspaces in conflict / review_requested / merge_planned for this repo). The wake message(s) also name a `workspaceId`.",
     "2. CLAIM it FIRST: `agent-relay workspace claim --id <id> --purpose steward`. This stops the deterministic auto-merge from racing you while you validate. The claim auto-expires, so a crash never blocks the repo.",
-    "3. Understand it: `agent-relay steward inspect <id>` — a diagnostics briefing (owner liveness, live git state, ahead/behind, recorded-vs-live branch mismatch, and a recommended action). `agent-relay steward checks <id>` suggests check commands from the changed files.",
-    "4. `cd` into the workspace's `worktreePath` and rebase the branch onto its `baseRef` (`git rebase <baseRef>`). Resolve conflicts faithfully — preserve the intent of EVERY side; never silently drop a change. If a conflict needs product judgment you can't make confidently, stop and escalate.",
+    "3. Understand it: `agent-relay steward inspect <id>` — a diagnostics briefing (the branch agent owner + its coordinator, owner liveness, live git state, ahead/behind, recorded-vs-live branch mismatch, and a recommended action). `agent-relay steward checks <id>` suggests check commands from the changed files.",
+    "4. `cd` into the workspace's `worktreePath` and rebase the branch onto its `baseRef` (`git rebase <baseRef>`). Resolve conflicts faithfully — preserve the intent of EVERY side; never silently drop a change. If a conflict needs product judgment you can't make confidently, stop and escalate to the coordinator.",
     "5. Run the repo's checks/tests (the suggested ones, plus the project's documented commands). Fix trivial breakage you caused while rebasing; do not paper over real failures.",
     "6. If green, land it: `agent-relay workspace land --id <id> --strategy rebase-ff`. The relay rebases onto the latest base, fast-forwards, and pushes to origin under the lease. The land consumes your claim.",
-    "7. If you must stop without landing, RELEASE the claim so automation can resume: `agent-relay workspace release --id <id>`, then send a clear, specific summary to the fallback/human target.",
+    "7. If you must stop without landing, RELEASE the claim so automation can resume: `agent-relay workspace release --id <id>`, then send a clear, specific summary to the workspace's COORDINATOR (the `coordinatorAgentId` from the wake message) — they own the work and can act on it. Reply to the wake message or message that agent id directly.",
     "",
-    "Escalate (do NOT merge) when: checks fail and you can't fix them, the conflict resolution is genuinely ambiguous, or anything looks risky. Never force-merge a red branch, never `git push --force` to a shared base, never discard committed work.",
+    `Escalate (do NOT merge) when: checks fail and you can't fix them, the conflict resolution is genuinely ambiguous, or anything looks risky. ALWAYS escalate to the branch agent's coordinator (the \`coordinatorAgentId\` in the wake message). Only if the wake message names NO coordinator, escalate to ${fallbackTarget}. Never force-merge a red branch, never \`git push --force\` to a shared base, never discard committed work.`,
     "",
     "Be terminal-efficient and decisive. When the queue is empty, you're done — you'll be woken again when there's more.",
   ].join("\n");

package/src/team-idle-tracker.ts

@@ -26,3 +26,22 @@ export interface TeamMemberWatch {
 
 // teamId → (memberId → watch)
 export const teamMemberWatchTracker = new Map<string, Map<string, TeamMemberWatch>>();
+
+// #412/#468 — per-team COORDINATOR liveness watch. The member watch above deliberately skips the
+// coordinator (it's the ping recipient, not a watched worker), so a coordinator that dies silently
+// is invisible to it. This tracker remembers the coordinator's last-alive observation so a later
+// silence escalates ONCE to the human/steward fallback instead of dead-ending — a coordinator-gone
+// team must never be silently abandoned.
+export interface TeamCoordinatorWatch {
+  /** Identity of the coordinator this watch is for; a re-elected/replaced coordinator resets it. */
+  coordinatorId: string;
+  /** Most recent time the coordinator was seen alive (non-offline, fresh heartbeat). */
+  lastActiveAt: number;
+  /** Whether the coordinator's CURRENT gone-spell already escalated (re-armed when it recovers). */
+  escalated: boolean;
+  /** Coordinator label for a readable escalation. */
+  label?: string;
+}
+
+// teamId → coordinator watch
+export const teamCoordinatorWatchTracker = new Map<string, TeamCoordinatorWatch>();

package/src/workspace-auto-merge.ts

@@ -25,6 +25,8 @@ import {
   listWorkspaces,
   patchWorkspaceMetadata,
 } from "./db";
+import { notifyBranchLandFailed, notifyStewardHandoff } from "./branch-landed";
+import { resolveStewardIdentity } from "./steward-identity";
 import { emitCommandEvent } from "./command-events";
 import { getLandingConfig, getStewardConfig } from "./config-store";
 import { getLifecycleManager } from "./lifecycle-manager";
@@ -51,20 +53,29 @@ export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): string | n
   if (lastWoke && Date.now() - lastWoke < STEWARD_WAKE_COOLDOWN_MS) return null;
   const policyName = ensureRepoSteward(ws.repoRoot);
   if (!policyName) return null;
+  // #640 — resolve the identity triangle so the steward isn't blind to the people-graph:
+  // the workspace's branch agent (owner) and that agent's coordinator (owner's `spawnedBy`).
+  const { ownerAgentId, coordinatorAgentId } = resolveStewardIdentity(ws);
+  const coordinatorLine = coordinatorAgentId
+    ? ` Escalate to the branch agent's coordinator \`${coordinatorAgentId}\` (not the system fallback) if you can't land it.`
+    : "";
+  const ownerLine = ownerAgentId ? ` Branch agent (owner): \`${ownerAgentId}\`.` : "";
   try {
     // Obligation (ttl null by type default): a steward task must survive until claimed/handled.
     routeNotification({
       type: "workspace.steward_task",
-      scope: { workspaceId: ws.id, repoRoot: ws.repoRoot },
+      scope: { workspaceId: ws.id, repoRoot: ws.repoRoot, author: ownerAgentId, parent: coordinatorAgentId ?? undefined },
       recipients: [`policy:${policyName}`],
       message: {
         subject: `Steward: ${ws.status} workspace needs attention`,
-        body: `Workspace \`${ws.branch ?? ws.id}\` (id ${ws.id}) in ${ws.repoRoot} is ${ws.status} and could not auto-land (${reason}). Claim it first so auto-merge yields: \`agent-relay workspace claim --id ${ws.id} --purpose steward\`. Inspect: \`agent-relay steward inspect ${ws.id}\`. Then cd into ${ws.worktreePath}, rebase onto ${ws.baseRef ?? "base"}, resolve, run checks, and land: \`agent-relay workspace land --id ${ws.id} --strategy rebase-ff\` — or \`agent-relay workspace release --id ${ws.id}\` and escalate if you can't.`,
-        payload: { kind: "workspace.steward-task", workspaceId: ws.id, repoRoot: ws.repoRoot, worktreePath: ws.worktreePath, branch: ws.branch, baseRef: ws.baseRef, status: ws.status, reason },
+        body: `Workspace \`${ws.branch ?? ws.id}\` (id ${ws.id}) in ${ws.repoRoot} is ${ws.status} and could not auto-land (${reason}).${ownerLine} Claim it first so auto-merge yields: \`agent-relay workspace claim --id ${ws.id} --purpose steward\`. Inspect: \`agent-relay steward inspect ${ws.id}\`. Then cd into ${ws.worktreePath}, rebase onto ${ws.baseRef ?? "base"}, resolve, run checks, and land: \`agent-relay workspace land --id ${ws.id} --strategy rebase-ff\` — or \`agent-relay workspace release --id ${ws.id}\` and escalate if you can't.${coordinatorLine}`,
+        payload: { kind: "workspace.steward-task", workspaceId: ws.id, repoRoot: ws.repoRoot, worktreePath: ws.worktreePath, branch: ws.branch, baseRef: ws.baseRef, status: ws.status, reason, ownerAgentId, coordinatorAgentId },
       },
     });
     getLifecycleManager().onMessageForPolicy(policyName);
-    patchWorkspaceMetadata(ws.id, { stewardWokenAt: Date.now(), stewardPolicy: policyName });
+    patchWorkspaceMetadata(ws.id, { stewardWokenAt: Date.now(), stewardPolicy: policyName, coordinatorAgentId: coordinatorAgentId ?? undefined });
+    // #641/#642 — tell the owner which steward took over and proactively loop in its coordinator.
+    notifyStewardHandoff({ workspace: ws, stewardId: policyName, coordinatorAgentId, reason });
     return policyName;
   } catch {
     return null;
@@ -117,12 +128,30 @@ function escalateNoProgressMerge(ws: WorkspaceRecord): void {
   if (getStewardConfig().enabled) {
     wakeRepoSteward(ws, `auto-merge backed off after ${MAX_NO_PROGRESS_MERGE_ATTEMPTS} no-progress attempts (${mergeError})`);
   }
+  // Emit the owner-facing `landed-failure` wake — the SAME actionable notice the live
+  // merge_planned→failed settle fires (routes/commands.ts → notifyBranchLandFailed). The
+  // loop-breaker stops dispatching once a lane hits the no-progress cap, so without this
+  // the dispatch-coupled wake never fires again and a backed-off lane (e.g. a dirty base
+  // checkout, #638/#644) goes silent until a human re-arms it via manual request-review.
+  // The branch worker is parked specifically on `landed-failure`; the `stranded_escalation`
+  // below is a separate fallback/steward signal, not the wake the owner awaits. Cooldown is
+  // already enforced by the early-return above, so this fires at most once per backoff.
+  // (Coordinator routing — owner's `spawnedBy` — is the #642 coordinator-awareness epic.)
+  notifyBranchLandFailed({
+    workspace: ws,
+    reason: `auto-merge backed off after ${MAX_NO_PROGRESS_MERGE_ATTEMPTS} no-progress attempts (${mergeError})`,
+  });
+  // #642 — escalation must reach the branch agent's coordinator (owner's `spawnedBy`), not
+  // just the configured fallback. When steward is enabled, wakeRepoSteward above already
+  // notified the coordinator of the handoff; including it here covers the steward-disabled
+  // path too so a backed-off lane always reaches the agent responsible for the work.
+  const { coordinatorAgentId } = resolveStewardIdentity(ws);
   const fallback = stewardFallbackTarget();
-  const recipients = [...new Set([fallback, ws.ownerAgentId].filter((r): r is string => Boolean(r)))];
+  const recipients = [...new Set([coordinatorAgentId, fallback, ws.ownerAgentId].filter((r): r is string => Boolean(r)))];
   if (recipients.length) {
     routeNotification({
       type: "workspace.stranded_escalation",
-      scope: { workspaceId: ws.id, repoRoot: ws.repoRoot, author: ws.ownerAgentId },
+      scope: { workspaceId: ws.id, repoRoot: ws.repoRoot, author: ws.ownerAgentId, parent: coordinatorAgentId ?? undefined },
       recipients,
       message: {
         subject: "Workspace can't auto-land — needs attention",

package/src/workspace-pr-completion.ts

@@ -155,6 +155,12 @@ function prRefreshForRelayMerge(workspace: WorkspaceRecord, preview: WorkspaceMe
   if (policy === "on-approval" && preview.reviewDecision !== "APPROVED") return { ok: false };
   if (preview.conflict === true) return { ok: false };
   if ((preview.behind ?? 0) <= 0) return { ok: false };
+  // #458: self-heal ONLY a PR that went red *purely because its base advanced* — its checks are
+  // failing AND the branch is behind base. A green (or still-pending) PR is never refreshed: it's
+  // left to the merge path / native auto-merge, so we don't re-trigger CI on a branch that isn't
+  // actually red. A red+up-to-date PR (behind=0, handled above) is red on its own merits, not by
+  // divergence, so it must not be refreshed into a loop either.
+  if (!statusChecksRed(preview.statusCheckRollup)) return { ok: false };
 
   const mergeable = text(preview.mergeable)?.toUpperCase();
   if (mergeable && mergeable !== "MERGEABLE") return { ok: false };
@@ -233,6 +239,26 @@ function statusChecksGreen(rollup: unknown): boolean {
   return rollup.every(statusNodeGreen);
 }
 
+// "Red" = at least one check has a definitive FAILING conclusion/state. Distinct from
+// `!statusChecksGreen` on purpose: a still-pending or no-checks-yet rollup is NOT red, so the
+// #458 base-divergence refresh never interrupts an in-flight run or a PR that hasn't reported.
+const FAILING_CONCLUSIONS = new Set(["FAILURE", "TIMED_OUT", "CANCELLED", "ACTION_REQUIRED", "STARTUP_FAILURE", "STALE"]);
+const FAILING_STATES = new Set(["FAILURE", "ERROR"]);
+
+function statusChecksRed(rollup: unknown): boolean {
+  if (!Array.isArray(rollup) || rollup.length === 0) return false;
+  return rollup.some(statusNodeRed);
+}
+
+function statusNodeRed(node: unknown): boolean {
+  if (!isRecord(node)) return false;
+  const conclusion = text(node.conclusion)?.toUpperCase();
+  if (conclusion) return FAILING_CONCLUSIONS.has(conclusion);
+  const state = text(node.state)?.toUpperCase();
+  if (state) return FAILING_STATES.has(state);
+  return false;
+}
+
 function prMergeable(preview: WorkspaceMergePreview): boolean {
   const mergeable = text(preview.mergeable)?.toUpperCase();
   if (mergeable !== "MERGEABLE") return false;