agent-relay-server 0.83.1 → 0.85.0

package/src/spawn-targets.ts

@@ -48,28 +48,39 @@ export function buildSpawnTargets(input: { callerId?: string; quota: number }):
 interface SpawnCatalogish {
   provider: SpawnProvider;
   defaultModel?: string;
-  models: Array<{ alias: string; providerModel: string; efforts: readonly string[]; defaultEffort?: string }>;
+  models: Array<{ alias: string; providerModel: string; efforts: readonly string[]; defaultEffort?: string; disabled?: boolean }>;
 }
 
 function providerMatrixFor(o: Orchestrator): Array<Record<string, unknown>> {
   const reported: ProviderCatalogSummary[] = o.providerCatalog ?? [];
   const global = effectiveProviderCatalogList();
+  // #643 — drop disabled models so relay_spawn_targets advertises only spawnable models (parity
+  // with the #485 dashboard pickers' `!model.disabled` filter). The disabled flag is a relay-side
+  // override (provider_model_overrides) that the effective catalog carries but the orchestrator-
+  // REPORTED catalog does not, so cross-reference the effective catalog by alias regardless of
+  // which source supplies the matrix entry — otherwise the reported path keeps advertising a model
+  // the server then rejects with "model is disabled".
+  const disabledAliasesFor = (provider: SpawnProvider): Set<string> =>
+    new Set(global.find((e) => e.provider === provider)?.models.filter((m) => m.disabled).map((m) => m.alias) ?? []);
   return o.providers.map((provider) => {
     const cat: SpawnCatalogish | undefined =
       reported.find((s) => s.provider === provider) ?? global.find((e) => e.provider === provider);
+    const disabled = disabledAliasesFor(provider);
     return {
       provider,
       available: true,
       ...(cat
         ? {
             defaultModel: cat.defaultModel,
-            models: cat.models.map((m) => ({
-              id: m.alias,
-              providerModel: m.providerModel,
-              default: cat.defaultModel === m.alias,
-              effortLevels: m.efforts,
-              ...(m.defaultEffort ? { defaultEffort: m.defaultEffort } : {}),
-            })),
+            models: cat.models
+              .filter((m) => !m.disabled && !disabled.has(m.alias))
+              .map((m) => ({
+                id: m.alias,
+                providerModel: m.providerModel,
+                default: cat.defaultModel === m.alias,
+                effortLevels: m.efforts,
+                ...(m.defaultEffort ? { defaultEffort: m.defaultEffort } : {}),
+              })),
           }
         : { models: [] }),
     };

package/src/team-idle-tracker.ts

@@ -11,3 +11,18 @@ export const teamIdleTracker = new Map<string, TeamIdleEntry>();
 export function getTeamIdleEntry(teamId: string): TeamIdleEntry | undefined {
   return teamIdleTracker.get(teamId);
 }
+
+// #634 — per-member heartbeat-gap watch. Kept in-memory (not derived from the live roster) ON
+// PURPOSE: a member that was ready/busy and then vanished from the roster entirely escapes any
+// roster-only enumeration, so the watch must remember it independently to detect the silence gap.
+export interface TeamMemberWatch {
+  /** Most recent time the member was seen alive (non-offline heartbeat). */
+  lastActiveAt: number;
+  /** Whether the member's CURRENT silence has already pinged the coordinator (re-armed on recovery). */
+  pinged: boolean;
+  /** Member label for a readable ping. */
+  label?: string;
+}
+
+// teamId → (memberId → watch)
+export const teamMemberWatchTracker = new Map<string, Map<string, TeamMemberWatch>>();

package/src/validation.ts

@@ -124,6 +124,8 @@ export function cleanTokenConstraints(value: unknown): TokenConstraints | undefi
   if (cwd) constraints.cwd = cwd;
   const teamId = cleanString(value.teamId, "constraints.teamId", { max: 120 });
   if (teamId) constraints.teamId = teamId;
+  const tenantId = cleanString(value.tenantId, "constraints.tenantId", { max: 120 });
+  if (tenantId) constraints.tenantId = tenantId;
   for (const key of ["terminalAttach", "logsRead", "canDelegate"] as const) {
     if (value[key] !== undefined) {
       if (typeof value[key] !== "boolean") throw new ValidationError(`constraints.${key} must be a boolean`);

package/src/workspace-actions.ts

@@ -25,7 +25,7 @@ import { isOwnerAlive, requestWorkspaceMerge } from "./workspace-merge";
 export { LAND_STRATEGIES, DEFAULT_MERGE_STRATEGY } from "./workspace-merge";
 import { claimMetadataPatch, workspaceActiveClaim } from "./workspace-claim";
 import { READY_TO_LAND_STATUSES, TERMINAL_WORKSPACE_STATUSES } from "./workspace-phase";
-import { scheduleRepoMerge } from "./workspace-auto-merge";
+import { CLEAR_MERGE_NO_PROGRESS, scheduleRepoMerge } from "./workspace-auto-merge";
 import type { Command, WorkspaceAutoMergePolicy, WorkspaceMergeStrategy, WorkspaceRecord, WorkspaceStatus } from "./types";
 
 // Single source of truth for the action verb set. The route's `optionalEnum` and
@@ -235,6 +235,10 @@ export function applyWorkspaceAction(workspace: WorkspaceRecord, input: ApplyWor
     ...metadata,
     ...(detail ? { detail } : {}),
     ...(agentId ? { updatedByAgentId: agentId } : {}),
+    // #638 — a fresh ready/request-review hand-off means the owner did more work, so a
+    // prior auto-merge no-progress back-off should clear: the new commits may resolve
+    // whatever blocked the land (e.g. a base that's since been synced). Re-arm the lane.
+    ...(READY_TO_LAND_STATUSES.has(nextStatus) ? CLEAR_MERGE_NO_PROGRESS : {}),
     lastWorkspaceAction: action,
     lastWorkspaceActionAt: Date.now(),
   });

package/src/workspace-auto-merge.ts

@@ -36,7 +36,7 @@ import { workspaceActiveClaim } from "./workspace-claim";
 import { DIRTY_WORKTREE_LAND_SKIP_REASON, READY_TO_LAND_STATUSES } from "./workspace-phase";
 import { fetchHostMergePreview, workspacePathWithinBase } from "./workspace-probe";
 import type { WorkspaceMergePreview, WorkspaceRecord } from "./types";
-import { STEWARD_WAKE_COOLDOWN_MS, workspaceAutoMergeEnabled } from "./config";
+import { STEWARD_WAKE_COOLDOWN_MS, stewardFallbackTarget, workspaceAutoMergeEnabled } from "./config";
 
 // Don't re-wake the managed steward for the same workspace more than once per this
 // window — a persistent conflict/behind row would otherwise re-ping every sweep.
@@ -71,6 +71,79 @@ export function wakeRepoSteward(ws: WorkspaceRecord, reason: string): string | n
   }
 }
 
+// #638 — auto-merge loop breaker. A `workspace.merge` the orchestrator settles `failed`
+// (no progress: origin moved ahead of local base, a replay conflict the clean-preview
+// didn't predict, a push race) returns the workspace to a landable status
+// (`review_requested`). The on-settle re-trigger (routes/commands.ts → scheduleRepoMerge)
+// then re-dispatches it within a few seconds — forever. That's a silent merge-command
+// storm that churns the per-repo lease and 409s every manual land (the live incident).
+// Cap consecutive no-progress attempts per workspace; past the cap, stop dispatching and
+// escalate. The count is incremented by the merge-result handler (server owns the
+// transition) and cleared on genuine progress or a fresh ready/request-review hand-off.
+export const MAX_NO_PROGRESS_MERGE_ATTEMPTS = 3;
+
+// Don't re-escalate a backed-off lane more than once per this window.
+const NO_PROGRESS_ESCALATION_COOLDOWN_MS = 30 * 60 * 1000;
+
+/** The recorded count of consecutive no-progress merge attempts for a workspace (#638). */
+export function mergeNoProgressCount(metadata: Record<string, unknown>): number {
+  const n = metadata.autoMergeNoProgressCount;
+  return typeof n === "number" && Number.isFinite(n) && n > 0 ? n : 0;
+}
+
+/** Metadata patch recording one more no-progress merge attempt (#638). Applied by the
+ * merge-result handler when a dispatched merge settles `failed` without landing. */
+export function noteMergeNoProgress(metadata: Record<string, unknown>): Record<string, unknown> {
+  return { autoMergeNoProgressCount: mergeNoProgressCount(metadata) + 1, autoMergeNoProgressAt: Date.now() };
+}
+
+/** Metadata patch that clears the no-progress back-off — genuine progress (a land/recycle)
+ * or a fresh hand-off means the lane is worth attempting again (#638). */
+export const CLEAR_MERGE_NO_PROGRESS: Record<string, unknown> = {
+  autoMergeNoProgressCount: 0,
+  autoMergeNoProgressAt: undefined,
+  autoMergeBackedOffAt: undefined,
+};
+
+// Surface a lane auto-merge can't advance (#638): wake the repo steward (if enabled) and
+// route a stranded-escalation notice to the configured fallback / workspace owner, so a
+// no-progress lane raises attention instead of churning silently. Cooldown-guarded so a
+// persistently-stuck row isn't re-escalated every sweep.
+function escalateNoProgressMerge(ws: WorkspaceRecord): void {
+  const meta = ws.metadata as Record<string, unknown>;
+  const lastEscalated = typeof meta.autoMergeBackedOffAt === "number" ? meta.autoMergeBackedOffAt : 0;
+  if (lastEscalated && Date.now() - lastEscalated < NO_PROGRESS_ESCALATION_COOLDOWN_MS) return;
+  const mergeError = typeof meta.mergeError === "string" && meta.mergeError ? meta.mergeError : "merge made no progress";
+  if (getStewardConfig().enabled) {
+    wakeRepoSteward(ws, `auto-merge backed off after ${MAX_NO_PROGRESS_MERGE_ATTEMPTS} no-progress attempts (${mergeError})`);
+  }
+  const fallback = stewardFallbackTarget();
+  const recipients = [...new Set([fallback, ws.ownerAgentId].filter((r): r is string => Boolean(r)))];
+  if (recipients.length) {
+    routeNotification({
+      type: "workspace.stranded_escalation",
+      scope: { workspaceId: ws.id, repoRoot: ws.repoRoot, author: ws.ownerAgentId },
+      recipients,
+      message: {
+        subject: "Workspace can't auto-land — needs attention",
+        body: `Workspace \`${ws.branch ?? ws.id}\` in ${ws.repoRoot} failed to land ${MAX_NO_PROGRESS_MERGE_ATTEMPTS}× without progress (${mergeError}). Auto-merge has backed off to stop a merge-command loop. Sync \`${ws.baseRef ?? "base"}\` and land manually, or release the workspace.`,
+        payload: { kind: "workspace.auto-merge-stuck", workspaceId: ws.id, repoRoot: ws.repoRoot, branch: ws.branch, baseRef: ws.baseRef, attempts: MAX_NO_PROGRESS_MERGE_ATTEMPTS, reason: mergeError },
+      },
+    });
+  }
+  patchWorkspaceMetadata(ws.id, { autoMergeBackedOffAt: Date.now() });
+  createActivityEvent({
+    clientId: `workspace-auto-merge-backoff-${ws.id}-${Date.now()}`,
+    kind: "state",
+    title: "Auto-merge backed off (no progress)",
+    body: `${ws.branch ?? ws.id} → ${ws.baseRef ?? "base"} failed to land ${MAX_NO_PROGRESS_MERGE_ATTEMPTS}× without progress — stopping the merge loop (${mergeError})`,
+    meta: ws.branch ?? ws.id,
+    icon: "ti-alert-octagon",
+    view: "orchestrators",
+    metadata: { source: "server", maintenanceJobId: "workspace-auto-merge", workspaceId: ws.id, attempts: MAX_NO_PROGRESS_MERGE_ATTEMPTS },
+  });
+}
+
 interface LandSweepResult {
   scanned: number;
   merged: string[];
@@ -78,12 +151,13 @@ interface LandSweepResult {
   heldByClaim: string[];
   heldByPause: string[];
   heldAwaitingApproval: string[];
+  backedOff: string[];
   leftForSteward: string[];
   wokeStewards: string[];
 }
 
 function emptyLandSweep(): LandSweepResult {
-  return { scanned: 0, merged: [], heldByLease: [], heldByClaim: [], heldByPause: [], heldAwaitingApproval: [], leftForSteward: [], wokeStewards: [] };
+  return { scanned: 0, merged: [], heldByLease: [], heldByClaim: [], heldByPause: [], heldAwaitingApproval: [], backedOff: [], leftForSteward: [], wokeStewards: [] };
 }
 
 // Deterministic auto-land (Layer 0, #167/#207/#242): land conflict-free ready work
@@ -95,13 +169,22 @@ async function landCandidates(
 ): Promise<LandSweepResult> {
   const stewardEnabled = getStewardConfig().enabled;
   const merged: string[] = [];
-  const heldByLease: string[] = [], heldByClaim: string[] = [], heldByPause: string[] = [], heldAwaitingApproval: string[] = [], leftForSteward: string[] = [], wokeStewards: string[] = [];
+  const heldByLease: string[] = [], heldByClaim: string[] = [], heldByPause: string[] = [], heldAwaitingApproval: string[] = [], backedOff: string[] = [], leftForSteward: string[] = [], wokeStewards: string[] = [];
 
   for (const ws of candidates) {
     if (isAwaitingReviewerApproval(ws)) { heldAwaitingApproval.push(ws.id); continue; }
     // A claimed workspace is being validated by a steward — don't race it (#208).
     if (workspaceActiveClaim(ws)) { heldByClaim.push(ws.id); continue; }
     if (getActiveMergePause(ws.repoRoot, ws.baseRef ?? "main")) { heldByPause.push(ws.id); continue; }
+    // #638 loop breaker: a lane whose last N merges settled `failed` without landing
+    // can't be advanced by re-dispatching — stop the storm, escalate once, and leave it
+    // for a steward/human. Cleared on genuine progress or a fresh hand-off (the agent did
+    // more work), so this is a back-off, not a permanent ban.
+    if (mergeNoProgressCount(ws.metadata) >= MAX_NO_PROGRESS_MERGE_ATTEMPTS) {
+      backedOff.push(ws.id);
+      escalateNoProgressMerge(ws);
+      continue;
+    }
     const orch = orchestrators.find((candidate) => workspacePathWithinBase(ws.sourceCwd, candidate.baseDir));
     if (!orch?.apiUrl) continue;
     const preview = await fetchHostMergePreview(orch.apiUrl, ws);
@@ -164,7 +247,7 @@ async function landCandidates(
     });
   }
 
-  return { scanned: candidates.length, merged, heldByLease, heldByClaim, heldByPause, heldAwaitingApproval, leftForSteward, wokeStewards };
+  return { scanned: candidates.length, merged, heldByLease, heldByClaim, heldByPause, heldAwaitingApproval, backedOff, leftForSteward, wokeStewards };
 }
 
 // Coalesced, fire-and-forget per-repo land trigger (see module header). A run already

package/src/workspace-merge.ts

@@ -3,6 +3,7 @@ import { isRecord, stringValue } from "agent-relay-sdk";
 import { createCommand } from "./commands-db";
 import {
   acquireMergeLease,
+  getActiveMergePause,
   getAgent,
   listAgents,
   listOrchestrators,
@@ -186,6 +187,15 @@ export function requestWorkspaceMerge(workspace: WorkspaceRecord, opts: RequestW
   if (workspace.mode !== "isolated" || !workspace.worktreePath) {
     return { ok: false, status: 422, error: `workspace ${workspace.id} has no worktree to merge` };
   }
+  // #638 — an active merge-pause is the operator's tourniquet for a merge loop on this
+  // repo. The auto-merge job already honors it (workspace-auto-merge.ts), but gate the
+  // shared dispatcher too as defense-in-depth: never let an auto-merge-originated request
+  // acquire the lease + storm while paused, regardless of which caller dispatched it. A
+  // manual/steward land (any other `requestedBy`) is the intended RECOVERY path during a
+  // pause, so it's allowed through.
+  if (opts.requestedBy === "auto-merge" && getActiveMergePause(workspace.repoRoot, workspace.baseRef ?? "main")) {
+    return { ok: false, status: 409, error: `auto-merge paused for ${workspace.repoRoot}` };
+  }
   const lease = acquireMergeLease(workspace.repoRoot, workspace.id, opts.requestedBy);
   if (!lease.ok) {
     return {