agent-relay-server 0.91.1 → 0.91.3

package/docs/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Agent Relay API",
-    "version": "0.91.1",
+    "version": "0.91.3",
     "description": "Real-time message bus for inter-agent communication. Agent-first: this spec is designed for machine consumption — agents can self-discover the full API surface via GET /api/spec.",
     "license": {
       "name": "MIT",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-server",
-  "version": "0.91.1",
+  "version": "0.91.3",
   "description": "Lightweight HTTP message relay for inter-agent communication across machines",
   "module": "src/index.ts",
   "type": "module",

package/runner/src/config.ts

@@ -99,6 +99,11 @@ export function registrationTimeoutMsFromEnv(): number {
   return Number.isFinite(parsed) && parsed > 0 ? parsed : 60_000;
 }
 
+export function nativeSelfResumeTimeoutMsFromEnv(): number {
+  const parsed = Number(process.env.AGENT_RELAY_NATIVE_SELF_RESUME_TIMEOUT_MS);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 120_000;
+}
+
 export function agentProfileNameFromEnv(): string | undefined {
   return process.env.AGENT_RELAY_AGENT_PROFILE;
 }

package/src/agent-death-diagnosis.ts

@@ -173,6 +173,10 @@ export function classifyAgentDeath(ev: DeathEvidence): AgentDeathDiagnosis {
     pushUnique(evidence, "log/error text matched a crash pattern (panic/segfault/traceback)");
     return verdict("provider-crash", "medium", "Likely provider crash (log pattern)");
   }
+  if (/self-resume-(?:exit-before-resume|relaunch-failed)/i.test(text)) {
+    pushUnique(evidence, "native self-resume exited before Relay observed a resumed provider");
+    return verdict("provider-crash", "high", "Self-resume failed before relaunch completed");
+  }
 
   // 5) killed — external SIGKILL/SIGTERM that isn't OOM and isn't a parent-initiated shutdown.
   if (signal && /KILL|TERM|INT|HUP|QUIT/i.test(signal)) {

package/src/bus.ts

@@ -11,6 +11,7 @@ import { reconcileTerminalProviderExit } from "./services/terminal-provider-exit
 import { registerAgent } from "./services/register-agent";
 import { authContextFromBus } from "./services/auth-context";
 import { commandAuthorizationResource, dispatchCommand } from "./services/dispatch-command";
+import { auditProviderStateTransition } from "./services/provider-state-audit";
 import { ServiceAuthError } from "./services/errors";
 import { ShutdownAuthError, ShutdownTargetError, shutdownAgent, shutdownInputFromBusFrame } from "./services/shutdown-agent";
 import { applyCommandToRecipe } from "./recipe-runner";
@@ -27,9 +28,8 @@ import {
   type RegisterFrame,
 } from "agent-relay-sdk/protocol";
 import { errMessage, isRecord, stringValue } from "agent-relay-sdk";
-import { agentProviderState } from "./provider-state";
 import { getComponentAuth, isComponentAuthorizedFor, isAuthorized, isOriginAllowed, unauthorized } from "./security";
-import type { AgentCard, Command, ComponentToken, ContextState, Message, ProviderCapabilities, QuotaState, RegisterAgentInput, Task } from "./types";
+import type { Command, ComponentToken, ContextState, Message, ProviderCapabilities, QuotaState, RegisterAgentInput, Task } from "./types";
 
 interface BusSocketData {
   kind: "bus";
@@ -562,36 +562,6 @@ function emitAgentStatusEvent(agentId: string): void {
   });
 }
 
-function auditProviderStateTransition(agentId: string, before: AgentCard | null | undefined, after: AgentCard | null | undefined): void {
-  const previous = agentProviderState(before);
-  const next = agentProviderState(after);
-  const previousKey = providerStateKey(previous);
-  const nextKey = providerStateKey(next);
-  if (previousKey === nextKey) return;
-  if (next?.state !== "blocked" && previous?.state !== "blocked") return;
-
-  const blocked = next?.state === "blocked";
-  const label = typeof next?.label === "string" ? next.label : typeof previous?.label === "string" ? previous.label : "provider blocked";
-  try {
-    createActivityEvent({
-      clientId: `server-agent-${agentId}-provider-state-${nextKey ?? "cleared"}-${Date.now()}`,
-      kind: "state",
-      title: blocked ? `Agent blocked: ${label}` : "Agent provider unblocked",
-      meta: agentId,
-      icon: blocked ? "ti-alert-circle" : "ti-circle-check",
-      view: "agents",
-      agentId,
-      metadata: {
-        source: "server",
-        previousProviderState: previous ?? null,
-        providerState: next ?? null,
-      },
-    });
-  } catch {
-    // Audit trail writes must never block bus status updates.
-  }
-}
-
 function auditRunnerTimelineEvent(agentId: string, timelineEvent: unknown): void {
   if (!isRecord(timelineEvent)) return;
   const metadata = isRecord(timelineEvent.metadata) ? timelineEvent.metadata : {};
@@ -633,11 +603,6 @@ function auditRunnerTimelineEvent(agentId: string, timelineEvent: unknown): void
   }
 }
 
-function providerStateKey(state: Record<string, unknown> | null): string | null {
-  if (!state) return null;
-  return [state.state, typeof state.reason === "string" ? state.reason : ""].join(":");
-}
-
 function sendCommandResult(
   ws: BusWebSocket,
   commandId: string,

package/src/notification-types.ts

@@ -12,6 +12,7 @@ export type NotificationType =
   | "agent.resume_budget_exhausted"
   | "agent.transient_land_hold"
   | "agent.rate_limit_resume"
+  | "agent.provider_blocked"
   | "team.idle"
   | "team.member_silent"
   | "team.coordinator_gone"
@@ -68,6 +69,7 @@ export const DEFAULT_TTL_MS: Record<NotificationType, number | null> = {
   "agent.resume_budget_exhausted": null,
   "agent.transient_land_hold": null,
   "agent.rate_limit_resume": 15 * MINUTES,
+  "agent.provider_blocked": null,
   "team.idle": null,
   "team.member_silent": null,
   "team.coordinator_gone": null,

package/src/provider-state.ts

@@ -38,6 +38,8 @@ export function isProviderBlocked(agent: HasMeta | null | undefined): boolean {
 
 /** The `reason` value a rate-limit hold carries (matches the runner's control-server seam). */
 export const RATE_LIMIT_BLOCK_REASON = "rate_limit";
+export const TRANSIENT_RATE_LIMIT_KIND = "transient_overload";
+export const TRANSIENT_RATE_LIMIT_BACKOFF_MS = [30_000, 2 * 60_000, 5 * 60_000] as const;
 
 /** True when the agent is specifically held on a provider usage/rate limit. */
 export function isRateLimitHold(agent: HasMeta | null | undefined): boolean {
@@ -45,6 +47,19 @@ export function isRateLimitHold(agent: HasMeta | null | undefined): boolean {
   return state?.state === "blocked" && state.reason === RATE_LIMIT_BLOCK_REASON;
 }
 
+export function isTransientRateLimitHold(state: ProviderStateView | null): boolean {
+  return state?.kind === TRANSIENT_RATE_LIMIT_KIND;
+}
+
+export function transientRateLimitAttempt(state: ProviderStateView | null): number {
+  const raw = state?.resumeAttempt;
+  return typeof raw === "number" && Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : 0;
+}
+
+export function transientRateLimitExhausted(state: ProviderStateView | null): boolean {
+  return isTransientRateLimitHold(state) && transientRateLimitAttempt(state) >= TRANSIENT_RATE_LIMIT_BACKOFF_MS.length;
+}
+
 // Fallback window when the provider gave no reset time: retry after this long
 // rather than waiting the full (unknown) limit window. Re-entering the hold on a
 // still-limited retry is cheap, and a subscription limit almost always carries a
@@ -56,6 +71,12 @@ export function isRateLimitHold(agent: HasMeta | null | undefined): boolean {
  */
 export function rateLimitResumeAt(state: ProviderStateView | null): number | null {
   if (!state) return null;
+  if (isTransientRateLimitHold(state)) {
+    if (typeof state.nextResumeAt === "number" && Number.isFinite(state.nextResumeAt)) return state.nextResumeAt;
+    if (transientRateLimitExhausted(state)) return null;
+    const base = typeof state.enteredAt === "number" && Number.isFinite(state.enteredAt) ? state.enteredAt : Date.now();
+    return base + TRANSIENT_RATE_LIMIT_BACKOFF_MS[transientRateLimitAttempt(state)]!;
+  }
   if (typeof state.resetAt === "number" && Number.isFinite(state.resetAt)) return state.resetAt;
   if (typeof state.enteredAt === "number" && Number.isFinite(state.enteredAt)) {
     return state.enteredAt + rateLimitFallbackMs();

package/src/services/provider-state-audit.ts

@@ -0,0 +1,52 @@
+import { createActivityEvent } from "../db";
+import { routeNotification } from "../notification-router";
+import { agentProviderState } from "../provider-state";
+import type { AgentCard } from "../types";
+
+export function auditProviderStateTransition(agentId: string, before: AgentCard | null | undefined, after: AgentCard | null | undefined): void {
+  const previous = agentProviderState(before);
+  const next = agentProviderState(after);
+  const previousKey = providerStateKey(previous);
+  const nextKey = providerStateKey(next);
+  if (previousKey === nextKey) return;
+  if (next?.state !== "blocked" && previous?.state !== "blocked") return;
+
+  const blocked = next?.state === "blocked";
+  const label = typeof next?.label === "string" ? next.label : typeof previous?.label === "string" ? previous.label : "provider blocked";
+  try {
+    createActivityEvent({
+      clientId: `server-agent-${agentId}-provider-state-${nextKey ?? "cleared"}-${Date.now()}`,
+      kind: "state",
+      title: blocked ? `Agent blocked: ${label}` : "Agent provider unblocked",
+      meta: agentId,
+      icon: blocked ? "ti-alert-circle" : "ti-circle-check",
+      view: "agents",
+      agentId,
+      metadata: {
+        source: "server",
+        previousProviderState: previous ?? null,
+        providerState: next ?? null,
+      },
+    });
+    if (!blocked || !after?.spawnedBy) return;
+    routeNotification({
+      type: "agent.provider_blocked",
+      scope: { agentId, parent: after.spawnedBy },
+      recipients: [after.spawnedBy],
+      message: {
+        subject: "Child held on provider block",
+        body: `Child agent ${after.label ?? after.name ?? agentId} is held: ${label}.`,
+        payload: { kind: "agent.provider_blocked", agentId, providerState: next ?? null },
+        replyExpected: false,
+        idempotencyKey: `provider-blocked:${agentId}:${nextKey ?? "blocked"}`,
+      },
+    });
+  } catch {
+    // Audit trail writes must never block bus status updates.
+  }
+}
+
+function providerStateKey(state: Record<string, unknown> | null): string | null {
+  if (!state) return null;
+  return [state.state, typeof state.reason === "string" ? state.reason : ""].join(":");
+}

package/src/services/rate-limit-resume.ts

@@ -1,6 +1,14 @@
-import { createActivityEvent, listAgents } from "../db";
+import { createActivityEvent, listAgents, mergeAgentMeta } from "../db";
 import { routeNotification } from "../notification-router";
-import { agentProviderState, isRateLimitHold, rateLimitResumeAt } from "../provider-state";
+import {
+  TRANSIENT_RATE_LIMIT_BACKOFF_MS,
+  agentProviderState,
+  isRateLimitHold,
+  isTransientRateLimitHold,
+  rateLimitResumeAt,
+  transientRateLimitAttempt,
+  transientRateLimitExhausted,
+} from "../provider-state";
 
 // #286 Phase 2: auto-resume agents held on a usage/rate-limit stall once their
 // window resets. The runner reports the hold as `providerState: blocked` (reason
@@ -22,14 +30,49 @@ const envMsOrDefault = (name: string, fallback: number): number => {
 // between delivery and the agent flipping to busy (which clears the hold).
 const resumeCooldownMs = () => envMsOrDefault("AGENT_RELAY_RATE_LIMIT_RESUME_COOLDOWN_MS", 5 * 60 * 1000);
 
-const resumeTracker = new Map<string, number>();
+interface ResumeTrackerEntry {
+  lastResumeAt?: number;
+  attempts?: number;
+  exhaustedNotified?: boolean;
+}
+
+const resumeTracker = new Map<string, ResumeTrackerEntry>();
 
 export function resetRateLimitResumeTrackerForTests(): void {
   resumeTracker.clear();
 }
 
 const RESUME_BODY =
-  "▶️ Your usage/rate limit has reset — resuming. Continue the task you were working on when the limit interrupted you. If you had already finished, no action is needed.";
+  "Your provider limit/overload hold is ready to retry — resuming. Continue the task you were working on when the provider interrupted you. If you had already finished, no action is needed.";
+
+function nextTransientState(state: Record<string, unknown>, now: number, attempts: number): Record<string, unknown> {
+  const nextAttempts = attempts + 1;
+  const nextDelay = TRANSIENT_RATE_LIMIT_BACKOFF_MS[nextAttempts];
+  return {
+    ...state,
+    resumeAttempt: nextAttempts,
+    lastResumeAt: now,
+    ...(nextDelay !== undefined ? { nextResumeAt: now + nextDelay } : {}),
+    updatedAt: now,
+  };
+}
+
+function notifyTransientExhausted(agent: { id: string; label?: string | null; spawnedBy?: string | null }, state: Record<string, unknown>, now: number): void {
+  mergeAgentMeta(agent.id, { providerState: { ...state, exhaustedAt: now, updatedAt: now } });
+  if (!agent.spawnedBy) return;
+  routeNotification({
+    type: "agent.provider_blocked",
+    scope: { agentId: agent.id, parent: agent.spawnedBy },
+    recipients: [agent.spawnedBy],
+    message: {
+      subject: "Child held on provider overload",
+      body: `Child agent ${agent.label ?? agent.id} is still held on transient provider overload after bounded auto-resume attempts. Manual check or restart may be needed.`,
+      payload: { kind: "agent.provider_blocked", agentId: agent.id, providerState: state, exhausted: true },
+      replyExpected: false,
+      idempotencyKey: `provider-overload-exhausted:${agent.id}:${state.enteredAt ?? "unknown"}`,
+    },
+  });
+}
 
 export function resumeRateLimitedAgents(): Record<string, unknown> {
   const now = Date.now();
@@ -47,9 +90,21 @@ export function resumeRateLimitedAgents(): Record<string, unknown> {
     const resumeAt = rateLimitResumeAt(state);
     if (resumeAt !== null && now < resumeAt) { waiting.push(agent.id); continue; }
 
-    const lastResume = resumeTracker.get(agent.id);
-    if (lastResume !== undefined && now - lastResume < cooldown) { cooling.push(agent.id); continue; }
-    resumeTracker.set(agent.id, now);
+    const transient = isTransientRateLimitHold(state);
+    const tracked = resumeTracker.get(agent.id) ?? {};
+    const attempts = transient ? Math.max(transientRateLimitAttempt(state), tracked.attempts ?? 0) : 0;
+    if (transient && (attempts >= TRANSIENT_RATE_LIMIT_BACKOFF_MS.length || transientRateLimitExhausted(state))) {
+      if (!tracked.exhaustedNotified) {
+        notifyTransientExhausted(agent, state as Record<string, unknown>, now);
+        resumeTracker.set(agent.id, { ...tracked, attempts, exhaustedNotified: true });
+      }
+      waiting.push(agent.id);
+      continue;
+    }
+
+    const lastResume = tracked.lastResumeAt;
+    if (!transient && lastResume !== undefined && now - lastResume < cooldown) { cooling.push(agent.id); continue; }
+    resumeTracker.set(agent.id, { ...tracked, attempts: transient ? attempts + 1 : undefined, lastResumeAt: now });
 
     try {
       // Short default TTL (per-type): a "limit reset, resume" push is pointless once stale — the
@@ -77,6 +132,8 @@ export function resumeRateLimitedAgents(): Record<string, unknown> {
       continue;
     }
 
+    if (transient) mergeAgentMeta(agent.id, { providerState: nextTransientState(state as Record<string, unknown>, now, attempts) });
+
     resumed.push(agent.id);
     createActivityEvent({
       clientId: `rate-limit-resume-${agent.id}-${now}`,
@@ -97,7 +154,10 @@ export function resumeRateLimitedAgents(): Record<string, unknown> {
   }
 
   // Forget agents no longer held so a future hold resumes immediately.
-  for (const id of resumeTracker.keys()) if (!seen.has(id)) resumeTracker.delete(id);
+  for (const [id, tracked] of resumeTracker.entries()) {
+    if (seen.has(id)) continue;
+    if (tracked.lastResumeAt === undefined || now - tracked.lastResumeAt > 15 * 60_000) resumeTracker.delete(id);
+  }
 
   return { held: seen.size, resumed, waiting, cooling, tracked: resumeTracker.size };
 }

package/src/services/terminal-provider-exit.ts

@@ -8,6 +8,7 @@ import { isRecord } from "agent-relay-sdk";
 import { clearAgentMetaKeys } from "../db";
 import { notifyAgentOffline } from "../agent-lifecycle-events";
 import { diagnoseAgentDeath } from "../agent-death-diagnosis";
+import { routeNotification } from "../notification-router";
 import type { AgentCard } from "../types";
 
 /**
@@ -29,5 +30,27 @@ export function reconcileTerminalProviderExit(agentId: string, before: AgentCard
   if (!isRecord(marker)) return;
   const reasonText = typeof marker.reason === "string" ? marker.reason : "provider exited";
   const reason = `provider exited (${reasonText})`;
-  notifyAgentOffline(agentId, reason, diagnoseAgentDeath({ agent: after, reason }));
+  const diagnosis = diagnoseAgentDeath({ agent: after, reason });
+  notifyAgentOffline(agentId, reason, diagnosis);
+  if (!after.spawnedBy && reasonText.startsWith("self-resume-")) {
+    routeNotification({
+      type: "agent.exited",
+      scope: { agentId },
+      recipients: ["user"],
+      message: {
+        subject: "Self-resume failed",
+        body: `Agent \`${agentId}\` exited during self-resume before the resumed session was confirmed. Manual recovery may need the provider resume id from the runner log.`,
+        payload: {
+          kind: "agent.self_resume_failed",
+          agentId,
+          reason,
+          diagnosis,
+          ...(typeof marker.claudeResumeId === "string" ? { claudeResumeId: marker.claudeResumeId } : {}),
+          ...(typeof marker.selfResumeCommandId === "string" ? { commandId: marker.selfResumeCommandId } : {}),
+          ...(typeof marker.selfResumeGeneration === "number" ? { generation: marker.selfResumeGeneration } : {}),
+        },
+        replyExpected: false,
+      },
+    });
+  }
 }