agent-relay-server 0.86.0 → 0.88.0

package/src/maintenance/teams.ts

@@ -2,6 +2,7 @@ import {
   clearTeamZeroMembersSince,
   countLiveTeamMembers,
   createActivityEvent,
+  getAgent,
   getCrashReport,
   hasTerminalEvent,
   getTeamBusyState,
@@ -9,18 +10,25 @@ import {
   listActiveTeams,
   setTeamZeroMembersSince,
 } from "../db";
-import { STALE_TTL_MS } from "../config";
+import { STALE_TTL_MS, stewardFallbackTarget } from "../config";
 import { agentProviderState } from "../provider-state";
 import { dissolveTeamById } from "../services/team";
+import { resolveCoordinatorAgentId } from "../steward-identity";
 import { getTeamsConfig } from "../teams-config-store";
 import { teamReapGraceMs } from "./constants";
 import { notifyTarget } from "./events";
-import { teamIdleTracker, teamMemberWatchTracker, type TeamMemberWatch } from "../team-idle-tracker";
+import {
+  teamCoordinatorWatchTracker,
+  teamIdleTracker,
+  teamMemberWatchTracker,
+  type TeamMemberWatch,
+} from "../team-idle-tracker";
 import type { AgentCard } from "../types";
 
 export function resetTeamIdleWatchdogForTests(): void {
   teamIdleTracker.clear();
   teamMemberWatchTracker.clear();
+  teamCoordinatorWatchTracker.clear();
 }
 
 /** A member counts as "alive" (worth watching for a future silence) when it's past onboarding and
@@ -84,6 +92,95 @@ function detectSilentMembers(
   }
 }
 
+/**
+ * #412/#468 — detect a team's COORDINATOR going silent and escalate to a human, instead of the team
+ * stalling forever. The member watchdog above SKIPS the coordinator (it's the ping recipient), and a
+ * missing/dead coordinator otherwise only gets recorded — never surfaced — so a silently-dead
+ * coordinator means nobody is driving and no member is even being listened to (#412's single point of
+ * failure). Reuses the SAME aliveness predicate (`memberIsAlive`) + terminal-event suppression
+ * (`getCrashReport || hasTerminalEvent`) as the member watch so a CLEANLY-exited coordinator does not
+ * false-positive. Escalates ONCE per gone-spell (re-arms on recovery) up the lineage — the gone
+ * coordinator's own coordinator (#639 resolver), falling back to the configured human/steward target
+ * — never dead-ending at `system`. A coordinator-gone team is always surfaced, never silently
+ * abandoned (no-silent-dissolve).
+ */
+function detectGoneCoordinator(
+  teamId: string,
+  coordinatorId: string | undefined,
+  now: number,
+  thresholdMs: number,
+  out: { escalatedTeamIds: string[]; suppressedTeamIds: string[]; goneCoordinatorIds: string[] },
+): void {
+  if (!coordinatorId) return; // no coordinator on the row — nothing to watch (idle path records this)
+  const freshCutoff = now - thresholdMs;
+  const existing = teamCoordinatorWatchTracker.get(teamId);
+  // A re-elected/claimed coordinator is a fresh identity — drop a watch left over from the old one.
+  if (existing && existing.coordinatorId !== coordinatorId) teamCoordinatorWatchTracker.delete(teamId);
+
+  const coord = getAgent(coordinatorId);
+  if (coord && memberIsAlive(coord, freshCutoff)) {
+    const prior = existing?.coordinatorId === coordinatorId ? existing : undefined;
+    teamCoordinatorWatchTracker.set(teamId, {
+      coordinatorId,
+      lastActiveAt: Math.max(coord.lastSeen, prior?.lastActiveAt ?? 0),
+      escalated: false, // alive again ⇒ re-arm the escalation
+      ...(coord.label ? { label: coord.label } : {}),
+    });
+    return;
+  }
+
+  // Coordinator is NOT alive (offline, heartbeat-stale, or vanished). Only escalate a coordinator we
+  // actually observed alive — never fire on cold/unknown startup state (mirrors the member watch).
+  const watch = teamCoordinatorWatchTracker.get(teamId);
+  if (!watch || watch.coordinatorId !== coordinatorId || watch.escalated) return;
+  watch.escalated = true; // one escalation per gone-spell, regardless of outcome below
+  out.goneCoordinatorIds.push(coordinatorId);
+
+  // A terminal marker means a #645 clean terminus (graceful shutdown) or a #633 crash the operator was
+  // already surfaced on — not a SILENT death. Suppress.
+  if (getCrashReport(coordinatorId) || hasTerminalEvent(coordinatorId)) {
+    out.suppressedTeamIds.push(teamId);
+    return;
+  }
+
+  // Escalate up the lineage, falling back to the configured human/steward target (#639/#642 pattern).
+  const escalationTarget = resolveCoordinatorAgentId(coordinatorId) ?? stewardFallbackTarget();
+  const label = watch.label ?? coordinatorId;
+  const sent = escalationTarget
+    ? notifyTarget(
+        "team.coordinator_gone",
+        { teamId, agentId: coordinatorId, parent: resolveCoordinatorAgentId(coordinatorId) ?? undefined },
+        escalationTarget,
+        "Team coordinator went silent",
+        `Coordinator \`${label}\` of team \`${teamId}\` has gone silent with no terminal event — nobody is driving the team and its members are no longer being listened to. Take over coordination, restart the team, or stand it down.`,
+        { kind: "team.coordinator_gone", teamId, coordinatorId, escalationTarget, label: watch.label },
+      )
+    : null;
+  if (sent) out.escalatedTeamIds.push(teamId);
+
+  // No-silent-dissolve: surface the coordinator-gone team on the dashboard even when there is no
+  // escalation target configured — it must never be silently abandoned.
+  createActivityEvent({
+    clientId: `team-coordinator-gone-${teamId}-${coordinatorId}-${now}`,
+    kind: "state",
+    title: "Team coordinator went silent",
+    body: sent
+      ? `Coordinator ${label} of team ${teamId} went silent — escalated to ${sent}.`
+      : `Coordinator ${label} of team ${teamId} went silent — no escalation target configured (set AGENT_RELAY_WORKSPACE_STEWARD_FALLBACK).`,
+    meta: teamId,
+    icon: "ti-user-x",
+    view: "teams",
+    agentId: coordinatorId,
+    metadata: {
+      source: "server",
+      maintenanceJobId: "team-idle-watchdog",
+      teamId,
+      coordinatorId,
+      escalationTarget: sent ?? null,
+    },
+  });
+}
+
 function teamIdlePayload(teamId: string, coordinatorId: string, thresholdMinutes: number, idleForMs: number, firstNoBusyAt: number) {
   const roster = getTeamRoster(teamId).map((member) => {
     const providerState = agentProviderState(member);
@@ -145,12 +242,17 @@ export function runTeamIdleWatchdog(input: { now?: number } = {}): Record<string
   const alreadyPingedTeamIds: string[] = [];
   const missingCoordinatorTeamIds: string[] = [];
   const memberSilence = { pingedMembers: [] as string[], watchedMembers: 0 };
+  const coordinatorGone = { escalatedTeamIds: [] as string[], suppressedTeamIds: [] as string[], goneCoordinatorIds: [] as string[] };
 
   for (const team of listActiveTeams()) {
     seen.add(team.id);
     // #634 — runs every cycle, independent of the team-level idle gate below: a single silent
     // member must surface even while OTHER members keep the team "busy".
     detectSilentMembers(team.id, team.coordinatorId, now, thresholdMs, memberSilence);
+    // #412/#468 — coordinator-gone escalation, also every cycle and independent of the idle gate: a
+    // dead coordinator must surface even if the team isn't (yet) idle, and the idle-ping path can't
+    // reach it (it would just ping the void).
+    detectGoneCoordinator(team.id, team.coordinatorId, now, thresholdMs, coordinatorGone);
     const busy = getTeamBusyState(team.id, cutoff);
     if (busy.busyMemberIds.length > 0) {
       teamIdleTracker.delete(team.id);
@@ -182,6 +284,7 @@ export function runTeamIdleWatchdog(input: { now?: number } = {}): Record<string
 
   for (const key of teamIdleTracker.keys()) if (!seen.has(key)) teamIdleTracker.delete(key);
   for (const key of teamMemberWatchTracker.keys()) if (!seen.has(key)) teamMemberWatchTracker.delete(key);
+  for (const key of teamCoordinatorWatchTracker.keys()) if (!seen.has(key)) teamCoordinatorWatchTracker.delete(key);
 
   return {
     enabled: true,
@@ -194,6 +297,9 @@ export function runTeamIdleWatchdog(input: { now?: number } = {}): Record<string
     missingCoordinatorTeamIds,
     silentMemberIds: memberSilence.pingedMembers,
     watchedMembers: memberSilence.watchedMembers,
+    coordinatorGoneTeamIds: coordinatorGone.escalatedTeamIds,
+    coordinatorGoneSuppressedTeamIds: coordinatorGone.suppressedTeamIds,
+    goneCoordinatorIds: coordinatorGone.goneCoordinatorIds,
   };
 }
 

package/src/mcp/index.ts

@@ -9,6 +9,7 @@ import { TEAM_TOOLS, relayTeamTool } from "../mcp-team-tools";
 import { BLACKBOARD_TOOLS, relayBlackboardTool } from "../mcp-blackboard-tools";
 import { PROJECT_TOOLS, relayProjectTool } from "../mcp-project-tools"; import { ISSUE_REF_TOOLS, relayIssueRefTool } from "../mcp-issue-ref-tools"; import { ISSUE_ASSOCIATION_TOOLS, relayIssueAssociationTool } from "../mcp-issue-association-tools"; import { ISSUE_ACTIVITY_TOOLS, relayIssueActivityTool } from "../mcp-issue-activity-tools"; import { ISSUE_LIFECYCLE_TOOLS, relayIssueLifecycleTool } from "../mcp-issue-lifecycle-tools";
 import { NOTIFICATION_SUBSCRIPTION_TOOLS, relayNotificationSubscriptionTool } from "../mcp-notification-subscription-tools"; import { MERGE_TOOLS, relayMergeTool } from "../mcp-merge-tools";
+import { NTFY_TOOLS, relayNotifyNtfy } from "../mcp-ntfy-tools";
 import { AGENT_ACTION_TOOL, relayAgentAction } from "../mcp-agent-action"; import { providerCatalogToolInputSchema } from "../mcp-provider-catalog";
 import type { ActivityKind } from "../types";
 import { worktreeMcpInstructions } from "../workspace-phase";
@@ -33,6 +34,7 @@ const TOOLS: ToolDefinition[] = [
   ...BLACKBOARD_TOOLS,
   ...PROJECT_TOOLS,
   ...NOTIFICATION_SUBSCRIPTION_TOOLS,
+  ...NTFY_TOOLS,
 ];
 
 export async function postMcp(req: Request): Promise<Response> {
@@ -144,6 +146,7 @@ async function callTool(auth: McpAuthContext, params: unknown): Promise<Record<s
     else if (name.startsWith("relay_blackboard_")) result = relayBlackboardTool(auth, name, args);
     else if (name.startsWith("relay_project_")) result = relayProjectTool(auth, name, args); else if (name.startsWith("relay_issue_ref_")) result = relayIssueRefTool(auth, name, args); else if (name.startsWith("relay_issue_association_")) result = relayIssueAssociationTool(auth, name, args); else if (name.startsWith("relay_issue_activity_")) result = relayIssueActivityTool(auth, name, args); else if (name.startsWith("relay_issue_lifecycle_")) result = await relayIssueLifecycleTool(auth, name, args);
     else if (name === "relay_subscribe" || name === "relay_unsubscribe" || name === "relay_list_subscriptions") result = relayNotificationSubscriptionTool(auth, name, args);
+    else if (name === "relay_notify_ntfy") result = await relayNotifyNtfy(args);
     else throw new ValidationError(`unknown tool: ${name}`);
 
     auditToolCall(auth, name, "ok", result);

package/src/mcp-ntfy-tools.ts

@@ -0,0 +1,35 @@
+import { MAX_BODY_BYTES } from "./config";
+import { sendNtfyNotification, NTFY_PRIORITIES, type NtfyNotificationResult, type NtfyPriority } from "./ntfy";
+import { optionalEnum } from "./validation";
+import type { ToolDefinition } from "./mcp/types";
+import { optionalString, optionalStringArray, stringField } from "./mcp/validation";
+
+export const NTFY_TOOLS: ToolDefinition[] = [
+  {
+    name: "relay_notify_ntfy",
+    description: "Send an egress-only ntfy notification to the configured operator topic. Supports title, body, priority min→urgent, optional tags, and click-through link.",
+    requiredScopes: ["notifications:write"],
+    inputSchema: {
+      type: "object",
+      properties: {
+        title: { type: "string" },
+        body: { type: "string" },
+        priority: { type: "string", enum: NTFY_PRIORITIES },
+        tags: { type: "array", items: { type: "string" } },
+        click: { type: "string" },
+      },
+      required: ["title", "body"],
+      additionalProperties: false,
+    },
+  },
+];
+
+export async function relayNotifyNtfy(args: Record<string, unknown>): Promise<NtfyNotificationResult> {
+  return sendNtfyNotification({
+    title: stringField(args.title, "title", { required: true, max: 200 }),
+    body: stringField(args.body, "body", { required: true, maxBytes: MAX_BODY_BYTES }),
+    priority: optionalEnum(args.priority, "priority", NTFY_PRIORITIES) as NtfyPriority | undefined,
+    tags: optionalStringArray(args.tags, "tags"),
+    click: optionalString(args.click, "click", 1000),
+  });
+}

package/src/notification-types.ts

@@ -5,6 +5,7 @@ export type NotificationType =
   | "agent.exited"
   | "agent.spawn_failed"
   | "agent.context_threshold"
+  | "agent.stuck_busy"
   | "agent.quota_threshold"
   | "agent.resume_budget_warning"
   | "agent.resume_budget_exhausted"
@@ -12,6 +13,7 @@ export type NotificationType =
   | "agent.rate_limit_resume"
   | "team.idle"
   | "team.member_silent"
+  | "team.coordinator_gone"
   | "plan.bound_agent_exited"
   | "workspace.steward_task"
   | "workspace.steward_handoff"
@@ -58,6 +60,7 @@ export const DEFAULT_TTL_MS: Record<NotificationType, number | null> = {
   "agent.exited": null,
   "agent.spawn_failed": null,
   "agent.context_threshold": 15 * MINUTES,
+  "agent.stuck_busy": null,
   "agent.quota_threshold": 15 * MINUTES,
   "agent.resume_budget_warning": null,
   "agent.resume_budget_exhausted": null,
@@ -65,6 +68,7 @@ export const DEFAULT_TTL_MS: Record<NotificationType, number | null> = {
   "agent.rate_limit_resume": 15 * MINUTES,
   "team.idle": null,
   "team.member_silent": null,
+  "team.coordinator_gone": null,
   "plan.bound_agent_exited": null,
   "workspace.steward_task": null,
   "workspace.steward_handoff": null,

package/src/ntfy.ts

@@ -0,0 +1,222 @@
+import { Buffer } from "node:buffer";
+import { MAX_BODY_BYTES } from "./config";
+import { getConfig } from "./config-store";
+import { upsertAgent, ValidationError } from "./db";
+import { registerChannelTypeSetting } from "./settings/modules";
+import { cleanString, cleanStringArray } from "./validation";
+import { errMessage, isRecord, type JSONSchema } from "agent-relay-sdk";
+
+export const NTFY_CONFIG_NAMESPACE = "channel-type";
+export const NTFY_CONFIG_KEY = "ntfy";
+export const NTFY_CHANNEL_ID = "ntfy:default";
+
+export const NTFY_PRIORITIES = ["min", "low", "default", "high", "urgent"] as const;
+export type NtfyPriority = (typeof NTFY_PRIORITIES)[number];
+
+export interface NtfyChannelConfig {
+  enabled: boolean;
+  serverUrl: string;
+  topic: string;
+  tokenEnv?: string;
+  username?: string;
+  passwordEnv?: string;
+  defaultTags: string[];
+  defaultClick?: string;
+  timeoutMs: number;
+}
+
+export interface NtfyNotificationInput {
+  title: string;
+  body: string;
+  priority?: NtfyPriority;
+  tags?: string[];
+  click?: string;
+}
+
+export interface NtfyNotificationResult {
+  ok: true;
+  status: number;
+  channelId: string;
+  topic: string;
+  priority: NtfyPriority;
+  tags: string[];
+}
+
+const DEFAULT_NTFY_CONFIG: NtfyChannelConfig = {
+  enabled: false,
+  serverUrl: "",
+  topic: "",
+  defaultTags: [],
+  timeoutMs: 15_000,
+};
+
+const NTFY_CONFIG_SCHEMA: JSONSchema = {
+  type: "object",
+  additionalProperties: false,
+  required: ["enabled", "serverUrl", "topic"],
+  properties: {
+    enabled: { type: "boolean" },
+    serverUrl: { type: "string", maxLength: 500 },
+    topic: { type: "string", maxLength: 200 },
+    tokenEnv: { type: "string", maxLength: 120, pattern: "^[A-Z0-9_]+$" },
+    username: { type: "string", maxLength: 120 },
+    passwordEnv: { type: "string", maxLength: 120, pattern: "^[A-Z0-9_]+$" },
+    defaultTags: { type: "array", maxItems: 20, items: { type: "string", maxLength: 40 } },
+    defaultClick: { type: "string", maxLength: 1000 },
+    timeoutMs: { type: "integer", minimum: 1_000, maximum: 60_000 },
+  },
+};
+
+registerChannelTypeSetting({
+  type: NTFY_CONFIG_KEY,
+  schema: NTFY_CONFIG_SCHEMA,
+  defaults: DEFAULT_NTFY_CONFIG,
+  display: {
+    label: "ntfy outgoing channel",
+    description: "Configures the built-in egress-only ntfy notification channel.",
+    order: 121,
+  },
+});
+
+export function getNtfyConfig(): NtfyChannelConfig {
+  const stored = getConfig<Partial<NtfyChannelConfig>>(NTFY_CONFIG_NAMESPACE, NTFY_CONFIG_KEY)?.value;
+  return normalizeNtfyConfig({ ...DEFAULT_NTFY_CONFIG, ...(isRecord(stored) ? stored : {}) });
+}
+
+export async function sendNtfyNotification(input: NtfyNotificationInput): Promise<NtfyNotificationResult> {
+  const config = getNtfyConfig();
+  validateSendConfig(config);
+  const title = cleanString(input.title, "title", { required: true, max: 200 })!;
+  const body = cleanString(input.body, "body", { required: true, max: MAX_BODY_BYTES })!;
+  const priority = input.priority ?? "default";
+  const tags = [...new Set([...(config.defaultTags ?? []), ...(input.tags ?? [])].map((tag) => tag.trim()).filter(Boolean))];
+  const click = cleanString(input.click ?? config.defaultClick, "click", { max: 1000 });
+  const url = ntfyTopicUrl(config);
+  const headers = ntfyHeaders({ config, title, priority, tags, click });
+
+  ensureNtfyChannelRegistered(config);
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), config.timeoutMs);
+  let response: Response;
+  try {
+    response = await fetch(url, {
+      method: "POST",
+      headers,
+      body,
+      signal: controller.signal,
+    });
+  } catch (error) {
+    if (controller.signal.aborted) throw new Error(`ntfy notify timed out after ${config.timeoutMs}ms`);
+    throw new Error(`ntfy notify failed: ${errMessage(error)}`);
+  } finally {
+    clearTimeout(timeout);
+  }
+
+  if (!response.ok) {
+    const text = await response.text().catch(() => "");
+    throw new Error(`ntfy notify failed: HTTP ${response.status}${text ? `: ${text.slice(0, 300)}` : ""}`);
+  }
+
+  return { ok: true, status: response.status, channelId: NTFY_CHANNEL_ID, topic: config.topic, priority, tags };
+}
+
+function normalizeNtfyConfig(value: unknown): NtfyChannelConfig {
+  if (!isRecord(value)) throw new ValidationError("ntfy channel config must be an object");
+  const timeoutMs = value.timeoutMs === undefined ? DEFAULT_NTFY_CONFIG.timeoutMs : value.timeoutMs;
+  if (typeof value.enabled !== "boolean") throw new ValidationError("ntfy.enabled must be a boolean");
+  if (typeof timeoutMs !== "number" || !Number.isSafeInteger(timeoutMs) || timeoutMs < 1_000 || timeoutMs > 60_000) {
+    throw new ValidationError("ntfy.timeoutMs must be an integer between 1000 and 60000");
+  }
+  return {
+    enabled: value.enabled,
+    serverUrl: cleanString(value.serverUrl, "ntfy.serverUrl", { max: 500 }) ?? "",
+    topic: cleanString(value.topic, "ntfy.topic", { max: 200 }) ?? "",
+    tokenEnv: cleanString(value.tokenEnv, "ntfy.tokenEnv", { max: 120 }),
+    username: cleanString(value.username, "ntfy.username", { max: 120 }),
+    passwordEnv: cleanString(value.passwordEnv, "ntfy.passwordEnv", { max: 120 }),
+    defaultTags: cleanStringArray(value.defaultTags, "ntfy.defaultTags", { itemMax: 40, maxItems: 20 }) ?? [],
+    defaultClick: cleanString(value.defaultClick, "ntfy.defaultClick", { max: 1000 }),
+    timeoutMs,
+  };
+}
+
+function validateSendConfig(config: NtfyChannelConfig): void {
+  if (!config.enabled) throw new ValidationError("ntfy channel is disabled");
+  if (!config.serverUrl || !config.topic) {
+    throw new ValidationError("ntfy channel requires channel-type/ntfy serverUrl and topic");
+  }
+  if (config.topic.includes("/") || config.topic.includes("?") || config.topic.includes("#")) {
+    throw new ValidationError("ntfy.topic must be a single topic name, not a path or URL");
+  }
+  if (config.tokenEnv && (config.username || config.passwordEnv)) {
+    throw new ValidationError("ntfy auth must use tokenEnv or username/passwordEnv, not both");
+  }
+  if (config.passwordEnv && !config.username) throw new ValidationError("ntfy.username required when passwordEnv is set");
+}
+
+function ntfyTopicUrl(config: NtfyChannelConfig): string {
+  let base: URL;
+  try {
+    base = new URL(config.serverUrl.endsWith("/") ? config.serverUrl : `${config.serverUrl}/`);
+  } catch {
+    throw new ValidationError("ntfy.serverUrl must be a valid URL");
+  }
+  return new URL(encodeURIComponent(config.topic), base).toString();
+}
+
+function ntfyHeaders(input: {
+  config: NtfyChannelConfig;
+  title: string;
+  priority: NtfyPriority;
+  tags: string[];
+  click?: string;
+}): Record<string, string> {
+  const headers: Record<string, string> = {
+    Title: input.title,
+    Priority: String(priorityValue(input.priority)),
+  };
+  if (input.tags.length) headers.Tags = input.tags.join(",");
+  if (input.click) headers.Click = input.click;
+  const authorization = authHeader(input.config);
+  if (authorization) headers.Authorization = authorization;
+  return headers;
+}
+
+function authHeader(config: NtfyChannelConfig): string | undefined {
+  if (config.tokenEnv) {
+    const token = cleanString(process.env[config.tokenEnv], config.tokenEnv, { required: true, max: 4_000 })!;
+    return `Bearer ${token}`;
+  }
+  if (config.username && config.passwordEnv) {
+    const password = cleanString(process.env[config.passwordEnv], config.passwordEnv, { required: true, max: 4_000 })!;
+    return `Basic ${Buffer.from(`${config.username}:${password}`).toString("base64")}`;
+  }
+  return undefined;
+}
+
+function priorityValue(priority: NtfyPriority): number {
+  return NTFY_PRIORITIES.indexOf(priority) + 1;
+}
+
+function ensureNtfyChannelRegistered(config: NtfyChannelConfig): void {
+  upsertAgent({
+    id: NTFY_CHANNEL_ID,
+    name: "ntfy",
+    kind: "channel",
+    tags: ["channel", "ntfy", "channel:ntfy"],
+    capabilities: ["channel", "ntfy", "notify"],
+    status: "online",
+    ready: true,
+    meta: {
+      kind: "channel",
+      provider: "ntfy",
+      accountId: "default",
+      channelType: "ntfy",
+      transport: "ntfy",
+      direction: "outbound",
+      displayName: "ntfy",
+      topicChannels: [config.topic],
+    },
+  });
+}

package/src/quota-advisory.ts

@@ -29,6 +29,7 @@ type AgentRecipientRow = {
 };
 
 const RESET_ROUNDING_PERCENT = 99;
+const QUOTA_THRESHOLD_EPSILON = 1e-9;
 
 export function installQuotaAdvisoryHandler(): void {
   setProviderQuotaAdvisoryHandler(handleProviderQuotaAdvisory);
@@ -62,14 +63,14 @@ function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string
       continue;
     }
 
-    if (remaining > config.warnRemaining) {
+    if (remaining - config.warnRemaining > QUOTA_THRESHOLD_EPSILON) {
       if (previous) delete nextState[window.name];
       continue;
     }
 
     const band = quotaRemainingBand(remaining, config.bandSize);
     if (previous?.lastQuotaAdvisoryBand === band) continue;
-    const stage: QuotaAdvisoryStage = remaining <= config.criticalRemaining ? "critical" : "warn";
+    const stage: QuotaAdvisoryStage = remaining - config.criticalRemaining <= QUOTA_THRESHOLD_EPSILON ? "critical" : "warn";
     notifyQuotaAdvisory({
       provider: input.provider,
       accountKey: input.accountKey,

package/src/quota-threshold-config.ts

@@ -4,7 +4,7 @@ import type { NotificationsConfig } from "./types";
 
 export const DEFAULT_QUOTA_THRESHOLD_CONFIG: NotificationsConfig["quotaThreshold"] = {
   enabled: true,
-  warnRemaining: 0.25,
+  warnRemaining: 0.15,
   criticalRemaining: 0.1,
   resetRemaining: 0.99,
   bandSize: 0.05,

package/src/routes/index.ts

@@ -14,7 +14,7 @@ import { deleteInboxDraftRoute, getChatHistoryImportsRoute, getInboxStateRoute,
 import { deleteMemoryRoute, getMemories, getMemoryBrokerInfo, getMemoryById, getMemoryStats, patchMemory, postMemory, postMemoryInject, postMemorySearch } from "./memory";
 import { deleteMessageById, getCursorRoute, getMessageById, getMessageDeliveryRoute, getMessageStatusRoute, getMessageThread, getMessages, getQueuedMessagesRoute, patchMessage, postClaimMessage, postMessage, postMessageDeliveryAction, postMessageDeliveryAttempt, postMessageReaction, postRenewMessageClaim, postSystemBroadcast } from "./messages";
 import { deleteOrchestratorById, getOrchestratorById, getOrchestrators, patchOrchestratorAgents, postOrchestrator, postOrchestratorAction, postOrchestratorHeartbeat } from "./orchestrator";
-import { deleteSpawnPolicyRoute, getSpawnPoliciesHealth, getSpawnPoliciesRoute, getSpawnPolicyHealth, getSpawnPolicyRoute, getSpawnPolicyStatus, postSpawnPolicyRestart, postSpawnPolicyStart, postSpawnPolicyStop, putSpawnPolicyRoute } from "./spawn-policy";
+import { deleteSpawnPolicyRoute, getSpawnPoliciesHealth, getSpawnPoliciesRoute, getSpawnPolicyDiagnostics, getSpawnPolicyHealth, getSpawnPolicyRoute, getSpawnPolicyStatus, postSpawnPolicyRestart, postSpawnPolicyStart, postSpawnPolicyStop, putSpawnPolicyRoute } from "./spawn-policy";
 import { deleteTokenProfileRoute, getTokenById, getTokenMe, getTokenProfiles, getTokens, patchTokenProfile, postIntegrationRuntimeToken, postInteractiveRunnerRuntimeToken, postMcpRuntimeToken, postOrchestratorRunnerToken, postRuntimeTokenRenew, postToken, postTokenProfile, postTokenRevoke } from "./tokens";
 import { deleteWorkspaceById, getWorkspaceById, getWorkspaceDiagnostics, getWorkspaceDiff, getWorkspaceGitState, getWorkspaceMergePreview, getWorkspaceOrphans, getWorkspaceRecoveryBranches, getWorkspaceStewards, getWorkspaces, postWorkspaceAction, postWorkspaceCleanupStale, postWorkspaceOrphanReclaim, postWorkspaceRecoveryBranchDiscard } from "./workspaces";
 import { error, type Handler } from "./_shared";
@@ -259,6 +259,7 @@ const routes: Route[] = [
   route("GET", "/api/spawn-policies/health", getSpawnPoliciesHealth),
   route("GET", "/api/spawn-policy/:name/status", getSpawnPolicyStatus),
   route("GET", "/api/spawn-policy/:name/health", getSpawnPolicyHealth),
+  route("GET", "/api/spawn-policy/:name/diagnostics", getSpawnPolicyDiagnostics),
   route("GET", "/api/providers", getProvidersRoute),
   route("GET", "/api/providers/config", getProviderConfigsRoute),
   route("GET", "/api/providers/quota-config", getProviderQuotaConfigsRoute),

package/src/routes/spawn-policy.ts

@@ -1,6 +1,6 @@
 // Auto-split from routes.ts (#299). Domain: spawn-policy.
-import { ValidationError, getOrchestrator } from "../db";
-import { auditEvent, authAuditMetadata, dashboardAttribution, emitCommand, error, json, parseBody, spawnRequestId, type Handler } from "./_shared";
+import { ValidationError, getCrashReport, getManagedPolicyTimeline, getOrchestrator } from "../db";
+import { auditEvent, authAuditMetadata, dashboardAttribution, emitCommand, error, json, parseBody, parseQueryInt, spawnRequestId, type Handler } from "./_shared";
 import { buildManagedSpawnParams } from "../managed-policy";
 import { cleanString } from "../validation";
 import { createCommand } from "../commands-db";
@@ -229,3 +229,20 @@ export const getSpawnPolicyHealth: Handler = (_req, params) => {
   const entry = getSpawnPolicy(params.name!);
   return entry ? json(policyStatusPayload(entry.value)) : error("spawn policy not found", 404);
 };
+
+export const getSpawnPolicyDiagnostics: Handler = (req, params) => {
+  const entry = getSpawnPolicy(params.name!);
+  if (!entry) return error("spawn policy not found", 404);
+  const url = new URL(req.url);
+  const limitRaw = parseQueryInt(url.searchParams.get("limit"), { min: 1, max: 200, clamp: true });
+  if (Number.isNaN(limitRaw)) return error("limit must be an integer between 1 and 200");
+  const sinceRaw = parseQueryInt(url.searchParams.get("since"), { min: 0, max: Number.MAX_SAFE_INTEGER });
+  if (Number.isNaN(sinceRaw)) return error("since must be a non-negative integer");
+  const status = policyStatusPayload(entry.value);
+  const agentId = typeof status.state.agentId === "string" ? status.state.agentId : undefined;
+  return json({
+    ...status,
+    timeline: getManagedPolicyTimeline(entry.value.name, { limit: limitRaw ?? 100, since: sinceRaw ?? undefined }),
+    crashReport: agentId ? getCrashReport(agentId) : null,
+  });
+};

package/src/team-idle-tracker.ts

@@ -26,3 +26,22 @@ export interface TeamMemberWatch {
 
 // teamId → (memberId → watch)
 export const teamMemberWatchTracker = new Map<string, Map<string, TeamMemberWatch>>();
+
+// #412/#468 — per-team COORDINATOR liveness watch. The member watch above deliberately skips the
+// coordinator (it's the ping recipient, not a watched worker), so a coordinator that dies silently
+// is invisible to it. This tracker remembers the coordinator's last-alive observation so a later
+// silence escalates ONCE to the human/steward fallback instead of dead-ending — a coordinator-gone
+// team must never be silently abandoned.
+export interface TeamCoordinatorWatch {
+  /** Identity of the coordinator this watch is for; a re-elected/replaced coordinator resets it. */
+  coordinatorId: string;
+  /** Most recent time the coordinator was seen alive (non-offline, fresh heartbeat). */
+  lastActiveAt: number;
+  /** Whether the coordinator's CURRENT gone-spell already escalated (re-armed when it recovers). */
+  escalated: boolean;
+  /** Coordinator label for a readable escalation. */
+  label?: string;
+}
+
+// teamId → coordinator watch
+export const teamCoordinatorWatchTracker = new Map<string, TeamCoordinatorWatch>();