agent-relay-server 0.106.0 → 0.108.0

package/src/services/task-semantic-events.ts

@@ -0,0 +1,137 @@
+import { getAgent } from "../db";
+import { emitRelayEvent } from "../events";
+import type { NotificationScope, NotificationType } from "../notification-types";
+import { routeNotification } from "../notification-router";
+import { resolveAudience } from "../notification-rules";
+import type { Task, TaskBlocker, TaskStatus } from "../types";
+
+export type TaskSemanticTransition = "created" | "started" | "blocked" | "needs-decision" | "landed" | "closed";
+
+const TASK_NOTIFICATION_TYPE: Record<TaskSemanticTransition, NotificationType> = {
+  created: "task.created",
+  started: "task.started",
+  blocked: "task.blocked",
+  "needs-decision": "task.needs_decision",
+  landed: "task.landed",
+  closed: "task.closed",
+};
+
+export function taskTransitionForStatus(status: TaskStatus, blockers: TaskBlocker[] | undefined): TaskSemanticTransition | null {
+  if (status === "claimed" || status === "in_progress") return "started";
+  if (status === "blocked") {
+    return blockers?.some((blocker) => blocker.type === "decision") ? "needs-decision" : "blocked";
+  }
+  if (status === "done" || status === "failed" || status === "canceled") return "closed";
+  return null;
+}
+
+export interface TaskSemanticNotificationInput {
+  task: Task;
+  transition: TaskSemanticTransition;
+  actor?: string;
+  recipients?: string[];
+  workspaceId?: string;
+  branch?: string;
+  sha?: string;
+  landKey?: string;
+  occurrenceKey?: string;
+  outcome?: Record<string, unknown>;
+}
+
+export function emitTaskSemanticNotification(input: TaskSemanticNotificationInput): void {
+  const type = TASK_NOTIFICATION_TYPE[input.transition];
+  const scope = taskNotificationScope(input);
+  const semanticEvent = `task.${input.transition}`;
+  const eventData = {
+    taskId: input.task.id,
+    transition: input.transition,
+    semanticEvent,
+    title: input.task.title,
+    status: input.task.status,
+    deliverableStage: input.task.deliverableStage,
+    ...(input.workspaceId ? { workspaceId: input.workspaceId } : {}),
+    ...(input.branch ? { branch: input.branch } : {}),
+    ...(input.sha ? { sha: input.sha } : {}),
+    ...(input.landKey ? { landKey: input.landKey } : {}),
+    ...(input.outcome ? { outcome: input.outcome } : {}),
+  };
+  emitRelayEvent({
+    type,
+    source: input.actor ?? "server",
+    subject: String(input.task.id),
+    data: eventData as Record<string, unknown>,
+  });
+
+  const subscriptionAudience = resolveAudience(type, scope).map((member) => ({
+    agentId: member.agentId,
+    matchedRule: member.matchedRule,
+  }));
+  const explicit = (input.recipients ?? [])
+    .map((agentId) => agentId.trim())
+    .filter(Boolean)
+    .map((agentId) => ({ agentId, matchedRule: "task-stakeholder" }));
+
+  const recipients = [...subscriptionAudience, ...explicit];
+  if (recipients.length === 0) return;
+
+  routeNotification({
+    type,
+    scope,
+    recipients,
+    ttlMs: null,
+    message: {
+      subject: taskNotificationSubject(input),
+      body: taskNotificationBody(input),
+      payload: {
+        kind: "task.transition",
+        notification: "task-transition",
+        taskId: input.task.id,
+        transition: input.transition,
+        semanticEvent,
+        ...(input.outcome ? { outcome: input.outcome } : {}),
+        ...(input.workspaceId ? { workspaceId: input.workspaceId } : {}),
+        ...(input.branch ? { branch: input.branch } : {}),
+        ...(input.sha ? { sha: input.sha } : {}),
+      },
+      idempotencyKey: taskNotificationIdempotencyKey(input),
+      replyExpected: false,
+    },
+  });
+}
+
+function taskNotificationIdempotencyKey(input: TaskSemanticNotificationInput): string | undefined {
+  if (input.landKey) return `task:${input.task.id}:transition:${input.transition}:${input.landKey}`;
+  if (input.occurrenceKey) return `task:${input.task.id}:transition:${input.transition}:occurrence:${input.occurrenceKey}`;
+  return undefined;
+}
+
+function taskNotificationScope(input: TaskSemanticNotificationInput): NotificationScope {
+  const owner = input.task.ownerAgentId;
+  const ownerAgent = owner ? getAgent(owner) : undefined;
+  const teamId = input.task.teamId ?? ownerAgent?.teamId;
+  return {
+    taskId: String(input.task.id),
+    transition: input.transition,
+    ...(input.landKey ? { landKey: input.landKey } : {}),
+    ...(owner ? { author: owner, agentId: owner } : {}),
+    ...(input.task.createdBy ? { parent: input.task.createdBy } : {}),
+    ...(teamId ? { teamId } : {}),
+    ...(input.workspaceId ? { workspaceId: input.workspaceId } : {}),
+  };
+}
+
+function taskNotificationSubject(input: TaskSemanticNotificationInput): string {
+  return `Task #${input.task.id} ${input.transition}`;
+}
+
+function taskNotificationBody(input: TaskSemanticNotificationInput): string {
+  if (input.transition === "landed") {
+    const bits = [`Task #${input.task.id} landed: ${input.task.title}`];
+    if (input.sha) bits.push(`sha ${input.sha.slice(0, 12)}`);
+    const gatesPassed = input.outcome?.gatesPassed;
+    if (typeof gatesPassed === "boolean") bits.push(gatesPassed ? "gates passed" : "gates not verified");
+    if (input.outcome?.statusAdvanced === true) bits.push("closed");
+    return bits.join("; ");
+  }
+  return `Task #${input.task.id} ${input.transition}: ${input.task.title}`;
+}

package/src/services/transient-agent-reaper.ts

@@ -4,7 +4,52 @@ import { createActivityEvent, getAgent, listAgents, listWorkspaces, recordTermin
 import { routeNotification } from "../notification-router";
 import { isProviderBlocked } from "../provider-state";
 import { TERMINAL_WORKSPACE_STATUSES } from "../workspace-phase";
-import type { WorkspaceRecord } from "../types";
+import type { AgentCard, Command, WorkspaceRecord } from "../types";
+
+// The single home for "dispose a transient/one-shot agent" — the canonical `agent.shutdown`
+// command + its events/terminal-marker/activity. Extracted so the idle-grace scan below AND
+// the #831 on-land reap signal emit the byte-identical disposal (one shutdown shape, not two).
+// Pure emission: the CALLER owns the decision (idle+grace vs landed+gates) and any dedup.
+export interface TransientShutdownOptions {
+  /** Disposal cause stamped on the command, agent.exited event, and terminal marker. */
+  reason: string;
+  /** Audit attribution (the command's requestedBy + the activity row's maintenanceJobId). */
+  requestedBy: string;
+  workspaceId?: string;
+  workspaceStatus?: string;
+  activityTitle: string;
+  activityBody: string;
+  /** Why the land was considered complete (idle-scan reason or "deliverable-landed"). */
+  landReason?: string;
+  now?: number;
+}
+
+export function requestTransientAgentShutdown(agent: AgentCard, opts: TransientShutdownOptions): Command {
+  const now = opts.now ?? Date.now();
+  const command = createCommand({
+    type: "agent.shutdown",
+    source: "system",
+    target: agent.id,
+    params: { action: "shutdown", agentId: agent.id, graceful: true, timeoutMs: 10_000, reason: opts.reason, requestedBy: opts.requestedBy, requestedAt: now },
+  });
+  emitRelayEvent({ type: "command.requested", source: command.source, subject: command.id, data: { command } });
+  emitRelayEvent({ type: "agent.exited", source: "server", subject: agent.id, data: { agentId: agent.id, parent: agent.spawnedBy, reason: opts.reason, workspaceId: opts.workspaceId, workspaceStatus: opts.workspaceStatus } });
+  // #645 — persist a clean terminal marker so the #634 heartbeat-gap watchdog reads this benign
+  // completion as a terminus (not a silent death) once the reaped worker stops heartbeating.
+  recordTerminalEvent({ agentId: agent.id, reason: opts.reason, kind: "agent.exited", epoch: agent.epoch, createdAt: now });
+  createActivityEvent({
+    clientId: `transient-agent-reaped-${agent.id}-${now}`,
+    kind: "state",
+    title: opts.activityTitle,
+    body: opts.activityBody,
+    meta: agent.id,
+    icon: "ti-check",
+    view: "agents",
+    agentId: agent.id,
+    metadata: { source: "server", maintenanceJobId: opts.requestedBy, commandId: command.id, reason: opts.reason, workspaceId: opts.workspaceId, landReason: opts.landReason },
+  });
+  return command;
+}
 
 const envMsOrDefault = (name: string, fallback: number): number => {
   const v = Number(process.env[name]);
@@ -37,6 +82,19 @@ function defaultCommitsAhead(ws: WorkspaceRecord): number | null {
   }
 }
 
+// #800 — A transient agent is protected from reaping ONLY by one of three evidence-based states:
+// it is not idle (still working), it is not ready (mid-startup/teardown), or it carries a REAL
+// provider block. `isProviderBlocked` reads the agent's own `meta.providerState`, which is set
+// exclusively from hard evidence — an HTTP 429 / StopFailure hook, the scraped CLI usage-limit
+// modal, an approval gate, or a model-unavailable hold (see runner rate-limit.ts / control-server).
+// It is NEVER derived from a quota-utilization number. The reaper deliberately does not read quota
+// utilization at all, so a provider merely sitting at high 7d/5h utilization can neither cause nor
+// prevent a reap. This keeps the false "high-util ⇒ blocked ⇒ reaped" chain (#800) impossible by
+// construction: utilization is not an input to this decision.
+function reapProtected(agent: ReturnType<typeof listAgents>[number]): boolean {
+  return agent.status !== "idle" || !agent.ready || isProviderBlocked(agent);
+}
+
 let _commitsAheadFn: (ws: WorkspaceRecord) => number | null = defaultCommitsAhead;
 
 export function overrideCommitsAheadForTests(fn: (ws: WorkspaceRecord) => number | null): void {
@@ -109,7 +167,9 @@ export function reapTransientAgents(): Record<string, unknown> {
     // A held agent (rate-limit hold #286, or any provider-blocked state) reads as
     // idle+ready but is NOT done — reaping it would kill work that's just waiting
     // for the limit to reset. Treat it as not-idle; the clock restarts on resume.
-    if (agent.status !== "idle" || !agent.ready || isProviderBlocked(agent)) { transientTracker.delete(agent.id); continue; }
+    // #800 — `reapProtected` is the single home for that decision: it honors only
+    // evidence-based block states, never a high quota-utilization number.
+    if (reapProtected(agent)) { transientTracker.delete(agent.id); continue; }
     const entry = transientTracker.get(agent.id) ?? { firstIdleAt: now };
     transientTracker.set(agent.id, entry);
     const idleForMs = now - entry.firstIdleAt;
@@ -127,27 +187,15 @@ export function reapTransientAgents(): Record<string, unknown> {
     }
     if (entry.lastReapAt) continue;
     entry.lastReapAt = now;
-    const command = createCommand({
-      type: "agent.shutdown",
-      source: "system",
-      target: agent.id,
-      params: { action: "shutdown", agentId: agent.id, graceful: true, timeoutMs: 10_000, reason: "transient-complete", requestedBy: "transient-agent-reaper", requestedAt: now },
-    });
-    emitRelayEvent({ type: "command.requested", source: command.source, subject: command.id, data: { command } });
-    emitRelayEvent({ type: "agent.exited", source: "server", subject: agent.id, data: { agentId: agent.id, parent: agent.spawnedBy, reason: "transient-complete", workspaceId: land.workspaceId, workspaceStatus: land.status } });
-    // #645 — persist a clean terminal marker so the #634 heartbeat-gap watchdog reads this benign
-    // completion as a terminus (not a silent death) once the reaped worker stops heartbeating.
-    recordTerminalEvent({ agentId: agent.id, reason: "transient-complete", kind: "agent.exited", epoch: agent.epoch, createdAt: now });
-    createActivityEvent({
-      clientId: `transient-agent-reaped-${agent.id}-${now}`,
-      kind: "state",
-      title: "Transient agent complete",
-      body: "Agent idle grace elapsed and branch work is land-complete; shutdown requested",
-      meta: agent.id,
-      icon: "ti-check",
-      view: "agents",
-      agentId: agent.id,
-      metadata: { source: "server", maintenanceJobId: "transient-agent-reaper", commandId: command.id, reason: "transient-complete", workspaceId: land.workspaceId, landReason: land.reason },
+    requestTransientAgentShutdown(agent, {
+      reason: "transient-complete",
+      requestedBy: "transient-agent-reaper",
+      workspaceId: land.workspaceId,
+      workspaceStatus: land.status,
+      activityTitle: "Transient agent complete",
+      activityBody: "Agent idle grace elapsed and branch work is land-complete; shutdown requested",
+      landReason: land.reason,
+      now,
     });
     reaped.push(agent.id);
   }

package/src/spawn-targets.ts

@@ -206,6 +206,29 @@ function quotaAdvisoryFor(
   };
 }
 
+// #839 — compact per-provider quota band view for the task dispatcher. Projects the same
+// advisory the spawn matrix surfaces (band / prefer / blockedNow) down to the fields the
+// dispatcher needs to protect scarce pools and load-shift onto abundant ones. Returns one
+// entry per provider that has quota records; providers without records are simply absent
+// (the dispatcher then treats them as band-unknown = "normal").
+export interface ProviderQuotaBand {
+  provider: SpawnProvider;
+  band: QuotaBand;
+  prefer: boolean;
+  blockedNow: boolean;
+  recommendedAction: string;
+}
+
+export function providerQuotaBands(now = Date.now()): ProviderQuotaBand[] {
+  return [...providerQuotaAdvisories(now).entries()].map(([provider, advisory]) => ({
+    provider,
+    band: advisory.quotaPenaltyBand,
+    prefer: advisory.prefer,
+    blockedNow: advisory.blockedNow,
+    recommendedAction: advisory.recommendedAction,
+  }));
+}
+
 function quotaWindowsFor(record: ProviderQuotaRecord, now: number, providerCfg?: ProviderQuotaRoutingConfig): SpawnQuotaBurnWindow[] {
   const quotaByName = new Map((record.quota?.windows ?? []).map((window) => [window.name, window]));
   return (record.burn?.windows ?? [])
@@ -381,9 +404,11 @@ export function quotaBandFromSignal(
 function recommendedAction(input: { quotaPenaltyBand: SpawnQuotaBurnBand; blockedNow: boolean; prefer: boolean; slack: number | null; windows: SpawnQuotaBurnWindow[] }): string {
   if (input.blockedNow) {
     const hasSevenDayBlock = input.windows.some((window) => window.role === "7d" && window.blockedNow);
+    // #827 — "at capacity" (util ≥ hardBackstop) is a routing-avoid advisory, not an evidence-based
+    // rate-limit hold. Use distinct wording so operators don't confuse it with an HTTP 429 block.
     return hasSevenDayBlock
-      ? "Quota is blocked now; switch provider or escalate unless this provider was explicitly requested."
-      : "5h quota is blocked now; wait for reset or use another provider unless the job is urgent.";
+      ? "Quota at capacity (routing-avoid); switch provider or wait for reset unless this provider was explicitly requested."
+      : "5h quota at capacity (routing-avoid); wait for reset or use another provider unless the job is urgent.";
   }
   return windowRecommendedAction(input.quotaPenaltyBand, input.slack);
 }