agent-relay-server 0.118.4 → 0.119.0

package/docs/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Agent Relay API",
-    "version": "0.118.4",
+    "version": "0.119.0",
     "description": "Real-time message bus for inter-agent communication. Agent-first: this spec is designed for machine consumption — agents can self-discover the full API surface via GET /api/spec.",
     "license": {
       "name": "MIT",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-server",
-  "version": "0.118.4",
+  "version": "0.119.0",
   "description": "Lightweight HTTP message relay for inter-agent communication across machines",
   "module": "src/index.ts",
   "type": "module",
@@ -36,9 +36,9 @@
     "CONTRIBUTING.md"
   ],
   "dependencies": {
-    "agent-relay-channels-host": "0.118.4",
+    "agent-relay-channels-host": "0.119.0",
     "agent-relay-providers": "0.104.1",
-    "agent-relay-sdk": "0.2.101",
+    "agent-relay-sdk": "0.2.102",
     "ajv": "^8.20.0"
   },
   "scripts": {

package/runner/plugins/claude/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "agent-relay-runner",
   "description": "Thin Agent Relay runner bridge for Claude Code",
-  "version": "0.118.4",
+  "version": "0.119.0",
   "agentRelayContracts": {
     "providerPluginProtocol": 1
   }

package/runner/plugins/claude/monitors/relay-monitor.provisioned.mjs

@@ -848,6 +848,61 @@ import { promisify } from "util";
 var execFileAsync = promisify(execFile);
 // sdk/src/teams-config.ts
 var TEAM_IDLE_THRESHOLD_MINUTES_MAX = 24 * 60;
+// sdk/src/admission/semaphore.ts
+class Semaphore {
+  #limit;
+  #inFlight = 0;
+  #waiters = [];
+  constructor(limit) {
+    if (!Number.isInteger(limit) || limit < 1) {
+      throw new RangeError(`Semaphore limit must be an integer >= 1, got ${limit}`);
+    }
+    this.#limit = limit;
+  }
+  get limit() {
+    return this.#limit;
+  }
+  get inFlight() {
+    return this.#inFlight;
+  }
+  get waiting() {
+    return this.#waiters.length;
+  }
+  get available() {
+    return this.#limit - this.#inFlight;
+  }
+  tryAcquire() {
+    if (this.#inFlight < this.#limit) {
+      this.#inFlight++;
+      return true;
+    }
+    return false;
+  }
+  acquire() {
+    if (this.tryAcquire())
+      return Promise.resolve();
+    return new Promise((resolve) => {
+      this.#waiters.push(resolve);
+    });
+  }
+  release() {
+    const next = this.#waiters.shift();
+    if (next) {
+      next();
+      return;
+    }
+    if (this.#inFlight > 0)
+      this.#inFlight--;
+  }
+  async run(fn) {
+    await this.acquire();
+    try {
+      return await fn();
+    } finally {
+      this.release();
+    }
+  }
+}
 // runner/src/config.ts
 function messageBodyMaxCharsFromEnv() {
   return process.env.AGENT_RELAY_MESSAGE_BODY_MAX_CHARS;

package/src/commands-db.ts

@@ -27,6 +27,36 @@ interface CommandFilters {
 }
 
 const ACTIVE_STATUSES: CommandStatus[] = ["pending", "accepted", "running"];
+const TERMINAL_STATUSES: CommandStatus[] = ["succeeded", "failed", "canceled", "timed_out"];
+
+// #930 — an `agent.spawn` command carries a hard TTL from CREATION (see
+// defaultExpiresAt). The #880 concurrency cap introduces REAL queuing, so a
+// legitimately-queued spawn can sit `accepted`/`running` in the orchestrator past
+// that create-time TTL and be swept to `timed_out` while the orchestrator still
+// intends to run it. Refresh the TTL from dispatch-intent (claim → accepted, or
+// slot-acquire → running) so the window measures from when the host committed to
+// the spawn, not from enqueue. Scoped to `agent.spawn`; every other command type
+// keeps create-time TTL semantics.
+const SPAWN_TTL_MS = 180_000;
+const SPAWN_DISPATCH_STATUSES: CommandStatus[] = ["accepted", "running"];
+
+function isTerminalStatus(status: CommandStatus): boolean {
+  return TERMINAL_STATUSES.includes(status);
+}
+
+// #930 — the two terminal→different transitions that ARE legitimate (verified by
+// checking every updateCommand caller). Everything else out of a terminal state is
+// state churn to be refused (esp. a backgrounded spawn's late `succeeded` racing the
+// TTL sweep: `timed_out`→`succeeded`).
+//   1. The spawn-registration watchdog corrects a premature spawn `succeeded` to
+//      `failed` when the agent never registered (src/services/spawn-registration-watchdog.ts).
+//   2. The upgrade reconcile flips a timed-out upgrade back to `succeeded` when the
+//      orchestrator eventually reports the desired version (src/routes/orchestrator.ts).
+function isLegitimateTerminalCorrection(type: string, from: CommandStatus, to: CommandStatus): boolean {
+  if (type === "agent.spawn" && from === "succeeded" && to === "failed") return true;
+  if (type === "orchestrator.upgrade" && from === "failed" && to === "succeeded") return true;
+  return false;
+}
 
 export function createCommand(input: CreateCommandInput): Command {
   const now = Date.now();
@@ -97,16 +127,41 @@ export function updateCommand(id: string, input: UpdateCommandInput): Command |
   const existing = getCommand(id);
   if (!existing) return null;
   const nextStatus = input.status ?? existing.status;
+  // #930 — terminal-state guard. A command already in a terminal state must not be
+  // silently transitioned to a DIFFERENT status. The concrete hazard: the backgrounded
+  // spawn dispatch (#880 cap) can call updateCommand("succeeded") AFTER expireCommands
+  // has already swept the row to `timed_out`, silently flipping timed_out→succeeded —
+  // state churn plus double-spawn risk if the requestor already reacted to the timeout.
+  // Idempotent terminal→same-terminal is fine; the only terminal→different transitions
+  // allowed are the checked-caller corrections in isLegitimateTerminalCorrection. Any
+  // other is an observable no-op (returns the row unchanged, logs) rather than a write.
+  if (
+    input.status &&
+    input.status !== existing.status &&
+    isTerminalStatus(existing.status) &&
+    !isLegitimateTerminalCorrection(existing.type, existing.status, input.status)
+  ) {
+    console.error(`[commands-db] refused terminal transition ${existing.status}→${input.status} for ${existing.type} command ${id}`);
+    return existing;
+  }
   const now = Date.now();
+  // #930 — refresh the spawn TTL from dispatch-intent when the command is claimed or
+  // dispatched (see SPAWN_DISPATCH_STATUSES). Other command types keep their existing
+  // expiry untouched.
+  const nextExpiresAt =
+    existing.type === "agent.spawn" && input.status && SPAWN_DISPATCH_STATUSES.includes(input.status)
+      ? now + SPAWN_TTL_MS
+      : existing.expiresAt ?? null;
   getDb().query(`
     UPDATE commands
-    SET status = ?, result = ?, error = ?, updated_at = ?
+    SET status = ?, result = ?, error = ?, updated_at = ?, expires_at = ?
     WHERE id = ?
   `).run(
     nextStatus,
     input.result !== undefined ? JSON.stringify(input.result) : existing.result ? JSON.stringify(existing.result) : null,
     input.error ?? existing.error ?? null,
     now,
+    nextExpiresAt,
     id,
   );
   const updated = getCommand(id);
@@ -131,7 +186,7 @@ export function expireCommands(now: number = Date.now()): Command[] {
 }
 
 function defaultExpiresAt(type: string, now: number): number | undefined {
-  if (type === "agent.spawn") return now + 180_000;
+  if (type === "agent.spawn") return now + SPAWN_TTL_MS;
   if (type === "agent.shutdown" || type === "agent.restart" || type === "agent.reconnect" || type === "agent.kill" || type === "agent.compact" || type === "agent.clearContext" || type === "agent.injectContext") return now + 30_000;
   return undefined;
 }

package/src/notification-rules.ts

@@ -255,6 +255,7 @@ const SPAWN_LIFECYCLE_AUDIENCE_TYPES = [
   "agent.exited",
   "agent.spawn_failed",
   "agent.spawn_recovered",
+  "agent.transient_empty_reap",
 ] as const satisfies readonly NotificationType[];
 
 for (const type of SPAWN_LIFECYCLE_AUDIENCE_TYPES) {

package/src/notification-settings.ts

@@ -129,7 +129,7 @@ export function notificationTypeEnabled(config: NotificationsConfig, type: Notif
 function notificationToggleForType(type: NotificationType): NotificationToggleKey | null {
   if (type === "branch.landed" || type === "landed-failure") return "branchLanded";
   if (type === "agent.ready") return "agentReady";
-  if (type === "agent.exited") return "agentExited";
+  if (type === "agent.exited" || type === "agent.transient_empty_reap") return "agentExited";
   if (type === "agent.spawn_failed") return "agentSpawnFailed";
   return null;
 }

package/src/notification-types.ts

@@ -22,6 +22,7 @@ export type NotificationType =
   | "agent.exited"
   | "agent.spawn_failed"
   | "agent.spawn_recovered"
+  | "agent.transient_empty_reap"
   | "agent.context_threshold"
   | "agent.stuck_busy"
   | "agent.quota_threshold"
@@ -120,6 +121,7 @@ export const DEFAULT_TTL_MS: Record<NotificationType, number | null> = {
   "agent.exited": null,
   "agent.spawn_failed": null,
   "agent.spawn_recovered": null,
+  "agent.transient_empty_reap": null,
   "agent.context_threshold": 15 * MINUTES,
   "agent.stuck_busy": null,
   "agent.quota_threshold": 15 * MINUTES,

package/src/services/task-context-hydration.ts

@@ -338,7 +338,9 @@ function joinBlocks(...parts: Array<string | undefined>): string | undefined {
  * single instruction source. Caller system append is preserved unless it already embeds the
  * hydrated brief, which avoids delivering the same brief through both provider channels.
  * The steer used as a first-message kick is sentinel-stripped so it cannot forge the data fence
- * in the (instruction-bearing) systemPromptAppend brief.
+ * in the (instruction-bearing) systemPromptAppend brief. #927 — for the append-system-prompt
+ * channel a spawn with no steer falls back to submitting the hydrated brief as turn 1, so the
+ * worker is never left with an empty prompt (which left task-bound Claude spawns idling).
  */
 export function projectTaskHydration(
   provider: SpawnProvider | string,
@@ -352,7 +354,10 @@ export function projectTaskHydration(
   if (taskHydrationChannel(provider) === "append-system-prompt") {
     return {
       systemPromptAppend: joinBlocks(context, callerAppend),
-      prompt: joinBlocks(steer ? stripSentinels(steer) : undefined),
+      // #927 — always submit a turn-1 message. Steer drives it when present (the brief lives in
+      // systemPromptAppend as durable context); with NO steer, fall back to submitting the brief
+      // itself so the spawn isn't left with an empty prompt and idling forever.
+      prompt: joinBlocks(steer ? stripSentinels(steer) : context),
     };
   }
   return {

package/src/services/task-entity.ts

@@ -9,6 +9,7 @@ import {
   createDeliverableTask,
   deliverableTaskIdsByOwnerAgent,
   deliverableTaskIdsBySpawnRequestId,
+  getAgent,
   getTaskDetail,
   listDeliverableTaskSummaries,
   listDeliverableTasks,
@@ -34,6 +35,8 @@ import {
 import { emitTaskChanged } from "../sse";
 import type { Task, TaskDetail, TaskEvent, TaskGateState, TaskListItem } from "../types";
 import { removeSubscriptionsForTask } from "../db/notification-subscriptions";
+import { emitRelayEvent } from "../events";
+import { routeNotification } from "../notification-router";
 import { isAgentExitDoneWhen, stageSatisfiesDoneWhen } from "./task-done-when";
 import { autoSubscribeTaskCoordinator, emitTaskSemanticNotification, taskTransitionForStage, taskTransitionForStatus } from "./task-semantic-events";
 
@@ -187,11 +190,11 @@ export function taskEntityHistory(id: number): TaskEvent[] {
   return listTaskEvents(id);
 }
 
-// #889 — finalize bound tasks for an reaped agent whose doneWhen is "agent-exit" / "reaped".
+// #889 — finalize bound tasks for a reaped agent whose doneWhen is "agent-exit" / "reaped".
 // Called by the transient reaper after requesting shutdown so ephemeral run-and-exit workers
 // don't leave their auto-scaffolded Task stuck in_progress forever.
 // Never overrides an already-terminal status. Idempotent (reapedAt guard inside markTaskAgentReaped).
-export function finalizeEphemeralTasksOnReap(agentId: string, opts: { spawnRequestId?: string; reason?: string } = {}): number[] {
+export function finalizeEphemeralTasksOnReap(agentId: string, opts: { spawnRequestId?: string; reason?: string; producedWork?: boolean; evidence?: string } = {}): number[] {
   const ids = new Set<number>();
   for (const id of deliverableTaskIdsByOwnerAgent(agentId)) ids.add(id);
   if (opts.spawnRequestId) {
@@ -203,6 +206,15 @@ export function finalizeEphemeralTasksOnReap(agentId: string, opts: { spawnReque
     if (!task) continue;
     if (task.status === "done" || task.status === "failed" || task.status === "canceled") continue;
     if (!isAgentExitDoneWhen(task.doneWhen)) continue;
+    if (opts.producedWork === false) {
+      try {
+        setTaskEntityStatus(taskId, { status: "open", note: "worker reaped without producing work; task reopened for retry", actor: "task-lifecycle" });
+        notifyEmptyTransientReap(agentId, task, opts);
+      } catch {
+        // best-effort; failures must not surface into the reaper scan
+      }
+      continue;
+    }
     try {
       if (!task.reapedAt) markTaskAgentReaped(taskId, { agentId, reason: opts.reason ?? "agent-exit" });
       setTaskEntityStatus(taskId, { status: "done", note: "doneWhen=agent-exit satisfied by worker reap", actor: "task-lifecycle" });
@@ -213,3 +225,32 @@ export function finalizeEphemeralTasksOnReap(agentId: string, opts: { spawnReque
   }
   return finalized;
 }
+
+function notifyEmptyTransientReap(agentId: string, task: TaskDetail, opts: { spawnRequestId?: string; reason?: string; evidence?: string }): void {
+  const agent = getAgent(agentId);
+  const parent = agent?.spawnedBy;
+  const recipients = [...new Set([parent, task.createdBy].filter((value): value is string => Boolean(value && value.trim())))];
+  const payload = {
+    kind: "agent.transient_empty_reap",
+    agentId,
+    parent,
+    taskId: task.id,
+    taskTitle: task.title,
+    spawnRequestId: opts.spawnRequestId,
+    reason: opts.reason ?? "agent-exit",
+    evidence: opts.evidence ?? "no-produced-work",
+  };
+  emitRelayEvent({ type: "agent.transient_empty_reap", source: "task-lifecycle", subject: agentId, data: payload });
+  routeNotification({
+    type: "agent.transient_empty_reap",
+    scope: { agentId, parent, taskId: String(task.id) },
+    recipients,
+    message: {
+      subject: "Worker reaped without producing work",
+      body: `Worker \`${agentId}\` was reaped without producing work for Task #${task.id} (${task.title}). The task was reopened for retry instead of being marked done.`,
+      payload,
+      idempotencyKey: `task:${task.id}:empty-reap:${agentId}:${opts.reason ?? "agent-exit"}`,
+      replyExpected: false,
+    },
+  });
+}

package/src/services/transient-agent-reaper.ts

@@ -105,24 +105,39 @@ export function resetCommitsAheadForTests(): void {
   _commitsAheadFn = defaultCommitsAhead;
 }
 
-function transientLandComplete(agentId: string): { complete: boolean; workspaceId?: string; status?: string; reason?: string } {
+function completedProviderTurns(agent: AgentCard): number {
+  const value = agent.meta?.completedProviderTurns;
+  return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
+}
+
+function workspaceProducedWork(ws: WorkspaceRecord | undefined): boolean {
+  if (!ws) return false;
+  if (ws.status === "merged") return true;
+  return typeof ws.metadata.mergedAt === "number";
+}
+
+function transientLandComplete(agent: AgentCard): { complete: boolean; producedWork: boolean; workspaceId?: string; status?: string; reason?: string } {
+  const agentId = agent.id;
   const isolated = listWorkspaces({ ownerAgentId: agentId }).filter((ws) => ws.mode === "isolated");
-  if (!isolated.length) return { complete: true, reason: "no-isolated-workspace" };
+  if (!isolated.length) return { complete: true, producedWork: completedProviderTurns(agent) > 0, reason: "no-isolated-workspace" };
   const live = isolated.find((ws) => !TERMINAL_WORKSPACE_STATUSES.has(ws.status));
-  if (!live) return { complete: true, workspaceId: isolated[0]?.id, status: isolated[0]?.status, reason: "terminal-workspace" };
+  if (!live) {
+    const latest = isolated[0];
+    return { complete: true, producedWork: workspaceProducedWork(latest), workspaceId: latest?.id, status: latest?.status, reason: "terminal-workspace" };
+  }
   const mergedAt = typeof live.metadata.mergedAt === "number" ? live.metadata.mergedAt : undefined;
   const landReportedAt = typeof live.metadata.landReportedAt === "number" ? live.metadata.landReportedAt : undefined;
   if (live.status === "active" && mergedAt !== undefined && mergedAt > (landReportedAt ?? 0)) {
-    return { complete: true, workspaceId: live.id, status: live.status, reason: "recycled-after-land" };
+    return { complete: true, producedWork: true, workspaceId: live.id, status: live.status, reason: "recycled-after-land" };
   }
   // Fast-path (#629): a zero-commit branch has nothing to preserve — reap immediately
   // instead of holding it pending a land that will never happen. Only fires when the
   // branch is provably empty (count === 0); null (unknown/git error) falls through to
   // the safe #614 hold-pending bias below.
   if (live.status === "active" && _commitsAheadFn(live) === 0) {
-    return { complete: true, workspaceId: live.id, status: live.status, reason: "empty-branch" };
+    return { complete: true, producedWork: false, workspaceId: live.id, status: live.status, reason: "empty-branch" };
   }
-  return { complete: false, workspaceId: live.id, status: live.status, reason: "land-pending" };
+  return { complete: false, producedWork: false, workspaceId: live.id, status: live.status, reason: "land-pending" };
 }
 
 function notifyTransientHeld(agentId: string, workspaceId: string | undefined, status: string | undefined, waitMs: number): void {
@@ -171,11 +186,14 @@ export function reapTransientAgents(): Record<string, unknown> {
     // #800 — `reapProtected` is the single home for that decision: it honors only
     // evidence-based block states, never a high quota-utilization number.
     if (reapProtected(agent)) { transientTracker.delete(agent.id); continue; }
+    // A just-registered transient worker can briefly report idle/ready before its first provider
+    // turn starts. Do not start the idle-reap clock until the runner has observed a completed turn.
+    if (completedProviderTurns(agent) === 0) { transientTracker.delete(agent.id); waiting.push(agent.id); continue; }
     const entry = transientTracker.get(agent.id) ?? { firstIdleAt: now };
     transientTracker.set(agent.id, entry);
     const idleForMs = now - entry.firstIdleAt;
     if (idleForMs < grace) { waiting.push(agent.id); continue; }
-    const land = transientLandComplete(agent.id);
+    const land = transientLandComplete(agent);
     if (!land.complete) {
       if (idleForMs >= grace + maxLandWait) {
         held.push(agent.id);
@@ -201,7 +219,7 @@ export function reapTransientAgents(): Record<string, unknown> {
     // #889 — finalize any bound task whose doneWhen is "agent-exit" (auto-scaffolded ephemeral
     // tasks that produce no deliverable and would otherwise hang in_progress forever).
     const spawnRequestId = typeof agent.meta?.spawnRequestId === "string" ? agent.meta.spawnRequestId : undefined;
-    finalizeEphemeralTasksOnReap(agent.id, { spawnRequestId, reason: "transient-complete" });
+    finalizeEphemeralTasksOnReap(agent.id, { spawnRequestId, reason: "transient-complete", producedWork: land.producedWork, evidence: land.reason });
     reaped.push(agent.id);
   }
   for (const key of transientTracker.keys()) if (!seen.has(key)) transientTracker.delete(key);

package/src/spawn-targets.ts

@@ -113,6 +113,26 @@ const DEFAULT_QUOTA_ADVISORY: SpawnQuotaAdvisory = {
   recommendedAction: "No quota burn projection available; spawn normally.",
 };
 
+// #926 — the orchestrator's REPORTED catalog is built from the static SDK manifest only
+// (orchestrator/src/provider-probe.ts uses providerCatalogList(), which has no DB access), while
+// spawn VALIDATION resolves against effectiveProviderCatalogList() (static catalog + DB
+// provider_model_overrides — see resolveSpawnModelParams in spawn-command.ts). A model enabled
+// only via a DB override (e.g. a newly added `sonnet-5`) is therefore spawnable but invisible to
+// this matrix even after an orchestrator restart, since the orchestrator's snapshot never reads
+// the DB. Merge by alias, with the DB-matrix (global) entry winning on collision, so every model
+// spawn validation would accept is discoverable here too.
+function mergeCatalogModels(reportedCat: SpawnCatalogish | undefined, globalCat: SpawnCatalogish | undefined): SpawnCatalogish | undefined {
+  if (!reportedCat && !globalCat) return undefined;
+  const byAlias = new Map<string, SpawnCatalogish["models"][number]>();
+  for (const m of reportedCat?.models ?? []) byAlias.set(m.alias, m);
+  for (const m of globalCat?.models ?? []) byAlias.set(m.alias, m);
+  return {
+    provider: (reportedCat ?? globalCat)!.provider,
+    defaultModel: reportedCat?.defaultModel ?? globalCat?.defaultModel,
+    models: [...byAlias.values()],
+  };
+}
+
 function providerMatrixFor(o: Orchestrator, quotaAdvisories: Map<SpawnProvider, SpawnQuotaAdvisory>): Array<Record<string, unknown>> {
   const reported: ProviderCatalogSummary[] = o.providerCatalog ?? [];
   const global = effectiveProviderCatalogList();
@@ -129,8 +149,7 @@ function providerMatrixFor(o: Orchestrator, quotaAdvisories: Map<SpawnProvider,
   const unavailableAliasesFor = (provider: SpawnProvider): Set<string> =>
     new Set(global.find((e) => e.provider === provider)?.models.filter((m) => m.unavailable).map((m) => m.alias) ?? []);
   return o.providers.map((provider) => {
-    const cat: SpawnCatalogish | undefined =
-      reported.find((s) => s.provider === provider) ?? global.find((e) => e.provider === provider);
+    const cat = mergeCatalogModels(reported.find((s) => s.provider === provider), global.find((e) => e.provider === provider));
     const disabled = disabledAliasesFor(provider);
     const unavailable = unavailableAliasesFor(provider);
     const quotaAdvisory = quotaAdvisories.get(provider) ?? DEFAULT_QUOTA_ADVISORY;