agent-relay-runner 0.129.11 → 0.129.12

package/src/rate-limit.ts

@@ -64,13 +64,20 @@ function formatResetClock(resetAtMs: number): string {
 }
 
 // The subscription limit line wordings seen in the wild (claude-auto-retry patterns +
-// claude-auto-retry#15: "session"/"weekly limit" slipped past older regexes). We REQUIRE a
-// limit phrase AND a reset/retry phrase on the same line so normal output mentioning
-// "rate limit" can't false-positive the hold.
+// claude-auto-retry#15: "session"/"weekly limit" slipped past older regexes). These match
+// PROSE as readily as the modal, so they are never sufficient on their own — see
+// parseClaudeUsageLimitModal for the structural gate that decides what is a real hold.
 const LIMIT_PHRASE_RE = /(?:hit (?:your|the)\s*(?:[\w-]+\s+)*limit|usage limit|\d+-hour limit|limit reached|out of (?:extra )?usage|rate limit(?:\s+(?:hit|reached|exceeded))?|weekly limit|session limit)/i;
 const RESET_AT_RE = /\bresets?\b(?:\s+at)?\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)?\s*(?:\(([^)]+)\))?/i;
 const TRY_AGAIN_RE = /\btry again in\s+(\d+)\s*(second|minute|hour|day)s?/i;
 
+// #1611 — the limit phrase and its reset/retry must sit on the same line or within this many
+// lines of each other. The pairing was originally allowed to span the WHOLE pane so a modal
+// wrapping across rows still matched; that made the two halves collectable from unrelated
+// places on screen. A modal wraps onto the very next row, so a tight window keeps the wrapped
+// case (proven in rate-limit.test.ts) without the pane-wide collection.
+const MAX_PAIR_GAP_LINES = 2;
+
 interface PaneRateLimit {
   /** The matched limit line, for the chat notice / observability. */
   message: string;
@@ -83,56 +90,119 @@ interface PaneRateLimit {
 
 /**
  * Detect a Claude usage/rate-limit modal in a captured tmux pane and parse its reset time.
- * Requires BOTH a limit phrase and a reset/retry phrase (anywhere in the pane, so it's robust
- * to a modal that wraps the two onto separate lines) — that pairing is what distinguishes the
- * real modal from normal output mentioning "rate limit". Returns null otherwise. `nowMs` is
- * injectable for tests.
+ * Returns null when the pane shows no live hold. `nowMs` is injectable for tests.
  */
 export function parseClaudeRateLimitPane(text: string, nowMs: number = Date.now()): PaneRateLimit | null {
   if (!text) return null;
   const overload = parseClaudeTransientOverloadPane(text);
   if (overload) return overload;
-  const limit = LIMIT_PHRASE_RE.exec(text);
-  if (!limit) return null;
-  const message = limitLineAt(text, limit.index);
-
-  const relative = TRY_AGAIN_RE.exec(text);
-  if (relative) {
-    const unitMs = { second: 1000, minute: 60_000, hour: 3_600_000, day: 86_400_000 }[relative[2]!.toLowerCase()] ?? 0;
-    const resetAt = clampReset(nowMs + Number(relative[1]) * unitMs, nowMs);
-    return { message, ...(resetAt ? { resetAt } : {}) };
-  }
-  const reset = RESET_AT_RE.exec(text);
-  if (reset) {
-    const resetAt = clampReset(parseClockReset(reset, nowMs), nowMs);
-    return { message, ...(resetAt ? { resetAt } : {}) };
+  return parseClaudeUsageLimitModal(text, nowMs);
+}
+
+/**
+ * #1611 — the usage-limit modal, distinguished from the agent's OWN pane output.
+ *
+ * The capture (`capture-pane -S -80`) contains the agent's rendered conversation, so a
+ * phrase test over the whole pane fires on any agent that WRITES about rate limits. It did:
+ * a coordinator briefing a worker rendered `usage limit · resets 21:00` — which is the very
+ * label buildRateLimitProviderState() above produces — and marked itself blocked. Requiring
+ * the limit/reset PAIRING does not help either, because that one string carries both halves;
+ * neither does any test of line shape, since an agent working on this file renders exact
+ * modal fixtures (this comment is itself on screen while you read it).
+ *
+ * So the gate is POSITIONAL, not textual. Claude Code pins its composer (input box +
+ * status/hint footer) to the bottom of the screen, separated from the conversation by a blank
+ * row, and draws blocking dialogs in that same spot. Everything the agent itself renders is
+ * therefore ABOVE the last blank row; the live modal is BELOW it. We only scan that trailing
+ * block — no phrase the agent can emit reaches it.
+ *
+ * Break condition (accepted, deliberate): if Claude Code ever draws the limit dialog ABOVE a
+ * blank row (e.g. above a persistent status bar), this stops firing and the hold is missed —
+ * a false negative, which costs one retry via the resume sweep. That is the cheap direction:
+ * a false positive marks a healthy agent blocked, and `isAgentUnavailable` (src/db/
+ * agent-predicates.ts) then QUEUES its inbound messages until the fabricated reset — real
+ * work idled on a fabricated signal. See #1611.
+ */
+function parseClaudeUsageLimitModal(text: string, nowMs: number): PaneRateLimit | null {
+  const block = livePaneBlock(text);
+  for (let i = 0; i < block.length; i++) {
+    const line = block[i]!;
+    if (!LIMIT_PHRASE_RE.test(line)) continue;
+    const message = cleanPaneLine(line);
+    // The reset may wrap onto a neighbouring row of the same modal, never further.
+    const window = block.slice(Math.max(0, i - MAX_PAIR_GAP_LINES), i + MAX_PAIR_GAP_LINES + 1).join("\n");
+
+    const relative = TRY_AGAIN_RE.exec(window);
+    if (relative) {
+      const unitMs = { second: 1000, minute: 60_000, hour: 3_600_000, day: 86_400_000 }[relative[2]!.toLowerCase()] ?? 0;
+      const resetAt = clampReset(nowMs + Number(relative[1]) * unitMs, nowMs);
+      return { message, ...(resetAt ? { resetAt } : {}) };
+    }
+    const reset = RESET_AT_RE.exec(window);
+    if (reset) {
+      const resetAt = clampReset(parseClockReset(reset, nowMs), nowMs);
+      return { message, ...(resetAt ? { resetAt } : {}) };
+    }
+    // Limit phrase but no reset/retry beside it — not confident it's the modal; skip.
   }
-  // Limit phrase but no reset/retry anywhere — not confident it's the modal; skip to
-  // avoid false-positiving on normal output (docs, the agent's own text) that says "limit".
   return null;
 }
 
-const TRANSIENT_API_ERROR_RE = /\bAPI Error:\s*((?:5\d\d)|529)\b[^\n]*(?:overload|server-side|temporar|try again|status\.claude\.com)?[^\n]*/i;
-const TRANSIENT_5XX_RE = /\b(?:5(?:00|02|03|04|20|29)|529)\b[^\n]*(?:overload|server-side|temporar|try again|status\.claude\.com)\b[^\n]*/i;
+/**
+ * The live bottom block of a captured pane: the last run of consecutive non-blank rows
+ * (trailing blanks ignored). On a healthy Claude pane that is the composer + footer chrome;
+ * on a held one it is the modal that replaced them. Panes with no blank row at all (unit-test
+ * one-liners, a short modal-only capture) yield the whole text, which is what we want.
+ */
+function livePaneBlock(text: string): string[] {
+  const lines = text.split(/\r?\n/);
+  let end = lines.length;
+  while (end > 0 && lines[end - 1]!.trim() === "") end--;
+  let start = end;
+  while (start > 0 && lines[start - 1]!.trim() !== "") start--;
+  return lines.slice(start, end);
+}
+
+// Unlike the usage-limit modal, Claude Code renders a transient API error as a CONVERSATION
+// row (`● API Error: 529 Overloaded…`), i.e. in the body, above the composer — so the
+// positional gate above cannot apply here without losing the detection entirely.
+// #1611 — but the arm had the same own-output hole: the phrase was matched anywhere on any
+// line, so an agent rendering this file's diff or a quoted error tripped it (reproduced live:
+// the fragment `+ane("● API Error: 529 Ove` in a rendered patch marked the agent blocked).
+// Claude's own error row STARTS with the error, after nothing but its bullet/box chrome, so
+// anchor there: text that merely embeds the phrase mid-line no longer matches. A genuine
+// error pane is unchanged.
+const TRANSIENT_API_ERROR_RE = /^API Error:\s*((?:5\d\d)|529)\b[^\n]*(?:overload|server-side|temporar|try again|status\.claude\.com)?[^\n]*/i;
+const TRANSIENT_5XX_RE = /^(?:5(?:00|02|03|04|20|29)|529)\b[^\n]*(?:overload|server-side|temporar|try again|status\.claude\.com)\b[^\n]*/i;
+// Row chrome Claude prefixes its rendered lines with: whitespace, bullets/spinners, and the
+// box-drawing block (U+2500–U+257F).
+const PANE_ROW_PREFIX_RE = /^[\s─-╿●⏺✳✻✽⎿>❯»→✗✘ו*-]+/;
 
 function parseClaudeTransientOverloadPane(text: string): PaneRateLimit | null {
-  const match = TRANSIENT_API_ERROR_RE.exec(text) ?? TRANSIENT_5XX_RE.exec(text);
-  if (!match) return null;
-  const message = limitLineAt(text, match.index);
-  const code = match[1] ?? /\b(5\d\d|529)\b/.exec(match[0])?.[1];
-  return {
-    message: message || match[0].trim(),
-    kind: "transient_overload",
-    errorType: code === "529" ? "overloaded" : "server_error",
-  };
+  for (const line of text.split(/\r?\n/)) {
+    const row = line.replace(PANE_ROW_PREFIX_RE, "");
+    const match = TRANSIENT_API_ERROR_RE.exec(row) ?? TRANSIENT_5XX_RE.exec(row);
+    if (!match) continue;
+    const code = match[1] ?? /\b(5\d\d|529)\b/.exec(match[0])?.[1];
+    return {
+      message: cleanPaneLine(line) || match[0].trim(),
+      kind: "transient_overload",
+      errorType: code === "529" ? "overloaded" : "server_error",
+    };
+  }
+  return null;
 }
 
-// The clean limit line containing the match offset, stripped of box-drawing chrome.
+// The clean line containing the match offset, stripped of box-drawing chrome.
 function limitLineAt(text: string, index: number): string {
   const start = text.lastIndexOf("\n", index) + 1;
   const endNl = text.indexOf("\n", index);
   const end = endNl === -1 ? text.length : endNl;
-  return text.slice(start, end).replace(/[│┌┐└┘─┤├╭╮╯╰]/g, " ").replace(/\s+/g, " ").trim();
+  return cleanPaneLine(text.slice(start, end));
+}
+
+function cleanPaneLine(line: string): string {
+  return line.replace(/[│┌┐└┘─┤├╭╮╯╰]/g, " ").replace(/\s+/g, " ").trim();
 }
 
 function clampReset(resetAt: number | undefined, nowMs: number): number | undefined {

package/src/relay-injection-events.ts

@@ -35,6 +35,30 @@ function launchInjectionEvents(input: RunnerLaunchInjectionInput): KeyedRelayInj
   const events: KeyedRelayInjectionEvent[] = (input.carried ?? []).map((event, index) => ({ key: `spawn-${index}`, event }));
   if (!isSpawnProvider(input.provider)) return events;
   const assembled = assembleLaunch(input.provider, input.config, input.providerConfig);
+  // #1565 P5.7 — the native-memory lever gets its OWN `memory`-category event (the channel #1612
+  // established for the spawn-seam memory block), not a line in the tools summary: this decides
+  // whether the agent has one memory store or two, and a SUPPRESSED lever must be visible even on
+  // a launch that provisions nothing else and would otherwise return early below.
+  const lever = assembled.nativeMemory;
+  if (lever.policy !== "default") {
+    events.push({
+      key: "native-memory",
+      event: {
+        category: "memory",
+        summary: lever.suppressed
+          ? `native memory ${lever.policy}: PARTIALLY applied — ${lever.suppressed.reason}`
+          : `native memory ${lever.policy} (${lever.applied.join(", ") || "no lever"})`,
+        detail: {
+          source: "native-memory-lever",
+          provider: assembled.provider,
+          base: assembled.base,
+          policy: lever.policy,
+          applied: lever.applied,
+          ...(lever.suppressed ? { suppressed: lever.suppressed } : {}),
+        },
+      },
+    });
+  }
   const toolCount = assembled.skills.length + assembled.plugins.length + assembled.mcp.servers.length
     + assembled.hooks.length
     + assembled.projectSkills.length

package/src/relay-instructions.ts

@@ -20,6 +20,15 @@ export function relayReplyReminderText(messageId: number): string {
   return `If this batch needs a response, send one useful reply to the latest relevant message with ${relayReplyActionText(messageId)}`;
 }
 
+// #1570 SUB-A — the 4-line reply-reminder block repeats a fact the agent already knows after
+// the first delivery: it can reply, and how. Show it on the first delivery (so the awareness
+// lands at least once) and then periodically, rather than on every replyable message.
+export const REMINDER_EVERY_DELIVERIES = 5;
+
+export function shouldShowReplyReminder(deliveryCount: number): boolean {
+  return deliveryCount <= 1 || deliveryCount % REMINDER_EVERY_DELIVERIES === 0;
+}
+
 export function workspaceDepsNote(input: { mode?: string | null; depsMode?: string | null }): string {
   if (input.mode !== "isolated") return "";
   switch (input.depsMode) {

package/src/relay-mcp-proxy.ts

@@ -1,5 +1,6 @@
 import { errMessage, isRecord } from "agent-relay-sdk";
 import { logger } from "./logger";
+import { RELAY_MCP_ONDEMAND_SURFACE_HEADER, RELAY_MCP_ONDEMAND_SURFACE_VALUE } from "./relay-mcp";
 
 // Loose fetch signature so tests can inject a plain async stub without Bun's `preconnect`
 // member; the real global `fetch` satisfies it.
@@ -30,6 +31,16 @@ export type FetchLike = (input: string | URL | Request, init?: RequestInit) => P
 // NEVER enforcement. The proxy can only ever surface a subset of what the relay's scope filter
 // already returned, so a proxy bug can never become an auth bypass — the relay owns the lock at
 // `tools/call`, the proxy owns the menu.
+//
+// #1450 (the #1570 unify) adds a SECOND narrowing dimension for Codex lean workers: the on-demand
+// relay set. The agent's DIRECT relay client has its on-demand tools hidden here so only the eager
+// hot-path set boots; the SAME tools are simultaneously fronted by the per-agent metaOnly callmux
+// (a second relay downstream over THIS proxy), reachable via callmux_call. callmux's requests carry
+// RELAY_MCP_ONDEMAND_SURFACE_HEADER, which opts OUT of this hiding so callmux sees the full set the
+// relay returned and can route to any of it. This is still strictly narrow-never-widen (the direct
+// client loses exactly what callmux gains; the header only restores the relay's own set, never the
+// call — tools/call stays scope-gated), and it is coupled at launch: a worker is only narrowed when
+// callmux is simultaneously fronting the on-demand set, so no capability is ever lost.
 
 const PROXY_PATH = "/mcp";
 // Relay-side failures we treat as "relay down" (buffer/serve-cache), as opposed to a real 4xx
@@ -60,6 +71,35 @@ const WORKTREE_ONLY_TOOLS = new Set<string>([
 interface ProxyContext {
   // The agent owns a live (non-terminal) isolated git worktree → workspace tools apply.
   isolatedWorktree: boolean;
+  // #1502 — this worktree agent's own workspace descriptor (from AGENT_RELAY_WORKSPACE_JSON). The
+  // relay writes the workspace ROW lazily via the orchestrator's managed-agents report, so a worker
+  // that delivers before that write (or whose registration was lost under concurrent-spawn load)
+  // would 404. We inject this descriptor into `relay_task_deliver` so the relay can self-heal the
+  // missing registration from the worker's own truth instead of stranding the delivery.
+  workspaceDescriptor?: Record<string, unknown>;
+}
+
+// #1502 — tool calls the proxy augments with the agent's workspace descriptor for self-heal.
+const WORKSPACE_SELF_HEAL_TOOLS = new Set(["relay_task_deliver"]);
+
+// #1502 — the minimal descriptor the relay needs to (re)register a dropped workspace row, distilled
+// from the agent's WorkspaceMetadata. Undefined unless this agent owns a live isolated worktree with
+// the required id/repo/worktree present, so a shared/non-worktree agent never sends one.
+export function isolatedWorkspaceDescriptor(
+  workspace: { id?: string; repoRoot?: string; worktreePath?: string; sourceCwd?: string; branch?: string; baseRef?: string; baseSha?: string } | undefined,
+  ownsIsolatedWorktree: boolean,
+): Record<string, unknown> | undefined {
+  if (!ownsIsolatedWorktree || !workspace?.id || !workspace.repoRoot || !workspace.worktreePath) return undefined;
+  return {
+    id: workspace.id,
+    repoRoot: workspace.repoRoot,
+    worktreePath: workspace.worktreePath,
+    mode: "isolated",
+    ...(workspace.sourceCwd ? { sourceCwd: workspace.sourceCwd } : {}),
+    ...(workspace.branch ? { branch: workspace.branch } : {}),
+    ...(workspace.baseRef ? { baseRef: workspace.baseRef } : {}),
+    ...(workspace.baseSha ? { baseSha: workspace.baseSha } : {}),
+  };
 }
 
 export interface BufferedToolCall {
@@ -80,6 +120,11 @@ export interface RelayMcpProxyOptions {
   enqueueBuffered(call: BufferedToolCall): void;
   initialContext?: ProxyContext;
   bufferableTools?: Set<string>;
+  // #1450 — the Codex-worker on-demand relay set to hide from the agent's DIRECT relay client so
+  // only the eager hot-path set boots. Undefined for every agent that isn't a lean Codex worker with
+  // callmux simultaneously fronting these tools (Claude, coordinators, reviewers, proxy-off) → no
+  // hiding, full surface. callmux's own relay downstream opts out via RELAY_MCP_ONDEMAND_SURFACE_HEADER.
+  onDemandTools?: Set<string>;
   // Test seam.
   fetchImpl?: FetchLike;
 }
@@ -102,6 +147,7 @@ export class RelayMcpProxy {
   private readonly authSecret: string;
   private readonly enqueueBuffered: (call: BufferedToolCall) => void;
   private readonly bufferableTools: Set<string>;
+  private readonly onDemandTools?: Set<string>;
   private readonly fetchImpl: FetchLike;
   private readonly encoder = new TextEncoder();
 
@@ -119,6 +165,7 @@ export class RelayMcpProxy {
     this.authSecret = options.authSecret;
     this.enqueueBuffered = options.enqueueBuffered;
     this.bufferableTools = options.bufferableTools ?? DEFAULT_BUFFERABLE_TOOLS;
+    this.onDemandTools = options.onDemandTools;
     this.fetchImpl = options.fetchImpl ?? fetch;
     this.context = options.initialContext ?? { isolatedWorktree: false };
   }
@@ -192,6 +239,10 @@ export class RelayMcpProxy {
   }
 
   private async handleRpc(req: Request): Promise<Response> {
+    // #1450 — read the on-demand-front header BEFORE consuming the body. The per-agent metaOnly
+    // callmux's relay downstream sets it to opt OUT of the on-demand hiding (it fronts those tools);
+    // the agent's direct relay client never sets it, so it gets the eager hot-path set only.
+    const frontsOnDemand = req.headers.get(RELAY_MCP_ONDEMAND_SURFACE_HEADER) === RELAY_MCP_ONDEMAND_SURFACE_VALUE;
     let message: JsonRpcMessage;
     try {
       const body = await req.json();
@@ -206,7 +257,7 @@ export class RelayMcpProxy {
     const method = message.method;
 
     if (method === "initialize") return this.handleInitialize(id, message);
-    if (method === "tools/list") return this.handleToolsList(id, message);
+    if (method === "tools/list") return this.handleToolsList(id, message, frontsOnDemand);
     if (method === "tools/call") return this.handleToolsCall(id, message);
     // Everything else (notifications/initialized, ping, …) forwards verbatim.
     return this.forwardRaw(message);
@@ -230,7 +281,7 @@ export class RelayMcpProxy {
     return Response.json(jsonRpcResult(id, result));
   }
 
-  private async handleToolsList(id: string | number | null, message: JsonRpcMessage): Promise<Response> {
+  private async handleToolsList(id: string | number | null, message: JsonRpcMessage, frontsOnDemand = false): Promise<Response> {
     const relay = await this.forward({ ...message, method: "tools/list", id }).catch(() => null);
     const tools = relay && isRecord(relay.body?.result) && Array.isArray((relay.body!.result as Record<string, unknown>).tools)
       ? ((relay.body!.result as Record<string, unknown>).tools as Array<Record<string, unknown>>)
@@ -239,17 +290,24 @@ export class RelayMcpProxy {
     // If the relay is down and we have no cache yet, surface an empty list rather than erroring —
     // the agent can still operate (writes buffer; tools/list refreshes on the next call).
     const base = tools ?? this.lastRelayTools;
-    const narrowed = this.narrow(base);
-    this.lastNarrowedNames = toolNames(narrowed);
+    // #1450 — callmux's relay downstream (frontsOnDemand) sees the full set so it can route
+    // callmux_call to any of it; the direct client has the on-demand set hidden.
+    const narrowed = this.narrow(base, frontsOnDemand);
+    if (!frontsOnDemand) this.lastNarrowedNames = toolNames(narrowed);
     return Response.json(jsonRpcResult(id, { tools: narrowed }));
   }
 
-  // Strict subset of the relay's already-scope-filtered list. Only removes — never adds. The one
-  // narrowing rule today: workspace tools apply only to an agent that owns a live worktree.
-  private narrow(tools: Array<Record<string, unknown>>): Array<Record<string, unknown>> {
+  // Strict subset of the relay's already-scope-filtered list. Only removes — never adds. Narrowing
+  // rules: (1) workspace tools apply only to an agent that owns a live worktree; (2) #1450 — the
+  // on-demand relay set is hidden from the agent's DIRECT client (only when `hideOnDemand`), since
+  // the per-agent metaOnly callmux fronts those tools on demand. `frontsOnDemand` (callmux's own
+  // request) skips rule 2 so callmux sees the full relay set.
+  private narrow(tools: Array<Record<string, unknown>>, frontsOnDemand = false): Array<Record<string, unknown>> {
+    const hideOnDemand = !frontsOnDemand && this.onDemandTools !== undefined && this.onDemandTools.size > 0;
     return tools.filter((tool) => {
       const name = typeof tool.name === "string" ? tool.name : "";
       if (WORKTREE_ONLY_TOOLS.has(name) && !this.context.isolatedWorktree) return false;
+      if (hideOnDemand && this.onDemandTools!.has(name)) return false;
       return true;
     });
   }
@@ -257,7 +315,20 @@ export class RelayMcpProxy {
   private async handleToolsCall(id: string | number | null, message: JsonRpcMessage): Promise<Response> {
     const params = isRecord(message.params) ? message.params : {};
     const toolName = typeof params.name === "string" ? params.name : "";
-    const args = isRecord(params.arguments) ? params.arguments : {};
+    let args = isRecord(params.arguments) ? params.arguments : {};
+
+    // #1502 — inject this agent's workspace descriptor so the relay can self-heal a missing/dropped
+    // workspace registration during delivery. Only for worktree agents, only when the agent didn't
+    // already supply one, and the augmented args are forwarded (not just used locally).
+    if (
+      WORKSPACE_SELF_HEAL_TOOLS.has(toolName) &&
+      this.context.isolatedWorktree &&
+      this.context.workspaceDescriptor &&
+      args.workspace === undefined
+    ) {
+      args = { ...args, workspace: this.context.workspaceDescriptor };
+      message = { ...message, params: { ...params, arguments: args } };
+    }
 
     const relay = await this.forward(message).catch((error) => {
       // A thrown fetch = transport failure (relay unreachable / DNS / connection refused).