agent-relay-runner 0.88.1 → 0.88.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-runner",
-  "version": "0.88.1",
+  "version": "0.88.3",
   "description": "Unified provider lifecycle runner for Agent Relay",
   "type": "module",
   "bin": {

package/plugins/claude/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "agent-relay-runner",
   "description": "Thin Agent Relay runner bridge for Claude Code",
-  "version": "0.88.1",
+  "version": "0.88.3",
   "agentRelayContracts": {
     "providerPluginProtocol": 1
   }

package/src/adapter.ts

@@ -25,6 +25,10 @@ export interface ProviderStatusEvent {
   label?: string;
   role?: string;
   parentId?: string;
+  // OS pids of the process(es) this busy work tracks (#650). When supplied, the
+  // runner reconciles them against actual liveness and clears the busy marker if
+  // every pid is dead — recovering from a tracked process killed out-of-band.
+  pids?: number[];
   providerState?: Record<string, unknown>;
   metadata?: Record<string, unknown>;
 }

package/src/adapters/claude.ts

@@ -12,7 +12,7 @@ import type { SessionEvent } from "../session-insights";
 import { prepareClaudeProfileHome, profileUsesHostProviderGlobals } from "../profile-home";
 import { assembleLaunch, materializeLaunchAssembly, sessionStatusLineSettingsArgs } from "../launch-assembly";
 import { claudeProviderMessageText } from "./claude-delivery";
-import { buildRateLimitProviderState, parseClaudeRateLimitPane } from "../rate-limit";
+import { buildRateLimitProviderState, gatedClaudeRateLimitStatus, parseClaudeRateLimitPane } from "../rate-limit";
 
 export class ClaudeAdapter implements ProviderAdapter {
   readonly provider = "claude";
@@ -27,6 +27,7 @@ export class ClaudeAdapter implements ProviderAdapter {
   // #286: true while a usage-limit modal is being held, so the pane watcher dismisses
   // it + emits the hold once (not every 2s tick). Cleared when the modal leaves the pane.
   private rateLimitReported = false;
+  private rateLimitPaneDetections = 0;
 
   onStatusChange(cb: (status: ProviderStatusUpdate) => void): void {
     this.statusCb = cb;
@@ -369,25 +370,24 @@ export class ClaudeAdapter implements ProviderAdapter {
         this.statusCb(status);
         return;
       }
-      // #286: the subscription usage-limit modal fires no hook, so detect it from the
-      // pane. Dismiss it (Escape = "wait", which just stops the turn) so the agent
-      // returns to a clean prompt the resume `continue` can land on, then hold it.
-      const rateLimit = claudeRateLimitStatus(pane, sessionName);
-      if (rateLimit) {
-        if (!this.rateLimitReported) {
-          this.rateLimitReported = true;
-          this.dismissRateLimitModal(sessionName, socketName);
-          this.statusCb(rateLimit);
-        }
-      } else if (this.rateLimitReported) {
-        // Modal gone (dismissed, or a real turn resumed) — re-arm for the next limit.
-        this.rateLimitReported = false;
+      // #286/#655: the subscription usage-limit modal fires no hook, so detect it
+      // from the pane. Act only after consecutive non-busy detections; a live turn
+      // mentioning quota/reset text must not be escaped or held.
+      const rateLimit = gatedClaudeRateLimitStatus(pane, {
+        consecutiveDetections: this.rateLimitPaneDetections,
+        reported: this.rateLimitReported,
+      }, { sessionName, busy: claudePaneIsBusy(pane) });
+      this.rateLimitPaneDetections = rateLimit.state.consecutiveDetections;
+      this.rateLimitReported = rateLimit.state.reported;
+      if (rateLimit.status) {
+        this.dismissRateLimitModal(sessionName, socketName);
+        this.statusCb(rateLimit.status);
       }
     }, 2000);
   }
 
-  // Send Escape to dismiss the usage-limit modal — the "wait" choice, which stops the
-  // turn (per the CC TUI) and returns the agent to a normal idle prompt. Best-effort:
+  // Send Escape to dismiss the confirmed usage-limit modal — the "wait" choice,
+  // which returns the agent to a normal idle prompt. Best-effort:
   // a failed send-keys must never wedge the pane watcher.
   private dismissRateLimitModal(sessionName: string, socketName?: string): void {
     try {

package/src/adapters/codex.ts

@@ -50,6 +50,9 @@ export class CodexAdapter implements ProviderAdapter {
   // Active turn id for the main thread, captured from turn/started so an interrupt
   // can target the in-flight turn. Cleared on turn/completed.
   private activeTurnId?: string;
+  // #653: monotonic id for a turn we synthesize from live item activity when a real
+  // turn/started never reached us — so a working Codex agent isn't reported `idle`.
+  private syntheticTurnSeq = 0;
   private readonly agentMessageCapture = new CodexAgentMessageCapture();
   private readonly itemTextBuffers = new Map<string, string>();
   private readonly itemTextBufferTypes = new Map<string, string>();
@@ -578,8 +581,26 @@ export class CodexAdapter implements ProviderAdapter {
     return [...this.sessionEvents];
   }
 
+  // #653: Codex busy-state normally flips on turn/started, but the app-server can stream
+  // item activity (which already drives the dashboard chat mirror) before — or without —
+  // a turn/started reaching us, leaving a visibly-working agent reported `idle` (a reap
+  // hazard). Derive busy from that SAME live activity: any in-flight item event with no
+  // active turn means a turn IS running we missed, so synthesize one and report busy.
+  // turn/completed → finishMainTurn clears it; we never synthesize on a closing
+  // item.completed, so a legitimately ended turn is not re-flipped to busy.
+  private ensureMainTurnActive(turnId?: string): void {
+    if (this.activeTurnId) return;
+    this.activeTurnId = turnId ?? `codex-synth-turn-${++this.syntheticTurnSeq}`;
+    this.statusCb({ status: "busy", reason: "provider-turn", id: this.activeTurnId });
+  }
+
   private handleCodexItemDelta(method: string, params: Record<string, unknown> | undefined): void {
     if (!method.includes("item/") && !method.includes("item.")) return;
+    // Live item stream (start/delta/progress) = a turn in flight. A closing
+    // item.completed is handled by handleCodexItem and must not synthesize a turn.
+    if (!method.includes("/completed") && !method.includes(".completed")) {
+      this.ensureMainTurnActive(stringValue(params?.turnId));
+    }
     const item = isRecord(params?.item) ? params.item : undefined;
     const itemId = codexItemId(params) ?? codexItemId(item);
     const type = codexItemTypeFromMethod(method) ?? stringValue(item?.type);

package/src/claim-tracker.ts

@@ -17,6 +17,10 @@ interface WorkRecord {
   role?: string;
   parentId?: string;
   metadata?: Record<string, unknown>;
+  // OS pids of the underlying process(es) this work tracks (#650). When present,
+  // a periodic liveness reconcile clears the record once every pid is dead — so a
+  // tracked process killed out-of-band (no exit event) can't strand the agent busy.
+  pids?: number[];
   startedAt: number;
 }
 
@@ -40,6 +44,7 @@ export class ClaimTracker {
       role: details.role ?? existing?.role,
       parentId: details.parentId ?? existing?.parentId,
       metadata: details.metadata ?? existing?.metadata,
+      pids: details.pids ?? existing?.pids,
       startedAt: existing?.startedAt ?? Date.now(),
     });
     return before !== this.currentStatus();
@@ -106,4 +111,30 @@ export class ClaimTracker {
   activeWork(): WorkRecord[] {
     return [...this.work.values()];
   }
+
+  /** True iff any active work carries pids the liveness reconcile can check (#650). */
+  hasLivenessTrackedWork(): boolean {
+    for (const item of this.work.values()) {
+      if (item.pids && item.pids.length > 0) return true;
+    }
+    return false;
+  }
+
+  /**
+   * Reconcile pid-bearing work against actual process liveness (#650): drop any
+   * record whose tracked pids are ALL dead — its underlying process died out-of-band
+   * (killed/crashed/orphaned) with no exit event to clear the busy marker. Records
+   * with no pids are never touched (no liveness signal to act on). Returns the removed
+   * records so the caller can emit the matching completion/abort + republish status.
+   */
+  reconcileLiveness(isAlive: (pid: number) => boolean): WorkRecord[] {
+    const removed: WorkRecord[] = [];
+    for (const [key, item] of this.work) {
+      if (!item.pids || item.pids.length === 0) continue;
+      if (item.pids.some(isAlive)) continue;
+      this.work.delete(key);
+      removed.push(item);
+    }
+    return removed;
+  }
 }

package/src/control-server.ts

@@ -446,6 +446,12 @@ async function handleStatus(req: Request, options: ControlServerOptions): Promis
   const isWorkKind = (item: unknown): item is "provider-turn" | "subagent" | "background-script" =>
     item === "provider-turn" || item === "subagent" || item === "background-script";
   const clear = Array.isArray(body?.clear) ? body.clear.filter(isWorkKind) : undefined;
+  // #650: optional pids the runner reconciles against process liveness so a tracked
+  // command/shell killed out-of-band clears the busy marker instead of stranding it.
+  const pidsInput = Array.isArray((body as { pids?: unknown })?.pids) ? (body as { pids: unknown[] }).pids : undefined;
+  const pids = pidsInput
+    ?.map((p) => (typeof p === "number" ? p : Number(p)))
+    .filter((p) => Number.isInteger(p) && p > 0);
   const timeline = statusTimelineEvent(body);
   const update: ProviderStatusEvent = {
     status,
@@ -454,6 +460,7 @@ async function handleStatus(req: Request, options: ControlServerOptions): Promis
     ...(clear?.length ? { clear: [...new Set(clear)] } : {}),
     ...(timeline ? { timeline } : {}),
     ...(typeof body?.id === "string" ? { id: body.id } : {}),
+    ...(pids?.length ? { pids } : {}),
     ...(typeof body?.label === "string" ? { label: body.label } : {}),
     ...(typeof body?.role === "string" ? { role: body.role } : {}),
     ...(typeof body?.parentId === "string" ? { parentId: body.parentId } : {}),

package/src/rate-limit.ts

@@ -1,4 +1,5 @@
 // #286 — shared rate-limit hold construction + Claude pane detection.
+import type { ProviderStatusUpdate } from "./adapter";
 //
 // Two detection arms feed the SAME provider-neutral `blocked` hold:
 //   1. The StopFailure hook (clean API-error turn-end: API-429/auth/billing) → control-server.
@@ -94,6 +95,37 @@ export function parseClaudeRateLimitPane(text: string, nowMs: number = Date.now(
   return null;
 }
 
+export interface ClaudeRateLimitPaneGateState {
+  consecutiveDetections: number;
+  reported: boolean;
+}
+
+export function gatedClaudeRateLimitStatus(
+  text: string,
+  state: ClaudeRateLimitPaneGateState,
+  options: { sessionName?: string; busy?: boolean } = {},
+): { status: ProviderStatusUpdate | null; state: ClaudeRateLimitPaneGateState } {
+  const parsed = parseClaudeRateLimitPane(text);
+  if (!parsed) return { status: null, state: { consecutiveDetections: 0, reported: false } };
+  if (options.busy) return { status: null, state: { consecutiveDetections: 0, reported: state.reported } };
+  const nextState = { consecutiveDetections: state.consecutiveDetections + 1, reported: state.reported };
+  if (nextState.reported || nextState.consecutiveDetections < 2) return { status: null, state: nextState };
+  nextState.reported = true;
+  return {
+    status: {
+      status: "idle",
+      clear: ["subagent"],
+      providerState: buildRateLimitProviderState({
+        errorType: "session_limit",
+        ...(parsed.resetAt ? { resetAt: parsed.resetAt } : {}),
+        message: parsed.message,
+        source: options.sessionName ? `claude-pane:${options.sessionName}` : "claude-pane",
+      }),
+    },
+    state: nextState,
+  };
+}
+
 // The clean limit line containing the match offset, stripped of box-drawing chrome.
 function limitLineAt(text: string, index: number): string {
   const start = text.lastIndexOf("\n", index) + 1;

package/src/runner-core.ts

@@ -5,6 +5,7 @@ import { dirname, join } from "node:path";
 import type { AgentLifecycle, AgentProfile, ContextState, Message, MessageSessionMeta, SendMessageInput, TaskStatusInput, WorkspaceMetadata } from "agent-relay-sdk";
 import { errMessage, RelayBusClient, RelayHttpClient } from "agent-relay-sdk";
 import { contextStateFromProbeMetrics, quotaStateFromProbeMetrics } from "agent-relay-sdk/context-probe";
+import { isPidAlive } from "agent-relay-sdk/process-utils";
 import { providerAttachmentText, providerMessageText } from "./adapter";
 import type { ManagedProcess, ProviderAdapter, ProviderConfig, ProviderPermissionDecision, ProviderPermissionDecisionInput, ProviderSessionEvent, ProviderStatusUpdate, RunnerSpawnConfig, SemanticStatus, TerminalAttachSpec } from "./adapter";
 import { messagesWithCachedAttachments } from "./attachment-cache";
@@ -109,6 +110,8 @@ interface ActiveTaskClaim {
 
 const CLAIM_RENEW_INTERVAL_MS = 5 * 60 * 1000;
 const HTTP_LIVENESS_INTERVAL_MS = 20_000;
+// #650: how often to reconcile pid-bearing busy work against actual process liveness.
+const BUSY_LIVENESS_RECONCILE_MS = 5_000;
 const HTTP_LIVENESS_LOG_INTERVAL_MS = 5 * 60 * 1000;
 const TOKEN_RENEW_RETRY_MS = 60_000;
 // Debounce reactive token recovery so a burst of 401-ing calls in the same window
@@ -204,6 +207,7 @@ export class AgentRunner {
   private drainTimer?: Timer;
   private claimRenewTimer?: Timer;
   private httpLivenessTimer?: Timer;
+  private livenessReconcileTimer?: Timer;
   private httpLivenessInFlight = false;
   private httpLivenessAuthFailed = false;
   private httpLivenessLastLog?: { key: string; at: number };
@@ -264,6 +268,7 @@ export class AgentRunner {
   // providerState; it clears when a new turn starts (the relay's resume message
   // wakes one) so the blocked badge lifts on its own.
   private rateLimitHold?: Record<string, unknown>;
+  private pendingRateLimitHold?: Record<string, unknown>;
   // Reasoning tailer (item 5): streams the in-flight turn's reasoning/tool steps
   // from the Claude transcript into chat as discreet session events.
   private reasoningTail?: ReasoningTailState;
@@ -471,6 +476,7 @@ export class AgentRunner {
     this.httpLivenessTimer = undefined;
     if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
     this.tokenRenewTimer = undefined;
+    this.disarmBusyLivenessReconcile();
     this.busyReconciler.disarm();
     this.stopReasoningTail();
     this.obligationCache.stop();
@@ -1150,16 +1156,28 @@ export class AgentRunner {
         ...(update.timeline.metadata ? { metadata: update.timeline.metadata } : {}),
       };
     }
+    let deferredRateLimitHold = false;
     if (typeof update !== "string" && update.providerState) {
-      const ps = update.providerState as { state?: unknown; reason?: unknown };
-      this.providerBlocked = ps.state === "blocked";
+      const ps = update.providerState as { state?: unknown; reason?: unknown; source?: unknown };
       // A rate-limit hold persists across the idle the turn ends on; any other
       // providerState supersedes it (clears a stale hold).
       if (ps.state === "blocked" && ps.reason === "rate_limit") {
-        const fresh = !this.rateLimitHold;
-        this.rateLimitHold = update.providerState;
-        if (fresh) this.publishRateLimitNotice(update.providerState);
+        const paneRateLimitHold = typeof ps.source === "string" && ps.source.startsWith("claude-pane");
+        if (status === "idle" && reason === "provider-turn" && paneRateLimitHold && this.claims.activeWork().some((work) => work.kind === "provider-turn")) {
+          this.pendingRateLimitHold = update.providerState;
+          this.providerBlocked = false;
+          deferredRateLimitHold = true;
+          this.sessionLog(`rate-limit hold deferred until active turn finishes (turn ${this.currentTurnId ?? "?"})`);
+        } else {
+          this.providerBlocked = true;
+          this.pendingRateLimitHold = undefined;
+          const fresh = !this.rateLimitHold;
+          this.rateLimitHold = update.providerState;
+          if (fresh) this.publishRateLimitNotice(update.providerState);
+        }
       } else {
+        this.providerBlocked = ps.state === "blocked";
+        this.pendingRateLimitHold = undefined;
         this.rateLimitHold = undefined;
       }
     } else if (status === "idle") {
@@ -1167,6 +1185,13 @@ export class AgentRunner {
     }
     // Forward progress (a real turn) lifts the hold so the blocked badge clears.
     if (status === "busy") this.rateLimitHold = undefined;
+    if (deferredRateLimitHold) {
+      if (typeof update !== "string") {
+        for (const kind of update.clear ?? []) this.claims.clearWorkKind(kind);
+      }
+      this.publishStatus();
+      return;
+    }
     if (typeof update !== "string" && status === "error") {
       const terminalReason = typeof update.metadata?.terminalFailureReason === "string"
         ? update.metadata.terminalFailureReason
@@ -1227,15 +1252,30 @@ export class AgentRunner {
     if (status === "busy" || status === "idle") this.terminalProviderExit = undefined;
     if (status === "busy") {
       this.claims.clearTerminalStatus();
-      this.claims.startWork(reason, id, typeof update === "string" ? {} : {
+      // #650: liveness-track the pids a reporter explicitly supplied; otherwise, for a
+      // provider-turn, fall back to the managed provider process itself. If that process
+      // dies out-of-band with no exit event reaching the runner (crash/orphan/external
+      // kill), the reconcile clears the stuck turn → idle. A dead pid means the turn is
+      // definitively over, so this can never falsely clear a live turn (#653's hazard).
+      const reportedPids = typeof update === "string" ? undefined : update.pids;
+      const livenessPids = reportedPids?.length
+        ? reportedPids
+        : reason === "provider-turn" && typeof this.process?.pid === "number" && this.process.pid > 0
+          ? [this.process.pid]
+          : undefined;
+      this.claims.startWork(reason, id, typeof update === "string" ? (livenessPids ? { pids: livenessPids } : {}) : {
         label: update.label,
         role: update.role,
         parentId: update.parentId,
+        ...(livenessPids ? { pids: livenessPids } : {}),
         metadata: {
           ...(update.metadata ?? {}),
           ...(update.providerState ? { providerState: update.providerState } : {}),
         },
       });
+      // A busy report with liveness-trackable pids arms the periodic reconcile (self-disarms
+      // once no pid-bearing work remains, so idle/untracked agents pay nothing).
+      if (livenessPids?.length) this.armBusyLivenessReconcile();
       if (reason === "provider-turn") {
         for (const claim of this.activeTaskClaims.values()) claim.observedProviderBusy = true;
       }
@@ -1253,6 +1293,14 @@ export class AgentRunner {
     if (status === "idle" && this.pendingInitialPrompt && !this.deliveringInitialPrompt) {
       void this.attemptInitialPromptDelivery(this.pendingInitialPrompt, INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS);
     }
+    if (status === "idle" && this.pendingRateLimitHold && !this.claims.activeWork().some((work) => work.kind === "provider-turn")) {
+      const hold = this.pendingRateLimitHold;
+      this.pendingRateLimitHold = undefined;
+      this.providerBlocked = true;
+      const fresh = !this.rateLimitHold;
+      this.rateLimitHold = hold;
+      if (fresh) this.publishRateLimitNotice(hold);
+    }
     this.publishStatus();
   }
 
@@ -1662,6 +1710,44 @@ export class AgentRunner {
     this.publishStatus();
   }
 
+  // #650: reconcile pid-bearing busy work against actual process liveness. A tracked
+  // command/shell that dies out-of-band (killed by the operator, crashes, orphaned)
+  // sends no exit event, so its busy marker would otherwise never clear and the agent
+  // shows `busy` forever — blocking transient auto-reap and (for Claude) holding quota.
+  // The timer self-disarms once no pid-bearing work remains; it is armed lazily when a
+  // busy report carrying pids arrives, so idle/non-tracked agents pay nothing.
+  private armBusyLivenessReconcile(): void {
+    if (this.livenessReconcileTimer || this.stopped) return;
+    this.livenessReconcileTimer = setInterval(() => this.runBusyLivenessReconcile(), BUSY_LIVENESS_RECONCILE_MS);
+  }
+
+  private disarmBusyLivenessReconcile(): void {
+    if (this.livenessReconcileTimer) clearInterval(this.livenessReconcileTimer);
+    this.livenessReconcileTimer = undefined;
+  }
+
+  private runBusyLivenessReconcile(): void {
+    if (this.stopped) {
+      this.disarmBusyLivenessReconcile();
+      return;
+    }
+    const removed = this.claims.reconcileLiveness(isPidAlive);
+    if (removed.length > 0) {
+      this.sessionLog(`cleared ${removed.length} stuck busy work item(s) whose tracked process died out-of-band: ${removed.map((w) => `${w.kind}:${w.id}`).join(", ")}`);
+      // A cleared provider-turn must also reset the turn bookkeeping so a later idle
+      // doesn't double-end it and the reasoning tail stops.
+      if (removed.some((w) => w.kind === "provider-turn")) {
+        this.currentTurnId = undefined;
+        this.currentTurnStartedAt = undefined;
+        this.compactionMidTurn = false;
+        this.busyReconciler.disarm();
+        this.stopReasoningTail();
+      }
+      this.publishStatus();
+    }
+    if (!this.claims.hasLivenessTrackedWork()) this.disarmBusyLivenessReconcile();
+  }
+
   // --- Turn-step tailer (item 5) ------------------------------------------------------
   // Tail the in-flight turn's Claude transcript and surface new narration/reasoning/tool
   // steps as session events, in transcript order. `narration` (the agent's intermediate