agent-relay-runner 0.88.2 → 0.88.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-runner",
-  "version": "0.88.2",
+  "version": "0.88.3",
   "description": "Unified provider lifecycle runner for Agent Relay",
   "type": "module",
   "bin": {

package/plugins/claude/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "agent-relay-runner",
   "description": "Thin Agent Relay runner bridge for Claude Code",
-  "version": "0.88.2",
+  "version": "0.88.3",
   "agentRelayContracts": {
     "providerPluginProtocol": 1
   }

package/src/adapter.ts

@@ -25,6 +25,10 @@ export interface ProviderStatusEvent {
   label?: string;
   role?: string;
   parentId?: string;
+  // OS pids of the process(es) this busy work tracks (#650). When supplied, the
+  // runner reconciles them against actual liveness and clears the busy marker if
+  // every pid is dead — recovering from a tracked process killed out-of-band.
+  pids?: number[];
   providerState?: Record<string, unknown>;
   metadata?: Record<string, unknown>;
 }

package/src/adapters/codex.ts

@@ -50,6 +50,9 @@ export class CodexAdapter implements ProviderAdapter {
   // Active turn id for the main thread, captured from turn/started so an interrupt
   // can target the in-flight turn. Cleared on turn/completed.
   private activeTurnId?: string;
+  // #653: monotonic id for a turn we synthesize from live item activity when a real
+  // turn/started never reached us — so a working Codex agent isn't reported `idle`.
+  private syntheticTurnSeq = 0;
   private readonly agentMessageCapture = new CodexAgentMessageCapture();
   private readonly itemTextBuffers = new Map<string, string>();
   private readonly itemTextBufferTypes = new Map<string, string>();
@@ -578,8 +581,26 @@ export class CodexAdapter implements ProviderAdapter {
     return [...this.sessionEvents];
   }
 
+  // #653: Codex busy-state normally flips on turn/started, but the app-server can stream
+  // item activity (which already drives the dashboard chat mirror) before — or without —
+  // a turn/started reaching us, leaving a visibly-working agent reported `idle` (a reap
+  // hazard). Derive busy from that SAME live activity: any in-flight item event with no
+  // active turn means a turn IS running we missed, so synthesize one and report busy.
+  // turn/completed → finishMainTurn clears it; we never synthesize on a closing
+  // item.completed, so a legitimately ended turn is not re-flipped to busy.
+  private ensureMainTurnActive(turnId?: string): void {
+    if (this.activeTurnId) return;
+    this.activeTurnId = turnId ?? `codex-synth-turn-${++this.syntheticTurnSeq}`;
+    this.statusCb({ status: "busy", reason: "provider-turn", id: this.activeTurnId });
+  }
+
   private handleCodexItemDelta(method: string, params: Record<string, unknown> | undefined): void {
     if (!method.includes("item/") && !method.includes("item.")) return;
+    // Live item stream (start/delta/progress) = a turn in flight. A closing
+    // item.completed is handled by handleCodexItem and must not synthesize a turn.
+    if (!method.includes("/completed") && !method.includes(".completed")) {
+      this.ensureMainTurnActive(stringValue(params?.turnId));
+    }
     const item = isRecord(params?.item) ? params.item : undefined;
     const itemId = codexItemId(params) ?? codexItemId(item);
     const type = codexItemTypeFromMethod(method) ?? stringValue(item?.type);

package/src/claim-tracker.ts

@@ -17,6 +17,10 @@ interface WorkRecord {
   role?: string;
   parentId?: string;
   metadata?: Record<string, unknown>;
+  // OS pids of the underlying process(es) this work tracks (#650). When present,
+  // a periodic liveness reconcile clears the record once every pid is dead — so a
+  // tracked process killed out-of-band (no exit event) can't strand the agent busy.
+  pids?: number[];
   startedAt: number;
 }
 
@@ -40,6 +44,7 @@ export class ClaimTracker {
       role: details.role ?? existing?.role,
       parentId: details.parentId ?? existing?.parentId,
       metadata: details.metadata ?? existing?.metadata,
+      pids: details.pids ?? existing?.pids,
       startedAt: existing?.startedAt ?? Date.now(),
     });
     return before !== this.currentStatus();
@@ -106,4 +111,30 @@ export class ClaimTracker {
   activeWork(): WorkRecord[] {
     return [...this.work.values()];
   }
+
+  /** True iff any active work carries pids the liveness reconcile can check (#650). */
+  hasLivenessTrackedWork(): boolean {
+    for (const item of this.work.values()) {
+      if (item.pids && item.pids.length > 0) return true;
+    }
+    return false;
+  }
+
+  /**
+   * Reconcile pid-bearing work against actual process liveness (#650): drop any
+   * record whose tracked pids are ALL dead — its underlying process died out-of-band
+   * (killed/crashed/orphaned) with no exit event to clear the busy marker. Records
+   * with no pids are never touched (no liveness signal to act on). Returns the removed
+   * records so the caller can emit the matching completion/abort + republish status.
+   */
+  reconcileLiveness(isAlive: (pid: number) => boolean): WorkRecord[] {
+    const removed: WorkRecord[] = [];
+    for (const [key, item] of this.work) {
+      if (!item.pids || item.pids.length === 0) continue;
+      if (item.pids.some(isAlive)) continue;
+      this.work.delete(key);
+      removed.push(item);
+    }
+    return removed;
+  }
 }

package/src/control-server.ts

@@ -446,6 +446,12 @@ async function handleStatus(req: Request, options: ControlServerOptions): Promis
   const isWorkKind = (item: unknown): item is "provider-turn" | "subagent" | "background-script" =>
     item === "provider-turn" || item === "subagent" || item === "background-script";
   const clear = Array.isArray(body?.clear) ? body.clear.filter(isWorkKind) : undefined;
+  // #650: optional pids the runner reconciles against process liveness so a tracked
+  // command/shell killed out-of-band clears the busy marker instead of stranding it.
+  const pidsInput = Array.isArray((body as { pids?: unknown })?.pids) ? (body as { pids: unknown[] }).pids : undefined;
+  const pids = pidsInput
+    ?.map((p) => (typeof p === "number" ? p : Number(p)))
+    .filter((p) => Number.isInteger(p) && p > 0);
   const timeline = statusTimelineEvent(body);
   const update: ProviderStatusEvent = {
     status,
@@ -454,6 +460,7 @@ async function handleStatus(req: Request, options: ControlServerOptions): Promis
     ...(clear?.length ? { clear: [...new Set(clear)] } : {}),
     ...(timeline ? { timeline } : {}),
     ...(typeof body?.id === "string" ? { id: body.id } : {}),
+    ...(pids?.length ? { pids } : {}),
     ...(typeof body?.label === "string" ? { label: body.label } : {}),
     ...(typeof body?.role === "string" ? { role: body.role } : {}),
     ...(typeof body?.parentId === "string" ? { parentId: body.parentId } : {}),

package/src/runner-core.ts

@@ -5,6 +5,7 @@ import { dirname, join } from "node:path";
 import type { AgentLifecycle, AgentProfile, ContextState, Message, MessageSessionMeta, SendMessageInput, TaskStatusInput, WorkspaceMetadata } from "agent-relay-sdk";
 import { errMessage, RelayBusClient, RelayHttpClient } from "agent-relay-sdk";
 import { contextStateFromProbeMetrics, quotaStateFromProbeMetrics } from "agent-relay-sdk/context-probe";
+import { isPidAlive } from "agent-relay-sdk/process-utils";
 import { providerAttachmentText, providerMessageText } from "./adapter";
 import type { ManagedProcess, ProviderAdapter, ProviderConfig, ProviderPermissionDecision, ProviderPermissionDecisionInput, ProviderSessionEvent, ProviderStatusUpdate, RunnerSpawnConfig, SemanticStatus, TerminalAttachSpec } from "./adapter";
 import { messagesWithCachedAttachments } from "./attachment-cache";
@@ -109,6 +110,8 @@ interface ActiveTaskClaim {
 
 const CLAIM_RENEW_INTERVAL_MS = 5 * 60 * 1000;
 const HTTP_LIVENESS_INTERVAL_MS = 20_000;
+// #650: how often to reconcile pid-bearing busy work against actual process liveness.
+const BUSY_LIVENESS_RECONCILE_MS = 5_000;
 const HTTP_LIVENESS_LOG_INTERVAL_MS = 5 * 60 * 1000;
 const TOKEN_RENEW_RETRY_MS = 60_000;
 // Debounce reactive token recovery so a burst of 401-ing calls in the same window
@@ -204,6 +207,7 @@ export class AgentRunner {
   private drainTimer?: Timer;
   private claimRenewTimer?: Timer;
   private httpLivenessTimer?: Timer;
+  private livenessReconcileTimer?: Timer;
   private httpLivenessInFlight = false;
   private httpLivenessAuthFailed = false;
   private httpLivenessLastLog?: { key: string; at: number };
@@ -472,6 +476,7 @@ export class AgentRunner {
     this.httpLivenessTimer = undefined;
     if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
     this.tokenRenewTimer = undefined;
+    this.disarmBusyLivenessReconcile();
     this.busyReconciler.disarm();
     this.stopReasoningTail();
     this.obligationCache.stop();
@@ -1247,15 +1252,30 @@ export class AgentRunner {
     if (status === "busy" || status === "idle") this.terminalProviderExit = undefined;
     if (status === "busy") {
       this.claims.clearTerminalStatus();
-      this.claims.startWork(reason, id, typeof update === "string" ? {} : {
+      // #650: liveness-track the pids a reporter explicitly supplied; otherwise, for a
+      // provider-turn, fall back to the managed provider process itself. If that process
+      // dies out-of-band with no exit event reaching the runner (crash/orphan/external
+      // kill), the reconcile clears the stuck turn → idle. A dead pid means the turn is
+      // definitively over, so this can never falsely clear a live turn (#653's hazard).
+      const reportedPids = typeof update === "string" ? undefined : update.pids;
+      const livenessPids = reportedPids?.length
+        ? reportedPids
+        : reason === "provider-turn" && typeof this.process?.pid === "number" && this.process.pid > 0
+          ? [this.process.pid]
+          : undefined;
+      this.claims.startWork(reason, id, typeof update === "string" ? (livenessPids ? { pids: livenessPids } : {}) : {
         label: update.label,
         role: update.role,
         parentId: update.parentId,
+        ...(livenessPids ? { pids: livenessPids } : {}),
         metadata: {
           ...(update.metadata ?? {}),
           ...(update.providerState ? { providerState: update.providerState } : {}),
         },
       });
+      // A busy report with liveness-trackable pids arms the periodic reconcile (self-disarms
+      // once no pid-bearing work remains, so idle/untracked agents pay nothing).
+      if (livenessPids?.length) this.armBusyLivenessReconcile();
       if (reason === "provider-turn") {
         for (const claim of this.activeTaskClaims.values()) claim.observedProviderBusy = true;
       }
@@ -1690,6 +1710,44 @@ export class AgentRunner {
     this.publishStatus();
   }
 
+  // #650: reconcile pid-bearing busy work against actual process liveness. A tracked
+  // command/shell that dies out-of-band (killed by the operator, crashes, orphaned)
+  // sends no exit event, so its busy marker would otherwise never clear and the agent
+  // shows `busy` forever — blocking transient auto-reap and (for Claude) holding quota.
+  // The timer self-disarms once no pid-bearing work remains; it is armed lazily when a
+  // busy report carrying pids arrives, so idle/non-tracked agents pay nothing.
+  private armBusyLivenessReconcile(): void {
+    if (this.livenessReconcileTimer || this.stopped) return;
+    this.livenessReconcileTimer = setInterval(() => this.runBusyLivenessReconcile(), BUSY_LIVENESS_RECONCILE_MS);
+  }
+
+  private disarmBusyLivenessReconcile(): void {
+    if (this.livenessReconcileTimer) clearInterval(this.livenessReconcileTimer);
+    this.livenessReconcileTimer = undefined;
+  }
+
+  private runBusyLivenessReconcile(): void {
+    if (this.stopped) {
+      this.disarmBusyLivenessReconcile();
+      return;
+    }
+    const removed = this.claims.reconcileLiveness(isPidAlive);
+    if (removed.length > 0) {
+      this.sessionLog(`cleared ${removed.length} stuck busy work item(s) whose tracked process died out-of-band: ${removed.map((w) => `${w.kind}:${w.id}`).join(", ")}`);
+      // A cleared provider-turn must also reset the turn bookkeeping so a later idle
+      // doesn't double-end it and the reasoning tail stops.
+      if (removed.some((w) => w.kind === "provider-turn")) {
+        this.currentTurnId = undefined;
+        this.currentTurnStartedAt = undefined;
+        this.compactionMidTurn = false;
+        this.busyReconciler.disarm();
+        this.stopReasoningTail();
+      }
+      this.publishStatus();
+    }
+    if (!this.claims.hasLivenessTrackedWork()) this.disarmBusyLivenessReconcile();
+  }
+
   // --- Turn-step tailer (item 5) ------------------------------------------------------
   // Tail the in-flight turn's Claude transcript and surface new narration/reasoning/tool
   // steps as session events, in transcript order. `narration` (the agent's intermediate