@stigmer/runner 3.0.8-dev.20260613085218 → 3.0.8

package/src/shared/__tests__/status-offload.test.ts

@@ -0,0 +1,256 @@
+import { describe, it, expect, vi } from "vitest";
+import { create, toBinary } from "@bufbuild/protobuf";
+import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import { AgentMessageSchema, ToolCallSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
+import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import type { ToolCall } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
+import type { ArtifactStorage } from "../artifact-storage.js";
+import {
+  offloadOversizedToolOutputs,
+  enforceStatusSizeLimit,
+  detectImagePayload,
+} from "../status-offload.js";
+
+function makeFakeStorage() {
+  const uploads: { key: string; size: number; contentType?: string }[] = [];
+  const storage: ArtifactStorage = {
+    upload: vi.fn(async (key: string, content: Buffer, contentType?: string) => {
+      uploads.push({ key, size: content.length, contentType });
+      return key;
+    }),
+    getDownloadUrl: vi.fn(async (key: string) => `https://artifacts.local/${key}`),
+    exists: vi.fn(async () => true),
+  };
+  return { storage, uploads };
+}
+
+function statusWithToolCall(tc: ToolCall): AgentExecutionStatus {
+  return create(AgentExecutionStatusSchema, {
+    messages: [create(AgentMessageSchema, { toolCalls: [tc] })],
+  });
+}
+
+function encodedSize(status: AgentExecutionStatus): number {
+  return toBinary(AgentExecutionStatusSchema, status).length;
+}
+
+const BIG_BASE64_IMAGE = Buffer.from("x".repeat(4096)).toString("base64");
+
+describe("detectImagePayload", () => {
+  it("extracts an MCP image content block", () => {
+    const result = JSON.stringify([
+      { type: "text", text: "Screenshot captured" },
+      { type: "image", data: BIG_BASE64_IMAGE, mimeType: "image/png" },
+    ]);
+    const img = detectImagePayload(result);
+    expect(img).not.toBeNull();
+    expect(img?.mimeType).toBe("image/png");
+    expect(img?.base64).toBe(BIG_BASE64_IMAGE);
+  });
+
+  it("extracts a raw data URL", () => {
+    const img = detectImagePayload(`data:image/jpeg;base64,${BIG_BASE64_IMAGE}`);
+    expect(img?.mimeType).toBe("image/jpeg");
+    expect(img?.base64).toBe(BIG_BASE64_IMAGE);
+  });
+
+  it("returns null for plain text", () => {
+    expect(detectImagePayload("just some logs")).toBeNull();
+  });
+
+  it("extracts an image from a serialized LangChain envelope (kwargs.content)", () => {
+    // Defensive shape: blocks nested under kwargs.content rather than a
+    // top-level array. The extractor normalizes to a top-level array, but
+    // detection must still cope if that ever drifts.
+    const result = JSON.stringify({
+      lc: 1,
+      type: "constructor",
+      id: ["langchain_core", "messages", "ToolMessage"],
+      kwargs: { content: [{ type: "image", data: BIG_BASE64_IMAGE, mimeType: "image/png" }] },
+    });
+    const img = detectImagePayload(result);
+    expect(img?.mimeType).toBe("image/png");
+    expect(img?.base64).toBe(BIG_BASE64_IMAGE);
+  });
+});
+
+describe("offloadOversizedToolOutputs", () => {
+  it("offloads an oversized image result and renders it inline-able", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    const result = JSON.stringify([
+      { type: "image", data: BIG_BASE64_IMAGE, mimeType: "image/png" },
+    ]);
+    const tc = create(ToolCallSchema, { id: "tc-1", name: "screenshot", result });
+    const status = statusWithToolCall(tc);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+      maxInlineBytes: 256,
+    });
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef).toBeDefined();
+    expect(out.outputRef?.isImage).toBe(true);
+    expect(out.outputRef?.mimeType).toBe("image/png");
+    expect(out.outputRef?.downloadUrl).toContain("artifacts/exec-1/toolcalls/tc-1.png");
+    // result is collapsed to a short label, no longer the giant blob.
+    expect(out.result.length).toBeLessThan(200);
+    expect(uploads).toHaveLength(1);
+    expect(uploads[0].contentType).toBe("image/png");
+  });
+
+  it("offloads oversized text with a preview head and text/plain upload", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    const result = "LOG ".repeat(2000); // ~8 KB
+    const tc = create(ToolCallSchema, { id: "tc-2", name: "Shell", result });
+    const status = statusWithToolCall(tc);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+      maxInlineBytes: 256,
+    });
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef?.isImage).toBe(false);
+    expect(out.outputRef?.mimeType).toBe("text/plain");
+    expect(out.outputRef?.truncatedPreview.length).toBeGreaterThan(0);
+    expect(out.result).toContain("view full output");
+    expect(uploads).toHaveLength(1);
+    expect(uploads[0].contentType).toBe("text/plain");
+  });
+
+  it("offloads a SMALL image regardless of size (output_ref is the only render path)", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    // A tiny image, far below the inline byte budget. It must still offload,
+    // because the UI can only render an image through ToolCallOutputRef.
+    const smallImage = Buffer.from("tiny-png").toString("base64");
+    const result = JSON.stringify([
+      { type: "image", data: smallImage, mimeType: "image/png" },
+    ]);
+    const tc = create(ToolCallSchema, { id: "tc-img", name: "screenshot", result });
+    const status = statusWithToolCall(tc);
+    expect(result.length).toBeLessThan(256);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+      maxInlineBytes: 256,
+    });
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef?.isImage).toBe(true);
+    expect(out.outputRef?.downloadUrl).toContain("artifacts/exec-1/toolcalls/tc-img.png");
+    expect(out.result).not.toContain(smallImage);
+    expect(uploads).toHaveLength(1);
+    expect(uploads[0].contentType).toBe("image/png");
+  });
+
+  it("leaves small results untouched (no artifact, no ref)", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    const tc = create(ToolCallSchema, { id: "tc-3", name: "Read", result: "small output" });
+    const status = statusWithToolCall(tc);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+      maxInlineBytes: 256,
+    });
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef).toBeUndefined();
+    expect(out.result).toBe("small output");
+    expect(uploads).toHaveLength(0);
+  });
+
+  it("is idempotent and dedupes uploads when result is re-inflated with identical bytes", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    const result = "LOG ".repeat(2000);
+    const tc = create(ToolCallSchema, { id: "tc-4", name: "Shell", result });
+    const status = statusWithToolCall(tc);
+    const ctx = { artifactStorage: storage, executionId: "exec-1", maxInlineBytes: 256 };
+
+    await offloadOversizedToolOutputs(status, ctx);
+    expect(uploads).toHaveLength(1);
+
+    // Simulate mergeToolCallEvent re-inflating the inline result with the SAME
+    // bytes on a subsequent persist; the ref's content hash still matches.
+    status.messages[0].toolCalls[0].result = result;
+    await offloadOversizedToolOutputs(status, ctx);
+
+    expect(uploads).toHaveLength(1); // not re-uploaded
+    expect(status.messages[0].toolCalls[0].result).toContain("view full output");
+  });
+
+  it("keeps the encoded status small after offloading a large blob", async () => {
+    const { storage } = makeFakeStorage();
+    const result = JSON.stringify([
+      { type: "image", data: Buffer.from("y".repeat(500_000)).toString("base64"), mimeType: "image/png" },
+    ]);
+    const tc = create(ToolCallSchema, { id: "tc-5", name: "screenshot", result });
+    const status = statusWithToolCall(tc);
+    expect(encodedSize(status)).toBeGreaterThan(500_000);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+    });
+
+    expect(encodedSize(status)).toBeLessThan(10_000);
+  });
+
+  it("falls back to inline truncation when the upload fails (never throws)", async () => {
+    const storage: ArtifactStorage = {
+      upload: vi.fn(async () => { throw new Error("storage down"); }),
+      getDownloadUrl: vi.fn(async (k: string) => k),
+      exists: vi.fn(async () => false),
+    };
+    const result = "LOG ".repeat(2000);
+    const tc = create(ToolCallSchema, { id: "tc-6", name: "Shell", result });
+    const status = statusWithToolCall(tc);
+    vi.spyOn(console, "warn").mockImplementation(() => {});
+
+    await expect(
+      offloadOversizedToolOutputs(status, {
+        artifactStorage: storage,
+        executionId: "exec-1",
+        maxInlineBytes: 256,
+      }),
+    ).resolves.toBeUndefined();
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef).toBeUndefined();
+    expect(out.result).toContain("offload failed");
+    expect(out.result.length).toBeLessThan(result.length);
+  });
+});
+
+describe("enforceStatusSizeLimit", () => {
+  it("returns false and changes nothing when under the limit", () => {
+    const tc = create(ToolCallSchema, { id: "tc", name: "Read", result: "tiny" });
+    const status = statusWithToolCall(tc);
+    expect(enforceStatusSizeLimit(status, 1024 * 1024)).toBe(false);
+    expect(status.messages[0].toolCalls[0].result).toBe("tiny");
+  });
+
+  it("elides the largest inline fields until the status fits", () => {
+    const big = "Z".repeat(50_000);
+    const status = create(AgentExecutionStatusSchema, {
+      messages: [
+        create(AgentMessageSchema, {
+          toolCalls: [
+            create(ToolCallSchema, { id: "a", name: "Shell", result: big }),
+            create(ToolCallSchema, { id: "b", name: "Read", result: big }),
+          ],
+        }),
+      ],
+    });
+    const before = encodedSize(status);
+    expect(before).toBeGreaterThan(80_000);
+
+    const elided = enforceStatusSizeLimit(status, 4_000);
+    expect(elided).toBe(true);
+    expect(encodedSize(status)).toBeLessThanOrEqual(4_000);
+  });
+});

package/src/shared/__tests__/status.test.ts

@@ -1,4 +1,5 @@
 import { describe, it, expect, vi } from "vitest";
+import { ConnectError, Code } from "@connectrpc/connect";
 import { create } from "@bufbuild/protobuf";
 import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
 import { ExecutionControlSignal, ExecutionPhase } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
@@ -123,6 +124,204 @@ describe("persistStatus", () => {
     expect(consoleSpy).toHaveBeenCalled();
     consoleSpy.mockRestore();
   });
+
+  it("hard-elides and retries once on a resource_exhausted (code 8) failure", async () => {
+    const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const updateStatus = vi
+      .fn()
+      .mockRejectedValueOnce({ code: 8, message: "resource_exhausted: exceeds maximum size" })
+      .mockResolvedValueOnce({ signal: ExecutionControlSignal.STOP });
+    const mockClient = { updateStatus } as any;
+    const status = create(AgentExecutionStatusSchema, {
+      phase: ExecutionPhase.EXECUTION_IN_PROGRESS,
+    });
+
+    const signal = await persistStatus(mockClient, "exec-too-big", status);
+    expect(signal).toBe(ExecutionControlSignal.STOP);
+    expect(updateStatus).toHaveBeenCalledTimes(2);
+    consoleSpy.mockRestore();
+  });
+
+  it("offloads oversized tool outputs before persisting when given an offload context", async () => {
+    const uploaded: string[] = [];
+    const offload = {
+      executionId: "exec-off",
+      artifactStorage: {
+        upload: vi.fn(async (key: string) => { uploaded.push(key); return key; }),
+        getDownloadUrl: vi.fn(async (key: string) => `https://artifacts.local/${key}`),
+        exists: vi.fn(async () => true),
+      },
+      maxInlineBytes: 256,
+    };
+    const mockClient = {
+      updateStatus: vi.fn().mockResolvedValue({ signal: ExecutionControlSignal.UNSPECIFIED }),
+    } as any;
+    const { AgentMessageSchema, ToolCallSchema } = await import(
+      "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb"
+    );
+    const status = create(AgentExecutionStatusSchema, {
+      phase: ExecutionPhase.EXECUTION_IN_PROGRESS,
+      messages: [
+        create(AgentMessageSchema, {
+          toolCalls: [create(ToolCallSchema, { id: "t", name: "Shell", result: "X".repeat(5000) })],
+        }),
+      ],
+    });
+
+    await persistStatus(mockClient, "exec-off", status, { offload });
+
+    expect(uploaded).toHaveLength(1);
+    const [, persisted] = mockClient.updateStatus.mock.calls[0];
+    expect(persisted.messages[0].toolCalls[0].outputRef).toBeDefined();
+    expect(persisted.messages[0].toolCalls[0].result.length).toBeLessThan(5000);
+  });
+});
+
+// Migrated from grpc-retry.test.ts: the transient-backoff loop now lives inside
+// persistStatus so every persist (streaming + terminal, every harness) shares
+// one chokepoint that bounds size AND retries transient transport errors.
+describe("persistStatus — transient retry", () => {
+  const noDelay = async () => {};
+  const emptyStatus = () => create(AgentExecutionStatusSchema, {});
+  function retryClient(impl: () => Promise<{ signal: ExecutionControlSignal }>) {
+    return { updateStatus: vi.fn(impl) } as any;
+  }
+
+  it("returns the signal on the first successful attempt", async () => {
+    const client = retryClient(async () => ({ signal: ExecutionControlSignal.STOP }));
+    const signal = await persistStatus(client, "exec-1", emptyStatus(), {
+      retry: { delayFn: noDelay },
+    });
+    expect(signal).toBe(ExecutionControlSignal.STOP);
+    expect(client.updateStatus).toHaveBeenCalledOnce();
+  });
+
+  it("retries a transient UNAVAILABLE and then succeeds", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      if (attempt === 1) throw new ConnectError("down", Code.Unavailable);
+      return { signal: ExecutionControlSignal.UNSPECIFIED };
+    });
+    const signal = await persistStatus(client, "exec-2", emptyStatus(), {
+      retry: { delayFn: noDelay },
+    });
+    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
+    expect(attempt).toBe(2);
+  });
+
+  it("does not retry terminal errors (NOT_FOUND)", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      throw new ConnectError("gone", Code.NotFound);
+    });
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const signal = await persistStatus(client, "exec-3", emptyStatus(), {
+      retry: { delayFn: noDelay },
+    });
+    errorSpy.mockRestore();
+    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
+    expect(attempt).toBe(1);
+  });
+
+  it("does not retry INTERNAL (neither retryable nor terminal)", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      throw new ConnectError("internal", Code.Internal);
+    });
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    await persistStatus(client, "exec-int", emptyStatus(), {
+      retry: { maxRetries: 3, delayFn: noDelay },
+    });
+    errorSpy.mockRestore();
+    expect(attempt).toBe(1);
+  });
+
+  it("applies exponential backoff and gives up after the retry budget", async () => {
+    const delays: number[] = [];
+    const client = retryClient(async () => {
+      throw new ConnectError("down", Code.Unavailable);
+    });
+    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const signal = await persistStatus(client, "exec-5", emptyStatus(), {
+      retry: {
+        baseDelayMs: 100,
+        backoffFactor: 2,
+        maxRetries: 3,
+        delayFn: async (ms) => { delays.push(ms); },
+      },
+    });
+    warnSpy.mockRestore();
+    errorSpy.mockRestore();
+    expect(delays).toEqual([100, 200, 400]);
+    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
+  });
+
+  it("respects a custom backoff factor", async () => {
+    const delays: number[] = [];
+    const client = retryClient(async () => {
+      throw new ConnectError("down", Code.Unavailable);
+    });
+    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    await persistStatus(client, "exec-bf", emptyStatus(), {
+      retry: {
+        baseDelayMs: 50,
+        backoffFactor: 3,
+        maxRetries: 3,
+        delayFn: async (ms) => { delays.push(ms); },
+      },
+    });
+    warnSpy.mockRestore();
+    errorSpy.mockRestore();
+    expect(delays).toEqual([50, 150, 450]);
+  });
+
+  it("maxRetries = 0 means a single attempt", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      throw new ConnectError("down", Code.Unavailable);
+    });
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    await persistStatus(client, "exec-0", emptyStatus(), {
+      retry: { maxRetries: 0, delayFn: noDelay },
+    });
+    errorSpy.mockRestore();
+    expect(attempt).toBe(1);
+  });
+
+  it("handles a mixed sequence: transient then terminal", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      if (attempt === 1) throw new ConnectError("down", Code.Unavailable);
+      throw new ConnectError("gone", Code.NotFound);
+    });
+    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const signal = await persistStatus(client, "exec-mix", emptyStatus(), {
+      retry: { delayFn: noDelay },
+    });
+    warnSpy.mockRestore();
+    errorSpy.mockRestore();
+    expect(attempt).toBe(2);
+    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
+  });
+
+  it("never throws regardless of error type", async () => {
+    const client = retryClient(async () => {
+      throw new ConnectError("server exploded", Code.Internal);
+    });
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    await expect(
+      persistStatus(client, "exec-7", emptyStatus(), { retry: { delayFn: noDelay } }),
+    ).resolves.toBe(ExecutionControlSignal.UNSPECIFIED);
+    errorSpy.mockRestore();
+  });
 });
 
 describe("reportSetupProgress", () => {

package/src/shared/grpc-retry.ts

@@ -1,20 +1,15 @@
 /**
- * Exponential-backoff retry wrapper for gRPC status persistence.
+ * gRPC error classification for status persistence retries.
  *
- * Wraps the raw StigmerClient.updateStatus() call with retry logic
- * that classifies gRPC error codes as retryable vs terminal. A failed
- * status update must never crash the streaming loop — on permanent
- * failure, the signal falls back to UNSPECIFIED.
- *
- * Used by the ExecuteDeepAgent streaming loop. The simpler
- * fire-and-forget persistStatus in shared/status.ts is preserved
- * for ExecuteCursor and non-streaming callers.
+ * Status persistence flows through a single chokepoint — `persistStatus` in
+ * status.ts — which uses these helpers to decide whether a failed
+ * `updateStatus` should back off and retry (transient transport errors) or
+ * fail fast (deterministic errors). This module is intentionally just the
+ * classification policy + its options type, kept small and separately tested;
+ * the persist loop that consumes it lives with the rest of the persist logic.
  */
 
 import { ConnectError, Code } from "@connectrpc/connect";
-import { ExecutionControlSignal } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
-import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
-import type { StigmerClient } from "../client/stigmer-client.js";
 
 export interface RetryOptions {
   /** Base delay before the first retry (ms). Default: 100. */
@@ -38,6 +33,7 @@ const TERMINAL_CODES = new Set<Code>([
   Code.PermissionDenied,
 ]);
 
+/** True for transient transport errors that are worth retrying with backoff. */
 export function isRetryableError(err: unknown): boolean {
   if (err instanceof ConnectError) {
     return RETRYABLE_CODES.has(err.code);
@@ -45,69 +41,10 @@ export function isRetryableError(err: unknown): boolean {
   return false;
 }
 
+/** True for deterministic errors that retrying cannot fix. */
 export function isTerminalError(err: unknown): boolean {
   if (err instanceof ConnectError) {
     return TERMINAL_CODES.has(err.code);
   }
   return false;
 }
-
-function defaultDelay(ms: number): Promise<void> {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-/**
- * Persist execution status with exponential-backoff retry.
- *
- * Returns the ExecutionControlSignal from the server on success.
- * On permanent or exhausted-retry failure, logs the error and
- * returns UNSPECIFIED (never throws).
- */
-export async function persistWithRetry(
-  client: StigmerClient,
-  executionId: string,
-  status: AgentExecutionStatus,
-  options?: RetryOptions,
-): Promise<ExecutionControlSignal> {
-  const baseDelay = options?.baseDelayMs ?? 100;
-  const factor = options?.backoffFactor ?? 2;
-  const maxRetries = options?.maxRetries ?? 3;
-  const delay = options?.delayFn ?? defaultDelay;
-
-  let lastError: unknown;
-
-  for (let attempt = 0; attempt <= maxRetries; attempt++) {
-    try {
-      const response = await client.updateStatus(executionId, status);
-      return response.signal;
-    } catch (err: unknown) {
-      lastError = err;
-
-      if (isTerminalError(err)) {
-        const code = (err as ConnectError).code;
-        console.error(
-          `[grpc-retry] Terminal error persisting status for ${executionId}: ` +
-          `code=${Code[code]} (attempt ${attempt + 1}/${maxRetries + 1})`,
-        );
-        return ExecutionControlSignal.UNSPECIFIED;
-      }
-
-      if (!isRetryableError(err) || attempt === maxRetries) {
-        break;
-      }
-
-      const delayMs = baseDelay * Math.pow(factor, attempt);
-      console.warn(
-        `[grpc-retry] Retryable error for ${executionId}: ` +
-        `code=${err instanceof ConnectError ? Code[err.code] : "unknown"} ` +
-        `(attempt ${attempt + 1}/${maxRetries + 1}, retry in ${delayMs}ms)`,
-      );
-      await delay(delayMs);
-    }
-  }
-
-  console.error(
-    `[grpc-retry] All retries exhausted for ${executionId}: ${lastError}`,
-  );
-  return ExecutionControlSignal.UNSPECIFIED;
-}

package/src/shared/stall-watchdog.ts

@@ -0,0 +1,122 @@
+/**
+ * Progress-based stall watchdog.
+ *
+ * Long-running agent turns are kept alive against Temporal by a periodic
+ * keep-alive heartbeat (see {@link ./heartbeat.ts}). That heartbeat proves the
+ * runner *process* is alive, but it says nothing about whether the agent is
+ * making *progress*: a turn that wedges inside the harness stream loop (a tool
+ * call that never returns, a model connection that silently dies) keeps
+ * heartbeating forever and never times out. The execution then sits at
+ * EXECUTION_IN_PROGRESS indefinitely.
+ *
+ * This watchdog closes that gap. It is an *out-of-band* timer: callers report
+ * progress via {@link StallWatchdog.recordActivity} on every stream event AND
+ * every token delta, and the watchdog fires `onStall` once if no progress is
+ * reported for `stallMs`. "Out-of-band" is the deliberate correctness property
+ * — the check runs on its own timer rather than inside the consumer's loop, so
+ * it catches a true "no new events for N minutes" hang. An in-band check that
+ * only runs when the loop advances cannot fire while the loop is blocked
+ * awaiting the next event, which is exactly the wedge we need to detect.
+ *
+ * This module is the convergence target for stall detection across harnesses:
+ * the Cursor harness wires it here, and the native deep-agent harness has its
+ * own (harness-local, in-band) check that should later converge onto this one.
+ */
+
+/**
+ * Canonical default stall window. Harnesses may override with a larger value
+ * when their tool calls routinely run longer without emitting stream activity.
+ */
+export const DEFAULT_STALL_TIMEOUT_MS = 120_000;
+
+/**
+ * Thrown / reported when an agent stream makes no progress for longer than the
+ * configured stall window. Carries the observed idle duration so callers can
+ * build an actionable, recognizable error message.
+ */
+export class StallTimeoutError extends Error {
+  constructor(
+    public readonly stalledMs: number,
+    detail?: string,
+  ) {
+    super(
+      `Agent stream stalled: no activity for ${Math.round(stalledMs / 1000)}s` +
+        (detail ? ` (${detail})` : ""),
+    );
+    this.name = "StallTimeoutError";
+  }
+}
+
+/**
+ * Recognizable prefix on every stall-induced failure message. Single source of
+ * truth so callers (and any future UI/log keying) match on one constant.
+ */
+export const STALL_ERROR_PREFIX = "[StallTimeoutError]";
+
+/**
+ * Build the user-facing failure text for a stall. Keeps the canonical wording
+ * (prefix + actionable "Retry or resume.") next to the error it describes.
+ */
+export function formatStallFailure(error: StallTimeoutError): string {
+  return `${STALL_ERROR_PREFIX} ${error.message}. Retry or resume.`;
+}
+
+export interface StallWatchdog {
+  /** Reset the idle timer. Call on every stream event AND every token delta. */
+  recordActivity(): void;
+  /** Disarm the watchdog. Idempotent; safe to call in a `finally`. */
+  stop(): void;
+}
+
+/**
+ * Start an out-of-band stall watchdog.
+ *
+ * @param stallMs  Idle window after which `onStall` fires.
+ * @param onStall  Invoked at most once with the observed idle duration (ms)
+ *                 when `Date.now() - lastActivityAt` exceeds `stallMs`. The
+ *                 watchdog disarms itself before invoking, so `onStall` runs
+ *                 exactly once even if its handler is slow.
+ *
+ * The poll interval is `stallMs / 4` capped at 15s: frequent enough to detect a
+ * stall promptly without busy-looping, and bounded so a large `stallMs` still
+ * polls on a sane cadence.
+ */
+export function startStallWatchdog(
+  stallMs: number,
+  onStall: (idleMs: number) => void,
+): StallWatchdog {
+  let lastActivityAt = Date.now();
+  let fired = false;
+  let timer: ReturnType<typeof setInterval> | undefined;
+
+  const tickMs = Math.min(Math.max(Math.floor(stallMs / 4), 1), 15_000);
+
+  const disarm = (): void => {
+    if (timer !== undefined) {
+      clearInterval(timer);
+      timer = undefined;
+    }
+  };
+
+  timer = setInterval(() => {
+    if (fired) return;
+    const idleMs = Date.now() - lastActivityAt;
+    if (idleMs >= stallMs) {
+      fired = true;
+      disarm();
+      onStall(idleMs);
+    }
+  }, tickMs);
+  // Do not keep the event loop alive solely for the watchdog.
+  timer.unref?.();
+
+  return {
+    recordActivity(): void {
+      lastActivityAt = Date.now();
+    },
+    stop(): void {
+      fired = true;
+      disarm();
+    },
+  };
+}