@stigmer/runner 3.0.8-dev.20260613085218 → 3.0.9-dev.20260615145121

package/src/shared/__tests__/stall-watchdog.test.ts

@@ -0,0 +1,193 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import {
+  startStallWatchdog,
+  StallTimeoutError,
+  formatStallFailure,
+  STALL_ERROR_PREFIX,
+  DEFAULT_STALL_TIMEOUT_MS,
+} from "../stall-watchdog.js";
+
+describe("startStallWatchdog", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+  });
+
+  it("fires onStall once after the idle window elapses with no activity", () => {
+    const onStall = vi.fn();
+    const wd = startStallWatchdog(60_000, onStall);
+
+    // Just before the window: not yet fired.
+    vi.advanceTimersByTime(59_000);
+    expect(onStall).not.toHaveBeenCalled();
+
+    // Cross the window: fires exactly once.
+    vi.advanceTimersByTime(2_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+    expect(onStall.mock.calls[0][0]).toBeGreaterThanOrEqual(60_000);
+
+    // It self-disarms — no further fires even as time marches on.
+    vi.advanceTimersByTime(120_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+
+    wd.stop();
+  });
+
+  it("recordActivity resets the idle timer so progress keeps it alive", () => {
+    const onStall = vi.fn();
+    const wd = startStallWatchdog(60_000, onStall);
+
+    // Report progress every 40s for a while: the window never elapses.
+    for (let i = 0; i < 10; i++) {
+      vi.advanceTimersByTime(40_000);
+      wd.recordActivity();
+    }
+    expect(onStall).not.toHaveBeenCalled();
+
+    // Now go quiet past the window. Detection has up to one poll interval
+    // (stallMs/4, capped 15s) of slop because ticks fire on absolute
+    // boundaries, so advance stallMs + a full tick to guarantee a fire.
+    vi.advanceTimersByTime(76_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+
+    wd.stop();
+  });
+
+  it("stop() disarms the watchdog so onStall never fires", () => {
+    const onStall = vi.fn();
+    const wd = startStallWatchdog(60_000, onStall);
+
+    wd.stop();
+    vi.advanceTimersByTime(600_000);
+    expect(onStall).not.toHaveBeenCalled();
+  });
+
+  it("is robust to stop() called after firing (idempotent)", () => {
+    const onStall = vi.fn();
+    const wd = startStallWatchdog(30_000, onStall);
+
+    vi.advanceTimersByTime(31_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+
+    expect(() => wd.stop()).not.toThrow();
+    vi.advanceTimersByTime(60_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe("StallTimeoutError", () => {
+  it("builds a readable message from the idle duration", () => {
+    const err = new StallTimeoutError(120_000);
+    expect(err.name).toBe("StallTimeoutError");
+    expect(err.stalledMs).toBe(120_000);
+    expect(err.message).toContain("120s");
+  });
+
+  it("appends an optional detail", () => {
+    const err = new StallTimeoutError(90_000, "last tool: browser_capture");
+    expect(err.message).toContain("90s");
+    expect(err.message).toContain("last tool: browser_capture");
+  });
+});
+
+describe("formatStallFailure", () => {
+  it("prefixes the recognizable tag and appends actionable guidance", () => {
+    const msg = formatStallFailure(new StallTimeoutError(180_000, "last tool: browser_capture"));
+    expect(msg.startsWith(STALL_ERROR_PREFIX)).toBe(true);
+    expect(msg).toContain("180s");
+    expect(msg).toContain("last tool: browser_capture");
+    expect(msg).toContain("Retry or resume.");
+  });
+});
+
+describe("DEFAULT_STALL_TIMEOUT_MS", () => {
+  it("matches the canonical 120s default", () => {
+    expect(DEFAULT_STALL_TIMEOUT_MS).toBe(120_000);
+  });
+});
+
+/**
+ * Integration contract for how execute-cursor/index.ts wires the watchdog to
+ * the SDK Run: on stall it must (1) record the stall, (2) build a
+ * StallTimeoutError, and (3) cancel the run when the SDK supports cancel —
+ * exactly the onStall closure in index.ts. We assert that contract here against
+ * a faithful fake Run rather than standing up the full executeCursor activity
+ * (which has no end-to-end test harness), keeping the test focused and robust.
+ */
+describe("stall -> run.cancel() wiring contract (mirrors execute-cursor/index.ts)", () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+  });
+
+  interface FakeRun {
+    supports: (op: string) => boolean;
+    cancel: () => Promise<void>;
+  }
+
+  function wire(run: FakeRun, stallMs: number) {
+    const state: { stallDetected: boolean; stallError?: StallTimeoutError } = {
+      stallDetected: false,
+    };
+    let lastToolName: string | undefined;
+    const wd = startStallWatchdog(stallMs, (idleMs) => {
+      state.stallDetected = true;
+      state.stallError = new StallTimeoutError(idleMs, lastToolName ? `last tool: ${lastToolName}` : undefined);
+      if (run.supports?.("cancel")) void run.cancel();
+    });
+    return {
+      state,
+      recordActivity: (tool?: string) => {
+        if (tool) lastToolName = tool;
+        wd.recordActivity();
+      },
+      stop: () => wd.stop(),
+    };
+  }
+
+  it("cancels the run and records a StallTimeoutError when the stream wedges", () => {
+    const cancel = vi.fn().mockResolvedValue(undefined);
+    const run: FakeRun = { supports: () => true, cancel };
+    const w = wire(run, 60_000);
+
+    // Simulate a tool call starting, then no further progress.
+    w.recordActivity("browser_capture");
+    vi.advanceTimersByTime(61_000);
+
+    expect(w.state.stallDetected).toBe(true);
+    expect(cancel).toHaveBeenCalledTimes(1);
+    expect(w.state.stallError).toBeInstanceOf(StallTimeoutError);
+    expect(formatStallFailure(w.state.stallError!)).toContain("last tool: browser_capture");
+    w.stop();
+  });
+
+  it("does not call cancel when the SDK does not support it", () => {
+    const cancel = vi.fn().mockResolvedValue(undefined);
+    const run: FakeRun = { supports: () => false, cancel };
+    const w = wire(run, 30_000);
+
+    vi.advanceTimersByTime(31_000);
+    expect(w.state.stallDetected).toBe(true);
+    expect(cancel).not.toHaveBeenCalled();
+    w.stop();
+  });
+
+  it("a stream that keeps making progress never stalls or cancels", () => {
+    const cancel = vi.fn().mockResolvedValue(undefined);
+    const run: FakeRun = { supports: () => true, cancel };
+    const w = wire(run, 60_000);
+
+    for (let i = 0; i < 20; i++) {
+      vi.advanceTimersByTime(30_000);
+      w.recordActivity("scroll");
+    }
+    expect(w.state.stallDetected).toBe(false);
+    expect(cancel).not.toHaveBeenCalled();
+    w.stop();
+  });
+});

package/src/shared/__tests__/status-offload.test.ts

@@ -0,0 +1,256 @@
+import { describe, it, expect, vi } from "vitest";
+import { create, toBinary } from "@bufbuild/protobuf";
+import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import { AgentMessageSchema, ToolCallSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
+import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import type { ToolCall } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
+import type { ArtifactStorage } from "../artifact-storage.js";
+import {
+  offloadOversizedToolOutputs,
+  enforceStatusSizeLimit,
+  detectImagePayload,
+} from "../status-offload.js";
+
+function makeFakeStorage() {
+  const uploads: { key: string; size: number; contentType?: string }[] = [];
+  const storage: ArtifactStorage = {
+    upload: vi.fn(async (key: string, content: Buffer, contentType?: string) => {
+      uploads.push({ key, size: content.length, contentType });
+      return key;
+    }),
+    getDownloadUrl: vi.fn(async (key: string) => `https://artifacts.local/${key}`),
+    exists: vi.fn(async () => true),
+  };
+  return { storage, uploads };
+}
+
+function statusWithToolCall(tc: ToolCall): AgentExecutionStatus {
+  return create(AgentExecutionStatusSchema, {
+    messages: [create(AgentMessageSchema, { toolCalls: [tc] })],
+  });
+}
+
+function encodedSize(status: AgentExecutionStatus): number {
+  return toBinary(AgentExecutionStatusSchema, status).length;
+}
+
+const BIG_BASE64_IMAGE = Buffer.from("x".repeat(4096)).toString("base64");
+
+describe("detectImagePayload", () => {
+  it("extracts an MCP image content block", () => {
+    const result = JSON.stringify([
+      { type: "text", text: "Screenshot captured" },
+      { type: "image", data: BIG_BASE64_IMAGE, mimeType: "image/png" },
+    ]);
+    const img = detectImagePayload(result);
+    expect(img).not.toBeNull();
+    expect(img?.mimeType).toBe("image/png");
+    expect(img?.base64).toBe(BIG_BASE64_IMAGE);
+  });
+
+  it("extracts a raw data URL", () => {
+    const img = detectImagePayload(`data:image/jpeg;base64,${BIG_BASE64_IMAGE}`);
+    expect(img?.mimeType).toBe("image/jpeg");
+    expect(img?.base64).toBe(BIG_BASE64_IMAGE);
+  });
+
+  it("returns null for plain text", () => {
+    expect(detectImagePayload("just some logs")).toBeNull();
+  });
+
+  it("extracts an image from a serialized LangChain envelope (kwargs.content)", () => {
+    // Defensive shape: blocks nested under kwargs.content rather than a
+    // top-level array. The extractor normalizes to a top-level array, but
+    // detection must still cope if that ever drifts.
+    const result = JSON.stringify({
+      lc: 1,
+      type: "constructor",
+      id: ["langchain_core", "messages", "ToolMessage"],
+      kwargs: { content: [{ type: "image", data: BIG_BASE64_IMAGE, mimeType: "image/png" }] },
+    });
+    const img = detectImagePayload(result);
+    expect(img?.mimeType).toBe("image/png");
+    expect(img?.base64).toBe(BIG_BASE64_IMAGE);
+  });
+});
+
+describe("offloadOversizedToolOutputs", () => {
+  it("offloads an oversized image result and renders it inline-able", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    const result = JSON.stringify([
+      { type: "image", data: BIG_BASE64_IMAGE, mimeType: "image/png" },
+    ]);
+    const tc = create(ToolCallSchema, { id: "tc-1", name: "screenshot", result });
+    const status = statusWithToolCall(tc);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+      maxInlineBytes: 256,
+    });
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef).toBeDefined();
+    expect(out.outputRef?.isImage).toBe(true);
+    expect(out.outputRef?.mimeType).toBe("image/png");
+    expect(out.outputRef?.downloadUrl).toContain("artifacts/exec-1/toolcalls/tc-1.png");
+    // result is collapsed to a short label, no longer the giant blob.
+    expect(out.result.length).toBeLessThan(200);
+    expect(uploads).toHaveLength(1);
+    expect(uploads[0].contentType).toBe("image/png");
+  });
+
+  it("offloads oversized text with a preview head and text/plain upload", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    const result = "LOG ".repeat(2000); // ~8 KB
+    const tc = create(ToolCallSchema, { id: "tc-2", name: "Shell", result });
+    const status = statusWithToolCall(tc);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+      maxInlineBytes: 256,
+    });
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef?.isImage).toBe(false);
+    expect(out.outputRef?.mimeType).toBe("text/plain");
+    expect(out.outputRef?.truncatedPreview.length).toBeGreaterThan(0);
+    expect(out.result).toContain("view full output");
+    expect(uploads).toHaveLength(1);
+    expect(uploads[0].contentType).toBe("text/plain");
+  });
+
+  it("offloads a SMALL image regardless of size (output_ref is the only render path)", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    // A tiny image, far below the inline byte budget. It must still offload,
+    // because the UI can only render an image through ToolCallOutputRef.
+    const smallImage = Buffer.from("tiny-png").toString("base64");
+    const result = JSON.stringify([
+      { type: "image", data: smallImage, mimeType: "image/png" },
+    ]);
+    const tc = create(ToolCallSchema, { id: "tc-img", name: "screenshot", result });
+    const status = statusWithToolCall(tc);
+    expect(result.length).toBeLessThan(256);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+      maxInlineBytes: 256,
+    });
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef?.isImage).toBe(true);
+    expect(out.outputRef?.downloadUrl).toContain("artifacts/exec-1/toolcalls/tc-img.png");
+    expect(out.result).not.toContain(smallImage);
+    expect(uploads).toHaveLength(1);
+    expect(uploads[0].contentType).toBe("image/png");
+  });
+
+  it("leaves small results untouched (no artifact, no ref)", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    const tc = create(ToolCallSchema, { id: "tc-3", name: "Read", result: "small output" });
+    const status = statusWithToolCall(tc);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+      maxInlineBytes: 256,
+    });
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef).toBeUndefined();
+    expect(out.result).toBe("small output");
+    expect(uploads).toHaveLength(0);
+  });
+
+  it("is idempotent and dedupes uploads when result is re-inflated with identical bytes", async () => {
+    const { storage, uploads } = makeFakeStorage();
+    const result = "LOG ".repeat(2000);
+    const tc = create(ToolCallSchema, { id: "tc-4", name: "Shell", result });
+    const status = statusWithToolCall(tc);
+    const ctx = { artifactStorage: storage, executionId: "exec-1", maxInlineBytes: 256 };
+
+    await offloadOversizedToolOutputs(status, ctx);
+    expect(uploads).toHaveLength(1);
+
+    // Simulate mergeToolCallEvent re-inflating the inline result with the SAME
+    // bytes on a subsequent persist; the ref's content hash still matches.
+    status.messages[0].toolCalls[0].result = result;
+    await offloadOversizedToolOutputs(status, ctx);
+
+    expect(uploads).toHaveLength(1); // not re-uploaded
+    expect(status.messages[0].toolCalls[0].result).toContain("view full output");
+  });
+
+  it("keeps the encoded status small after offloading a large blob", async () => {
+    const { storage } = makeFakeStorage();
+    const result = JSON.stringify([
+      { type: "image", data: Buffer.from("y".repeat(500_000)).toString("base64"), mimeType: "image/png" },
+    ]);
+    const tc = create(ToolCallSchema, { id: "tc-5", name: "screenshot", result });
+    const status = statusWithToolCall(tc);
+    expect(encodedSize(status)).toBeGreaterThan(500_000);
+
+    await offloadOversizedToolOutputs(status, {
+      artifactStorage: storage,
+      executionId: "exec-1",
+    });
+
+    expect(encodedSize(status)).toBeLessThan(10_000);
+  });
+
+  it("falls back to inline truncation when the upload fails (never throws)", async () => {
+    const storage: ArtifactStorage = {
+      upload: vi.fn(async () => { throw new Error("storage down"); }),
+      getDownloadUrl: vi.fn(async (k: string) => k),
+      exists: vi.fn(async () => false),
+    };
+    const result = "LOG ".repeat(2000);
+    const tc = create(ToolCallSchema, { id: "tc-6", name: "Shell", result });
+    const status = statusWithToolCall(tc);
+    vi.spyOn(console, "warn").mockImplementation(() => {});
+
+    await expect(
+      offloadOversizedToolOutputs(status, {
+        artifactStorage: storage,
+        executionId: "exec-1",
+        maxInlineBytes: 256,
+      }),
+    ).resolves.toBeUndefined();
+
+    const out = status.messages[0].toolCalls[0];
+    expect(out.outputRef).toBeUndefined();
+    expect(out.result).toContain("offload failed");
+    expect(out.result.length).toBeLessThan(result.length);
+  });
+});
+
+describe("enforceStatusSizeLimit", () => {
+  it("returns false and changes nothing when under the limit", () => {
+    const tc = create(ToolCallSchema, { id: "tc", name: "Read", result: "tiny" });
+    const status = statusWithToolCall(tc);
+    expect(enforceStatusSizeLimit(status, 1024 * 1024)).toBe(false);
+    expect(status.messages[0].toolCalls[0].result).toBe("tiny");
+  });
+
+  it("elides the largest inline fields until the status fits", () => {
+    const big = "Z".repeat(50_000);
+    const status = create(AgentExecutionStatusSchema, {
+      messages: [
+        create(AgentMessageSchema, {
+          toolCalls: [
+            create(ToolCallSchema, { id: "a", name: "Shell", result: big }),
+            create(ToolCallSchema, { id: "b", name: "Read", result: big }),
+          ],
+        }),
+      ],
+    });
+    const before = encodedSize(status);
+    expect(before).toBeGreaterThan(80_000);
+
+    const elided = enforceStatusSizeLimit(status, 4_000);
+    expect(elided).toBe(true);
+    expect(encodedSize(status)).toBeLessThanOrEqual(4_000);
+  });
+});

package/src/shared/__tests__/status.test.ts

@@ -1,4 +1,5 @@
 import { describe, it, expect, vi } from "vitest";
+import { ConnectError, Code } from "@connectrpc/connect";
 import { create } from "@bufbuild/protobuf";
 import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
 import { ExecutionControlSignal, ExecutionPhase } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
@@ -123,6 +124,204 @@ describe("persistStatus", () => {
     expect(consoleSpy).toHaveBeenCalled();
     consoleSpy.mockRestore();
   });
+
+  it("hard-elides and retries once on a resource_exhausted (code 8) failure", async () => {
+    const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const updateStatus = vi
+      .fn()
+      .mockRejectedValueOnce({ code: 8, message: "resource_exhausted: exceeds maximum size" })
+      .mockResolvedValueOnce({ signal: ExecutionControlSignal.STOP });
+    const mockClient = { updateStatus } as any;
+    const status = create(AgentExecutionStatusSchema, {
+      phase: ExecutionPhase.EXECUTION_IN_PROGRESS,
+    });
+
+    const signal = await persistStatus(mockClient, "exec-too-big", status);
+    expect(signal).toBe(ExecutionControlSignal.STOP);
+    expect(updateStatus).toHaveBeenCalledTimes(2);
+    consoleSpy.mockRestore();
+  });
+
+  it("offloads oversized tool outputs before persisting when given an offload context", async () => {
+    const uploaded: string[] = [];
+    const offload = {
+      executionId: "exec-off",
+      artifactStorage: {
+        upload: vi.fn(async (key: string) => { uploaded.push(key); return key; }),
+        getDownloadUrl: vi.fn(async (key: string) => `https://artifacts.local/${key}`),
+        exists: vi.fn(async () => true),
+      },
+      maxInlineBytes: 256,
+    };
+    const mockClient = {
+      updateStatus: vi.fn().mockResolvedValue({ signal: ExecutionControlSignal.UNSPECIFIED }),
+    } as any;
+    const { AgentMessageSchema, ToolCallSchema } = await import(
+      "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb"
+    );
+    const status = create(AgentExecutionStatusSchema, {
+      phase: ExecutionPhase.EXECUTION_IN_PROGRESS,
+      messages: [
+        create(AgentMessageSchema, {
+          toolCalls: [create(ToolCallSchema, { id: "t", name: "Shell", result: "X".repeat(5000) })],
+        }),
+      ],
+    });
+
+    await persistStatus(mockClient, "exec-off", status, { offload });
+
+    expect(uploaded).toHaveLength(1);
+    const [, persisted] = mockClient.updateStatus.mock.calls[0];
+    expect(persisted.messages[0].toolCalls[0].outputRef).toBeDefined();
+    expect(persisted.messages[0].toolCalls[0].result.length).toBeLessThan(5000);
+  });
+});
+
+// Migrated from grpc-retry.test.ts: the transient-backoff loop now lives inside
+// persistStatus so every persist (streaming + terminal, every harness) shares
+// one chokepoint that bounds size AND retries transient transport errors.
+describe("persistStatus — transient retry", () => {
+  const noDelay = async () => {};
+  const emptyStatus = () => create(AgentExecutionStatusSchema, {});
+  function retryClient(impl: () => Promise<{ signal: ExecutionControlSignal }>) {
+    return { updateStatus: vi.fn(impl) } as any;
+  }
+
+  it("returns the signal on the first successful attempt", async () => {
+    const client = retryClient(async () => ({ signal: ExecutionControlSignal.STOP }));
+    const signal = await persistStatus(client, "exec-1", emptyStatus(), {
+      retry: { delayFn: noDelay },
+    });
+    expect(signal).toBe(ExecutionControlSignal.STOP);
+    expect(client.updateStatus).toHaveBeenCalledOnce();
+  });
+
+  it("retries a transient UNAVAILABLE and then succeeds", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      if (attempt === 1) throw new ConnectError("down", Code.Unavailable);
+      return { signal: ExecutionControlSignal.UNSPECIFIED };
+    });
+    const signal = await persistStatus(client, "exec-2", emptyStatus(), {
+      retry: { delayFn: noDelay },
+    });
+    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
+    expect(attempt).toBe(2);
+  });
+
+  it("does not retry terminal errors (NOT_FOUND)", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      throw new ConnectError("gone", Code.NotFound);
+    });
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const signal = await persistStatus(client, "exec-3", emptyStatus(), {
+      retry: { delayFn: noDelay },
+    });
+    errorSpy.mockRestore();
+    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
+    expect(attempt).toBe(1);
+  });
+
+  it("does not retry INTERNAL (neither retryable nor terminal)", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      throw new ConnectError("internal", Code.Internal);
+    });
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    await persistStatus(client, "exec-int", emptyStatus(), {
+      retry: { maxRetries: 3, delayFn: noDelay },
+    });
+    errorSpy.mockRestore();
+    expect(attempt).toBe(1);
+  });
+
+  it("applies exponential backoff and gives up after the retry budget", async () => {
+    const delays: number[] = [];
+    const client = retryClient(async () => {
+      throw new ConnectError("down", Code.Unavailable);
+    });
+    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const signal = await persistStatus(client, "exec-5", emptyStatus(), {
+      retry: {
+        baseDelayMs: 100,
+        backoffFactor: 2,
+        maxRetries: 3,
+        delayFn: async (ms) => { delays.push(ms); },
+      },
+    });
+    warnSpy.mockRestore();
+    errorSpy.mockRestore();
+    expect(delays).toEqual([100, 200, 400]);
+    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
+  });
+
+  it("respects a custom backoff factor", async () => {
+    const delays: number[] = [];
+    const client = retryClient(async () => {
+      throw new ConnectError("down", Code.Unavailable);
+    });
+    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    await persistStatus(client, "exec-bf", emptyStatus(), {
+      retry: {
+        baseDelayMs: 50,
+        backoffFactor: 3,
+        maxRetries: 3,
+        delayFn: async (ms) => { delays.push(ms); },
+      },
+    });
+    warnSpy.mockRestore();
+    errorSpy.mockRestore();
+    expect(delays).toEqual([50, 150, 450]);
+  });
+
+  it("maxRetries = 0 means a single attempt", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      throw new ConnectError("down", Code.Unavailable);
+    });
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    await persistStatus(client, "exec-0", emptyStatus(), {
+      retry: { maxRetries: 0, delayFn: noDelay },
+    });
+    errorSpy.mockRestore();
+    expect(attempt).toBe(1);
+  });
+
+  it("handles a mixed sequence: transient then terminal", async () => {
+    let attempt = 0;
+    const client = retryClient(async () => {
+      attempt++;
+      if (attempt === 1) throw new ConnectError("down", Code.Unavailable);
+      throw new ConnectError("gone", Code.NotFound);
+    });
+    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const signal = await persistStatus(client, "exec-mix", emptyStatus(), {
+      retry: { delayFn: noDelay },
+    });
+    warnSpy.mockRestore();
+    errorSpy.mockRestore();
+    expect(attempt).toBe(2);
+    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
+  });
+
+  it("never throws regardless of error type", async () => {
+    const client = retryClient(async () => {
+      throw new ConnectError("server exploded", Code.Internal);
+    });
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    await expect(
+      persistStatus(client, "exec-7", emptyStatus(), { retry: { delayFn: noDelay } }),
+    ).resolves.toBe(ExecutionControlSignal.UNSPECIFIED);
+    errorSpy.mockRestore();
+  });
 });
 
 describe("reportSetupProgress", () => {