@stigmer/runner 3.0.8-dev.20260613085218 → 3.0.8

package/src/shared/__tests__/grpc-retry-extended.test.ts

@@ -1,30 +1,13 @@
 /**
- * Extended gRPC retry tests — ported from Python test_grpc_retry.py.
- *
- * Covers config validation, backoff timing, INTERNAL error handling,
- * mixed error sequences, and max-retries edge cases beyond the core
- * 15 tests in grpc-retry.test.ts.
+ * Extended gRPC error-classification tests — ported from Python
+ * test_grpc_retry.py. Covers additional status codes beyond the core
+ * grpc-retry.test.ts. The persist-with-backoff loop that consumes these
+ * classifiers is tested in status.test.ts ("persistStatus — transient retry").
  */
 
-import { describe, it, expect, vi } from "vitest";
+import { describe, it, expect } from "vitest";
 import { ConnectError, Code } from "@connectrpc/connect";
-import { create } from "@bufbuild/protobuf";
-import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
-import { ExecutionControlSignal } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
-import { persistWithRetry, isRetryableError, isTerminalError } from "../grpc-retry.js";
-import type { StigmerClient } from "../../client/stigmer-client.js";
-
-function makeStatus() {
-  return create(AgentExecutionStatusSchema, {});
-}
-
-function mockClient(
-  impl: (...args: unknown[]) => Promise<{ signal: ExecutionControlSignal }>,
-): StigmerClient {
-  return { updateStatus: impl } as unknown as StigmerClient;
-}
-
-const noDelay = async () => {};
+import { isRetryableError, isTerminalError } from "../grpc-retry.js";
 
 describe("isRetryableError — extended codes", () => {
   it("returns false for INTERNAL", () => {
@@ -53,124 +36,3 @@ describe("isTerminalError — extended codes", () => {
     expect(isTerminalError(new Error("random"))).toBe(false);
   });
 });
-
-describe("persistWithRetry — extended scenarios", () => {
-  it("does not retry INTERNAL errors (neither retryable nor terminal)", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new ConnectError("internal", Code.Internal);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-int", makeStatus(), {
-      maxRetries: 3,
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(attempt).toBe(1);
-  });
-
-  it("does not retry ALREADY_EXISTS", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new ConnectError("exists", Code.AlreadyExists);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-ae", makeStatus(), {
-      maxRetries: 3,
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(attempt).toBe(1);
-  });
-
-  it("handles mixed error sequence: retryable then terminal", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      if (attempt === 1) throw new ConnectError("down", Code.Unavailable);
-      throw new ConnectError("gone", Code.NotFound);
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-mix", makeStatus(), {
-      delayFn: noDelay,
-    });
-    warnSpy.mockRestore();
-    errorSpy.mockRestore();
-
-    expect(attempt).toBe(2);
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-  });
-
-  it("handles retryable then success", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      if (attempt <= 2) throw new ConnectError("timeout", Code.DeadlineExceeded);
-      return { signal: ExecutionControlSignal.STOP };
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-ret", makeStatus(), {
-      maxRetries: 5,
-      delayFn: noDelay,
-    });
-    warnSpy.mockRestore();
-
-    expect(signal).toBe(ExecutionControlSignal.STOP);
-    expect(attempt).toBe(3);
-  });
-
-  it("maxRetries = 0 means one attempt only", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new ConnectError("down", Code.Unavailable);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-0", makeStatus(), {
-      maxRetries: 0,
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(attempt).toBe(1);
-  });
-
-  it("respects custom backoff factor", async () => {
-    const delays: number[] = [];
-    const client = mockClient(async () => {
-      throw new ConnectError("down", Code.Unavailable);
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-bf", makeStatus(), {
-      baseDelayMs: 50,
-      backoffFactor: 3,
-      maxRetries: 3,
-      delayFn: async (ms) => { delays.push(ms); },
-    });
-    warnSpy.mockRestore();
-    errorSpy.mockRestore();
-
-    expect(delays).toEqual([50, 150, 450]);
-  });
-
-  it("uses default options when none provided", async () => {
-    const client = mockClient(async () => ({
-      signal: ExecutionControlSignal.UNSPECIFIED,
-    }));
-
-    const signal = await persistWithRetry(client, "exec-default", makeStatus());
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-  });
-});

package/src/shared/__tests__/grpc-retry.test.ts

@@ -1,26 +1,10 @@
-import { describe, it, expect, vi } from "vitest";
+import { describe, it, expect } from "vitest";
 import { ConnectError, Code } from "@connectrpc/connect";
-import { create } from "@bufbuild/protobuf";
-import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
-import { ExecutionControlSignal } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
-import {
-  persistWithRetry,
-  isRetryableError,
-  isTerminalError,
-} from "../grpc-retry.js";
-import type { StigmerClient } from "../../client/stigmer-client.js";
+import { isRetryableError, isTerminalError } from "../grpc-retry.js";
 
-function makeStatus() {
-  return create(AgentExecutionStatusSchema, {});
-}
-
-function mockClient(
-  updateStatusImpl: (...args: unknown[]) => Promise<{ signal: ExecutionControlSignal }>,
-): StigmerClient {
-  return { updateStatus: updateStatusImpl } as unknown as StigmerClient;
-}
-
-const noDelay = async () => {};
+// The persist-with-backoff loop that consumes these classifiers lives in
+// status.ts and is exercised by status.test.ts ("persistStatus — transient
+// retry"). This file covers the classification policy in isolation.
 
 describe("isRetryableError", () => {
   it("returns true for UNAVAILABLE", () => {
@@ -57,116 +41,3 @@ describe("isTerminalError", () => {
     expect(isTerminalError(new ConnectError("down", Code.Unavailable))).toBe(false);
   });
 });
-
-describe("persistWithRetry", () => {
-  it("returns signal on successful first attempt", async () => {
-    const client = mockClient(async () => ({
-      signal: ExecutionControlSignal.STOP,
-    }));
-
-    const signal = await persistWithRetry(client, "exec-1", makeStatus(), {
-      delayFn: noDelay,
-    });
-
-    expect(signal).toBe(ExecutionControlSignal.STOP);
-  });
-
-  it("retries on UNAVAILABLE and succeeds", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      if (attempt === 1) throw new ConnectError("down", Code.Unavailable);
-      return { signal: ExecutionControlSignal.UNSPECIFIED };
-    });
-
-    const signal = await persistWithRetry(client, "exec-2", makeStatus(), {
-      delayFn: noDelay,
-    });
-
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-    expect(attempt).toBe(2);
-  });
-
-  it("does not retry terminal errors (NOT_FOUND)", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new ConnectError("gone", Code.NotFound);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-3", makeStatus(), {
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-    expect(attempt).toBe(1);
-  });
-
-  it("returns UNSPECIFIED after all retries exhausted", async () => {
-    const client = mockClient(async () => {
-      throw new ConnectError("down", Code.Unavailable);
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-4", makeStatus(), {
-      maxRetries: 3,
-      delayFn: noDelay,
-    });
-    warnSpy.mockRestore();
-    errorSpy.mockRestore();
-
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-  });
-
-  it("uses correct backoff delays", async () => {
-    const delays: number[] = [];
-    const client = mockClient(async () => {
-      throw new ConnectError("down", Code.Unavailable);
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-5", makeStatus(), {
-      baseDelayMs: 100,
-      backoffFactor: 2,
-      maxRetries: 3,
-      delayFn: async (ms) => { delays.push(ms); },
-    });
-    warnSpy.mockRestore();
-    errorSpy.mockRestore();
-
-    expect(delays).toEqual([100, 200, 400]);
-  });
-
-  it("handles non-ConnectError by not retrying", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new Error("unexpected");
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-6", makeStatus(), {
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-    expect(attempt).toBe(1);
-  });
-
-  it("never throws regardless of error type", async () => {
-    const client = mockClient(async () => {
-      throw new ConnectError("server exploded", Code.Internal);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await expect(
-      persistWithRetry(client, "exec-7", makeStatus(), { delayFn: noDelay }),
-    ).resolves.toBe(ExecutionControlSignal.UNSPECIFIED);
-    errorSpy.mockRestore();
-  });
-});

package/src/shared/__tests__/stall-watchdog.test.ts

@@ -0,0 +1,193 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import {
+  startStallWatchdog,
+  StallTimeoutError,
+  formatStallFailure,
+  STALL_ERROR_PREFIX,
+  DEFAULT_STALL_TIMEOUT_MS,
+} from "../stall-watchdog.js";
+
+describe("startStallWatchdog", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+  });
+
+  it("fires onStall once after the idle window elapses with no activity", () => {
+    const onStall = vi.fn();
+    const wd = startStallWatchdog(60_000, onStall);
+
+    // Just before the window: not yet fired.
+    vi.advanceTimersByTime(59_000);
+    expect(onStall).not.toHaveBeenCalled();
+
+    // Cross the window: fires exactly once.
+    vi.advanceTimersByTime(2_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+    expect(onStall.mock.calls[0][0]).toBeGreaterThanOrEqual(60_000);
+
+    // It self-disarms — no further fires even as time marches on.
+    vi.advanceTimersByTime(120_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+
+    wd.stop();
+  });
+
+  it("recordActivity resets the idle timer so progress keeps it alive", () => {
+    const onStall = vi.fn();
+    const wd = startStallWatchdog(60_000, onStall);
+
+    // Report progress every 40s for a while: the window never elapses.
+    for (let i = 0; i < 10; i++) {
+      vi.advanceTimersByTime(40_000);
+      wd.recordActivity();
+    }
+    expect(onStall).not.toHaveBeenCalled();
+
+    // Now go quiet past the window. Detection has up to one poll interval
+    // (stallMs/4, capped 15s) of slop because ticks fire on absolute
+    // boundaries, so advance stallMs + a full tick to guarantee a fire.
+    vi.advanceTimersByTime(76_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+
+    wd.stop();
+  });
+
+  it("stop() disarms the watchdog so onStall never fires", () => {
+    const onStall = vi.fn();
+    const wd = startStallWatchdog(60_000, onStall);
+
+    wd.stop();
+    vi.advanceTimersByTime(600_000);
+    expect(onStall).not.toHaveBeenCalled();
+  });
+
+  it("is robust to stop() called after firing (idempotent)", () => {
+    const onStall = vi.fn();
+    const wd = startStallWatchdog(30_000, onStall);
+
+    vi.advanceTimersByTime(31_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+
+    expect(() => wd.stop()).not.toThrow();
+    vi.advanceTimersByTime(60_000);
+    expect(onStall).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe("StallTimeoutError", () => {
+  it("builds a readable message from the idle duration", () => {
+    const err = new StallTimeoutError(120_000);
+    expect(err.name).toBe("StallTimeoutError");
+    expect(err.stalledMs).toBe(120_000);
+    expect(err.message).toContain("120s");
+  });
+
+  it("appends an optional detail", () => {
+    const err = new StallTimeoutError(90_000, "last tool: browser_capture");
+    expect(err.message).toContain("90s");
+    expect(err.message).toContain("last tool: browser_capture");
+  });
+});
+
+describe("formatStallFailure", () => {
+  it("prefixes the recognizable tag and appends actionable guidance", () => {
+    const msg = formatStallFailure(new StallTimeoutError(180_000, "last tool: browser_capture"));
+    expect(msg.startsWith(STALL_ERROR_PREFIX)).toBe(true);
+    expect(msg).toContain("180s");
+    expect(msg).toContain("last tool: browser_capture");
+    expect(msg).toContain("Retry or resume.");
+  });
+});
+
+describe("DEFAULT_STALL_TIMEOUT_MS", () => {
+  it("matches the canonical 120s default", () => {
+    expect(DEFAULT_STALL_TIMEOUT_MS).toBe(120_000);
+  });
+});
+
+/**
+ * Integration contract for how execute-cursor/index.ts wires the watchdog to
+ * the SDK Run: on stall it must (1) record the stall, (2) build a
+ * StallTimeoutError, and (3) cancel the run when the SDK supports cancel —
+ * exactly the onStall closure in index.ts. We assert that contract here against
+ * a faithful fake Run rather than standing up the full executeCursor activity
+ * (which has no end-to-end test harness), keeping the test focused and robust.
+ */
+describe("stall -> run.cancel() wiring contract (mirrors execute-cursor/index.ts)", () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+  });
+
+  interface FakeRun {
+    supports: (op: string) => boolean;
+    cancel: () => Promise<void>;
+  }
+
+  function wire(run: FakeRun, stallMs: number) {
+    const state: { stallDetected: boolean; stallError?: StallTimeoutError } = {
+      stallDetected: false,
+    };
+    let lastToolName: string | undefined;
+    const wd = startStallWatchdog(stallMs, (idleMs) => {
+      state.stallDetected = true;
+      state.stallError = new StallTimeoutError(idleMs, lastToolName ? `last tool: ${lastToolName}` : undefined);
+      if (run.supports?.("cancel")) void run.cancel();
+    });
+    return {
+      state,
+      recordActivity: (tool?: string) => {
+        if (tool) lastToolName = tool;
+        wd.recordActivity();
+      },
+      stop: () => wd.stop(),
+    };
+  }
+
+  it("cancels the run and records a StallTimeoutError when the stream wedges", () => {
+    const cancel = vi.fn().mockResolvedValue(undefined);
+    const run: FakeRun = { supports: () => true, cancel };
+    const w = wire(run, 60_000);
+
+    // Simulate a tool call starting, then no further progress.
+    w.recordActivity("browser_capture");
+    vi.advanceTimersByTime(61_000);
+
+    expect(w.state.stallDetected).toBe(true);
+    expect(cancel).toHaveBeenCalledTimes(1);
+    expect(w.state.stallError).toBeInstanceOf(StallTimeoutError);
+    expect(formatStallFailure(w.state.stallError!)).toContain("last tool: browser_capture");
+    w.stop();
+  });
+
+  it("does not call cancel when the SDK does not support it", () => {
+    const cancel = vi.fn().mockResolvedValue(undefined);
+    const run: FakeRun = { supports: () => false, cancel };
+    const w = wire(run, 30_000);
+
+    vi.advanceTimersByTime(31_000);
+    expect(w.state.stallDetected).toBe(true);
+    expect(cancel).not.toHaveBeenCalled();
+    w.stop();
+  });
+
+  it("a stream that keeps making progress never stalls or cancels", () => {
+    const cancel = vi.fn().mockResolvedValue(undefined);
+    const run: FakeRun = { supports: () => true, cancel };
+    const w = wire(run, 60_000);
+
+    for (let i = 0; i < 20; i++) {
+      vi.advanceTimersByTime(30_000);
+      w.recordActivity("scroll");
+    }
+    expect(w.state.stallDetected).toBe(false);
+    expect(cancel).not.toHaveBeenCalled();
+    w.stop();
+  });
+});