@stigmer/runner 3.0.8-dev.20260613085218 → 3.0.9-dev.20260615145121

package/src/shared/grpc-retry.ts

@@ -1,20 +1,15 @@
 /**
- * Exponential-backoff retry wrapper for gRPC status persistence.
+ * gRPC error classification for status persistence retries.
  *
- * Wraps the raw StigmerClient.updateStatus() call with retry logic
- * that classifies gRPC error codes as retryable vs terminal. A failed
- * status update must never crash the streaming loop — on permanent
- * failure, the signal falls back to UNSPECIFIED.
- *
- * Used by the ExecuteDeepAgent streaming loop. The simpler
- * fire-and-forget persistStatus in shared/status.ts is preserved
- * for ExecuteCursor and non-streaming callers.
+ * Status persistence flows through a single chokepoint — `persistStatus` in
+ * status.ts — which uses these helpers to decide whether a failed
+ * `updateStatus` should back off and retry (transient transport errors) or
+ * fail fast (deterministic errors). This module is intentionally just the
+ * classification policy + its options type, kept small and separately tested;
+ * the persist loop that consumes it lives with the rest of the persist logic.
  */
 
 import { ConnectError, Code } from "@connectrpc/connect";
-import { ExecutionControlSignal } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
-import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
-import type { StigmerClient } from "../client/stigmer-client.js";
 
 export interface RetryOptions {
   /** Base delay before the first retry (ms). Default: 100. */
@@ -38,6 +33,7 @@ const TERMINAL_CODES = new Set<Code>([
   Code.PermissionDenied,
 ]);
 
+/** True for transient transport errors that are worth retrying with backoff. */
 export function isRetryableError(err: unknown): boolean {
   if (err instanceof ConnectError) {
     return RETRYABLE_CODES.has(err.code);
@@ -45,69 +41,10 @@ export function isRetryableError(err: unknown): boolean {
   return false;
 }
 
+/** True for deterministic errors that retrying cannot fix. */
 export function isTerminalError(err: unknown): boolean {
   if (err instanceof ConnectError) {
     return TERMINAL_CODES.has(err.code);
   }
   return false;
 }
-
-function defaultDelay(ms: number): Promise<void> {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-/**
- * Persist execution status with exponential-backoff retry.
- *
- * Returns the ExecutionControlSignal from the server on success.
- * On permanent or exhausted-retry failure, logs the error and
- * returns UNSPECIFIED (never throws).
- */
-export async function persistWithRetry(
-  client: StigmerClient,
-  executionId: string,
-  status: AgentExecutionStatus,
-  options?: RetryOptions,
-): Promise<ExecutionControlSignal> {
-  const baseDelay = options?.baseDelayMs ?? 100;
-  const factor = options?.backoffFactor ?? 2;
-  const maxRetries = options?.maxRetries ?? 3;
-  const delay = options?.delayFn ?? defaultDelay;
-
-  let lastError: unknown;
-
-  for (let attempt = 0; attempt <= maxRetries; attempt++) {
-    try {
-      const response = await client.updateStatus(executionId, status);
-      return response.signal;
-    } catch (err: unknown) {
-      lastError = err;
-
-      if (isTerminalError(err)) {
-        const code = (err as ConnectError).code;
-        console.error(
-          `[grpc-retry] Terminal error persisting status for ${executionId}: ` +
-          `code=${Code[code]} (attempt ${attempt + 1}/${maxRetries + 1})`,
-        );
-        return ExecutionControlSignal.UNSPECIFIED;
-      }
-
-      if (!isRetryableError(err) || attempt === maxRetries) {
-        break;
-      }
-
-      const delayMs = baseDelay * Math.pow(factor, attempt);
-      console.warn(
-        `[grpc-retry] Retryable error for ${executionId}: ` +
-        `code=${err instanceof ConnectError ? Code[err.code] : "unknown"} ` +
-        `(attempt ${attempt + 1}/${maxRetries + 1}, retry in ${delayMs}ms)`,
-      );
-      await delay(delayMs);
-    }
-  }
-
-  console.error(
-    `[grpc-retry] All retries exhausted for ${executionId}: ${lastError}`,
-  );
-  return ExecutionControlSignal.UNSPECIFIED;
-}

package/src/shared/stall-watchdog.ts

@@ -0,0 +1,122 @@
+/**
+ * Progress-based stall watchdog.
+ *
+ * Long-running agent turns are kept alive against Temporal by a periodic
+ * keep-alive heartbeat (see {@link ./heartbeat.ts}). That heartbeat proves the
+ * runner *process* is alive, but it says nothing about whether the agent is
+ * making *progress*: a turn that wedges inside the harness stream loop (a tool
+ * call that never returns, a model connection that silently dies) keeps
+ * heartbeating forever and never times out. The execution then sits at
+ * EXECUTION_IN_PROGRESS indefinitely.
+ *
+ * This watchdog closes that gap. It is an *out-of-band* timer: callers report
+ * progress via {@link StallWatchdog.recordActivity} on every stream event AND
+ * every token delta, and the watchdog fires `onStall` once if no progress is
+ * reported for `stallMs`. "Out-of-band" is the deliberate correctness property
+ * — the check runs on its own timer rather than inside the consumer's loop, so
+ * it catches a true "no new events for N minutes" hang. An in-band check that
+ * only runs when the loop advances cannot fire while the loop is blocked
+ * awaiting the next event, which is exactly the wedge we need to detect.
+ *
+ * This module is the convergence target for stall detection across harnesses:
+ * the Cursor harness wires it here, and the native deep-agent harness has its
+ * own (harness-local, in-band) check that should later converge onto this one.
+ */
+
+/**
+ * Canonical default stall window. Harnesses may override with a larger value
+ * when their tool calls routinely run longer without emitting stream activity.
+ */
+export const DEFAULT_STALL_TIMEOUT_MS = 120_000;
+
+/**
+ * Thrown / reported when an agent stream makes no progress for longer than the
+ * configured stall window. Carries the observed idle duration so callers can
+ * build an actionable, recognizable error message.
+ */
+export class StallTimeoutError extends Error {
+  constructor(
+    public readonly stalledMs: number,
+    detail?: string,
+  ) {
+    super(
+      `Agent stream stalled: no activity for ${Math.round(stalledMs / 1000)}s` +
+        (detail ? ` (${detail})` : ""),
+    );
+    this.name = "StallTimeoutError";
+  }
+}
+
+/**
+ * Recognizable prefix on every stall-induced failure message. Single source of
+ * truth so callers (and any future UI/log keying) match on one constant.
+ */
+export const STALL_ERROR_PREFIX = "[StallTimeoutError]";
+
+/**
+ * Build the user-facing failure text for a stall. Keeps the canonical wording
+ * (prefix + actionable "Retry or resume.") next to the error it describes.
+ */
+export function formatStallFailure(error: StallTimeoutError): string {
+  return `${STALL_ERROR_PREFIX} ${error.message}. Retry or resume.`;
+}
+
+export interface StallWatchdog {
+  /** Reset the idle timer. Call on every stream event AND every token delta. */
+  recordActivity(): void;
+  /** Disarm the watchdog. Idempotent; safe to call in a `finally`. */
+  stop(): void;
+}
+
+/**
+ * Start an out-of-band stall watchdog.
+ *
+ * @param stallMs  Idle window after which `onStall` fires.
+ * @param onStall  Invoked at most once with the observed idle duration (ms)
+ *                 when `Date.now() - lastActivityAt` exceeds `stallMs`. The
+ *                 watchdog disarms itself before invoking, so `onStall` runs
+ *                 exactly once even if its handler is slow.
+ *
+ * The poll interval is `stallMs / 4` capped at 15s: frequent enough to detect a
+ * stall promptly without busy-looping, and bounded so a large `stallMs` still
+ * polls on a sane cadence.
+ */
+export function startStallWatchdog(
+  stallMs: number,
+  onStall: (idleMs: number) => void,
+): StallWatchdog {
+  let lastActivityAt = Date.now();
+  let fired = false;
+  let timer: ReturnType<typeof setInterval> | undefined;
+
+  const tickMs = Math.min(Math.max(Math.floor(stallMs / 4), 1), 15_000);
+
+  const disarm = (): void => {
+    if (timer !== undefined) {
+      clearInterval(timer);
+      timer = undefined;
+    }
+  };
+
+  timer = setInterval(() => {
+    if (fired) return;
+    const idleMs = Date.now() - lastActivityAt;
+    if (idleMs >= stallMs) {
+      fired = true;
+      disarm();
+      onStall(idleMs);
+    }
+  }, tickMs);
+  // Do not keep the event loop alive solely for the watchdog.
+  timer.unref?.();
+
+  return {
+    recordActivity(): void {
+      lastActivityAt = Date.now();
+    },
+    stop(): void {
+      fired = true;
+      disarm();
+    },
+  };
+}

package/src/shared/status-offload.ts

@@ -0,0 +1,342 @@
+/**
+ * Size-bounding guard for the persisted AgentExecutionStatus payload.
+ *
+ * Tool outputs (an MCP screenshot's base64 image, a giant accessibility-tree
+ * dump, a multi-MB shell log, a huge file write) are stored inline in
+ * `ToolCall.result`/`args_preview`, which live in `status.messages` and are
+ * re-serialized whole on every `persistStatus` -> `updateStatus` gRPC call.
+ * Left unchecked, a single large result pushes the message past the server's
+ * 4 MiB gRPC receive cap; the call fails with `resource_exhausted`, progress
+ * stops persisting, and the live UI freezes mid-execution.
+ *
+ * This module enforces, at the single persist boundary, that the payload stays
+ * under the limit:
+ *
+ *   1. offloadOversizedToolOutputs — per-tool-call, SIZE-DRIVEN (not keyed on
+ *      tool type): any result over the byte threshold is spilled to artifact
+ *      storage and replaced with a short head plus a typed ToolCallOutputRef.
+ *      Images are uploaded as their decoded bytes (so the UI can render an
+ *      <img>); other large output is uploaded as text with a preview head.
+ *      Idempotent and content-hash-deduped so the throttled, repeated persists
+ *      (and result re-inflation by mergeToolCallEvent) upload each blob once.
+ *
+ *   2. enforceStatusSizeLimit — an aggregate, type-agnostic backstop that runs
+ *      even when no artifact storage is available: if the encoded status still
+ *      exceeds a soft cap (comfortably under 4 MiB), it elides the largest
+ *      remaining inline fields in place until the payload fits.
+ *
+ * Both operate ONLY on what is persisted/streamed; the agent's working context
+ * is managed by the harness/SDK separately, so reasoning is unaffected.
+ */
+
+import { createHash } from "node:crypto";
+import { create, toBinary } from "@bufbuild/protobuf";
+import {
+  AgentExecutionStatusSchema,
+} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import { ToolCallOutputRefSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
+import type { ToolCall } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
+import type { ArtifactStorage } from "./artifact-storage.js";
+
+/**
+ * A single tool output (result or args_preview) larger than this many bytes is
+ * offloaded to artifact storage instead of inlined into the persisted status.
+ * 256 KiB is generous for ordinary tool output yet far below the 4 MiB cap, so
+ * even a handful of large-but-sub-threshold results cannot aggregate past it.
+ */
+export const INLINE_TOOL_OUTPUT_MAX_BYTES = 256 * 1024;
+
+/** Head of an offloaded text result kept inline for an at-a-glance preview. */
+export const TEXT_PREVIEW_HEAD_CHARS = 4_000;
+
+/**
+ * Soft cap for the whole encoded status. Kept comfortably under the 4 MiB gRPC
+ * receive limit so the aggregate backstop trims before the server ever rejects.
+ */
+export const STATUS_PAYLOAD_SOFT_LIMIT_BYTES = 3 * 1024 * 1024;
+
+/**
+ * Tighter cap used only after the server has already rejected a payload as too
+ * large, to maximize the chance the retry succeeds.
+ */
+export const STATUS_PAYLOAD_HARD_LIMIT_BYTES = 2 * 1024 * 1024;
+
+/** Marker left in place of an aggregate-elided inline field. */
+const ELISION_MARKER = "[output elided to keep status under the size limit]";
+
+/** Below this size an inline field is not worth eliding (the marker is ~50B). */
+const ELISION_MIN_BYTES = 1_024;
+
+export interface ToolOutputOffloadContext {
+  readonly artifactStorage: ArtifactStorage;
+  readonly executionId: string;
+  /** Override the per-result byte threshold (tests use a small value). */
+  readonly maxInlineBytes?: number;
+}
+
+interface ImagePayload {
+  readonly base64: string;
+  readonly mimeType: string;
+}
+
+function byteLen(s: string | undefined): number {
+  return s ? Buffer.byteLength(s, "utf8") : 0;
+}
+
+function sha256(s: string): string {
+  return createHash("sha256").update(s, "utf8").digest("hex");
+}
+
+function headChars(s: string, n: number): string {
+  return s.length <= n ? s : s.slice(0, n);
+}
+
+function formatBytes(n: number): string {
+  if (n >= 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(1)} MB`;
+  if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`;
+  return `${n} B`;
+}
+
+function extFromMime(mimeType: string): string {
+  switch (mimeType) {
+    case "image/png": return "png";
+    case "image/jpeg": return "jpg";
+    case "image/gif": return "gif";
+    case "image/webp": return "webp";
+    case "image/svg+xml": return "svg";
+    default: return "bin";
+  }
+}
+
+function matchDataUrl(s: string): ImagePayload | null {
+  const m = s.match(/^data:(image\/[a-zA-Z0-9.+-]+);base64,([\s\S]+)$/);
+  if (!m) return null;
+  return { mimeType: m[1], base64: m[2].replace(/\s+/g, "") };
+}
+
+function contentBlocks(parsed: unknown): unknown[] {
+  if (Array.isArray(parsed)) return parsed;
+  if (parsed && typeof parsed === "object") {
+    const obj = parsed as Record<string, unknown>;
+    if (Array.isArray(obj.content)) return obj.content;
+    // Defensive: a serialized LangChain ToolMessage envelope nests its blocks
+    // under kwargs.content. The deep-agent extractor already normalizes image
+    // results to a top-level array (status-builder-shared.ts), so this branch is
+    // insurance against future shape drift — not the primary path.
+    const kwargs = obj.kwargs;
+    if (kwargs && typeof kwargs === "object" && Array.isArray((kwargs as Record<string, unknown>).content)) {
+      return (kwargs as Record<string, unknown>).content as unknown[];
+    }
+  }
+  return [];
+}
+
+/**
+ * Best-effort extraction of an inline base64 image from a tool result string.
+ * Handles a raw data URL, MCP image content blocks ({type:"image", data,
+ * mimeType}) and OpenAI-style image_url blocks. Returns null for non-image or
+ * unparseable results so the caller falls back to text offload.
+ */
+export function detectImagePayload(result: string): ImagePayload | null {
+  const direct = matchDataUrl(result.trim());
+  if (direct) return direct;
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(result);
+  } catch {
+    return null;
+  }
+
+  for (const block of contentBlocks(parsed)) {
+    if (!block || typeof block !== "object") continue;
+    const obj = block as Record<string, unknown>;
+    const type = typeof obj.type === "string" ? obj.type : "";
+    if (type !== "image" && type !== "image_url") continue;
+
+    const raw =
+      typeof obj.data === "string" ? obj.data :
+      typeof obj.image === "string" ? obj.image :
+      undefined;
+    if (raw) {
+      const asUrl = matchDataUrl(raw);
+      if (asUrl) return asUrl;
+      const mimeType =
+        typeof obj.mimeType === "string" ? obj.mimeType :
+        typeof obj.mime_type === "string" ? obj.mime_type :
+        "image/png";
+      return { mimeType, base64: raw.replace(/\s+/g, "") };
+    }
+
+    const imageUrl = obj.image_url;
+    if (imageUrl && typeof imageUrl === "object") {
+      const url = (imageUrl as Record<string, unknown>).url;
+      if (typeof url === "string") {
+        const asUrl = matchDataUrl(url);
+        if (asUrl) return asUrl;
+      }
+    }
+  }
+  return null;
+}
+
+function collapsedResultFor(ref: { isImage: boolean; sizeBytes: bigint; truncatedPreview: string }): string {
+  if (ref.isImage) {
+    return `[image output — ${formatBytes(Number(ref.sizeBytes))}, view inline]`;
+  }
+  const tail = `\n\n[output truncated — ${formatBytes(Number(ref.sizeBytes))} total; view full output]`;
+  return ref.truncatedPreview + tail;
+}
+
+async function maybeOffloadToolCall(
+  tc: ToolCall,
+  ctx: ToolOutputOffloadContext,
+  maxBytes: number,
+): Promise<void> {
+  const result = tc.result;
+  if (!result) return;
+
+  const hash = sha256(result);
+
+  // Idempotent: the same content was already offloaded on a prior persist (or
+  // mergeToolCallEvent re-inflated `result` with identical bytes). Re-collapse
+  // the inline copy without re-uploading.
+  if (tc.outputRef && tc.outputRef.contentHash === hash) {
+    tc.result = collapsedResultFor(tc.outputRef);
+    return;
+  }
+
+  // Images are offloaded regardless of size, BEFORE the size gate below: a
+  // `ToolCallOutputRef` is the only path the UI has to render an image inline,
+  // so even a sub-threshold screenshot must be lifted out of `result` into a
+  // renderable ref. Non-image text only offloads when it exceeds maxBytes.
+  const image = detectImagePayload(result);
+  if (image) {
+    const bytes = Buffer.from(image.base64, "base64");
+    const key = `artifacts/${ctx.executionId}/toolcalls/${tc.id}.${extFromMime(image.mimeType)}`;
+    await ctx.artifactStorage.upload(key, bytes, image.mimeType);
+    const downloadUrl = await ctx.artifactStorage.getDownloadUrl(key);
+    tc.outputRef = create(ToolCallOutputRefSchema, {
+      storageKey: key,
+      downloadUrl,
+      sizeBytes: BigInt(bytes.length),
+      contentHash: hash,
+      mimeType: image.mimeType,
+      isImage: true,
+      truncatedPreview: "",
+    });
+    tc.result = collapsedResultFor(tc.outputRef);
+    return;
+  }
+
+  // Non-image text below the inline budget stays inline; only oversized text is
+  // spilled (with a head preview) to keep the persisted payload bounded.
+  if (byteLen(result) <= maxBytes) return;
+
+  const content = Buffer.from(result, "utf8");
+  const key = `artifacts/${ctx.executionId}/toolcalls/${tc.id}.txt`;
+  await ctx.artifactStorage.upload(key, content, "text/plain");
+  const downloadUrl = await ctx.artifactStorage.getDownloadUrl(key);
+  tc.outputRef = create(ToolCallOutputRefSchema, {
+    storageKey: key,
+    downloadUrl,
+    sizeBytes: BigInt(content.length),
+    contentHash: hash,
+    mimeType: "text/plain",
+    isImage: false,
+    truncatedPreview: headChars(result, TEXT_PREVIEW_HEAD_CHARS),
+  });
+  tc.result = collapsedResultFor(tc.outputRef);
+}
+
+/**
+ * Offload every oversized tool result in the status to artifact storage,
+ * replacing the inline value with a short head + ToolCallOutputRef. Per-tool
+ * failures fall back to an inline truncation (a bounded result beats a failed
+ * persist) and never throw, so a storage hiccup cannot fail the execution.
+ */
+export async function offloadOversizedToolOutputs(
+  status: AgentExecutionStatus,
+  ctx: ToolOutputOffloadContext,
+): Promise<void> {
+  const maxBytes = ctx.maxInlineBytes ?? INLINE_TOOL_OUTPUT_MAX_BYTES;
+  for (const msg of status.messages) {
+    for (const tc of msg.toolCalls) {
+      try {
+        await maybeOffloadToolCall(tc, ctx, maxBytes);
+      } catch (err) {
+        const original = tc.result ?? "";
+        tc.result =
+          headChars(original, TEXT_PREVIEW_HEAD_CHARS) +
+          `\n\n[output truncated — offload failed: ${err instanceof Error ? err.message : String(err)}]`;
+        console.warn(
+          `[status-offload] execution=${ctx.executionId} tool=${tc.name} ` +
+          `offload failed (non-fatal); truncated inline`,
+        );
+      }
+    }
+  }
+}
+
+function encodedSize(status: AgentExecutionStatus): number {
+  return toBinary(AgentExecutionStatusSchema, status).length;
+}
+
+/**
+ * Aggregate, type-agnostic backstop. If the encoded status exceeds
+ * `softLimitBytes`, elide the largest inline tool fields (and, as a last
+ * resort, message content) in place until it fits. Returns true if anything
+ * was elided. This guarantees a bounded payload even without artifact storage
+ * (e.g. offload disabled) or when many medium results sum past the limit.
+ */
+export function enforceStatusSizeLimit(
+  status: AgentExecutionStatus,
+  softLimitBytes: number = STATUS_PAYLOAD_SOFT_LIMIT_BYTES,
+): boolean {
+  if (encodedSize(status) <= softLimitBytes) return false;
+
+  const toolCalls: ToolCall[] = [];
+  for (const msg of status.messages) {
+    for (const tc of msg.toolCalls) toolCalls.push(tc);
+  }
+  // Largest inline footprint first so we shed the most bytes per elision.
+  toolCalls.sort(
+    (a, b) =>
+      byteLen(b.result) + byteLen(b.argsPreview) -
+      (byteLen(a.result) + byteLen(a.argsPreview)),
+  );
+
+  let elidedAny = false;
+  for (const tc of toolCalls) {
+    if (encodedSize(status) <= softLimitBytes) return elidedAny;
+    if (byteLen(tc.result) > ELISION_MIN_BYTES) {
+      tc.result = ELISION_MARKER;
+      elidedAny = true;
+    }
+    if (byteLen(tc.argsPreview) > ELISION_MIN_BYTES) {
+      tc.argsPreview = ELISION_MARKER;
+      elidedAny = true;
+    }
+    if (tc.args !== undefined) {
+      tc.args = undefined;
+      elidedAny = true;
+    }
+  }
+
+  // Last resort: oversized message content (e.g. a huge AI response).
+  if (encodedSize(status) > softLimitBytes) {
+    const byContent = [...status.messages].sort(
+      (a, b) => byteLen(b.content) - byteLen(a.content),
+    );
+    for (const msg of byContent) {
+      if (encodedSize(status) <= softLimitBytes) break;
+      if (byteLen(msg.content) > ELISION_MIN_BYTES) {
+        msg.content = headChars(msg.content, TEXT_PREVIEW_HEAD_CHARS) + `\n\n${ELISION_MARKER}`;
+        elidedAny = true;
+      }
+    }
+  }
+
+  return elidedAny;
+}