@stigmer/runner 3.0.8-dev.20260613085218 → 3.0.9-dev.20260615145121

package/src/shared/status.ts

@@ -19,28 +19,162 @@ import {
   ExecutionPhase,
 } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
 import type { StigmerClient } from "../client/stigmer-client.js";
+import {
+  offloadOversizedToolOutputs,
+  enforceStatusSizeLimit,
+  STATUS_PAYLOAD_HARD_LIMIT_BYTES,
+  type ToolOutputOffloadContext,
+} from "./status-offload.js";
+import {
+  isRetryableError,
+  isTerminalError,
+  type RetryOptions,
+} from "./grpc-retry.js";
 
 export function utcTimestamp(): string {
   return new Date().toISOString().replace("+00:00", "Z");
 }
 
+/**
+ * A connect-rpc `resource_exhausted` (gRPC code 8) — the payload exceeded the
+ * server's receive cap. Matched by code and, defensively, by the message buf's
+ * size check emits ("exceeds maximum size").
+ */
+function isPayloadTooLarge(err: unknown): boolean {
+  if (typeof err === "object" && err !== null && "code" in err) {
+    if ((err as { code?: unknown }).code === 8) return true;
+  }
+  const msg = err instanceof Error ? err.message : String(err);
+  return /resource_exhausted|exceeds maximum size/i.test(msg);
+}
+
+/** Options controlling how a status payload is bounded and persisted. */
+export interface PersistStatusOptions {
+  /**
+   * When supplied, oversized tool outputs (a screenshot's base64, a multi-MB
+   * dump) are spilled to artifact storage before persisting — the only path
+   * that populates a renderable `ToolCallOutputRef`. When omitted, the
+   * aggregate size backstop still runs, so the payload is bounded regardless.
+   */
+  readonly offload?: ToolOutputOffloadContext;
+  /**
+   * Transient-error backoff policy. Omitted fields fall back to
+   * {@link DEFAULT_PERSIST_RETRY}; an omitted `retry` uses it entirely.
+   */
+  readonly retry?: RetryOptions;
+}
+
+/**
+ * Default transient-retry policy: three attempts with exponential backoff.
+ * Applied to every persist so a transient drop (server restart, brief
+ * unavailability) recovers silently instead of dropping a status update.
+ */
+const DEFAULT_PERSIST_RETRY = {
+  baseDelayMs: 100,
+  backoffFactor: 2,
+  maxRetries: 3,
+} as const;
+
+function defaultDelay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 /**
  * Persist execution status via gRPC. Returns the control signal from the
- * backend (e.g. STOP to abort the execution). Errors are logged and
- * swallowed — a failed status update should not crash the activity.
+ * backend (e.g. STOP to abort the execution).
+ *
+ * This is the single, unforgeable chokepoint for status persistence across
+ * every harness (streaming and terminal, native and Cursor). Routing all
+ * persistence through here guarantees three invariants that must never be
+ * skipped by an individual call site:
+ *
+ *   1. Size bounding — when `offload` is supplied, oversized tool outputs are
+ *      spilled to artifact storage first (the path that makes screenshots
+ *      render); an aggregate backstop then runs unconditionally so callers
+ *      without storage are still protected from the 4 MiB gRPC cap.
+ *   2. Transient resilience — `Unavailable`/`DeadlineExceeded` back off and
+ *      retry (default three attempts); terminal errors never retry.
+ *   3. Oversize recovery — a `resource_exhausted` rejection is never silently
+ *      swallowed: we hard-elide and retry once, so an oversized payload can
+ *      never freeze persistence invisibly (the bug that stalled the UI).
  */
 export async function persistStatus(
   client: StigmerClient,
   executionId: string,
   status: AgentExecutionStatus,
+  options: PersistStatusOptions = {},
 ): Promise<ExecutionControlSignal> {
-  try {
-    const response = await client.updateStatus(executionId, status);
-    return response.signal;
-  } catch (err) {
-    console.error(`Failed to persist status for ${executionId}:`, err);
-    return ExecutionControlSignal.UNSPECIFIED;
+  const { offload, retry } = options;
+
+  if (offload) {
+    try {
+      await offloadOversizedToolOutputs(status, offload);
+    } catch (err) {
+      console.warn(`[persistStatus] ${executionId}: tool-output offload failed (non-fatal): ${err}`);
+    }
   }
+  if (enforceStatusSizeLimit(status)) {
+    console.warn(
+      `[persistStatus] ${executionId}: status exceeded the soft size limit; ` +
+      `elided oversized inline fields to fit under the gRPC cap`,
+    );
+  }
+
+  const baseDelayMs = retry?.baseDelayMs ?? DEFAULT_PERSIST_RETRY.baseDelayMs;
+  const backoffFactor = retry?.backoffFactor ?? DEFAULT_PERSIST_RETRY.backoffFactor;
+  const maxRetries = retry?.maxRetries ?? DEFAULT_PERSIST_RETRY.maxRetries;
+  const delay = retry?.delayFn ?? defaultDelay;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const response = await client.updateStatus(executionId, status);
+      return response.signal;
+    } catch (err) {
+      // Oversize rejection is recoverable exactly once and out-of-band from the
+      // transient budget: hard-elide the largest inline fields and retry now.
+      if (isPayloadTooLarge(err)) {
+        console.error(
+          `[persistStatus] ${executionId}: payload rejected as too large despite ` +
+          `the size guard; hard-eliding and retrying once`,
+        );
+        enforceStatusSizeLimit(status, STATUS_PAYLOAD_HARD_LIMIT_BYTES);
+        try {
+          const response = await client.updateStatus(executionId, status);
+          return response.signal;
+        } catch (retryErr) {
+          console.error(`[persistStatus] ${executionId}: still failing after hard elide:`, retryErr);
+          return ExecutionControlSignal.UNSPECIFIED;
+        }
+      }
+
+      // Terminal errors (invalid argument, not found, permission denied) are
+      // deterministic — retrying cannot help.
+      if (isTerminalError(err)) {
+        console.error(
+          `[persistStatus] ${executionId}: terminal error persisting status ` +
+          `(attempt ${attempt + 1}/${maxRetries + 1}): ${err}`,
+        );
+        return ExecutionControlSignal.UNSPECIFIED;
+      }
+
+      // Transient errors back off and retry until the budget is exhausted.
+      if (isRetryableError(err) && attempt < maxRetries) {
+        const delayMs = baseDelayMs * Math.pow(backoffFactor, attempt);
+        console.warn(
+          `[persistStatus] ${executionId}: retryable error ` +
+          `(attempt ${attempt + 1}/${maxRetries + 1}, retry in ${delayMs}ms): ${err}`,
+        );
+        await delay(delayMs);
+        continue;
+      }
+
+      // Non-retryable, non-terminal (e.g. a bare Error), or retries exhausted.
+      console.error(`Failed to persist status for ${executionId}:`, err);
+      return ExecutionControlSignal.UNSPECIFIED;
+    }
+  }
+
+  return ExecutionControlSignal.UNSPECIFIED;
 }
 
 /**

package/src/{activities/execute-deep-agent → shared}/streaming-scheduler.ts

@@ -13,6 +13,10 @@
  *     OR firstUpdate
  *
  * Ported from Python StreamingUpdateScheduler.
+ *
+ * Shared infrastructure: both the native deep-agent harness and the Cursor
+ * harness drive their incremental status persistence through this scheduler so
+ * the live stream cadence is identical across harnesses.
  */
 
 export enum UpdateReason {

package/dist/activities/execute-deep-agent/streaming-scheduler.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"streaming-scheduler.js","sourceRoot":"","sources":["../../../src/activities/execute-deep-agent/streaming-scheduler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,CAAN,IAAY,YAMX;AAND,WAAY,YAAY;IACtB,iDAAiC,CAAA;IACjC,qDAAqC,CAAA;IACrC,uCAAuB,CAAA;IACvB,6CAA6B,CAAA;IAC7B,6BAAa,CAAA;AACf,CAAC,EANW,YAAY,KAAZ,YAAY,QAMvB;AAWD,MAAM,cAAc,GAAoB;IACtC,aAAa,EAAE,GAAG;IAClB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,EAAE;CACnB,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,mBAAmB;IACjC,MAAM,aAAa,GAAG,gBAAgB,CACpC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EACrC,cAAc,CAAC,aAAa,EAC5B,2BAA2B,CAC5B,CAAC;IACF,IAAI,aAAa,GAAG,gBAAgB,CAClC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EACrC,cAAc,CAAC,aAAa,EAC5B,2BAA2B,CAC5B,CAAC;IACF,MAAM,cAAc,GAAG,gBAAgB,CACrC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EACrC,cAAc,CAAC,cAAc,EAC7B,2BAA2B,CAC5B,CAAC;IAEF,IAAI,aAAa,GAAG,aAAa,EAAE,CAAC;QAClC,OAAO,CAAC,IAAI,CACV,8BAA8B,aAAa,MAAM;YACjD,8BAA8B,aAAa,8BAA8B,CAC1E,CAAC;QACF,aAAa,GAAG,aAAa,CAAC;IAChC,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,gBAAgB,CACvB,GAAuB,EACvB,QAAgB,EAChB,OAAe;IAEf,IAAI,CAAC,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACzE,OAAO,CAAC,IAAI,CACV,WAAW,OAAO,KAAK,GAAG,qBAAqB,QAAQ,EAAE,CAC1D,CAAC;QACF,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,wBAAwB;IAC1B,MAAM,CAAkB;IAEzB,cAAc,CAAS;IACvB,gBAAgB,GAAG,CAAC,CAAC;IACrB,UAAU,GAAiB,YAAY,CAAC,IAAI,CAAC;IAC7C,UAAU,GAAG,IAAI,CAAC;IAE1B;;;OAGG;IACH,YAAY,MAAwB,EAAE,KAAc;QAClD,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,cAAc,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,KAAK,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;IACnD,CAAC;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,eAAuB,EAAE,KAAc;QACtD,MAAM,GAAG,GAAG,KAAK,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;QACvC,MAAM,eAAe,GAAG,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC;QAClD,MAAM,eAAe,GAAG,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAEhE,IAAI,IAAI,CAAC,UAAU,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,YAAY,CAAC;YAC5C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,cAAc,CAAC;YAC9C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAClD,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,gBAAgB,CAAC;YAChD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YACjD,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC;YACzC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC;QACpC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,qDAAqD;IACrD,cAAc,CAAC,eAAuB,EAAE,KAAc;QACpD,IAAI,CAAC,cAAc,GAAG,KAAK,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;QACjD,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;IAC1B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,qBAAqB,CAAC,KAAc;QAClC,OAAO,CAAC,KAAK,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC;IAC5D,CAAC;IAED,qBAAqB,CAAC,eAAuB;QAC3C,OAAO,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACjD,CAAC;CACF"}