@stigmer/react 3.0.8-dev.20260612122433 → 3.0.8-dev.20260613051837

package/src/execution/useExecutionStream.ts

@@ -11,14 +11,23 @@ import {
 } from "react";
 import type { AgentExecution } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
 import { ExecutionPhase } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+import { isTransientStreamError } from "@stigmer/sdk";
 import { useStigmer } from "../hooks";
 import { toError } from "../internal/toError";
 import { useStreamRate } from "../internal/dev";
 import {
   StreamController,
+  DEFAULT_CONNECT_TIMEOUT_MS,
+  DEFAULT_SLOW_THRESHOLD_MS,
   type StreamControllerSink,
 } from "../internal/stream-controller";
-import { ConversationStore, type StreamState } from "../internal/store";
+import {
+  computeBackoffDelay,
+  sleep,
+  DEFAULT_RECONNECT_MAX_ATTEMPTS,
+  type BackoffOptions,
+} from "../internal/backoff";
+import { ConversationStore } from "../internal/store";
 import { isTerminalPhase } from "./execution-phases";
 
 /** Return value of {@link useExecutionStream}. */
@@ -37,14 +46,46 @@ export interface UseExecutionStreamReturn {
   readonly isStreaming: boolean;
   /** `true` after subscription starts but before the first snapshot arrives. */
   readonly isConnecting: boolean;
-  /** Error from the last failed stream attempt, or `null` when healthy. */
+  /**
+   * `true` while a transient drop is being retried automatically in the
+   * background. The last snapshot stays visible and `error` remains `null` —
+   * surface a subtle "Reconnecting…" affordance, not an error. Becomes
+   * `false` once a snapshot is received (back to `isStreaming`) or retries
+   * are exhausted (then `error` is set).
+   */
+  readonly isReconnecting: boolean;
+  /** 1-based count of the in-flight reconnect attempt; `0` when not reconnecting. */
+  readonly reconnectAttempt: number;
+  /**
+   * Error from the last failed stream attempt, or `null` when healthy.
+   *
+   * Only set once auto-reconnect has exhausted its attempts (or for a
+   * non-transient failure that is not retried). It stays `null` throughout
+   * background reconnection so a recoverable hiccup never shows as an error.
+   */
   readonly error: Error | null;
+  /**
+   * `true` when the stream opened but no first snapshot arrived within the
+   * connect-timeout window, even after one silent self-heal. Distinct from
+   * `error` (a thrown/closed stream that auto-reconnect retries): this is the
+   * *silent* hang the retry loop can never observe. Surface an actionable
+   * "the agent hasn't started — Retry" affordance wired to `reconnect()`.
+   * Cleared by the next snapshot or a manual `reconnect()`. Never auto-aborts.
+   */
+  readonly connectTimedOut: boolean;
+  /**
+   * `true` when a non-terminal stream has produced no new snapshot for the
+   * slow-stall window. Purely informational ("still working — taking longer
+   * than usual"); cleared by the next snapshot. Never an error, never aborts.
+   */
+  readonly isSlow: boolean;
   /**
    * Reset error state and re-establish the stream subscription.
    *
-   * Works in any lifecycle state — error, complete, or mid-stream.
-   * Uses the `connectKey` counter pattern consistent with `refetch()`
-   * in other SDK hooks.
+   * The fallback after auto-reconnect exhausts, and a manual escape hatch in
+   * any lifecycle state — error, complete, or mid-stream. Resets the retry
+   * counter and preserves the last snapshot (no flash to empty). Uses the
+   * `connectKey` counter pattern consistent with `refetch()` in other SDK hooks.
    */
   readonly reconnect: () => void;
 }
@@ -65,6 +106,33 @@ export interface UseExecutionStreamOptions {
    * preserving backward compatibility for standalone usage.
    */
   readonly store?: ConversationStore;
+  /**
+   * Automatically re-establish the subscription with exponential backoff
+   * when a non-terminal stream drops (transport error, idle timeout, laptop
+   * sleep). Defaults to `true`. Set `false` to opt out and surface every
+   * drop as an immediate `error` for manual `reconnect()`.
+   */
+  readonly autoReconnect?: boolean;
+  /**
+   * Tune the auto-reconnect backoff schedule and attempt cap. Omitted fields
+   * fall back to SDK defaults (base 1s, ×2, max 30s, 10 attempts).
+   */
+  readonly reconnectOptions?: BackoffOptions & {
+    /** Max attempts before surfacing a terminal `error`. */
+    readonly maxAttempts?: number;
+  };
+  /**
+   * Hard connect-timeout in ms: how long the stream may stay `connecting`
+   * (no first snapshot) before the watchdog self-heals once and then surfaces
+   * `connectTimedOut`. Defaults to {@link DEFAULT_CONNECT_TIMEOUT_MS} (10s).
+   */
+  readonly connectTimeoutMs?: number;
+  /**
+   * Soft slow-stall threshold in ms: how long a non-terminal stream may go
+   * without a new snapshot before `isSlow` flips on. Defaults to
+   * {@link DEFAULT_SLOW_THRESHOLD_MS} (60s).
+   */
+  readonly slowThresholdMs?: number;
 }
 
 /**
@@ -73,7 +141,18 @@ export interface UseExecutionStreamOptions {
  *
  * Manages the full subscription lifecycle through a finite state
  * machine: connection establishment, rAF-coalesced snapshot streaming,
- * terminal-phase detection, error handling, and manual reconnection.
+ * terminal-phase detection, automatic reconnection with exponential
+ * backoff on transient drops, and manual reconnection as the fallback.
+ *
+ * **Resilience:** a non-terminal stream drop — whether a thrown transport
+ * error (WebKit "Load failed", `fetch failed`, `Unavailable`) or a graceful
+ * server close mid-run (idle timeout, load-balancer recycle) — is retried
+ * automatically with backoff. The last snapshot stays visible
+ * (`isReconnecting`), the access token is re-read on each attempt via the
+ * per-request interceptor, and `error` is surfaced only once attempts are
+ * exhausted. Completion is decided by the terminal phase, never by the
+ * stream merely ending (a graceful close of a running execution reconnects
+ * rather than falsely reporting "complete"). Opt out via `autoReconnect: false`.
  *
  * **Performance characteristics:**
  * - Non-terminal snapshots are coalesced via `requestAnimationFrame`
@@ -122,6 +201,17 @@ export function useExecutionStream(
   }
   const store = options?.store ?? internalStoreRef.current!;
 
+  // -- Reconnect ------------------------------------------------------------
+  const [connectKey, setConnectKey] = useState(0);
+
+  // Self-heal-once guard for the connect-timeout watchdog. A ref so it survives
+  // the reconnect that the watchdog itself triggers (the controller is reset on
+  // every teardown and therefore cannot hold it). Reset only on a genuinely
+  // fresh start — a new execution, a healthy snapshot, or a manual reconnect —
+  // never on the watchdog's own self-heal hop, so the policy is exactly:
+  // first silent timeout → reconnect once; second → surface.
+  const connectAutoRetriedRef = useRef(false);
+
   // -- Controller setup -----------------------------------------------------
   const controllerRef = useRef<StreamController | null>(null);
   if (!controllerRef.current) {
@@ -136,22 +226,53 @@ export function useExecutionStream(
           store.setStreamState(state);
         });
       },
+      onConnectTimeout() {
+        if (connectAutoRetriedRef.current) {
+          // Already self-healed once and still silent — surface an actionable
+          // signal. The store dedupes, so this never causes a render storm.
+          store.setConnectTimedOut(true);
+        } else {
+          // First silent timeout: re-establish the subscription once. Most
+          // transient hangs (buffering proxy, cold start) clear on the retry.
+          connectAutoRetriedRef.current = true;
+          setConnectKey((k) => k + 1);
+        }
+      },
+      setSlow(value) {
+        store.setSlow(value);
+      },
     };
-    controllerRef.current = new StreamController(sink);
+    controllerRef.current = new StreamController(sink, undefined, undefined, {
+      connectTimeoutMs: options?.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS,
+      slowThresholdMs: options?.slowThresholdMs ?? DEFAULT_SLOW_THRESHOLD_MS,
+    });
   }
   const controller = controllerRef.current;
 
-  // -- Reconnect ------------------------------------------------------------
-  const [connectKey, setConnectKey] = useState(0);
   const reconnect = useCallback(() => {
+    // A manual retry gives a fresh self-heal budget and clears the surfaced
+    // timeout so the affordance disappears the instant the user acts.
+    connectAutoRetriedRef.current = false;
+    store.setConnectTimedOut(false);
     setConnectKey((k) => k + 1);
-  }, []);
+  }, [store]);
 
   // -- Stream rate instrumentation ------------------------------------------
   const streamRate = useStreamRate();
   const streamRateRef = useRef(streamRate);
   streamRateRef.current = streamRate;
 
+  // -- Reconnect config (ref-backed so option identity churn never resubscribes)
+  const autoReconnect = options?.autoReconnect ?? true;
+  const reconnectOptions = options?.reconnectOptions;
+  const configRef = useRef({ autoReconnect, reconnectOptions });
+  configRef.current = { autoReconnect, reconnectOptions };
+
+  // Tracks the execution the store currently holds, so we reset the store on
+  // a genuine identity change (A → B) but preserve it across reconnects of the
+  // SAME execution. Mirrors useWorkflowExecutionEventStream / useFetch.
+  const prevExecutionIdRef = useRef<string | null>(null);
+
   // -- Subscription effect --------------------------------------------------
   // Note: controller, store, and streamRate are ref-backed stable objects —
   // they MUST NOT appear in the deps array. Including them would cause
@@ -160,45 +281,126 @@ export function useExecutionStream(
     if (!executionId) {
       controller.reset();
       store.reset();
+      prevExecutionIdRef.current = null;
+      connectAutoRetriedRef.current = false;
       return;
     }
 
+    // Reset only when switching to a different execution. Crucially we do NOT
+    // reset the store on reconnect (connectKey bump) or on cleanup — that
+    // would wipe the conversation to an empty "Connecting…" on every retry.
+    // The full-snapshot subscribe re-delivers the entire state on reconnect,
+    // so keeping the last-known-good snapshot is both correct and seamless.
+    if (
+      prevExecutionIdRef.current !== null &&
+      prevExecutionIdRef.current !== executionId
+    ) {
+      store.reset();
+      // A genuinely different execution earns a fresh self-heal budget; the
+      // bump that re-runs this effect on reconnect must NOT reset the guard.
+      connectAutoRetriedRef.current = false;
+    }
+    prevExecutionIdRef.current = executionId;
+
     const abortController = new AbortController();
+    const signal = abortController.signal;
     controller.start(executionId);
 
     (async () => {
-      try {
-        for await (const snapshot of stigmer.agentExecution.subscribe(
-          executionId,
-          abortController.signal,
-        )) {
-          if (abortController.signal.aborted) return;
-
-          controller.handleSnapshot(snapshot);
-          streamRateRef.current.tick(
-            snapshot.status?.messages?.length ?? 0,
-          );
-
-          const phase =
-            snapshot.status?.phase ??
-            ExecutionPhase.EXECUTION_PHASE_UNSPECIFIED;
-          if (isTerminalPhase(phase)) break;
+      const { autoReconnect: auto, reconnectOptions: backoff } =
+        configRef.current;
+      const maxAttempts = backoff?.maxAttempts ?? DEFAULT_RECONNECT_MAX_ATTEMPTS;
+
+      // 1-based count of consecutive failed attempts. Reset to 0 by any
+      // successful snapshot, so each healthy stretch gets a fresh backoff
+      // budget rather than inheriting the previous outage's attempt count.
+      let attempt = 0;
+
+      // Schedule the next retry after `error`, or stop. Returns `true` when
+      // the loop should continue (a retry was scheduled), `false` when it
+      // should exit (opted out, exhausted, or aborted). Shared by the
+      // thrown-error and premature-end paths so both converge on one policy.
+      const scheduleRetry = async (error: Error): Promise<boolean> => {
+        if (!auto || attempt >= maxAttempts) {
+          controller.handleError(error);
+          return false;
         }
+        attempt += 1;
+        controller.handleReconnecting(attempt, error);
+        try {
+          await sleep(computeBackoffDelay(attempt, backoff), signal);
+        } catch {
+          return false; // aborted mid-backoff
+        }
+        return !signal.aborted;
+      };
+
+      while (!signal.aborted) {
+        let sawTerminal = false;
+        try {
+          for await (const snapshot of stigmer.agentExecution.subscribe(
+            executionId,
+            signal,
+          )) {
+            if (signal.aborted) return;
+
+            attempt = 0; // a snapshot proves the connection is healthy
+            // The first snapshot also resolves the connect watchdog: clear any
+            // surfaced timeout and refresh the self-heal budget for any future
+            // silent stretch on this same execution.
+            connectAutoRetriedRef.current = false;
+            store.setConnectTimedOut(false);
+            controller.handleSnapshot(snapshot);
+            streamRateRef.current.tick(snapshot.status?.messages?.length ?? 0);
+
+            const phase =
+              snapshot.status?.phase ??
+              ExecutionPhase.EXECUTION_PHASE_UNSPECIFIED;
+            if (isTerminalPhase(phase)) {
+              sawTerminal = true;
+              break;
+            }
+          }
+        } catch (err) {
+          if (signal.aborted) return;
+          const error = toError(err);
+          // Only known-transient transport noise is retried. A non-transient
+          // error (not-found, invalid-argument, …) is deterministic — the
+          // same request would fail identically, so surface it immediately.
+          if (!auto || !isTransientStreamError(error)) {
+            controller.handleError(error);
+            return;
+          }
+          if (await scheduleRetry(error)) continue;
+          return;
+        }
+
+        if (signal.aborted) return;
 
-        if (!abortController.signal.aborted) {
+        if (sawTerminal) {
+          // handleSnapshot already transitioned to `complete`; flush any
+          // buffered frame and finish. Completion is decided by the terminal
+          // phase, never by the stream merely ending.
           controller.handleStreamEnd();
           streamRateRef.current.summary();
+          return;
         }
-      } catch (err) {
-        if (abortController.signal.aborted) return;
-        controller.handleError(toError(err));
+
+        // The iterator finished without a terminal phase: the server closed a
+        // still-running stream (idle timeout, load-balancer recycle, pod
+        // restart). This is transient by definition — reconnect and the next
+        // full snapshot reconciles whatever changed (including, if it ended
+        // meanwhile, the terminal state we missed).
+        if (await scheduleRetry(new Error("The connection was interrupted."))) {
+          continue;
+        }
+        return;
       }
     })();
 
     return () => {
       abortController.abort();
       controller.reset();
-      store.reset();
     };
   }, [executionId, stigmer, connectKey]);
 
@@ -208,6 +410,11 @@ export function useExecutionStream(
     store.subscribe,
     store.getStreamState,
   );
+  const connectTimedOut = useSyncExternalStore(
+    store.subscribe,
+    store.getConnectTimedOut,
+  );
+  const isSlow = useSyncExternalStore(store.subscribe, store.getSlow);
 
   // -- Derive public return values ------------------------------------------
   const phase = useMemo(
@@ -218,8 +425,21 @@ export function useExecutionStream(
 
   const isStreaming = streamState.stage === "streaming";
   const isConnecting = streamState.stage === "connecting";
-  const error =
-    streamState.stage === "error" ? streamState.error : null;
+  const isReconnecting = streamState.stage === "reconnecting";
+  const reconnectAttempt =
+    streamState.stage === "reconnecting" ? streamState.attempt : 0;
+  const error = streamState.stage === "error" ? streamState.error : null;
 
-  return { execution, phase, isStreaming, isConnecting, error, reconnect };
+  return {
+    execution,
+    phase,
+    isStreaming,
+    isConnecting,
+    isReconnecting,
+    reconnectAttempt,
+    error,
+    connectTimedOut,
+    isSlow,
+    reconnect,
+  };
 }

package/src/internal/VirtualizedThread.tsx

@@ -41,6 +41,8 @@ export interface VirtualizedThreadProps {
   readonly org?: string;
   readonly planActionsDisabled?: boolean;
   readonly centerContent?: boolean;
+  readonly onRetrySend?: () => void;
+  readonly onRetryExecution?: (message: string) => void;
 }
 
 // ---------------------------------------------------------------------------
@@ -101,6 +103,8 @@ export function VirtualizedThread({
   org,
   planActionsDisabled,
   centerContent,
+  onRetrySend,
+  onRetryExecution,
 }: VirtualizedThreadProps) {
   const virtuosoRef = useRef<VirtuosoHandle>(null);
   const scrollerRef = useRef<HTMLDivElement | null>(null);
@@ -132,8 +136,10 @@ export function VirtualizedThread({
       onBuildFromPlan,
       org,
       planActionsDisabled,
+      onRetrySend,
+      onRetryExecution,
     }),
-    [formatToolCallSummary, onApprovalSubmit, submittingApprovalIds, onBuildFromPlan, org, planActionsDisabled],
+    [formatToolCallSummary, onApprovalSubmit, submittingApprovalIds, onBuildFromPlan, org, planActionsDisabled, onRetrySend, onRetryExecution],
   );
 
   return (

package/src/internal/__tests__/backoff.test.ts

@@ -0,0 +1,99 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import {
+  computeBackoffDelay,
+  sleep,
+  AbortError,
+  DEFAULT_RECONNECT_BASE_DELAY_MS,
+  DEFAULT_RECONNECT_MAX_DELAY_MS,
+} from "../backoff";
+
+describe("computeBackoffDelay", () => {
+  // random=()=>1 collapses full jitter to its upper bound, exposing the raw
+  // exponential schedule for exact assertions.
+  const noJitter = () => 1;
+
+  it("grows exponentially from the base delay", () => {
+    expect(computeBackoffDelay(1, undefined, noJitter)).toBe(
+      DEFAULT_RECONNECT_BASE_DELAY_MS,
+    );
+    expect(computeBackoffDelay(2, undefined, noJitter)).toBe(2_000);
+    expect(computeBackoffDelay(3, undefined, noJitter)).toBe(4_000);
+    expect(computeBackoffDelay(5, undefined, noJitter)).toBe(16_000);
+  });
+
+  it("caps at maxDelayMs", () => {
+    // attempt 6 → 32_000 raw, clamped to the 30_000 ceiling.
+    expect(computeBackoffDelay(6, undefined, noJitter)).toBe(
+      DEFAULT_RECONNECT_MAX_DELAY_MS,
+    );
+    expect(computeBackoffDelay(50, undefined, noJitter)).toBe(
+      DEFAULT_RECONNECT_MAX_DELAY_MS,
+    );
+  });
+
+  it("applies full jitter within [0, capped]", () => {
+    expect(computeBackoffDelay(3, undefined, () => 0)).toBe(0);
+    expect(computeBackoffDelay(3, undefined, () => 0.5)).toBe(2_000);
+    for (let i = 0; i < 200; i++) {
+      const d = computeBackoffDelay(4); // real Math.random
+      expect(d).toBeGreaterThanOrEqual(0);
+      expect(d).toBeLessThanOrEqual(8_000);
+    }
+  });
+
+  it("honors custom options", () => {
+    const opts = { baseDelayMs: 100, factor: 3, maxDelayMs: 1_000 };
+    expect(computeBackoffDelay(1, opts, noJitter)).toBe(100);
+    expect(computeBackoffDelay(2, opts, noJitter)).toBe(300);
+    expect(computeBackoffDelay(3, opts, noJitter)).toBe(900);
+    expect(computeBackoffDelay(4, opts, noJitter)).toBe(1_000); // 2700 capped
+  });
+
+  it("treats attempt < 1 as the first attempt", () => {
+    expect(computeBackoffDelay(0, undefined, noJitter)).toBe(
+      DEFAULT_RECONNECT_BASE_DELAY_MS,
+    );
+    expect(computeBackoffDelay(-5, undefined, noJitter)).toBe(
+      DEFAULT_RECONNECT_BASE_DELAY_MS,
+    );
+  });
+});
+
+describe("sleep", () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  it("resolves after the delay", async () => {
+    const settled = vi.fn();
+    const p = sleep(1_000).then(settled);
+    await vi.advanceTimersByTimeAsync(999);
+    expect(settled).not.toHaveBeenCalled();
+    await vi.advanceTimersByTimeAsync(1);
+    await p;
+    expect(settled).toHaveBeenCalledOnce();
+  });
+
+  it("rejects immediately with AbortError when the signal is already aborted", async () => {
+    const ac = new AbortController();
+    ac.abort();
+    await expect(sleep(1_000, ac.signal)).rejects.toBeInstanceOf(AbortError);
+  });
+
+  it("rejects when aborted mid-wait and leaves no pending timer", async () => {
+    const ac = new AbortController();
+    const p = sleep(10_000, ac.signal);
+    ac.abort();
+    await expect(p).rejects.toBeInstanceOf(AbortError);
+    // No timer should survive the abort — advancing time settles nothing.
+    expect(vi.getTimerCount()).toBe(0);
+  });
+
+  it("does not reject after resolving (listener removed on success)", async () => {
+    const ac = new AbortController();
+    const p = sleep(500, ac.signal);
+    await vi.advanceTimersByTimeAsync(500);
+    await expect(p).resolves.toBeUndefined();
+    // Aborting after the fact must not produce an unhandled rejection.
+    expect(() => ac.abort()).not.toThrow();
+  });
+});