@stigmer/runner 3.0.8-dev.20260613085218 → 3.0.9-dev.20260615145121

package/src/runner-manager.ts

@@ -24,10 +24,22 @@ import {
 } from "@temporalio/worker";
 import type { PayloadCodec } from "@temporalio/common";
 import type { Config } from "./config.js";
+import { DEFAULT_CURSOR_STREAM_STALL_TIMEOUT_MS } from "./config.js";
 import type { WorkerActivities } from "./worker.js";
 import { resolveWorkflowSource, OTEL_WORKFLOW_INTERCEPTOR_MODULE } from "./workflow-source.js";
 import { resolveRunnerBootstrap, refreshRunnerAccessToken } from "./bootstrap.js";
 import { createRunnerTokenCoordinator } from "./runner-token-coordinator.js";
+// Per-task-queue in-flight activity tracking lives in ./in-flight.ts so the
+// activity interceptor (no manager-closure handle) and unit tests can reach it.
+// This is what keeps a session worker alive while ExecuteCursor is running, so a
+// view close can no longer reap the worker mid-run.
+import {
+  activityStartedOnQueue,
+  activityFinishedOnQueue,
+  inFlightCountForQueue,
+  setQueueDrainCallback,
+  forgetQueue,
+} from "./in-flight.js";
 
 const SESSION_QUEUE_PREFIX = "session:";
 const WFEXEC_QUEUE_PREFIX = "wfexec:";
@@ -76,6 +88,9 @@ export interface RunnerManagerOptions {
   /** Default LLM model identifier. @default "gpt-4.1" */
   readonly primaryModel?: string;
 
+  /** No-progress bound for the Cursor harness stream (ms). @default 180000 */
+  readonly cursorStreamStallTimeoutMs?: number;
+
   /** Checkpointer type for LangGraph agent state. @default "memory" (or "http" if proxyEndpoint is set) */
   readonly checkpointerType?: "memory" | "http";
 
@@ -141,6 +156,12 @@ interface ManagedSession {
   worker: Worker;
   runPromise: Promise<void>;
   shutdownController: AbortController;
+  /**
+   * Set when a remove was requested while an activity was still in flight, so
+   * the worker is kept alive in the background and torn down only once the last
+   * activity finishes. Cleared if the session is re-opened before it drains.
+   */
+  pendingClose: boolean;
 }
 
 /**
@@ -324,7 +345,84 @@ export async function createStigmerRunnerManager(
       );
     });
 
-    return { worker, runPromise, shutdownController };
+    return { worker, runPromise, shutdownController, pendingClose: false };
+  }
+
+  /**
+   * Re-opening a session/execution whose teardown was deferred: cancel the
+   * pending close so the worker keeps serving the reused queue. Returns true
+   * when an existing managed worker was found (caller should not recreate one).
+   */
+  function reuseExistingWorker(
+    registry: Map<string, ManagedSession>,
+    id: string,
+    taskQueue: string,
+    kind: string,
+  ): boolean {
+    const existing = registry.get(id);
+    if (!existing) return false;
+    if (existing.pendingClose) {
+      existing.pendingClose = false;
+      setQueueDrainCallback(taskQueue, undefined);
+      console.log(`[runner-manager] Re-opened ${kind} ${id}; cancelled deferred teardown`);
+    }
+    return true;
+  }
+
+  /**
+   * Graceful teardown of a managed worker. Never aborts: callers only reach the
+   * actual teardown once no activity is in flight, so a plain worker.shutdown()
+   * drains the (idle) queue cleanly. The abort-before-shutdown that used to live
+   * here is what killed running activities on a view close.
+   */
+  async function teardownManaged(
+    registry: Map<string, ManagedSession>,
+    id: string,
+    taskQueue: string,
+    kind: string,
+  ): Promise<void> {
+    const managed = registry.get(id);
+    if (!managed) return;
+    managed.worker.shutdown();
+    await managed.runPromise;
+    registry.delete(id);
+    shutdownSignals.delete(taskQueue);
+    _shutdownSignalRegistry.delete(taskQueue);
+    forgetQueue(taskQueue);
+    console.log(`[runner-manager] Removed ${kind} ${id} (active=${registry.size})`);
+  }
+
+  /**
+   * Remove a managed worker, deferring teardown while activities are in flight.
+   * This is the server-side safety invariant that lets a run continue in the
+   * background after its session view closes: the worker is reaped only when the
+   * last activity finishes (or immediately if the queue is already idle).
+   */
+  async function removeManaged(
+    registry: Map<string, ManagedSession>,
+    id: string,
+    taskQueue: string,
+    kind: string,
+  ): Promise<void> {
+    const managed = registry.get(id);
+    if (!managed) return;
+
+    if (inFlightCountForQueue(taskQueue) > 0) {
+      managed.pendingClose = true;
+      setQueueDrainCallback(taskQueue, () => {
+        // Skip if a full shutdown() is already reaping every worker, so we
+        // never call worker.shutdown() twice on the same worker.
+        if (shuttingDown) return;
+        void teardownManaged(registry, id, taskQueue, kind);
+      });
+      console.log(
+        `[runner-manager] Deferring teardown of ${kind} ${id} — ` +
+        `${inFlightCountForQueue(taskQueue)} activity(ies) still in flight (runs in background)`,
+      );
+      return;
+    }
+
+    await teardownManaged(registry, id, taskQueue, kind);
   }
 
   return {
@@ -332,11 +430,11 @@ export async function createStigmerRunnerManager(
       if (shuttingDown) {
         throw new Error("RunnerManager is shutting down");
       }
-      if (sessions.has(sessionId)) {
+      const taskQueue = SESSION_QUEUE_PREFIX + sessionId;
+      if (reuseExistingWorker(sessions, sessionId, taskQueue, "session")) {
         return;
       }
 
-      const taskQueue = SESSION_QUEUE_PREFIX + sessionId;
       const managed = await createWorkerOnQueue(taskQueue);
       sessions.set(sessionId, managed);
       console.log(
@@ -345,20 +443,8 @@ export async function createStigmerRunnerManager(
     },
 
     async removeSession(sessionId: string): Promise<void> {
-      const session = sessions.get(sessionId);
-      if (!session) {
-        return;
-      }
-
-      const taskQueue = SESSION_QUEUE_PREFIX + sessionId;
-      session.shutdownController.abort();
-      session.worker.shutdown();
-      await session.runPromise;
-      sessions.delete(sessionId);
-      shutdownSignals.delete(taskQueue);
-      _shutdownSignalRegistry.delete(taskQueue);
-      console.log(
-        `[runner-manager] Removed session ${sessionId} (active=${sessions.size})`,
+      await removeManaged(
+        sessions, sessionId, SESSION_QUEUE_PREFIX + sessionId, "session",
       );
     },
 
@@ -370,11 +456,11 @@ export async function createStigmerRunnerManager(
       if (shuttingDown) {
         throw new Error("RunnerManager is shutting down");
       }
-      if (workflowExecutions.has(executionId)) {
+      const taskQueue = WFEXEC_QUEUE_PREFIX + executionId;
+      if (reuseExistingWorker(workflowExecutions, executionId, taskQueue, "workflow execution")) {
         return;
       }
 
-      const taskQueue = WFEXEC_QUEUE_PREFIX + executionId;
       const managed = await createWorkerOnQueue(taskQueue);
       workflowExecutions.set(executionId, managed);
       console.log(
@@ -383,20 +469,9 @@ export async function createStigmerRunnerManager(
     },
 
     async removeWorkflowExecution(executionId: string): Promise<void> {
-      const execution = workflowExecutions.get(executionId);
-      if (!execution) {
-        return;
-      }
-
-      const taskQueue = WFEXEC_QUEUE_PREFIX + executionId;
-      execution.shutdownController.abort();
-      execution.worker.shutdown();
-      await execution.runPromise;
-      workflowExecutions.delete(executionId);
-      shutdownSignals.delete(taskQueue);
-      _shutdownSignalRegistry.delete(taskQueue);
-      console.log(
-        `[runner-manager] Removed workflow execution ${executionId} (active=${workflowExecutions.size})`,
+      await removeManaged(
+        workflowExecutions, executionId, WFEXEC_QUEUE_PREFIX + executionId,
+        "workflow execution",
       );
     },
 
@@ -499,6 +574,8 @@ export function mapManagerOptionsToConfig(
     checkpointerProxyEndpoint:
       options.checkpointerProxyEndpoint ?? options.proxyEndpoint ?? null,
     primaryModel: options.primaryModel ?? "gpt-4.1",
+    cursorStreamStallTimeoutMs:
+      options.cursorStreamStallTimeoutMs ?? DEFAULT_CURSOR_STREAM_STALL_TIMEOUT_MS,
   };
 }
 
@@ -600,6 +677,23 @@ async function buildInterceptorConfig(): Promise<InterceptorConfig> {
   let sinks: InjectedSinks<any> = { ...createWorkflowMetricsSinks() };
   const workflowInterceptorModules: string[] = [];
 
+  // In-flight activity counter: keeps a session/wfexec worker alive while one of
+  // its activities (notably the long ExecuteCursor) is running, so a view close
+  // can no longer reap the worker mid-run. Always installed; cheap and global.
+  activityInterceptors.push((ctx) => ({
+    inbound: {
+      async execute(input, next) {
+        const taskQueue = ctx.info.taskQueue;
+        activityStartedOnQueue(taskQueue);
+        try {
+          return await next(input);
+        } finally {
+          activityFinishedOnQueue(taskQueue);
+        }
+      },
+    },
+  }));
+
   if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) {
     const { OpenTelemetryActivityInboundInterceptor, makeWorkflowExporter } =
       await import("@temporalio/interceptors-opentelemetry");

package/src/runner.ts

@@ -17,6 +17,7 @@ import { join } from "node:path";
 import { homedir, tmpdir } from "node:os";
 import type { PayloadCodec } from "@temporalio/common";
 import type { Config } from "./config.js";
+import { DEFAULT_CURSOR_STREAM_STALL_TIMEOUT_MS } from "./config.js";
 import type { WorkerActivities } from "./worker.js";
 import { resolveRunnerBootstrap } from "./bootstrap.js";
 
@@ -65,6 +66,9 @@ export interface StigmerRunnerOptions {
   /** Default LLM model identifier. @default "gpt-4.1" */
   readonly primaryModel?: string;
 
+  /** No-progress bound for the Cursor harness stream (ms). @default 180000 */
+  readonly cursorStreamStallTimeoutMs?: number;
+
   /** Checkpointer type for LangGraph agent state. @default "memory" (or "http" if proxyEndpoint is set) */
   readonly checkpointerType?: "memory" | "http";
 
@@ -268,6 +272,8 @@ export function mapOptionsToConfig(options: StigmerRunnerOptions): Config {
       ?? options.proxyEndpoint
       ?? null,
     primaryModel: options.primaryModel ?? "gpt-4.1",
+    cursorStreamStallTimeoutMs:
+      options.cursorStreamStallTimeoutMs ?? DEFAULT_CURSOR_STREAM_STALL_TIMEOUT_MS,
   };
 }
 

package/src/shared/__tests__/artifact-storage.test.ts

@@ -315,6 +315,7 @@ describe("loadArtifactStorageConfig", () => {
     checkpointerType: "memory" as const,
     checkpointerProxyEndpoint: null,
     primaryModel: "gpt-4.1",
+    cursorStreamStallTimeoutMs: 180000,
   };
 
   afterEach(() => {

package/src/shared/__tests__/grpc-retry-extended.test.ts

@@ -1,30 +1,13 @@
 /**
- * Extended gRPC retry tests — ported from Python test_grpc_retry.py.
- *
- * Covers config validation, backoff timing, INTERNAL error handling,
- * mixed error sequences, and max-retries edge cases beyond the core
- * 15 tests in grpc-retry.test.ts.
+ * Extended gRPC error-classification tests — ported from Python
+ * test_grpc_retry.py. Covers additional status codes beyond the core
+ * grpc-retry.test.ts. The persist-with-backoff loop that consumes these
+ * classifiers is tested in status.test.ts ("persistStatus — transient retry").
  */
 
-import { describe, it, expect, vi } from "vitest";
+import { describe, it, expect } from "vitest";
 import { ConnectError, Code } from "@connectrpc/connect";
-import { create } from "@bufbuild/protobuf";
-import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
-import { ExecutionControlSignal } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
-import { persistWithRetry, isRetryableError, isTerminalError } from "../grpc-retry.js";
-import type { StigmerClient } from "../../client/stigmer-client.js";
-
-function makeStatus() {
-  return create(AgentExecutionStatusSchema, {});
-}
-
-function mockClient(
-  impl: (...args: unknown[]) => Promise<{ signal: ExecutionControlSignal }>,
-): StigmerClient {
-  return { updateStatus: impl } as unknown as StigmerClient;
-}
-
-const noDelay = async () => {};
+import { isRetryableError, isTerminalError } from "../grpc-retry.js";
 
 describe("isRetryableError — extended codes", () => {
   it("returns false for INTERNAL", () => {
@@ -53,124 +36,3 @@ describe("isTerminalError — extended codes", () => {
     expect(isTerminalError(new Error("random"))).toBe(false);
   });
 });
-
-describe("persistWithRetry — extended scenarios", () => {
-  it("does not retry INTERNAL errors (neither retryable nor terminal)", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new ConnectError("internal", Code.Internal);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-int", makeStatus(), {
-      maxRetries: 3,
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(attempt).toBe(1);
-  });
-
-  it("does not retry ALREADY_EXISTS", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new ConnectError("exists", Code.AlreadyExists);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-ae", makeStatus(), {
-      maxRetries: 3,
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(attempt).toBe(1);
-  });
-
-  it("handles mixed error sequence: retryable then terminal", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      if (attempt === 1) throw new ConnectError("down", Code.Unavailable);
-      throw new ConnectError("gone", Code.NotFound);
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-mix", makeStatus(), {
-      delayFn: noDelay,
-    });
-    warnSpy.mockRestore();
-    errorSpy.mockRestore();
-
-    expect(attempt).toBe(2);
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-  });
-
-  it("handles retryable then success", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      if (attempt <= 2) throw new ConnectError("timeout", Code.DeadlineExceeded);
-      return { signal: ExecutionControlSignal.STOP };
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-ret", makeStatus(), {
-      maxRetries: 5,
-      delayFn: noDelay,
-    });
-    warnSpy.mockRestore();
-
-    expect(signal).toBe(ExecutionControlSignal.STOP);
-    expect(attempt).toBe(3);
-  });
-
-  it("maxRetries = 0 means one attempt only", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new ConnectError("down", Code.Unavailable);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-0", makeStatus(), {
-      maxRetries: 0,
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(attempt).toBe(1);
-  });
-
-  it("respects custom backoff factor", async () => {
-    const delays: number[] = [];
-    const client = mockClient(async () => {
-      throw new ConnectError("down", Code.Unavailable);
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-bf", makeStatus(), {
-      baseDelayMs: 50,
-      backoffFactor: 3,
-      maxRetries: 3,
-      delayFn: async (ms) => { delays.push(ms); },
-    });
-    warnSpy.mockRestore();
-    errorSpy.mockRestore();
-
-    expect(delays).toEqual([50, 150, 450]);
-  });
-
-  it("uses default options when none provided", async () => {
-    const client = mockClient(async () => ({
-      signal: ExecutionControlSignal.UNSPECIFIED,
-    }));
-
-    const signal = await persistWithRetry(client, "exec-default", makeStatus());
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-  });
-});

package/src/shared/__tests__/grpc-retry.test.ts

@@ -1,26 +1,10 @@
-import { describe, it, expect, vi } from "vitest";
+import { describe, it, expect } from "vitest";
 import { ConnectError, Code } from "@connectrpc/connect";
-import { create } from "@bufbuild/protobuf";
-import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
-import { ExecutionControlSignal } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
-import {
-  persistWithRetry,
-  isRetryableError,
-  isTerminalError,
-} from "../grpc-retry.js";
-import type { StigmerClient } from "../../client/stigmer-client.js";
+import { isRetryableError, isTerminalError } from "../grpc-retry.js";
 
-function makeStatus() {
-  return create(AgentExecutionStatusSchema, {});
-}
-
-function mockClient(
-  updateStatusImpl: (...args: unknown[]) => Promise<{ signal: ExecutionControlSignal }>,
-): StigmerClient {
-  return { updateStatus: updateStatusImpl } as unknown as StigmerClient;
-}
-
-const noDelay = async () => {};
+// The persist-with-backoff loop that consumes these classifiers lives in
+// status.ts and is exercised by status.test.ts ("persistStatus — transient
+// retry"). This file covers the classification policy in isolation.
 
 describe("isRetryableError", () => {
   it("returns true for UNAVAILABLE", () => {
@@ -57,116 +41,3 @@ describe("isTerminalError", () => {
     expect(isTerminalError(new ConnectError("down", Code.Unavailable))).toBe(false);
   });
 });
-
-describe("persistWithRetry", () => {
-  it("returns signal on successful first attempt", async () => {
-    const client = mockClient(async () => ({
-      signal: ExecutionControlSignal.STOP,
-    }));
-
-    const signal = await persistWithRetry(client, "exec-1", makeStatus(), {
-      delayFn: noDelay,
-    });
-
-    expect(signal).toBe(ExecutionControlSignal.STOP);
-  });
-
-  it("retries on UNAVAILABLE and succeeds", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      if (attempt === 1) throw new ConnectError("down", Code.Unavailable);
-      return { signal: ExecutionControlSignal.UNSPECIFIED };
-    });
-
-    const signal = await persistWithRetry(client, "exec-2", makeStatus(), {
-      delayFn: noDelay,
-    });
-
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-    expect(attempt).toBe(2);
-  });
-
-  it("does not retry terminal errors (NOT_FOUND)", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new ConnectError("gone", Code.NotFound);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-3", makeStatus(), {
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-    expect(attempt).toBe(1);
-  });
-
-  it("returns UNSPECIFIED after all retries exhausted", async () => {
-    const client = mockClient(async () => {
-      throw new ConnectError("down", Code.Unavailable);
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-4", makeStatus(), {
-      maxRetries: 3,
-      delayFn: noDelay,
-    });
-    warnSpy.mockRestore();
-    errorSpy.mockRestore();
-
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-  });
-
-  it("uses correct backoff delays", async () => {
-    const delays: number[] = [];
-    const client = mockClient(async () => {
-      throw new ConnectError("down", Code.Unavailable);
-    });
-
-    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await persistWithRetry(client, "exec-5", makeStatus(), {
-      baseDelayMs: 100,
-      backoffFactor: 2,
-      maxRetries: 3,
-      delayFn: async (ms) => { delays.push(ms); },
-    });
-    warnSpy.mockRestore();
-    errorSpy.mockRestore();
-
-    expect(delays).toEqual([100, 200, 400]);
-  });
-
-  it("handles non-ConnectError by not retrying", async () => {
-    let attempt = 0;
-    const client = mockClient(async () => {
-      attempt++;
-      throw new Error("unexpected");
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    const signal = await persistWithRetry(client, "exec-6", makeStatus(), {
-      delayFn: noDelay,
-    });
-    errorSpy.mockRestore();
-
-    expect(signal).toBe(ExecutionControlSignal.UNSPECIFIED);
-    expect(attempt).toBe(1);
-  });
-
-  it("never throws regardless of error type", async () => {
-    const client = mockClient(async () => {
-      throw new ConnectError("server exploded", Code.Internal);
-    });
-
-    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
-    await expect(
-      persistWithRetry(client, "exec-7", makeStatus(), { delayFn: noDelay }),
-    ).resolves.toBe(ExecutionControlSignal.UNSPECIFIED);
-    errorSpy.mockRestore();
-  });
-});