@stigmer/runner 3.12.3 → 3.12.4

package/src/activities/__tests__/call-llm.test.ts

@@ -236,6 +236,83 @@ describe("callLlmAction", () => {
     });
   });
 
+  describe("schema validation failure channels (#686)", () => {
+    const schemaConfig: LlmCallConfig = {
+      model: "gpt-4o-mini",
+      prompt: "classify",
+      response_schema: {
+        type: "object",
+        properties: { answer: { type: "number" } },
+        required: ["answer"],
+      },
+    };
+
+    it("default policy throws LLM_SCHEMA_VALIDATION after exactly one model call (no blind retry loop)", async () => {
+      process.env.OPENAI_API_KEY = "sk-test";
+      mockFetchWithRegistry(() => openAIJsonResponse('{"wrong_key": true}'));
+
+      await expect(
+        callLlmAction(schemaConfig, {}, "exec-1"),
+      ).rejects.toMatchObject({ type: "LLM_SCHEMA_VALIDATION" });
+
+      // Before maxRetries was pinned to 0, LangChain blind-retried the
+      // identical call ~7 times on every schema miss (#686).
+      const llmCalls = mockFetch.mock.calls.filter(
+        (call: unknown[]) => !(call[0] as string).includes("model-registry"),
+      );
+      expect(llmCalls).toHaveLength(1);
+    });
+
+    it("soft policies return parse_error instead of throwing so the engine can retry/branch", async () => {
+      process.env.OPENAI_API_KEY = "sk-test";
+      mockFetchWithRegistry(() => openAIJsonResponse('{"wrong_key": true}'));
+
+      const result = await callLlmAction(
+        { ...schemaConfig, on_invalid: "ON_INVALID_RETRY", max_retries: 2 },
+        {},
+        "exec-1",
+      );
+
+      expect(result.parse_error).toContain("answer");
+      expect(result.result).toBeUndefined();
+    });
+
+    it("a conforming response with a soft policy carries no parse_error", async () => {
+      process.env.OPENAI_API_KEY = "sk-test";
+      mockFetchWithRegistry(() => openAIJsonResponse('{"answer": 42}'));
+
+      const result = await callLlmAction(
+        { ...schemaConfig, on_invalid: "ON_INVALID_RETRY" },
+        {},
+        "exec-1",
+      );
+
+      expect(result.parse_error).toBeUndefined();
+      expect(result.result).toEqual({ answer: 42 });
+    });
+  });
+
+  describe("task timeout (#686)", () => {
+    it("times out with non-retryable LLM_TIMEOUT when config.timeout is breached", async () => {
+      process.env.OPENAI_API_KEY = "sk-test";
+      mockFetch.mockImplementation((url: string, init?: RequestInit) => {
+        if ((url as string).includes("model-registry")) {
+          return Promise.resolve(new Response("{}", { status: 404 }));
+        }
+        // Hang until the SDK's timeout signal aborts the request.
+        return new Promise((_, reject) => {
+          init?.signal?.addEventListener("abort", () =>
+            reject(Object.assign(new Error("Request timed out."), { name: "APIConnectionTimeoutError" })),
+          );
+        });
+      });
+
+      await expect(
+        callLlmAction({ model: "gpt-4o-mini", prompt: "hi", timeout: 1 }, {}, "exec-1"),
+      ).rejects.toMatchObject({ type: "LLM_TIMEOUT", nonRetryable: true });
+    }, 15_000);
+  });
+
   describe("direct mode — Anthropic", () => {
     it("calls Anthropic API and returns structured result", async () => {
       process.env.ANTHROPIC_API_KEY = "sk-ant-test";

package/src/activities/call-http.ts

@@ -101,10 +101,27 @@ export async function callHttpAction(
     fetchOptions.redirect = "manual";
   }
 
+  // Honor the author's timeout_seconds (#686). Non-retryable by design:
+  // the workflow author declared the budget, so a breach is a task failure
+  // they can catch with try/catch — Temporal re-running the same slow call
+  // up to 5 times would multiply the wait without changing the outcome.
+  const timeoutMs = resolved.timeout_seconds ? resolved.timeout_seconds * 1000 : undefined;
+  if (timeoutMs !== undefined) {
+    fetchOptions.signal = AbortSignal.timeout(timeoutMs);
+  }
+
   let response: Response;
   try {
     response = await fetch(uri, fetchOptions);
   } catch (err: unknown) {
+    if (timeoutMs !== undefined && err instanceof Error &&
+        (err.name === "TimeoutError" || err.name === "AbortError")) {
+      throw ApplicationFailure.nonRetryable(
+        `HTTP ${method} ${uri} timed out after ${resolved.timeout_seconds}s (task timeout_seconds)`,
+        "HTTP_CALL_TIMEOUT",
+        { timeoutSeconds: resolved.timeout_seconds },
+      );
+    }
     throw new Error(
       `HTTP ${method} ${uri} failed: ${err instanceof Error ? err.message : String(err)}`,
     );

package/src/activities/call-llm.ts

@@ -31,6 +31,7 @@ import {
 import { computeLlmCostMicros, ensureLoaded as ensurePricingLoaded } from "../shared/model-pricing.js";
 import { resolveToApiModelId } from "../shared/model-registry.js";
 import { buildChatModel } from "../shared/model-client.js";
+import { getRunnerSecret } from "../shared/runner-credential-store.js";
 import { checkDirectCredentials } from "../shared/llm-backend.js";
 import { classifyModelCallError } from "../shared/model-error.js";
 
@@ -41,7 +42,25 @@ export interface LlmCallConfig {
   readonly response_schema?: Record<string, unknown>;
   readonly temperature?: number;
   readonly max_tokens?: number;
+  /**
+   * Author-declared call budget in seconds (proto: LlmCallTaskConfig.timeout,
+   * 1-600). Bounds the provider request via buildChatModel's timeout seam;
+   * a breach fails non-retryably with LLM_TIMEOUT (#686). The engine widens
+   * the activity's startToClose to fit values above the default 5m.
+   */
   readonly timeout?: number;
+  /**
+   * Schema-validation policy (proto: LlmCallTaskConfig.on_invalid). The
+   * retry/fallback ORCHESTRATION lives in the workflow engine
+   * (call-function.ts, mirroring call-agent.ts); this activity only reads
+   * it to pick the failure channel: soft policies get a `parse_error`
+   * result the engine can act on, the default throws LLM_SCHEMA_VALIDATION.
+   */
+  readonly on_invalid?: string;
+  /** Engine-owned (see on_invalid); accepted here so config passes through. */
+  readonly max_retries?: number;
+  /** Engine-owned (see on_invalid); accepted here so config passes through. */
+  readonly fallback_task?: string;
 }
 
 export interface LlmCallResult {
@@ -142,6 +161,18 @@ function classifyAndThrowLlmError(
   );
 }
 
+/**
+ * Matches the request-timeout errors the provider SDKs raise when
+ * buildChatModel's timeout bound fires (OpenAI: APIConnectionTimeoutError
+ * "Request timed out."; Anthropic mirrors the shape). Only consulted when
+ * the task declared an explicit `timeout`, so the loose message match
+ * cannot reclassify errors on unbudgeted calls.
+ */
+function isRequestTimeoutError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false;
+  return err.name === "APIConnectionTimeoutError" || /timed?\s*out/i.test(err.message);
+}
+
 export async function callLlmAction(
   config: LlmCallConfig,
   runtimeEnv: Record<string, unknown>,
@@ -162,7 +193,10 @@ export async function callLlmAction(
   await ensurePricingLoaded().catch(() => {});
 
   const proxyEndpoint = process.env.STIGMER_PROXY_ENDPOINT;
-  const stigmerToken = process.env.STIGMER_TOKEN;
+  // Per-call store read: rotation (manager updateToken / static renewal)
+  // must be visible to in-flight workers, exactly like the env read it
+  // replaced (#508).
+  const stigmerToken = getRunnerSecret("STIGMER_TOKEN");
   const proxyActive = !!(proxyEndpoint && stigmerToken);
 
   console.log(
@@ -185,6 +219,12 @@ export async function callLlmAction(
   // Anthropic requires an explicit maxTokens; preserve the 4096 default here
   // (buildChatModel intentionally imposes none). resolvedModel is already an
   // API id, so buildChatModel's resolve step is a no-op for it.
+  //
+  // maxRetries: 0 — Temporal owns retries for this activity (callProxy
+  // retries up to 5x). LangChain's default retry loop underneath would
+  // multiply that, and it also blind-retries schema-validation failures:
+  // before this was pinned to 0, one non-conforming structured response
+  // burned ~7 identical model calls before surfacing (#686).
   const { model } = await buildChatModel({
     modelName: resolvedModel,
     proxyEndpoint: proxyActive ? proxyEndpoint : undefined,
@@ -192,6 +232,10 @@ export async function callLlmAction(
     headerScope: { workflowExecutionId: executionId },
     temperature: config.temperature,
     maxTokens: provider === "anthropic" ? (config.max_tokens ?? 4096) : config.max_tokens,
+    // The author's per-call budget rides the same seam as the operator's
+    // STIGMER_LLM_REQUEST_TIMEOUT_MS (#468); an explicit value wins there.
+    timeoutMs: config.timeout ? config.timeout * 1000 : undefined,
+    maxRetries: 0,
   });
 
   const messages: (HumanMessage | SystemMessage)[] = [];
@@ -200,6 +244,14 @@ export async function callLlmAction(
   }
   messages.push(new HumanMessage(config.prompt));
 
+  // Soft schema-failure channel: when the engine will orchestrate
+  // ON_INVALID_RETRY / ON_INVALID_FALLBACK, a validation miss is a signal
+  // (parse_error result), not a failure — the engine re-prompts or
+  // branches. ON_INVALID_FAIL (and unset) keeps the throwing contract.
+  const softSchemaFailure =
+    config.on_invalid === "ON_INVALID_RETRY" ||
+    config.on_invalid === "ON_INVALID_FALLBACK";
+
   let result: LlmCallResult;
 
   try {
@@ -231,7 +283,31 @@ export async function callLlmAction(
       };
     }
   } catch (err) {
-    classifyAndThrowLlmError(err, modelId, provider, proxyActive);
+    if (softSchemaFailure && err instanceof z.ZodError) {
+      // Usage is unrecoverable here — the structured runnable throws
+      // before exposing the raw response (same loss as the throwing path).
+      result = {
+        input_tokens: 0,
+        output_tokens: 0,
+        result: undefined,
+        model: modelId,
+        provider,
+        parse_error: err.errors
+          .map((e) => `${e.path.join(".")}: ${e.message}`)
+          .join("; "),
+      };
+    } else if (config.timeout && isRequestTimeoutError(err)) {
+      // The author declared this budget in the task config; breaching it is
+      // a task failure they can catch, not a transient to retry — Temporal
+      // re-running the same over-budget call 5x would multiply the wait.
+      throw ApplicationFailure.nonRetryable(
+        `LLM call for model "${modelId}" timed out after ${config.timeout}s (task timeout)`,
+        "LLM_TIMEOUT",
+        { timeoutSeconds: config.timeout },
+      );
+    } else {
+      classifyAndThrowLlmError(err, modelId, provider, proxyActive);
+    }
   }
 
   const costMicros = computeLlmCostMicros(config.model, result.input_tokens, result.output_tokens);

package/src/activities/execute-cursor/__tests__/agent-session-cache.test.ts

@@ -0,0 +1,220 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import {
+  cacheSessionAgent,
+  closeAllCachedAgents,
+  computeAgentFingerprint,
+  evictSessionAgent,
+  takeCachedAgent,
+  _parkedAgentCountForTests,
+  _resetAgentSessionCacheForTests,
+  type CacheableAgent,
+} from "../agent-session-cache.js";
+
+function makeAgent(agentId = "agent-1"): CacheableAgent & { close: ReturnType<typeof vi.fn> } {
+  return { agentId, close: vi.fn() };
+}
+
+const FP = "fingerprint-a";
+const OTHER_FP = "fingerprint-b";
+
+describe("agent-session-cache", () => {
+  beforeEach(() => {
+    _resetAgentSessionCacheForTests();
+    delete process.env.STIGMER_CURSOR_AGENT_CACHE_TTL_MS;
+  });
+
+  afterEach(() => {
+    _resetAgentSessionCacheForTests();
+    vi.useRealTimers();
+  });
+
+  describe("checkout semantics", () => {
+    it("park then take returns the same agent without closing it", () => {
+      const agent = makeAgent();
+      cacheSessionAgent("ses-1", agent, FP);
+
+      const taken = takeCachedAgent("ses-1", FP, "agent-1");
+
+      expect(taken).toBe(agent);
+      expect(agent.close).not.toHaveBeenCalled();
+    });
+
+    it("checkout is exclusive — a second take misses", () => {
+      cacheSessionAgent("ses-1", makeAgent(), FP);
+
+      expect(takeCachedAgent("ses-1", FP, "agent-1")).toBeDefined();
+      expect(takeCachedAgent("ses-1", FP, "agent-1")).toBeUndefined();
+    });
+
+    it("an empty expectedAgentId (first-execution shape) still matches", () => {
+      const agent = makeAgent();
+      cacheSessionAgent("ses-1", agent, FP);
+
+      expect(takeCachedAgent("ses-1", FP, "")).toBe(agent);
+    });
+
+    it("misses for a different session", () => {
+      cacheSessionAgent("ses-1", makeAgent(), FP);
+
+      expect(takeCachedAgent("ses-2", FP, "agent-1")).toBeUndefined();
+      expect(_parkedAgentCountForTests()).toBe(1);
+    });
+  });
+
+  describe("reuse guards", () => {
+    it("fingerprint drift closes the parked agent and misses", () => {
+      const agent = makeAgent();
+      cacheSessionAgent("ses-1", agent, FP);
+
+      const taken = takeCachedAgent("ses-1", OTHER_FP, "agent-1");
+
+      expect(taken).toBeUndefined();
+      expect(agent.close).toHaveBeenCalledTimes(1);
+      expect(_parkedAgentCountForTests()).toBe(0);
+    });
+
+    it("agentId mismatch (harnessStateId replaced elsewhere) closes and misses", () => {
+      const agent = makeAgent("agent-old");
+      cacheSessionAgent("ses-1", agent, FP);
+
+      const taken = takeCachedAgent("ses-1", FP, "agent-new");
+
+      expect(taken).toBeUndefined();
+      expect(agent.close).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe("lifetime bounds", () => {
+    it("re-parking a session closes the displaced agent", () => {
+      const first = makeAgent("agent-1");
+      const second = makeAgent("agent-2");
+      cacheSessionAgent("ses-1", first, FP);
+      cacheSessionAgent("ses-1", second, FP);
+
+      expect(first.close).toHaveBeenCalledTimes(1);
+      expect(takeCachedAgent("ses-1", FP, "agent-2")).toBe(second);
+    });
+
+    it("idle TTL evicts and closes", () => {
+      vi.useFakeTimers();
+      process.env.STIGMER_CURSOR_AGENT_CACHE_TTL_MS = "1000";
+      const agent = makeAgent();
+      cacheSessionAgent("ses-1", agent, FP);
+
+      vi.advanceTimersByTime(1001);
+
+      expect(agent.close).toHaveBeenCalledTimes(1);
+      expect(_parkedAgentCountForTests()).toBe(0);
+    });
+
+    it("checkout before the TTL fires disarms the eviction timer", () => {
+      vi.useFakeTimers();
+      process.env.STIGMER_CURSOR_AGENT_CACHE_TTL_MS = "1000";
+      const agent = makeAgent();
+      cacheSessionAgent("ses-1", agent, FP);
+
+      const taken = takeCachedAgent("ses-1", FP, "agent-1");
+      vi.advanceTimersByTime(5000);
+
+      expect(taken).toBe(agent);
+      expect(agent.close).not.toHaveBeenCalled();
+    });
+
+    it("the LRU cap closes the oldest parked agent", () => {
+      vi.useFakeTimers();
+      const first = makeAgent("agent-0");
+      cacheSessionAgent("ses-0", first, FP);
+      for (let i = 1; i < 32; i++) {
+        vi.advanceTimersByTime(1);
+        cacheSessionAgent(`ses-${i}`, makeAgent(`agent-${i}`), FP);
+      }
+
+      vi.advanceTimersByTime(1);
+      cacheSessionAgent("ses-32", makeAgent("agent-32"), FP);
+
+      expect(first.close).toHaveBeenCalledTimes(1);
+      expect(_parkedAgentCountForTests()).toBe(32);
+    });
+
+    it("an empty sessionId is never parked — the lease releases immediately", () => {
+      const agent = makeAgent();
+      cacheSessionAgent("", agent, FP);
+
+      expect(agent.close).toHaveBeenCalledTimes(1);
+      expect(_parkedAgentCountForTests()).toBe(0);
+    });
+  });
+
+  describe("explicit release", () => {
+    it("evictSessionAgent closes the parked agent", () => {
+      const agent = makeAgent();
+      cacheSessionAgent("ses-1", agent, FP);
+
+      evictSessionAgent("ses-1");
+
+      expect(agent.close).toHaveBeenCalledTimes(1);
+      expect(_parkedAgentCountForTests()).toBe(0);
+    });
+
+    it("closeAllCachedAgents closes everything (worker shutdown)", () => {
+      const a = makeAgent("agent-a");
+      const b = makeAgent("agent-b");
+      cacheSessionAgent("ses-a", a, FP);
+      cacheSessionAgent("ses-b", b, FP);
+
+      closeAllCachedAgents();
+
+      expect(a.close).toHaveBeenCalledTimes(1);
+      expect(b.close).toHaveBeenCalledTimes(1);
+      expect(_parkedAgentCountForTests()).toBe(0);
+    });
+
+    it("a close() that throws never breaks eviction", () => {
+      const agent = makeAgent();
+      agent.close.mockImplementation(() => {
+        throw new Error("already disposed");
+      });
+      cacheSessionAgent("ses-1", agent, FP);
+
+      expect(() => evictSessionAgent("ses-1")).not.toThrow();
+      expect(_parkedAgentCountForTests()).toBe(0);
+    });
+  });
+
+  describe("computeAgentFingerprint", () => {
+    const baseOptions = {
+      apiKey: "sk-secret-key",
+      model: "gpt-5",
+      modelParams: [{ key: "tier", value: "standard" }],
+      workspaceDirs: ["/workspace/app"],
+      mcpServers: { github: { command: "npx", env: { TOKEN: "t-1" } } },
+    };
+
+    it("is stable across key ordering", () => {
+      const reordered = {
+        mcpServers: { github: { env: { TOKEN: "t-1" }, command: "npx" } },
+        workspaceDirs: ["/workspace/app"],
+        modelParams: [{ key: "tier", value: "standard" }],
+        model: "gpt-5",
+        apiKey: "sk-secret-key",
+      };
+      expect(computeAgentFingerprint(baseOptions)).toBe(computeAgentFingerprint(reordered));
+    });
+
+    it("changes when any acquisition input changes", () => {
+      const base = computeAgentFingerprint(baseOptions);
+      expect(computeAgentFingerprint({ ...baseOptions, apiKey: "sk-rotated" })).not.toBe(base);
+      expect(computeAgentFingerprint({ ...baseOptions, model: "gpt-5-mini" })).not.toBe(base);
+      expect(
+        computeAgentFingerprint({
+          ...baseOptions,
+          mcpServers: { github: { command: "npx", env: { TOKEN: "t-2" } } },
+        }),
+      ).not.toBe(base);
+    });
+
+    it("never embeds the raw api key", () => {
+      expect(computeAgentFingerprint(baseOptions)).not.toContain("sk-secret-key");
+    });
+  });
+});

package/src/activities/execute-cursor/__tests__/service-tier.test.ts

@@ -19,10 +19,10 @@ vi.mock("@cursor/sdk", () => ({
 }));
 
 import {
-  resolveEffectiveServiceTier,
   resolveServiceTierParams,
   resetCatalogCacheForTests,
 } from "../service-tier.js";
+import { resolveEffectiveServiceTier } from "../../../shared/service-tier.js";
 
 const CATALOG = [
   {