@stigmer/runner 3.9.0 → 3.11.0

package/src/activities/__tests__/discover-mcp-server.test.ts

@@ -10,6 +10,14 @@ vi.mock("../../idle-watchdog.js", () => ({
   activityFinished: vi.fn(),
 }));
 
+// The factory starts a Temporal heartbeat loop; outside an activity context
+// its ticks would throw, so replace it with a stop-spy and assert the
+// start/stop contract instead.
+const mockHeartbeatStop = vi.fn();
+vi.mock("../../shared/heartbeat.js", () => ({
+  startHeartbeat: vi.fn(() => ({ stop: mockHeartbeatStop, cancelled: false, workerShutdown: false })),
+}));
+
 const mockInitializeConnections = vi.fn();
 const mockGetClient = vi.fn();
 const mockClose = vi.fn().mockResolvedValue(undefined);
@@ -600,6 +608,122 @@ describe("DiscoverMcpServer activity", () => {
       expect(result.newToolsFingerprint).toBe("");
     });
 
+    it("fails closed when the EC read errors and the server requires credentials", async () => {
+      // Issue #239: limping ahead with an empty env either died later as an
+      // opaque PlaceholderResolutionError or dialed the endpoint with a
+      // garbage credential and wedged in the 4xx → SSE-fallback limbo. The
+      // failure must name the root cause: credential delivery.
+      const { discoverMcpServer, CredentialResolutionError } = await import("../discover-mcp-server.js");
+
+      const spec = makeHttpSpec("https://mcp.monday.com/mcp");
+      spec.env = { MONDAY_ACCESS_TOKEN: { isSecret: true } };
+      const mockClient = makeMockStigmerClient({
+        mcpServer: makeMcpServer({ metadata: { slug: "monday" }, spec }),
+      });
+      mockClient.getExecutionContextByExecutionId = vi
+        .fn()
+        .mockRejectedValue(new Error("PERMISSION_DENIED: not scope-bound"));
+
+      await expect(
+        discoverMcpServer(
+          { mcpServerId: "mcp-monday", executionContextId: "ctx-1" },
+          { stigmerClient: mockClient as any, transportPosture: "stdio-forbidden" },
+        ),
+      ).rejects.toThrow(CredentialResolutionError);
+      expect(mockInitializeConnections).not.toHaveBeenCalled();
+    });
+
+    it("keeps the lenient path when the EC read errors but no env is declared", async () => {
+      const { discoverMcpServer } = await import("../discover-mcp-server.js");
+
+      const mockClient = makeMockStigmerClient({
+        mcpServer: makeMcpServer({
+          metadata: { slug: "no-env" },
+          spec: makeHttpSpec("https://mcp.example.com"),
+        }),
+      });
+      mockClient.getExecutionContextByExecutionId = vi
+        .fn()
+        .mockRejectedValue(new Error("transient"));
+
+      mockInitializeConnections.mockResolvedValue({});
+      mockGetClient.mockResolvedValue(makeMockMcpClient({ tools: [] }));
+
+      const result = await discoverMcpServer(
+        { mcpServerId: "mcp-lenient", executionContextId: "ctx-2" },
+        { stigmerClient: mockClient as any, transportPosture: "stdio-forbidden" },
+      );
+      expect(result.tools).toEqual([]);
+    });
+
+    it("keeps the lenient path when all declared env vars are optional", async () => {
+      // Caller-identity-style servers declare only optional keys; discovery
+      // legitimately proceeds without an EC for them (the injections below
+      // resolveEnvVarsForDiscovery supply the values).
+      const { discoverMcpServer } = await import("../discover-mcp-server.js");
+
+      const spec = makeHttpSpec("https://mcp.example.com");
+      spec.env = { STIGMER_CALLER_IDENTITY_KIND: { optional: true } };
+      const mockClient = makeMockStigmerClient({
+        mcpServer: makeMcpServer({ metadata: { slug: "optional-only" }, spec }),
+      });
+      mockClient.getExecutionContextByExecutionId = vi
+        .fn()
+        .mockRejectedValue(new Error("transient"));
+
+      mockInitializeConnections.mockResolvedValue({});
+      mockGetClient.mockResolvedValue(makeMockMcpClient({ tools: [] }));
+
+      const result = await discoverMcpServer(
+        { mcpServerId: "mcp-optional", executionContextId: "ctx-3" },
+        { stigmerClient: mockClient as any, transportPosture: "stdio-forbidden" },
+      );
+      expect(result.tools).toEqual([]);
+    });
+
+    it("fails closed when the EC is empty and the server requires credentials", async () => {
+      const { discoverMcpServer, CredentialResolutionError } = await import("../discover-mcp-server.js");
+
+      const spec = makeHttpSpec("https://mcp.monday.com/mcp");
+      spec.env = { MONDAY_ACCESS_TOKEN: { isSecret: true } };
+      const mockClient = makeMockStigmerClient({
+        mcpServer: makeMcpServer({ metadata: { slug: "monday" }, spec }),
+        executionContext: { spec: { data: {} } },
+      });
+
+      const promise = discoverMcpServer(
+        { mcpServerId: "mcp-monday", executionContextId: "ctx-4" },
+        { stigmerClient: mockClient as any, transportPosture: "stdio-forbidden" },
+      );
+      await expect(promise).rejects.toThrow(CredentialResolutionError);
+      await expect(promise).rejects.toThrow(/delivered no credentials/);
+      expect(mockInitializeConnections).not.toHaveBeenCalled();
+    });
+
+    it("fails closed when a delivered credential is the redaction sentinel", async () => {
+      // A redacted value means the server-side decrypt gate refused this
+      // runner's credential class/scope — dialing with the literal sentinel
+      // can only produce a misleading 401 (the issue-#239 M2 mechanism).
+      const { discoverMcpServer, CredentialResolutionError } = await import("../discover-mcp-server.js");
+
+      const spec = makeHttpSpec("https://mcp.monday.com/mcp");
+      spec.env = { MONDAY_ACCESS_TOKEN: { isSecret: true } };
+      const mockClient = makeMockStigmerClient({
+        mcpServer: makeMcpServer({ metadata: { slug: "monday" }, spec }),
+        executionContext: {
+          spec: { data: { MONDAY_ACCESS_TOKEN: { value: "***REDACTED***", isSecret: true } } },
+        },
+      });
+
+      const promise = discoverMcpServer(
+        { mcpServerId: "mcp-monday", executionContextId: "ctx-5" },
+        { stigmerClient: mockClient as any, transportPosture: "stdio-forbidden" },
+      );
+      await expect(promise).rejects.toThrow(CredentialResolutionError);
+      await expect(promise).rejects.toThrow(/MONDAY_ACCESS_TOKEN.*delivered redacted|delivered redacted.*MONDAY_ACCESS_TOKEN/s);
+      expect(mockInitializeConnections).not.toHaveBeenCalled();
+    });
+
     it("closes MCP client even when discovery fails", async () => {
       const { discoverMcpServer } = await import("../discover-mcp-server.js");
 
@@ -620,6 +744,85 @@ describe("DiscoverMcpServer activity", () => {
     });
   });
 
+  // ─────────────────────────────────────────────────────────────────────────
+  // Transport-aware init bounds (issue #239)
+  // ─────────────────────────────────────────────────────────────────────────
+
+  describe("transport-aware init bounds", () => {
+    it("bounds HTTP endpoints at 30s and stdio at 270s", async () => {
+      // HTTP has no cold-start excuse: a healthy endpoint completes the MCP
+      // handshake in seconds, so a short bound converts the silent-SSE hang
+      // into a fast actionable failure. stdio keeps the cold-start allowance
+      // (issue #243). The OSS server's connect-workflow budget (connectTimeout,
+      // controller/connect.go — pinned there) is derived from the stdio bound
+      // + the classification floor, so both bounds stay reachable under it.
+      const { initTimeoutMsFor } = await import("../discover-mcp-server.js");
+      expect(initTimeoutMsFor("http")).toBe(30_000);
+      expect(initTimeoutMsFor("sse")).toBe(30_000);
+      expect(initTimeoutMsFor("stdio")).toBe(270_000);
+    });
+
+    it("names the endpoint URL in the HTTP timeout message", async () => {
+      const { initTimeoutMessageFor } = await import("../discover-mcp-server.js");
+      const message = initTimeoutMessageFor("monday", {
+        slug: "monday",
+        connectionType: "http",
+        url: "https://mcp.monday.com/mcp",
+        toolApprovals: [],
+        pinnedToolApprovals: [],
+        toolApprovalOverrides: [],
+        discoveredCapabilitiesEmpty: true,
+      });
+      expect(message).toContain("https://mcp.monday.com/mcp");
+      expect(message).toContain("30s");
+      expect(message).toContain("SSE fallback");
+    });
+
+    it("names the command in the stdio timeout message", async () => {
+      const { initTimeoutMessageFor } = await import("../discover-mcp-server.js");
+      const message = initTimeoutMessageFor("filesystem", {
+        slug: "filesystem",
+        connectionType: "stdio",
+        command: "npx",
+        toolApprovals: [],
+        pinnedToolApprovals: [],
+        toolApprovalOverrides: [],
+        discoveredCapabilitiesEmpty: true,
+      });
+      expect(message).toContain("npx");
+      expect(message).toContain("270s");
+      expect(message).toContain("cold start");
+    });
+  });
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // Heartbeat contract (issue #239)
+  // ─────────────────────────────────────────────────────────────────────────
+
+  describe("heartbeat contract", () => {
+    it("starts a heartbeat for the activity and stops it on completion", async () => {
+      // The connect workflow proxies this activity with a 60s heartbeatTimeout.
+      // Without a heartbeat loop, any discovery slower than 60s was killed by
+      // Temporal with an opaque heartbeat timeout — never reaching the
+      // actionable init-timeout errors (issues #239/#243).
+      const { createDiscoverMcpServerActivities } = await import("../discover-mcp-server.js");
+      const { startHeartbeat } = await import("../../shared/heartbeat.js");
+
+      vi.mocked(startHeartbeat).mockClear();
+      mockHeartbeatStop.mockClear();
+
+      const { DiscoverMcpServerCapabilities } = createDiscoverMcpServerActivities(makeConfig());
+      try {
+        await DiscoverMcpServerCapabilities({ mcpServerId: "test" });
+      } catch {
+        // Expected — no real backend behind the factory's own client.
+      }
+
+      expect(startHeartbeat).toHaveBeenCalledTimes(1);
+      expect(mockHeartbeatStop).toHaveBeenCalledTimes(1);
+    });
+  });
+
   // ─────────────────────────────────────────────────────────────────────────
   // Idle Watchdog Integration
   // ─────────────────────────────────────────────────────────────────────────

package/src/activities/__tests__/workflow-event-activities.test.ts

@@ -7,11 +7,15 @@ import type { WorkflowEventDescriptor } from "../../workflow-engine/types.js";
 
 const mockGetEventLogHighWaterMark = vi.fn<(executionId: string) => Promise<bigint>>();
 const mockGetWorkflowExecution = vi.fn<(executionId: string) => Promise<unknown>>();
+const mockUpdateStatus = vi.fn<(input: unknown) => Promise<unknown>>();
 
 vi.mock("../../client/stigmer-client.js", () => ({
   StigmerClient: vi.fn().mockImplementation(() => ({
     getEventLogHighWaterMark: (...args: unknown[]) => mockGetEventLogHighWaterMark(...(args as [string])),
     getWorkflowExecution: (...args: unknown[]) => mockGetWorkflowExecution(...(args as [string])),
+    workflowExecutionCommand: {
+      updateStatus: (...args: unknown[]) => mockUpdateStatus(args[0]),
+    },
   })),
 }));
 
@@ -29,8 +33,9 @@ describe("initSequenceFromEventLog", () => {
     mockGetEventLogHighWaterMark.mockReset();
   });
 
-  it("sets counter to 0 when executionId is empty", async () => {
-    await initSequenceFromEventLog("");
+  it("returns 0 and resets the legacy counter when executionId is empty", async () => {
+    const highWaterMark = await initSequenceFromEventLog("");
+    expect(highWaterMark).toBe(0);
 
     const evt = toProtoEvent({
       type: "task_started",
@@ -43,10 +48,11 @@ describe("initSequenceFromEventLog", () => {
     expect(mockGetEventLogHighWaterMark).not.toHaveBeenCalled();
   });
 
-  it("sets counter to 0 when server returns no events", async () => {
+  it("returns 0 when server has no events", async () => {
     mockGetEventLogHighWaterMark.mockResolvedValue(BigInt(0));
 
-    await initSequenceFromEventLog("wfx_test-1");
+    const highWaterMark = await initSequenceFromEventLog("wfx_test-1");
+    expect(highWaterMark).toBe(0);
 
     const evt = toProtoEvent({
       type: "task_started",
@@ -58,7 +64,15 @@ describe("initSequenceFromEventLog", () => {
     expect(evt.sequenceNumber).toBe(BigInt(1));
   });
 
-  it("continues from high-water mark when events exist", async () => {
+  it("returns the high-water mark as a plain number (Temporal payload converter cannot carry BigInt)", async () => {
+    mockGetEventLogHighWaterMark.mockResolvedValue(BigInt(42));
+
+    const highWaterMark = await initSequenceFromEventLog("wfx_recovery-1");
+    expect(highWaterMark).toBe(42);
+    expect(typeof highWaterMark).toBe("number");
+  });
+
+  it("seeds the legacy counter so pre-patch replays continue from N+1", async () => {
     mockGetEventLogHighWaterMark.mockResolvedValue(BigInt(42));
 
     await initSequenceFromEventLog("wfx_recovery-1");
@@ -87,7 +101,8 @@ describe("initSequenceFromEventLog", () => {
   it("handles large sequence numbers", async () => {
     mockGetEventLogHighWaterMark.mockResolvedValue(BigInt(999999));
 
-    await initSequenceFromEventLog("wfx_large-seq");
+    const highWaterMark = await initSequenceFromEventLog("wfx_large-seq");
+    expect(highWaterMark).toBe(999999);
 
     const evt = toProtoEvent({
       type: "task_started",
@@ -112,7 +127,52 @@ describe("toProtoEvent", () => {
   });
 
   describe("sequence numbering", () => {
-    it("assigns monotonically increasing sequence numbers", () => {
+    it("honors the workflow-assigned sequenceNumber when stamped on the descriptor", () => {
+      const evt = toProtoEvent({
+        type: "task_started",
+        taskName: "t1",
+        occurredAt: NOW,
+        taskKind: "set",
+        attemptNumber: 1,
+        sequenceNumber: 77,
+      });
+
+      expect(evt.sequenceNumber).toBe(BigInt(77));
+    });
+
+    it("workflow-assigned sequences are stable across conversions (retry idempotency)", () => {
+      const desc: WorkflowEventDescriptor = {
+        type: "task_started",
+        taskName: "t1",
+        occurredAt: NOW,
+        taskKind: "set",
+        attemptNumber: 1,
+        sequenceNumber: 5,
+      };
+
+      // Simulates an activity retry re-converting the same descriptors:
+      // the sequence must not change between attempts.
+      expect(toProtoEvent(desc).sequenceNumber).toBe(BigInt(5));
+      expect(toProtoEvent(desc).sequenceNumber).toBe(BigInt(5));
+    });
+
+    it("a stamped descriptor does not consume the legacy counter", () => {
+      const unstamped: WorkflowEventDescriptor = {
+        type: "task_started",
+        taskName: "t",
+        occurredAt: NOW,
+        taskKind: "set",
+        attemptNumber: 1,
+      };
+
+      const e1 = toProtoEvent(unstamped);
+      toProtoEvent({ ...unstamped, sequenceNumber: 99 });
+      const e2 = toProtoEvent(unstamped);
+
+      expect(e2.sequenceNumber).toBe(e1.sequenceNumber + BigInt(1));
+    });
+
+    it("legacy path: assigns monotonically increasing sequence numbers when unstamped", () => {
       const desc: WorkflowEventDescriptor = {
         type: "task_started",
         taskName: "t1",
@@ -130,7 +190,7 @@ describe("toProtoEvent", () => {
       expect(e3.sequenceNumber).toBe(BigInt(3));
     });
 
-    it("resets to 1 after re-initializing with empty executionId", async () => {
+    it("legacy path: resets to 1 after re-initializing with empty executionId", async () => {
       toProtoEvent({
         type: "task_started",
         taskName: "t",
@@ -654,8 +714,13 @@ describe("toProtoEvent", () => {
 });
 
 describe("emitWorkflowEvents", () => {
+  beforeEach(() => {
+    mockUpdateStatus.mockReset();
+  });
+
   it("returns silently for empty events array", async () => {
     await expect(emitWorkflowEvents("exec-1", [])).resolves.toBeUndefined();
+    expect(mockUpdateStatus).not.toHaveBeenCalled();
   });
 
   it("returns silently for empty executionId", async () => {
@@ -667,6 +732,40 @@ describe("emitWorkflowEvents", () => {
       attemptNumber: 1,
     }];
     await expect(emitWorkflowEvents("", events)).resolves.toBeUndefined();
+    expect(mockUpdateStatus).not.toHaveBeenCalled();
+  });
+
+  it("sends workflow-assigned sequence numbers through to the RPC", async () => {
+    mockUpdateStatus.mockResolvedValue({});
+
+    await emitWorkflowEvents("exec-1", [{
+      type: "task_started",
+      taskName: "t",
+      occurredAt: NOW,
+      taskKind: "set",
+      attemptNumber: 1,
+      sequenceNumber: 12,
+    }]);
+
+    expect(mockUpdateStatus).toHaveBeenCalledTimes(1);
+    const input = mockUpdateStatus.mock.calls[0][0] as { events: { sequenceNumber: bigint }[] };
+    expect(input.events).toHaveLength(1);
+    expect(input.events[0].sequenceNumber).toBe(BigInt(12));
+  });
+
+  it("propagates RPC errors so the local activity retry policy fires", async () => {
+    mockUpdateStatus.mockRejectedValue(new Error("server unavailable"));
+
+    const events: WorkflowEventDescriptor[] = [{
+      type: "task_started",
+      taskName: "t",
+      occurredAt: NOW,
+      taskKind: "set",
+      attemptNumber: 1,
+      sequenceNumber: 1,
+    }];
+
+    await expect(emitWorkflowEvents("exec-1", events)).rejects.toThrow("server unavailable");
   });
 });
 

package/src/activities/call-llm.ts

@@ -31,6 +31,7 @@ import {
 import { computeLlmCostMicros, ensureLoaded as ensurePricingLoaded } from "../shared/model-pricing.js";
 import { resolveToApiModelId } from "../shared/model-registry.js";
 import { buildChatModel } from "../shared/model-client.js";
+import { checkDirectCredentials } from "../shared/llm-backend.js";
 import { classifyModelCallError } from "../shared/model-error.js";
 
 export interface LlmCallConfig {
@@ -169,23 +170,15 @@ export async function callLlmAction(
     `structured=${!!config.response_schema} execution=${executionId}`,
   );
 
-  // Direct mode (no proxy) requires the provider's own API key. Validated here
-  // rather than in buildChatModel so the shared module stays free of Temporal
-  // failure types.
+  // Direct mode (no proxy) requires a usable credential path — the provider's
+  // own API key, or a configured model backend that authenticates itself
+  // (vertex uses Google ADC; requiring an Anthropic key there would reject a
+  // correctly-configured deployment). Validated here rather than in
+  // buildChatModel so the shared module stays free of Temporal failure types.
   if (!proxyActive) {
-    if (provider === "openai" && !process.env.OPENAI_API_KEY) {
-      throw ApplicationFailure.nonRetryable(
-        `OPENAI_API_KEY is not set and no proxy is configured. ` +
-        `Set the API key in your environment or connect to a Stigmer Cloud deployment.`,
-        "LLM_MISSING_API_KEY",
-      );
-    }
-    if (provider === "anthropic" && !process.env.ANTHROPIC_API_KEY) {
-      throw ApplicationFailure.nonRetryable(
-        `ANTHROPIC_API_KEY is not set and no proxy is configured. ` +
-        `Set the API key in your environment or connect to a Stigmer Cloud deployment.`,
-        "LLM_MISSING_API_KEY",
-      );
+    const missing = checkDirectCredentials(provider);
+    if (missing !== null) {
+      throw ApplicationFailure.nonRetryable(missing, "LLM_MISSING_API_KEY");
     }
   }
 

package/src/activities/classify-tool-approvals.ts

@@ -23,6 +23,8 @@ import { SystemMessage, HumanMessage } from "@langchain/core/messages";
 import { activityStarted, activityFinished } from "../idle-watchdog.js";
 import { getSummarizationModel } from "../shared/model-registry.js";
 import { buildChatModel } from "../shared/model-client.js";
+import { checkDirectCredentials } from "../shared/llm-backend.js";
+import { tryInferProvider } from "../shared/llm-proxy.js";
 import type { Config } from "../config.js";
 
 const BATCH_SIZE = 40;
@@ -149,7 +151,8 @@ Output one classification per tool, maintaining the input order.`;
 // ─────────────────────────────────────────────────────────────────────────────
 
 export interface ClassifyToolsOptions {
-  proxyEndpoint: string;
+  /** Null when the deployment has no proxy — the runner calls providers directly. */
+  proxyEndpoint: string | null;
   stigmerToken: string | null;
   primaryModel: string;
 }
@@ -166,6 +169,26 @@ export async function classifyTools(
 
   const model = await getSummarizationModel(options.primaryModel);
 
+  // Direct mode with no credential path fails closed up front, with one
+  // actionable message instead of a provider 401 per batch. The message
+  // names the economy model and its provider because classification's
+  // provider follows the economy-model resolution, not the operator's key:
+  // "OPENAI_API_KEY is missing" is baffling to an operator holding an
+  // Anthropic key unless the log says which model classification wanted.
+  if (!options.proxyEndpoint) {
+    const provider = tryInferProvider(model);
+    const missing = provider === null ? null : checkDirectCredentials(provider);
+    if (missing !== null) {
+      console.warn(
+        `[ClassifyToolApprovals] Cannot classify ${tools.length} tool(s) for ` +
+        `'${serverName}': the classifier model '${model}' (${provider}) has no ` +
+        `credential path — failing closed, every tool requires approval until ` +
+        `a reconnect re-classifies it. ${missing}`,
+      );
+      return fallbackApprovals(tools);
+    }
+  }
+
   const batches: ToolDescriptor[][] = [];
   for (let i = 0; i < tools.length; i += BATCH_SIZE) {
     batches.push(tools.slice(i, i + BATCH_SIZE));
@@ -236,7 +259,7 @@ interface ClassifyBatchParams {
   serverName: string;
   serverDescription: string;
   model: string;
-  proxyEndpoint: string;
+  proxyEndpoint: string | null;
   stigmerToken: string | null;
   mcpServerId: string | null;
   batchIdx: number;
@@ -256,7 +279,7 @@ async function classifyBatch(params: ClassifyBatchParams): Promise<ToolApprovalR
   // primary this routes to Claude, not the hardcoded OpenAI path it used to.
   const { model: llm } = await buildChatModel({
     modelName: model,
-    proxyEndpoint,
+    proxyEndpoint: proxyEndpoint ?? undefined,
     stigmerToken: stigmerToken ?? undefined,
     headerScope: { mcpServerId: mcpServerId ?? undefined },
     maxTokens,
@@ -388,8 +411,15 @@ export function createClassifyToolApprovalsActivities(config: Config) {
           `[ClassifyToolApprovals] Activity started: ${input.tools.length} tools for '${input.serverName}'`,
         );
 
+        // Only a real proxy is an LLM proxy. stigmerBackendEndpoint (the
+        // gRPC control plane) serves no /v1/proxy/llm route — coercing it
+        // into one here used to 404 every classification in unproxied OSS
+        // and silently disable the vertex backend for this activity. (The
+        // registry fetch legitimately falls back to the backend endpoint —
+        // see shared/registry-endpoint.ts — because the control plane DOES
+        // serve /v1/proxy/model-registry; that fallback does not port here.)
         return await classifyTools(input, {
-          proxyEndpoint: config.proxyEndpoint ?? config.stigmerBackendEndpoint,
+          proxyEndpoint: config.proxyEndpoint,
           stigmerToken: config.stigmerToken,
           primaryModel: config.primaryModel,
         });