@stigmer/runner 3.0.1

package/src/shared/connect-backfill.ts

@@ -0,0 +1,162 @@
+/**
+ * Connect backfill for MCP servers without discovered capabilities.
+ *
+ * When an MCP server has empty discovered_capabilities, triggers the
+ * connect RPC to discover tools and classify approval policies via
+ * the LLM classifier. The connect RPC starts a Temporal workflow
+ * (discover + classify) and blocks until completion (~30 seconds).
+ *
+ * Backfill trigger: discovered_capabilities is empty or absent.
+ *
+ * Non-fatal: if connect fails for any server (permissions, timeout,
+ * unreachable), the original servers are kept and execution continues
+ * with empty policies. All tools from that server will require
+ * approval by default (fail-closed).
+ *
+ * This module is harness-agnostic — both ExecuteCursor and
+ * ExecuteDeepAgent use the same backfill logic. Each harness maps
+ * the returned ResolvedMcpServer[] into its SDK-specific format.
+ */
+
+import type { ResolvedMcpServer } from "./mcp-resolver.js";
+import { resolveMcpServers } from "./mcp-resolver.js";
+import type { StigmerClient } from "../client/stigmer-client.js";
+import type { McpServerUsage } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/spec_pb";
+
+const CONNECT_TIMEOUT_MS = 60_000;
+
+/**
+ * When true, skip connect backfill entirely. The runner discovers
+ * tools directly via stdio/HTTP when it connects MCP servers, so
+ * the backfill (which routes through the Java service's Temporal
+ * workflow) is only needed for LLM-based approval classification.
+ * Offline tests set this because no worker polls the stigmer_runner
+ * queue that the Connect workflow dispatches to.
+ */
+const SKIP_BACKFILL = process.env.SKIP_MCP_CONNECT_BACKFILL === "true";
+
+/**
+ * Check whether an MCP server needs a connect backfill.
+ *
+ * Returns true if the server has never been discovered (empty
+ * capabilities). Exported for testing.
+ */
+export function needsBackfill(server: ResolvedMcpServer): boolean {
+  if (SKIP_BACKFILL) return false;
+  return server.discoveredCapabilitiesEmpty;
+}
+
+/**
+ * Run connect backfill for MCP servers that need it, then re-resolve
+ * to pick up the newly populated approval policies.
+ *
+ * Triggers the connect RPC synchronously for each server with empty
+ * discovered_capabilities. The RPC starts a Temporal workflow
+ * (discover + classify) and returns the updated McpServer.
+ *
+ * After at least one successful backfill, re-resolves ALL servers
+ * to pick up fresh policies. If no backfills succeed, returns the
+ * original servers unchanged.
+ */
+export async function backfillMcpServersIfNeeded(
+  client: StigmerClient,
+  currentServers: ResolvedMcpServer[],
+  usages: McpServerUsage[],
+  envVars: Record<string, string>,
+  org: string,
+  onHeartbeat?: () => void,
+  secretKeys?: ReadonlySet<string>,
+): Promise<ResolvedMcpServer[]> {
+  const serversNeedingBackfill = currentServers.filter(needsBackfill);
+
+  if (serversNeedingBackfill.length === 0) {
+    return currentServers;
+  }
+
+  console.log(
+    `[connect-backfill] Backfill needed for ${serversNeedingBackfill.length} MCP server(s): ` +
+    serversNeedingBackfill.map((s) => s.slug).join(", "),
+  );
+
+  let anyBackfilled = false;
+
+  for (const server of serversNeedingBackfill) {
+    try {
+      const serverRef = usages.find((u) => u.mcpServerRef?.slug === server.slug);
+      if (!serverRef?.mcpServerRef) continue;
+
+      const fullServer = await client.getMcpServerByReference(serverRef.mcpServerRef);
+      const serverId = fullServer.metadata?.id;
+      if (!serverId) continue;
+
+      const runtimeEnv = extractRuntimeEnvForServer(fullServer, envVars, secretKeys);
+
+      console.log(
+        `[connect-backfill] Triggering connect for "${server.slug}" (${serverId})`,
+      );
+      onHeartbeat?.();
+
+      const updated = await Promise.race([
+        client.connectMcpServer(serverId, org, runtimeEnv),
+        new Promise<never>((_, reject) =>
+          setTimeout(
+            () => reject(new Error(
+              `Connect timed out after ${CONNECT_TIMEOUT_MS / 1000}s`,
+            )),
+            CONNECT_TIMEOUT_MS,
+          ),
+        ),
+      ]);
+
+      const toolCount = updated.status?.discoveredCapabilities?.tools.length ?? 0;
+      const approvalCount = updated.status?.toolApprovals?.length ?? 0;
+      console.log(
+        `[connect-backfill] "${server.slug}" — ` +
+        `discovered ${toolCount} tool(s), classified ${approvalCount} approval policy(ies)`,
+      );
+      anyBackfilled = true;
+      onHeartbeat?.();
+    } catch (err) {
+      console.warn(
+        `[connect-backfill] Failed for "${server.slug}": ` +
+        `${err instanceof Error ? err.message : err}. ` +
+        `Continuing with empty approval policies for this server.`,
+      );
+    }
+  }
+
+  if (!anyBackfilled) {
+    return currentServers;
+  }
+
+  const refreshed = await resolveMcpServers(client, usages, envVars);
+  return refreshed.resolvedServers;
+}
+
+/**
+ * Extract the MCP server's required env vars from the execution
+ * environment. Returns only the keys declared in spec.env that are
+ * present in the merged environment.
+ *
+ * isSecret is derived from the MCP server's env declaration first,
+ * then from the execution-level secretKeys set, defaulting to false.
+ */
+export function extractRuntimeEnvForServer(
+  server: { spec?: { env?: Record<string, unknown> } },
+  mergedEnv: Record<string, string>,
+  secretKeys?: ReadonlySet<string>,
+): Record<string, { value: string; isSecret: boolean }> | undefined {
+  const envDecls = server.spec?.env;
+  if (!envDecls || Object.keys(envDecls).length === 0) return undefined;
+
+  const runtime: Record<string, { value: string; isSecret: boolean }> = {};
+  for (const key of Object.keys(envDecls)) {
+    if (key in mergedEnv) {
+      const decl = envDecls[key] as { isSecret?: boolean } | undefined;
+      const isSecret = decl?.isSecret ?? secretKeys?.has(key) ?? false;
+      runtime[key] = { value: mergedEnv[key], isSecret };
+    }
+  }
+
+  return Object.keys(runtime).length > 0 ? runtime : undefined;
+}

package/src/shared/extract-json.ts

@@ -0,0 +1,153 @@
+/**
+ * Extract JSON from agent text responses.
+ *
+ * Three-tier extraction with end-first scanning:
+ *
+ *   Tier 1:    JSON.parse on the full trimmed text.
+ *   Tier 1.5:  Code-fence extraction — scan fences last-to-first,
+ *              only attempt parse when content starts with { or [.
+ *   Tier 1.75: Last-brace extraction — find the last balanced {...}
+ *              by scanning backwards from the end of the text.
+ *
+ * End-first scanning aligns with the prompt injection contract that
+ * tells the agent to put JSON as its "final response." When multiple
+ * JSON fragments exist (e.g. intermediate debug output followed by
+ * the actual result), the last fragment is overwhelmingly the correct
+ * one. If the heuristic picks wrong, downstream schema validation
+ * catches it and the caller falls through to LLM-based extraction.
+ *
+ * Trailing-comma repair (stripTrailingCommas) is applied before every
+ * JSON.parse attempt across all tiers.
+ */
+
+/**
+ * Extract JSON from agent text. Returns the parsed value on success,
+ * or undefined so the caller can fall through to LLM-based extraction.
+ */
+export function extractJsonFromText(text: string): unknown | undefined {
+  if (!text) return undefined;
+
+  const trimmed = text.trim();
+
+  // Tier 1: full text is valid JSON
+  const direct = tryParse(trimmed);
+  if (direct !== undefined) return direct;
+
+  // Tier 1.5: JSON inside a code fence
+  const fenced = extractFromCodeFences(trimmed);
+  if (fenced !== undefined) return fenced;
+
+  // Tier 1.75: last balanced {...} in the text
+  const braced = extractLastJsonObject(trimmed);
+  if (braced !== undefined) return braced;
+
+  return undefined;
+}
+
+// ---------------------------------------------------------------------------
+// Tier 1.5 — code-fence extraction (last-to-first)
+// ---------------------------------------------------------------------------
+
+const CODE_FENCE_RE = /```(?:json|JSON)?\s*\n([\s\S]*?)```/g;
+
+function extractFromCodeFences(text: string): unknown | undefined {
+  const fences: string[] = [];
+  let match: RegExpExecArray | null;
+  while ((match = CODE_FENCE_RE.exec(text)) !== null) {
+    fences.push(match[1]);
+  }
+  CODE_FENCE_RE.lastIndex = 0;
+
+  // Scan last-to-first — the final fence is most likely the structured output
+  for (let i = fences.length - 1; i >= 0; i--) {
+    const content = fences[i].trim();
+    if (!content.startsWith("{") && !content.startsWith("[")) continue;
+    const result = tryParse(content);
+    if (result !== undefined) return result;
+  }
+
+  return undefined;
+}
+
+// ---------------------------------------------------------------------------
+// Tier 1.75 — last balanced brace extraction
+// ---------------------------------------------------------------------------
+
+function extractLastJsonObject(text: string): unknown | undefined {
+  const lastClose = text.lastIndexOf("}");
+  if (lastClose === -1) return undefined;
+
+  // Walk backwards from lastClose to find the matching open brace,
+  // respecting nesting and string literals.
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+
+  for (let i = lastClose; i >= 0; i--) {
+    const ch = text[i];
+
+    if (inString) {
+      if (escaped) {
+        escaped = false;
+        continue;
+      }
+      if (ch === "\\") {
+        escaped = true;
+        continue;
+      }
+      if (ch === '"') {
+        inString = false;
+      }
+      continue;
+    }
+
+    if (ch === '"') {
+      inString = true;
+      continue;
+    }
+
+    if (ch === "}") {
+      depth++;
+    } else if (ch === "{") {
+      depth--;
+      if (depth === 0) {
+        const candidate = text.slice(i, lastClose + 1);
+        return tryParse(candidate);
+      }
+    }
+  }
+
+  return undefined;
+}
+
+// ---------------------------------------------------------------------------
+// Shared helpers
+// ---------------------------------------------------------------------------
+
+function tryParse(candidate: string): unknown | undefined {
+  try {
+    return JSON.parse(candidate);
+  } catch {
+    // fall through
+  }
+
+  const repaired = stripTrailingCommas(candidate);
+  if (repaired !== candidate) {
+    try {
+      return JSON.parse(repaired);
+    } catch {
+      // repair didn't help
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Remove trailing commas before } and ] — a common LLM quirk that
+ * produces invalid JSON. Only removes commas that are followed by
+ * optional whitespace and a closing delimiter.
+ */
+function stripTrailingCommas(json: string): string {
+  return json.replace(/,\s*([}\]])/g, "$1");
+}

package/src/shared/grpc-retry.ts

@@ -0,0 +1,113 @@
+/**
+ * Exponential-backoff retry wrapper for gRPC status persistence.
+ *
+ * Wraps the raw StigmerClient.updateStatus() call with retry logic
+ * that classifies gRPC error codes as retryable vs terminal. A failed
+ * status update must never crash the streaming loop — on permanent
+ * failure, the signal falls back to UNSPECIFIED.
+ *
+ * Used by the ExecuteDeepAgent streaming loop. The simpler
+ * fire-and-forget persistStatus in shared/status.ts is preserved
+ * for ExecuteCursor and non-streaming callers.
+ */
+
+import { ConnectError, Code } from "@connectrpc/connect";
+import { ExecutionControlSignal } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import type { StigmerClient } from "../client/stigmer-client.js";
+
+export interface RetryOptions {
+  /** Base delay before the first retry (ms). Default: 100. */
+  readonly baseDelayMs?: number;
+  /** Backoff multiplier between retries. Default: 2. */
+  readonly backoffFactor?: number;
+  /** Maximum number of retry attempts. Default: 3. */
+  readonly maxRetries?: number;
+  /** Injectable delay function for testing. */
+  readonly delayFn?: (ms: number) => Promise<void>;
+}
+
+const RETRYABLE_CODES = new Set<Code>([
+  Code.Unavailable,
+  Code.DeadlineExceeded,
+]);
+
+const TERMINAL_CODES = new Set<Code>([
+  Code.InvalidArgument,
+  Code.NotFound,
+  Code.PermissionDenied,
+]);
+
+export function isRetryableError(err: unknown): boolean {
+  if (err instanceof ConnectError) {
+    return RETRYABLE_CODES.has(err.code);
+  }
+  return false;
+}
+
+export function isTerminalError(err: unknown): boolean {
+  if (err instanceof ConnectError) {
+    return TERMINAL_CODES.has(err.code);
+  }
+  return false;
+}
+
+function defaultDelay(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Persist execution status with exponential-backoff retry.
+ *
+ * Returns the ExecutionControlSignal from the server on success.
+ * On permanent or exhausted-retry failure, logs the error and
+ * returns UNSPECIFIED (never throws).
+ */
+export async function persistWithRetry(
+  client: StigmerClient,
+  executionId: string,
+  status: AgentExecutionStatus,
+  options?: RetryOptions,
+): Promise<ExecutionControlSignal> {
+  const baseDelay = options?.baseDelayMs ?? 100;
+  const factor = options?.backoffFactor ?? 2;
+  const maxRetries = options?.maxRetries ?? 3;
+  const delay = options?.delayFn ?? defaultDelay;
+
+  let lastError: unknown;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const response = await client.updateStatus(executionId, status);
+      return response.signal;
+    } catch (err: unknown) {
+      lastError = err;
+
+      if (isTerminalError(err)) {
+        const code = (err as ConnectError).code;
+        console.error(
+          `[grpc-retry] Terminal error persisting status for ${executionId}: ` +
+          `code=${Code[code]} (attempt ${attempt + 1}/${maxRetries + 1})`,
+        );
+        return ExecutionControlSignal.UNSPECIFIED;
+      }
+
+      if (!isRetryableError(err) || attempt === maxRetries) {
+        break;
+      }
+
+      const delayMs = baseDelay * Math.pow(factor, attempt);
+      console.warn(
+        `[grpc-retry] Retryable error for ${executionId}: ` +
+        `code=${err instanceof ConnectError ? Code[err.code] : "unknown"} ` +
+        `(attempt ${attempt + 1}/${maxRetries + 1}, retry in ${delayMs}ms)`,
+      );
+      await delay(delayMs);
+    }
+  }
+
+  console.error(
+    `[grpc-retry] All retries exhausted for ${executionId}: ${lastError}`,
+  );
+  return ExecutionControlSignal.UNSPECIFIED;
+}

package/src/shared/heartbeat.ts

@@ -0,0 +1,70 @@
+/**
+ * Activity heartbeat utility.
+ *
+ * Provides a periodic heartbeat loop that activities can start for
+ * long-running operations. The loop sends heartbeat details at a
+ * configurable interval and stops when cancelled or when the
+ * returned abort function is called.
+ *
+ * Heartbeats serve two purposes:
+ * 1. Prevent Temporal from timing out the activity (heartbeatTimeout)
+ * 2. Detect parent workflow cancellation (CancelledFailure from heartbeat)
+ */
+
+import { Context, CancelledFailure } from "@temporalio/activity";
+
+export interface HeartbeatHandle {
+  stop(): void;
+  readonly cancelled: boolean;
+  readonly workerShutdown: boolean;
+}
+
+export interface HeartbeatOptions {
+  shutdownSignal?: AbortSignal;
+}
+
+export function startHeartbeat(
+  intervalMs: number,
+  getDetails?: () => Record<string, unknown>,
+  options?: HeartbeatOptions,
+): HeartbeatHandle {
+  let stopped = false;
+  let wasCancelled = false;
+  let wasWorkerShutdown = false;
+
+  const timer = setInterval(() => {
+    if (stopped) return;
+    try {
+      Context.current().heartbeat(getDetails?.());
+    } catch (err) {
+      if (err instanceof CancelledFailure) {
+        if (options?.shutdownSignal?.aborted) {
+          wasWorkerShutdown = true;
+        } else {
+          wasCancelled = true;
+        }
+        stopped = true;
+        clearInterval(timer);
+      }
+    }
+  }, intervalMs);
+
+  return {
+    stop() {
+      stopped = true;
+      clearInterval(timer);
+    },
+    get cancelled() {
+      return wasCancelled;
+    },
+    get workerShutdown() {
+      return wasWorkerShutdown;
+    },
+  };
+}
+
+export function checkCancellation(): void {
+  if (Context.current().cancellationSignal.aborted) {
+    throw new CancelledFailure("Activity cancelled by workflow");
+  }
+}

package/src/shared/json-schema-to-zod.ts

@@ -0,0 +1,53 @@
+/**
+ * Convert a JSON Schema object to a Zod schema.
+ *
+ * Used by multiple activity layers (call:llm, ExecuteCursor extraction,
+ * ExecuteDeepAgent responseFormat) to produce Zod schemas for
+ * `withStructuredOutput()` / function-calling APIs.
+ *
+ * Handles the subset of JSON Schema used by workflow output schemas:
+ * object types with required/optional fields, string/number/boolean/array
+ * primitives, enum constraints, and null types.
+ */
+
+import { z } from "zod";
+
+export function jsonSchemaToZod(schema: Record<string, unknown>): z.ZodType {
+  const type = schema.type as string | undefined;
+
+  if (type === "object") {
+    const properties = schema.properties as Record<string, Record<string, unknown>> | undefined;
+    const required = new Set(schema.required as string[] | undefined ?? []);
+
+    if (!properties) return z.object({}).passthrough();
+
+    const shape: Record<string, z.ZodType> = {};
+    for (const [key, propSchema] of Object.entries(properties)) {
+      let fieldType = jsonSchemaToZod(propSchema);
+      if (!required.has(key)) {
+        fieldType = fieldType.nullable();
+      }
+      shape[key] = fieldType;
+    }
+    return z.object(shape).passthrough();
+  }
+
+  if (type === "array") {
+    const items = schema.items as Record<string, unknown> | undefined;
+    return z.array(items ? jsonSchemaToZod(items) : z.unknown());
+  }
+
+  if (type === "string") {
+    const enumValues = schema.enum as string[] | undefined;
+    if (enumValues && enumValues.length > 0) {
+      return z.enum(enumValues as [string, ...string[]]);
+    }
+    return z.string();
+  }
+
+  if (type === "number" || type === "integer") return z.number();
+  if (type === "boolean") return z.boolean();
+  if (type === "null") return z.null();
+
+  return z.unknown();
+}

package/src/shared/llm-proxy.ts

@@ -0,0 +1,138 @@
+/**
+ * LLM proxy routing utilities — shared between ExecuteDeepAgent and
+ * ClassifyToolApprovals (and any future LLM-calling activities).
+ *
+ * Responsibilities:
+ * 1. Infer the LLM provider from a model name string (prefix heuristics).
+ * 2. Construct the correct proxy base URL for each provider, matching the
+ *    path pattern expected by stigmer-cloud's LlmProxyController:
+ *      - Anthropic: {proxy}/v1/proxy/llm/anthropic
+ *      - OpenAI:    {proxy}/v1/proxy/llm/openai/v1
+ * 3. Build authorization and scope headers for proxy requests.
+ *
+ * These utilities mirror the Python agent-runner's `LLMConfig.build_llm_kwargs`
+ * but as composable functions rather than a config class method.
+ */
+
+export type LlmProvider = "anthropic" | "openai";
+
+/**
+ * Provider-specific path suffixes appended to `STIGMER_PROXY_ENDPOINT`.
+ *
+ * OpenAI's SDK includes `/v1` in its default base URL (https://api.openai.com/v1),
+ * so the proxy path includes it: `/v1/proxy/llm/openai/v1`.
+ *
+ * Anthropic's SDK does NOT include `/v1` in its base URL (https://api.anthropic.com),
+ * so the proxy path omits it: `/v1/proxy/llm/anthropic`.
+ */
+const PROXY_PATHS: Readonly<Record<LlmProvider, string>> = {
+  anthropic: "/v1/proxy/llm/anthropic",
+  openai: "/v1/proxy/llm/openai/v1",
+};
+
+const OPENAI_PREFIXES = ["gpt", "o1", "o3", "o4", "chatgpt"] as const;
+
+/**
+ * Infer the LLM provider from a model name.
+ *
+ * Resolution order:
+ * 1. Explicit prefix syntax: "provider:model-name" (e.g. "openai:gpt-4.1")
+ * 2. Name prefix heuristics (claude → anthropic, gpt/o1/o3/o4 → openai)
+ *
+ * Throws if the provider cannot be determined — callers should not silently
+ * fall back to a default provider when an unknown model is requested.
+ */
+export function inferProvider(modelName: string): LlmProvider {
+  if (!modelName) {
+    throw new Error("Model name is required but received an empty string");
+  }
+
+  const colonIdx = modelName.indexOf(":");
+  if (colonIdx > 0 && colonIdx < modelName.length - 1) {
+    const prefix = modelName.slice(0, colonIdx).toLowerCase();
+    if (prefix === "anthropic" || prefix === "openai") {
+      return prefix;
+    }
+  }
+
+  const lower = modelName.toLowerCase();
+
+  if (lower.startsWith("claude")) {
+    return "anthropic";
+  }
+
+  for (const prefix of OPENAI_PREFIXES) {
+    if (lower.startsWith(prefix)) {
+      return "openai";
+    }
+  }
+
+  throw new Error(
+    `Cannot determine LLM provider for model "${modelName}". ` +
+    `Known prefixes: claude* (anthropic), gpt*/o1*/o3*/o4*/chatgpt* (openai). ` +
+    `Use explicit prefix syntax "provider:model-name" for non-standard model names.`,
+  );
+}
+
+/**
+ * Strip an explicit "provider:" prefix from the model name, returning
+ * just the model ID portion. If no explicit prefix is present, returns
+ * the original name unchanged.
+ */
+export function stripProviderPrefix(modelName: string): string {
+  const colonIdx = modelName.indexOf(":");
+  if (colonIdx > 0 && colonIdx < modelName.length - 1) {
+    const prefix = modelName.slice(0, colonIdx).toLowerCase();
+    if (prefix === "anthropic" || prefix === "openai") {
+      return modelName.slice(colonIdx + 1);
+    }
+  }
+  return modelName;
+}
+
+/**
+ * Construct the proxy base URL for a given provider.
+ *
+ * The returned URL is what gets passed as `baseURL` / `clientOptions.baseURL`
+ * to the LangChain model constructor. The LangChain SDK appends provider-
+ * specific API paths (e.g. `/v1/messages` for Anthropic, `/chat/completions`
+ * for OpenAI).
+ *
+ * Example:
+ *   resolveProxyBaseUrl("https://api.stigmer.ai", "openai")
+ *   → "https://api.stigmer.ai/v1/proxy/llm/openai/v1"
+ */
+export function resolveProxyBaseUrl(
+  proxyEndpoint: string,
+  provider: LlmProvider,
+): string {
+  const base = proxyEndpoint.replace(/\/+$/, "");
+  return `${base}${PROXY_PATHS[provider]}`;
+}
+
+/**
+ * Build the HTTP headers for proxy-routed LLM requests.
+ *
+ * Always includes Bearer authorization. Optionally includes scope headers
+ * used by the proxy for FGA authorization and billing attribution.
+ */
+export function buildProxyHeaders(
+  token: string,
+  options?: { executionId?: string; mcpServerId?: string; workflowExecutionId?: string },
+): Record<string, string> {
+  const headers: Record<string, string> = {
+    Authorization: `Bearer ${token}`,
+  };
+
+  if (options?.executionId) {
+    headers["X-Stigmer-Execution-Id"] = options.executionId;
+  }
+  if (options?.mcpServerId) {
+    headers["X-Stigmer-Mcp-Server-Id"] = options.mcpServerId;
+  }
+  if (options?.workflowExecutionId) {
+    headers["X-Stigmer-Workflow-Execution-Id"] = options.workflowExecutionId;
+  }
+
+  return headers;
+}