@desplega.ai/agent-swarm 2.5.4 → 2.7.0

package/src/utils/internal-ai/credentials.ts

@@ -25,6 +25,7 @@ import { registerBunOAuthFlows } from "@earendil-works/pi-ai/bun-oauth";
 import { getEnvApiKey } from "@earendil-works/pi-ai/compat";
 import { builtinModels } from "@earendil-works/pi-ai/providers/all";
 import { getValidCodexOAuth, persistCodexOAuth } from "../../providers/codex-oauth/storage.js";
+import { selectCredential } from "../credentials.js";
 import { type CredentialKind, DEFAULT_MODEL, resolveRaterModelString } from "./models.js";
 
 registerBunOAuthFlows();
@@ -135,7 +136,7 @@ export async function resolveCredential(
   if (openrouterKey) {
     return {
       kind: "openrouter",
-      apiKey: openrouterKey,
+      apiKey: selectCredential(openrouterKey, undefined, "OPENROUTER_API_KEY").selected,
       modelDefault: resolveRaterModelString("openrouter"),
     };
   }
@@ -151,7 +152,7 @@ export async function resolveCredential(
   if (anthropicKey) {
     return {
       kind: "anthropic",
-      apiKey: anthropicKey,
+      apiKey: selectCredential(anthropicKey, undefined, "ANTHROPIC_API_KEY").selected,
       modelDefault: resolveRaterModelString("anthropic"),
     };
   }
@@ -161,7 +162,7 @@ export async function resolveCredential(
   if (openaiKey) {
     return {
       kind: "openai",
-      apiKey: openaiKey,
+      apiKey: selectCredential(openaiKey, undefined, "OPENAI_API_KEY").selected,
       modelDefault: resolveRaterModelString("openai"),
     };
   }

package/src/utils/provider-key-probes.ts

@@ -0,0 +1,328 @@
+/** Shared credential probes. This module does not import worker harnesses. */
+import {
+  type OpenAICompatibleProvider,
+  providerBaseUrl,
+  providerForKeyVar,
+} from "@desplega/model-catalog";
+import {
+  createScopedFetch,
+  deriveDefaultRoute,
+  getRouteProvider,
+  type ModelRoute,
+} from "@desplega/model-routing";
+import { assertUrlSafe, publicEndpointSsrfOptions } from "../oauth/mcp-wrapper";
+import { getOpenRouterAttributionHeaders, getOpenRouterBaseUrl } from "./openrouter-base-url";
+import { scrubSecrets } from "./secret-scrubber";
+
+interface ProbeOptions {
+  env?: Record<string, string | undefined>;
+  publicEndpoint?: boolean;
+}
+
+const LIVE_TEST_TIMEOUT_MS = 5_000;
+
+function scrubProbeBody(body: string, init: RequestInit): string {
+  const headers = new Headers(init.headers);
+  const apiKey =
+    headers.get("x-api-key") ?? headers.get("authorization")?.replace(/^Bearer\s+/i, "");
+  return scrubSecrets(apiKey ? body.replaceAll(apiKey, "[REDACTED]") : body);
+}
+
+export interface LiveKeyResult {
+  keyType: string;
+  index: number;
+  suffix: string;
+  ok: boolean;
+  error?: string;
+  httpStatus?: number;
+  latency_ms: number;
+}
+
+export interface LiveValidationResult {
+  ok: boolean;
+  error?: string;
+  httpStatus?: number;
+  warning?: string;
+  keys?: LiveKeyResult[];
+  latency_ms: number;
+  /** No free live check exists for this credential (cloud routes): report presence only. */
+  skipped?: boolean;
+}
+
+export async function timedFetch(
+  url: string,
+  init: RequestInit,
+  options: ProbeOptions = {},
+): Promise<{ ok: boolean; status: number; bodyText: string; latency_ms: number }> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), LIVE_TEST_TIMEOUT_MS);
+  const start = Date.now();
+  try {
+    if (options.publicEndpoint) assertUrlSafe(url, publicEndpointSsrfOptions());
+    const res = await fetch(url, {
+      ...init,
+      ...(options.publicEndpoint ? { redirect: "manual" } : {}),
+      signal: controller.signal,
+    });
+    let bodyText = "";
+    try {
+      bodyText = scrubProbeBody(await res.text(), init);
+    } catch {
+      // The HTTP status is sufficient when the response body cannot be read.
+    }
+    return {
+      ok: res.ok,
+      status: res.status,
+      bodyText,
+      latency_ms: Date.now() - start,
+    };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export async function checkAnthropicApiKey(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(
+    "https://api.anthropic.com/v1/models",
+    {
+      method: "GET",
+      headers: { "x-api-key": apiKey, "anthropic-version": "2023-06-01" },
+    },
+    options,
+  );
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+export async function checkOpenAiApiKey(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(
+    "https://api.openai.com/v1/models",
+    {
+      method: "GET",
+      headers: { Authorization: `Bearer ${apiKey}` },
+    },
+    options,
+  );
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+/** `GET /v1/models` is the call the Grok CLI makes first; it runs no inference. */
+export async function checkXaiApiKey(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(
+    "https://api.x.ai/v1/models",
+    {
+      method: "GET",
+      headers: { Authorization: `Bearer ${apiKey}` },
+    },
+    options,
+  );
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+/** Fetch a key-authenticated GET and normalize it into a `LiveValidationResult`. */
+async function checkBearerGet(
+  url: string,
+  init: RequestInit,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(url, { method: "GET", ...init }, options);
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+/**
+ * OpenRouter's `/models` is public and answers 200 for any key, so the check
+ * reads `GET /key` (the key's own info, 401 for a bad key). A custom
+ * `OPENROUTER_BASE_URL` without `/key` (404/405) falls back to `/models`.
+ */
+export async function checkOpenRouter(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const base = getOpenRouterBaseUrl(options.env);
+  const get = (url: string) =>
+    checkBearerGet(
+      url,
+      {
+        headers: {
+          ...getOpenRouterAttributionHeaders(url, options.env),
+          Authorization: `Bearer ${apiKey}`,
+        },
+      },
+      options,
+    );
+  const keyResult = await get(`${base}/key`);
+  if (keyResult.httpStatus === 404 || keyResult.httpStatus === 405) return get(`${base}/models`);
+  return keyResult;
+}
+
+/** `GET /models` on the official DeepSeek API: lists models, runs no inference. */
+export function checkDeepSeekApiKey(
+  apiKey: string,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  return checkBearerGet(
+    "https://api.deepseek.com/models",
+    {
+      headers: { Authorization: `Bearer ${apiKey}` },
+    },
+    options,
+  );
+}
+
+/**
+ * `GET <baseUrl>/models` with a Bearer key against an OpenAI-compatible
+ * registry provider (NaN, Helmcode). Lists models, runs no inference.
+ * `baseUrl` defaults to the registry entry; the probe passes the
+ * `<ID>_BASE_URL` override, and tests inject a fake server.
+ */
+export async function checkOpenAICompatibleProvider(
+  provider: OpenAICompatibleProvider,
+  apiKey: string,
+  baseUrl: string = provider.baseUrl,
+  options: ProbeOptions = {},
+): Promise<LiveValidationResult> {
+  const r = await timedFetch(
+    `${baseUrl.replace(/\/+$/, "")}/models`,
+    {
+      method: "GET",
+      headers: { Authorization: `Bearer ${apiKey}` },
+    },
+    options,
+  );
+  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+  return {
+    ok: false,
+    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+    httpStatus: r.status,
+    latency_ms: r.latency_ms,
+  };
+}
+
+/**
+ * Run the route provider's own check. Requests stay on the route's origin and
+ * path prefix, go to public hosts only, and never follow a redirect.
+ */
+async function checkRoute(
+  route: ModelRoute,
+  env: Record<string, string | undefined>,
+): Promise<LiveValidationResult> {
+  const startedAt = Date.now();
+  const provider = getRouteProvider(route.provider);
+  const baseUrl = route.baseUrl ?? provider?.defaultBaseUrl;
+  if (!provider?.validate || !baseUrl) return { ok: true, skipped: true, latency_ms: 0 };
+  const publicFetch = async (url: string | URL | Request, init?: RequestInit) => {
+    assertUrlSafe(String(url), publicEndpointSsrfOptions());
+    const res = await fetch(url, init);
+    if (res.ok || !res.body) return res;
+    return new Response(scrubProbeBody(await res.text(), init ?? {}), {
+      status: res.status,
+      statusText: res.statusText,
+      headers: res.headers,
+    });
+  };
+  const result = await provider.validate({
+    route,
+    env,
+    signal: AbortSignal.timeout(LIVE_TEST_TIMEOUT_MS),
+    fetch: createScopedFetch(baseUrl, publicFetch as typeof fetch),
+  });
+  const latency_ms = Date.now() - startedAt;
+  if (result.status === "verified") return { ok: true, latency_ms };
+  return {
+    ok: result.status === "configured",
+    ...(result.status === "configured" ? { skipped: true } : { httpStatus: result.httpStatus }),
+    error: scrubSecrets(result.reason),
+    latency_ms,
+  };
+}
+
+/** Check one newly submitted pool member without loading a worker harness. */
+export async function checkProviderPoolKey(
+  keyType: string,
+  apiKey: string,
+  env: Record<string, string | undefined> = process.env,
+): Promise<LiveValidationResult> {
+  const startedAt = Date.now();
+  const options: ProbeOptions = { env, publicEndpoint: true };
+  try {
+    switch (keyType) {
+      case "ANTHROPIC_API_KEY": {
+        // Behind a gateway (`ANTHROPIC_BASE_URL`), Claude Code sends this key
+        // to the gateway, so check it there and not at api.anthropic.com.
+        const keyEnv = { ...env, ANTHROPIC_API_KEY: apiKey };
+        const route = deriveDefaultRoute("claude", keyEnv);
+        if (
+          route?.provider === "anthropic-gateway" &&
+          "secretKey" in route.auth &&
+          route.auth.secretKey === "ANTHROPIC_API_KEY"
+        ) {
+          return await checkRoute(route, keyEnv);
+        }
+        return await checkAnthropicApiKey(apiKey, options);
+      }
+      case "OPENAI_API_KEY":
+        return await checkOpenAiApiKey(apiKey, options);
+      case "OPENROUTER_API_KEY":
+        return await checkOpenRouter(apiKey, options);
+      case "DEEPSEEK_API_KEY":
+        return await checkDeepSeekApiKey(apiKey, options);
+      case "XAI_API_KEY":
+        return await checkXaiApiKey(apiKey, options);
+      case "CLAUDE_CODE_OAUTH_TOKEN": {
+        const tokenEnv = { CLAUDE_CODE_OAUTH_TOKEN: apiKey };
+        return await checkRoute(deriveDefaultRoute("claude", tokenEnv)!, tokenEnv);
+      }
+      default: {
+        const provider = providerForKeyVar(keyType);
+        if (provider) {
+          return await checkOpenAICompatibleProvider(
+            provider,
+            apiKey,
+            providerBaseUrl(provider, env),
+            options,
+          );
+        }
+        return { ok: true, skipped: true, latency_ms: 0 };
+      }
+    }
+  } catch (err) {
+    const raw = err instanceof Error ? err.message : String(err);
+    return {
+      ok: false,
+      error: scrubSecrets(raw.split(apiKey).join("[REDACTED]")) || "Unknown error",
+      latency_ms: Date.now() - startedAt,
+    };
+  }
+}

package/src/utils/transcription.ts

@@ -20,6 +20,7 @@
  * auto-detected key always goes to its provider's own endpoint. Deepgram and AssemblyAI use other request shapes and are not
  * supported. Read at call time, so a config reload applies without a restart.
  */
+import { selectCredential } from "./credentials";
 import { createValueScrubber, scrubSecrets } from "./secret-scrubber";
 
 const TRANSCRIPTION_TIMEOUT_MS = 60_000;
@@ -119,7 +120,9 @@ export function resolveTranscriptionConfig(): TranscriptionConfig | null {
     : AUTO_DETECT_ORDER;
   for (const name of candidates) {
     const { keyEnv } = PROVIDERS[name];
-    const apiKey = envValue(keyEnv);
+    const value = envValue(keyEnv);
+    // OPENAI_API_KEY may be a comma pool: send one member.
+    const apiKey = value ? selectCredential(value, undefined, keyEnv).selected : undefined;
     if (apiKey) return { provider: name, apiKey, keyEnv, source: "auto" };
   }
   return null;

package/src/workflows/engine.ts

@@ -51,7 +51,7 @@ import { FOREACH_TERMINAL_STEP_STATUSES, resolveForeachParent } from "./foreach-
 import { getSecretInputKeys, redactSecretsForStorage, resolveInputs } from "./input";
 import { validateJsonSchema } from "./json-schema-validator";
 import { getMaxWorkflowStepsPerRun } from "./limits";
-import { resolveNodeTimeoutMs } from "./node-timeout";
+import { resolveNodeTimeoutMs, startStepDeadline } from "./node-timeout";
 import { findWorkflowReadinessProblems, formatReadinessRunError } from "./readiness";
 import { deepInterpolate } from "./template";
 import { runStepValidation, type ValidationRunResult } from "./validation";
@@ -78,6 +78,13 @@ export interface WorkflowExecutionOptions {
    * HTTP trigger records its `Idempotency-Key` here.
    */
   accept?: (run: { runId: string; skipped: boolean }) => Promise<void>;
+  /**
+   * Return once the run is committed and walk it in the background, as a
+   * child run does. By default a top-level start resolves only after the
+   * walk reaches its end or its first async step, so a caller that only
+   * needs the run id waits for every instant step before it.
+   */
+  detach?: boolean;
 }
 
 /**
@@ -209,9 +216,12 @@ export async function startWorkflowExecution(
       options,
     );
     // A child run walks on its own, outside the parent executor's timeout. Its
-    // terminal status wakes the parent step (`resumeFromChildRun`).
-    if (options.parentStepId) {
-      walk.catch((err) => console.error(`[workflows] Child workflow run ${runId} failed:`, err));
+    // terminal status wakes the parent step (`resumeFromChildRun`). A detached
+    // top-level run walks the same way; the walk holds the run before its
+    // first await, so releasing the start hold below leaves no gap.
+    if (options.parentStepId || options.detach) {
+      const kind = options.parentStepId ? "Child" : "Detached";
+      walk.catch((err) => console.error(`[workflows] ${kind} workflow run ${runId} failed:`, err));
       return runId;
     }
     await walk;
@@ -1045,7 +1055,7 @@ async function runClaimedStep(
     inputCtx: interpolationCtx,
   };
 
-  const timeoutMs = resolveNodeTimeoutMs(node);
+  const deadline = startStepDeadline(resolveNodeTimeoutMs(node));
 
   let result: Awaited<ReturnType<typeof executor.run>>;
   try {
@@ -1053,10 +1063,10 @@ async function runClaimedStep(
       executor.run({
         config: interpolatedConfig,
         context: executionCtx,
-        meta,
+        meta: { ...meta, signal: deadline.signal },
       }),
-      timeoutPromise(timeoutMs),
-    ]);
+      deadline.expired,
+    ]).finally(deadline.clear);
   } catch (err) {
     const errorMsg = err instanceof Error ? err.message : String(err);
 
@@ -1459,11 +1469,3 @@ export function scriptBodyInterpolationError(nodeId: string, tokens: string[]):
     "Pass dynamic trigger or upstream values through config.args and read them from argv."
   );
 }
-
-// ─── Helpers ───────────────────────────────────────────────
-
-function timeoutPromise(ms: number): Promise<never> {
-  return new Promise((_, reject) => {
-    setTimeout(() => reject(new Error(`Step timed out after ${ms}ms`)), ms);
-  });
-}

package/src/workflows/executors/sub-workflow.ts

@@ -133,7 +133,14 @@ export class SubWorkflowExecutor extends BaseExecutor<
       throw err;
     }
     const child = await db.getWorkflowRun(childRunId);
-    if (!child) return { status: "failed", error: `Child workflow run ${childRunId} not found` };
+    // The start committed the row, so a missing one was deleted with its workflow.
+    if (!child) {
+      return {
+        status: "failed",
+        error: `Child workflow run ${childRunId} not found`,
+        retryable: false,
+      };
+    }
     return this.settle(child, config.workflowId);
   }
 

package/src/workflows/executors/swarm-script.ts

@@ -123,6 +123,7 @@ export class SwarmScriptExecutor extends BaseExecutor<
       source: resolved.source,
       args: config.args,
       fsMode: "none",
+      executor: resolved.script.executor,
       agentId: agentId ?? "workflow",
       mcpBaseUrl: getMcpBaseUrl(),
       egressSecrets: credentials.egressSecrets,
@@ -132,6 +133,10 @@ export class SwarmScriptExecutor extends BaseExecutor<
       apiConnections: getScriptApiConnectionDescriptors({ agentId: agentId ?? undefined }),
       mcpConnections: getScriptMcpConnectionDescriptors({ agentId: agentId ?? undefined }),
       timeoutMs: config.timeoutMs,
+      // The step watchdog aborts this when the node budget runs out, which
+      // kills the script process: a step reported as timed out must not go
+      // on to perform the script's side effects.
+      signal: meta.signal,
     });
     if (output.exitCode === 0 && !output.error && !output.runtimeError) {
       await touchScratchScriptLastUsed(resolved.script.id);

package/src/workflows/foreach-driver.ts

@@ -47,7 +47,7 @@ import {
   settleForeachChild,
 } from "./foreach-join";
 import { getSecretInputKeys } from "./input";
-import { resolveNodeTimeoutMs } from "./node-timeout";
+import { resolveNodeTimeoutMs, startStepDeadline } from "./node-timeout";
 import { rebuildNodeExecutionInput } from "./step-input";
 
 export { FOREACH_ITEMS_WAIT_FOR };
@@ -480,27 +480,21 @@ async function startChild(input: {
   }
 
   const executor = registry.get(bodyType);
-  const timeoutMs = resolveNodeTimeoutMs({ type: bodyType, config });
-  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = startStepDeadline(resolveNodeTimeoutMs({ type: bodyType, config }));
   let result: Awaited<ReturnType<typeof executor.run>>;
   try {
     result = await Promise.race([
       executor.run({
         config,
         context: { ...ctx, ...bodyCtx },
-        meta: { ...meta, stepId: child.id, nodeId: child.nodeId },
-      }),
-      new Promise<never>((_, reject) => {
-        timer = setTimeout(
-          () => reject(new Error(`Step timed out after ${timeoutMs}ms`)),
-          timeoutMs,
-        );
+        meta: { ...meta, stepId: child.id, nodeId: child.nodeId, signal: deadline.signal },
       }),
+      deadline.expired,
     ]);
   } catch (err) {
     return { status: "failed", error: err instanceof Error ? err.message : String(err) };
   } finally {
-    if (timer) clearTimeout(timer);
+    deadline.clear();
   }
 
   if (result.status === "failed") {

package/src/workflows/node-timeout.ts

@@ -19,3 +19,33 @@ export function resolveNodeTimeoutMs(node: Pick<WorkflowNode, "type" | "config">
   if (node.type === "script" && typeof config.timeout === "number") return config.timeout;
   return DEFAULT_NODE_TIMEOUT_MS;
 }
+
+export interface StepDeadline {
+  /** Aborted when the budget runs out. The executor gets it as `meta.signal`. */
+  signal: AbortSignal;
+  /** Rejects with "Step timed out after <ms>ms" when the budget runs out. */
+  expired: Promise<never>;
+  /** Stop the timer once the executor call settled. */
+  clear: () => void;
+}
+
+/**
+ * The watchdog for one executor call. Racing `expired` alone settles the step
+ * but leaves the executor running, so a step reported as timed out could still
+ * perform its side effects. On expiry the deadline also aborts `signal`, so an
+ * executor that forwards it (swarm-script kills its script process) stops.
+ */
+export function startStepDeadline(timeoutMs: number): StepDeadline {
+  const controller = new AbortController();
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const expired = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      const error = new Error(`Step timed out after ${timeoutMs}ms`);
+      // Reject first so the race settles on the timeout, not on the
+      // executor's "killed" result that the abort produces.
+      reject(error);
+      controller.abort(error);
+    }, timeoutMs);
+  });
+  return { signal: controller.signal, expired, clear: () => clearTimeout(timer) };
+}

package/templates/ai-toolbox.manifest.json

@@ -1,5 +1,5 @@
 {
-  "commit": "5a577022800d9b43ccec5376bfe16c7b24eecd91",
+  "commit": "c81f2bb62f49b45603e9e702558e1ca22e98fcdd",
   "excludedSkills": [
     "feedback"
   ],
@@ -44,7 +44,7 @@
     "templates/skills/questioning/content.md": "66f826a191e96d0041adca00a7c197a458de8e6fe164c1599b8708098d4109ab",
     "templates/skills/questioning/files/template.md": "a68d72801a0b10380bf773b07e9d2f8d4c5b4b1c609ce954289e38ffc78cc223",
     "templates/skills/researching/config.json": "921c062cab599097cc7677d2dad4797ec4d5844d2ee73607fbd417ec3fa487d4",
-    "templates/skills/researching/content.md": "92951142327415696726a2577defdb2379cc611cdafa9cdac92431a5e60f618f",
+    "templates/skills/researching/content.md": "87ced42d15f1fd63d1b4e8d5ed8161537202f5b05a85df38dea273e9e3d6ad08",
     "templates/skills/researching/files/template.md": "411d34b98c610ef715edf0f24d09f828ba2d0f65f688e7879ef33a9453007949",
     "templates/skills/reviewing/config.json": "09c40bb098e9738c60a848995fa9fe7568d158631a089c76ba02c0dd955445c7",
     "templates/skills/reviewing/content.md": "1bbc285c9ad8cc3312f1ee2c558b04ffee914ffb6f5c63fb3741fcbb3e300fa5",