@desplega.ai/agent-swarm 2.5.4 → 2.7.0

package/src/commands/credential-refresh.ts

@@ -1,12 +1,17 @@
+import type { AgentCredStatus } from "../types";
+import { CREDENTIAL_POOL_VARS } from "../utils/credentials";
+import { scrubSecrets } from "../utils/secret-scrubber";
 import {
   buildCredStatusReport,
   isCredCheckDisabled,
+  type LiveKeyResult,
   sendCredStatusReport,
   shouldRefreshBedrockStatus,
 } from "./provider-credentials";
 import type { ApiConfig } from "./runner";
 
 export const CREDENTIAL_RETRY_INTERVAL_MS = 30_000;
+export const KEY_RECHECK_INTERVAL_MS = 10 * 60_000;
 const BEDROCK_REFRESH_INTERVAL_MS = 5 * 60 * 1000;
 
 export interface CredentialRefreshState {
@@ -15,6 +20,64 @@ export interface CredentialRefreshState {
   ready: boolean | null;
   lastRefreshAt: number;
   inFlight: boolean;
+  fingerprint: string | null;
+  rejectedKeys: number;
+}
+
+export function credentialFingerprint(env: Record<string, string | undefined>): string {
+  return Bun.hash(
+    JSON.stringify(
+      [...CREDENTIAL_POOL_VARS, "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL"].map(
+        (key) => env[key] ?? null,
+      ),
+    ),
+  ).toString();
+}
+
+/**
+ * Failed keys worth the periodic recheck. A Claude subscription token is left
+ * out: its check calls an undocumented Anthropic account endpoint, so it runs
+ * only once per token value (boot or change). A failed test with no per-key
+ * results counts as one.
+ */
+export function recheckableFailures(liveTest: AgentCredStatus["liveTest"] | undefined): number {
+  const keys = liveTest?.keys;
+  if (keys?.length) {
+    return keys.filter((key) => !key.ok && key.keyType !== "CLAUDE_CODE_OAUTH_TOKEN").length;
+  }
+  return liveTest?.ok === false ? 1 : 0;
+}
+
+/** Report each pool separately. A telemetry failure must not stop credential checks. */
+export async function reportKeyValidation(api: ApiConfig, keys: LiveKeyResult[]): Promise<void> {
+  for (const keyType of new Set(keys.map((key) => key.keyType))) {
+    try {
+      const response = await fetch(`${api.apiUrl}/api/keys/report-validation`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${api.apiKey}`,
+          "X-Agent-ID": api.agentId,
+        },
+        body: JSON.stringify({
+          keyType,
+          keys: keys
+            .filter((key) => key.keyType === keyType)
+            .map((key) => ({
+              keyIndex: key.index,
+              keySuffix: key.suffix,
+              ok: key.ok,
+              httpStatus: key.httpStatus,
+            })),
+        }),
+        signal: AbortSignal.timeout(5000),
+      });
+      if (!response.ok) throw new Error(`HTTP ${response.status}`);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      console.warn(scrubSecrets(`[credentials] Failed to report key validation: ${message}`));
+    }
+  }
 }
 
 /**
@@ -31,11 +94,17 @@ export async function refreshCredentialStatus(
 ): Promise<void> {
   if (isCredCheckDisabled(env) || state.inFlight) return;
   const providerChanged = state.harnessProvider !== harnessProvider;
+  const fingerprint = credentialFingerprint(env);
+  const fingerprintChanged = state.fingerprint !== fingerprint;
   const retryDue =
     state.ready !== true && now - state.lastRefreshAt >= CREDENTIAL_RETRY_INTERVAL_MS;
+  const keyRecheckDue =
+    state.rejectedKeys > 0 && now - state.lastRefreshAt >= KEY_RECHECK_INTERVAL_MS;
   if (
     !providerChanged &&
+    !fingerprintChanged &&
     !retryDue &&
+    !keyRecheckDue &&
     !shouldRefreshBedrockStatus({
       harnessProvider,
       env,
@@ -61,7 +130,13 @@ export async function refreshCredentialStatus(
       missing: snapshot.missing,
       credStatus: snapshot,
     });
-    if (state.harnessProvider === harnessProvider) state.ready = snapshot.ready;
+    if (state.harnessProvider === harnessProvider) {
+      state.ready = snapshot.ready;
+      state.fingerprint = fingerprint;
+      state.rejectedKeys = recheckableFailures(snapshot.liveTest);
+      const keys = snapshot.liveTest?.keys;
+      if (keys?.length) await reportKeyValidation(api, keys);
+    }
   } finally {
     // Errors propagate to the runner's non-fatal logger, leaving ready unknown
     // so even a lost ready:true write is retried without a restart.

package/src/commands/credential-wait.ts

@@ -12,7 +12,7 @@
  *   3. Re-checks; if ready, returns.
  *   4. Logs a `[boot] waiting for …` line and invokes `onTick(status)` so
  *      callers can report state to the API.
- *   5. Sleeps with exponential backoff (2s → 30s, cap configurable).
+ *   5. Sleeps with exponential backoff (2s to 10s, cap configurable).
  *   6. If `BOOT_MAX_WAIT_SECONDS` is set and exceeded, throws a
  *      `BootMaxWaitExceededError` so the runner can exit with a distinct
  *      code. Default 0 = wait forever.
@@ -96,7 +96,7 @@ function resolveBackoff(
   };
   return {
     initialMs: override?.initialMs ?? parsePositive(env.BOOT_INITIAL_BACKOFF_MS, 2000),
-    maxMs: override?.maxMs ?? parsePositive(env.BOOT_MAX_BACKOFF_MS, 30000),
+    maxMs: override?.maxMs ?? parsePositive(env.BOOT_MAX_BACKOFF_MS, 10000),
     // 0 = wait forever — the runner can override with a finite ceiling per
     // worker if monitoring wants a "config never arrived" signal.
     maxWaitSeconds: override?.maxWaitSeconds ?? parsePositive(env.BOOT_MAX_WAIT_SECONDS, 0),

package/src/commands/onboard/compose-generator.ts

@@ -243,9 +243,11 @@ export function generateCompose(state: OnboardState): string {
     lines.push(`      - "${port}:3000"`);
     lines.push("");
     lines.push("    volumes:");
-    lines.push("      - swarm_logs:/app/logs");
-    lines.push("      - swarm_shared:/app/shared");
-    lines.push(`      - swarm_${svc.sanitizedName}:/app/agent`);
+    // Same targets as docker-compose.example.yml: the worker logs to /logs and
+    // keeps its clones and files under /workspace.
+    lines.push("      - swarm_logs:/logs");
+    lines.push("      - swarm_shared:/workspace/shared");
+    lines.push(`      - swarm_${svc.sanitizedName}:/workspace/personal`);
     if (state.provider === "bedrock" && state.awsProfile) {
       lines.push(engine === "podman" ? AWS_PROFILE_VOLUME_SELINUX : AWS_PROFILE_VOLUME);
     }

package/src/commands/provider-credentials.ts

@@ -19,7 +19,6 @@
  */
 
 import {
-  type OpenAICompatibleProvider,
   openAICompatibleKeyVars,
   openAICompatibleProviders,
   providerBaseUrl,
@@ -45,13 +44,36 @@ import type {
   ProviderName,
   ReasoningEffort,
 } from "../types";
-import { CLAUDE_CREDENTIALS_HINT } from "../utils/credentials";
 import {
-  getOpenRouterAttributionHeaders,
-  getOpenRouterBaseUrl,
-} from "../utils/openrouter-base-url";
+  CLAUDE_CREDENTIALS_HINT,
+  CREDENTIAL_POOL_VARS,
+  splitCredentialPool,
+} from "../utils/credentials";
+import { isEnvFlagEnabled } from "../utils/env-flag";
+import {
+  checkAnthropicApiKey,
+  checkDeepSeekApiKey,
+  checkOpenAICompatibleProvider,
+  checkOpenAiApiKey,
+  checkOpenRouter,
+  checkXaiApiKey,
+  type LiveKeyResult,
+  type LiveValidationResult,
+  timedFetch,
+} from "../utils/provider-key-probes";
 import { scrubSecrets } from "../utils/secret-scrubber";
 
+export {
+  checkAnthropicApiKey,
+  checkDeepSeekApiKey,
+  checkOpenAICompatibleProvider,
+  checkOpenAiApiKey,
+  checkOpenRouter,
+  checkXaiApiKey,
+  type LiveKeyResult,
+  type LiveValidationResult,
+} from "../utils/provider-key-probes";
+
 export type SupportedProvider =
   | "claude"
   | "claude-managed"
@@ -215,46 +237,8 @@ export async function checkProviderCredentials(
 // Mirrors `checkProviderCredentials` but issues a real (cheapest possible)
 // upstream call so the user can flip the `harness` milestone to `verified`.
 //
-// Pure function — no DB writes. Lives here (not on `ProviderAdapter`) because
-// adapters are runtime-loaded by workers, while this dispatcher is API-server
-// safe. All errors run through `scrubSecrets` so we never leak the user's
-// own key shape back through the JSON response.
-
-const LIVE_TEST_TIMEOUT_MS = 5_000;
-
-export interface LiveValidationResult {
-  ok: boolean;
-  error?: string;
-  latency_ms: number;
-  /** No free live check exists for this credential (cloud routes): report presence only. */
-  skipped?: boolean;
-}
-
-async function timedFetch(
-  url: string,
-  init: RequestInit,
-): Promise<{ ok: boolean; status: number; bodyText: string; latency_ms: number }> {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), LIVE_TEST_TIMEOUT_MS);
-  const start = Date.now();
-  try {
-    const res = await fetch(url, { ...init, signal: controller.signal });
-    let bodyText = "";
-    try {
-      bodyText = await res.text();
-    } catch {
-      // ignore body read errors — status alone is enough for ok/not-ok
-    }
-    return {
-      ok: res.ok,
-      status: res.status,
-      bodyText,
-      latency_ms: Date.now() - start,
-    };
-  } finally {
-    clearTimeout(timer);
-  }
-}
+// Worker-only dispatch uses the shared probes in `provider-key-probes.ts`.
+// The API imports those probes directly to avoid loading worker harnesses.
 
 function asLiveError(err: unknown, latency_ms: number): LiveValidationResult {
   const raw = err instanceof Error ? err.message : String(err);
@@ -265,6 +249,69 @@ function asLiveError(err: unknown, latency_ms: number): LiveValidationResult {
   };
 }
 
+/**
+ * Run `check` on every member of a comma pool in parallel. The pool passes
+ * when one key works (the runner skips a rejected key); `warning` names the
+ * keys that failed, and the check fails only when every key fails.
+ */
+export async function checkEachKey(
+  keyType: string,
+  value: string,
+  check: (key: string) => Promise<LiveValidationResult>,
+): Promise<LiveValidationResult> {
+  const members = splitCredentialPool(value);
+  if (members.length === 0) members.push(value.trim());
+  const results = await Promise.all(
+    members.map(async (member) => {
+      const startedAt = Date.now();
+      try {
+        return await check(member);
+      } catch (err) {
+        return asLiveError(err, Date.now() - startedAt);
+      }
+    }),
+  );
+  const latency_ms = Math.max(...results.map((result) => result.latency_ms));
+  if (results.every((result) => result.skipped)) return { ok: true, skipped: true, latency_ms };
+  const keys = results.map(
+    (result, index): LiveKeyResult => ({
+      keyType,
+      index,
+      suffix: members[index]!.slice(-5),
+      ok: result.ok,
+      ...(result.error !== undefined ? { error: result.error } : {}),
+      ...(result.httpStatus !== undefined ? { httpStatus: result.httpStatus } : {}),
+      latency_ms: result.latency_ms,
+    }),
+  );
+  const failed = keys.filter((key) => !key.ok);
+  if (failed.length === 0) return { ok: true, latency_ms, keys };
+  const failures = failed
+    .map(
+      (key) =>
+        `key ending ${key.suffix} (${key.httpStatus !== undefined ? `HTTP ${key.httpStatus}` : (key.error ?? "Unknown error")})`,
+    )
+    .join(", ");
+  if (failed.length < keys.length) {
+    return {
+      ok: true,
+      latency_ms,
+      keys,
+      warning: `${failed.length} of ${keys.length} keys failed: ${failures}.`,
+    };
+  }
+  if (keys.length === 1) {
+    return {
+      ok: false,
+      error: failed[0]!.error,
+      httpStatus: failed[0]!.httpStatus,
+      latency_ms,
+      keys,
+    };
+  }
+  return { ok: false, error: `All ${keys.length} keys failed: ${failures}.`, latency_ms, keys };
+}
+
 // ─── Per-endpoint live-call helpers ──────────────────────────────────────────
 // Each helper accepts the auth material it needs and returns a normalized
 // `LiveValidationResult`. The harness-level dispatcher below picks which
@@ -272,22 +319,10 @@ function asLiveError(err: unknown, latency_ms: number): LiveValidationResult {
 // users no longer see "ANTHROPIC_API_KEY is not set" when their adapter is
 // happily running off `CLAUDE_CODE_OAUTH_TOKEN`.
 
-async function checkAnthropicApiKey(apiKey: string): Promise<LiveValidationResult> {
-  const r = await timedFetch("https://api.anthropic.com/v1/models", {
-    method: "GET",
-    headers: { "x-api-key": apiKey, "anthropic-version": "2023-06-01" },
-  });
-  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
-  return {
-    ok: false,
-    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
-    latency_ms: r.latency_ms,
-  };
-}
-
 /**
- * OAuth credentials (Claude Pro/Max via `claude` CLI login, ChatGPT via Codex
- * OAuth) are treated as a presence check — we don't issue a live upstream call.
+ * Codex OAuth credentials (ChatGPT login) are treated as a presence check — we
+ * don't issue a live upstream call. (Claude subscription tokens get a real
+ * check through the route: see `claudeSubscriptionProvider`.)
  *
  * Why: OAuth flows include their own refresh logic (handled at adapter boot,
  * not here), the OAuth-bearer-with-/v1/models contract isn't a stable public
@@ -299,70 +334,6 @@ function presenceCheckOk(): LiveValidationResult {
   return { ok: true, latency_ms: 0 };
 }
 
-async function checkOpenAiApiKey(apiKey: string): Promise<LiveValidationResult> {
-  const r = await timedFetch("https://api.openai.com/v1/models", {
-    method: "GET",
-    headers: { Authorization: `Bearer ${apiKey}` },
-  });
-  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
-  return {
-    ok: false,
-    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
-    latency_ms: r.latency_ms,
-  };
-}
-
-/** `GET /v1/models` is the call the Grok CLI makes first; it runs no inference. */
-async function checkXaiApiKey(apiKey: string): Promise<LiveValidationResult> {
-  const r = await timedFetch("https://api.x.ai/v1/models", {
-    method: "GET",
-    headers: { Authorization: `Bearer ${apiKey}` },
-  });
-  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
-  return {
-    ok: false,
-    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
-    latency_ms: r.latency_ms,
-  };
-}
-
-async function checkOpenRouter(apiKey: string): Promise<LiveValidationResult> {
-  const url = `${getOpenRouterBaseUrl()}/models`;
-  const r = await timedFetch(url, {
-    method: "GET",
-    headers: { ...getOpenRouterAttributionHeaders(url), Authorization: `Bearer ${apiKey}` },
-  });
-  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
-  return {
-    ok: false,
-    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
-    latency_ms: r.latency_ms,
-  };
-}
-
-/**
- * `GET <baseUrl>/models` with a Bearer key against an OpenAI-compatible
- * registry provider (NaN, Helmcode). Lists models, runs no inference.
- * `baseUrl` defaults to the registry entry; the probe passes the
- * `<ID>_BASE_URL` override, and tests inject a fake server.
- */
-export async function checkOpenAICompatibleProvider(
-  provider: OpenAICompatibleProvider,
-  apiKey: string,
-  baseUrl: string = provider.baseUrl,
-): Promise<LiveValidationResult> {
-  const r = await timedFetch(`${baseUrl.replace(/\/+$/, "")}/models`, {
-    method: "GET",
-    headers: { Authorization: `Bearer ${apiKey}` },
-  });
-  if (r.ok) return { ok: true, latency_ms: r.latency_ms };
-  return {
-    ok: false,
-    error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
-    latency_ms: r.latency_ms,
-  };
-}
-
 /**
  * Mirror of the disk check in `checkCodexCredentials` — true when the worker
  * has a materialised `~/.codex/auth.json`, which is the canonical "logged in"
@@ -402,14 +373,15 @@ function parseCodexOAuthAccess(blob: string | undefined): string | null {
  *
  * | Harness          | Accepted credentials (in resolution order)                              | Endpoint                       |
  * |------------------|-------------------------------------------------------------------------|--------------------------------|
- * | `claude`         | default route: Foundry / Bedrock / Vertex → gateway (`ANTHROPIC_BASE_URL` + `ANTHROPIC_AUTH_TOKEN` or `ANTHROPIC_API_KEY`) → `CLAUDE_CODE_OAUTH_TOKEN` → `ANTHROPIC_API_KEY` | the route's own `/v1/models`; OAuth presence-only; cloud routes skipped |
+ * | `claude`         | default route: Foundry / Bedrock / Vertex → gateway (`ANTHROPIC_BASE_URL` + `ANTHROPIC_AUTH_TOKEN` or `ANTHROPIC_API_KEY`) → `CLAUDE_CODE_OAUTH_TOKEN` → `ANTHROPIC_API_KEY` | the route's own `/v1/models`; OAuth: `/api/oauth/profile` (403 scope = ok); cloud routes skipped |
  * | `claude-managed` | `ANTHROPIC_API_KEY` (managed agents always use API key + managed envs)  | Anthropic `/v1/models`         |
  * | `codex`          | `~/.codex/auth.json` (file) → `CODEX_OAUTH` (env OAuth) → `OPENAI_API_KEY` | OpenAI `/v1/models` (api-key path only) |
- * | `opencode`       | auth.json (file) → `OPENROUTER_API_KEY` → `ANTHROPIC_API_KEY` → `OPENAI_API_KEY` → each registry provider key (`NAN_API_KEY` or `HELMCODE_API_NAN`, `HELMCODE_API_KEY`) | file presence-only, otherwise provider's `/v1/models` |
+ * | `opencode`       | auth.json (file) → `OPENROUTER_API_KEY` → `ANTHROPIC_API_KEY` → `OPENAI_API_KEY` → each registry provider key (`NAN_API_KEY` or `HELMCODE_API_NAN`, `HELMCODE_API_KEY`) | file presence-only, otherwise OpenRouter `/key` or the provider's `/v1/models` |
  * | `pi`             | auth.json (file) → provider API keys, including model-specific keys   | file/model-specific key presence-only, otherwise provider's `/v1/models` |
  * | `acp`            | target-specific (the ACP target process owns its own auth)              | presence-only (validated by the target process) |
  * | `pi` (bedrock)   | `MODEL_OVERRIDE=amazon-bedrock/*` → AWS SDK default credential chain    | presence-only (real check is the worker-side Bedrock enumeration) |
  * | `devin`          | `DEVIN_API_KEY` (+ `DEVIN_API_BASE_URL` override)                       | `${baseUrl}/v3/self`            |
+ * | `dsh`            | `DEEPSEEK_API_KEY` → `OPENROUTER_API_KEY`                               | DeepSeek `/models`, or OpenRouter `/key` |
  * | `amp`            | `AMP_API_KEY`                                                           | `amp usage` (credit balance, no inference; 5s timeout) |
  *
  * Returns `{ok: true, latency_ms}` on 2xx, `{ok: false, error, latency_ms}`
@@ -429,8 +401,9 @@ export async function validateProviderCredentials(
       case "claude": {
         // The default route decides where the check goes: a gateway key is
         // checked against the gateway only (scoped fetch), never against
-        // api.anthropic.com. OAuth stays presence-only; cloud routes have no
-        // free check and report `configured` (no live test).
+        // api.anthropic.com. A subscription token is checked against
+        // `/api/oauth/profile`; cloud routes have no free check and report
+        // `configured` (no live test).
         const route = deriveDefaultRoute("claude", env);
         if (!route) {
           return {
@@ -439,16 +412,40 @@ export async function validateProviderCredentials(
             latency_ms: Date.now() - startedAt,
           };
         }
-        const result = await validateRoute(route, env);
-        const latency_ms = Date.now() - startedAt;
-        if (result.status === "verified") return { ok: true, latency_ms };
-        if (result.status === "configured") return { ok: true, skipped: true, latency_ms };
-        return { ok: false, error: scrubSecrets(result.reason), latency_ms };
+        const checkRoute = async (
+          routeEnv: Record<string, string | undefined>,
+        ): Promise<LiveValidationResult> => {
+          const start = Date.now();
+          const result = await validateRoute(route, routeEnv);
+          const latency_ms = Date.now() - start;
+          if (result.status === "verified") return { ok: true, latency_ms };
+          if (result.status === "configured") return { ok: true, skipped: true, latency_ms };
+          return {
+            ok: false,
+            error: scrubSecrets(result.reason),
+            httpStatus: result.httpStatus,
+            latency_ms,
+          };
+        };
+        const keyVar =
+          route.auth.kind === "subscription"
+            ? "CLAUDE_CODE_OAUTH_TOKEN"
+            : "secretKey" in route.auth
+              ? route.auth.secretKey
+              : undefined;
+        const value = keyVar ? env[keyVar] : undefined;
+        // Only pool vars are split: the runner never splits a gateway token.
+        if (keyVar && value && (CREDENTIAL_POOL_VARS as readonly string[]).includes(keyVar)) {
+          return checkEachKey(keyVar, value, (member) => checkRoute({ ...env, [keyVar]: member }));
+        }
+        return checkRoute(env);
       }
       case "claude-managed": {
         // Managed agents always run with an API key — OAuth not supported on
         // the managed-agents path today.
-        if (env.ANTHROPIC_API_KEY) return checkAnthropicApiKey(env.ANTHROPIC_API_KEY);
+        if (env.ANTHROPIC_API_KEY) {
+          return checkEachKey("ANTHROPIC_API_KEY", env.ANTHROPIC_API_KEY, checkAnthropicApiKey);
+        }
         return {
           ok: false,
           error: "ANTHROPIC_API_KEY is not set.",
@@ -456,6 +453,22 @@ export async function validateProviderCredentials(
         };
       }
       case "codex": {
+        // A gateway route is checked against the gateway only (scoped fetch
+        // to its own `/models`), never against api.openai.com, and wins over
+        // every OAuth / auth.json / OPENAI_API_KEY check below.
+        const gateway = deriveDefaultRoute("codex", env);
+        if (gateway) {
+          const result = await validateRoute(gateway, env);
+          const latency_ms = Date.now() - startedAt;
+          if (result.status === "verified") return { ok: true, latency_ms };
+          if (result.status === "configured") return { ok: true, skipped: true, latency_ms };
+          return {
+            ok: false,
+            error: scrubSecrets(result.reason),
+            httpStatus: result.httpStatus,
+            latency_ms,
+          };
+        }
         // Resolution order matches `checkCodexCredentials`:
         //   1) `~/.codex/auth.json` on disk (canonical state once `codex login`
         //      has run, or when the entrypoint pre-materialised it from
@@ -474,7 +487,9 @@ export async function validateProviderCredentials(
         if (codexAuthFileExists(env)) return presenceCheckOk();
         if (parseCodexOAuthAccess(env.CODEX_OAUTH)) return presenceCheckOk();
         if (hasCodexOAuthPoolSlots(env)) return presenceCheckOk();
-        if (env.OPENAI_API_KEY) return checkOpenAiApiKey(env.OPENAI_API_KEY);
+        if (env.OPENAI_API_KEY) {
+          return checkEachKey("OPENAI_API_KEY", env.OPENAI_API_KEY, checkOpenAiApiKey);
+        }
         return {
           ok: false,
           error:
@@ -503,13 +518,21 @@ export async function validateProviderCredentials(
         // registry providers in declaration order (NaN, canonical key or
         // alias → Helmcode) at their `<ID>_BASE_URL`, so existing env
         // combinations keep their probe.
-        if (env.OPENROUTER_API_KEY) return checkOpenRouter(env.OPENROUTER_API_KEY);
-        if (env.ANTHROPIC_API_KEY) return checkAnthropicApiKey(env.ANTHROPIC_API_KEY);
-        if (env.OPENAI_API_KEY) return checkOpenAiApiKey(env.OPENAI_API_KEY);
+        if (env.OPENROUTER_API_KEY) {
+          return checkEachKey("OPENROUTER_API_KEY", env.OPENROUTER_API_KEY, checkOpenRouter);
+        }
+        if (env.ANTHROPIC_API_KEY) {
+          return checkEachKey("ANTHROPIC_API_KEY", env.ANTHROPIC_API_KEY, checkAnthropicApiKey);
+        }
+        if (env.OPENAI_API_KEY) {
+          return checkEachKey("OPENAI_API_KEY", env.OPENAI_API_KEY, checkOpenAiApiKey);
+        }
         for (const direct of openAICompatibleProviders()) {
           const key = providerKeyFromEnv(direct, env);
           if (key) {
-            return checkOpenAICompatibleProvider(direct, key, providerBaseUrl(direct, env));
+            return checkEachKey(direct.envVar, key, (member) =>
+              checkOpenAICompatibleProvider(direct, member, providerBaseUrl(direct, env)),
+            );
           }
         }
         // Model-specific keys (e.g. GEMINI_API_KEY) can satisfy readiness
@@ -534,28 +557,36 @@ export async function validateProviderCredentials(
           };
         }
         const baseUrl = env.DEVIN_API_BASE_URL ?? "https://api.devin.ai";
-        const r = await timedFetch(`${baseUrl.replace(/\/+$/, "")}/v3/self`, {
-          method: "GET",
-          headers: {
-            Authorization: `Bearer ${apiKey}`,
-            "Content-Type": "application/json",
-          },
+        return checkEachKey("DEVIN_API_KEY", apiKey, async (member) => {
+          const r = await timedFetch(`${baseUrl.replace(/\/+$/, "")}/v3/self`, {
+            method: "GET",
+            headers: {
+              Authorization: `Bearer ${member}`,
+              "Content-Type": "application/json",
+            },
+          });
+          if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+          return {
+            ok: false,
+            error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
+            httpStatus: r.status,
+            latency_ms: r.latency_ms,
+          };
         });
-        if (r.ok) return { ok: true, latency_ms: r.latency_ms };
+      }
+      case "dsh": {
+        // Same precedence as `checkDshCredentials`: the DeepSeek key, then OpenRouter.
+        const deepseekKey = env.DEEPSEEK_API_KEY?.trim();
+        if (deepseekKey) return checkEachKey("DEEPSEEK_API_KEY", deepseekKey, checkDeepSeekApiKey);
+        const openRouterKey = env.OPENROUTER_API_KEY?.trim();
+        if (openRouterKey)
+          return checkEachKey("OPENROUTER_API_KEY", openRouterKey, checkOpenRouter);
         return {
           ok: false,
-          error: scrubSecrets(`HTTP ${r.status}: ${r.bodyText.slice(0, 200)}`),
-          latency_ms: r.latency_ms,
+          error: "Set DEEPSEEK_API_KEY or OPENROUTER_API_KEY for dsh.",
+          latency_ms: Date.now() - startedAt,
         };
       }
-      case "dsh":
-        return checkDshCredentials(env).ready
-          ? presenceCheckOk()
-          : {
-              ok: false,
-              error: "Set DEEPSEEK_API_KEY or OPENROUTER_API_KEY for dsh.",
-              latency_ms: Date.now() - startedAt,
-            };
       case "amp": {
         if (!checkAmpCredentials(env).ready) {
           return {
@@ -565,14 +596,17 @@ export async function validateProviderCredentials(
           };
         }
         // `amp usage` reads the credit balance and runs no inference.
-        const live = await liveTestAmpCredentials(env as Record<string, string | undefined>);
-        return live.ok
-          ? { ok: true, latency_ms: Date.now() - startedAt }
-          : {
-              ok: false,
-              error: live.error ?? "amp usage failed",
-              latency_ms: Date.now() - startedAt,
-            };
+        return checkEachKey("AMP_API_KEY", env.AMP_API_KEY!, async (member) => {
+          const start = Date.now();
+          const live = await liveTestAmpCredentials({ ...env, AMP_API_KEY: member });
+          return live.ok
+            ? { ok: true, latency_ms: Date.now() - start }
+            : {
+                ok: false,
+                error: live.error ?? "amp usage failed",
+                latency_ms: Date.now() - start,
+              };
+        });
       }
       case "cursor": {
         const apiKey = env.CURSOR_API_KEY?.trim();
@@ -584,16 +618,21 @@ export async function validateProviderCredentials(
           };
         }
         const { liveTestCursorKey } = await import("../providers/cursor-adapter");
-        const result = await liveTestCursorKey(apiKey);
-        return result.ok
-          ? { ok: true, latency_ms: Date.now() - startedAt }
-          : { ok: false, error: result.error, latency_ms: Date.now() - startedAt };
+        return checkEachKey("CURSOR_API_KEY", apiKey, async (member) => {
+          const start = Date.now();
+          const result = await liveTestCursorKey(member);
+          return result.ok
+            ? { ok: true, latency_ms: Date.now() - start }
+            : { ok: false, error: result.error, latency_ms: Date.now() - start };
+        });
       }
       case "grok": {
         const apiKey = env.XAI_API_KEY?.trim();
-        if (apiKey) return checkXaiApiKey(apiKey);
+        if (apiKey) return checkEachKey("XAI_API_KEY", apiKey, checkXaiApiKey);
         const openRouterKey = env.OPENROUTER_API_KEY?.trim();
-        if (openRouterKey) return checkOpenRouter(openRouterKey);
+        if (openRouterKey) {
+          return checkEachKey("OPENROUTER_API_KEY", openRouterKey, checkOpenRouter);
+        }
         return {
           ok: false,
           error: "Set XAI_API_KEY or OPENROUTER_API_KEY for grok.",
@@ -626,7 +665,7 @@ export async function validateProviderCredentials(
  * the dashboard surfaces "unreported".
  */
 export function isCredCheckDisabled(env: NodeJS.ProcessEnv): boolean {
-  return env.CRED_CHECK_DISABLE === "1";
+  return isEnvFlagEnabled("CRED_CHECK_DISABLE", false, env);
 }
 
 /**
@@ -653,6 +692,8 @@ export async function buildCredStatusReport(
       error: live.error ?? null,
       latency_ms: live.latency_ms,
       testedAt: Date.now(),
+      ...(live.warning !== undefined ? { warning: live.warning } : {}),
+      ...(live.keys !== undefined ? { keys: live.keys } : {}),
     };
   }
   // Include the Bedrock enumeration block when the pi probe ran in Bedrock SDK