@vellumai/assistant 0.11.5 → 0.11.6-staging.1

package/src/providers/inference/__tests__/profile-probe-classify.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, test } from "bun:test";
+
+import { ProviderError } from "../../../util/errors.js";
+import { classifyProbeFailure } from "../profile-probe.js";
+
+describe("classifyProbeFailure", () => {
+  test("blames the provider connection for a rejected credential", () => {
+    const result = classifyProbeFailure(
+      new ProviderError("401 invalid api key", "openai-compatible", 401, {
+        reason: "invalid_credentials",
+      }),
+    );
+    expect(result.blame).toBe("provider");
+    expect(result.reason).toBe("invalid_credentials");
+  });
+
+  test("blames the profile for an unknown model", () => {
+    const result = classifyProbeFailure(
+      new ProviderError("model does not exist", "openai-compatible", 404, {
+        reason: "model_not_found",
+      }),
+    );
+    expect(result.blame).toBe("profile");
+  });
+
+  test("blames the profile for an impossible token budget", () => {
+    const result = classifyProbeFailure(
+      new ProviderError(
+        "ContextWindowExceededError: maximum context length is 300000 tokens",
+        "openai-compatible",
+        400,
+        { reason: "context_overflow" },
+      ),
+    );
+    expect(result.blame).toBe("profile");
+    expect(result.detail).toContain("300000");
+  });
+
+  test("marks rate limits as transient", () => {
+    expect(
+      classifyProbeFailure(
+        new ProviderError("429", "openai", 429, { reason: "rate_limited" }),
+      ).blame,
+    ).toBe("transient");
+  });
+
+  test("blames the provider for transport failures", () => {
+    // The adapters stamp reason network_error on SDK connection errors
+    // (normalizeOpenAIAPIError), so the probe classifies by reason alone.
+    const result = classifyProbeFailure(
+      new ProviderError(
+        "OpenAI-compatible API error (unknown status): Connection error.",
+        "openai-compatible",
+        undefined,
+        { reason: "network_error" },
+      ),
+    );
+    expect(result.blame).toBe("provider");
+    expect(result.reason).toBe("network_error");
+  });
+
+  test("falls back to unknown for unclassified errors", () => {
+    expect(classifyProbeFailure(new Error("boom")).blame).toBe("unknown");
+    expect(classifyProbeFailure(new Error("boom")).detail).toBe("boom");
+  });
+});

package/src/providers/inference/adapter-factory.ts

@@ -136,10 +136,6 @@ const ADAPTER_FACTORIES: Record<string, AdapterFactory> = {
       // Replay thinking as `reasoning_content` so DeepSeek-compatible
       // thinking-mode upstreams accept follow-up requests that include tools.
       assistantReasoningField: "reasoning_content",
-      // Generic OpenAI-compat proxies may front a strict backend (vLLM,
-      // DeepSeek, Portkey). Backfill empty assistant turns so the request
-      // satisfies `content or tool_calls must be set`.
-      backfillEmptyAssistantContent: true,
       ...(baseURL ? { baseURL } : {}),
     }),
   // Keyless openai-compatible endpoints (e.g. LM Studio) ignore the key; the
@@ -152,11 +148,6 @@ const ADAPTER_FACTORIES: Record<string, AdapterFactory> = {
       // Replay thinking as `reasoning_content` so DeepSeek-compatible
       // thinking-mode endpoints accept follow-up requests that include tools.
       assistantReasoningField: "reasoning_content",
-      // Custom endpoints (Portkey, vLLM, LM Studio, DeepSeek-compat) often
-      // reject `{ role: "assistant", content: null }` after a Stop mid-stream
-      // or a reasoning-only turn. Same guard as OpenRouter and Vercel AI
-      // Gateway.
-      backfillEmptyAssistantContent: true,
       // Custom OpenAI-compatible endpoints may front strict reasoning
       // models (DeepSeek thinking) that 400 on any explicit tool_choice.
       omitToolChoiceWhenReasoning: true,

package/src/providers/inference/credential-rotation.ts

@@ -0,0 +1,61 @@
+/**
+ * Refreshes provider state after a credential rotation.
+ */
+
+import { getConfig, invalidateConfigCache } from "../../config/loader.js";
+import { evictConversationsForReload } from "../../daemon/conversation-store.js";
+import { clearEmbeddingBackendCache } from "../../persistence/embeddings/embedding-backend.js";
+import { credentialKey } from "../../security/credential-key.js";
+import { getLogger } from "../../util/logger.js";
+import { initializeProviders } from "../registry.js";
+import { findConnectionsUsingCredential } from "./credential-usage.js";
+
+const log = getLogger("credential-rotation");
+
+export async function refreshProvidersAfterSecretChange(): Promise<void> {
+  clearEmbeddingBackendCache();
+  invalidateConfigCache();
+  await initializeProviders(getConfig());
+
+  // Provider instances are captured when conversations are created, so a key
+  // change must evict or mark them stale before the next turn. Best-effort:
+  // the credential write has already succeeded, so a disposal failure must not
+  // surface as a 500 that makes clients think the secret change failed.
+  try {
+    evictConversationsForReload();
+  } catch (err) {
+    log.warn(
+      { error: err instanceof Error ? err.message : String(err) },
+      "Error evicting conversations after credential change (non-fatal)",
+    );
+  }
+}
+
+export async function refreshProvidersForRotatedCredential(
+  service: string,
+  field: string,
+): Promise<void> {
+  const key = credentialKey(service, field);
+
+  // A connection resolves its auth through this credential account, and
+  // the resolved adapter bakes the key it read into its headers, so the
+  // rotation only reaches dispatch once the cached adapter is dropped.
+  // The credential write has already succeeded: an enumeration failure
+  // refreshes rather than surfacing as a 500.
+  try {
+    const connections = findConnectionsUsingCredential(key);
+    if (connections.length > 0) {
+      await refreshProvidersAfterSecretChange();
+    }
+  } catch (err) {
+    log.warn(
+      {
+        service,
+        field,
+        error: err instanceof Error ? err.message : String(err),
+      },
+      "Failed to inspect provider connections after credential update; refreshing providers anyway",
+    );
+    await refreshProvidersAfterSecretChange();
+  }
+}

package/src/providers/inference/endpoint-probe.ts

@@ -0,0 +1,115 @@
+/**
+ * Save-time probe for connections with a custom base URL.
+ *
+ * A wrong base path (e.g. NVIDIA without `/v1`) otherwise stays invisible
+ * until the first chat turn fails with an opaque provider 404. The probe
+ * fires one minimal chat-completions request against the stored endpoint and
+ * reports the outcome as a non-blocking hint. The save always succeeds.
+ */
+
+import { z } from "zod";
+
+import { getLogger } from "../../util/logger.js";
+import type { Auth, ConnectionModel } from "./auth.js";
+import { resolveAuth } from "./resolve-auth.js";
+
+const log = getLogger("inference-endpoint-probe");
+
+/**
+ * Bound on every save-path probe (this endpoint probe and the profile probe)
+ * so a dead host can't hang a save. Matches the API-key validation timeout.
+ */
+export const PROBE_TIMEOUT_MS = 10_000;
+
+export const EndpointCheckSchema = z
+  .object({
+    ok: z.boolean(),
+    status: z.number().int().optional(),
+    resolved_url: z.string(),
+    error_class: z.enum(["http_error", "timeout", "network"]).optional(),
+    /** Human-readable warning, present when `ok` is false. */
+    hint: z.string().optional(),
+  })
+  .meta({ id: "EndpointCheck" });
+
+export type EndpointCheck = z.infer<typeof EndpointCheckSchema>;
+
+function hintForStatus(status: number): string {
+  if (status === 404) {
+    return "The endpoint returned 404 for a test request: check the base path for this provider (e.g. NVIDIA needs /v1, OpenRouter needs /api/v1). Some providers gate requests behind auth, so this may be a false alarm.";
+  }
+  if (status === 401 || status === 403) {
+    return `The endpoint rejected the credential for a test request (HTTP ${status}). Check the API key.`;
+  }
+  return `The endpoint returned HTTP ${status} for a test request.`;
+}
+
+/**
+ * Probe a connection's custom endpoint with a minimal chat-completions
+ * request (`max_tokens: 1`). Returns `null` when there is nothing to probe:
+ * no custom base URL, no model id to send, or auth that cannot be resolved
+ * (a missing credential surfaces through its own error path).
+ */
+export async function testInferenceConnection(
+  connection: {
+    provider: string;
+    auth: Auth;
+    baseUrl?: string | null;
+    models?: ConnectionModel[] | null;
+  },
+  fetchImpl: typeof fetch = fetch,
+): Promise<EndpointCheck | null> {
+  const model = connection.models?.[0]?.id;
+  if (!connection.baseUrl || !model) {
+    return null;
+  }
+
+  const resolved = await resolveAuth(connection.auth, connection.provider, {
+    baseUrl: connection.baseUrl,
+  });
+  if (!resolved.ok || resolved.resolved.kind === "runtime_proxy") {
+    return null;
+  }
+  const authHeaders =
+    resolved.resolved.kind === "header" ? resolved.resolved.headers : {};
+
+  const url = `${connection.baseUrl.replace(/\/+$/, "")}/chat/completions`;
+  try {
+    const res = await fetchImpl(url, {
+      method: "POST",
+      headers: { ...authHeaders, "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model,
+        messages: [{ role: "user", content: "ping" }],
+        max_tokens: 1,
+        stream: false,
+      }),
+      signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+    });
+    void res.body?.cancel();
+    if (res.ok) {
+      return { ok: true, status: res.status, resolved_url: url };
+    }
+    return {
+      ok: false,
+      status: res.status,
+      resolved_url: url,
+      error_class: "http_error",
+      hint: hintForStatus(res.status),
+    };
+  } catch (err) {
+    const isTimeout = err instanceof Error && err.name === "TimeoutError";
+    log.info(
+      { url, error: err instanceof Error ? err.message : String(err) },
+      "Endpoint probe failed to reach the endpoint",
+    );
+    return {
+      ok: false,
+      resolved_url: url,
+      error_class: isTimeout ? "timeout" : "network",
+      hint: isTimeout
+        ? `The endpoint did not respond within ${PROBE_TIMEOUT_MS / 1000}s for a test request.`
+        : `Could not reach the endpoint for a test request: ${err instanceof Error ? err.message : String(err)}`,
+    };
+  }
+}

package/src/providers/inference/profile-probe.ts

@@ -0,0 +1,256 @@
+/**
+ * Live save-time probe for a user-authored inference profile: one minimal
+ * request dispatched through the profile's own resolved model and connection,
+ * so a wrong model name, a rejected key, or an impossible token budget
+ * surfaces when the profile is saved instead of on the first chat message.
+ *
+ * The verdict is advisory and classified by which object the user should fix
+ * (this profile vs its provider connection), using the semantic
+ * `ProviderErrorReason` the provider adapters stamp at their throw sites.
+ * Managed and routing-identity profiles are never probed: their pins are
+ * hand-validated, and a probe there would spend managed credits on every
+ * save.
+ */
+
+import type { ResolutionFallbackReason } from "../../config/llm-resolver.js";
+import {
+  resolveCallSiteConfig,
+  selectWinningProfile,
+} from "../../config/llm-resolver.js";
+import { getConfigReadOnly } from "../../config/loader.js";
+import { getDb } from "../../persistence/db-connection.js";
+import { ProviderError, type ProviderErrorReason } from "../../util/errors.js";
+import { getLogger } from "../../util/logger.js";
+import { dispatchProviderResolvable } from "../connection-resolution.js";
+import {
+  createTimeout,
+  getConfiguredProvider,
+  userMessage,
+} from "../provider-send-message.js";
+import { ROUTING_IDENTITY_PROVIDERS } from "./auth.js";
+import { getConnection, MANAGED_CONNECTION_NAMES } from "./connections.js";
+import { PROBE_TIMEOUT_MS } from "./endpoint-probe.js";
+
+const log = getLogger("inference-profile-probe");
+
+/** Which object the user should fix when the probe fails. */
+export type ProfileCheckBlame =
+  | "profile"
+  | "provider"
+  | "transient"
+  | "unknown";
+
+export interface ProfileCheck {
+  ok: boolean;
+  blame?: ProfileCheckBlame;
+  /** Semantic reason (ProviderErrorReason, or "resolution"/"not_configured"). */
+  reason?: string;
+  /** Upstream or resolver error text, verbatim. */
+  detail?: string;
+  /** The connection to inspect when `blame` is "provider". */
+  connection?: string;
+  /** English summary for non-localized surfaces (the CLI). */
+  message?: string;
+}
+
+const PROVIDER_BLAME_REASONS: ReadonlySet<ProviderErrorReason> = new Set([
+  "invalid_credentials",
+  "insufficient_credits",
+  "daily_limit_reached",
+  "network_error",
+  "server_error",
+]);
+
+const TRANSIENT_BLAME_REASONS: ReadonlySet<ProviderErrorReason> = new Set([
+  "rate_limited",
+  "overloaded",
+]);
+
+/**
+ * Map a failed probe dispatch onto the object the user should fix. Reasons
+ * the adapters stamp about the request itself (model unknown or restricted,
+ * parameter and token-budget rejections) blame the profile; credential,
+ * billing, and reachability reasons blame the provider connection.
+ */
+export function classifyProbeFailure(err: unknown): {
+  blame: ProfileCheckBlame;
+  reason?: string;
+  detail: string;
+} {
+  const detail = err instanceof Error ? err.message : String(err);
+  if (!(err instanceof ProviderError) || err.reason === undefined) {
+    return { blame: "unknown", detail };
+  }
+  if (PROVIDER_BLAME_REASONS.has(err.reason)) {
+    return { blame: "provider", reason: err.reason, detail };
+  }
+  if (TRANSIENT_BLAME_REASONS.has(err.reason)) {
+    return { blame: "transient", reason: err.reason, detail };
+  }
+  if (err.reason === "unknown") {
+    return { blame: "unknown", reason: err.reason, detail };
+  }
+  return { blame: "profile", reason: err.reason, detail };
+}
+
+function checkMessage(check: {
+  blame: ProfileCheckBlame;
+  detail: string;
+  connection?: string;
+}): string {
+  switch (check.blame) {
+    case "profile":
+      return `A test request through this profile failed: ${check.detail} Check the profile's model and parameters.`;
+    case "provider":
+      return (
+        `A test request through this profile failed: ${check.detail} ` +
+        (check.connection
+          ? `Check the provider connection "${check.connection}".`
+          : `Check the profile's provider connection.`)
+      );
+    case "transient":
+      return `A test request through this profile could not complete: ${check.detail} This may be temporary.`;
+    default:
+      return `A test request through this profile failed: ${check.detail}`;
+  }
+}
+
+/**
+ * Probe a saved profile with one minimal request. Returns `null` when there
+ * is no verdict to give: the profile does not exist or is disabled, it rides
+ * a managed or routing-identity route (probing would bill managed credits),
+ * or the probe timed out without an answer.
+ */
+export async function probeInferenceProfile(
+  name: string,
+): Promise<ProfileCheck | null> {
+  const { llm } = getConfigReadOnly();
+  const entry = llm.profiles?.[name] as Record<string, unknown> | undefined;
+  if (!entry || entry.status === "disabled" || entry.source === "managed") {
+    return null;
+  }
+
+  const selectionSeed = crypto.randomUUID();
+
+  // If the resolver would fall back past this profile, the probe must not run:
+  // it would silently exercise (and bill) a different profile. The fallback
+  // reason is itself the verdict.
+  let fallbackReason: ResolutionFallbackReason | undefined;
+  const winner = selectWinningProfile("inference", llm, {
+    overrideProfile: name,
+    selectionSeed,
+    // The same resolvability predicate dispatch applies: without it this
+    // pre-check can report the requested profile as winner while the actual
+    // dispatch below falls back to (and bills) a different profile.
+    isResolvableProvider: dispatchProviderResolvable,
+    onResolutionFallback: (info) => {
+      if (info.requested === name) {
+        fallbackReason = info.reason;
+      }
+    },
+  });
+  if (winner.profileName !== name) {
+    const detail = `The profile is ${fallbackReason ?? "unusable"} and requests would silently run on ${winner.profileName ?? "the default"} instead.`;
+    return {
+      ok: false,
+      blame: "profile",
+      reason: "resolution",
+      detail,
+      message: checkMessage({ blame: "profile", detail }),
+    };
+  }
+
+  // Billing guards run on the fully resolved call-site config — the same
+  // composition dispatch consumes (mix arm expanded, call-site overrides
+  // applied) — so neither a mix nor an `llm.callSites.inference` provider
+  // tweak can route the probe onto a managed or platform-billed path the
+  // stored entry hid. Same seed as the dispatch below, so the judged arm is
+  // the dispatched arm.
+  const resolved = resolveCallSiteConfig("inference", llm, {
+    overrideProfile: name,
+    selectionSeed,
+  });
+  const provider = resolved.provider ?? "";
+  if (ROUTING_IDENTITY_PROVIDERS.has(provider)) {
+    return null;
+  }
+  const connection = resolved.provider_connection ?? (provider || undefined);
+  // Canonical managed names always dispatch platform-billed: routing ignores
+  // a user-owned row that merely claims such a name
+  // (tryResolveProviderForConnectionName), so the name gates the probe
+  // regardless of the row's stored auth.
+  if (connection && MANAGED_CONNECTION_NAMES.has(connection)) {
+    return null;
+  }
+  // A legacy profile can declare a concrete provider while staying bound to
+  // a platform-billed connection row; the row's auth is the billing fact, so
+  // it gates the probe too.
+  const boundRow = connection ? getConnection(getDb(), connection) : null;
+  if (boundRow?.auth.type === "platform") {
+    return null;
+  }
+
+  const timeout = createTimeout(PROBE_TIMEOUT_MS);
+  try {
+    const providerInstance = await getConfiguredProvider("inference", {
+      overrideProfile: name,
+      selectionSeed,
+    });
+    if (!providerInstance) {
+      const detail = "No dispatchable provider is configured for this profile.";
+      return {
+        ok: false,
+        blame: "provider",
+        reason: "not_configured",
+        detail,
+        connection,
+        message: checkMessage({ blame: "provider", detail, connection }),
+      };
+    }
+    // Final billing gate on the route dispatch ACTUALLY resolved: a profile
+    // without a binding gets its row auto-selected inside
+    // getConfiguredProvider, so only the stamped attribution names the row
+    // this request would sign with.
+    const route = providerInstance.routeAttribution;
+    if (route?.isManagedRoute) {
+      return null;
+    }
+    const routedRow = route?.connectionName
+      ? getConnection(getDb(), route.connectionName)
+      : null;
+    if (routedRow?.auth.type === "platform") {
+      return null;
+    }
+    await providerInstance.sendMessage(
+      [userMessage("Reply with only the word OK.")],
+      {
+        signal: timeout.signal,
+        config: {
+          callSite: "inference",
+          overrideProfile: name,
+          selectionSeed,
+        },
+      },
+    );
+    return { ok: true };
+  } catch (err) {
+    if (timeout.signal.aborted) {
+      // No verdict: a slow model and a dead endpoint look identical here,
+      // and the connection-level probe already covers unreachable hosts.
+      log.info({ profile: name }, "Profile probe timed out without a verdict");
+      return null;
+    }
+    const classified = classifyProbeFailure(err);
+    return {
+      ok: false,
+      ...classified,
+      ...(classified.blame === "provider" ? { connection } : {}),
+      message: checkMessage({
+        ...classified,
+        ...(classified.blame === "provider" ? { connection } : {}),
+      }),
+    };
+  } finally {
+    timeout.cleanup();
+  }
+}