@vellumai/assistant 0.11.4 → 0.11.5-staging.2

package/src/providers/inference/auth.ts

@@ -175,12 +175,23 @@ export type ConnectionModel = z.infer<typeof ConnectionModelSchema>;
 /**
  * Providers whose connections require an explicit `baseUrl` and non-empty
  * `models` list (openai-compatible endpoints have no fixed upstream, so the
- * user must supply both). Every other provider derives these from its catalog
- * entry and rejects a client-supplied `baseUrl`.
+ * user must supply both).
  */
 export const PROVIDERS_REQUIRING_BASE_URL_AND_MODELS: ReadonlySet<string> =
   new Set(["openai-compatible"]);
 
+/**
+ * Providers that persist a client-supplied `baseUrl`. openai-compatible
+ * requires one (see above); ollama has a well-known local default and treats
+ * a stored URL as an optional override. Every other provider derives its
+ * upstream from the catalog and rejects a client-supplied `baseUrl` so a
+ * keyed connection cannot be pointed at an attacker-controlled host.
+ */
+export const PROVIDERS_ALLOWING_CUSTOM_BASE_URL: ReadonlySet<string> = new Set([
+  ...PROVIDERS_REQUIRING_BASE_URL_AND_MODELS,
+  "ollama",
+]);
+
 // ---------------------------------------------------------------------------
 // Full connection shape used by CRUD layer
 // ---------------------------------------------------------------------------

package/src/providers/inference/credential-usage.ts

@@ -0,0 +1,37 @@
+/**
+ * Which provider connections dispatch through a given credential.
+ *
+ * Deleting a credential silently breaks every connection that resolves its
+ * auth through it, so the delete surfaces ask this before removing anything.
+ */
+
+import { getDb } from "../../persistence/db-connection.js";
+import { normalizeCredentialRef } from "../../security/credential-key.js";
+import { listConnections } from "./connections.js";
+
+/**
+ * Names of the provider connections whose auth resolves to
+ * `credentialAccount`, sorted for a stable message.
+ *
+ * Both sides are run through `normalizeCredentialRef` so a row storing the
+ * short `openrouter:api_key` form matches the canonical
+ * `credential/openrouter/api_key` account. Auth variants without a credential
+ * (`platform`, `none`) reference no account and never match.
+ *
+ * Read failures propagate: a caller that cannot enumerate the dependants must
+ * refuse the delete rather than treat the credential as unused, which is the
+ * exact silent breakage this check exists to prevent.
+ */
+export function findConnectionsUsingCredential(
+  credentialAccount: string,
+): string[] {
+  const account = normalizeCredentialRef(credentialAccount);
+  return listConnections(getDb())
+    .filter(
+      (connection) =>
+        "credential" in connection.auth &&
+        normalizeCredentialRef(connection.auth.credential) === account,
+    )
+    .map((connection) => connection.name)
+    .sort();
+}

package/src/providers/inference/missing-credential-guard.ts

@@ -0,0 +1,110 @@
+/**
+ * Receive-side guard for credential-backed connections.
+ *
+ * A provider instance bakes in the credential it resolved at construction, so
+ * a credential removed afterwards is still dispatched with. Several upstreams
+ * answer an unusable key with `200` and an empty `content` array rather than a
+ * `401`, which reaches the user as a blank assistant turn with nothing to act
+ * on. This wrapper converts that pair (no usable content, and the
+ * connection's credential definitively absent from the vault) into the
+ * same `ConnectionResolutionError` the pre-dispatch preflight raises, so the
+ * conversation layer renders its actionable "missing credential" error.
+ *
+ * A response with content passes through untouched, and an unreachable
+ * credential store (`indeterminate`) is never reported as a missing
+ * credential.
+ */
+
+import type { ProviderRouteAttribution } from "../../util/errors.js";
+import { getLogger } from "../../util/logger.js";
+import { hasToolUse, hasVisibleText } from "../content-blocks.js";
+import { checkCredentialPresence } from "../provider-availability.js";
+import { ConnectionResolutionError } from "../routing-identity.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+} from "../types.js";
+
+const log = getLogger("providers/missing-credential-guard");
+
+export class MissingCredentialGuardProvider implements Provider {
+  readonly name: string;
+  readonly tokenEstimationProvider?: string;
+  readonly defaultModel?: string;
+  readonly supportsNativeWebSearch?: boolean;
+  readonly countInputTokens?: NonNullable<Provider["countInputTokens"]>;
+
+  constructor(
+    private readonly inner: Provider,
+    private readonly connection: {
+      name: string;
+      credentialAccount: string;
+    },
+  ) {
+    this.name = inner.name;
+    this.tokenEstimationProvider = inner.tokenEstimationProvider;
+    this.defaultModel = inner.defaultModel;
+    this.supportsNativeWebSearch = inner.supportsNativeWebSearch;
+    if (inner.countInputTokens) {
+      this.countInputTokens = inner.countInputTokens.bind(inner);
+    }
+  }
+
+  /**
+   * Route attribution is assigned onto the outermost provider after
+   * construction (`attachProviderRoute`), and read back off it by usage
+   * accounting, so it reads and writes through to the wrapped instance.
+   */
+  get routeAttribution(): ProviderRouteAttribution | undefined {
+    return this.inner.routeAttribution;
+  }
+
+  set routeAttribution(attribution: ProviderRouteAttribution | undefined) {
+    this.inner.routeAttribution = attribution;
+  }
+
+  supportsNativeWebSearchFor(options?: SendMessageOptions): boolean {
+    return this.inner.supportsNativeWebSearchFor
+      ? this.inner.supportsNativeWebSearchFor(options)
+      : this.inner.supportsNativeWebSearch === true;
+  }
+
+  async sendMessage(
+    messages: Message[],
+    options?: SendMessageOptions,
+  ): Promise<ProviderResponse> {
+    const response = await this.inner.sendMessage(messages, options);
+    if (hasVisibleText(response.content) || hasToolUse(response.content)) {
+      return response;
+    }
+
+    // An empty turn has causes other than a deleted key (a genuinely empty
+    // completion, a refusal, a stop-sequence hit), so the credential read is
+    // what decides, and only its definitive `absent` answer does.
+    const presence = await checkCredentialPresence(
+      this.connection.credentialAccount,
+    );
+    if (presence !== "absent") {
+      return response;
+    }
+
+    log.warn(
+      {
+        connectionName: this.connection.name,
+        credential: this.connection.credentialAccount,
+        model: response.model,
+        stopReason: response.stopReason,
+      },
+      "Upstream returned no usable content for a connection whose credential is absent, so a configuration error is surfaced rather than an empty turn",
+    );
+
+    throw new ConnectionResolutionError(
+      this.connection.name,
+      "missing_credential",
+      `provider_connection "${this.connection.name}" answered with an empty response and has no credential stored: its API key was removed`,
+      { model: response.model },
+    );
+  }
+}

package/src/providers/inference/resolve-auth.ts

@@ -17,7 +17,7 @@ import {
 } from "../platform-proxy/context.js";
 import {
   type Auth,
-  PROVIDERS_REQUIRING_BASE_URL_AND_MODELS,
+  PROVIDERS_ALLOWING_CUSTOM_BASE_URL,
   type ResolvedAuth,
 } from "./auth.js";
 
@@ -36,14 +36,14 @@ export async function resolveAuth(
   { ok: true; resolved: ResolvedAuth } | { ok: false; error: ResolveAuthError }
 > {
   // Defense-in-depth: strip baseUrl for providers that should not accept one.
-  // The route layer rejects base_url for non-openai-compatible providers, but
-  // this guard catches any code path that bypasses route validation (e.g.
-  // corrupted DB rows, direct calls from internal code).
+  // The route layer rejects base_url for those providers, but this guard
+  // catches any code path that bypasses route validation (e.g. corrupted DB
+  // rows, direct calls from internal code).
   let safeBaseUrl = opts.baseUrl;
-  if (safeBaseUrl && !PROVIDERS_REQUIRING_BASE_URL_AND_MODELS.has(provider)) {
+  if (safeBaseUrl && !PROVIDERS_ALLOWING_CUSTOM_BASE_URL.has(provider)) {
     log.warn(
       { provider, baseUrl: safeBaseUrl },
-      `Stripping baseUrl for provider "${provider}" — base_url is only valid for openai-compatible providers.`,
+      `Stripping baseUrl for provider "${provider}": base_url is only valid for openai-compatible and ollama providers.`,
     );
     safeBaseUrl = null;
   }
@@ -85,7 +85,7 @@ export async function resolveAuth(
 
     case "none":
       // Keyless endpoints still need their custom baseUrl threaded through
-      // (LM Studio / vLLM openai-compatible servers).
+      // (Ollama, LM Studio / vLLM openai-compatible servers).
       return {
         ok: true,
         resolved: {

package/src/providers/media-resolve.ts

@@ -12,13 +12,21 @@
  * can keep reading `block.source.data` exactly as before.
  *
  * Blocks that already carry base64 (a live, in-flight turn) pass through
- * untouched — no disk read — so only reloaded history pays the resolution cost,
+ * untouched (no disk read) so only reloaded history pays the resolution cost,
  * and only on its first send after reload. The walk is pure: it returns fresh
  * block/message objects and never mutates the caller's in-memory history.
+ *
+ * The walk is also the last place an image block can be checked before a
+ * provider serializes it, so it is where an image the provider cannot decode
+ * is replaced by a text note: see {@link UNSENDABLE_IMAGE_FORMAT_NOTE}.
  */
 
 import { optimizeImageForTransport } from "../agent/image-optimize.js";
 import { getAttachmentContent } from "../persistence/attachments-store.js";
+import {
+  heifImageMimeType,
+  sniffImageMimeType,
+} from "../util/image-conversion.js";
 import { getLogger } from "../util/logger.js";
 import type {
   Base64MediaSource,
@@ -43,6 +51,65 @@ export function mediaSourceBytes(source: MediaSource): Buffer | null {
   return getAttachmentContent(source.attachmentId);
 }
 
+/**
+ * Whether an image source's bytes are not one of the formats providers accept,
+ * so {@link resolveMediaReferences} replaces the block with
+ * {@link UNSENDABLE_IMAGE_FORMAT_NOTE} instead of sending it. Applies the same
+ * rule as {@link resolveImageBlock}, including its treatment of a reference
+ * whose payload is gone: that is an unavailable attachment, not an unreadable
+ * format. Lets a caller holding a persisted block ask whether the model will
+ * see the image.
+ *
+ * Answers for the formats every provider reads, so HEIF counts as unsendable
+ * even though a Gemini turn ({@link MediaResolveOptions}) would carry it.
+ */
+export function isUnsendableImageSource(source: MediaSource): boolean {
+  const bytes =
+    source.type === "base64"
+      ? imageHeadBytes(source.data)
+      : getAttachmentContent(source.attachmentId);
+  return bytes != null && sendableImageMimeType(bytes, {}) === null;
+}
+
+/**
+ * Media types a provider accepts for an image block beyond the four every
+ * provider reads (PNG, JPEG, GIF, WebP).
+ *
+ * HEIF is the one format where providers genuinely disagree: Gemini decodes
+ * `image/heic` and `image/heif` directly
+ * (https://ai.google.dev/gemini-api/docs/image-understanding), while
+ * OpenAI-compatible and Anthropic endpoints answer HTTP 400 for the whole
+ * request. HEIF bytes reach a provider whenever transcoding was unavailable
+ * (`convertImageToJpeg` is backed by macOS `sips`), so a Linux-hosted assistant
+ * on Gemini depends on them passing through.
+ */
+export interface MediaResolveOptions {
+  acceptsHeif?: boolean;
+}
+
+// 16 base64 chars decode to the 12 bytes the longest signature needs.
+function imageHeadBytes(dataBase64: string): Buffer {
+  return Buffer.from(dataBase64.slice(0, 16), "base64");
+}
+
+/**
+ * Media type the provider should be told an image is, or null when it cannot
+ * read the bytes at all.
+ */
+function sendableImageMimeType(
+  bytes: Uint8Array,
+  options: MediaResolveOptions,
+): string | null {
+  const sniffed = sniffImageMimeType(bytes);
+  if (sniffed) {
+    return sniffed;
+  }
+  if (options.acceptsHeif) {
+    return heifImageMimeType(bytes);
+  }
+  return null;
+}
+
 /**
  * Narrow a media source to its base64 arm. After {@link resolveMediaReferences}
  * runs, provider transforms only ever see base64 sources; this asserts that
@@ -96,9 +163,57 @@ export function resolveMediaSourceData(
   return { data: bytes.toString("base64"), media_type: source.media_type };
 }
 
-async function resolveImageBlock(block: ImageContent): Promise<ContentBlock> {
+/**
+ * Note that replaces an image block whose bytes are not one of the formats
+ * providers accept (PNG, JPEG, GIF, WebP). Sending the block instead costs the
+ * user the whole turn: an OpenAI-compatible endpoint answers HTTP 400 ("The
+ * image data you provided does not represent a valid image") for the entire
+ * request, so one unreadable image in a batch of eight blocks the reply to all
+ * of them. The note keeps the turn alive and tells the model what it is
+ * missing, and the daemon posts a system card naming the file so the user
+ * knows too.
+ */
+export const UNSENDABLE_IMAGE_FORMAT_NOTE =
+  "[Image omitted: its format is not one the model can read (PNG, JPEG, GIF, and WebP are)]";
+
+/**
+ * Resolve an image block into one the provider can accept, keyed on the bytes
+ * rather than on the declared media type.
+ *
+ * The declaration is only a claim: web clients derive it from the filename
+ * extension, so a HEIC photo saved as `.png` arrives declared `image/png`. One
+ * sniff covers both failure modes that claim produces. Bytes that sniff as
+ * nothing the target provider reads are unsendable (a corrupt head, or a format
+ * outside both the universal four and the provider's own extras: AVIF, BMP,
+ * TIFF, SVG) and become {@link UNSENDABLE_IMAGE_FORMAT_NOTE}. Bytes that sniff
+ * as something other than the declaration are relabeled, since a mismatch is
+ * fatal on its own (Anthropic answers "image does not match the provided media
+ * type").
+ *
+ * The sniff runs ahead of the transport optimization so sendability cannot
+ * depend on an unrelated size threshold: optimization rewrites only the images
+ * it decides to rescale.
+ */
+async function resolveImageBlock(
+  block: ImageContent,
+  options: MediaResolveOptions,
+): Promise<ContentBlock> {
   if (block.source.type === "base64") {
-    return block;
+    const sniffed = sendableImageMimeType(
+      imageHeadBytes(block.source.data),
+      options,
+    );
+    if (!sniffed) {
+      log.warn(
+        { declaredMediaType: block.source.media_type },
+        "Inline image is not in a provider-readable format; substituting a text note",
+      );
+      return { type: "text", text: UNSENDABLE_IMAGE_FORMAT_NOTE };
+    }
+    return {
+      type: "image",
+      source: { type: "base64", media_type: sniffed, data: block.source.data },
+    };
   }
   const bytes = getAttachmentContent(block.source.attachmentId);
   if (!bytes) {
@@ -111,11 +226,22 @@ async function resolveImageBlock(block: ImageContent): Promise<ContentBlock> {
       text: "[Attachment unavailable: image could not be loaded]",
     };
   }
+  const sniffed = sendableImageMimeType(bytes, options);
+  if (!sniffed) {
+    log.warn(
+      {
+        attachmentId: block.source.attachmentId,
+        declaredMediaType: block.source.media_type,
+      },
+      "Referenced image is not in a provider-readable format; substituting a text note",
+    );
+    return { type: "text", text: UNSENDABLE_IMAGE_FORMAT_NOTE };
+  }
   // Apply the same transport optimization the inline-base64 path used, so a
   // reloaded (reference) turn sends the model the same bytes a live turn would.
   const { data, mediaType } = await optimizeImageForTransport(
     bytes.toString("base64"),
-    block.source.media_type,
+    sniffed,
   );
   return {
     type: "image",
@@ -161,10 +287,13 @@ function resolveFileBlock(block: FileContent): ContentBlock {
   };
 }
 
-async function resolveBlock(block: ContentBlock): Promise<ContentBlock> {
+async function resolveBlock(
+  block: ContentBlock,
+  options: MediaResolveOptions,
+): Promise<ContentBlock> {
   switch (block.type) {
     case "image":
-      return resolveImageBlock(block);
+      return resolveImageBlock(block, options);
     case "file":
       return resolveFileBlock(block);
     case "tool_result": {
@@ -174,7 +303,9 @@ async function resolveBlock(block: ContentBlock): Promise<ContentBlock> {
       }
       return {
         ...block,
-        contentBlocks: await Promise.all(block.contentBlocks.map(resolveBlock)),
+        contentBlocks: await Promise.all(
+          block.contentBlocks.map((nested) => resolveBlock(nested, options)),
+        ),
       };
     }
     default:
@@ -182,13 +313,40 @@ async function resolveBlock(block: ContentBlock): Promise<ContentBlock> {
   }
 }
 
-function contentHasReference(content: ContentBlock[]): boolean {
+/**
+ * Whether an inline image needs rebuilding before it can be sent: its payload
+ * is unreadable, or its declared media type is not exactly what the bytes are.
+ * The comparison is exact so a non-canonical spelling of a real format is
+ * rewritten too (`image/jpg` is not in any provider's accepted set, and clients
+ * derive it from a `.jpg` extension). Sniffing here, rather than rebuilding
+ * every inline block unconditionally, keeps the clean live turn on the identity
+ * fast path below at the cost of decoding a 12-byte head per image.
+ */
+function base64ImageNeedsRewrite(
+  source: Base64MediaSource,
+  options: MediaResolveOptions,
+): boolean {
+  return (
+    sendableImageMimeType(imageHeadBytes(source.data), options) !==
+    source.media_type
+  );
+}
+
+function contentNeedsResolution(
+  content: ContentBlock[],
+  options: MediaResolveOptions,
+): boolean {
   return content.some((block) => {
-    if (block.type === "image" || block.type === "file") {
+    if (block.type === "image") {
+      return block.source.type === "workspace_ref"
+        ? true
+        : base64ImageNeedsRewrite(block.source, options);
+    }
+    if (block.type === "file") {
       return block.source.type === "workspace_ref";
     }
     if (block.type === "tool_result" && block.contentBlocks?.length) {
-      return contentHasReference(block.contentBlocks);
+      return contentNeedsResolution(block.contentBlocks, options);
     }
     return false;
   });
@@ -196,21 +354,24 @@ function contentHasReference(content: ContentBlock[]): boolean {
 
 /**
  * Return a copy of `messages` with every {@link WorkspaceRefMediaSource}
- * resolved to inline base64. Messages with no references are returned unchanged
- * (same object reference) so the common all-base64 live turn does no allocation
- * or disk I/O.
+ * resolved to inline base64 and every image block reconciled with its own
+ * bytes. Messages that need neither are returned unchanged (same object
+ * reference) so the common live turn does no allocation or disk I/O.
  */
 export function resolveMediaReferences(
   messages: Message[],
+  options: MediaResolveOptions = {},
 ): Promise<Message[]> {
   return Promise.all(
     messages.map(async (message) => {
-      if (!contentHasReference(message.content)) {
+      if (!contentNeedsResolution(message.content, options)) {
         return message;
       }
       return {
         ...message,
-        content: await Promise.all(message.content.map(resolveBlock)),
+        content: await Promise.all(
+          message.content.map((block) => resolveBlock(block, options)),
+        ),
       };
     }),
   );