@vellumai/assistant 0.8.5 → 0.8.7

package/src/providers/gemini/client.ts

@@ -1,9 +1,14 @@
 import type * as genai from "@google/genai";
 import { ApiError, GoogleGenAI, ThinkingLevel } from "@google/genai";
 
+import {
+  THINKING_LEVELS,
+  type ThinkingLevel as ThinkingLevelName,
+} from "../../config/schemas/llm.js";
 import { isAbortReason } from "../../util/abort-reasons.js";
 import { ProviderError } from "../../util/errors.js";
 import { getLogger } from "../../util/logger.js";
+import { PROVIDER_CATALOG } from "../model-catalog.js";
 import { createStreamTimeout } from "../stream-timeout.js";
 import type {
   ContentBlock,
@@ -11,12 +16,16 @@ import type {
   Provider,
   ProviderResponse,
   SendMessageOptions,
-  ToolDefinition,
 } from "../types.js";
 import {
   ContextOverflowError,
   extractOverflowTokensFromMessage,
 } from "../types.js";
+import {
+  base64ByteLength,
+  GEMINI_MAX_INLINE_AUDIO_BYTES,
+  normalizeGeminiAudioMime,
+} from "./inline-media.js";
 
 /**
  * Token/context-specific phrases that reliably indicate context-overflow
@@ -33,40 +42,100 @@ function isGemini3Model(model: string): boolean {
   return model.startsWith("gemini-3") || model.startsWith("models/gemini-3");
 }
 
-const THINKING_LEVEL_BY_NAME: Record<string, ThinkingLevel> = {
+const THINKING_LEVEL_BY_NAME: Record<ThinkingLevelName, ThinkingLevel> = {
   minimal: ThinkingLevel.MINIMAL,
   low: ThinkingLevel.LOW,
   medium: ThinkingLevel.MEDIUM,
   high: ThinkingLevel.HIGH,
 };
 
+/**
+ * Default thinking level for Gemini Pro models when the profile doesn't pin
+ * one. Pro rejects `"minimal"` and an absent level resolves to `"minimal"`
+ * upstream, so we pin Google's documented Pro default (`"high"`) — always a
+ * supported value.
+ */
+const GEMINI_PRO_DEFAULT_THINKING_LEVEL: ThinkingLevelName = "high";
+
+/**
+ * Gemini 3.x Pro family accepts only `low`/`medium`/`high` (no `"minimal"`) and
+ * cannot fully disable thinking. Matches `gemini-3.1-pro-preview`,
+ * `gemini-3.1-pro-preview-customtools`, and future `gemini-3*pro*`.
+ */
+function isGeminiProModel(model: string): boolean {
+  const normalized = model.startsWith("models/")
+    ? model.slice("models/".length)
+    : model;
+  return /^gemini-3.*pro/.test(normalized);
+}
+
+/**
+ * Lowest thinking level the model accepts. Pro's floor is `"low"`; every other
+ * thinking-capable Gemini model accepts `"minimal"`.
+ */
+function geminiThinkingFloor(model: string): ThinkingLevelName {
+  return isGeminiProModel(model) ? "low" : "minimal";
+}
+
+/**
+ * Raise `level` to `floor` when it sits below it, so we never send a level the
+ * model rejects (e.g. `"minimal"` to a Pro model).
+ */
+function clampThinkingLevelToFloor(
+  level: ThinkingLevelName,
+  floor: ThinkingLevelName,
+): ThinkingLevelName {
+  return THINKING_LEVELS.indexOf(level) < THINKING_LEVELS.indexOf(floor)
+    ? floor
+    : level;
+}
+
 /**
  * Translate the resolved wire-shape `thinking` config into Gemini's
- * `thinkingConfig`. Returns `undefined` when no thinking config was supplied,
- * which lets Google's per-model default apply (e.g. `gemini-3.5-flash`
- * defaults to dynamic medium-level thinking).
+ * `thinkingConfig`, guaranteeing the emitted `thinkingLevel` is one the model
+ * accepts. Returns `undefined` when nothing needs to be set, which lets
+ * Google's per-model default apply (e.g. `gemini-3.5-flash` defaults to
+ * dynamic medium-level thinking).
+ *
+ * - `enabled: false` maps to the model's floor — the most "off" state it
+ *   allows (`"minimal"` for most models, `"low"` for Pro, which can't disable
+ *   thinking).
+ * - An explicit `level` below the floor is raised to the floor.
+ * - When no `level` is pinned, Pro models get the documented default (`"high"`)
+ *   because an absent level resolves to the unsupported `"minimal"` upstream;
+ *   other models keep Google's per-model default by leaving the level unset.
  *
- * `enabled: false` maps to `thinkingLevel: MINIMAL` because Gemini 3.x cannot
- * fully disable thinking — `"minimal"` is the floor. `includeThoughts` is
- * gated on `streamThinking` so callers that opted out of streaming thoughts
- * don't pay for thought tokens in the response.
+ * `includeThoughts` is gated on `streamThinking` so callers that opted out of
+ * streaming thoughts don't pay for thought tokens in the response.
  */
 function buildThinkingConfig(
   thinking: Record<string, unknown> | undefined,
+  model: string,
 ): genai.ThinkingConfig | undefined {
   if (!thinking) return undefined;
+  const floor = geminiThinkingFloor(model);
+
   if (thinking.type === "disabled") {
     return {
-      thinkingLevel: ThinkingLevel.MINIMAL,
+      thinkingLevel: THINKING_LEVEL_BY_NAME[floor],
       includeThoughts: false,
     };
   }
   if (thinking.type !== "adaptive") return undefined;
 
   const result: genai.ThinkingConfig = {};
-  if (typeof thinking.level === "string") {
-    const mapped = THINKING_LEVEL_BY_NAME[thinking.level];
-    if (mapped) result.thinkingLevel = mapped;
+  if (
+    typeof thinking.level === "string" &&
+    thinking.level in THINKING_LEVEL_BY_NAME
+  ) {
+    const clamped = clampThinkingLevelToFloor(
+      thinking.level as ThinkingLevelName,
+      floor,
+    );
+    result.thinkingLevel = THINKING_LEVEL_BY_NAME[clamped];
+  } else if (isGeminiProModel(model)) {
+    result.thinkingLevel =
+      THINKING_LEVEL_BY_NAME[GEMINI_PRO_DEFAULT_THINKING_LEVEL];
   }
   if (typeof thinking.streamThinking === "boolean") {
     result.includeThoughts = thinking.streamThinking;
@@ -74,6 +143,24 @@ function buildThinkingConfig(
   return Object.keys(result).length > 0 ? result : undefined;
 }
 
+/**
+ * Whether the active Gemini model accepts a `thinkingConfig`. Non-thinking
+ * models (e.g. `gemini-2.5-flash-lite`) reject thinking params, and gemini is
+ * in `THINKING_AWARE_PROVIDERS` so `providers/retry.ts` no longer strips them —
+ * so we gate on the catalog's `supportsThinking` capability here. Unknown or
+ * uncatalogued models default to allowing thinking config (preserving prior
+ * behavior); only an explicit `supportsThinking: false` suppresses it.
+ */
+function geminiModelSupportsThinking(model: string): boolean {
+  const normalized = model.startsWith("models/")
+    ? model.slice("models/".length)
+    : model;
+  const catalogModel = PROVIDER_CATALOG.find(
+    (provider) => provider.id === "gemini",
+  )?.models.find((m) => m.id === normalized);
+  return catalogModel?.supportsThinking !== false;
+}
+
 function stripGeminiHttpOptions(
   config: genai.GenerateContentConfig,
 ): genai.GenerateContentConfig {
@@ -204,21 +291,22 @@ export class GeminiProvider implements Provider {
 
   async sendMessage(
     messages: Message[],
-    tools?: ToolDefinition[],
-    systemPrompt?: string,
     options?: SendMessageOptions,
   ): Promise<ProviderResponse> {
-    const { config, onEvent, signal } = options ?? {};
+    const { tools, systemPrompt, config, onEvent, signal } = options ?? {};
     const configObj = config as Record<string, unknown> | undefined;
     const maxTokens = configObj?.max_tokens as number | undefined;
     const modelOverride = configObj?.model as string | undefined;
     const usageAttributionHeaders = configObj?.usageAttributionHeaders as
       | Record<string, string>
       | undefined;
-    const thinkingConfig = buildThinkingConfig(
-      configObj?.thinking as Record<string, unknown> | undefined,
-    );
     const activeModel = modelOverride ?? this.model;
+    const thinkingConfig = geminiModelSupportsThinking(activeModel)
+      ? buildThinkingConfig(
+          configObj?.thinking as Record<string, unknown> | undefined,
+          activeModel,
+        )
+      : undefined;
 
     try {
       const geminiContents = this.toGeminiContents(messages, activeModel);
@@ -275,6 +363,7 @@ export class GeminiProvider implements Provider {
       let finishReason = "unknown";
       let promptTokens = 0;
       let outputTokens = 0;
+      let cachedTokens = 0;
       let responseModel = activeModel;
 
       try {
@@ -322,6 +411,11 @@ export class GeminiProvider implements Provider {
           if (chunk.usageMetadata) {
             promptTokens = chunk.usageMetadata.promptTokenCount ?? 0;
             outputTokens = chunk.usageMetadata.candidatesTokenCount ?? 0;
+            // Gemini 2.5+/3.x cache a stable request prefix implicitly (on by
+            // default). promptTokenCount already includes these cached tokens,
+            // so cachedContentTokenCount is the read subset — surface it so the
+            // pricing layer applies the discounted cache-read rate.
+            cachedTokens = chunk.usageMetadata.cachedContentTokenCount ?? 0;
           }
 
           if (chunk.modelVersion) {
@@ -365,13 +459,18 @@ export class GeminiProvider implements Provider {
         usageMetadata: {
           promptTokenCount: promptTokens,
           candidatesTokenCount: outputTokens,
+          cachedContentTokenCount: cachedTokens,
         },
       };
 
       return {
         content,
         model: responseModel,
-        usage: { inputTokens: promptTokens, outputTokens },
+        usage: {
+          inputTokens: promptTokens,
+          outputTokens,
+          ...(cachedTokens > 0 ? { cacheReadInputTokens: cachedTokens } : {}),
+        },
         stopReason: finishReason,
         rawRequest,
         rawResponse,
@@ -435,7 +534,7 @@ export class GeminiProvider implements Provider {
 
     for (const msg of messages) {
       const role = msg.role === "assistant" ? "model" : "user";
-      const { parts, toolResultImageParts } = this.toGeminiParts(
+      const { parts, toolResultMediaParts } = this.toGeminiParts(
         msg.content,
         toolCallNames,
         model,
@@ -447,8 +546,8 @@ export class GeminiProvider implements Provider {
       // Gemini requires that a Content with functionResponse parts must not
       // contain non-functionResponse parts. Emit tool-result images in a
       // separate user Content entry.
-      if (toolResultImageParts.length > 0) {
-        result.push({ role: "user", parts: toolResultImageParts });
+      if (toolResultMediaParts.length > 0) {
+        result.push({ role: "user", parts: toolResultMediaParts });
       }
     }
 
@@ -461,9 +560,9 @@ export class GeminiProvider implements Provider {
     toolCallNames: Map<string, string>,
     model: string,
     role: "model" | "user",
-  ): { parts: genai.Part[]; toolResultImageParts: genai.Part[] } {
+  ): { parts: genai.Part[]; toolResultMediaParts: genai.Part[] } {
     const parts: genai.Part[] = [];
-    const toolResultImageParts: genai.Part[] = [];
+    const toolResultMediaParts: genai.Part[] = [];
 
     for (const block of blocks) {
       switch (block.type) {
@@ -478,14 +577,27 @@ export class GeminiProvider implements Provider {
             },
           });
           break;
-        case "file":
+        case "file": {
           if (this.supportsGeminiInlineFile(block.source.media_type)) {
-            parts.push({
-              inlineData: {
-                mimeType: block.source.media_type,
-                data: block.source.data,
-              },
-            });
+            // Normalize audio MIME onto Gemini's spelling (e.g. audio/mpeg →
+            // audio/mp3); PDFs pass through unchanged. Guard the 20 MB inline
+            // request limit for audio so an oversize clip degrades to a text
+            // note rather than 400ing the whole request.
+            const audioMime = normalizeGeminiAudioMime(block.source.media_type);
+            const rawBytes = base64ByteLength(block.source.data);
+            if (audioMime && rawBytes > GEMINI_MAX_INLINE_AUDIO_BYTES) {
+              const approxMb = Math.round(rawBytes / (1024 * 1024));
+              parts.push({
+                text: `[Audio file too large to send inline: ${block.source.filename} (${block.source.media_type}, ~${approxMb}MB). Gemini's inline request limit is 20MB; this file was omitted. Ask the user for a shorter clip.]`,
+              });
+            } else {
+              parts.push({
+                inlineData: {
+                  mimeType: audioMime ?? block.source.media_type,
+                  data: block.source.data,
+                },
+              });
+            }
           } else {
             const fallback = block.extracted_text?.trim()
               ? `[Attached file: ${block.source.filename} (${block.source.media_type})]\n${block.extracted_text}`
@@ -493,6 +605,7 @@ export class GeminiProvider implements Provider {
             parts.push({ text: fallback });
           }
           break;
+        }
         case "tool_use":
           {
             const functionCallPart: genai.Part = {
@@ -521,16 +634,39 @@ export class GeminiProvider implements Provider {
             if (extraText.length > 0) {
               outputText = outputText + "\n" + extraText.join("\n");
             }
-            // Collect images separately — Gemini rejects mixing inlineData
-            // with functionResponse in the same Content entry.
+            // Collect images and inline-able audio separately — Gemini rejects
+            // mixing inlineData with functionResponse in the same Content entry.
             for (const cb of block.contentBlocks) {
               if (cb.type === "image") {
-                toolResultImageParts.push({
+                toolResultMediaParts.push({
                   inlineData: {
                     mimeType: cb.source.media_type,
                     data: cb.source.data,
                   },
                 });
+              } else if (cb.type === "file") {
+                const audioMime = normalizeGeminiAudioMime(
+                  cb.source.media_type,
+                );
+                if (
+                  audioMime &&
+                  base64ByteLength(cb.source.data) <=
+                    GEMINI_MAX_INLINE_AUDIO_BYTES
+                ) {
+                  toolResultMediaParts.push({
+                    inlineData: { mimeType: audioMime, data: cb.source.data },
+                  });
+                } else if (audioMime) {
+                  // Oversize audio: note it in the functionResponse output
+                  // rather than a media part (a text part can't ride the
+                  // separate media Content, and inline audio would blow
+                  // Gemini's request-size limit).
+                  outputText =
+                    outputText +
+                    `\n[Audio too large to send inline: ${cb.source.filename}. Ask for a shorter clip.]`;
+                }
+                // Non-inline-able file sub-blocks (m4a/opus/pdf) are skipped
+                // here; the tool's text output already conveys the file.
               }
             }
           }
@@ -556,7 +692,7 @@ export class GeminiProvider implements Provider {
       this.addGemini3UnsignedToolCallFallback(parts, model);
     }
 
-    return { parts, toolResultImageParts };
+    return { parts, toolResultMediaParts };
   }
 
   private addGemini3UnsignedToolCallFallback(
@@ -580,6 +716,9 @@ export class GeminiProvider implements Provider {
   }
 
   private supportsGeminiInlineFile(mimeType: string): boolean {
-    return mimeType === "application/pdf";
+    return (
+      mimeType === "application/pdf" ||
+      normalizeGeminiAudioMime(mimeType) !== null
+    );
   }
 }

package/src/providers/gemini/inline-media.ts

@@ -0,0 +1,74 @@
+/**
+ * Shared source of truth for how Gemini ingests inline audio: which MIME types
+ * it accepts, how our stored MIME types normalize onto Gemini's spelling, the
+ * inline-request size ceiling, and the token cost.
+ *
+ * Imported by both the Gemini serializer (`client.ts`) and the token estimator
+ * (`context/token-estimator.ts`) so the two cannot drift — if they disagreed on
+ * which audio is sent inline, the context budgeter would mis-count what
+ * actually goes on the wire.
+ *
+ * Ref: https://ai.google.dev/gemini-api/docs/audio
+ *   - Inline-accepted audio: wav, mp3, aiff, aac, ogg, flac
+ *   - Inline request size limit: 20 MB total (prompt + system + inline files)
+ *   - Audio is billed at ~32 tokens/second
+ */
+
+/**
+ * Audio MIME types Gemini accepts as inline data, in Gemini's own spelling
+ * (the values we emit on the wire). Our upload allowlist stores mp3 as
+ * `audio/mpeg`; {@link normalizeGeminiAudioMime} maps that onto `audio/mp3`.
+ */
+export const GEMINI_INLINE_AUDIO_MIME_TYPES = new Set([
+  "audio/wav",
+  "audio/mp3",
+  "audio/aiff",
+  "audio/aac",
+  "audio/ogg",
+  "audio/flac",
+]);
+
+/**
+ * Map a stored attachment MIME type onto Gemini's inline-audio spelling, or
+ * `null` when Gemini cannot take it inline.
+ *
+ * m4a (`audio/x-m4a`, `audio/mp4`) and opus are intentionally unsupported:
+ * m4a bytes are an MP4 container, not raw AAC, so we must not relabel them as
+ * `audio/aac` — they fall through to a text placeholder instead.
+ */
+export function normalizeGeminiAudioMime(mimeType: string): string | null {
+  const normalized = mimeType.toLowerCase().split(";")[0]?.trim() ?? "";
+  const remapped = normalized === "audio/mpeg" ? "audio/mp3" : normalized;
+  return GEMINI_INLINE_AUDIO_MIME_TYPES.has(remapped) ? remapped : null;
+}
+
+/** Raw byte length backing a base64 string (4 base64 chars → 3 bytes). */
+export function base64ByteLength(base64Data: string): number {
+  return Math.floor((base64Data.length * 3) / 4);
+}
+
+/**
+ * Max raw bytes of audio we send inline to Gemini. Gemini's total inline
+ * request limit is 20 MB and base64 inflates payloads ~33%, so we cap raw
+ * audio at ~12 MB (≈16 MB encoded) to leave headroom for the prompt, system
+ * instruction, and conversation history. Larger files degrade to a text
+ * placeholder (the Gemini Files API is a future enhancement).
+ */
+export const GEMINI_MAX_INLINE_AUDIO_BYTES = 12 * 1024 * 1024;
+
+/**
+ * Gemini bills audio at ~32 tokens/second, NOT by payload size. We don't have
+ * the decoded duration at estimate time, so approximate it from byte size
+ * assuming ~128 kbps (~16 KB/s) typical compressed audio. This keeps the
+ * context budgeter within range instead of the ~170x over-count that treating
+ * the base64 payload as text produces. (Uncompressed WAV is under-counted, but
+ * the inline size guard caps it at {@link GEMINI_MAX_INLINE_AUDIO_BYTES}.)
+ */
+const GEMINI_AUDIO_TOKENS_PER_SECOND = 32;
+const APPROX_AUDIO_BYTES_PER_SECOND = 16_000;
+
+export function estimateGeminiAudioTokens(base64Data: string): number {
+  const approxSeconds =
+    base64ByteLength(base64Data) / APPROX_AUDIO_BYTES_PER_SECOND;
+  return Math.ceil(approxSeconds * GEMINI_AUDIO_TOKENS_PER_SECOND);
+}

package/src/providers/inference/__tests__/adapter-factory-openai-compatible.test.ts

@@ -24,7 +24,6 @@ describe("openai-compatible adapter factory", () => {
       name: "my-vllm",
       provider: "openai-compatible",
       auth: { type: "api_key", credential: "cred-vllm" },
-      status: "active",
       label: "vLLM",
       baseUrl: "http://localhost:8080/v1",
       models: [{ id: "my-model" }],
@@ -52,7 +51,6 @@ describe("openai-compatible adapter factory", () => {
       name: "my-vllm",
       provider: "openai-compatible",
       auth: { type: "none" },
-      status: "active",
       label: null,
       baseUrl: "http://localhost:8080/v1",
       models: [{ id: "my-model" }],

package/src/providers/inference/__tests__/base-url-security.test.ts

@@ -7,6 +7,7 @@ import { migrateCreateProviderConnections } from "../../../memory/migrations/243
 import { migrateProviderConnectionStatusLabel } from "../../../memory/migrations/244-provider-connection-status-label.js";
 import { migrateProviderConnectionBaseUrlAndModels } from "../../../memory/migrations/250-provider-connection-base-url-and-models.js";
 import { migrateStripBaseUrlNonOpenaiCompatible } from "../../../memory/migrations/257-strip-base-url-non-openai-compatible.js";
+import { migrateDropProviderConnectionStatus } from "../../../memory/migrations/265-drop-provider-connection-status.js";
 import * as schema from "../../../memory/schema.js";
 import { providerConnections } from "../../../memory/schema/inference.js";
 import { getConnection } from "../connections.js";
@@ -22,6 +23,7 @@ function bootDb() {
   const db = createTestDb();
   migrateCreateProviderConnections(db);
   migrateProviderConnectionStatusLabel(db);
+  migrateDropProviderConnectionStatus(db);
   migrateProviderConnectionBaseUrlAndModels(db);
   return db;
 }
@@ -42,7 +44,6 @@ describe("migration 257: strip base_url from non-openai-compatible connections",
         name: "bad-anthropic",
         provider: "anthropic",
         auth: JSON.stringify({ type: "api_key", credential: "cred-abc" }),
-        status: "active",
         baseUrl: "https://evil.example.com/v1",
         createdAt: now,
         updatedAt: now,
@@ -65,7 +66,6 @@ describe("migration 257: strip base_url from non-openai-compatible connections",
         name: "good-vllm",
         provider: "openai-compatible",
         auth: JSON.stringify({ type: "api_key", credential: "cred-vllm" }),
-        status: "active",
         baseUrl: "https://my-vllm.example.com/v1",
         models: JSON.stringify([{ id: "my-model" }]),
         createdAt: now,
@@ -89,7 +89,6 @@ describe("migration 257: strip base_url from non-openai-compatible connections",
         name: "bad-openai",
         provider: "openai",
         auth: JSON.stringify({ type: "api_key", credential: "cred-abc" }),
-        status: "active",
         baseUrl: "https://evil.example.com/v1",
         createdAt: now,
         updatedAt: now,