@vellumai/assistant 0.10.12 → 0.11.0-staging.2

package/src/providers/content-block-schema.ts

@@ -131,6 +131,31 @@ const webSearchToolResultContentSchema = z
   })
   .passthrough();
 
+/**
+ * Client-rendered surface cards. `data`, `actions`, and `display` stay loose
+ * because the concrete shape is chosen per `surfaceType` by
+ * `daemon/message-types/surfaces.ts` — validating them strictly here would
+ * reject renderable surfaces at the read boundary, which is exactly the
+ * failure this variant exists to prevent.
+ *
+ * `display` is a bare string, not the `inline`/`panel` enum the
+ * `ui_surface_show` WIRE event uses: persisted surfaces carry whatever the
+ * `ui_show` tool wrote (`CurrentTurnSurface.display` is `string`), so an
+ * LLM-chosen value must not fail the row.
+ */
+const uiSurfaceContentSchema = z
+  .object({
+    type: z.literal("ui_surface"),
+    surfaceId: z.string(),
+    surfaceType: z.string(),
+    title: z.string().optional(),
+    data: z.record(z.string(), z.unknown()).optional(),
+    actions: z.array(z.unknown()).optional(),
+    display: z.string().optional(),
+    completed: z.boolean().optional(),
+  })
+  .passthrough();
+
 export const contentBlockSchema: z.ZodType<ContentBlock> = z.discriminatedUnion(
   "type",
   [
@@ -143,6 +168,7 @@ export const contentBlockSchema: z.ZodType<ContentBlock> = z.discriminatedUnion(
     toolResultContentSchema,
     serverToolUseContentSchema,
     webSearchToolResultContentSchema,
+    uiSurfaceContentSchema,
   ],
 );
 

package/src/providers/inference/__tests__/connections-credential-normalize.test.ts

@@ -0,0 +1,87 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { migrateCreateProviderConnections } from "../../../persistence/migrations/243-provider-connections.js";
+import { migrateProviderConnectionStatusLabel } from "../../../persistence/migrations/244-provider-connection-status-label.js";
+import { migrateProviderConnectionBaseUrlAndModels } from "../../../persistence/migrations/250-provider-connection-base-url-and-models.js";
+import { migrateDropProviderConnectionStatus } from "../../../persistence/migrations/265-drop-provider-connection-status.js";
+import * as schema from "../../../persistence/schema/index.js";
+import { providerConnections } from "../../../persistence/schema/inference.js";
+import { normalizeCredentialRef } from "../../../security/credential-key.js";
+import { createConnection, getConnection } from "../connections.js";
+
+function bootDb() {
+  const sqlite = new Database(":memory:");
+  sqlite.exec("PRAGMA journal_mode=WAL");
+  const db = drizzle(sqlite, { schema });
+  migrateCreateProviderConnections(db);
+  migrateProviderConnectionStatusLabel(db);
+  migrateDropProviderConnectionStatus(db);
+  migrateProviderConnectionBaseUrlAndModels(db);
+  return db;
+}
+
+describe("normalizeCredentialRef", () => {
+  test("maps the secrets-API wire name to the vault key", () => {
+    expect(normalizeCredentialRef("openrouter:api_key")).toBe(
+      "credential/openrouter/api_key",
+    );
+  });
+
+  test("splits compound service names at the last colon", () => {
+    expect(normalizeCredentialRef("integration:google:access_token")).toBe(
+      "credential/integration:google/access_token",
+    );
+  });
+
+  test("leaves vault keys and other refs untouched", () => {
+    expect(normalizeCredentialRef("credential/openrouter/api_key")).toBe(
+      "credential/openrouter/api_key",
+    );
+    expect(normalizeCredentialRef("openrouter")).toBe("openrouter");
+    expect(normalizeCredentialRef("openrouter:")).toBe("openrouter:");
+    expect(normalizeCredentialRef(":api_key")).toBe(":api_key");
+  });
+});
+
+describe("connection credential normalization", () => {
+  test("createConnection stores the vault-key form for a wire-name credential", () => {
+    const db = bootDb();
+    const result = createConnection(db, {
+      name: "openrouter-conn",
+      provider: "openrouter",
+      auth: { type: "api_key", credential: "openrouter:api_key" },
+    });
+    expect(result.ok).toBe(true);
+    if (result.ok && result.connection.auth.type === "api_key") {
+      expect(result.connection.auth.credential).toBe(
+        "credential/openrouter/api_key",
+      );
+    }
+  });
+
+  test("getConnection heals a stored wire-name credential without a migration", () => {
+    const db = bootDb();
+    const now = Date.now();
+    db.insert(providerConnections)
+      .values({
+        name: "legacy-openrouter",
+        provider: "openrouter",
+        auth: JSON.stringify({
+          type: "api_key",
+          credential: "openrouter:api_key",
+        }),
+        createdAt: now,
+        updatedAt: now,
+      })
+      .run();
+
+    const conn = getConnection(db, "legacy-openrouter");
+    expect(conn).not.toBeNull();
+    if (conn?.auth.type === "api_key") {
+      expect(conn.auth.credential).toBe("credential/openrouter/api_key");
+    }
+  });
+});

package/src/providers/inference/adapter-factory.ts

@@ -29,6 +29,7 @@ import { OllamaProvider } from "../ollama/client.js";
 import { OpenAIChatCompletionsProvider } from "../openai/chat-completions-provider.js";
 import { OpenAIResponsesProvider } from "../openai/responses-provider.js";
 import { OpenRouterProvider } from "../openrouter/client.js";
+import { PoolsideProvider } from "../poolside/client.js";
 import { RetryProvider } from "../retry.js";
 import { TogetherProvider } from "../together/client.js";
 import type { Provider } from "../types.js";
@@ -151,6 +152,11 @@ const ADAPTER_FACTORIES: Record<string, AdapterFactory> = {
       streamTimeoutMs,
       ...(baseURL ? { baseURL } : {}),
     }),
+  poolside: ({ apiKey, model, streamTimeoutMs, baseURL }) =>
+    new PoolsideProvider(apiKey, model, {
+      streamTimeoutMs,
+      ...(baseURL ? { baseURL } : {}),
+    }),
 };
 
 /**

package/src/providers/inference/connections.ts

@@ -3,6 +3,7 @@ import { z } from "zod";
 
 import type { DrizzleDb } from "../../persistence/db-connection.js";
 import { providerConnections } from "../../persistence/schema/inference.js";
+import { normalizeCredentialRef } from "../../security/credential-key.js";
 import { getLogger } from "../../util/logger.js";
 import { clearConnectionProviderCache } from "../registry.js";
 import {
@@ -29,6 +30,21 @@ const log = getLogger("providers/inference/connections");
 // Helpers
 // ---------------------------------------------------------------------------
 
+/**
+ * Parse an auth value (stored JSON or caller input), normalizing any
+ * credential reference to the vault-key form. Normalizing on read as well as
+ * write heals rows that were stored with the secrets-API wire name (e.g.
+ * `openrouter:api_key`) without a migration.
+ */
+function parseAuth(raw: unknown): Auth | null {
+  const parsed = AuthSchema.safeParse(raw);
+  if (!parsed.success) return null;
+  const auth = parsed.data;
+  return "credential" in auth
+    ? { ...auth, credential: normalizeCredentialRef(auth.credential) }
+    : auth;
+}
+
 function parseModelsColumn(raw: string | null): ConnectionModel[] | null {
   if (raw === null || raw === "") return null;
   try {
@@ -56,14 +72,14 @@ export function listConnections(
     : db.select().from(providerConnections).all();
 
   return rows.flatMap((row) => {
-    const auth = AuthSchema.safeParse(JSON.parse(row.auth));
-    if (!auth.success) return [];
+    const auth = parseAuth(JSON.parse(row.auth));
+    if (!auth) return [];
     const provider = ConnectionProviderSchema.safeParse(row.provider);
     if (!provider.success) return [];
     return [
       {
         ...row,
-        auth: auth.data,
+        auth,
         provider: provider.data,
         label: row.label ?? null,
         baseUrl: row.baseUrl ?? null,
@@ -85,13 +101,13 @@ export function getConnection(
     .get();
 
   if (!row) return null;
-  const auth = AuthSchema.safeParse(JSON.parse(row.auth));
-  if (!auth.success) return null;
+  const auth = parseAuth(JSON.parse(row.auth));
+  if (!auth) return null;
   const provider = ConnectionProviderSchema.safeParse(row.provider);
   if (!provider.success) return null;
   return {
     ...row,
-    auth: auth.data,
+    auth,
     provider: provider.data,
     label: row.label ?? null,
     baseUrl: row.baseUrl ?? null,
@@ -152,8 +168,8 @@ export function createConnection(
   // Safe cast: VALID_CONNECTION_PROVIDERS.includes() guards above.
   const provider = input.provider as ConnectionProvider;
 
-  const authResult = AuthSchema.safeParse(input.auth);
-  if (!authResult.success) {
+  const auth = parseAuth(input.auth);
+  if (!auth) {
     return { ok: false, error: { code: "invalid_auth" } };
   }
 
@@ -182,7 +198,7 @@ export function createConnection(
     .values({
       name: input.name,
       provider,
-      auth: JSON.stringify(authResult.data),
+      auth: JSON.stringify(auth),
       label,
       baseUrl,
       models: models === null ? null : JSON.stringify(models),
@@ -200,7 +216,7 @@ export function createConnection(
     connection: {
       name: input.name,
       provider,
-      auth: authResult.data,
+      auth,
       label,
       baseUrl,
       models,
@@ -223,8 +239,8 @@ export function updateConnection(
     return { ok: false, error: { code: "not_found" } };
   }
 
-  const authResult = AuthSchema.safeParse(input.auth);
-  if (!authResult.success) {
+  const auth = parseAuth(input.auth);
+  if (!auth) {
     return { ok: false, error: { code: "invalid_auth" } };
   }
 
@@ -248,7 +264,7 @@ export function updateConnection(
     label?: string | null;
     baseUrl?: string | null;
     models?: string | null;
-  } = { auth: JSON.stringify(authResult.data), updatedAt: now };
+  } = { auth: JSON.stringify(auth), updatedAt: now };
   if (input.label !== undefined) setClause.label = input.label;
   if (input.baseUrl !== undefined) setClause.baseUrl = input.baseUrl;
   if (input.models !== undefined)
@@ -267,7 +283,7 @@ export function updateConnection(
     ok: true,
     connection: {
       ...existing,
-      auth: authResult.data,
+      auth,
       label: input.label !== undefined ? input.label : existing.label,
       baseUrl: nextBaseUrl,
       models: nextModels,

package/src/providers/model-catalog.ts

@@ -52,6 +52,13 @@ export interface CatalogModel {
   adaptiveThinkingOnly?: boolean;
   supportsCaching?: boolean;
   supportsVision?: boolean;
+  /**
+   * The model's serving surface accepts OpenAI chat-completions `input_audio`
+   * content parts (base64 wav/mp3), so eligible audio attachments are sent
+   * inline instead of as a text placeholder. Daemon-only: not projected into
+   * the client catalog (see scripts/sync-llm-catalog.ts).
+   */
+  supportsAudioInput?: boolean;
   supportsToolUse?: boolean;
   pricing?: CatalogModelPricing;
   /**
@@ -825,6 +832,22 @@ const RAW_PROVIDER_CATALOG: ProviderCatalogEntry[] = [
       linkLabel: "Open Fireworks Dashboard",
     },
     models: [
+      {
+        id: "accounts/fireworks/models/kimi-k3",
+        displayName: "Kimi K3",
+        contextWindowTokens: 1048576,
+        maxOutputTokens: 131072,
+        supportsThinking: true,
+        adaptiveThinkingOnly: true,
+        supportsCaching: true,
+        supportsVision: true,
+        supportsToolUse: true,
+        pricing: {
+          inputPer1mTokens: 3,
+          outputPer1mTokens: 15,
+          cacheReadPer1mTokens: 0.3,
+        },
+      },
       {
         id: "accounts/fireworks/models/kimi-k2p6",
         displayName: "Kimi K2.6",
@@ -2082,6 +2105,9 @@ const RAW_PROVIDER_CATALOG: ProviderCatalogEntry[] = [
         supportsThinking: true,
         supportsCaching: true,
         supportsVision: true,
+        // Inkling ingests audio natively (dMel tokens); Baseten's serving
+        // surface accepts `input_audio` parts for it.
+        supportsAudioInput: true,
         supportsToolUse: true,
         // Baseten's reasoning_effort for Inkling tops out at "xhigh" (no
         // "max"), matching the chat-completions client's default ceiling.
@@ -2097,6 +2123,45 @@ const RAW_PROVIDER_CATALOG: ProviderCatalogEntry[] = [
     apiKeyUrl: "https://app.baseten.co/settings/api_keys",
     apiKeyPlaceholder: "Your Baseten API key",
   },
+  {
+    id: "poolside",
+    displayName: "Poolside",
+    subtitle:
+      "Laguna models from Poolside (OpenAI-compatible). Requires a Poolside API key.",
+    setupMode: "api-key",
+    setupHint: "Enter your Poolside API key to enable Laguna models.",
+    envVar: "POOLSIDE_API_KEY",
+    credentialsGuide: {
+      description: "Sign in to Poolside and create an API key.",
+      url: "https://poolside.ai",
+      linkLabel: "Open Poolside",
+    },
+    models: [
+      {
+        id: "poolside/laguna-s-2.1",
+        displayName: "Laguna S 2.1",
+        contextWindowTokens: 1050000,
+        maxOutputTokens: 131072,
+        supportsThinking: true,
+        supportsCaching: false,
+        supportsVision: false,
+        supportsToolUse: true,
+      },
+      {
+        id: "poolside/laguna-xs-2.1",
+        displayName: "Laguna XS 2.1",
+        contextWindowTokens: 262144,
+        maxOutputTokens: 32768,
+        supportsThinking: true,
+        supportsCaching: false,
+        supportsVision: false,
+        supportsToolUse: true,
+      },
+    ],
+    defaultModel: "poolside/laguna-s-2.1",
+    apiKeyUrl: "https://poolside.ai",
+    apiKeyPlaceholder: "Your Poolside API key",
+  },
 ];
 
 export const PROVIDER_CATALOG: ProviderCatalogEntry[] =
@@ -2170,3 +2235,15 @@ export function isAdaptiveThinkingOnlyModel(modelId: string): boolean {
     p.models.some((m) => m.id === modelId && m.adaptiveThinkingOnly === true),
   );
 }
+
+/**
+ * Whether a model's serving surface accepts OpenAI chat-completions
+ * `input_audio` content parts, driven by the `supportsAudioInput` capability
+ * in the catalog. Matches the model ID across every provider (same pattern as
+ * {@link isAdaptiveThinkingOnlyModel}).
+ */
+export function modelSupportsAudioInput(modelId: string): boolean {
+  return PROVIDER_CATALOG.some((p) =>
+    p.models.some((m) => m.id === modelId && m.supportsAudioInput === true),
+  );
+}

package/src/providers/openai/__tests__/chat-completions-input-audio.test.ts

@@ -0,0 +1,260 @@
+/**
+ * Serialization tests for capability-gated native audio input on
+ * OpenAI-compatible providers: `file` blocks carrying eligible audio reach
+ * audio-capable models (catalog `supportsAudioInput`, e.g. Inkling) as
+ * `input_audio` content parts, while every other model keeps the
+ * text-placeholder serialization byte-for-byte.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import type { ContentBlock } from "../../types.js";
+import { OpenAIChatCompletionsProvider } from "../chat-completions-provider.js";
+
+type MockChunk = {
+  choices: Array<{
+    delta: { content?: string | null };
+    finish_reason?: string | null;
+  }>;
+  usage?: { prompt_tokens: number; completion_tokens: number };
+};
+
+const OK_CHUNKS: MockChunk[] = [
+  {
+    choices: [{ delta: { content: "ok" }, finish_reason: "stop" }],
+    usage: { prompt_tokens: 1, completion_tokens: 1 },
+  },
+];
+
+function makeStream(chunks: MockChunk[]): AsyncIterable<MockChunk> {
+  return {
+    async *[Symbol.asyncIterator]() {
+      for (const c of chunks) yield c;
+    },
+  };
+}
+
+type CapturedParams = {
+  messages: Array<{
+    role: string;
+    content: string | Array<Record<string, unknown>>;
+  }>;
+};
+
+/** Swap `create` for a canned stream that records the request params. */
+function stubCreateCapture(provider: OpenAIChatCompletionsProvider): {
+  params: () => CapturedParams;
+} {
+  let captured: unknown;
+  const inner = provider as unknown as {
+    client: {
+      chat: {
+        completions: {
+          create: (params: unknown) => Promise<AsyncIterable<MockChunk>>;
+        };
+      };
+    };
+  };
+  inner.client.chat.completions.create = async (params) => {
+    captured = params;
+    return makeStream(OK_CHUNKS);
+  };
+  return { params: () => captured as CapturedParams };
+}
+
+// In the catalog with `supportsAudioInput: true` (Baseten's Inkling).
+const AUDIO_MODEL = "thinkingmachines/inkling";
+// Not in the catalog — must keep the text-placeholder serialization.
+const TEXT_MODEL = "test-model";
+
+function makeProvider(model: string): OpenAIChatCompletionsProvider {
+  return new OpenAIChatCompletionsProvider("test-key", model, {
+    providerName: "baseten",
+    providerLabel: "Baseten",
+  });
+}
+
+function audioFileBlock(mediaType: string, data = "QUJDRA=="): ContentBlock {
+  return {
+    type: "file",
+    source: {
+      type: "base64",
+      media_type: mediaType,
+      data,
+      filename: "clip.audio",
+    },
+  };
+}
+
+function userContentParts(params: CapturedParams): Record<string, unknown>[] {
+  return params.messages
+    .filter((m) => m.role === "user" && Array.isArray(m.content))
+    .flatMap((m) => m.content as Array<Record<string, unknown>>);
+}
+
+describe("chat-completions input_audio serialization", () => {
+  test("wav file block on an audio-capable model becomes an input_audio part", async () => {
+    const provider = makeProvider(AUDIO_MODEL);
+    const captured = stubCreateCapture(provider);
+
+    await provider.sendMessage([
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "can you hear this?" },
+          audioFileBlock("audio/wav"),
+        ],
+      },
+    ]);
+
+    const parts = userContentParts(captured.params());
+    expect(parts).toContainEqual({
+      type: "input_audio",
+      input_audio: { data: "QUJDRA==", format: "wav" },
+    });
+    expect(JSON.stringify(parts)).not.toContain("<attached_file");
+  });
+
+  test("mp3 mime maps onto the mp3 format enum", async () => {
+    const provider = makeProvider(AUDIO_MODEL);
+    const captured = stubCreateCapture(provider);
+
+    await provider.sendMessage([
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "listen" },
+          audioFileBlock("audio/mpeg"),
+        ],
+      },
+    ]);
+
+    const parts = userContentParts(captured.params());
+    expect(parts).toContainEqual({
+      type: "input_audio",
+      input_audio: { data: "QUJDRA==", format: "mp3" },
+    });
+  });
+
+  test("audio on a model without the capability keeps the text placeholder", async () => {
+    const provider = makeProvider(TEXT_MODEL);
+    const captured = stubCreateCapture(provider);
+
+    await provider.sendMessage([
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "can you hear this?" },
+          audioFileBlock("audio/wav"),
+        ],
+      },
+    ]);
+
+    const parts = userContentParts(captured.params());
+    expect(JSON.stringify(parts)).not.toContain("input_audio");
+    expect(parts).toContainEqual({
+      type: "text",
+      text: `<attached_file name="clip.audio" type="audio/wav" />\nNo extracted text available.`,
+    });
+  });
+
+  test("oversize audio falls back to the text placeholder", async () => {
+    const provider = makeProvider(AUDIO_MODEL);
+    const captured = stubCreateCapture(provider);
+
+    // ~12.75 MB decoded (> 12 MB inline cap) without being a real payload.
+    const oversize = "A".repeat(17_000_000);
+    await provider.sendMessage([
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "big one" },
+          audioFileBlock("audio/wav", oversize),
+        ],
+      },
+    ]);
+
+    const parts = userContentParts(captured.params());
+    expect(JSON.stringify(parts)).not.toContain("input_audio");
+  });
+
+  test("unsupported audio mime (m4a) falls back to the text placeholder", async () => {
+    const provider = makeProvider(AUDIO_MODEL);
+    const captured = stubCreateCapture(provider);
+
+    await provider.sendMessage([
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "voice memo" },
+          audioFileBlock("audio/x-m4a"),
+        ],
+      },
+    ]);
+
+    const parts = userContentParts(captured.params());
+    expect(JSON.stringify(parts)).not.toContain("input_audio");
+    expect(JSON.stringify(parts)).toContain("audio/x-m4a");
+  });
+
+  test("tool_result nested audio is hoisted into the trailing user message when capable", async () => {
+    const provider = makeProvider(AUDIO_MODEL);
+    const captured = stubCreateCapture(provider);
+
+    await provider.sendMessage([
+      { role: "user", content: [{ type: "text", text: "read the song file" }] },
+      {
+        role: "assistant",
+        content: [{ type: "tool_use", id: "t1", name: "file_read", input: {} }],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "t1",
+            content: "read 4 bytes",
+            contentBlocks: [audioFileBlock("audio/wav")],
+          },
+        ],
+      },
+    ]);
+
+    const params = captured.params();
+    const toolMessages = params.messages.filter((m) => m.role === "tool");
+    expect(toolMessages).toHaveLength(1);
+    expect(JSON.stringify(toolMessages)).not.toContain("input_audio");
+    expect(userContentParts(params)).toContainEqual({
+      type: "input_audio",
+      input_audio: { data: "QUJDRA==", format: "wav" },
+    });
+  });
+
+  test("tool_result nested audio stays dropped on a model without the capability", async () => {
+    const provider = makeProvider(TEXT_MODEL);
+    const captured = stubCreateCapture(provider);
+
+    await provider.sendMessage([
+      { role: "user", content: [{ type: "text", text: "read the song file" }] },
+      {
+        role: "assistant",
+        content: [{ type: "tool_use", id: "t1", name: "file_read", input: {} }],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "t1",
+            content: "read 4 bytes",
+            contentBlocks: [audioFileBlock("audio/wav")],
+          },
+        ],
+      },
+    ]);
+
+    const serialized = JSON.stringify(captured.params());
+    expect(serialized).not.toContain("input_audio");
+    expect(serialized).not.toContain("QUJDRA==");
+  });
+});