@vellumai/assistant 0.8.4 → 0.8.6

package/src/providers/__tests__/retry-callsite.test.ts

@@ -425,7 +425,7 @@ describe("RetryProvider — callSite resolution", () => {
     expect(config.temperature).toBe(0.5);
   });
 
-  test("strips effort/speed/thinking for providers that don't support them", async () => {
+  test("strips effort/speed for providers that don't support them (e.g. fireworks)", async () => {
     setLlmConfig({
       default: {
         provider: "anthropic",
@@ -434,14 +434,14 @@ describe("RetryProvider — callSite resolution", () => {
         speed: "fast",
       },
       callSites: {
-        memoryRetrieval: { thinking: { enabled: true } },
+        memoryRetrieval: { thinking: { enabled: false } },
       },
     });
 
     let seen: SendMessageOptions | undefined;
-    // gemini does not support effort/speed/thinking — they must be stripped.
+    // fireworks does not support speed or thinking — they must be stripped.
     const wrapped = new RetryProvider(
-      makeProvider("gemini", (options) => {
+      makeProvider("fireworks", (options) => {
         seen = options;
       }),
     );
@@ -451,13 +451,93 @@ describe("RetryProvider — callSite resolution", () => {
     });
 
     const config = seen?.config as Record<string, unknown>;
-    expect(config.effort).toBeUndefined();
     expect(config.speed).toBeUndefined();
     expect(config.thinking).toBeUndefined();
     // Model still comes through.
     expect(config.model).toBe("claude-opus-4-7");
   });
 
+  test("preserves thinking + level for Gemini provider", async () => {
+    setLlmConfig({
+      default: {
+        provider: "gemini",
+        model: "gemini-3.5-flash",
+        thinking: { enabled: true, streamThinking: true, level: "high" },
+      },
+      callSites: { mainAgent: {} },
+    });
+
+    let seen: SendMessageOptions | undefined;
+    const wrapped = new RetryProvider(
+      makeProvider("gemini", (options) => {
+        seen = options;
+      }),
+    );
+
+    await wrapped.sendMessage(DUMMY_MESSAGES, undefined, undefined, {
+      config: { callSite: "mainAgent" },
+    });
+
+    const config = seen?.config as Record<string, unknown>;
+    expect(config.thinking).toEqual({
+      type: "adaptive",
+      level: "high",
+      streamThinking: true,
+    });
+  });
+
+  test("Gemini disabled thinking carries the wire `disabled` discriminator", async () => {
+    setLlmConfig({
+      default: {
+        provider: "gemini",
+        model: "gemini-3.5-flash",
+        thinking: { enabled: false, streamThinking: false },
+      },
+      callSites: { mainAgent: {} },
+    });
+
+    let seen: SendMessageOptions | undefined;
+    const wrapped = new RetryProvider(
+      makeProvider("gemini", (options) => {
+        seen = options;
+      }),
+    );
+
+    await wrapped.sendMessage(DUMMY_MESSAGES, undefined, undefined, {
+      config: { callSite: "mainAgent" },
+    });
+
+    const config = seen?.config as Record<string, unknown>;
+    expect(config.thinking).toEqual({ type: "disabled" });
+  });
+
+  test("scrubs Gemini-only thinking extras (level, streamThinking) for Anthropic", async () => {
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-opus-4-7",
+        thinking: { enabled: true, streamThinking: true, level: "high" },
+      },
+      callSites: { mainAgent: {} },
+    });
+
+    let seen: SendMessageOptions | undefined;
+    const wrapped = new RetryProvider(
+      makeProvider("anthropic", (options) => {
+        seen = options;
+      }),
+    );
+
+    await wrapped.sendMessage(DUMMY_MESSAGES, undefined, undefined, {
+      config: { callSite: "mainAgent" },
+    });
+
+    const config = seen?.config as Record<string, unknown>;
+    // Anthropic's SDK rejects unknown keys inside the `thinking` object with
+    // "Extra inputs are not permitted" — must be exactly `{ type }`.
+    expect(config.thinking).toEqual({ type: "adaptive" });
+  });
+
   test("explicit per-call config.model wins over resolved callSite model", async () => {
     setLlmConfig({
       default: { provider: "anthropic", model: "resolved-model" },

package/src/providers/anthropic/client.ts

@@ -1,6 +1,5 @@
 import Anthropic from "@anthropic-ai/sdk";
 
-import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/system-prompt.js";
 import { isAbortReason } from "../../util/abort-reasons.js";
 import { ProviderError } from "../../util/errors.js";
 import { getLogger } from "../../util/logger.js";
@@ -819,6 +818,15 @@ export class AnthropicProvider implements Provider {
       ((config as Record<string, unknown> | undefined)?.cacheTtl as
         | "5m"
         | "1h") ?? "1h";
+    // Opt-out for callers (e.g. the memory router) that send a single
+    // user message per call with content that changes every time. The
+    // turn-start cache breakpoint below is only useful when the same
+    // prefix is re-sent on a subsequent call (typical for the main agent
+    // loop's tool-use iterations); one-shot callers pay cache_creation
+    // cost without a future hit.
+    const disableTurnStartCache =
+      (config as Record<string, unknown> | undefined)?.disableTurnStartCache ===
+      true;
     let sentMessages: Anthropic.MessageParam[] | undefined;
     const startedAt = Date.now();
     // Hoisted so the catch block can distinguish our inner stream timeout
@@ -980,14 +988,11 @@ export class AnthropicProvider implements Provider {
       // followed by user tool_result). Replaying stale thinking blocks from
       // earlier turns causes 400 errors when the signature is no longer
       // valid (e.g. after a provider/model/profile switch).
-      const activeToolUseStart =
-        findActiveToolUseContinuationStart(formatted);
+      const activeToolUseStart = findActiveToolUseContinuationStart(formatted);
       for (let i = 0; i < activeToolUseStart; i++) {
         const msg = formatted[i];
         if (msg.role !== "assistant" || !Array.isArray(msg.content)) continue;
-        const stripped = (
-          msg.content as Anthropic.ContentBlockParam[]
-        ).filter(
+        const stripped = (msg.content as Anthropic.ContentBlockParam[]).filter(
           (b) =>
             typeof b === "string" ||
             (b.type !== "thinking" && b.type !== "redacted_thinking"),
@@ -1009,6 +1014,7 @@ export class AnthropicProvider implements Provider {
         speed,
         output_config,
         cacheTtl: _cacheTtl,
+        disableTurnStartCache: _disableTurnStartCache,
         max_tokens: callerMaxTokens,
         usageAttributionHeaders,
         ...restConfig
@@ -1065,38 +1071,17 @@ export class AnthropicProvider implements Provider {
       };
 
       if (systemPrompt) {
-        const boundaryIdx = systemPrompt.indexOf(SYSTEM_PROMPT_CACHE_BOUNDARY);
-        if (boundaryIdx >= 0) {
-          // Split into two cache blocks: static instructions (stable across
-          // turns) and dynamic workspace content (changes when files are
-          // edited).  The static prefix stays cached even when workspace
-          // files change, saving ~8-10K tokens of cache creation per turn.
-          // Both blocks use 1-hour cache TTL to avoid repeated cache misses
-          // for conversations with turn gaps exceeding the default 5-minute
-          // window.
-          const staticBlock = systemPrompt.slice(0, boundaryIdx);
-          const dynamicBlock = systemPrompt.slice(
-            boundaryIdx + SYSTEM_PROMPT_CACHE_BOUNDARY.length,
-          );
-          const systemBlocks = [staticBlock, dynamicBlock]
-            .filter((text) => text.length > 0)
-            .map((text) => ({
-              type: "text" as const,
-              text,
-              cache_control: cacheControl,
-            }));
-          if (systemBlocks.length > 0) {
-            params.system = systemBlocks;
-          }
-        } else {
-          params.system = [
-            {
-              type: "text" as const,
-              text: systemPrompt,
-              cache_control: cacheControl,
-            },
-          ];
-        }
+        // The whole system prompt is rendered as a single cached
+        // block.  A 1-hour cache TTL is used (when supported by the
+        // model) so the breakpoint survives turn gaps that exceed the
+        // default 5-minute window.
+        params.system = [
+          {
+            type: "text" as const,
+            text: systemPrompt,
+            cache_control: cacheControl,
+          },
+        ];
       }
 
       if (tools && tools.length > 0) {
@@ -1160,7 +1145,9 @@ export class AnthropicProvider implements Provider {
         }
       };
       const turnStartIdx = findUserTextMsgIdx(msgs.length - 1);
-      if (turnStartIdx >= 0) applyCacheControlToLastBlock(turnStartIdx);
+      if (turnStartIdx >= 0 && !disableTurnStartCache) {
+        applyCacheControlToLastBlock(turnStartIdx);
+      }
 
       // Previous-turn anchor: when this request is the first of a new turn
       // (turn-start is the very last message — no tool-use loop yet), also
@@ -1172,9 +1159,8 @@ export class AnthropicProvider implements Provider {
       // cache_creation tokens per new turn). Skipped during tool-use loops
       // where the current turn-start already covers the same prefix and a
       // second anchor would blow the 4-breakpoint budget.
-      let prevTurnAnchorIdx = -1;
       if (turnStartIdx === msgs.length - 1 && turnStartIdx > 0) {
-        prevTurnAnchorIdx = findUserTextMsgIdx(turnStartIdx - 1);
+        const prevTurnAnchorIdx = findUserTextMsgIdx(turnStartIdx - 1);
         if (prevTurnAnchorIdx >= 0)
           applyCacheControlToLastBlock(prevTurnAnchorIdx);
       }
@@ -1185,7 +1171,6 @@ export class AnthropicProvider implements Provider {
       // cheaply without conflicting with the 1h breakpoints above.
       // Skip thinking/redacted_thinking blocks — Anthropic doesn't allow
       // cache_control on those types.
-      let tailBreakpointApplied = false;
       if (turnStartIdx >= 0 && turnStartIdx < sentMessages.length - 1) {
         const lastMsg = sentMessages[sentMessages.length - 1];
         if (Array.isArray(lastMsg.content) && lastMsg.content.length > 0) {
@@ -1207,34 +1192,15 @@ export class AnthropicProvider implements Provider {
           if (tailBlock && typeof tailBlock !== "string") {
             (tailBlock as unknown as Record<string, unknown>).cache_control =
               tailCacheControl;
-            tailBreakpointApplied = true;
           }
         }
       }
 
-      // Enforce Anthropic API maximum of 4 cache_control blocks.
-      // With the system prompt boundary split into 2 cached blocks AND
-      // tools + turn-start + (tail OR prev-turn-anchor), we'd have 5.
-      // Drop the static system block's breakpoint — it's small (<1K
-      // tokens) so the re-read cost is negligible, while the dynamic
-      // block (workspace context) rarely changes mid-session and
-      // benefits more from caching. Tail and prev-turn-anchor are
-      // mutually exclusive (prev-turn-anchor only fires when turn-start
-      // is the last message, which is the exact condition that suppresses
-      // the tail), so we never exceed 5.
-      const hasToolCacheBreakpoint =
-        params.tools?.some(
-          (t) => "cache_control" in t && t.cache_control != null,
-        ) ?? false;
-      if (
-        (tailBreakpointApplied || prevTurnAnchorIdx >= 0) &&
-        Array.isArray(params.system) &&
-        params.system.length === 2 &&
-        hasToolCacheBreakpoint
-      ) {
-        delete (params.system[0] as unknown as Record<string, unknown>)
-          .cache_control;
-      }
+      // Cache-breakpoint accounting: system(1) + tools(1) + turn-start(1) +
+      // (tail OR prev-turn-anchor)(1) = 4 — exactly Anthropic's per-request
+      // cap.  Tail and prev-turn-anchor are mutually exclusive (the latter
+      // only fires when turn-start is the last message, which suppresses
+      // the tail), so the total can't drift past 4.
 
       // Strip orphaned UTF-16 surrogates so the Anthropic JSON parser never
       // sees invalid strings produced by upstream surrogate-splitting `.slice()` calls.

package/src/providers/call-site-routing.ts

@@ -24,6 +24,10 @@ import { AsyncLocalStorage } from "node:async_hooks";
 import { resolveCallSiteConfig } from "../config/llm-resolver.js";
 import { getConfig } from "../config/loader.js";
 import { getDb } from "../memory/db-connection.js";
+import {
+  describeSubscriptionModelIncompatibility,
+  isConnectionCompatibleWithModel,
+} from "./connection-model-compat.js";
 import {
   ConnectionResolutionError,
   tryResolveProviderForConnectionName,
@@ -73,10 +77,15 @@ export class CallSiteRoutingProvider implements Provider {
      * `expectedProvider` is the provider name the resolved profile
      * declared. The hook verifies the connection's provider matches
      * and throws on mismatch.
+     *
+     * `model` is the resolved call-site model, threaded through so the
+     * connection lookup can gate `oauth_subscription` (Codex) connections
+     * by model compatibility.
      */
     private readonly resolveByConnection: (
       connectionName: string,
       expectedProvider: string,
+      model: string | undefined,
     ) => Promise<Provider | null>,
   ) {
     this.tokenEstimationProvider = defaultProvider.tokenEstimationProvider;
@@ -140,22 +149,33 @@ export class CallSiteRoutingProvider implements Provider {
     if (!callSite) return this.defaultProvider;
 
     const overrideProfile = options?.config?.overrideProfile;
+    // Forward the per-conversation mix seed so transport selection picks the
+    // same mix arm as wire-param normalization in `retry.ts` — otherwise a mix
+    // spanning providers could route the transport to a different arm than the
+    // request params.
+    const selectionSeed = options?.config?.selectionSeed;
     const resolved = resolveCallSiteConfig(callSite, getConfig().llm, {
       overrideProfile,
+      selectionSeed,
     });
 
     let connectionName = resolved.provider_connection;
 
     // When no connection is set and the provider differs from the default,
-    // auto-resolve to an active connection for the provider (handles the
-    // "Any active X connection" case where the profile set provider but
-    // not provider_connection, and the merge didn't inherit one).
+    // auto-resolve a connection for the provider (handles the case where the
+    // profile set provider but not provider_connection, and the merge didn't
+    // inherit one).
+    let autoResolveCandidates:
+      | import("./inference/auth.js").ProviderConnection[]
+      | undefined;
     if (!connectionName && resolved.provider !== this.defaultProvider.name) {
       try {
-        const candidates = listConnections(getDb(), {
+        autoResolveCandidates = listConnections(getDb(), {
           provider: resolved.provider,
         });
-        const active = candidates.find((c) => c.status === "active");
+        const active = autoResolveCandidates.find((c) =>
+          isConnectionCompatibleWithModel(c, resolved.model),
+        );
         if (active) {
           connectionName = active.name;
         }
@@ -168,6 +188,7 @@ export class CallSiteRoutingProvider implements Provider {
       const connectionProvider = await this.resolveByConnection(
         connectionName,
         resolved.provider,
+        resolved.model,
       );
       if (connectionProvider) return connectionProvider;
       return this.defaultProvider;
@@ -177,6 +198,20 @@ export class CallSiteRoutingProvider implements Provider {
       return this.defaultProvider;
     }
 
+    if (autoResolveCandidates) {
+      const incompatMsg = describeSubscriptionModelIncompatibility(
+        autoResolveCandidates,
+        resolved.model,
+      );
+      if (incompatMsg) {
+        throw new ConnectionResolutionError(
+          "<resolved-callsite>",
+          "model_incompatible",
+          incompatMsg,
+        );
+      }
+    }
+
     throw new ConnectionResolutionError(
       "<resolved-callsite>",
       "missing_connection",
@@ -200,11 +235,12 @@ export function wrapWithCallSiteRouting(
 ): Provider {
   return new CallSiteRoutingProvider(
     base,
-    (connectionName, expectedProvider) =>
+    (connectionName, expectedProvider, model) =>
       tryResolveProviderForConnectionName(
         connectionName,
         config,
         expectedProvider,
+        model,
       ),
   );
 }

package/src/providers/connection-model-compat.ts

@@ -0,0 +1,61 @@
+/**
+ * Model-compatibility gate for auto-resolved provider connections.
+ *
+ * When a profile uses "Any active <provider> connection" (no
+ * `provider_connection` pinned), the daemon auto-picks an active connection
+ * for the provider. `oauth_subscription` connections (ChatGPT Codex) hard-
+ * route every request to the Codex endpoint, which rejects non-Codex models
+ * with HTTP 400. This helper lets the auto-resolution sites skip such a
+ * connection when the requested model is not Codex-compatible.
+ */
+
+import type { ProviderConnection } from "./inference/auth.js";
+import { isCodexSubscriptionModel } from "./openai/codex-models.js";
+
+/**
+ * Whether `connection` can serve a request for `model` during
+ * auto-resolution.
+ *
+ * `oauth_subscription` connections route through the ChatGPT Codex endpoint,
+ * so they are only compatible with Codex models. Every other auth type
+ * imposes no model restriction and is always compatible.
+ *
+ * `model` may be undefined when the call site has no resolved model; in that
+ * case no model gating is applied (returns true) so resolution behaviour is
+ * unchanged.
+ *
+ * This gate applies to auto-resolution only — an explicitly pinned
+ * `provider_connection` bypasses connection selection entirely and is used
+ * regardless of model.
+ */
+export function isConnectionCompatibleWithModel(
+  connection: Pick<ProviderConnection, "auth">,
+  model: string | undefined,
+): boolean {
+  if (connection.auth.type !== "oauth_subscription") return true;
+  if (!model) return true;
+  return isCodexSubscriptionModel(model);
+}
+
+/**
+ * When auto-resolution found candidates but none were model-compatible,
+ * return a user-facing explanation if the incompatibility is specifically
+ * due to all candidates being `oauth_subscription` (ChatGPT) connections.
+ *
+ * Returns `null` when the incompatibility has a different cause (callers
+ * should fall through to their existing generic error).
+ */
+export function describeSubscriptionModelIncompatibility(
+  candidates: Pick<ProviderConnection, "auth">[],
+  model: string | undefined,
+): string | null {
+  if (!model || candidates.length === 0) return null;
+  if (candidates.some((c) => isConnectionCompatibleWithModel(c, model))) {
+    return null;
+  }
+  const allSubscription = candidates.every(
+    (c) => c.auth.type === "oauth_subscription",
+  );
+  if (!allSubscription) return null;
+  return `Model "${model}" isn't available through your ChatGPT subscription. Select a supported model or add an OpenAI API key connection.`;
+}

package/src/providers/connection-resolution.ts

@@ -30,6 +30,10 @@
 import { resolveCallSiteConfig } from "../config/llm-resolver.js";
 import { getDb } from "../memory/db-connection.js";
 import { getLogger } from "../util/logger.js";
+import {
+  describeSubscriptionModelIncompatibility,
+  isConnectionCompatibleWithModel,
+} from "./connection-model-compat.js";
 import { getConnection, listConnections } from "./inference/connections.js";
 import type { ProvidersConfig } from "./registry.js";
 import { resolveProviderFromConnection } from "./registry.js";
@@ -51,7 +55,8 @@ export class ConnectionResolutionError extends Error {
       | "lookup_failed"
       | "not_found"
       | "provider_mismatch"
-      | "missing_connection",
+      | "missing_connection"
+      | "model_incompatible",
     message: string,
     public readonly cause?: unknown,
   ) {
@@ -79,11 +84,16 @@ export class ConnectionResolutionError extends Error {
  * `expectedProvider` is the provider name the resolving profile declared.
  * Pass `undefined` to skip the mismatch check (callers that don't yet
  * know the expected provider).
+ *
+ * `model` is the resolved call-site model. It gates the `provider_mismatch`
+ * auto-recovery below so a non-Codex model is never rerouted onto an
+ * `oauth_subscription` (ChatGPT Codex) connection.
  */
 export async function tryResolveProviderForConnectionName(
   connectionName: string,
   config: ProvidersConfig,
   expectedProvider?: string,
+  model?: string,
 ): Promise<Provider | null> {
   let connection;
   try {
@@ -110,10 +120,13 @@ export async function tryResolveProviderForConnectionName(
     // "anthropic-managed" leaked through). Try to find an active connection
     // for the expected provider before giving up.
     let resolved = false;
+    let mismatchCandidates: import("./inference/auth.js").ProviderConnection[] | undefined;
     try {
       const db = getDb();
-      const candidates = listConnections(db, { provider: expectedProvider });
-      const active = candidates.find((c) => c.status === "active");
+      mismatchCandidates = listConnections(db, { provider: expectedProvider });
+      const active = mismatchCandidates.find((c) =>
+        isConnectionCompatibleWithModel(c, model),
+      );
       if (active) {
         log.info(
           {
@@ -121,7 +134,7 @@ export async function tryResolveProviderForConnectionName(
             resolvedConnection: active.name,
             expectedProvider,
           },
-          "Auto-resolved stale provider_connection to matching active connection",
+          "Auto-resolved stale provider_connection to matching connection",
         );
         connection = active;
         resolved = true;
@@ -130,6 +143,16 @@ export async function tryResolveProviderForConnectionName(
       // DB not available — fall through to the original error.
     }
     if (!resolved) {
+      const incompatMsg = mismatchCandidates
+        ? describeSubscriptionModelIncompatibility(mismatchCandidates, model)
+        : null;
+      if (incompatMsg) {
+        throw new ConnectionResolutionError(
+          connectionName,
+          "model_incompatible",
+          incompatMsg,
+        );
+      }
       throw new ConnectionResolutionError(
         connectionName,
         "provider_mismatch",
@@ -137,13 +160,6 @@ export async function tryResolveProviderForConnectionName(
       );
     }
   }
-  if (connection.status === "disabled") {
-    log.debug(
-      { connectionName, provider: connection.provider },
-      "provider_connection is disabled — returning null",
-    );
-    return null;
-  }
   // `resolveProviderFromConnection` reaches into auth resolution (credential
   // reads, managed-proxy context). A transient failure there is a soft
   // miss — log and return null so the caller can treat it the same as
@@ -151,7 +167,7 @@ export async function tryResolveProviderForConnectionName(
   // catch is specifically for in-flight failures that should not take
   // dispatch offline.
   try {
-    return await resolveProviderFromConnection(connection, config);
+    return await resolveProviderFromConnection(connection, config, { model });
   } catch (err) {
     log.warn(
       { err, connectionName },
@@ -187,12 +203,15 @@ export async function resolveDefaultProvider(
     // provider without a connection ("Any active" selection), and the merge
     // cleared or failed to inherit one. Try to find an active connection
     // for the provider before giving up.
+    let autoResolveCandidates: import("./inference/auth.js").ProviderConnection[] | undefined;
     if (resolved.provider) {
       try {
-        const candidates = listConnections(getDb(), {
+        autoResolveCandidates = listConnections(getDb(), {
           provider: resolved.provider,
         });
-        const active = candidates.find((c) => c.status === "active");
+        const active = autoResolveCandidates.find((c) =>
+          isConnectionCompatibleWithModel(c, resolved.model),
+        );
         if (active) {
           log.info(
             { provider: resolved.provider, resolvedConnection: active.name },
@@ -205,6 +224,19 @@ export async function resolveDefaultProvider(
       }
     }
     if (!connectionName) {
+      const incompatMsg = autoResolveCandidates
+        ? describeSubscriptionModelIncompatibility(
+            autoResolveCandidates,
+            resolved.model,
+          )
+        : null;
+      if (incompatMsg) {
+        throw new ConnectionResolutionError(
+          "<llm.default>",
+          "model_incompatible",
+          incompatMsg,
+        );
+      }
       throw new ConnectionResolutionError(
         "<llm.default>",
         "missing_connection",
@@ -216,5 +248,6 @@ export async function resolveDefaultProvider(
     connectionName,
     config,
     resolved.provider,
+    resolved.model,
   );
 }

package/src/providers/fireworks/client.ts

@@ -34,6 +34,7 @@ export class FireworksProvider extends OpenAIChatCompletionsProvider {
       // overrides (e.g. DeepSeek V4 → "max") come from
       // {@link resolveMaxReasoningEffort}.
       maxReasoningEffort: "high",
+      assistantReasoningField: "reasoning_content",
     });
   }