@vellumai/assistant 0.6.4 → 0.6.5

package/src/providers/retry.ts

@@ -1,3 +1,5 @@
+import { resolveCallSiteConfig } from "../config/llm-resolver.js";
+import { getConfig } from "../config/loader.js";
 import { ProviderError } from "../util/errors.js";
 import { getLogger } from "../util/logger.js";
 import {
@@ -7,13 +9,13 @@ import {
   isRetryableNetworkError,
   sleep,
 } from "../util/retry.js";
-import { isModelIntent, resolveModelIntent } from "./model-intents.js";
-import type {
-  Message,
-  Provider,
-  ProviderResponse,
-  SendMessageOptions,
-  ToolDefinition,
+import {
+  isContextOverflowError,
+  type Message,
+  type Provider,
+  type ProviderResponse,
+  type SendMessageOptions,
+  type ToolDefinition,
 } from "./types.js";
 
 const log = getLogger("retry");
@@ -61,6 +63,19 @@ function isRetryableProviderMessage(error: unknown): boolean {
 }
 
 function isRetryableError(error: unknown): boolean {
+  // Context overflow is deterministic — retrying the same oversized prompt
+  // will never succeed. Short-circuit before the generic 429/5xx check so
+  // ContextOverflowError (which extends ProviderError and may carry a 429
+  // statusCode on Gemini/Vertex) never triggers exponential backoff.
+  if (isContextOverflowError(error)) return false;
+  // Daemon/user-initiated aborts are never retryable. The catch-site tags
+  // these with `abortReason` exactly when `signal.aborted` was true at the
+  // time of failure, so this short-circuits before any message-based pattern
+  // matches — which matters because transport-level aborts (retryable) and
+  // caller-cancels both surface as "Request was aborted" from the SDK.
+  if (error instanceof ProviderError && error.abortReason !== undefined) {
+    return false;
+  }
   if (error instanceof ProviderError && error.statusCode !== undefined) {
     if (error.statusCode === 429 || error.statusCode >= 500) return true;
   }
@@ -69,6 +84,27 @@ function isRetryableError(error: unknown): boolean {
   return isRetryableNetworkError(error);
 }
 
+/**
+ * Normalize per-call options before handing them to the wrapped provider.
+ *
+ * When `config.callSite` is set, resolves model/maxTokens/effort/speed/
+ * temperature/thinking via `resolveCallSiteConfig` and writes them into
+ * `nextConfig` using the wire-format names that downstream provider clients
+ * consume (`max_tokens` snake-case for the token cap; camelCase for the rest,
+ * which matches the resolver's shape). Per-call explicit overrides on the
+ * original `config` object win over the resolved values, so callers can pin
+ * a model or other parameter for a single request. `contextWindow` and
+ * `provider` are intentionally excluded from the written fields — they are
+ * server-side routing/overflow concerns, not provider request parameters,
+ * and forwarding them would leak unknown fields into provider request bodies
+ * (strict-schema clients like Anthropic reject the request).
+ *
+ * Whether or not `callSite` is set, this function applies per-provider
+ * stripping (`thinking`/`effort`/`speed`) based on the wrapped provider's
+ * name — agent-loop callers that pre-resolve provider/model still need this
+ * stripping so they don't accidentally send Anthropic-only knobs to OpenAI
+ * etc.
+ */
 function normalizeSendMessageOptions(
   providerName: string,
   options?: SendMessageOptions,
@@ -76,34 +112,77 @@ function normalizeSendMessageOptions(
   const config = options?.config;
   if (!config) return options;
 
-  const explicitModel =
-    typeof config.model === "string" && config.model.trim().length > 0
-      ? config.model.trim()
-      : undefined;
-  const intent = isModelIntent(config.modelIntent)
-    ? config.modelIntent
-    : undefined;
-  const hasIntent = config.modelIntent !== undefined;
-
-  const needsThinkingStrip =
-    !THINKING_AWARE_PROVIDERS.has(providerName) && config.thinking !== undefined;
-  const needsEffortStrip =
-    !EFFORT_SUPPORTED_PROVIDERS.has(providerName) && config.effort !== undefined;
-  const needsSpeedStrip =
-    providerName !== "anthropic" && config.speed !== undefined;
+  const nextConfig: Record<string, unknown> = { ...config };
 
-  if (
-    !hasIntent &&
-    explicitModel === config.model &&
-    !needsThinkingStrip &&
-    !needsEffortStrip &&
-    !needsSpeedStrip
-  ) {
-    return options;
-  }
+  if (config.callSite !== undefined) {
+    const resolved = resolveCallSiteConfig(config.callSite, getConfig().llm);
 
-  const nextConfig: Record<string, unknown> = { ...config };
-  delete nextConfig.modelIntent;
+    const explicitModel =
+      typeof config.model === "string" && config.model.trim().length > 0
+        ? config.model.trim()
+        : undefined;
+
+    // Routing key is consumed by the RetryProvider layer and must not leak
+    // downstream.
+    delete nextConfig.callSite;
+
+    // Apply resolved values, letting per-call explicit fields win where set.
+    nextConfig.model = explicitModel ?? resolved.model;
+    if (nextConfig.max_tokens === undefined) {
+      nextConfig.max_tokens = resolved.maxTokens;
+    }
+    if (nextConfig.effort === undefined) {
+      nextConfig.effort = resolved.effort;
+    }
+    if (nextConfig.speed === undefined) {
+      nextConfig.speed = resolved.speed;
+    }
+    // `temperature` defaults to `null` in the LLM schema (meaning "no opinion
+    // — let the provider pick its own default"). Only forward when the
+    // resolved value is an actual number; passing `temperature: null` to
+    // provider clients would either be a wire error or silently override
+    // sensible provider defaults. Mirrors the legacy non-callSite path which
+    // never set `temperature` on `providerConfig`.
+    if (
+      nextConfig.temperature === undefined &&
+      resolved.temperature !== null &&
+      resolved.temperature !== undefined
+    ) {
+      nextConfig.temperature = resolved.temperature;
+    }
+    if (nextConfig.thinking === undefined) {
+      // Convert the schema-shape `{ enabled, streamThinking }` into the
+      // Anthropic wire-format `{ type: "adaptive" }` (or omit when disabled).
+      // Mirrors the non-callSite path in `agent/loop.ts` which sets
+      // `providerConfig.thinking = { type: "adaptive" }` only when enabled.
+      // Without this conversion, `thinking` arrives at `AnthropicProvider`
+      // with a shape the SDK doesn't accept (`ThinkingConfigParam` requires
+      // a `type` discriminator), and OpenRouter's truthy check would treat
+      // a disabled config as enabled.
+      if (resolved.thinking?.enabled === true) {
+        nextConfig.thinking = { type: "adaptive" };
+      }
+    }
+    // Forward OpenRouter-only routing preferences so `OpenRouterProvider` can
+    // translate `openrouter.only` into the wire-format `provider: { only: [...] }`
+    // body field on both the OpenAI-compat and Anthropic-compat endpoints.
+    if (
+      providerName === "openrouter" &&
+      nextConfig.openrouter === undefined &&
+      Array.isArray(resolved.openrouter?.only) &&
+      resolved.openrouter.only.length > 0
+    ) {
+      nextConfig.openrouter = { only: resolved.openrouter.only };
+    }
+    // `contextWindow` and `provider` are server-side concerns, not provider
+    // request parameters: `contextWindow` is consumed by the agent loop's
+    // overflow recovery and the conversation manager directly from
+    // `config.llm.default.contextWindow.*`; `provider` selection is handled
+    // by `CallSiteRoutingProvider` upstream. Forwarding them as per-call
+    // config leaks unknown fields into provider request bodies — Anthropic
+    // (and other strict-schema clients) reject the request with
+    // "Extra inputs are not permitted".
+  }
 
   // thinking is Anthropic-specific on the wire; OpenRouter reads it as a
   // signal for its unified reasoning parameter. Strip it for other providers.
@@ -114,6 +193,33 @@ function normalizeSendMessageOptions(
     delete nextConfig.thinking;
   }
 
+  // Anthropic (and OpenRouter fronting Anthropic) rejects requests that
+  // combine extended thinking with forced tool use (`tool_choice.type` of
+  // `"tool"` or `"any"`).  Strip thinking when both are present so the
+  // request doesn't fail with a 400 "Thinking may not be enabled when
+  // tool_choice forces tool use."  `tool_choice: { type: "auto" }` is
+  // compatible with thinking and left untouched.
+  //
+  // For OpenRouter, only strip when routing to an `anthropic/*` model —
+  // non-Anthropic reasoning models (e.g. xAI Grok) translate `thinking`
+  // into OpenRouter's `reasoning` parameter via `buildExtraCreateParams`
+  // and may support reasoning with forced tool_choice.
+  const isThinkingForcedToolConflict = (() => {
+    if (nextConfig.thinking == null) return false;
+    const tc = nextConfig.tool_choice as Record<string, unknown> | undefined;
+    if (tc == null || (tc.type !== "tool" && tc.type !== "any")) return false;
+    if (providerName === "anthropic") return true;
+    if (providerName === "openrouter") {
+      const model =
+        typeof nextConfig.model === "string" ? nextConfig.model : "";
+      return model.startsWith("anthropic/");
+    }
+    return false;
+  })();
+  if (isThinkingForcedToolConflict) {
+    delete nextConfig.thinking;
+  }
+
   // effort is supported by Anthropic, OpenAI, and OpenAI-compatible providers; strip for others
   if (
     !EFFORT_SUPPORTED_PROVIDERS.has(providerName) &&
@@ -127,12 +233,10 @@ function normalizeSendMessageOptions(
     delete nextConfig.speed;
   }
 
-  if (explicitModel) {
-    nextConfig.model = explicitModel;
-  } else if (intent) {
-    nextConfig.model = resolveModelIntent(providerName, intent);
-  } else {
-    delete nextConfig.model;
+  // `openrouter.only` is OpenRouter-specific routing; strip for other
+  // providers so strict-schema clients don't see an unknown field.
+  if (providerName !== "openrouter" && nextConfig.openrouter !== undefined) {
+    delete nextConfig.openrouter;
   }
 
   return {
@@ -141,9 +245,21 @@ function normalizeSendMessageOptions(
   };
 }
 
+/**
+ * `RetryProvider` sets `retriesExhausted = true` on the final thrown error
+ * when the retry loop burned through all attempts against a retryable error
+ * (transient network, 5xx, provider-overloaded, mid-stream corruption).
+ * Consumers can read it via `(err as { retriesExhausted?: boolean })` to
+ * suppress Sentry captures for user-network-flap noise — the retry loop
+ * already did its job, and no engineering action would change the outcome.
+ */
 export class RetryProvider implements Provider {
   public readonly name: string;
 
+  get tokenEstimationProvider(): string | undefined {
+    return this.inner.tokenEstimationProvider;
+  }
+
   constructor(private readonly inner: Provider) {
     this.name = inner.name;
   }
@@ -155,6 +271,7 @@ export class RetryProvider implements Provider {
     options?: SendMessageOptions,
   ): Promise<ProviderResponse> {
     let lastError: unknown;
+    let didRetry = false;
 
     const normalizedOptions = normalizeSendMessageOptions(this.name, options);
 
@@ -202,14 +319,32 @@ export class RetryProvider implements Provider {
             },
             "Retrying after transient error",
           );
+          didRetry = true;
           await sleep(delay);
           continue;
         }
 
+        // If we exhausted retries on a retryable error, tag the error so
+        // downstream consumers (Sentry capture, etc.) can recognize that the
+        // retry loop already tried its best. The catch-site logic above only
+        // stops retrying when either (a) retries are exhausted, or (b) the
+        // error isn't retryable — so we check the retryable predicate here to
+        // distinguish the two cases.
+        if (didRetry && isRetryableError(error) && error instanceof Error) {
+          (error as Error & { retriesExhausted?: boolean }).retriesExhausted =
+            true;
+        }
+
         throw error;
       }
     }
 
+    // Unreachable in practice — the loop body always either returns or throws —
+    // but mark the last error in case execution somehow falls through.
+    if (lastError instanceof Error && isRetryableError(lastError)) {
+      (lastError as Error & { retriesExhausted?: boolean }).retriesExhausted =
+        true;
+    }
     throw lastError;
   }
 }

package/src/providers/speech-to-text/__tests__/resolve.test.ts

@@ -70,6 +70,7 @@ mock.module("../../../security/credential-key.js", () => ({
 const deepgramCtorCalls: Array<{ apiKey: string; options: unknown }> = [];
 const geminiCtorCalls: Array<{ apiKey: string; options: unknown }> = [];
 const whisperCtorCalls: Array<{ apiKey: string; options: unknown }> = [];
+const xaiCtorCalls: Array<{ apiKey: string; options: unknown }> = [];
 
 mock.module("../deepgram-realtime.js", () => ({
   DeepgramRealtimeTranscriber: class {
@@ -101,6 +102,16 @@ mock.module("../openai-whisper-stream.js", () => ({
   },
 }));
 
+mock.module("../xai-realtime.js", () => ({
+  XAIRealtimeTranscriber: class {
+    readonly providerId = "xai" as const;
+    readonly boundaryId = "daemon-streaming" as const;
+    constructor(apiKey: string, options: unknown) {
+      xaiCtorCalls.push({ apiKey, options });
+    }
+  },
+}));
+
 // ---------------------------------------------------------------------------
 // Subject import (after mocks)
 //
@@ -541,6 +552,7 @@ describe("telephony routing catalog alignment", () => {
       "openai-whisper": "openai",
       deepgram: "deepgram",
       "google-gemini": "gemini",
+      xai: "xai",
     };
 
     for (const id of listProviderIds()) {
@@ -714,6 +726,7 @@ describe("resolveStreamingTranscriber diarize preference", () => {
     deepgramCtorCalls.length = 0;
     geminiCtorCalls.length = 0;
     whisperCtorCalls.length = 0;
+    xaiCtorCalls.length = 0;
     loggerWarnings.length = 0;
   });
 
@@ -825,4 +838,46 @@ describe("resolveStreamingTranscriber diarize preference", () => {
     expect(options.sampleRate).toBe(48000);
     expect(options.diarize).toBe(true);
   });
+
+  // -------------------------------------------------------------------------
+  // xAI realtime streaming
+  // -------------------------------------------------------------------------
+
+  test("resolves a non-null xai transcriber when xai is configured and credentials are available", async () => {
+    mockProviderKeys["xai"] = "xai-key";
+    mockConfig = buildConfig({ provider: "xai" });
+
+    const transcriber = await resolveStreamingTranscriber();
+
+    expect(transcriber).not.toBeNull();
+    expect(transcriber!.providerId).toBe("xai");
+    expect(transcriber!.boundaryId).toBe("daemon-streaming");
+    expect(xaiCtorCalls).toHaveLength(1);
+  });
+
+  test("diarize: 'required' with xai constructs the transcriber (does not reject)", async () => {
+    mockProviderKeys["xai"] = "xai-key";
+    mockConfig = buildConfig({ provider: "xai" });
+
+    const transcriber = await resolveStreamingTranscriber({
+      diarize: "required",
+    });
+
+    expect(transcriber).not.toBeNull();
+    expect(xaiCtorCalls).toHaveLength(1);
+    const options = xaiCtorCalls[0]!.options as Record<string, unknown>;
+    expect(options.diarize).toBe(true);
+    // No warning logged — xai supports diarization per the catalog.
+    expect(loggerWarnings).toHaveLength(0);
+  });
+
+  test("returns null for xai when no credential is set", async () => {
+    mockProviderKeys = {};
+    mockConfig = buildConfig({ provider: "xai" });
+
+    const transcriber = await resolveStreamingTranscriber();
+
+    expect(transcriber).toBeNull();
+    expect(xaiCtorCalls).toHaveLength(0);
+  });
 });

package/src/providers/speech-to-text/google-gemini-live-stream.ts

@@ -250,10 +250,9 @@ export class GoogleGeminiLiveStreamingTranscriber implements StreamingTranscribe
     let timedOut = false;
     connectPromise
       .then((session) => {
-        if (timedOut) {
-          // Never assign a session after the start() timeout: if we did,
-          // `this.session` would persist past the failed start() and a
-          // retry would incorrectly trip the "start() called twice" guard.
+        if (timedOut || this.closed || this.stopping) {
+          // Never assign a session after shutdown or timeout: if we did,
+          // `this.session` would persist as an orphaned live WebSocket.
           try {
             session.close();
           } catch {
@@ -398,6 +397,7 @@ export class GoogleGeminiLiveStreamingTranscriber implements StreamingTranscribe
       serverContent.turnComplete === true;
 
     if (isComplete) {
+      if (this.finalEmittedForCurrentTurn) return;
       const finalText = this.currentTurnText;
       this.currentTurnText = "";
       this.lastEmittedPartial = "";

package/src/providers/speech-to-text/provider-catalog.ts

@@ -219,6 +219,23 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
       },
     },
   ],
+  [
+    "xai",
+    {
+      id: "xai",
+      credentialProvider: "xai",
+      supportedBoundaries: new Set<SttBoundaryId>([
+        "daemon-batch",
+        "daemon-streaming",
+      ]),
+      telephonyMode: "batch-only",
+      conversationStreamingMode: "realtime-ws",
+      supportsDiarization: true,
+      telephonyRouting: {
+        strategyKind: "media-stream-custom",
+      },
+    },
+  ],
 ]);
 
 // ---------------------------------------------------------------------------

package/src/providers/speech-to-text/resolve.ts

@@ -381,6 +381,13 @@ async function createStreamingTranscriber(
         pcmSampleRate: options.sampleRate,
       });
     }
+    case "xai": {
+      const { XAIRealtimeTranscriber } = await import("./xai-realtime.js");
+      return new XAIRealtimeTranscriber(apiKey, {
+        sampleRate: options.sampleRate,
+        ...(options.diarize ? { diarize: true } : {}),
+      });
+    }
     default: {
       const _exhaustive: never = providerId;
       return null;