@vellumai/assistant 0.6.3 → 0.6.5

package/src/providers/speech-to-text/xai.test.ts

@@ -0,0 +1,155 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { XAIProvider } from "./xai.js";
+
+const TEST_API_KEY = "xai-test-key-for-unit-tests";
+
+describe("XAIProvider", () => {
+  let originalFetch: typeof globalThis.fetch;
+
+  beforeEach(() => {
+    originalFetch = globalThis.fetch;
+  });
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+  });
+
+  test("successful transcription returns trimmed text", async () => {
+    globalThis.fetch = (async (
+      _url: string | URL | Request,
+      _init?: RequestInit,
+    ) => {
+      return new Response(JSON.stringify({ text: "  hello  " }), {
+        status: 200,
+        headers: { "Content-Type": "application/json" },
+      });
+    }) as typeof fetch;
+
+    const provider = new XAIProvider(TEST_API_KEY);
+    const result = await provider.transcribe(
+      Buffer.from("fake-audio"),
+      "audio/ogg",
+    );
+
+    expect(result).toEqual({ text: "hello" });
+  });
+
+  test("API error throws with status and partial body", async () => {
+    const errorBody = JSON.stringify({
+      error: {
+        message: "Invalid API key provided",
+        type: "invalid_request_error",
+      },
+    });
+
+    globalThis.fetch = (async (
+      _url: string | URL | Request,
+      _init?: RequestInit,
+    ) => {
+      return new Response(errorBody, {
+        status: 401,
+        headers: { "Content-Type": "application/json" },
+      });
+    }) as typeof fetch;
+
+    const provider = new XAIProvider(TEST_API_KEY);
+
+    await expect(
+      provider.transcribe(Buffer.from("fake-audio"), "audio/wav"),
+    ).rejects.toThrow("xAI STT error (401)");
+  });
+
+  test("sends correct FormData structure (no model field, file blob with correct MIME)", async () => {
+    let capturedBody: FormData | undefined;
+    let capturedHeaders: HeadersInit | undefined;
+
+    globalThis.fetch = (async (
+      url: string | URL | Request,
+      init?: RequestInit,
+    ) => {
+      capturedBody = init?.body as FormData;
+      capturedHeaders = init?.headers;
+      expect(url).toBe("https://api.x.ai/v1/stt");
+      expect(init?.method).toBe("POST");
+
+      return new Response(JSON.stringify({ text: "transcribed" }), {
+        status: 200,
+        headers: { "Content-Type": "application/json" },
+      });
+    }) as typeof fetch;
+
+    const provider = new XAIProvider(TEST_API_KEY);
+    await provider.transcribe(Buffer.from("fake-audio"), "audio/mpeg");
+
+    // Verify authorization header
+    expect(capturedHeaders).toEqual({
+      Authorization: `Bearer ${TEST_API_KEY}`,
+    });
+
+    // Verify FormData contents
+    expect(capturedBody).toBeInstanceOf(FormData);
+
+    // xAI does not use a `model` field.
+    expect(capturedBody!.get("model")).toBeNull();
+
+    const file = capturedBody!.get("file");
+    expect(file).toBeInstanceOf(Blob);
+    expect((file as Blob).type).toBe("audio/mpeg");
+  });
+
+  test("returns empty text when API returns empty text field", async () => {
+    globalThis.fetch = (async () => {
+      return new Response(JSON.stringify({ text: "" }), {
+        status: 200,
+        headers: { "Content-Type": "application/json" },
+      });
+    }) as unknown as typeof fetch;
+
+    const provider = new XAIProvider(TEST_API_KEY);
+    const result = await provider.transcribe(
+      Buffer.from("silence"),
+      "audio/wav",
+    );
+
+    expect(result).toEqual({ text: "" });
+  });
+
+  test("returns empty text when API response has no text property", async () => {
+    globalThis.fetch = (async () => {
+      return new Response(JSON.stringify({}), {
+        status: 200,
+        headers: { "Content-Type": "application/json" },
+      });
+    }) as unknown as typeof fetch;
+
+    const provider = new XAIProvider(TEST_API_KEY);
+    const result = await provider.transcribe(
+      Buffer.from("silence"),
+      "audio/wav",
+    );
+
+    expect(result).toEqual({ text: "" });
+  });
+
+  test("error body is truncated to 300 characters", async () => {
+    const longBody = "x".repeat(500);
+
+    globalThis.fetch = (async () => {
+      return new Response(longBody, { status: 500 });
+    }) as unknown as typeof fetch;
+
+    const provider = new XAIProvider(TEST_API_KEY);
+
+    try {
+      await provider.transcribe(Buffer.from("audio"), "audio/wav");
+      expect.unreachable("should have thrown");
+    } catch (err) {
+      const msg = (err as Error).message;
+      expect(msg).toContain("xAI STT error (500)");
+      // The body portion should be at most 300 chars
+      const bodyPart = msg.replace("xAI STT error (500): ", "");
+      expect(bodyPart.length).toBeLessThanOrEqual(300);
+    }
+  });
+});

package/src/providers/speech-to-text/xai.ts

@@ -0,0 +1,97 @@
+import type { SttTranscribeResult } from "../../stt/types.js";
+
+export const XAI_STT_URL = "https://api.x.ai/v1/stt";
+export const DEFAULT_TIMEOUT_MS = 60_000;
+
+/**
+ * Derive a filename extension from a MIME type so the xAI STT API can detect
+ * the audio format. Falls back to "audio" when the MIME type is unrecognised.
+ */
+export function extensionFromMime(mimeType: string): string {
+  const map: Record<string, string> = {
+    "audio/wav": "wav",
+    "audio/x-wav": "wav",
+    "audio/mpeg": "mp3",
+    "audio/mp3": "mp3",
+    "audio/ogg": "ogg",
+    "audio/opus": "opus",
+    "audio/webm": "webm",
+    "audio/mp4": "m4a",
+    "audio/x-m4a": "m4a",
+    "audio/flac": "flac",
+  };
+  const base = mimeType.split(";")[0].trim().toLowerCase();
+  return map[base] ?? "audio";
+}
+
+/**
+ * Build a FormData payload for the xAI `/v1/stt` endpoint.
+ *
+ * xAI does not require a `model` field. The xAI docs explicitly require the
+ * `file` field to be appended LAST in the multipart body, so we only append
+ * the file here (no other fields in v1).
+ */
+export function buildXaiFormData(audio: Buffer, mimeType: string): FormData {
+  const ext = extensionFromMime(mimeType);
+
+  const formData = new FormData();
+  // xAI requires the `file` field to be LAST in the multipart body.
+  formData.append(
+    "file",
+    new Blob([new Uint8Array(audio)], { type: mimeType }),
+    `audio.${ext}`,
+  );
+
+  return formData;
+}
+
+/**
+ * Send audio to the xAI STT API and return the transcribed text.
+ *
+ * xAI returns a richer shape (`{ text, language, duration, words }`) — we only
+ * consume `text`.
+ */
+export async function xaiTranscribe(
+  apiKey: string,
+  audio: Buffer,
+  mimeType: string,
+  signal?: AbortSignal,
+): Promise<string> {
+  const formData = buildXaiFormData(audio, mimeType);
+
+  const effectiveSignal = signal ?? AbortSignal.timeout(DEFAULT_TIMEOUT_MS);
+
+  const response = await fetch(XAI_STT_URL, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${apiKey}` },
+    body: formData,
+    signal: effectiveSignal,
+  });
+
+  if (!response.ok) {
+    const body = await response.text().catch(() => "");
+    throw new Error(
+      `xAI STT error (${response.status}): ${body.slice(0, 300)}`,
+    );
+  }
+
+  const result = (await response.json()) as { text?: string };
+  return result.text?.trim() ?? "";
+}
+
+export class XAIProvider {
+  private readonly apiKey: string;
+
+  constructor(apiKey: string) {
+    this.apiKey = apiKey;
+  }
+
+  async transcribe(
+    audio: Buffer,
+    mimeType: string,
+    signal?: AbortSignal,
+  ): Promise<SttTranscribeResult> {
+    const text = await xaiTranscribe(this.apiKey, audio, mimeType, signal);
+    return { text };
+  }
+}

package/src/providers/types.ts

@@ -1,3 +1,6 @@
+import type { LLMCallSite } from "../config/schemas/llm.js";
+import { ProviderError } from "../util/errors.js";
+
 export interface TextContent {
   type: "text";
   text: string;
@@ -126,12 +129,23 @@ export type ProviderEvent =
       toolUseId: string;
       input: Record<string, unknown>;
     }
-  | { type: "server_tool_complete"; toolUseId: string; isError: boolean; content?: unknown[] };
+  | {
+      type: "server_tool_complete";
+      toolUseId: string;
+      isError: boolean;
+      content?: unknown[];
+    };
 
 export interface SendMessageConfig {
   model?: string;
-  modelIntent?: ModelIntent;
-  effort?: "low" | "medium" | "high" | "max";
+  /**
+   * LLM call-site identifier. `RetryProvider` resolves
+   * provider/model/maxTokens/effort/speed/temperature/thinking/contextWindow
+   * via `resolveCallSiteConfig(callSite, config.llm)`, falling back to
+   * `llm.default` when no callSite-specific entry is present.
+   */
+  callSite?: LLMCallSite;
+  effort?: "low" | "medium" | "high" | "xhigh" | "max";
   speed?: "standard" | "fast";
   [key: string]: unknown;
 }
@@ -144,6 +158,15 @@ export interface SendMessageOptions {
 
 export interface Provider {
   name: string;
+  /**
+   * Provider key used by the local token estimator to select model-family
+   * specific rules (e.g. Anthropic's `width * height / 750` image sizing).
+   * Wrapper providers that route to another provider's API — e.g. OpenRouter
+   * calling Anthropic's Messages endpoint for `anthropic/*` models — override
+   * this so the estimator matches what the upstream API will actually charge.
+   * Falls back to `name` when unset.
+   */
+  tokenEstimationProvider?: string;
   sendMessage(
     messages: Message[],
     tools?: ToolDefinition[],
@@ -151,3 +174,79 @@ export interface Provider {
     options?: SendMessageOptions,
   ): Promise<ProviderResponse>;
 }
+
+// ── Context-overflow error ────────────────────────────────────────────
+
+export interface ContextOverflowErrorOptions {
+  /** Actual tokens the request was estimated/measured to consume, when the provider reports it. */
+  actualTokens?: number;
+  /** Context-window cap the provider enforced, when reported in the error body. */
+  maxTokens?: number;
+  /** HTTP status reported by the provider. Defaults to 400. */
+  statusCode?: number;
+  /** Underlying error to preserve the cause chain (standard Error.cause). */
+  cause?: unknown;
+}
+
+/**
+ * Thrown by provider clients when the request exceeded the model's context
+ * window (HTTP 400 `context_length_exceeded`, Anthropic's `prompt_too_long`,
+ * Gemini's resource-exhausted category, etc.).
+ *
+ * Extends `ProviderError` so existing `instanceof ProviderError` classifiers
+ * (`util/retry.ts`, `daemon/conversation-error.ts`) continue to see it as a
+ * typed 4xx provider error and apply the right policy. The
+ * `actualTokens`/`maxTokens` fields carry structured counts when the
+ * provider reports them, avoiding brittle regex parsing at the caller.
+ *
+ * A regex-on-message fallback still exists in
+ * `daemon/parse-actual-tokens-from-error.ts` as a safety net for adapters
+ * that rewrap the error (e.g. managed-proxy) before it reaches the agent
+ * loop.
+ */
+export class ContextOverflowError extends ProviderError {
+  public readonly actualTokens?: number;
+  public readonly maxTokens?: number;
+
+  constructor(
+    message: string,
+    provider: string,
+    options: ContextOverflowErrorOptions = {},
+  ) {
+    super(
+      message,
+      provider,
+      options.statusCode ?? 400,
+      options.cause !== undefined ? { cause: options.cause } : undefined,
+    );
+    this.name = "ContextOverflowError";
+    this.actualTokens = options.actualTokens;
+    this.maxTokens = options.maxTokens;
+  }
+}
+
+export function isContextOverflowError(
+  err: unknown,
+): err is ContextOverflowError {
+  return err instanceof ContextOverflowError;
+}
+
+/**
+ * Extract `actualTokens` / `maxTokens` from provider overflow messages of the
+ * form "N tokens > M maximum" or bare "N > M". Returns an empty object when
+ * neither count is parseable — callers should treat this as "matched the
+ * overflow signal but counts unknown".
+ */
+export function extractOverflowTokensFromMessage(message: string): {
+  actualTokens?: number;
+  maxTokens?: number;
+} {
+  const match = message.match(/(\d[\d,]*)\s*(?:tokens?\s*)?[>≥]\s*(\d[\d,]*)/i);
+  if (!match) return {};
+  const actual = parseInt(match[1].replace(/,/g, ""), 10);
+  const max = parseInt(match[2].replace(/,/g, ""), 10);
+  const out: { actualTokens?: number; maxTokens?: number } = {};
+  if (!isNaN(actual) && actual > 0) out.actualTokens = actual;
+  if (!isNaN(max) && max > 0) out.maxTokens = max;
+  return out;
+}

package/src/runtime/AGENTS.md

@@ -18,9 +18,9 @@ Approvals are **orthogonal to message sending**. The assistant asks for approval
 
 - **Discovery**: Clients discover pending approvals via SSE events (`confirmation_request`, `secret_request`) which include a `requestId`.
 - **Resolution**: Clients respond via standalone endpoints keyed by `requestId`:
-  - `POST /v1/confirm` — `{ requestId, decision, selectedPattern?, selectedScope? }`. Valid decisions: `"allow"`, `"allow_10m"`, `"allow_conversation"`, `"deny"`, `"always_allow"`, `"always_deny"`, `"always_allow_high_risk"`. For persistent decisions (`always_allow`, `always_deny`, `always_allow_high_risk`), `selectedPattern` and `selectedScope` are validated against the server-provided allowlist/scope options from the original confirmation request before trust rules are persisted.
+  - `POST /v1/confirm` — `{ requestId, decision, selectedPattern?, selectedScope? }`. Valid decisions: `"allow"`, `"allow_10m"`, `"allow_conversation"`, `"deny"`, `"always_allow"`, `"always_deny"`. For persistent decisions (`always_allow`, `always_deny`), `selectedPattern` and `selectedScope` are validated against the server-provided allowlist/scope options from the original confirmation request before trust rules are persisted.
   - `POST /v1/secret` — `{ requestId, value, delivery }`
-  - `POST /v1/trust-rules` — `{ requestId, pattern, scope, decision, allowHighRisk? }`. Validates pattern/scope against server-provided options. Does not resolve the confirmation itself.
+  - `POST /v1/trust-rules` — `{ requestId, pattern, scope, decision }`. Validates pattern/scope against server-provided options. Does not resolve the confirmation itself.
 - **Tracking**: The `pending-interactions` tracker (`assistant/src/runtime/pending-interactions.ts`) maps `requestId → conversation`. Use `register()` to track, `resolve()` to consume, `getByConversation()` to query.
 
 Do NOT couple approval handling to message sending. Do NOT add run/status tracking to the send path.
@@ -69,12 +69,54 @@ Unlike the SSE-based host proxies used by the macOS client, `host_browser_reques
 
 A single guardian may have multiple parallel extension installs connected at once (two Chrome profiles, two desktops sharing a sync identity). Each install generates a stable `clientInstanceId` on first run, persists it in `chrome.storage.local`, and sends it on every WebSocket handshake as a query param (`clientInstanceId=...`) or header (`x-client-instance-id`). The registry keys inner entries by that id so sibling installs don't evict each other on register/unregister. The default `send(guardianId, msg)` path routes to whichever instance has the most recent activity (`lastActiveAt`); `sendToInstance(guardianId, clientInstanceId, msg)` pins a specific install. Older extension builds that omit the id get a connection-scoped `legacy:<connectionId>` fallback key so they degrade gracefully to single-instance semantics.
 
-`Conversation.hostBrowserSenderOverride` is the integration point between the turn layer and the registry. When a turn for a chrome-extension interface enters the routes layer, `conversation-routes.ts` resolves the active registry entry for the caller's guardian and sets the override to a sender that writes to that WebSocket. `Conversation.restoreBrowserProxyAvailability()` re-threads the override on queue drain — without this, the drain path would clobber the registry-routed sender with the default `sendToClient` (pointed at the SSE hub) and `host_browser_request` frames would stop reaching the extension mid-queue.
+`Conversation.hostBrowserSenderOverride` is the integration point between the turn layer and the registry. When any turn enters the routes layer and the guardian has an active extension connection in the `ChromeExtensionRegistry`, `conversation-routes.ts` resolves the registry entry and sets the override to a sender that writes to that WebSocket. This applies to chrome-extension turns (where the registry is the only transport) and macOS turns (where the extension connection lets browser tools route through the user's real Chrome session instead of cdp-inspect/local). `Conversation.restoreBrowserProxyAvailability()` re-threads the override on queue drain — without this, the drain path would clobber the registry-routed sender with the default `sendToClient` (pointed at the SSE hub) and `host_browser_request` frames would stop reaching the extension mid-queue.
 
 Capability token bootstrap for self-hosted deployments is handled by `routes/browser-extension-pair-routes.ts` (loopback-only; mints a guardian-bound HMAC capability token via `capability-tokens.ts`). Cloud deployments issue guardian-bound JWTs via the gateway's WorkOS-backed flow — `browser-extension-pair-routes.ts` is not involved.
 
 See `docs/browser-use-architecture-phase2.md` for the full wire diagram and component inventory.
 
+### Canonical browser backend precedence (macOS)
+
+On macOS-originated turns, the CDP factory (`tools/browser/cdp-client/factory.ts`) evaluates three browser backends in strict priority order. Each candidate is tried lazily; if the first command fails with a transport-level error, the factory falls over to the next candidate. CDP protocol errors (the browser understood the command but rejected it) do NOT trigger failover.
+
+| Priority | Backend         | Condition                                                                                                                                                                                           | Source                                                                                 |
+| -------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
+| 1        | **Extension**   | `hostBrowserProxy` present AND `isAvailable()` returns `true` (registry-routed WebSocket is connected)                                                                                              | `ChromeExtensionRegistry` via `resolveHostBrowserSender()` in `conversation-routes.ts` |
+| 2        | **cdp-inspect** | (a) `hostBrowser.cdpInspect.enabled` is `true` in config, OR (b) `transportInterface === "macos"` AND `desktopAuto.enabled` is `true` (default) AND the cooldown from a prior failure is not active | Config + `desktopAuto` policy in factory                                               |
+| 3        | **Local**       | Always present as the final fallback                                                                                                                                                                | Playwright sacrificial-profile browser managed by `browserManager`                     |
+
+**Fallback criteria for cdp-inspect (desktop-auto):**
+
+- On macOS, `desktopAuto.enabled` defaults to `true`, so cdp-inspect is attempted even when the top-level `cdpInspect.enabled` is `false`.
+- If the cdp-inspect probe fails (Chrome was not launched with `--remote-debugging-port`, or the endpoint is unreachable), the factory records a cooldown timestamp (`desktopAuto.cooldownMs`, default 30 seconds).
+- While the cooldown is active, subsequent macOS turns skip the cdp-inspect candidate entirely and go straight to local, bounding the per-call latency penalty to one `probeTimeoutMs` (default 500ms) per cooldown window.
+- The cooldown only applies to desktop-auto candidates (reason starts with `"desktopAuto:"`). Explicitly configured cdp-inspect (`enabled: true`) is never cooldown-suppressed.
+
+**After the first successful CDP command**, the selected backend becomes **sticky** for the remainder of the tool invocation. Subsequent commands always route through the same backend so multi-command tool flows do not hop transports mid-step.
+
+### Per-tool `browser_mode` override
+
+All CDP-backed browser tools (`browser_navigate`, `browser_snapshot`, `browser_screenshot`, `browser_click`, `browser_type`, `browser_hover`, `browser_scroll`, `browser_press_key`, `browser_select_option`, `browser_wait_for`, `browser_extract`, `browser_fill_credential`, `browser_attach`, `browser_detach`, `browser_close`, `browser_status`) accept an optional `browser_mode` input parameter that overrides the automatic backend selection for that invocation.
+
+| Value            | Behavior                                                                 |
+| ---------------- | ------------------------------------------------------------------------ |
+| `auto` (default) | Existing priority-ordered fallback: extension -> cdp-inspect -> local    |
+| `extension`      | Pin to chrome-extension backend. Fails immediately if proxy unavailable. |
+| `cdp-inspect`    | Pin to CDP inspect/debugger backend. Fails if endpoint unreachable.      |
+| `local`          | Pin to local Playwright-managed browser. No fallback.                    |
+| `cdp-debugger`   | Alias for `cdp-inspect`.                                                 |
+| `playwright`     | Alias for `local`.                                                       |
+
+**Strict pinned-mode semantics**: When `browser_mode` is set to a specific backend (not `auto`), the factory builds exactly one candidate and disables failover. If the pinned backend is unavailable, the tool returns a detailed error including:
+
+- The requested mode
+- An ordered list of attempted backends with exact failure reasons
+- A remediation checklist tailored by backend and failure code (e.g. "Ensure Chrome is running with --remote-debugging-port=9222")
+
+**Auto-mode fallback logging**: In auto mode, fallback transitions are logged at `warn` level with structured metadata including the full candidate sequence and per-candidate failure reasons. This ensures fallback events are always observable in production logs.
+
+**Test coverage:** Regression tests for `browser_mode` wiring live in `__tests__/headless-browser-mode.test.ts`. E2E regression tests for backend precedence live in `__tests__/host-browser-e2e-cloud.test.ts` (extension path) and `__tests__/conversation-routes-disk-view.test.ts` (macOS fallback path). Unit tests for pinned candidate construction and failover live in `tools/browser/cdp-client/__tests__/factory.test.ts`.
+
 ### Channel approvals (Telegram, Slack)
 
 Channel approval flows use `requestId` (not `runId`) as the primary identifier: