@vellumai/assistant 0.11.6 → 0.11.7-staging.2

package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts

@@ -0,0 +1,296 @@
+/**
+ * Managed Flux STT through the gateway's speech relay
+ * (gateway -> velay -> Deepgram Flux; velay contact is gateway-only).
+ *
+ * Composes {@link DeepgramFluxRealtimeTranscriber} against the relay's
+ * managed STT v2 endpoint with `contract=flux`, so Flux's own turn events
+ * arrive verbatim. Without that opt-in velay translates them into the
+ * released v1 Deepgram dialect, which drops `EagerEndOfTurn` and
+ * `TurnResumed` entirely and leaves live voice with no provider turn end to
+ * act on.
+ *
+ * This wrapper owns what is relay-specific, mirroring
+ * `VellumManagedRealtimeTranscriber` on the v1 path:
+ *
+ * - Dialing the gateway (`/v2/speech/stt/stream`, `?key=` carries a
+ *   daemon-minted service token, no `model` param since velay derives and
+ *   prices the Flux model from `language`).
+ * - Surfacing `velay_error` control frames as categorized SttErrors instead
+ *   of generic socket failures.
+ * - Transparent re-dial on velay's 30-minute session cap, so one live-voice
+ *   session can outlive it.
+ *
+ * There is deliberately no `finalizeUtterance`: Flux commits turns itself,
+ * and the interface makes the method optional so live voice feature-detects
+ * its absence and never asks for a flush.
+ */
+
+import {
+  type StreamingTranscriber,
+  SttError,
+  type SttStreamServerEvent,
+} from "../../stt/types.js";
+import { getLogger } from "../../util/logger.js";
+import { DeepgramFluxRealtimeTranscriber } from "./deepgram-flux-realtime.js";
+import {
+  mapVelayError,
+  probeVelayRejection,
+  type SpeechRelayConnection,
+  type VelayErrorInfo,
+} from "./vellum-speech-relay-connection.js";
+
+const log = getLogger("vellum-managed-flux-realtime");
+
+/** Managed STT v2: velay resolves this to Deepgram's Flux `/v2/listen`. */
+const STT_STREAM_PATH = "/v2/speech/stt/stream";
+
+export interface VellumManagedFluxRealtimeOptions {
+  /** Audio sample rate in Hz (default: 16000). */
+  sampleRate?: number;
+  /**
+   * BCP-47 language code (or `"multi"`), forwarded to the relay. This is
+   * how velay picks between `flux-general-en` and `flux-general-multi`, so
+   * it is the only lever over the Flux model from this side.
+   */
+  language?: string;
+}
+
+export class VellumManagedFluxRealtimeTranscriber implements StreamingTranscriber {
+  readonly providerId = "vellum-flux" as const;
+  readonly boundaryId = "daemon-streaming" as const;
+
+  private readonly connection: SpeechRelayConnection;
+  private readonly options: VellumManagedFluxRealtimeOptions;
+
+  private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
+  /** The current relay session; null while re-dialing after a session cap. */
+  private inner: DeepgramFluxRealtimeTranscriber | null = null;
+  /**
+   * The most recent `velay_error` frame from the current session. Set just
+   * before velay closes the socket; consumed when the inner adapter turns
+   * that close into an error event.
+   */
+  private pendingRelayError: VelayErrorInfo | null = null;
+  /**
+   * Set when the current session hit velay's duration cap: the swap to a
+   * fresh session is deferred to the capped session's `closed` event so its
+   * close cleanup drains through first.
+   */
+  private redialOnClose = false;
+  private stopping = false;
+  private closedEmitted = false;
+
+  constructor(
+    connection: SpeechRelayConnection,
+    options: VellumManagedFluxRealtimeOptions = {},
+  ) {
+    this.connection = connection;
+    this.options = options;
+  }
+
+  async start(onEvent: (event: SttStreamServerEvent) => void): Promise<void> {
+    if (this.onEvent) {
+      throw new Error(
+        "VellumManagedFluxRealtimeTranscriber: start() called twice",
+      );
+    }
+    this.onEvent = onEvent;
+    try {
+      const inner = await this.dial();
+      if (this.stopping) {
+        // stop() raced the initial dial: it saw no inner to tear down and
+        // already emitted closed, so the freshly opened relay session must
+        // not be kept (a leaked session keeps velay metering).
+        inner.stop();
+        return;
+      }
+      this.inner = inner;
+    } catch (err) {
+      // A failed WebSocket upgrade exposes no HTTP details; replay the
+      // request as a plain GET to recover the relay's {code, detail}
+      // rejection.
+      const rejection = await probeVelayRejection(this.probeUrl());
+      if (rejection) {
+        const mapped = mapVelayError(rejection);
+        throw new SttError(mapped.category, mapped.message, {
+          userFacing: true,
+        });
+      }
+      throw err;
+    }
+  }
+
+  sendAudio(audio: Buffer, mimeType: string): void {
+    // Audio arriving during a session-cap re-dial (sub-second) is dropped,
+    // mirroring the inner adapter's backpressure-drop semantics.
+    this.inner?.sendAudio(audio, mimeType);
+  }
+
+  stop(): void {
+    if (this.stopping) {
+      return;
+    }
+    this.stopping = true;
+    if (this.inner) {
+      this.inner.stop();
+      return;
+    }
+    // Mid-redial stop: there is no session to drain.
+    this.emitClosedOnce();
+  }
+
+  // -- Relay session management ------------------------------------------
+
+  private async dial(): Promise<DeepgramFluxRealtimeTranscriber> {
+    this.pendingRelayError = null;
+    // Fresh per dial: a session-cap re-dial happens ~30 minutes in, long
+    // after any previously minted token expired.
+    const inner = new DeepgramFluxRealtimeTranscriber(
+      this.connection.mintServiceToken(),
+      {
+        baseUrl: this.connection.wsBaseUrl,
+        path: STT_STREAM_PATH,
+        queryAuth: true,
+        omitModelParam: true,
+        nativeContract: true,
+        ...(this.options.sampleRate !== undefined
+          ? { sampleRate: this.options.sampleRate }
+          : {}),
+        ...(this.options.language ? { language: this.options.language } : {}),
+        onUnhandledFrame: (frame) => this.handleRelayFrame(frame),
+      },
+    );
+    await inner.start((event) => this.handleInnerEvent(inner, event));
+    return inner;
+  }
+
+  private probeUrl(): string {
+    const key = encodeURIComponent(this.connection.mintServiceToken());
+    return `${this.connection.httpBaseUrl}${STT_STREAM_PATH}?key=${key}`;
+  }
+
+  private handleRelayFrame(frame: Record<string, unknown>): void {
+    if (frame.type !== "velay_error" || typeof frame.code !== "string") {
+      return;
+    }
+    this.pendingRelayError = {
+      code: frame.code,
+      ...(typeof frame.detail === "string" ? { detail: frame.detail } : {}),
+    };
+    log.info(
+      { code: frame.code, detail: frame.detail },
+      "Received velay_error frame on managed Flux relay",
+    );
+  }
+
+  private handleInnerEvent(
+    source: DeepgramFluxRealtimeTranscriber,
+    event: SttStreamServerEvent,
+  ): void {
+    // Events from a superseded session (the one replaced by a re-dial) must
+    // not leak through: its close/error was already accounted for.
+    if (source !== this.inner) {
+      return;
+    }
+
+    if (event.type === "error") {
+      const relayError = this.pendingRelayError;
+      if (relayError?.code === "session_duration_exceeded") {
+        // Velay's 30-minute cap, not a failure. Swallow the error but keep
+        // the capped session current so its close cleanup drains first; the
+        // re-dial happens on its `closed` event.
+        this.redialOnClose = !this.stopping;
+        return;
+      }
+      if (relayError) {
+        const mapped = mapVelayError(relayError);
+        this.emit({
+          type: "error",
+          category: mapped.category,
+          message: mapped.message,
+        });
+        return;
+      }
+      this.emit(event);
+      return;
+    }
+
+    if (event.type === "closed") {
+      if (this.redialOnClose && !this.stopping) {
+        this.redialOnClose = false;
+        this.beginRedial();
+        return;
+      }
+      this.emitClosedOnce();
+      return;
+    }
+
+    this.emit(event);
+  }
+
+  /**
+   * Replace the capped session with a fresh one. Detached: audio sent during
+   * the swap is dropped (see {@link sendAudio}); a dial failure is a real
+   * error and ends the stream.
+   */
+  private beginRedial(): void {
+    log.info("Managed Flux relay session hit velay's duration cap, re-dialing");
+    this.inner = null;
+    void (async () => {
+      try {
+        const inner = await this.dial();
+        if (this.stopping) {
+          inner.stop();
+          return;
+        }
+        this.inner = inner;
+        log.info("Managed Flux relay re-dial complete; session continues");
+      } catch (err) {
+        if (this.stopping) {
+          return;
+        }
+        const rejection = await probeVelayRejection(this.probeUrl());
+        if (rejection) {
+          const mapped = mapVelayError(rejection);
+          this.emit({
+            type: "error",
+            category: mapped.category,
+            message: mapped.message,
+          });
+        } else {
+          this.emit({
+            type: "error",
+            category: "provider-error",
+            message: `Managed Flux relay re-dial failed after the session cap: ${
+              err instanceof Error ? err.message : String(err)
+            }`,
+          });
+        }
+        this.emitClosedOnce();
+      }
+    })();
+  }
+
+  // -- Event emission -----------------------------------------------------
+
+  private emit(event: SttStreamServerEvent): void {
+    if (!this.onEvent || this.closedEmitted) {
+      return;
+    }
+    if (event.type === "closed") {
+      this.closedEmitted = true;
+    }
+    try {
+      this.onEvent(event);
+    } catch (err) {
+      log.warn(
+        { error: err },
+        "Listener error in vellum managed Flux realtime adapter",
+      );
+    }
+  }
+
+  private emitClosedOnce(): void {
+    this.emit({ type: "closed" });
+  }
+}

package/src/providers/types.ts

@@ -1,6 +1,16 @@
 import type { ToolDefinition } from "../tools/tool-types.js";
 export type { ToolDefinition };
 
+/**
+ * The tool name that activates provider-native web search. Callers append a
+ * tool under this name only to instances reporting
+ * {@link Provider.supportsNativeWebSearch}; anywhere a request can change
+ * routes after that decision (see `RetryProvider`'s backup-profile
+ * escalation), the tool has to be dropped again when the new route cannot
+ * serve it, or the model answers with a tool call nothing can execute.
+ */
+export const NATIVE_WEB_SEARCH_TOOL_NAME = "web_search";
+
 import type { LLMCallSite } from "../config/schemas/llm.js";
 import {
   ProviderError,
@@ -191,6 +201,15 @@ export interface ProviderResponse {
   model: string;
   /** Provider that actually produced this response, which may differ from a wrapper provider name. */
   actualProvider?: string;
+  /**
+   * Inference profile key that actually governed this response when a
+   * wrapper re-routed the request away from the caller's own resolution
+   * (`RetryProvider`'s fallback-route escalation). `UsageTrackingProvider`
+   * prefers this over re-resolving from the original request options, so a
+   * successful fallback serve is attributed to the backup profile rather
+   * than the failed primary's. Absent on the normal (non-rerouted) path.
+   */
+  actualInferenceProfile?: string;
   /**
    * Base URL the provider's HTTP client actually resolved to for this request,
    * read from the live SDK client instance rather than re-derived from config.
@@ -281,6 +300,18 @@ export interface SendMessageConfig {
    * request.
    */
   forceOverrideProfile?: boolean;
+  /**
+   * True when the caller appended {@link NATIVE_WEB_SEARCH_TOOL_NAME} purely
+   * to activate the route's provider-native web search, rather than passing an
+   * app-executed search tool of the same name (which is what runs when a
+   * search backend like Brave or the platform search proxy is configured).
+   * Only the caller can tell those apart, and a route change after that
+   * decision has to: `RetryProvider` drops the tool on a backup that runs no
+   * native search, and must not touch it when the daemon executes it itself.
+   * A resolution/routing-time concern only; stripped before any provider wire
+   * request.
+   */
+  nativeWebSearchSentinel?: boolean;
   /**
    * Per-conversation seed for deterministic `mix`-profile expansion. The agent
    * loop sets this to the conversation id so every resolver call this send
@@ -316,6 +347,14 @@ export interface SendMessageConfig {
    * JSON request bodies.
    */
   usageAttributionHeaders?: Record<string, string>;
+  /**
+   * Per-request HTTP headers merged onto the transport. `RetryProvider`
+   * stamps these for providers that need support-lookup headers (OpenCode
+   * `x-opencode-session` / `x-opencode-request`). Provider clients pass
+   * them through SDK request options only and must never include this
+   * object in provider JSON request bodies.
+   */
+  requestHeaders?: Record<string, string>;
   /**
    * Controls local usage-ledger writes for attributed provider calls.
    * Defaults to `auto`; conversation paths that aggregate usage separately

package/src/providers/usage-tracking.ts

@@ -66,9 +66,20 @@ export class UsageTrackingProvider implements Provider {
     }
 
     try {
+      // A fallback-serving wrapper (`RetryProvider`'s escalation) stamps the
+      // profile that actually governed the response; prefer it over the
+      // original request's resolution so a degraded serve bills under the
+      // backup profile, not the failed primary's. `forceOverrideProfile`
+      // mirrors how the fallback dispatch floated that profile above the
+      // call-site layers.
+      const appliedOverride =
+        response.actualInferenceProfile ?? config.overrideProfile;
       const attribution = resolveUsageAttribution({
         callSite: config.callSite,
-        overrideProfile: config.overrideProfile,
+        overrideProfile: appliedOverride,
+        ...(response.actualInferenceProfile !== undefined
+          ? { forceOverrideProfile: true }
+          : {}),
       });
       const providerName = response.actualProvider ?? this.inner.name;
       const pricingUsage = buildPricingUsageFromResponse(

package/src/providers/vellum/client.ts

@@ -0,0 +1,29 @@
+import { OpenAIChatCompletionsProvider } from "../openai/chat-completions-provider.js";
+
+export interface VellumProviderOptions {
+  apiKey?: string;
+  baseURL?: string;
+  streamTimeoutMs?: number;
+}
+
+/**
+ * OpenAI-compatible client for Vellum-hosted GPU inference (vLLM).
+ * Managed requests set `baseURL` to the platform `/v1/runtime-proxy/vellum`
+ * path and authenticate with the assistant API key. These models have no
+ * bring-your-own-key path.
+ */
+export class VellumProvider extends OpenAIChatCompletionsProvider {
+  constructor(
+    apiKey: string,
+    model: string,
+    options: VellumProviderOptions = {},
+  ) {
+    super(apiKey || "not-needed", model, {
+      providerName: "vellum",
+      providerLabel: "Vellum",
+      streamTimeoutMs: options.streamTimeoutMs,
+      omitToolChoiceWhenReasoning: true,
+      ...(options.baseURL ? { baseURL: options.baseURL } : {}),
+    });
+  }
+}

package/src/providers/vellum-model-routing.test.ts

@@ -41,6 +41,7 @@ describe("vellum-model-routing", () => {
 
   test("getManagedUpstream resolves a bare catalog id to its owner", () => {
     expect(getManagedUpstream("claude-fable-5")).toBe("anthropic");
+    expect(getManagedUpstream("qwen/qwen3-8b")).toBe("vellum");
   });
 
   test("getManagedUpstream resolves a routing string by its prefix", () => {
@@ -72,6 +73,7 @@ describe("vellum-model-routing", () => {
       "gemini",
       "openai",
       "together",
+      "vellum",
     ]);
   });
 });

package/src/providers/vellum-model-routing.ts

@@ -36,9 +36,9 @@ export const MANAGED_ROUTABLE_PROVIDERS: ReadonlySet<string> = new Set(
  * Sentinel provider id for the single, provider-agnostic Vellum-managed
  * connection. Unlike the per-provider `*-managed` connections, this one does
  * not name an upstream provider on its DB row — the upstream is determined
- * per-request from the resolving profile. This id is never a real catalog
- * entry, so routing code must substitute the effective provider before any
- * catalog/adapter lookup.
+ * per-request from the resolving profile. The same id is the catalog owner
+ * of Vellum-hosted GPU models. Dispatch still substitutes a concrete
+ * upstream (including `vellum` for those GPU models) before adapter lookup.
  */
 export const VELLUM_MANAGED_PROVIDER = "vellum";
 

package/src/runtime/AGENTS.md

@@ -41,9 +41,8 @@ Approvals are **orthogonal to message sending**. The assistant asks for approval
 
 - **Discovery**: Clients discover pending approvals via SSE events (`confirmation_request`, `secret_request`) which include a `requestId`.
 - **Resolution**: Clients respond via standalone endpoints keyed by `requestId`:
-  - `POST /v1/confirm` — `{ requestId, decision, selectedPattern?, selectedScope? }`. Valid decisions: `"allow"`, `"allow_10m"`, `"allow_conversation"`, `"deny"`, `"always_allow"`, `"always_deny"`. For persistent decisions (`always_allow`, `always_deny`), `selectedPattern` and `selectedScope` are validated against the server-provided allowlist/scope options from the original confirmation request before trust rules are persisted.
+  - `POST /v1/confirm` — `{ requestId, decision }`. Valid decisions: `"allow"`, `"deny"`. The confirm route resolves the pending interaction and nothing else: durable Always-Allow rules are minted through the gateway trust-rules API (`POST /v1/trust-rules`, also served assistant-scoped), which carries the rule's tool, pattern, and risk. The daemon's own `trust-rules` surface is read-only (`GET /v1/trust-rules` list, `POST /v1/trust-rules/suggest`).
   - `POST /v1/secret` — `{ requestId, value, delivery }`
-  - `POST /v1/trust-rules` — `{ requestId, pattern, scope, decision }`. Validates pattern/scope against server-provided options. Does not resolve the confirmation itself.
 - **Tracking**: The `pending-interactions` tracker (`assistant/src/runtime/pending-interactions.ts`) maps `requestId → conversation`. Use `register()` to track, `resolve()` to consume, `getByConversation()` to query.
 
 Do NOT couple approval handling to message sending. Do NOT add run/status tracking to the send path.

package/src/runtime/__tests__/agent-wake.test.ts

@@ -266,6 +266,7 @@ const recordRequestLogCalls: Array<{
 }> = [];
 const recordUsageCalls: Array<{
   conversationId: string;
+  providerName: string | undefined;
   inputTokens: number;
   outputTokens: number;
   model: string;
@@ -279,7 +280,7 @@ const recordUsageCalls: Array<{
 }> = [];
 mock.module("../../daemon/conversation-usage.js", () => ({
   recordUsage: (
-    ctx: { conversationId: string },
+    ctx: { conversationId: string; providerName?: string },
     inputTokens: number,
     outputTokens: number,
     model: string,
@@ -300,6 +301,7 @@ mock.module("../../daemon/conversation-usage.js", () => ({
   ) => {
     recordUsageCalls.push({
       conversationId: ctx.conversationId,
+      providerName: ctx.providerName,
       inputTokens,
       outputTokens,
       model,
@@ -1748,7 +1750,13 @@ describe("wakeAgentForOpportunity", () => {
       { resolveTarget: async () => conversation },
     );
 
-    expect(result).toEqual({ invoked: true, producedToolCalls: false });
+    // The result carries how the run ended, so a caller that treats output as
+    // a finished answer can see that something else ended this one.
+    expect(result).toEqual({
+      invoked: true,
+      producedToolCalls: false,
+      exitReason: "error",
+    });
     // The checkpoint's flush pushed + persisted the partial tail.
     expect(conversation.pushedMessages.length).toBeGreaterThan(0);
     expect(conversation.persistedTailCalls.length).toBeGreaterThan(0);
@@ -1770,7 +1778,11 @@ describe("wakeAgentForOpportunity", () => {
       { resolveTarget: async () => conversation },
     );
 
-    expect(result).toEqual({ invoked: true, producedToolCalls: false });
+    expect(result).toEqual({
+      invoked: true,
+      producedToolCalls: false,
+      exitReason: "no_tool_calls",
+    });
   });
 
   test("reports no_output on a clean empty reply when the caller requires usable output", async () => {
@@ -2986,6 +2998,96 @@ describe("wakeAgentForOpportunity", () => {
     });
   });
 
+  test("rerouted wake records usage under the profile that actually served", async () => {
+    // The wake resolved its own profile before the run; the request then hit
+    // an outage-shaped failure and `RetryProvider` escalated to a backup
+    // profile on a different provider, stamping both `actualProvider` and
+    // `actualInferenceProfile` on the response the loop reports.
+    const usageEvent: AgentEvent = {
+      type: "usage",
+      inputTokens: 100,
+      outputTokens: 5,
+      model: "claude-sonnet-5",
+      actualProvider: "anthropic",
+      actualInferenceProfile: "backupProfile",
+      providerDurationMs: 10,
+      rawRequest: { request: "rerouted wake" },
+      rawResponse: { response: "real reply" },
+    };
+    const conversation = makeWakeConversation({
+      baseline: [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      scriptedEvents: [usageEvent],
+      scriptedAssistant: {
+        role: "assistant",
+        content: [{ type: "text", text: "real reply" }],
+      },
+    });
+
+    await wakeAgentForOpportunity(
+      {
+        conversationId: conversation.conversationId,
+        hint: "do reply",
+        source: "unit-test",
+        callSite: "memoryRetrospective",
+      },
+      { resolveTarget: async () => conversation },
+    );
+
+    // All three attribution facets of the row must describe the same call.
+    // Provider and model already followed the backup off the event; the
+    // profile has to follow it too or the row contradicts itself.
+    expect(recordUsageCalls).toHaveLength(1);
+    expect(recordUsageCalls[0]).toMatchObject({
+      providerName: "anthropic",
+      model: "claude-sonnet-5",
+      overrideProfile: "backupProfile",
+      forceOverrideProfile: true,
+      selectionSeed: conversation.conversationId,
+    });
+  });
+
+  test("non-rerouted wake keeps its own profile resolution", async () => {
+    // No reroute: the event carries no `actualInferenceProfile`, so the wake's
+    // own pre-run resolution stands and nothing is floated above the call site.
+    const usageEvent: AgentEvent = {
+      type: "usage",
+      inputTokens: 100,
+      outputTokens: 5,
+      model: "test-model",
+      actualProvider: "test-provider",
+      providerDurationMs: 10,
+      rawRequest: { request: "normal wake" },
+      rawResponse: { response: "real reply" },
+    };
+    const conversation = makeWakeConversation({
+      baseline: [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      scriptedEvents: [usageEvent],
+      scriptedAssistant: {
+        role: "assistant",
+        content: [{ type: "text", text: "real reply" }],
+      },
+    });
+
+    await wakeAgentForOpportunity(
+      {
+        conversationId: conversation.conversationId,
+        hint: "do reply",
+        source: "unit-test",
+        callSite: "memoryRetrospective",
+      },
+      { resolveTarget: async () => conversation },
+    );
+
+    expect(recordUsageCalls).toHaveLength(1);
+    expect(recordUsageCalls[0]).toMatchObject({
+      providerName: "test-provider",
+      model: "test-model",
+      overrideProfile: null,
+      forceOverrideProfile: false,
+      selectionSeed: conversation.conversationId,
+    });
+  });
+
   test("silent no-op wake still records usage even though its request log is dropped", async () => {
     const usageEvent: AgentEvent = {
       type: "usage",