@vellumai/assistant 0.10.4 → 0.10.5-dev.202607022329.d89bf2f

package/src/providers/speech-to-text/provider-catalog.ts

@@ -17,68 +17,6 @@ import type {
   TelephonySttMode,
 } from "../../stt/types.js";
 
-// ---------------------------------------------------------------------------
-// Telephony routing metadata
-// ---------------------------------------------------------------------------
-
-/**
- * Strategy kind for telephony call setup.
- *
- * Determines how the telephony routing resolver (`telephony-stt-routing.ts`)
- * wires the STT provider into a Twilio call:
- *
- * - `"conversation-relay-native"` — the provider is natively supported by
- *   Twilio ConversationRelay. TwiML includes `transcriptionProvider` /
- *   `speechModel` attributes and Twilio handles audio ingestion.
- * - `"media-stream-custom"` — the provider is not natively supported by
- *   Twilio. A `<Stream>` media-stream is opened and the daemon transcribes
- *   audio server-side via the provider's batch API.
- */
-type TelephonyStrategyKind =
-  | "conversation-relay-native"
-  | "media-stream-custom";
-
-/**
- * Twilio-native ConversationRelay provider name.
- *
- * These are the values Twilio accepts in the `transcriptionProvider` TwiML
- * attribute on `<ConversationRelay>`.
- */
-export type TwilioNativeProvider = "Deepgram" | "Google";
-
-/**
- * Twilio-native mapping details for providers routed through
- * ConversationRelay. Only present when `strategyKind` is
- * `"conversation-relay-native"`.
- */
-interface TwilioNativeMapping {
-  /** Twilio-native provider name for the TwiML `transcriptionProvider` attribute. */
-  readonly provider: TwilioNativeProvider;
-  /**
-   * Default ASR speech model identifier, or `undefined` to use the
-   * provider's default model. Individual providers override as needed
-   * (e.g. Deepgram defaults to `"nova-3"`).
-   */
-  readonly defaultSpeechModel: string | undefined;
-}
-
-/**
- * Telephony routing metadata — the single source of truth for how a
- * provider is wired into Twilio call setup.
- *
- * The telephony routing resolver reads these fields from the catalog
- * instead of maintaining its own hardcoded maps.
- */
-interface TelephonyRouting {
-  /** Which Twilio call-setup strategy this provider uses. */
-  readonly strategyKind: TelephonyStrategyKind;
-  /**
-   * Twilio-native mapping details. Present when `strategyKind` is
-   * `"conversation-relay-native"`, absent for `"media-stream-custom"`.
-   */
-  readonly twilioNativeMapping?: TwilioNativeMapping;
-}
-
 // ---------------------------------------------------------------------------
 // Client display metadata
 // ---------------------------------------------------------------------------
@@ -161,13 +99,6 @@ interface SttProviderEntry {
    */
   readonly supportsDiarization: boolean;
 
-  /**
-   * Telephony routing metadata — describes how this provider is wired
-   * into Twilio call setup. This is the single source of truth for
-   * strategy selection and Twilio-native mapping details.
-   */
-  readonly telephonyRouting: TelephonyRouting;
-
   /** Guide for obtaining API credentials from this provider. */
   readonly credentialsGuide?: SttCredentialsGuide;
 }
@@ -206,13 +137,6 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
       telephonyMode: "realtime-ws",
       conversationStreamingMode: "realtime-ws",
       supportsDiarization: true,
-      telephonyRouting: {
-        strategyKind: "conversation-relay-native",
-        twilioNativeMapping: {
-          provider: "Deepgram",
-          defaultSpeechModel: "nova-3",
-        },
-      },
       credentialsGuide: {
         description:
           "Sign in to the Deepgram console, navigate to API Keys, and create a new key.",
@@ -239,13 +163,6 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
       telephonyMode: "batch-only",
       conversationStreamingMode: "realtime-ws",
       supportsDiarization: false,
-      telephonyRouting: {
-        strategyKind: "conversation-relay-native",
-        twilioNativeMapping: {
-          provider: "Google",
-          defaultSpeechModel: undefined,
-        },
-      },
       credentialsGuide: {
         description:
           "Visit Google AI Studio, sign in with your Google account, and create an API key.",
@@ -271,9 +188,6 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
       telephonyMode: "batch-only",
       conversationStreamingMode: "incremental-batch",
       supportsDiarization: false,
-      telephonyRouting: {
-        strategyKind: "media-stream-custom",
-      },
       credentialsGuide: {
         description:
           "Log in to the OpenAI platform, go to API Keys, and generate a new secret key.",
@@ -299,9 +213,6 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
       telephonyMode: "batch-only",
       conversationStreamingMode: "realtime-ws",
       supportsDiarization: true,
-      telephonyRouting: {
-        strategyKind: "media-stream-custom",
-      },
       credentialsGuide: {
         description:
           "Sign in to the xAI console, navigate to API Keys, and create a new key.",

package/src/providers/speech-to-text/resolve.ts

@@ -260,6 +260,17 @@ export interface ResolveStreamingTranscriberOptions {
    * See {@link DiarizePreference} for semantics.
    */
   diarize?: DiarizePreference;
+  /**
+   * Emit `final` events only at utterance boundaries. Supported only by
+   * providers whose catalog `telephonyMode` is `"realtime-ws"` (Deepgram,
+   * where it also enables `utterance_end_ms` endpointing). All other
+   * providers resolve to `null` so the caller falls back to per-turn
+   * batch transcription — e.g. openai-whisper fires `final` only from
+   * `stop()` (end-of-stream, not end-of-utterance) and xAI emits a
+   * `final` per committed segment. Used by telephony call ingestion.
+   * Default: false.
+   */
+  utteranceBoundaryFinals?: boolean;
 }
 
 /**
@@ -277,6 +288,8 @@ export interface ResolveStreamingTranscriberOptions {
  * - No credentials are configured for the resolved provider.
  * - No streaming adapter exists for the configured provider.
  * - `diarize` is `"required"` but the configured provider cannot diarize.
+ * - `utteranceBoundaryFinals` is set but the configured provider's catalog
+ *   `telephonyMode` is not `"realtime-ws"`.
  */
 export async function resolveStreamingTranscriber(
   options: ResolveStreamingTranscriberOptions = {},
@@ -298,6 +311,26 @@ export async function resolveStreamingTranscriber(
     return null;
   }
 
+  // Boundary-requiring callers (telephony) can only stream on providers
+  // whose catalog telephonyMode is "realtime-ws" (Deepgram gates finals on
+  // utterance boundaries). Everything else fires `final` either only from
+  // stop() — end-of-stream, not end-of-utterance (openai-whisper) — or per
+  // committed segment (xAI), so streaming would yield no replies until
+  // hangup, or mid-sentence replies. Resolve to null so the caller falls
+  // back to per-turn batch transcription.
+  if (options.utteranceBoundaryFinals) {
+    const telephonyMode = getProviderEntry(
+      provider as SttProviderId,
+    )?.telephonyMode;
+    if (telephonyMode !== "realtime-ws") {
+      log.warn(
+        { providerId: provider, telephonyMode },
+        "utterance-boundary finals requested but the configured STT provider has no realtime telephony streaming — falling back to batch transcription",
+      );
+      return null;
+    }
+  }
+
   // Resolve diarization capability against the catalog. For `"required"`
   // callers, bail early (with a warning) when the configured provider can't
   // diarize so the caller can surface a clear error to the user.
@@ -323,9 +356,18 @@ export async function resolveStreamingTranscriber(
   return createStreamingTranscriber(apiKey, provider as SttProviderId, {
     sampleRate: options.sampleRate,
     diarize: enableDiarization,
+    utteranceBoundaryFinals: options.utteranceBoundaryFinals ?? false,
   });
 }
 
+/**
+ * Deepgram `utterance_end_ms` used when utterance-boundary finals are
+ * requested. Deepgram requires >= 1000 ms; this is the pause length after
+ * which an `UtteranceEnd` frame confirms the utterance is complete even
+ * when `speech_final` endpointing never fired (e.g. background noise).
+ */
+const UTTERANCE_BOUNDARY_END_MS = 1_000;
+
 /**
  * Options forwarded to individual streaming adapter constructors.
  */
@@ -338,6 +380,13 @@ interface CreateStreamingTranscriberOptions {
    * support.
    */
   diarize?: boolean;
+  /**
+   * Whether `final` events should be gated on utterance boundaries.
+   * Only forwarded to Deepgram; the resolver never sets this for
+   * providers without realtime telephony streaming (they resolve to
+   * `null` instead).
+   */
+  utteranceBoundaryFinals?: boolean;
 }
 
 /**
@@ -361,6 +410,12 @@ async function createStreamingTranscriber(
       return new DeepgramRealtimeTranscriber(apiKey, {
         sampleRate: options.sampleRate,
         ...(options.diarize ? { diarize: true } : {}),
+        ...(options.utteranceBoundaryFinals
+          ? {
+              utteranceBoundaryFinals: true,
+              utteranceEndMs: UTTERANCE_BOUNDARY_END_MS,
+            }
+          : {}),
       });
     }
     case "google-gemini": {

package/src/providers/tool-progress-events.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, test } from "bun:test";
+
+import { createToolProgressEmitter } from "./tool-progress-events.js";
+import type { ProviderEvent } from "./types.js";
+
+describe("createToolProgressEmitter", () => {
+  test("flushes the latest throttled argument progress after activity pauses", async () => {
+    const events: ProviderEvent[] = [];
+    const progress = createToolProgressEmitter((event) => events.push(event));
+
+    progress.emitInputJsonDelta("call-1", "app_create", '{"a":1}');
+    progress.emitInputJsonDelta("call-1", "app_create", '{"a":12}');
+
+    expect(events).toEqual([
+      {
+        type: "tool_use_preview_start",
+        toolUseId: "call-1",
+        toolName: "app_create",
+      },
+      {
+        type: "input_json_delta",
+        toolUseId: "call-1",
+        toolName: "app_create",
+        accumulatedJson: '{"a":1}',
+      },
+    ]);
+
+    await Bun.sleep(180);
+
+    expect(events).toEqual([
+      {
+        type: "tool_use_preview_start",
+        toolUseId: "call-1",
+        toolName: "app_create",
+      },
+      {
+        type: "input_json_delta",
+        toolUseId: "call-1",
+        toolName: "app_create",
+        accumulatedJson: '{"a":1}',
+      },
+      {
+        type: "input_json_delta",
+        toolUseId: "call-1",
+        toolName: "app_create",
+        accumulatedJson: '{"a":12}',
+      },
+    ]);
+  });
+});

package/src/providers/tool-progress-events.ts

@@ -0,0 +1,99 @@
+import type { ProviderEvent } from "./types.js";
+
+const INPUT_JSON_DELTA_EMIT_INTERVAL_MS = 150;
+
+export function createToolProgressEmitter(
+  onEvent: ((event: ProviderEvent) => void) | undefined,
+): {
+  emitPreviewStart: (toolUseId: string, toolName: string) => void;
+  emitInputJsonDelta: (
+    toolUseId: string,
+    toolName: string,
+    accumulatedJson: string,
+    opts?: { force?: boolean },
+  ) => void;
+} {
+  const previewedToolUseIds = new Set<string>();
+  const lastInputEmitByToolUseId = new Map<
+    string,
+    { emittedAt: number; accumulatedJson: string }
+  >();
+  const pendingInputEmitByToolUseId = new Map<
+    string,
+    {
+      timeout: ReturnType<typeof setTimeout>;
+      toolName: string;
+      accumulatedJson: string;
+    }
+  >();
+
+  const emitPreviewStart = (toolUseId: string, toolName: string): void => {
+    if (!onEvent || !toolUseId || !toolName) return;
+    if (previewedToolUseIds.has(toolUseId)) return;
+    previewedToolUseIds.add(toolUseId);
+    onEvent({ type: "tool_use_preview_start", toolUseId, toolName });
+  };
+
+  const emitInputJsonDelta = (
+    toolUseId: string,
+    toolName: string,
+    accumulatedJson: string,
+    opts?: { force?: boolean },
+  ): void => {
+    if (!onEvent || !toolUseId || !toolName || !accumulatedJson) return;
+    emitPreviewStart(toolUseId, toolName);
+
+    const last = lastInputEmitByToolUseId.get(toolUseId);
+    if (last?.accumulatedJson === accumulatedJson) return;
+
+    const now = Date.now();
+    if (
+      !opts?.force &&
+      last &&
+      now - last.emittedAt < INPUT_JSON_DELTA_EMIT_INTERVAL_MS
+    ) {
+      const pending = pendingInputEmitByToolUseId.get(toolUseId);
+      if (pending) {
+        pending.toolName = toolName;
+        pending.accumulatedJson = accumulatedJson;
+        return;
+      }
+      const delay = INPUT_JSON_DELTA_EMIT_INTERVAL_MS - (now - last.emittedAt);
+      const timeout = setTimeout(() => {
+        const latest = pendingInputEmitByToolUseId.get(toolUseId);
+        if (!latest) return;
+        pendingInputEmitByToolUseId.delete(toolUseId);
+        lastInputEmitByToolUseId.set(toolUseId, {
+          emittedAt: Date.now(),
+          accumulatedJson: latest.accumulatedJson,
+        });
+        onEvent({
+          type: "input_json_delta",
+          toolName: latest.toolName,
+          toolUseId,
+          accumulatedJson: latest.accumulatedJson,
+        });
+      }, delay);
+      timeout.unref?.();
+      pendingInputEmitByToolUseId.set(toolUseId, {
+        timeout,
+        toolName,
+        accumulatedJson,
+      });
+      return;
+    }
+
+    const pending = pendingInputEmitByToolUseId.get(toolUseId);
+    if (pending) {
+      clearTimeout(pending.timeout);
+      pendingInputEmitByToolUseId.delete(toolUseId);
+    }
+    lastInputEmitByToolUseId.set(toolUseId, {
+      emittedAt: now,
+      accumulatedJson,
+    });
+    onEvent({ type: "input_json_delta", toolName, toolUseId, accumulatedJson });
+  };
+
+  return { emitPreviewStart, emitInputJsonDelta };
+}

package/src/providers/types.ts

@@ -1,4 +1,4 @@
-import type { ToolDefinition } from "@vellumai/skill-host-contracts";
+import type { ToolDefinition } from "../tools/tool-types.js";
 export type { ToolDefinition };
 
 import type { LLMCallSite } from "../config/schemas/llm.js";

package/src/providers/vellum-model-routing.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  formatVellumModel,
+  MANAGED_ROUTABLE_PROVIDERS,
+  parseVellumModel,
+} from "./vellum-model-routing.js";
+
+describe("vellum-model-routing", () => {
+  test("round-trips a slashy native Fireworks id losslessly", () => {
+    const native = "accounts/fireworks/models/minimax-m3";
+    const encoded = formatVellumModel("fireworks", native);
+    expect(encoded).toBe("fireworks/accounts/fireworks/models/minimax-m3");
+    expect(parseVellumModel(encoded)).toEqual({
+      provider: "fireworks",
+      model: native,
+    });
+  });
+
+  test("round-trips a bare Anthropic id", () => {
+    const encoded = formatVellumModel("anthropic", "claude-fable-5");
+    expect(parseVellumModel(encoded)).toEqual({
+      provider: "anthropic",
+      model: "claude-fable-5",
+    });
+  });
+
+  test("format rejects a non-managed provider", () => {
+    expect(() => formatVellumModel("openrouter", "x")).toThrow();
+    expect(() => formatVellumModel("fireworks", "")).toThrow();
+  });
+
+  test("parse returns null for non-routed strings", () => {
+    expect(parseVellumModel("claude-opus-4-8")).toBeNull(); // no slash
+    expect(parseVellumModel("openrouter/whatever")).toBeNull(); // not managed
+    expect(parseVellumModel("minimax/minimax-m3")).toBeNull(); // not managed
+    expect(parseVellumModel("fireworks/")).toBeNull(); // empty model
+    expect(parseVellumModel("/foo")).toBeNull(); // empty provider
+  });
+
+  test("managed set matches the platform proxy table", () => {
+    // Guards against drift if PLATFORM_PROVIDER_META changes.
+    expect([...MANAGED_ROUTABLE_PROVIDERS].sort()).toEqual([
+      "anthropic",
+      "fireworks",
+      "gemini",
+      "openai",
+      "together",
+    ]);
+  });
+});

package/src/providers/vellum-model-routing.ts

@@ -0,0 +1,84 @@
+/**
+ * Codec for Vellum-managed model routing strings.
+ *
+ * A single `vellum` provider connection is provider-agnostic: unlike the
+ * per-provider `*-managed` connections it does not carry the upstream provider
+ * on its DB row. The upstream provider therefore has to travel *with* the
+ * model. Where a routing site has the provider as a sibling field it uses that
+ * directly; where only a single model string is available (telemetry headers
+ * like `X-Vellum-Resolved-Model`, persisted model overrides, display), the
+ * provider is encoded as a `<provider>/<model>` prefix, e.g.
+ * `fireworks/accounts/fireworks/models/minimax-m3`.
+ *
+ * This module is the single source of truth for that encoding. It is a pure
+ * codec — no I/O, no wiring — so it can be adopted incrementally.
+ */
+
+import { PLATFORM_PROVIDER_META } from "./platform-proxy/constants.js";
+
+/**
+ * Provider ids that can front a Vellum-managed (platform-proxied) route.
+ * Only these are valid prefixes for a routing string; everything else
+ * (openrouter, ollama, openai-compatible, or a raw native model id) is not a
+ * Vellum-routed string and `parseVellumModel` returns null for it.
+ */
+export const MANAGED_ROUTABLE_PROVIDERS: ReadonlySet<string> = new Set(
+  Object.values(PLATFORM_PROVIDER_META)
+    .filter((m) => m.managed)
+    .map((m) => m.name),
+);
+
+export interface VellumModelRoute {
+  /** Upstream provider id, e.g. "fireworks". Always a managed-routable id. */
+  provider: string;
+  /** Native upstream model id, slashes intact, e.g. "accounts/fireworks/models/minimax-m3". */
+  model: string;
+}
+
+/**
+ * Encode a provider + native model id into a Vellum routing string.
+ * Throws on a non-routable provider — that is a caller bug, not user input.
+ */
+export function formatVellumModel(provider: string, model: string): string {
+  if (!MANAGED_ROUTABLE_PROVIDERS.has(provider)) {
+    throw new Error(
+      `formatVellumModel: "${provider}" is not a Vellum-managed provider ` +
+        `(expected one of ${[...MANAGED_ROUTABLE_PROVIDERS].join(", ")})`,
+    );
+  }
+  if (!model) {
+    throw new Error("formatVellumModel: model must be non-empty");
+  }
+  return `${provider}/${model}`;
+}
+
+/**
+ * Decode a Vellum routing string back into provider + native model id.
+ *
+ * Splits on the FIRST slash only so native ids that themselves contain
+ * slashes (Fireworks `accounts/fireworks/models/...`) round-trip losslessly.
+ * Returns null when the string is not a Vellum-routed model — no slash, empty
+ * model, or a prefix that is not a managed-routable provider.
+ *
+ * Note: `anthropic/…` is also OpenRouter's native id shape, so this codec
+ * must only be applied in a Vellum-connection context. OpenRouter is
+ * `managed:false` and never routes through a vellum connection, so the
+ * collision is unreachable in practice; the guard here is belt-and-suspenders.
+ */
+export function parseVellumModel(
+  routingString: string,
+): VellumModelRoute | null {
+  const slash = routingString.indexOf("/");
+  if (slash <= 0) {
+    return null;
+  }
+  const provider = routingString.slice(0, slash);
+  const model = routingString.slice(slash + 1);
+  if (!model) {
+    return null;
+  }
+  if (!MANAGED_ROUTABLE_PROVIDERS.has(provider)) {
+    return null;
+  }
+  return { provider, model };
+}

package/src/runtime/AGENTS.md

@@ -163,15 +163,16 @@ Verification SESSION state (pending sessions, codes, resend, rate-limit) is assi
 
 The verified outcome is written in-process by the gateway: the HTTP guardian-attest handler calls `ContactStore.markChannelVerified` directly (verifiedVia "manual"), and the inbound code-match path (`gateway/src/verification/text-verification.ts`) writes via `upsertVerifiedContactChannel` / `createGuardianBinding` (verifiedVia "challenge"). The revoke/downgrade outcome is relayed from the daemon via `ipcCallPersistent("mark_channel_revoked", …)` to `ContactStore.markChannelRevoked`.
 
-The `mark_channel_verified` IPC method exists as the daemon/CLI relay surface (symmetric with `mark_channel_revoked`) but has no caller: the trusted-contact CLI path sends codes only, and the outcome arrives via the inbound code-match path.
-
 ## Rate Limiting & Diagnostics
 
-All `/v1/*` endpoints share a per-client-IP sliding-window rate limiter (`middleware/rate-limiter.ts`):
+Most `/v1/*` endpoints share a per-client-IP sliding-window rate limiter (`middleware/rate-limiter.ts`):
 
-- **Authenticated**: 300 requests/minute
+- **Authenticated (loopback)**: 1200 requests/minute — desktop app, CLI, anything on the daemon's own host (`127.0.0.0/8`, `::1`). A cold sidebar load at thousands of conversations legitimately bursts far beyond the remote budget.
+- **Authenticated (remote)**: 300 requests/minute — proxied non-loopback clients, keyed by the forwarded client IP.
 - **Unauthenticated**: 20 requests/minute
 
+**Exempt endpoints** (`isRateLimitExemptEndpoint`): the SSE stream (`events`) and liveness/readiness probes (`health`, `healthz`, `readyz`) bypass the per-minute limiter entirely. The stream is one long-lived connection, not a burst of requests, and 429-ing it drops the stream — which drives a client reconnect + full re-bootstrap loop that generates far more load than the limiter saves. Liveness probes must always answer or the client treats the assistant as down and reconnects harder. The events route still enforces auth downstream, and stream memory is bounded by SSE backpressure shedding + subscriber caps.
+
 When the limit is exceeded, the limiter returns 429 and logs a structured warning (module: `rate-limiter`) with the denied endpoint and a breakdown of which endpoints consumed the budget in the current window. This makes it easy to identify whether the cause is rapid conversation switching, polling, or unexpected request volume.
 
 Logs rotate daily into `$VELLUM_WORKSPACE_DIR/data/logs/assistant-YYYY-MM-DD.log` (or into the directory configured via `logFile.dir`). To watch rate limit events in real time: