@vellumai/assistant 0.10.8 → 0.10.9-dev.202607152225.ec6fdbf

package/src/platform/managed-speech.test.ts

@@ -0,0 +1,218 @@
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+interface MockPlatformClient {
+  platformAssistantId: string;
+  fetch: ReturnType<typeof mock>;
+}
+
+let mockClient: MockPlatformClient | null = null;
+
+mock.module("./client.js", () => ({
+  VellumPlatformClient: {
+    create: async () => mockClient,
+  },
+}));
+
+import {
+  managedSpeechSynthesize,
+  managedSpeechTranscribe,
+} from "./managed-speech.js";
+
+const AUDIO = Buffer.from([0x01, 0x02, 0x03, 0xff]);
+
+describe("managedSpeechTranscribe", () => {
+  beforeEach(() => {
+    mockClient = {
+      platformAssistantId: "asst-123",
+      fetch: mock(
+        async () =>
+          new Response(
+            JSON.stringify({
+              text: "hello world",
+              providerId: "deepgram",
+              model: "nova-3",
+              durationSeconds: 12.5,
+            }),
+            { status: 200, headers: { "content-type": "application/json" } },
+          ),
+      ),
+    };
+  });
+
+  afterEach(() => {
+    mockClient = null;
+  });
+
+  test("posts base64 audio to the assistant's transcribe endpoint", async () => {
+    const result = await managedSpeechTranscribe({
+      audio: AUDIO,
+      mimeType: "audio/wav",
+      source: "dictation",
+    });
+
+    expect(result).toEqual({
+      ok: true,
+      value: { text: "hello world", durationSeconds: 12.5 },
+    });
+    const [path, init] = mockClient!.fetch.mock.calls[0] as [
+      string,
+      RequestInit,
+    ];
+    expect(path).toBe("/v1/assistants/asst-123/managed-speech/stt/transcribe/");
+    expect(init.method).toBe("POST");
+    const body = JSON.parse(init.body as string);
+    expect(body).toEqual({
+      audioBase64: AUDIO.toString("base64"),
+      mimeType: "audio/wav",
+      source: "dictation",
+    });
+  });
+
+  test("omits source when not provided", async () => {
+    await managedSpeechTranscribe({ audio: AUDIO, mimeType: "audio/wav" });
+    const [, init] = mockClient!.fetch.mock.calls[0] as [string, RequestInit];
+    expect(JSON.parse(init.body as string)).not.toHaveProperty("source");
+  });
+
+  test("returns unavailable when no platform client resolves", async () => {
+    mockClient = null;
+    const result = await managedSpeechTranscribe({
+      audio: AUDIO,
+      mimeType: "audio/wav",
+    });
+    expect(result).toMatchObject({ ok: false, kind: "unavailable" });
+  });
+
+  test("returns unavailable when the assistant ID is missing", async () => {
+    mockClient!.platformAssistantId = "";
+    const result = await managedSpeechTranscribe({
+      audio: AUDIO,
+      mimeType: "audio/wav",
+    });
+    expect(result).toMatchObject({ ok: false, kind: "unavailable" });
+  });
+
+  test("surfaces the platform error code on 402", async () => {
+    mockClient!.fetch = mock(
+      async () =>
+        new Response(
+          JSON.stringify({
+            code: "insufficient_balance",
+            detail: "Insufficient balance. Please add funds to continue.",
+          }),
+          { status: 402, headers: { "content-type": "application/json" } },
+        ),
+    );
+    const result = await managedSpeechTranscribe({
+      audio: AUDIO,
+      mimeType: "audio/wav",
+    });
+    expect(result).toMatchObject({
+      ok: false,
+      kind: "platform-error",
+      status: 402,
+      code: "insufficient_balance",
+      message: "Insufficient balance. Please add funds to continue.",
+    });
+  });
+
+  test("treats a malformed 200 body as a platform error", async () => {
+    mockClient!.fetch = mock(
+      async () =>
+        new Response(JSON.stringify({ unexpected: true }), {
+          status: 200,
+          headers: { "content-type": "application/json" },
+        }),
+    );
+    const result = await managedSpeechTranscribe({
+      audio: AUDIO,
+      mimeType: "audio/wav",
+    });
+    expect(result).toMatchObject({ ok: false, kind: "platform-error" });
+  });
+});
+
+describe("managedSpeechSynthesize", () => {
+  const MP3_BYTES = Buffer.from([0x49, 0x44, 0x33, 0x04, 0x00]);
+
+  beforeEach(() => {
+    mockClient = {
+      platformAssistantId: "asst-123",
+      fetch: mock(
+        async () =>
+          new Response(new Uint8Array(MP3_BYTES), {
+            status: 200,
+            headers: { "content-type": "audio/mpeg" },
+          }),
+      ),
+    };
+  });
+
+  afterEach(() => {
+    mockClient = null;
+  });
+
+  test("posts text and format, round-trips binary audio and content type", async () => {
+    const result = await managedSpeechSynthesize({
+      text: "hello",
+      format: "pcm_16000",
+    });
+
+    expect(result.ok).toBe(true);
+    if (result.ok) {
+      expect(Buffer.compare(result.value.audio, MP3_BYTES)).toBe(0);
+      expect(result.value.contentType).toBe("audio/mpeg");
+    }
+    const [path, init] = mockClient!.fetch.mock.calls[0] as [
+      string,
+      RequestInit,
+    ];
+    expect(path).toBe("/v1/assistants/asst-123/managed-speech/tts/synthesize/");
+    expect(JSON.parse(init.body as string)).toEqual({
+      text: "hello",
+      format: "pcm_16000",
+    });
+  });
+
+  test("falls back to audio/mpeg when no content type is returned", async () => {
+    mockClient!.fetch = mock(
+      async () => new Response(new Uint8Array(MP3_BYTES), { status: 200 }),
+    );
+    const result = await managedSpeechSynthesize({
+      text: "hello",
+      format: "mp3",
+    });
+    expect(result.ok && result.value.contentType).toBe("audio/mpeg");
+  });
+
+  test("treats empty audio as a platform error", async () => {
+    mockClient!.fetch = mock(
+      async () => new Response(new Uint8Array(0), { status: 200 }),
+    );
+    const result = await managedSpeechSynthesize({
+      text: "hello",
+      format: "mp3",
+    });
+    expect(result).toMatchObject({ ok: false, kind: "platform-error" });
+  });
+
+  test("surfaces insufficient_balance on 402", async () => {
+    mockClient!.fetch = mock(
+      async () =>
+        new Response(
+          JSON.stringify({ code: "insufficient_balance", detail: "Top up." }),
+          { status: 402, headers: { "content-type": "application/json" } },
+        ),
+    );
+    const result = await managedSpeechSynthesize({
+      text: "hello",
+      format: "mp3",
+    });
+    expect(result).toMatchObject({
+      ok: false,
+      kind: "platform-error",
+      status: 402,
+      code: "insufficient_balance",
+    });
+  });
+});

package/src/platform/managed-speech.ts

@@ -0,0 +1,204 @@
+/**
+ * Client for the platform's managed speech endpoints: Vellum-held Deepgram
+ * key, billed to the org's credits. The daemon sends complete audio (STT) or
+ * text (TTS) and the platform does the provider call — no speech credentials
+ * ever live on this machine.
+ *
+ * Contract (mirrors `vellum-assistant-platform` — do not change unilaterally):
+ * - `POST /v1/assistants/{id}/managed-speech/stt/transcribe/`
+ *   `{audioBase64, mimeType, source?}` → 200 `{text, providerId, model,
+ *   durationSeconds}`.
+ * - `POST /v1/assistants/{id}/managed-speech/tts/synthesize/`
+ *   `{text, format?}` → 200 binary audio, `Content-Type` from upstream.
+ * - Errors: `{code, detail}`; notable codes: `insufficient_balance` (402),
+ *   `missing_price` (400), `provider_configuration_error` (500).
+ *
+ * Pure platform HTTP client: no provider-catalog, config-schema, or resolver
+ * imports — the vellum STT/TTS providers build on top of it.
+ */
+
+import { VellumPlatformClient } from "./client.js";
+
+export type ManagedSpeechResult<T> =
+  | { ok: true; value: T }
+  | {
+      ok: false;
+      kind: "unavailable" | "platform-error";
+      status?: number;
+      /** Platform error code (e.g. `insufficient_balance`) when the body carried one. */
+      code?: string;
+      message: string;
+    };
+
+export interface ManagedSpeechTranscription {
+  text: string;
+  durationSeconds: number;
+}
+
+export interface ManagedSpeechSynthesis {
+  audio: Buffer;
+  contentType: string;
+}
+
+export type ManagedSpeechTtsFormat = "mp3" | "wav_8000" | "pcm_16000";
+
+async function resolveClient(): Promise<
+  { client: VellumPlatformClient } | { error: ManagedSpeechResult<never> }
+> {
+  const client = await VellumPlatformClient.create();
+  if (!client) {
+    return {
+      error: {
+        ok: false,
+        kind: "unavailable",
+        message:
+          "Managed speech is unavailable: no Vellum platform connection.",
+      },
+    };
+  }
+  if (!client.platformAssistantId) {
+    return {
+      error: {
+        ok: false,
+        kind: "unavailable",
+        message:
+          "Managed speech is unavailable: platform assistant ID is missing.",
+      },
+    };
+  }
+  return { client };
+}
+
+/**
+ * Whether managed speech can be used at all: a platform connection whose
+ * assistant identity is fully provisioned. A stored API key alone is not
+ * enough — synthesis/transcription would fail before any request is made.
+ */
+export async function managedSpeechAvailable(): Promise<boolean> {
+  return !("error" in (await resolveClient()));
+}
+
+export async function managedSpeechTranscribe(input: {
+  audio: Buffer;
+  mimeType: string;
+  source?: string;
+  signal?: AbortSignal;
+}): Promise<ManagedSpeechResult<ManagedSpeechTranscription>> {
+  const resolved = await resolveClient();
+  if ("error" in resolved) {
+    return resolved.error;
+  }
+  const { client } = resolved;
+
+  const response = await client.fetch(
+    `/v1/assistants/${encodeURIComponent(client.platformAssistantId)}/managed-speech/stt/transcribe/`,
+    {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        audioBase64: input.audio.toString("base64"),
+        mimeType: input.mimeType,
+        ...(input.source !== undefined ? { source: input.source } : {}),
+      }),
+      signal: input.signal,
+    },
+  );
+
+  if (!response.ok) {
+    return await platformError(response, "transcription");
+  }
+
+  const body: unknown = await response.json().catch(() => null);
+  if (
+    !body ||
+    typeof body !== "object" ||
+    typeof (body as { text?: unknown }).text !== "string" ||
+    typeof (body as { durationSeconds?: unknown }).durationSeconds !== "number"
+  ) {
+    return {
+      ok: false,
+      kind: "platform-error",
+      status: response.status,
+      message: "Managed speech transcription returned a malformed response.",
+    };
+  }
+  const parsed = body as { text: string; durationSeconds: number };
+  return {
+    ok: true,
+    value: { text: parsed.text, durationSeconds: parsed.durationSeconds },
+  };
+}
+
+export async function managedSpeechSynthesize(input: {
+  text: string;
+  format: ManagedSpeechTtsFormat;
+  signal?: AbortSignal;
+}): Promise<ManagedSpeechResult<ManagedSpeechSynthesis>> {
+  const resolved = await resolveClient();
+  if ("error" in resolved) {
+    return resolved.error;
+  }
+  const { client } = resolved;
+
+  const response = await client.fetch(
+    `/v1/assistants/${encodeURIComponent(client.platformAssistantId)}/managed-speech/tts/synthesize/`,
+    {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ text: input.text, format: input.format }),
+      signal: input.signal,
+    },
+  );
+
+  if (!response.ok) {
+    return await platformError(response, "synthesis");
+  }
+
+  const audio = Buffer.from(await response.arrayBuffer());
+  if (audio.length === 0) {
+    return {
+      ok: false,
+      kind: "platform-error",
+      status: response.status,
+      message: "Managed speech synthesis returned empty audio.",
+    };
+  }
+  return {
+    ok: true,
+    value: {
+      audio,
+      contentType: response.headers.get("content-type") ?? "audio/mpeg",
+    },
+  };
+}
+
+async function platformError(
+  response: Response,
+  operation: string,
+): Promise<ManagedSpeechResult<never>> {
+  let code: string | undefined;
+  let detail: string | undefined;
+  try {
+    const body = (await response.json()) as {
+      code?: unknown;
+      detail?: unknown;
+    };
+    if (typeof body?.code === "string") {
+      code = body.code;
+    }
+    if (typeof body?.detail === "string") {
+      detail = body.detail;
+    }
+  } catch {
+    // Non-JSON error body; status alone will have to do.
+  }
+  return {
+    ok: false,
+    kind: "platform-error",
+    status: response.status,
+    code,
+    message:
+      detail ??
+      `Managed speech ${operation} failed (platform returned ${response.status}).`,
+  };
+}

package/src/plugin-api/conversation-turn.ts

@@ -0,0 +1,250 @@
+/**
+ * Plugin-facing facade for running a full conversation agent-loop turn.
+ *
+ * This is the conversation-scoped equivalent of the daemon's
+ * `processMessage` path: it persists a user message, runs the agent loop
+ * (with all its machinery -- system prompt construction, conversation
+ * history, tool use cycles, compaction, injections), and returns the
+ * assistant's full content-block response.
+ *
+ * Plugins that need to drive conversation turns (e.g. meeting-bot
+ * flushing a transcript excerpt) should prefer this over the stateless
+ * `provider.sendMessage()` call, which has no history, no tools, and no
+ * context management.
+ */
+
+import type { LLMCallSite } from "../config/schemas/llm.js";
+import type { ServerMessage } from "../daemon/message-protocol.js";
+import type { UserMessageAttachment } from "../daemon/message-types/shared.js";
+import type { ContentBlock, MediaSource } from "../providers/types.js";
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+export interface RunConversationTurnOptions {
+  /**
+   * Conversation to run the turn in. If omitted, a new conversation is
+   * created (its ID is generated with `uuidv7` and returned in the result).
+   */
+  conversationId?: string;
+  /**
+   * User message content blocks for this turn. Text blocks become the
+   * user message body; image/file blocks are resolved to inline
+   * attachments. Other block types (tool_use, tool_result, thinking) are
+   * ignored as they are not valid user input.
+   */
+  content: ContentBlock[];
+  /**
+   * LLM call-site for inference profile resolution. Defaults to
+   * `"mainAgent"` inside the agent loop when omitted.
+   */
+  callSite?: LLMCallSite;
+  /**
+   * Abort signal. When aborted, the conversation's internal abort
+   * controller fires, terminating the in-flight agent loop.
+   */
+  signal?: AbortSignal;
+}
+
+export interface RunConversationTurnResult {
+  /** The assistant's full content blocks for this turn (text, tool_use, etc.). */
+  content: ContentBlock[];
+  /** The user message row ID assigned by the persistence layer. */
+  userMessageId: string;
+  /** The conversation this turn ran in. */
+  conversationId: string;
+  /** True when the message was queued because the conversation was busy. */
+  queued?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Content conversion
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract a plain-text content string and attachment list from
+ * {@link ContentBlock} input. Text blocks are concatenated (newline
+ * separated); image and file blocks are converted to
+ * {@link UserMessageAttachment} entries with their media source resolved
+ * to inline base64. Other block types are ignored.
+ */
+function extractContentAndAttachments(
+  blocks: ContentBlock[],
+  resolveMedia: (
+    source: MediaSource,
+  ) => { data: string; media_type: string } | null,
+): { text: string; attachments: UserMessageAttachment[] } {
+  const textParts: string[] = [];
+  const attachments: UserMessageAttachment[] = [];
+
+  for (const block of blocks) {
+    if (block.type === "text") {
+      textParts.push(block.text);
+    } else if (block.type === "image" || block.type === "file") {
+      const source = block.source;
+      const resolved = resolveMedia(source);
+      if (resolved) {
+        attachments.push({
+          filename:
+            source.filename ?? (block.type === "image" ? "image" : "file"),
+          mimeType: resolved.media_type,
+          data: resolved.data,
+          ...(block.type === "file" && block.extracted_text
+            ? { extractedText: block.extracted_text }
+            : {}),
+        });
+      }
+    }
+  }
+
+  return { text: textParts.join("\n"), attachments };
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a full conversation agent-loop turn: persist the user message, execute
+ * the agent loop (history, tools, compaction, injections), and return the
+ * assistant's full content-block response.
+ *
+ * When `conversationId` is omitted, a new conversation is created and its ID
+ * is returned in the result.
+ *
+ * Events are fanned out to SSE/event-hub subscribers via `broadcastMessage`
+ * (the same path the daemon's HTTP message route uses) while also being
+ * collected internally so the caller receives the final response content.
+ * For streaming use cases, subscribe to the conversation's events via the
+ * host event hub (`assistantEventHub`).
+ *
+ * If the conversation is currently processing another turn, the message is
+ * queued via `enqueueMessage` and the result carries `queued: true` with
+ * empty content -- the queued turn will execute automatically when the
+ * current turn finishes.
+ */
+export async function runConversationTurn(
+  options: RunConversationTurnOptions,
+): Promise<RunConversationTurnResult> {
+  const { v7: uuidv7 } = await import("uuid");
+  const { getOrCreateConversation } =
+    await import("../daemon/conversation-store.js");
+  const { broadcastMessage } =
+    await import("../runtime/assistant-event-hub.js");
+  const { resolveMediaSourceData } =
+    await import("../providers/media-resolve.js");
+  const { ensureConversationExists, getMessageById, getMessages } =
+    await import("../persistence/conversation-crud.js");
+  const { publishConversationListAndMetadataChanged } =
+    await import("../runtime/sync/resource-sync-events.js");
+
+  const conversationId = options.conversationId ?? uuidv7();
+  const conversation = await getOrCreateConversation(conversationId);
+
+  // `getOrCreateConversation` only builds the in-memory conversation (provider,
+  // system prompt, history hydration) — it never writes a `conversations` row.
+  // On the first turn of a brand-new chat (a minted id, or a caller-supplied id
+  // with no persisted row), the user-message persist inside `processMessage`
+  // would insert into `messages` against a nonexistent FK target. Ensure the
+  // row exists (idempotent) before persisting, mirroring the live-voice ingress
+  // which faces the same adopted-id-without-row gap.
+  const createdConversation = ensureConversationExists(conversationId);
+  if (createdConversation) {
+    // The row was created outside the normal send-message route, which is where
+    // sibling clients/sidebars learn about a new conversation. Emit the same
+    // "created" list invalidation that route does so they see it without
+    // waiting for a reload.
+    publishConversationListAndMetadataChanged("created", conversationId);
+  }
+
+  // Wire the external abort signal to the conversation's internal abort
+  // controller so aborting the signal terminates the in-flight agent loop.
+  if (options.signal) {
+    options.signal.addEventListener("abort", () => {
+      conversation.abortController?.abort();
+    });
+  }
+
+  // Convert ContentBlock[] input to the text + attachments shape the
+  // conversation's processMessage path expects.
+  const { text, attachments } = extractContentAndAttachments(
+    options.content,
+    resolveMediaSourceData,
+  );
+
+  // Build the event emitter: fan out to SSE/event-hub subscribers via
+  // broadcastMessage, then invoke the collector callback. This mirrors the
+  // buildEventEmitter pattern from process-message.ts so plugin-driven
+  // turns reach the same subscribers as HTTP-driven turns.
+  let assistantMessageId: string | undefined;
+  const onEvent = (msg: ServerMessage): void => {
+    broadcastMessage(msg, conversationId);
+    if (msg.type === "message_complete" && msg.messageId) {
+      assistantMessageId = msg.messageId;
+    }
+  };
+
+  // When the conversation is busy, enqueue the message instead of rejecting.
+  // The queue is drained automatically when the current turn finishes.
+  if (conversation.isProcessing()) {
+    const requestId = uuidv7();
+    const enqueueResult = conversation.enqueueMessage({
+      content: text,
+      attachments,
+      onEvent,
+      requestId,
+      isInteractive: false,
+    });
+    if (enqueueResult.rejected) {
+      throw new Error(
+        "Conversation is busy and its message queue is full. Try again later.",
+      );
+    }
+    return {
+      content: [],
+      userMessageId: requestId,
+      conversationId,
+      queued: true,
+    };
+  }
+
+  const userMessageId = await conversation.processMessage({
+    content: text,
+    attachments,
+    onEvent,
+    isInteractive: false,
+    ...(options.callSite ? { callSite: options.callSite } : {}),
+  });
+
+  // Retrieve the assistant's full content blocks from the persisted
+  // message row. The message_complete event carries the assistant
+  // message ID; if it was captured, use it directly. Otherwise fall back
+  // to scanning the conversation's messages for the first assistant
+  // message after our user message.
+  let assistantContent: ContentBlock[] = [];
+  if (assistantMessageId) {
+    const assistantRow = getMessageById(assistantMessageId, conversationId);
+    if (assistantRow) {
+      assistantContent = assistantRow.content;
+    }
+  }
+  if (assistantContent.length === 0) {
+    const allMessages = getMessages(conversationId);
+    const userIdx = allMessages.findIndex((m) => m.id === userMessageId);
+    if (userIdx >= 0) {
+      for (let i = allMessages.length - 1; i > userIdx; i--) {
+        if (allMessages[i].role === "assistant") {
+          assistantContent = allMessages[i].content;
+          break;
+        }
+      }
+    }
+  }
+
+  return {
+    content: assistantContent,
+    userMessageId,
+    conversationId,
+  };
+}

package/src/plugin-api/index.ts

@@ -235,3 +235,24 @@ export {
   syncMessageToDisk,
   updateMessageMetadata,
 } from "../persistence/conversation-plugin-facade.js";
+// Synthesize text to speech through the assistant's globally configured TTS
+// provider (ElevenLabs, Fish Audio, etc.). Plugins that need voice output —
+// e.g. a meeting bot speaking into a live call — use this instead of managing
+// TTS credentials and provider config themselves. Returns a Buffer + MIME type.
+// Text is sanitized internally (markdown/URLs/emoji stripped) so callers can
+// pass raw model output directly.
+export type { SynthesizeTextOptions } from "../tts/synthesize-text.js";
+export { synthesizeText, TtsSynthesisError } from "../tts/synthesize-text.js";
+export type { TtsSynthesisResult } from "../tts/types.js";
+// Conversation agent-loop turn — run a full conversation turn (persist user
+// message, execute the agent loop with history/tools/compaction/injections,
+// return the assistant's full content-block response). Accepts ContentBlock[]
+// input (text, images, files) and an optional conversationId (creates a new
+// conversation when omitted). Plugins that need to drive conversation turns
+// (e.g. meeting-bot flushing a transcript excerpt) should prefer this over the
+// stateless `provider.sendMessage()` call.
+export type {
+  RunConversationTurnOptions,
+  RunConversationTurnResult,
+} from "./conversation-turn.js";
+export { runConversationTurn } from "./conversation-turn.js";