@vellumai/assistant 0.7.3 → 0.8.1

package/src/runtime/routes/stt-routes.ts

@@ -1,17 +1,28 @@
 /**
  * Transport-agnostic route definitions for speech-to-text.
  *
- * GET  /v1/stt/providers  — list available STT providers and their metadata
- * POST /v1/stt/transcribe — transcribe base64-encoded audio to text
+ * GET  /v1/stt/providers        — list available STT providers and their metadata
+ * POST /v1/stt/transcribe       — transcribe base64-encoded audio to text
+ * POST /v1/stt/transcribe-file  — transcribe audio/video file to text (full ffmpeg + chunking flow)
  */
 
+import { randomUUID } from "node:crypto";
+import { access, mkdir, readdir, readFile, rm, unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { extname, join } from "node:path";
+
 import { z } from "zod";
 
 import { listProviderEntries } from "../../providers/speech-to-text/provider-catalog.js";
 import { resolveBatchTranscriber } from "../../providers/speech-to-text/resolve.js";
 import { normalizeSttError } from "../../stt/daemon-batch-transcriber.js";
-import type { SttErrorCategory } from "../../stt/types.js";
+import type { BatchTranscriber, SttErrorCategory } from "../../stt/types.js";
 import { getLogger } from "../../util/logger.js";
+import {
+  FFMPEG_TRANSCODE_TIMEOUT_MS,
+  FFPROBE_TIMEOUT_MS,
+  spawnWithTimeout,
+} from "../../util/spawn.js";
 import {
   BadGatewayError,
   BadRequestError,
@@ -28,6 +39,155 @@ const log = getLogger("stt-routes");
 /** Timeout for a single transcription request. */
 const TRANSCRIPTION_TIMEOUT_MS = 30_000;
 
+// ---------------------------------------------------------------------------
+// File-transcription constants and helpers (used by stt_transcribe_file)
+// ---------------------------------------------------------------------------
+
+const VIDEO_EXTENSIONS = new Set([
+  ".mp4",
+  ".mov",
+  ".avi",
+  ".mkv",
+  ".webm",
+  ".m4v",
+  ".mpeg",
+  ".mpg",
+]);
+const AUDIO_EXTENSIONS = new Set([
+  ".mp3",
+  ".wav",
+  ".m4a",
+  ".aac",
+  ".ogg",
+  ".flac",
+  ".aiff",
+  ".wma",
+]);
+
+/** Max file size for a single STT chunk request (25MB). */
+const STT_CHUNK_MAX_BYTES = 25 * 1024 * 1024;
+
+/** Duration per chunk when splitting for large files (10 minutes). */
+const CHUNK_DURATION_SECS = 600;
+
+/** Timeout for a single STT transcription request. */
+const STT_REQUEST_TIMEOUT_MS = 300_000;
+
+async function getAudioDuration(audioPath: string): Promise<number> {
+  const result = await spawnWithTimeout(
+    [
+      "ffprobe",
+      "-v",
+      "error",
+      "-show_entries",
+      "format=duration",
+      "-of",
+      "csv=p=0",
+      audioPath,
+    ],
+    FFPROBE_TIMEOUT_MS,
+  );
+  if (result.exitCode !== 0) return 0;
+  return parseFloat(result.stdout.trim()) || 0;
+}
+
+async function splitAudio(
+  audioPath: string,
+  chunkDir: string,
+  chunkDurationSecs: number,
+): Promise<string[]> {
+  const chunkPattern = join(chunkDir, "chunk-%03d.wav");
+  const result = await spawnWithTimeout(
+    [
+      "ffmpeg",
+      "-y",
+      "-i",
+      audioPath,
+      "-f",
+      "segment",
+      "-segment_time",
+      String(chunkDurationSecs),
+      "-acodec",
+      "pcm_s16le",
+      "-ar",
+      "16000",
+      "-ac",
+      "1",
+      chunkPattern,
+    ],
+    FFMPEG_TRANSCODE_TIMEOUT_MS,
+  );
+  if (result.exitCode !== 0) {
+    throw new Error(`Failed to split audio: ${result.stderr.slice(0, 300)}`);
+  }
+  const files = await readdir(chunkDir);
+  return files
+    .filter((f) => f.startsWith("chunk-") && f.endsWith(".wav"))
+    .sort()
+    .map((f) => join(chunkDir, f));
+}
+
+/** Convert source to 16kHz mono WAV for consistent processing. */
+async function toWav(inputPath: string, isVideo: boolean): Promise<string> {
+  const wavPath = join(tmpdir(), `vellum-transcribe-${randomUUID()}.wav`);
+  const args = ["ffmpeg", "-y", "-i", inputPath];
+  if (isVideo) args.push("-vn");
+  args.push("-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", wavPath);
+  const result = await spawnWithTimeout(args, FFMPEG_TRANSCODE_TIMEOUT_MS);
+  if (result.exitCode !== 0) {
+    throw new Error(`ffmpeg failed: ${result.stderr.slice(0, 500)}`);
+  }
+  return wavPath;
+}
+
+async function transcribeWithProvider(
+  audioPath: string,
+  transcriber: BatchTranscriber,
+): Promise<string> {
+  const duration = await getAudioDuration(audioPath);
+  const fileSize = Bun.file(audioPath).size;
+
+  // If small enough, send directly
+  if (fileSize <= STT_CHUNK_MAX_BYTES) {
+    const audioBuffer = await readFile(audioPath);
+    const result = await transcriber.transcribe({
+      audio: audioBuffer,
+      mimeType: "audio/wav",
+      signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS),
+    });
+    return result.text;
+  }
+
+  // Split into chunks for large files
+  const chunkDir = join(tmpdir(), `vellum-transcribe-chunks-${randomUUID()}`);
+  await mkdir(chunkDir, { recursive: true });
+
+  try {
+    process.stderr.write(
+      `Large file (${Math.round(duration / 60)}min) - splitting into chunks...\n`,
+    );
+    const chunks = await splitAudio(audioPath, chunkDir, CHUNK_DURATION_SECS);
+    const parts: string[] = [];
+
+    for (let i = 0; i < chunks.length; i++) {
+      process.stderr.write(
+        `  Transcribing chunk ${i + 1}/${chunks.length}...\n`,
+      );
+      const audioBuffer = await readFile(chunks[i]);
+      const result = await transcriber.transcribe({
+        audio: audioBuffer,
+        mimeType: "audio/wav",
+        signal: AbortSignal.timeout(STT_REQUEST_TIMEOUT_MS),
+      });
+      if (result.text) parts.push(result.text);
+    }
+
+    return parts.join(" ");
+  } finally {
+    await rm(chunkDir, { recursive: true, force: true }).catch(() => {});
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Error category → RouteError mapping
 // ---------------------------------------------------------------------------
@@ -156,6 +316,63 @@ async function handleTranscribe({ body }: RouteHandlerArgs) {
   }
 }
 
+async function handleTranscribeFile({ body }: RouteHandlerArgs) {
+  const filePath = body?.filePath;
+  if (typeof filePath !== "string" || !filePath) {
+    throw new BadRequestError("filePath is required");
+  }
+
+  try {
+    await access(filePath);
+  } catch {
+    throw new BadRequestError(`File not found: ${filePath}`);
+  }
+
+  const ext = extname(filePath).toLowerCase();
+  const isVideo = VIDEO_EXTENSIONS.has(ext);
+  const isAudio = AUDIO_EXTENSIONS.has(ext);
+  if (!isVideo && !isAudio) {
+    throw new BadRequestError(
+      `Unsupported file type: ${ext}. Only audio and video files can be transcribed.`,
+    );
+  }
+
+  let transcriber;
+  try {
+    transcriber = await resolveBatchTranscriber();
+  } catch (err) {
+    log.error({ err }, "Failed to resolve STT transcriber");
+    throw new ServiceUnavailableError("STT provider is not available");
+  }
+  if (!transcriber) {
+    throw new ServiceUnavailableError(
+      "No speech-to-text provider is configured",
+    );
+  }
+
+  const startTime = Date.now();
+  let wavPath: string | null = null;
+  try {
+    wavPath = await toWav(filePath, isVideo);
+    const text = await transcribeWithProvider(wavPath, transcriber);
+    const durationSeconds = (Date.now() - startTime) / 1000;
+    return {
+      transcript: text,
+      provider: transcriber.providerId,
+      durationSeconds,
+    };
+  } catch (err) {
+    log.error({ err, filePath }, "File transcription failed");
+    throw new BadGatewayError(
+      err instanceof Error ? err.message : "Transcription failed",
+    );
+  } finally {
+    if (wavPath) {
+      await unlink(wavPath).catch(() => {});
+    }
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Route definitions
 // ---------------------------------------------------------------------------
@@ -220,4 +437,24 @@ export const ROUTES: RouteDefinition[] = [
     }),
     handler: handleTranscribe,
   },
+  {
+    operationId: "stt_transcribe_file",
+    endpoint: "stt/transcribe-file",
+    method: "POST",
+    policyKey: "stt/transcribe-file",
+    requirePolicyEnforcement: true,
+    summary: "Transcribe audio/video file to text",
+    description:
+      "Transcribe an audio or video file to text using the configured STT provider. Handles ffmpeg conversion, large-file chunking, and sequential chunk transcription.",
+    tags: ["stt"],
+    requestBody: z.object({
+      filePath: z.string().describe("Absolute path to the audio or video file"),
+    }),
+    responseBody: z.object({
+      transcript: z.string(),
+      provider: z.string(),
+      durationSeconds: z.number(),
+    }),
+    handler: handleTranscribeFile,
+  },
 ];

package/src/runtime/routes/surface-action-routes.ts

@@ -11,10 +11,13 @@ import { z } from "zod";
 
 import type { ChannelId } from "../../channels/types.js";
 import { isHttpAuthDisabled } from "../../config/env.js";
+import type { Conversation } from "../../daemon/conversation.js";
 import {
   findConversation,
   findConversationBySurfaceId,
+  getOrCreateConversation,
 } from "../../daemon/conversation-store.js";
+import { rawGet } from "../../memory/raw-query.js";
 import { getLogger } from "../../util/logger.js";
 import { DAEMON_INTERNAL_ASSISTANT_ID } from "../assistant-scope.js";
 import { healGuardianBindingDrift } from "../guardian-vellum-migration.js";
@@ -87,6 +90,37 @@ function applyTrustContext(
   }
 }
 
+/**
+ * Resolve the conversation owning a surface, rehydrating from the DB when
+ * the in-memory lookup misses (daemon restart or LRU eviction). The DB scan
+ * is also the source of truth that the surface exists — without it,
+ * `getOrCreateConversation` would silently create a phantom conversation
+ * for any caller-supplied id.
+ */
+async function resolveSurfaceConversation(
+  conversationId: string | null | undefined,
+  surfaceId: string,
+): Promise<Conversation | undefined> {
+  const found = conversationId
+    ? findConversation(conversationId)
+    : findConversationBySurfaceId(surfaceId);
+  if (found) return found;
+
+  // Escape LIKE wildcards so a `surfaceId` like "%" or "_" can't match
+  // unrelated rows.
+  const escaped = surfaceId.replace(/[\\%_]/g, "\\$&");
+  const row = rawGet<{ conversation_id: string }>(
+    `SELECT conversation_id FROM messages
+     WHERE content LIKE ? ESCAPE '\\'
+     ORDER BY created_at DESC
+     LIMIT 1`,
+    `%"surfaceId":"${escaped}"%`,
+  );
+  if (!row) return undefined;
+  if (conversationId && conversationId !== row.conversation_id) return undefined;
+  return await getOrCreateConversation(row.conversation_id);
+}
+
 // ---------------------------------------------------------------------------
 // Handlers
 // ---------------------------------------------------------------------------
@@ -113,9 +147,10 @@ async function handleSurfaceAction({
     throw new BadRequestError("conversationId must be a string");
   }
 
-  const conversation = conversationId
-    ? findConversation(conversationId)
-    : findConversationBySurfaceId(surfaceId);
+  const conversation = await resolveSurfaceConversation(
+    conversationId,
+    surfaceId,
+  );
 
   if (!conversation) {
     throw new NotFoundError("No active conversation found");
@@ -175,7 +210,7 @@ async function handleSurfaceAction({
   }
 }
 
-function handleSurfaceUndo({ body, pathParams }: RouteHandlerArgs) {
+async function handleSurfaceUndo({ body, pathParams }: RouteHandlerArgs) {
   const surfaceId = pathParams?.id;
   if (!surfaceId) {
     throw new BadRequestError("surfaceId path parameter is required");
@@ -186,9 +221,10 @@ function handleSurfaceUndo({ body, pathParams }: RouteHandlerArgs) {
     throw new BadRequestError("conversationId must be a string");
   }
 
-  const conversation = conversationId
-    ? findConversation(conversationId)
-    : findConversationBySurfaceId(surfaceId);
+  const conversation = await resolveSurfaceConversation(
+    conversationId,
+    surfaceId,
+  );
 
   if (!conversation) {
     throw new NotFoundError("No active conversation found");

package/src/runtime/routes/tts-routes.ts

@@ -20,6 +20,7 @@ import {
   TtsSynthesisError,
 } from "../../tts/synthesize-text.js";
 import { resolveTtsConfig } from "../../tts/tts-config-resolver.js";
+import type { TtsUseCase } from "../../tts/types.js";
 import { getLogger } from "../../util/logger.js";
 import {
   BadGatewayError,
@@ -155,6 +156,45 @@ async function handleSynthesizeTts({ body }: RouteHandlerArgs) {
   return doSynthesize(sanitizedText, { context: body.context });
 }
 
+async function handleSynthesizeCliTts({ body }: RouteHandlerArgs) {
+  if (!body?.text || typeof body.text !== "string") {
+    throw new BadRequestError("text is required");
+  }
+
+  const sanitizedText = sanitizeForTts(body.text).trim();
+  if (!sanitizedText) {
+    throw new BadRequestError(
+      "Text has no speakable content after sanitization",
+    );
+  }
+
+  const useCase: TtsUseCase =
+    (body.useCase as TtsUseCase | undefined) ?? "message-playback";
+  const voiceId =
+    body.voiceId && typeof body.voiceId === "string"
+      ? body.voiceId
+      : undefined;
+
+  try {
+    const result = await synthesizeText({ text: sanitizedText, useCase, voiceId });
+    return {
+      audioBase64: Buffer.from(result.audio).toString("base64"),
+      contentType: result.contentType,
+    };
+  } catch (err) {
+    log.error({ err }, "TTS CLI synthesis failed");
+
+    if (
+      err instanceof TtsSynthesisError &&
+      err.code === "TTS_PROVIDER_NOT_CONFIGURED"
+    ) {
+      throw new ServiceUnavailableError("TTS provider is not configured");
+    }
+
+    throw new BadGatewayError("TTS synthesis failed");
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Route definitions
 // ---------------------------------------------------------------------------
@@ -206,4 +246,31 @@ export const ROUTES: RouteDefinition[] = [
     responseHeaders: ttsResponseHeaders,
     handler: handleSynthesizeTts,
   },
+  {
+    operationId: "tts_synthesize_cli",
+    endpoint: "tts/synthesize-cli",
+    method: "POST",
+    policyKey: "tts/synthesize-cli",
+    requirePolicyEnforcement: true,
+    summary: "Synthesize text to speech (CLI)",
+    description:
+      "Synthesize arbitrary text to audio. Returns base64-encoded audio + content type for CLI consumption.",
+    tags: ["tts"],
+    requestBody: z.object({
+      text: z.string().describe("Text to synthesize into speech"),
+      useCase: z
+        .enum(["message-playback", "phone-call"])
+        .optional()
+        .default("message-playback"),
+      voiceId: z
+        .string()
+        .optional()
+        .describe("Provider-specific voice identifier override"),
+    }),
+    responseBody: z.object({
+      audioBase64: z.string().describe("Base64-encoded audio bytes"),
+      contentType: z.string().describe("MIME type of the audio (e.g. audio/mpeg)"),
+    }),
+    handler: handleSynthesizeCliTts,
+  },
 ];

package/src/runtime/routes/types.ts

@@ -136,4 +136,36 @@ export interface RouteDefinition {
    * RouteError subclasses rather than explicit Response objects.
    */
   additionalResponses?: Record<string, { description: string }>;
+  /**
+   * When true, the route expects a raw binary body (e.g. file uploads).
+   * The HTTP adapter already reads `rawBody` for non-JSON content types;
+   * this flag is a declarative signal for documentation and tooling.
+   */
+  rawBody?: boolean;
+  /**
+   * Per-route request-log control. Routes that opt in can suppress the
+   * per-request INFO log line after a confirmed run of successful
+   * responses — useful for high-frequency probes like `/v1/health` where
+   * the first few responses confirm the route works and every line after
+   * that is just noise. Non-success responses (status >= 400) always log.
+   */
+  logging?: RouteLoggingConfig;
+}
+
+/**
+ * Logging behavior for a single route. Currently only the success-suppression
+ * counter is supported; new knobs (sampling, periodic summary lines) can be
+ * added here as separate fields without changing call sites.
+ */
+export interface RouteLoggingConfig {
+  /**
+   * After this many successful (status < 400) responses, suppress the
+   * per-request INFO log line for further successful responses on the
+   * same route (keyed by `operationId`, so all path-param variants
+   * share a single counter).
+   *
+   * Counters are process-local and reset on restart. Warning (4xx) and
+   * error (5xx) log lines are always emitted regardless of this setting.
+   */
+  silenceSuccessAfter?: number;
 }