@vellumai/assistant 0.6.2 → 0.6.4

package/src/runtime/routes/stt-routes.ts

@@ -0,0 +1,233 @@
+/**
+ * HTTP route definitions for speech-to-text transcription.
+ *
+ * POST /v1/stt/transcribe — transcribe base64-encoded audio to text
+ *
+ * Uses the globally configured STT provider via the `services.stt`
+ * abstraction. Provider selection is resolved via `resolveBatchTranscriber()`
+ * from `providers/speech-to-text/resolve.ts`.
+ */
+
+import { z } from "zod";
+
+import { resolveBatchTranscriber } from "../../providers/speech-to-text/resolve.js";
+import { normalizeSttError } from "../../stt/daemon-batch-transcriber.js";
+import type { SttErrorCategory } from "../../stt/types.js";
+import { getLogger } from "../../util/logger.js";
+import { httpError } from "../http-errors.js";
+import type { RouteDefinition } from "../http-router.js";
+
+const log = getLogger("stt-routes");
+
+/** Timeout for a single transcription request. */
+const TRANSCRIPTION_TIMEOUT_MS = 30_000;
+
+// ---------------------------------------------------------------------------
+// Error category -> HTTP status / message mapping
+// ---------------------------------------------------------------------------
+
+const STT_ERROR_MAP: Record<
+  SttErrorCategory,
+  { status: number; code: string; message: string }
+> = {
+  auth: {
+    status: 401,
+    code: "UNAUTHORIZED",
+    message: "STT provider credentials are invalid or missing",
+  },
+  "rate-limit": {
+    status: 429,
+    code: "RATE_LIMITED",
+    message: "STT provider rate limit exceeded",
+  },
+  timeout: {
+    status: 504,
+    code: "INTERNAL_ERROR",
+    message: "STT transcription timed out",
+  },
+  "invalid-audio": {
+    status: 400,
+    code: "BAD_REQUEST",
+    message: "Audio payload was rejected by the STT provider",
+  },
+  "provider-error": {
+    status: 502,
+    code: "INTERNAL_ERROR",
+    message: "STT provider error",
+  },
+};
+
+// ---------------------------------------------------------------------------
+// Route definitions
+// ---------------------------------------------------------------------------
+
+export function sttRouteDefinitions(): RouteDefinition[] {
+  return [
+    {
+      endpoint: "stt/transcribe",
+      method: "POST",
+      policyKey: "stt/transcribe",
+      summary: "Transcribe audio to text",
+      description:
+        "Transcribe base64-encoded audio to text using the configured STT provider. " +
+        "Provider selection is resolved globally via config.",
+      tags: ["stt"],
+      requestBody: z.object({
+        audioBase64: z
+          .string()
+          .describe("Base64-encoded audio data to transcribe"),
+        mimeType: z
+          .string()
+          .describe(
+            'MIME type of the audio data (must start with "audio/", e.g. "audio/wav", "audio/ogg")',
+          ),
+        source: z
+          .string()
+          .optional()
+          .describe(
+            "Optional source identifier for analytics (e.g. 'dictation', 'voice-mode')",
+          ),
+      }),
+      handler: async ({ req }) => {
+        // -- Parse body -------------------------------------------------------
+        let body: {
+          audioBase64?: unknown;
+          mimeType?: unknown;
+          source?: unknown;
+        };
+        try {
+          body = (await req.json()) as typeof body;
+        } catch {
+          return httpError("BAD_REQUEST", "Invalid JSON body", 400);
+        }
+
+        if (!body || typeof body !== "object") {
+          return httpError("BAD_REQUEST", "Invalid JSON body", 400);
+        }
+
+        // -- Validate audioBase64 ---------------------------------------------
+        if (
+          !body.audioBase64 ||
+          typeof body.audioBase64 !== "string" ||
+          body.audioBase64.length === 0
+        ) {
+          return httpError(
+            "BAD_REQUEST",
+            "audioBase64 is required and must be a non-empty string",
+            400,
+          );
+        }
+
+        // -- Validate mimeType ------------------------------------------------
+        if (
+          !body.mimeType ||
+          typeof body.mimeType !== "string" ||
+          !body.mimeType.startsWith("audio/")
+        ) {
+          return httpError(
+            "BAD_REQUEST",
+            'mimeType is required and must start with "audio/"',
+            400,
+          );
+        }
+
+        // -- Decode audio -----------------------------------------------------
+        // Buffer.from(str, "base64") silently accepts malformed input rather
+        // than throwing, so we validate the characters explicitly first.
+        const base64Str = body.audioBase64 as string;
+        if (
+          !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(
+            base64Str,
+          )
+        ) {
+          return httpError(
+            "BAD_REQUEST",
+            "Invalid base64 encoding in audioBase64",
+            400,
+          );
+        }
+
+        let audioBuffer: Buffer;
+        try {
+          audioBuffer = Buffer.from(base64Str, "base64");
+        } catch {
+          return httpError(
+            "BAD_REQUEST",
+            "audioBase64 could not be decoded",
+            400,
+          );
+        }
+
+        if (audioBuffer.length === 0) {
+          return httpError(
+            "BAD_REQUEST",
+            "Decoded audio payload is empty",
+            400,
+          );
+        }
+
+        // -- Resolve transcriber ----------------------------------------------
+        let transcriber;
+        try {
+          transcriber = await resolveBatchTranscriber();
+        } catch (err) {
+          log.error({ err }, "Failed to resolve STT transcriber");
+          return httpError(
+            "SERVICE_UNAVAILABLE",
+            "STT provider is not available",
+            503,
+          );
+        }
+
+        if (!transcriber) {
+          return httpError(
+            "SERVICE_UNAVAILABLE",
+            "No speech-to-text provider is configured",
+            503,
+          );
+        }
+
+        // -- Transcribe with timeout ------------------------------------------
+        const abortController = new AbortController();
+        const timeoutId = setTimeout(
+          () => abortController.abort(),
+          TRANSCRIPTION_TIMEOUT_MS,
+        );
+
+        try {
+          const result = await transcriber.transcribe({
+            audio: audioBuffer,
+            mimeType: body.mimeType as string,
+            signal: abortController.signal,
+          });
+
+          return Response.json({
+            text: result.text,
+            providerId: transcriber.providerId,
+            boundaryId: transcriber.boundaryId,
+          });
+        } catch (err) {
+          const sttErr = normalizeSttError(err);
+          const mapped = STT_ERROR_MAP[sttErr.category];
+
+          log.warn(
+            {
+              category: sttErr.category,
+              message: sttErr.message,
+              source: body.source,
+            },
+            "STT transcription failed",
+          );
+
+          return httpError(
+            mapped.code as Parameters<typeof httpError>[0],
+            mapped.message,
+            mapped.status,
+          );
+        } finally {
+          clearTimeout(timeoutId);
+        }
+      },
+    },
+  ];
+}

package/src/runtime/routes/subagents-routes.ts

@@ -7,7 +7,7 @@
  */
 import { z } from "zod";
 
-import { getMessages } from "../../memory/conversation-crud.js";
+import { getMessages, type MessageRow } from "../../memory/conversation-crud.js";
 import { getSubagentManager } from "../../subagent/index.js";
 import { getLogger } from "../../util/logger.js";
 import { httpError } from "../http-errors.js";
@@ -31,16 +31,25 @@ export interface SubagentDetailResult {
     content: string;
     toolName?: string;
     isError?: boolean;
+    messageId?: string;
   }>;
 }
 
+const FORK_DIRECTIVE_RE =
+  /^⎯⎯⎯ FORK TASK ⎯⎯⎯\n[\s\S]*?Complete this task directly and return only your findings:\n\n([\s\S]*?)\n⎯⎯⎯+$/;
+
+function stripForkDirectiveFraming(text: string): string {
+  const match = FORK_DIRECTIVE_RE.exec(text);
+  return match ? match[1] : text;
+}
+
 /**
  * Parse raw message rows into subagent detail events. Extracted as a pure
  * function so it can be unit-tested without a database.
  */
 export function parseSubagentMessages(
   subagentId: string,
-  messages: Array<{ role: string; content: string }>,
+  messages: MessageRow[],
 ): SubagentDetailResult {
   // Extract objective from the first user message
   let objective: string | undefined;
@@ -53,7 +62,7 @@ export function parseSubagentMessages(
           (b: Record<string, unknown>) => isRecord(b) && b.type === "text",
         );
         if (textBlock && typeof textBlock.text === "string") {
-          objective = textBlock.text;
+          objective = stripForkDirectiveFraming(textBlock.text);
         }
       }
     } catch {
@@ -62,12 +71,7 @@ export function parseSubagentMessages(
   }
 
   // Extract events from both assistant and user messages.
-  const events: Array<{
-    type: string;
-    content: string;
-    toolName?: string;
-    isError?: boolean;
-  }> = [];
+  const events: SubagentDetailResult["events"] = [];
   const pendingTools = new Map<string, string>();
   for (const m of messages) {
     if (m.role !== "assistant" && m.role !== "user") continue;
@@ -86,7 +90,7 @@ export function parseSubagentMessages(
         block.type === "text" &&
         typeof block.text === "string"
       ) {
-        events.push({ type: "text", content: block.text });
+        events.push({ type: "text", content: block.text, messageId: m.id });
       } else if (block.type === "tool_use") {
         const name = typeof block.name === "string" ? block.name : "unknown";
         const input = isRecord(block.input)

package/src/runtime/routes/surface-action-routes.ts

@@ -27,7 +27,7 @@ interface SurfaceActionTarget {
     surfaceId: string,
     actionId: string,
     data?: Record<string, unknown>,
-  ): void;
+  ): void | Promise<unknown>;
   handleSurfaceUndo?(surfaceId: string): void;
   setTrustContext?(ctx: {
     trustClass: "guardian" | "trusted_contact" | "unknown";
@@ -143,11 +143,44 @@ export async function handleSurfaceAction(
   }
 
   try {
-    conversation.handleSurfaceAction(surfaceId, actionId, data);
+    // Most action paths return `void` (regular button/selection forwards);
+    // the `launch_conversation` dispatch branch returns a structured result
+    // so we can surface validation errors (e.g. missing title / seedPrompt)
+    // as 4xx responses instead of silently reporting success.
+    const raw = await conversation.handleSurfaceAction(
+      surfaceId,
+      actionId,
+      data,
+    );
+    const result =
+      raw && typeof raw === "object" && "accepted" in raw
+        ? (raw as
+            | { accepted: true; conversationId?: string }
+            | { accepted: false; error: string })
+        : undefined;
+    if (result && result.accepted === false) {
+      log.warn(
+        {
+          conversationId: conversationId ?? undefined,
+          surfaceId,
+          actionId,
+          error: result.error,
+        },
+        "Surface action rejected",
+      );
+      return httpError("BAD_REQUEST", result.error, 400);
+    }
     log.info(
       { conversationId: conversationId ?? undefined, surfaceId, actionId },
       "Surface action handled via HTTP",
     );
+    if (
+      result &&
+      result.accepted === true &&
+      typeof result.conversationId === "string"
+    ) {
+      return Response.json({ ok: true, conversationId: result.conversationId });
+    }
     return Response.json({ ok: true });
   } catch (err) {
     log.error(
@@ -239,6 +272,12 @@ export function surfaceActionRouteDefinitions(deps: {
       }),
       responseBody: z.object({
         ok: z.boolean(),
+        conversationId: z
+          .string()
+          .describe(
+            "Id of a newly launched conversation when the action dispatched one (e.g. launch_conversation). Omitted otherwise.",
+          )
+          .optional(),
       }),
       handler: async ({ req, authContext }) => {
         if (!deps.findConversation) {

package/src/runtime/routes/tts-routes.ts

@@ -1,17 +1,22 @@
 /**
- * HTTP route definitions for message text-to-speech synthesis.
+ * HTTP route definitions for text-to-speech synthesis.
  *
  * POST /v1/messages/:id/tts?conversationId=... — synthesize message text to audio
+ * POST /v1/tts/synthesize                      — synthesize arbitrary text to audio
  *
- * Gated behind the `message-tts` assistant feature flag.
- * Uses Fish Audio for synthesis when configured.
+ * Both endpoints use the globally configured TTS provider via the provider
+ * abstraction. The message endpoint is gated behind the `message-tts`
+ * assistant feature flag; the generic endpoint is always available when a
+ * TTS provider is configured.
  */
 
-import { synthesizeWithFishAudio } from "../../calls/fish-audio-client.js";
+import { z } from "zod";
+
 import { sanitizeForTts } from "../../calls/tts-text-sanitizer.js";
 import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
 import { getConfig } from "../../config/loader.js";
 import { getMessageContent } from "../../daemon/handlers/conversation-history.js";
+import { synthesizeText } from "../../tts/synthesize-text.js";
 import { getLogger } from "../../util/logger.js";
 import { httpError } from "../http-errors.js";
 import type { RouteDefinition } from "../http-router.js";
@@ -32,7 +37,7 @@ export function ttsRouteDefinitions(): RouteDefinition[] {
       policyKey: "messages/tts",
       summary: "Synthesize message to speech",
       description:
-        "Synthesize a message's text content to audio using Fish Audio TTS.",
+        "Synthesize a message's text content to audio using the configured TTS provider.",
       tags: ["messages"],
       queryParams: [
         {
@@ -70,35 +75,114 @@ export function ttsRouteDefinitions(): RouteDefinition[] {
           );
         }
 
-        const { fishAudio } = config;
-        if (!fishAudio?.referenceId) {
-          return httpError(
-            "SERVICE_UNAVAILABLE",
-            "Fish Audio TTS is not configured",
-            503,
-          );
+        try {
+          const { audio, contentType } = await synthesizeText({
+            text: sanitizedText,
+            useCase: "message-playback",
+          });
+
+          return new Response(new Uint8Array(audio), {
+            status: 200,
+            headers: { "Content-Type": contentType },
+          });
+        } catch (err) {
+          log.error({ err, messageId }, "TTS synthesis failed");
+
+          // Surface provider-not-configured as 503
+          if (
+            err instanceof Error &&
+            "code" in err &&
+            (err as { code: string }).code === "TTS_PROVIDER_NOT_CONFIGURED"
+          ) {
+            return httpError(
+              "SERVICE_UNAVAILABLE",
+              "TTS provider is not configured",
+              503,
+            );
+          }
+
+          return httpError("INTERNAL_ERROR", "TTS synthesis failed", 502);
         }
+      },
+    },
 
+    // -- Generic text synthesis -----------------------------------------------
+
+    {
+      endpoint: "tts/synthesize",
+      method: "POST",
+      policyKey: "tts/synthesize",
+      summary: "Synthesize text to speech",
+      description:
+        "Synthesize arbitrary text to audio using the configured TTS provider. " +
+        "Provider selection is resolved globally via config — callers do not " +
+        "specify a provider.",
+      tags: ["tts"],
+      requestBody: z.object({
+        text: z.string().describe("Text to synthesize into speech"),
+        context: z
+          .string()
+          .optional()
+          .describe(
+            "Optional context hint for output policy or capability selection (e.g. voice-mode). " +
+              "Does not affect provider selection.",
+          ),
+        conversationId: z
+          .string()
+          .optional()
+          .describe("Optional conversation ID for scoping or analytics."),
+      }),
+      handler: async ({ req }) => {
+        let body: { text?: string; context?: string; conversationId?: string };
         try {
-          const audioBuffer = await synthesizeWithFishAudio(
-            sanitizedText,
-            fishAudio,
+          body = (await req.json()) as typeof body;
+        } catch {
+          return httpError("BAD_REQUEST", "Invalid JSON body", 400);
+        }
+
+        if (!body || typeof body !== "object") {
+          return httpError("BAD_REQUEST", "Invalid JSON body", 400);
+        }
+
+        if (!body.text || typeof body.text !== "string") {
+          return httpError("BAD_REQUEST", "text is required", 400);
+        }
+
+        const sanitizedText = sanitizeForTts(body.text).trim();
+        if (!sanitizedText) {
+          return httpError(
+            "BAD_REQUEST",
+            "Text has no speakable content after sanitization",
+            400,
           );
+        }
 
-          const format = fishAudio.format ?? "mp3";
-          const contentType =
-            format === "wav"
-              ? "audio/wav"
-              : format === "opus"
-                ? "audio/opus"
-                : "audio/mpeg";
+        try {
+          const { audio, contentType } = await synthesizeText({
+            text: sanitizedText,
+            useCase: "message-playback",
+          });
 
-          return new Response(new Uint8Array(audioBuffer), {
+          return new Response(new Uint8Array(audio), {
             status: 200,
             headers: { "Content-Type": contentType },
           });
         } catch (err) {
-          log.error({ err, messageId }, "TTS synthesis failed");
+          log.error({ err, context: body.context }, "TTS synthesis failed");
+
+          // Surface provider-not-configured as 503
+          if (
+            err instanceof Error &&
+            "code" in err &&
+            (err as { code: string }).code === "TTS_PROVIDER_NOT_CONFIGURED"
+          ) {
+            return httpError(
+              "SERVICE_UNAVAILABLE",
+              "TTS provider is not configured",
+              503,
+            );
+          }
+
           return httpError("INTERNAL_ERROR", "TTS synthesis failed", 502);
         }
       },

package/src/runtime/routes/usage-routes.ts

@@ -3,7 +3,7 @@
  *
  * GET /v1/usage/totals?from=&to=              — aggregate totals for a time range
  * GET /v1/usage/daily?from=&to=               — per-day buckets for a time range
- * GET /v1/usage/breakdown?from=&to=&groupBy=  — grouped breakdown (actor, provider, model)
+ * GET /v1/usage/breakdown?from=&to=&groupBy=  — grouped breakdown (actor, provider, model, or conversation)
  */
 
 import { z } from "zod";
@@ -14,10 +14,29 @@ import {
   getUsageHourBuckets,
   getUsageTotals,
 } from "../../memory/llm-usage-store.js";
+import { validateTimezone } from "../../memory/usage-buckets.js";
 import { httpError } from "../http-errors.js";
 import type { RouteDefinition } from "../http-router.js";
 
-const VALID_GROUP_BY = new Set(["actor", "provider", "model"]);
+const VALID_GROUP_BY = new Set(["actor", "provider", "model", "conversation"]);
+
+/**
+ * Resolve the optional `tz` query param to a validated IANA identifier.
+ * Returns the tz string, or an error Response if the value is not a valid tz.
+ */
+function resolveTimezone(url: URL): string | Response {
+  const tz = url.searchParams.get("tz") ?? "UTC";
+  try {
+    validateTimezone(tz);
+  } catch (err) {
+    return httpError(
+      "BAD_REQUEST",
+      (err as Error).message,
+      400,
+    );
+  }
+  return tz;
+}
 
 /**
  * Parse and validate the `from` and `to` epoch-millis query parameters.
@@ -110,6 +129,12 @@ export function usageRouteDefinitions(): RouteDefinition[] {
           schema: { type: "string", enum: ["daily", "hourly"] },
           description: 'Bucket granularity: "daily" (default) or "hourly"',
         },
+        {
+          name: "tz",
+          schema: { type: "string" },
+          description:
+            'IANA timezone identifier (e.g. "America/Los_Angeles"). Bucket boundaries and display labels are computed in this timezone. Defaults to "UTC" for backwards compatibility.',
+        },
       ],
       responseBody: z.object({
         buckets: z.array(z.unknown()).describe("Usage bucket objects"),
@@ -125,10 +150,13 @@ export function usageRouteDefinitions(): RouteDefinition[] {
             400,
           );
         }
+        const tz = resolveTimezone(url);
+        if (tz instanceof Response) return tz;
+        // The chart wants a continuous time axis, so fill empty buckets.
         const buckets =
           granularity === "hourly"
-            ? getUsageHourBuckets(range)
-            : getUsageDayBuckets(range);
+            ? getUsageHourBuckets(range, tz, { fillEmpty: true })
+            : getUsageDayBuckets(range, tz, { fillEmpty: true });
         return Response.json({ buckets });
       },
     },
@@ -137,7 +165,7 @@ export function usageRouteDefinitions(): RouteDefinition[] {
       method: "GET",
       summary: "Get usage breakdown",
       description:
-        "Return grouped usage breakdown (by actor, provider, or model).",
+        "Return grouped usage breakdown (by actor, provider, model, or conversation).",
       tags: ["usage"],
       queryParams: [
         {
@@ -153,7 +181,8 @@ export function usageRouteDefinitions(): RouteDefinition[] {
         {
           name: "groupBy",
           schema: { type: "string" },
-          description: "Group by: actor, provider, or model (required)",
+          description:
+            "Group by: actor, provider, model, or conversation (required)",
         },
       ],
       responseBody: z.object({
@@ -167,21 +196,21 @@ export function usageRouteDefinitions(): RouteDefinition[] {
         if (!groupBy) {
           return httpError(
             "BAD_REQUEST",
-            'Missing required query parameter: "groupBy" (one of: actor, provider, model)',
+            'Missing required query parameter: "groupBy" (one of: actor, provider, model, conversation)',
             400,
           );
         }
         if (!VALID_GROUP_BY.has(groupBy)) {
           return httpError(
             "BAD_REQUEST",
-            `Invalid "groupBy" value: "${groupBy}". Must be one of: actor, provider, model`,
+            `Invalid "groupBy" value: "${groupBy}". Must be one of: actor, provider, model, conversation`,
             400,
           );
         }
 
         const breakdown = getUsageGroupBreakdown(
           range,
-          groupBy as "actor" | "provider" | "model",
+          groupBy as "actor" | "provider" | "model" | "conversation",
         );
         return Response.json({ breakdown });
       },