@vellumai/assistant 0.12.4-staging.1 → 0.12.4-staging.3

package/src/runtime/routes/__tests__/dictation-processing.test.ts

@@ -0,0 +1,174 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { DictationProfile } from "../../../daemon/dictation-profile-store.js";
+
+let profile: DictationProfile;
+let nextToolInput: { mode?: string; text?: string } | null;
+let providerAvailable: boolean;
+let providerFails: boolean;
+let providerResolutions: number;
+let requestedProperties: Record<string, unknown> | undefined;
+
+mock.module("../../../providers/provider-send-message.js", () => ({
+  getConfiguredProvider: async () => {
+    providerResolutions += 1;
+    if (!providerAvailable) {
+      return null;
+    }
+    return {
+      sendMessage: async (
+        _messages: unknown,
+        options: {
+          tools: Array<{
+            input_schema: { properties: Record<string, unknown> };
+          }>;
+        },
+      ) => {
+        if (providerFails) {
+          throw new Error("Provider unavailable");
+        }
+        requestedProperties = options.tools[0]?.input_schema.properties;
+        return {
+          content: nextToolInput
+            ? [{ type: "tool_use", input: nextToolInput }]
+            : [{ type: "text", text: "Rewritten text." }],
+        };
+      },
+    };
+  },
+  extractToolUse: (response: { content: Array<{ type: string }> }) =>
+    response.content.find((block) => block.type === "tool_use"),
+  userMessage: (text: string) => ({ role: "user", content: text }),
+  createTimeout: () => ({ signal: undefined, cleanup: () => undefined }),
+}));
+
+mock.module("../../../daemon/dictation-profile-store.js", () => ({
+  resolveProfile: () => ({ profile, source: "default" }),
+}));
+
+const { ROUTES, computeMaxTokens } = await import("../diagnostics-routes.js");
+const route = ROUTES.find((entry) => entry.operationId === "dictation_post")!;
+
+async function dictate(
+  transcription: string,
+  context: { cursorInTextField?: boolean; selectedText?: string } = {
+    cursorInTextField: true,
+  },
+) {
+  return (await route.handler({
+    body: { transcription, context },
+  } as unknown as Parameters<typeof route.handler>[0])) as {
+    text: string;
+    mode: string;
+    actionPlan?: string;
+  };
+}
+
+beforeEach(() => {
+  profile = { id: "general", name: "General" };
+  nextToolInput = { mode: "dictation", text: "We should ship it." };
+  providerAvailable = true;
+  providerFails = false;
+  providerResolutions = 0;
+  requestedProperties = undefined;
+});
+
+describe("dictation preserves recognized words", () => {
+  test.each([
+    "I think we probably shouldn't ship it, at least not yet.",
+    "Um, I don't, I don't want to say yes. Maybe next week?",
+    "First onions, then tomatoes. Actually, no tomatoes.",
+    "  Please don't send that.\nI mean it.  ",
+    "send me the draft, but don't send it to anyone else",
+  ])(
+    "preserves text-field dictation despite an attempted model rewrite: %s",
+    async (words) => {
+      expect(await dictate(words)).toMatchObject({
+        mode: "dictation",
+        text: words,
+      });
+      expect(providerResolutions).toBe(1);
+    },
+  );
+
+  test("a style profile cannot automatically rewrite ordinary dictation", async () => {
+    profile.stylePrompt = "Make every sentence confident and professional.";
+    const words = "I guess maybe we shouldn't do this.";
+
+    expect(await dictate(words)).toMatchObject({ text: words });
+    expect(providerResolutions).toBe(1);
+  });
+
+  test("whitespace-only selection is ordinary dictation", async () => {
+    expect(
+      await dictate("Don't change my words.", {
+        cursorInTextField: true,
+        selectedText: "  \n ",
+      }),
+    ).toMatchObject({ mode: "dictation", text: "Don't change my words." });
+    expect(providerResolutions).toBe(1);
+  });
+
+  test("keeps explicitly configured snippet and dictionary replacements", async () => {
+    profile.snippets = [
+      { trigger: "my sign off", expansion: "Thanks, user one" },
+    ];
+    profile.dictionary = [{ spoken: "user one", written: "Example User" }];
+
+    expect(await dictate("Maybe later. my sign off")).toMatchObject({
+      mode: "dictation",
+      text: "Maybe later. Thanks, Example User",
+    });
+    expect(providerResolutions).toBe(1);
+  });
+
+  test("classification outside a text field cannot supply replacement text", async () => {
+    const words = "I think we probably shouldn't ship it.";
+
+    expect(await dictate(words, {})).toMatchObject({
+      mode: "dictation",
+      text: words,
+    });
+    expect(providerResolutions).toBe(1);
+    expect(Object.keys(requestedProperties ?? {})).toEqual(["mode"]);
+  });
+
+  test.each([true, false])(
+    "action classification preserves the original command with cursorInTextField=%s",
+    async (cursorInTextField) => {
+      nextToolInput = { mode: "action", text: "Send the files." };
+      profile.snippets = [{ trigger: "files", expansion: "all files" }];
+      const words = "Send the files, but not the private ones.";
+
+      expect(await dictate(words, { cursorInTextField })).toMatchObject({
+        mode: "action",
+        text: words,
+        actionPlan: `User wants to: ${words}`,
+      });
+    },
+  );
+
+  test.each([
+    "missing-provider",
+    "provider-error",
+    "missing-tool",
+    "invalid-mode",
+  ])("classification fallback preserves words when %s", async (failure) => {
+    providerAvailable = failure !== "missing-provider";
+    providerFails = failure === "provider-error";
+    nextToolInput = failure === "missing-tool" ? null : { mode: "invalid" };
+    const words = "I don't think that's what I meant, you know?";
+
+    expect(await dictate(words, {})).toMatchObject({
+      mode: "dictation",
+      text: words,
+    });
+  });
+});
+
+describe("selected-text output budget", () => {
+  test("reserves space for the edit and tool-call JSON", () => {
+    expect(computeMaxTokens(900)).toBeGreaterThan(600);
+    expect(computeMaxTokens(5)).toBe(512);
+  });
+});

package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts

@@ -4,8 +4,9 @@ import { CALL_SITE_DEFAULTS } from "../../../config/call-site-defaults.js";
 import {
   type LLMCallSite,
   LLMCallSiteEnum,
+  LLMSchema,
 } from "../../../config/schemas/llm.js";
-import { ROUTES } from "../llm-call-sites-routes.js";
+import { buildCallSiteCatalog, ROUTES } from "../llm-call-sites-routes.js";
 
 const route = ROUTES.find((r) => r.operationId === "llm_call_sites_list")!;
 
@@ -89,6 +90,20 @@ describe("llm-call-sites-routes", () => {
     }
   });
 
+  test("omits an unavailable managed-only shipped default under BYOK", () => {
+    const managed = buildCallSiteCatalog(LLMSchema.parse({})).find(
+      ({ id }) => id === "voiceEscalationJudge",
+    );
+    expect(managed?.defaultProfile).toBe("jev-managed");
+    expect(managed?.shippedDefaultProfile).toBe("jev-managed");
+
+    const byok = buildCallSiteCatalog(
+      LLMSchema.parse({ defaultProvider: { provider: "anthropic" } }),
+    ).find(({ id }) => id === "voiceEscalationJudge");
+    expect(byok?.defaultProfile).toBeUndefined();
+    expect(byok?.shippedDefaultProfile).toBeUndefined();
+  });
+
   test("domains have non-empty id and displayName", async () => {
     const result = (await route.handler({})) as {
       domains: Array<{ id: string; displayName: string }>;

package/src/runtime/routes/conversation-routes.ts

@@ -42,7 +42,7 @@ import {
   supportsHostProxy,
 } from "../../channels/types.js";
 import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
-import { getUserSelectableProfilesForProvider } from "../../config/default-profile-catalog.js";
+import { getConversationProfilesForProvider } from "../../config/default-profile-catalog.js";
 import { isHttpAuthDisabled } from "../../config/env.js";
 import { getConfig } from "../../config/loader.js";
 import { isModeSessionRecoveryHealthy } from "../../config/session-groups-gate.js";
@@ -1754,7 +1754,9 @@ export async function handleSendMessage(
   }
   if (requestedInferenceProfile !== undefined) {
     const { llm } = getConfig();
-    const profiles = getUserSelectableProfilesForProvider(
+    // The conversation view: a sticky pin runs the main agent's turns, so a
+    // profile whose model returns verdicts rather than text is not offered.
+    const profiles = getConversationProfilesForProvider(
       llm.profiles,
       llm.defaultProvider ?? null,
     );

package/src/runtime/routes/diagnostics-routes.ts

@@ -22,7 +22,6 @@ import {
   getConfiguredProvider,
   userMessage,
 } from "../../providers/provider-send-message.js";
-import { isMaxTokensStopReason } from "../../providers/stop-reasons.js";
 import { getLogger } from "../../util/logger.js";
 import { safeStringSlice } from "../../util/unicode.js";
 import { ACTOR_PRINCIPALS } from "../auth/route-policy.js";
@@ -73,56 +72,14 @@ function buildAppMetadataBlock(context: DictationContext): string {
   ].join("\n");
 }
 
-function buildCombinedDictationPrompt(
-  body: DictationBody,
-  stylePrompt?: string,
-): string {
-  // Every line here is read on every hold, so the prompt says each thing
-  // once and leaves the model no reasoning to write.
-  const sections = [
-    "You are a voice input assistant. Given a speech transcription, classify it and, if it is dictation, clean it up.",
-    "",
-    "## Classification",
+function buildDictationClassificationPrompt(body: DictationBody): string {
+  return [
+    "Classify the speech transcription without rewriting it.",
     "dictation: the user is composing text to be typed as-is.",
-    "action: the user is asking an assistant to do something (send, message, open, search, create, schedule). Return the transcription unchanged.",
+    "action: the user is asking an assistant to do something (send, message, open, search, create, schedule).",
     `Cursor in text field: ${body.context.cursorInTextField ? "yes" : "no"}. If yes, lean toward dictation unless the intent to command is clear.`,
-    "",
-    "## Cleanup",
-    "- Fix grammar, punctuation and capitalization; remove filler words",
-    "- Rewrite hedging into clear statements, keeping the speaker's meaning",
-    "- When the speaker enumerates items, lay them out as a list, one per line",
-    "- Keep the user's natural voice; do not over-formalize casual speech",
-  ];
-
-  if (stylePrompt) {
-    sections.push(
-      "",
-      "## User Style (highest priority)",
-      "The user's own writing preferences. They override the tone guidance below.",
-      "",
-      stylePrompt,
-    );
-  }
-
-  sections.push(
-    "",
-    "## Tone",
-    stylePrompt
-      ? "Fallback guidance where the User Style above is silent:"
-      : "Adapt tone to the active application:",
-    "- Email: professional but warm, greetings and sign-offs where they fit",
-    "- Slack: casual and conversational",
-    "- Code editors: technical and concise",
-    "- Terminal: terse, command-like",
-    "- Messages: very casual, short sentences",
-    "- Notes and docs: neutral, clear writing",
-    "- Otherwise: the user's natural voice",
-    "The window title may name the recipient; adapt formality to the apparent relationship.",
-    "",
     buildAppMetadataBlock(body.context),
-  );
-
-  return sections.join("\n");
+  ].join("\n");
 }
 
 function buildCommandPrompt(body: DictationBody, stylePrompt?: string): string {
@@ -184,44 +141,10 @@ function buildCommandPrompt(body: DictationBody, stylePrompt?: string): string {
 
 export function computeMaxTokens(inputLength: number): number {
   const estimatedInputTokens = Math.ceil(inputLength / 3);
-  // The cleanup tool call carries the whole transcript back in its `text`
-  // argument, plus a `reasoning` string and the JSON scaffolding around both,
-  // so the budget has to cover well more than the input alone. `max_tokens` is
-  // a ceiling rather than a target: a call that stops short of it costs no
-  // extra latency.
+  // Leave room for the transformed selection and tool-call JSON.
   return Math.max(512, estimatedInputTokens * 2 + 256);
 }
 
-export type CleanupRejection = "truncated";
-
-/**
- * Decide whether the cleanup model's rewrite is safe to use as the payload.
- *
- * The rewrite replaces what the user actually said. A token-cap stop means the
- * tool JSON was cut mid-argument, so however complete the `text` argument
- * happens to look, it is a fragment; the raw transcript is the safer payload
- * then. Unpolished beats wrong.
- *
- * Whether a shorter rewrite still preserved the meaning is the model's call
- * (the prompt asks it to drop fillers and tighten phrasing), so there is no
- * length check here: a deterministic ratio cannot tell a concise cleanup from
- * a summary.
- */
-export function resolveCleanedDictation(
-  raw: string,
-  cleaned: string,
-  stopReason: string | null | undefined,
-): { text: string; rejected: CleanupRejection | null } {
-  const trimmed = cleaned.trim();
-  if (!trimmed) {
-    return { text: raw, rejected: null };
-  }
-  if (isMaxTokensStopReason(stopReason)) {
-    return { text: raw, rejected: "truncated" };
-  }
-  return { text: trimmed, rejected: null };
-}
-
 interface DictationResult {
   text: string;
   mode: DictationMode;
@@ -263,174 +186,88 @@ async function handleDictation(body: DictationBody): Promise<DictationResult> {
     return handleCommandMode(body, profile, profileMeta, stylePrompt);
   }
 
-  // Non-command: single LLM call that classifies AND cleans in one shot
-  const transcription = expandSnippets(body.transcription, profile.snippets);
+  const mode = await classifyDictation(body);
+  if (mode === "action") {
+    return {
+      text: body.transcription,
+      mode,
+      actionPlan: `User wants to: ${body.transcription}`,
+      ...profileMeta,
+    };
+  }
 
-  // Covers provider resolution as well as the call, which is what the caller
-  // waits for.
-  const modelStartedAt = Date.now();
+  // Only explicit user-authored replacements may change dictated words.
+  const text = applyDictionary(
+    expandSnippets(body.transcription, profile.snippets),
+    profile.dictionary,
+  );
+  return { text, mode, ...profileMeta };
+}
+
+async function classifyDictation(
+  body: DictationBody,
+): Promise<"dictation" | "action"> {
   try {
     const provider = await getConfiguredProvider("interactionClassifier");
-    if (!provider) {
-      log.warn(
-        "Dictation: no provider available, using heuristic + raw transcription",
+    if (provider) {
+      const { signal, cleanup } = createTimeout(
+        DICTATION_CLASSIFICATION_TIMEOUT_MS,
       );
-      // Build a compatible msg for the heuristic
-      const mode = detectDictationModeHeuristic({
-        type: "dictation_request",
-        transcription: body.transcription,
-        context: body.context,
-      } as DictationRequest);
-      const normalizedText = applyDictionary(transcription, profile.dictionary);
-      if (mode === "action") {
-        return {
-          text: body.transcription,
-          mode: "action",
-          actionPlan: `User wants to: ${body.transcription}`,
-          ...profileMeta,
-        };
-      }
-      return {
-        text: normalizedText,
-        mode,
-        ...profileMeta,
-      };
-    }
-
-    const systemPrompt = buildCombinedDictationPrompt(body, stylePrompt);
-    const maxTokens = computeMaxTokens(transcription.length);
-    const { signal, cleanup } = createTimeout(
-      DICTATION_CLASSIFICATION_TIMEOUT_MS,
-    );
-
-    try {
-      const response = await provider.sendMessage(
-        [userMessage(`Transcription: "${transcription}"`)],
-        {
-          tools: [
-            {
-              name: "process_dictation",
-              description: "Classify the voice input and return cleaned text",
-              input_schema: {
-                type: "object" as const,
-                properties: {
-                  mode: {
-                    type: "string",
-                    enum: ["dictation", "action"],
-                    description:
-                      "dictation = user wants text inserted/cleaned up for typing. action = user wants the assistant to perform a task.",
-                  },
-                  text: {
-                    type: "string",
-                    description:
-                      "If dictation: the cleaned/formatted text ready for insertion. If action: the raw transcription unchanged.",
+      try {
+        const response = await provider.sendMessage(
+          [userMessage(body.transcription)],
+          {
+            tools: [
+              {
+                name: "process_dictation",
+                description: "Classify the voice input",
+                input_schema: {
+                  type: "object" as const,
+                  properties: {
+                    mode: {
+                      type: "string",
+                      enum: ["dictation", "action"],
+                      description:
+                        "dictation = user wants text inserted. action = user wants the assistant to perform a task.",
+                    },
                   },
+                  required: ["mode"],
                 },
-                required: ["mode", "text"],
               },
+            ],
+            systemPrompt: buildDictationClassificationPrompt(body),
+            config: {
+              callSite: "interactionClassifier",
+              max_tokens: 512,
+              tool_choice: { type: "tool" as const, name: "process_dictation" },
             },
-          ],
-          systemPrompt,
-          config: {
-            callSite: "interactionClassifier",
-            max_tokens: maxTokens,
-            tool_choice: {
-              type: "tool" as const,
-              name: "process_dictation",
-            },
+            signal,
           },
-          signal,
-        },
-      );
-      cleanup();
-
-      const toolBlock = extractToolUse(response);
-      if (toolBlock) {
-        const input = toolBlock.input as {
-          mode?: string;
-          text?: string;
-        };
-        const mode: DictationMode =
-          input.mode === "action" ? "action" : "dictation";
-        log.info(
-          {
-            mode,
-            modelMs: Date.now() - modelStartedAt,
-            inChars: transcription.length,
-            outChars: input.text?.length ?? 0,
-          },
-          "LLM dictation classify+clean",
         );
-
-        if (mode === "action") {
-          return {
-            text: body.transcription,
-            mode: "action",
-            actionPlan: `User wants to: ${body.transcription}`,
-            ...profileMeta,
-          };
+        const input = extractToolUse(response)?.input as
+          | { mode?: string }
+          | undefined;
+        if (input?.mode === "action" || input?.mode === "dictation") {
+          return input.mode;
         }
-        const { text: cleanedText, rejected } = resolveCleanedDictation(
-          transcription,
-          input.text ?? "",
-          response.stopReason,
-        );
-        if (rejected) {
-          // Lengths only -- transcript content must never be logged.
-          log.warn(
-            {
-              rejected,
-              stopReason: response.stopReason,
-              rawChars: transcription.length,
-              cleanedChars: input.text?.trim().length ?? 0,
-            },
-            "Dictation cleanup rejected, using raw transcription",
-          );
-        }
-        const normalizedText = applyDictionary(cleanedText, profile.dictionary);
-        return {
-          text: normalizedText,
-          mode: "dictation",
-          ...profileMeta,
-        };
+      } finally {
+        cleanup();
       }
-
-      log.warn(
-        { modelMs: Date.now() - modelStartedAt },
-        "No tool_use block in combined dictation call, using heuristic",
-      );
-    } finally {
-      cleanup();
     }
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
     log.warn(
-      { err: message, modelMs: Date.now() - modelStartedAt },
-      "Combined dictation LLM call failed, using heuristic",
+      { err: message },
+      "Dictation classification failed, using heuristic",
     );
   }
 
-  // Heuristic fallback
-  const fallbackMode = detectDictationModeHeuristic({
+  const mode = detectDictationModeHeuristic({
     type: "dictation_request",
     transcription: body.transcription,
     context: body.context,
   } as DictationRequest);
-  log.info({ mode: fallbackMode }, "Using heuristic fallback");
-  if (fallbackMode === "action") {
-    return {
-      text: body.transcription,
-      mode: "action",
-      actionPlan: `User wants to: ${body.transcription}`,
-      ...profileMeta,
-    };
-  }
-  const normalizedText = applyDictionary(transcription, profile.dictionary);
-  return {
-    text: normalizedText,
-    mode: fallbackMode,
-    ...profileMeta,
-  };
+  return mode === "action" ? "action" : "dictation";
 }
 
 async function handleCommandMode(
@@ -560,7 +397,7 @@ export const ROUTES: RouteDefinition[] = [
     },
     summary: "Process dictation",
     description:
-      "Classify voice input as dictation or action, clean up text, and apply user style preferences.",
+      "Preserve dictated words with explicit dictionary and snippet replacements, classify actions, or edit selected text when instructed.",
     tags: ["diagnostics"],
     requestBody: DictationRequestSchema,
     responseBody: z.object({

package/src/runtime/routes/llm-call-sites-routes.ts

@@ -1,14 +1,16 @@
 import { z } from "zod";
 
-import { CALL_SITE_DEFAULTS } from "../../config/call-site-defaults.js";
 import { getUserSelectableProfilesForProvider } from "../../config/default-profile-catalog.js";
-import { resolveDefaultProfileKey } from "../../config/llm-resolver.js";
+import {
+  resolveDefaultProfileKey,
+  resolveShippedDefaultProfileKey,
+} from "../../config/llm-resolver.js";
 import { loadConfig } from "../../config/loader.js";
 import {
   CALL_SITE_CATALOG,
   CALL_SITE_DOMAINS,
 } from "../../config/schemas/call-site-catalog.js";
-import type { LLMCallSite } from "../../config/schemas/llm.js";
+import type { LLMCallSite, LLMConfig } from "../../config/schemas/llm.js";
 import { ACTOR_PRINCIPALS } from "../auth/route-policy.js";
 import type { RouteDefinition } from "./types.js";
 
@@ -30,7 +32,8 @@ const callSiteEntrySchema = z.object({
    * `defaultProfile` would scatter pinned or remapped sites across their
    * winners. Profileless call sites report `balanced`: the resolver's
    * fallback anchor consults the balanced remap for them, so they follow
-   * the Balanced tier like any balanced-keyed site.
+   * the Balanced tier like any balanced-keyed site. A managed-only tier is
+   * omitted when it has no body under the workspace's default provider.
    */
   shippedDefaultProfile: z.string().optional(),
 });
@@ -40,16 +43,22 @@ const callSiteCatalogResponseSchema = z.object({
   callSites: z.array(callSiteEntrySchema),
 });
 
+export function buildCallSiteCatalog(llm: LLMConfig) {
+  return CALL_SITE_CATALOG.map((entry) => ({
+    ...entry,
+    defaultProfile: resolveDefaultProfileKey(entry.id as LLMCallSite, llm),
+    shippedDefaultProfile: resolveShippedDefaultProfileKey(
+      entry.id as LLMCallSite,
+      llm,
+    ),
+  }));
+}
+
 async function handleGetCallSites() {
   const { llm } = loadConfig();
   return {
     domains: CALL_SITE_DOMAINS,
-    callSites: CALL_SITE_CATALOG.map((entry) => ({
-      ...entry,
-      defaultProfile: resolveDefaultProfileKey(entry.id as LLMCallSite, llm),
-      shippedDefaultProfile:
-        CALL_SITE_DEFAULTS[entry.id]?.profile ?? "balanced",
-    })),
+    callSites: buildCallSiteCatalog(llm),
   };
 }