@vellumai/assistant 0.11.2-dev.202608072015.e50ee9a → 0.11.2-dev.202608072144.9eaee43

package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts

@@ -6,6 +6,16 @@
  * assistant through (Slack, Telegram, WhatsApp, phone, …) plus a couple of
  * internal ids (`vellum` for native app conversations, `platform` for the
  * internal control plane). This is the single source of truth for that set:
+ *
+ * One id, `plugin`, does not name a surface: it names *every* surface a plugin
+ * brings. A plugin channel's real identity is the plugin, which is workspace
+ * state and cannot be a compile-time union member, so the plugin name travels
+ * in `sourceMetadata.plugin` and is prefixed onto every external id the gateway
+ * forwards (`imessage:+15551234567`). Two plugins therefore share a channel
+ * row — one admission floor, one set of channel-wide defaults — while their
+ * conversations, contacts, and trust records stay disjoint. See
+ * `gateway/src/channels/plugin-inbound.ts` for what that concedes.
+ *
  * the assistant adopts it wholesale as its `ChannelId`, and the gateway
  * asserts its own (narrower) inbound list is a subset of it so the two sides
  * cannot silently drift.
@@ -30,6 +40,7 @@ export const CHANNEL_IDS = [
   "platform",
   "a2a",
   "discord",
+  "plugin",
 ] as const;
 
 export type ChannelId = (typeof CHANNEL_IDS)[number];

package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts

@@ -6,6 +6,16 @@
  * assistant through (Slack, Telegram, WhatsApp, phone, …) plus a couple of
  * internal ids (`vellum` for native app conversations, `platform` for the
  * internal control plane). This is the single source of truth for that set:
+ *
+ * One id, `plugin`, does not name a surface: it names *every* surface a plugin
+ * brings. A plugin channel's real identity is the plugin, which is workspace
+ * state and cannot be a compile-time union member, so the plugin name travels
+ * in `sourceMetadata.plugin` and is prefixed onto every external id the gateway
+ * forwards (`imessage:+15551234567`). Two plugins therefore share a channel
+ * row — one admission floor, one set of channel-wide defaults — while their
+ * conversations, contacts, and trust records stay disjoint. See
+ * `gateway/src/channels/plugin-inbound.ts` for what that concedes.
+ *
  * the assistant adopts it wholesale as its `ChannelId`, and the gateway
  * asserts its own (narrower) inbound list is a subset of it so the two sides
  * cannot silently drift.
@@ -30,6 +40,7 @@ export const CHANNEL_IDS = [
   "platform",
   "a2a",
   "discord",
+  "plugin",
 ] as const;
 
 export type ChannelId = (typeof CHANNEL_IDS)[number];

package/node_modules/@vellumai/service-contracts/src/channels.ts

@@ -6,6 +6,16 @@
  * assistant through (Slack, Telegram, WhatsApp, phone, …) plus a couple of
  * internal ids (`vellum` for native app conversations, `platform` for the
  * internal control plane). This is the single source of truth for that set:
+ *
+ * One id, `plugin`, does not name a surface: it names *every* surface a plugin
+ * brings. A plugin channel's real identity is the plugin, which is workspace
+ * state and cannot be a compile-time union member, so the plugin name travels
+ * in `sourceMetadata.plugin` and is prefixed onto every external id the gateway
+ * forwards (`imessage:+15551234567`). Two plugins therefore share a channel
+ * row — one admission floor, one set of channel-wide defaults — while their
+ * conversations, contacts, and trust records stay disjoint. See
+ * `gateway/src/channels/plugin-inbound.ts` for what that concedes.
+ *
  * the assistant adopts it wholesale as its `ChannelId`, and the gateway
  * asserts its own (narrower) inbound list is a subset of it so the two sides
  * cannot silently drift.
@@ -30,6 +40,7 @@ export const CHANNEL_IDS = [
   "platform",
   "a2a",
   "discord",
+  "plugin",
 ] as const;
 
 export type ChannelId = (typeof CHANNEL_IDS)[number];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.2-dev.202608072015.e50ee9a",
+  "version": "0.11.2-dev.202608072144.9eaee43",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/call-controller.test.ts

@@ -377,6 +377,8 @@ function setupController(
     requiresPcmAudio?: boolean;
     /** Simulate a transport that gates teardown on playback drain. */
     awaitPlaybackDrained?: () => Promise<void>;
+    /** Synthesis-language resolver, as the media-stream server wires it. */
+    resolveSynthesisLanguage?: () => string | undefined;
   },
 ) {
   ensureConversation("conv-ctrl-test");
@@ -400,6 +402,7 @@ function setupController(
   const controller = new CallController(session.id, transport, task ?? null, {
     assistantId: opts?.assistantId,
     trustContext: opts?.trustContext,
+    resolveSynthesisLanguage: opts?.resolveSynthesisLanguage,
   });
   return { session, relay: transport, controller };
 }
@@ -3470,6 +3473,123 @@ describe("call-controller", () => {
     controller.destroy();
   });
 
+  test("synthesized provider: the resolved language rides every segment's synthesis request", async () => {
+    const requestLanguages: Array<string | undefined> = [];
+    registerFishAudioSegmentRecorder({
+      onSynthesizeStream: async (_text, request) => {
+        requestLanguages.push(request.language);
+      },
+    });
+    mockStartVoiceTurn.mockImplementation(
+      createMockVoiceTurn(["Hola desde aqui. ", "Segunda frase."]),
+    );
+    const { controller } = setupController(undefined, {
+      resolveSynthesisLanguage: () => "es",
+    });
+
+    await controller.handleCallerUtterance("Hola");
+
+    expect(requestLanguages).toEqual(["es", "es"]);
+
+    controller.destroy();
+  });
+
+  test("synthesized provider: no language on the request when nothing resolves", async () => {
+    const requestLanguages: Array<string | undefined> = [];
+    registerFishAudioSegmentRecorder({
+      onSynthesizeStream: async (_text, request) => {
+        requestLanguages.push(request.language);
+      },
+    });
+    mockStartVoiceTurn.mockImplementation(
+      createMockVoiceTurn(["Hello from here."]),
+    );
+    // Default resolver + default config (multilingual STT pin): no hint.
+    const { controller } = setupController();
+
+    await controller.handleCallerUtterance("Hi");
+
+    expect(requestLanguages).toEqual([undefined]);
+
+    controller.destroy();
+  });
+
+  function registerDeepgramRequestRecorder(): {
+    requests: Array<{ voiceId?: string; language?: string }>;
+  } {
+    const cfg = loadConfig();
+    cfg.services.tts.provider = "deepgram";
+
+    _resetTtsProviderOverridesForTests();
+    const requests: Array<{ voiceId?: string; language?: string }> = [];
+    const deepgramRecorder: TtsProvider = {
+      id: "deepgram",
+      capabilities: {
+        supportsStreaming: false,
+        supportedFormats: ["mp3", "wav", "opus"],
+      },
+      async synthesize(request) {
+        requests.push({
+          voiceId: request.voiceId,
+          language: request.language,
+        });
+        return { audio: Buffer.from("audio"), contentType: "audio/mpeg" };
+      },
+    };
+    _setTtsProviderForTests(deepgramRecorder);
+    return { requests };
+  }
+
+  test("synthesized provider: a configured languageVoices entry for the call language rides the request as voiceId", async () => {
+    const cfg = loadConfig();
+    cfg.services.tts.providers.deepgram.languageVoices = {
+      hi: "aura-2-hindi-voice",
+    };
+    try {
+      const { requests } = registerDeepgramRequestRecorder();
+      mockStartVoiceTurn.mockImplementation(
+        createMockVoiceTurn(["Namaste doston."]),
+      );
+      const { controller } = setupController(undefined, {
+        resolveSynthesisLanguage: () => "hi",
+      });
+
+      await controller.handleCallerUtterance("Namaste");
+
+      expect(requests).toEqual([
+        { voiceId: "aura-2-hindi-voice", language: "hi" },
+      ]);
+
+      controller.destroy();
+    } finally {
+      delete cfg.services.tts.providers.deepgram.languageVoices;
+    }
+  });
+
+  test("synthesized provider: no voiceId when languageVoices has no entry for the call language", async () => {
+    const cfg = loadConfig();
+    cfg.services.tts.providers.deepgram.languageVoices = {
+      hi: "aura-2-hindi-voice",
+    };
+    try {
+      const { requests } = registerDeepgramRequestRecorder();
+      mockStartVoiceTurn.mockImplementation(
+        createMockVoiceTurn(["Hola desde aqui."]),
+      );
+      const { controller } = setupController(undefined, {
+        resolveSynthesisLanguage: () => "es",
+      });
+
+      await controller.handleCallerUtterance("Hola");
+
+      expect(requests).toEqual([{ voiceId: undefined, language: "es" }]);
+
+      controller.destroy();
+    } finally {
+      delete cfg.services.tts.providers.deepgram.languageVoices;
+    }
+  });
+
   test("synthesized provider: segments synthesize serially in order even when the first is slow", async () => {
     const events: string[] = [];
     registerFishAudioSegmentRecorder({

package/src/__tests__/config-loader-backfill.test.ts

@@ -548,6 +548,25 @@ describe("loadConfig startup behavior", () => {
     expect(raw.daemon?.standaloneRecording).toBe(false);
   });
 
+  test("strips calls.voice.language from existing user configs", () => {
+    // `calls.voice.language` is a retired no-op knob: nothing read it.
+    // The spoken language lives at `services.stt.language` and per-language
+    // TTS voices at `services.tts.providers.<id>.languageVoices`. Existing
+    // configs that have it written to disk should load cleanly with the
+    // field stripped and its siblings preserved.
+    writeConfig({
+      provider: "anthropic",
+      calls: { voice: { language: "es-ES", telephonyStreaming: false } },
+    });
+
+    loadConfig();
+
+    const raw = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+    expect(raw.calls?.voice?.language).toBeUndefined();
+    // Sibling fields under calls.voice are preserved
+    expect(raw.calls?.voice?.telephonyStreaming).toBe(false);
+  });
+
   test("still writes a default config on first launch when file is absent", () => {
     // Discoverability: when no config.json exists, write one populated with
     // all schema defaults so users can see and edit available options.

package/src/__tests__/config-schema.test.ts

@@ -888,7 +888,6 @@ describe("AssistantConfigSchema", () => {
         denyCategories: [],
       },
       voice: {
-        language: "en-US",
         interruptSensitivity: "low",
         telephonyStreaming: true,
         utteranceEndMs: 1000,
@@ -933,6 +932,9 @@ describe("AssistantConfigSchema", () => {
         silenceThresholdMs: 1200,
         maxTurnDurationMs: 30000,
         bargeInMinSpeechMs: 250,
+        echoBargeInMargin: 1.5,
+        echoEmaHalfLifeMs: 400,
+        echoDrainSlackMs: 300,
       },
       frontModel: {
         endpointDecisionTimeoutMs: 1200,
@@ -1079,7 +1081,6 @@ describe("AssistantConfigSchema", () => {
 
   test("config without calls.voice parses correctly and produces defaults", () => {
     const result = AssistantConfigSchema.parse({});
-    expect(result.calls.voice.language).toBe("en-US");
     expect(result.calls.voice.interruptSensitivity).toBe("low");
     expect(result.calls.voice.telephonyStreaming).toBe(true);
     expect(result.calls.voice.utteranceEndMs).toBe(1000);
@@ -1089,17 +1090,26 @@ describe("AssistantConfigSchema", () => {
     const result = AssistantConfigSchema.parse({
       calls: {
         voice: {
-          language: "es-ES",
           telephonyStreaming: false,
           utteranceEndMs: 2500,
         },
       },
     });
-    expect(result.calls.voice.language).toBe("es-ES");
     expect(result.calls.voice.telephonyStreaming).toBe(false);
     expect(result.calls.voice.utteranceEndMs).toBe(2500);
   });
 
+  test("language is no longer part of the voice config schema", () => {
+    // The retired knob was read by nothing; Zod strips the unrecognized key
+    // so persisted configs that still carry it keep parsing.
+    const result = AssistantConfigSchema.parse({
+      calls: { voice: { language: "es-ES" } },
+    });
+    expect(
+      (result.calls.voice as Record<string, unknown>).language,
+    ).toBeUndefined();
+  });
+
   test("rejects calls.voice.utteranceEndMs outside the 1000-5000 range", () => {
     for (const utteranceEndMs of [999, 5001]) {
       const result = AssistantConfigSchema.safeParse({
@@ -1417,6 +1427,131 @@ describe("AssistantConfigSchema", () => {
     expect(result.services.tts.providers.deepgram.format).toBe("opus");
   });
 
+  test("accepts services.tts.providers languageVoices maps", () => {
+    const result = AssistantConfigSchema.parse({
+      services: {
+        tts: {
+          providers: {
+            elevenlabs: {
+              languageVoices: { hi: "voice-hindi", ja: "voice-japanese" },
+            },
+            deepgram: { languageVoices: { hi: "aura-2-hi-voice" } },
+            vellum: { languageVoices: { hi: "aura-2-hi-voice" } },
+          },
+        },
+      },
+    });
+    expect(result.services.tts.providers.elevenlabs.languageVoices).toEqual({
+      hi: "voice-hindi",
+      ja: "voice-japanese",
+    });
+    expect(result.services.tts.providers.deepgram.languageVoices).toEqual({
+      hi: "aura-2-hi-voice",
+    });
+    expect(result.services.tts.providers.vellum.languageVoices).toEqual({
+      hi: "aura-2-hi-voice",
+    });
+  });
+
+  test("normalizes user-entered languageVoices keys to lowercase base subtags", () => {
+    // "hi-IN" or "HI" typed into config must still match a spoken "hi"
+    // instead of silently never applying; the schema normalizes on parse.
+    const result = AssistantConfigSchema.parse({
+      services: {
+        tts: {
+          providers: {
+            elevenlabs: {
+              languageVoices: {
+                "hi-IN": "voice-hindi",
+                JA: "voice-japanese",
+                es_419: "voice-spanish",
+                " PT ": "voice-portuguese",
+              },
+            },
+          },
+        },
+      },
+    });
+    expect(result.services.tts.providers.elevenlabs.languageVoices).toEqual({
+      hi: "voice-hindi",
+      ja: "voice-japanese",
+      es: "voice-spanish",
+      pt: "voice-portuguese",
+    });
+  });
+
+  test("keeps the first entry when languageVoices keys collide after normalization", () => {
+    const result = AssistantConfigSchema.parse({
+      services: {
+        tts: {
+          providers: {
+            deepgram: {
+              languageVoices: { "hi-IN": "voice-first", hi: "voice-second" },
+            },
+          },
+        },
+      },
+    });
+    expect(result.services.tts.providers.deepgram.languageVoices).toEqual({
+      hi: "voice-first",
+    });
+  });
+
+  test("drops languageVoices keys that normalize to the empty string", () => {
+    const result = AssistantConfigSchema.parse({
+      services: {
+        tts: {
+          providers: {
+            vellum: {
+              languageVoices: {
+                "": "voice-empty",
+                "  ": "voice-blank",
+                "-IN": "voice-region-only",
+                hi: "voice-hindi",
+              },
+            },
+          },
+        },
+      },
+    });
+    expect(result.services.tts.providers.vellum.languageVoices).toEqual({
+      hi: "voice-hindi",
+    });
+  });
+
+  test("languageVoices defaults to unset", () => {
+    const result = AssistantConfigSchema.parse({});
+    expect(
+      result.services.tts.providers.elevenlabs.languageVoices,
+    ).toBeUndefined();
+    expect(
+      result.services.tts.providers.deepgram.languageVoices,
+    ).toBeUndefined();
+    expect(result.services.tts.providers.vellum.languageVoices).toBeUndefined();
+  });
+
+  test("rejects non-string services.tts.providers languageVoices values", () => {
+    for (const provider of ["elevenlabs", "deepgram", "vellum"]) {
+      const result = AssistantConfigSchema.safeParse({
+        services: {
+          tts: {
+            providers: {
+              [provider]: { languageVoices: { hi: 42 } },
+            },
+          },
+        },
+      });
+      expect(result.success).toBe(false);
+      if (!result.success) {
+        expect(
+          result.error.issues.some((issue) =>
+            issue.path.join(".").includes("languageVoices"),
+          ),
+        ).toBe(true);
+      }
+    }
+  });
+
   // Legacy config objects on disk may carry a `mode` key. Parsing must strip
   // it rather than reject the config — and must not treat it as a managed
   // selection, which is what migration 130 rewrites to `provider: "vellum"`.
@@ -2432,7 +2567,9 @@ describe("loadConfig with schema validation", () => {
     expect(config.calls.userConsultTimeoutSeconds).toBe(120);
     expect(config.calls.disclosure.enabled).toBe(true);
     expect(config.calls.safety.denyCategories).toEqual([]);
-    expect(config.calls.voice.language).toBe("en-US");
+    expect(
+      (config.calls.voice as Record<string, unknown>).language,
+    ).toBeUndefined();
     expect(
       (config.calls.voice as Record<string, unknown>).transcriptionProvider,
     ).toBeUndefined();

package/src/__tests__/media-stream-output.test.ts

@@ -26,6 +26,7 @@ import {
 } from "../calls/media-stream-audio-transcode.js";
 import { MediaStreamOutput } from "../calls/media-stream-output.js";
 import { resolveCallTtsProvider } from "../calls/resolve-call-tts-provider.js";
+import { setConfig } from "./helpers/set-config.js";
 
 const mockResolveCallTtsProvider = resolveCallTtsProvider as ReturnType<
   typeof jest.fn
@@ -1213,6 +1214,180 @@ describe("MediaStreamOutput", () => {
     });
   });
 
+  // ---------------------------------------------------------------------------
+  // Synthesis language hint
+  // ---------------------------------------------------------------------------
+
+  describe("synthesis language hint", () => {
+    /** Synthesize one turn and return the provider request it produced. */
+    async function synthesizeOneTurn(
+      output: MediaStreamOutput,
+    ): Promise<{ language?: string }> {
+      mockSynthesize.mockResolvedValue({
+        audio: makeWavBuffer([1000, 2000, 3000, 4000]),
+        contentType: "audio/wav",
+      });
+      output.sendTextToken("Hello caller.", true);
+      await drain(() => mockSynthesize.mock.calls.length > 0);
+      return mockSynthesize.mock.calls[0][0] as { language?: string };
+    }
+
+    test("the resolver's language rides the provider request", async () => {
+      const { ws } = createMockWs();
+      const output = makeOutput(ws, "stream-lang");
+      output.setSynthesisLanguageResolver(() => "es");
+
+      const request = await synthesizeOneTurn(output);
+
+      expect(request.language).toBe("es");
+    });
+
+    test("fixed system copy synthesizes without the caller-language hint while model text keeps it", async () => {
+      const { ws } = createMockWs();
+      const output = makeOutput(ws, "stream-lang");
+      output.setSynthesisLanguageResolver(() => "ja");
+      mockSynthesize.mockResolvedValue({
+        audio: makeWavBuffer([1000, 2000, 3000, 4000]),
+        contentType: "audio/wav",
+      });
+
+      output.sendTextToken("Model reply.", true);
+      output.sendTextToken("Are you still there?", true, { systemCopy: true });
+      await drain(() => mockSynthesize.mock.calls.length >= 2);
+
+      const requests = mockSynthesize.mock.calls.map(
+        (call) => call[0] as { text: string; language?: string },
+      );
+      expect(requests[0]?.text).toBe("Model reply.");
+      expect(requests[0]?.language).toBe("ja");
+      expect(requests[1]?.text).toBe("Are you still there?");
+      expect(requests[1]?.language).toBeUndefined();
+    });
+
+    test("no language when nothing resolves (default multilingual pin)", async () => {
+      const { ws } = createMockWs();
+      const output = makeOutput(ws, "stream-lang");
+
+      const request = await synthesizeOneTurn(output);
+
+      expect(request.language).toBeUndefined();
+    });
+
+    test("a monolingual pin on a manual-selection provider resolves as the hint", async () => {
+      setConfig("services", {
+        stt: { provider: "deepgram", language: "es-ES" },
+      });
+      try {
+        const { ws } = createMockWs();
+        const output = makeOutput(ws, "stream-lang");
+
+        const request = await synthesizeOneTurn(output);
+
+        expect(request.language).toBe("es");
+      } finally {
+        setConfig("services", {});
+      }
+    });
+
+    test("the pin is ignored for auto-detecting providers", async () => {
+      setConfig("services", {
+        stt: { provider: "openai-whisper", language: "es" },
+      });
+      try {
+        const { ws } = createMockWs();
+        const output = makeOutput(ws, "stream-lang");
+
+        const request = await synthesizeOneTurn(output);
+
+        expect(request.language).toBeUndefined();
+      } finally {
+        setConfig("services", {});
+      }
+    });
+  });
+
+  // ---------------------------------------------------------------------------
+  // Per-language voice override (languageVoices)
+  // ---------------------------------------------------------------------------
+
+  describe("per-language voice override", () => {
+    afterEach(() => {
+      setConfig("services", {});
+    });
+
+    /** Seed a deepgram languageVoices entry and resolve deepgram as the provider. */
+    function useDeepgramWithHindiVoice(): void {
+      setConfig("services", {
+        tts: {
+          provider: "deepgram",
+          providers: {
+            deepgram: { languageVoices: { hi: "aura-2-hindi-voice" } },
+          },
+        },
+      });
+      useProvider({
+        id: "deepgram",
+        capabilities: { supportsStreaming: false, supportedFormats: ["wav"] },
+        synthesize: mockSynthesize,
+      });
+      mockSynthesize.mockResolvedValue({
+        audio: makeWavBuffer([1000, 2000, 3000, 4000]),
+        contentType: "audio/wav",
+      });
+    }
+
+    test("a configured languageVoices entry for the resolved language rides the request as voiceId", async () => {
+      useDeepgramWithHindiVoice();
+      const { ws } = createMockWs();
+      const output = makeOutput(ws, "stream-voice");
+      output.setSynthesisLanguageResolver(() => "hi");
+
+      output.sendTextToken("Namaste.", true);
+      await drain(() => mockSynthesize.mock.calls.length > 0);
+
+      const request = mockSynthesize.mock.calls[0][0] as {
+        voiceId?: string;
+        language?: string;
+      };
+      expect(request.language).toBe("hi");
+      expect(request.voiceId).toBe("aura-2-hindi-voice");
+    });
+
+    test("no voiceId when the map has no entry for the resolved language", async () => {
+      useDeepgramWithHindiVoice();
+      const { ws } = createMockWs();
+      const output = makeOutput(ws, "stream-voice");
+      output.setSynthesisLanguageResolver(() => "es");
+
+      output.sendTextToken("Hola.", true);
+      await drain(() => mockSynthesize.mock.calls.length > 0);
+
+      const request = mockSynthesize.mock.calls[0][0] as {
+        voiceId?: string;
+        language?: string;
+      };
+      expect(request.language).toBe("es");
+      expect(request.voiceId).toBeUndefined();
+    });
+
+    test("system copy gets neither the language hint nor the language voice", async () => {
+      useDeepgramWithHindiVoice();
+      const { ws } = createMockWs();
+      const output = makeOutput(ws, "stream-voice");
+      output.setSynthesisLanguageResolver(() => "hi");
+
+      output.sendTextToken("Are you still there?", true, { systemCopy: true });
+      await drain(() => mockSynthesize.mock.calls.length > 0);
+
+      const request = mockSynthesize.mock.calls[0][0] as {
+        voiceId?: string;
+        language?: string;
+      };
+      expect(request.language).toBeUndefined();
+      expect(request.voiceId).toBeUndefined();
+    });
+  });
+
   // ---------------------------------------------------------------------------
   // Streaming PCM synthesis (incremental transcode)
   // ---------------------------------------------------------------------------