@vellumai/assistant 0.11.3-staging.2 → 0.11.3-staging.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.3-staging.2",
+  "version": "0.11.3-staging.3",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/call-controller.test.ts

@@ -377,6 +377,8 @@ function setupController(
     requiresPcmAudio?: boolean;
     /** Simulate a transport that gates teardown on playback drain. */
     awaitPlaybackDrained?: () => Promise<void>;
+    /** Synthesis-language resolver, as the media-stream server wires it. */
+    resolveSynthesisLanguage?: () => string | undefined;
   },
 ) {
   ensureConversation("conv-ctrl-test");
@@ -400,6 +402,7 @@ function setupController(
   const controller = new CallController(session.id, transport, task ?? null, {
     assistantId: opts?.assistantId,
     trustContext: opts?.trustContext,
+    resolveSynthesisLanguage: opts?.resolveSynthesisLanguage,
   });
   return { session, relay: transport, controller };
 }
@@ -3470,6 +3473,123 @@ describe("call-controller", () => {
     controller.destroy();
   });
 
+  test("synthesized provider: the resolved language rides every segment's synthesis request", async () => {
+    const requestLanguages: Array<string | undefined> = [];
+    registerFishAudioSegmentRecorder({
+      onSynthesizeStream: async (_text, request) => {
+        requestLanguages.push(request.language);
+      },
+    });
+    mockStartVoiceTurn.mockImplementation(
+      createMockVoiceTurn(["Hola desde aqui. ", "Segunda frase."]),
+    );
+    const { controller } = setupController(undefined, {
+      resolveSynthesisLanguage: () => "es",
+    });
+
+    await controller.handleCallerUtterance("Hola");
+
+    expect(requestLanguages).toEqual(["es", "es"]);
+
+    controller.destroy();
+  });
+
+  test("synthesized provider: no language on the request when nothing resolves", async () => {
+    const requestLanguages: Array<string | undefined> = [];
+    registerFishAudioSegmentRecorder({
+      onSynthesizeStream: async (_text, request) => {
+        requestLanguages.push(request.language);
+      },
+    });
+    mockStartVoiceTurn.mockImplementation(
+      createMockVoiceTurn(["Hello from here."]),
+    );
+    // Default resolver + default config (multilingual STT pin): no hint.
+    const { controller } = setupController();
+
+    await controller.handleCallerUtterance("Hi");
+
+    expect(requestLanguages).toEqual([undefined]);
+
+    controller.destroy();
+  });
+
+  function registerDeepgramRequestRecorder(): {
+    requests: Array<{ voiceId?: string; language?: string }>;
+  } {
+    const cfg = loadConfig();
+    cfg.services.tts.provider = "deepgram";
+
+    _resetTtsProviderOverridesForTests();
+    const requests: Array<{ voiceId?: string; language?: string }> = [];
+    const deepgramRecorder: TtsProvider = {
+      id: "deepgram",
+      capabilities: {
+        supportsStreaming: false,
+        supportedFormats: ["mp3", "wav", "opus"],
+      },
+      async synthesize(request) {
+        requests.push({
+          voiceId: request.voiceId,
+          language: request.language,
+        });
+        return { audio: Buffer.from("audio"), contentType: "audio/mpeg" };
+      },
+    };
+    _setTtsProviderForTests(deepgramRecorder);
+    return { requests };
+  }
+
+  test("synthesized provider: a configured languageVoices entry for the call language rides the request as voiceId", async () => {
+    const cfg = loadConfig();
+    cfg.services.tts.providers.deepgram.languageVoices = {
+      hi: "aura-2-hindi-voice",
+    };
+    try {
+      const { requests } = registerDeepgramRequestRecorder();
+      mockStartVoiceTurn.mockImplementation(
+        createMockVoiceTurn(["Namaste doston."]),
+      );
+      const { controller } = setupController(undefined, {
+        resolveSynthesisLanguage: () => "hi",
+      });
+
+      await controller.handleCallerUtterance("Namaste");
+
+      expect(requests).toEqual([
+        { voiceId: "aura-2-hindi-voice", language: "hi" },
+      ]);
+
+      controller.destroy();
+    } finally {
+      delete cfg.services.tts.providers.deepgram.languageVoices;
+    }
+  });
+
+  test("synthesized provider: no voiceId when languageVoices has no entry for the call language", async () => {
+    const cfg = loadConfig();
+    cfg.services.tts.providers.deepgram.languageVoices = {
+      hi: "aura-2-hindi-voice",
+    };
+    try {
+      const { requests } = registerDeepgramRequestRecorder();
+      mockStartVoiceTurn.mockImplementation(
+        createMockVoiceTurn(["Hola desde aqui."]),
+      );
+      const { controller } = setupController(undefined, {
+        resolveSynthesisLanguage: () => "es",
+      });
+
+      await controller.handleCallerUtterance("Hola");
+
+      expect(requests).toEqual([{ voiceId: undefined, language: "es" }]);
+
+      controller.destroy();
+    } finally {
+      delete cfg.services.tts.providers.deepgram.languageVoices;
+    }
+  });
+
   test("synthesized provider: segments synthesize serially in order even when the first is slow", async () => {
     const events: string[] = [];
     registerFishAudioSegmentRecorder({

package/src/__tests__/config-loader-backfill.test.ts

@@ -548,6 +548,25 @@ describe("loadConfig startup behavior", () => {
     expect(raw.daemon?.standaloneRecording).toBe(false);
   });
 
+  test("strips calls.voice.language from existing user configs", () => {
+    // `calls.voice.language` is a retired no-op knob: nothing read it.
+    // The spoken language lives at `services.stt.language` and per-language
+    // TTS voices at `services.tts.providers.<id>.languageVoices`. Existing
+    // configs that have it written to disk should load cleanly with the
+    // field stripped and its siblings preserved.
+    writeConfig({
+      provider: "anthropic",
+      calls: { voice: { language: "es-ES", telephonyStreaming: false } },
+    });
+
+    loadConfig();
+
+    const raw = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+    expect(raw.calls?.voice?.language).toBeUndefined();
+    // Sibling fields under calls.voice are preserved
+    expect(raw.calls?.voice?.telephonyStreaming).toBe(false);
+  });
+
   test("still writes a default config on first launch when file is absent", () => {
     // Discoverability: when no config.json exists, write one populated with
     // all schema defaults so users can see and edit available options.

package/src/__tests__/config-schema.test.ts

@@ -888,7 +888,6 @@ describe("AssistantConfigSchema", () => {
         denyCategories: [],
       },
       voice: {
-        language: "en-US",
         interruptSensitivity: "low",
         telephonyStreaming: true,
         utteranceEndMs: 1000,
@@ -1071,7 +1070,6 @@ describe("AssistantConfigSchema", () => {
 
   test("config without calls.voice parses correctly and produces defaults", () => {
     const result = AssistantConfigSchema.parse({});
-    expect(result.calls.voice.language).toBe("en-US");
     expect(result.calls.voice.interruptSensitivity).toBe("low");
     expect(result.calls.voice.telephonyStreaming).toBe(true);
     expect(result.calls.voice.utteranceEndMs).toBe(1000);
@@ -1081,17 +1079,26 @@ describe("AssistantConfigSchema", () => {
     const result = AssistantConfigSchema.parse({
       calls: {
         voice: {
-          language: "es-ES",
           telephonyStreaming: false,
           utteranceEndMs: 2500,
         },
       },
     });
-    expect(result.calls.voice.language).toBe("es-ES");
     expect(result.calls.voice.telephonyStreaming).toBe(false);
     expect(result.calls.voice.utteranceEndMs).toBe(2500);
   });
 
+  test("language is no longer part of the voice config schema", () => {
+    // The retired knob was read by nothing; Zod strips the unrecognized key
+    // so persisted configs that still carry it keep parsing.
+    const result = AssistantConfigSchema.parse({
+      calls: { voice: { language: "es-ES" } },
+    });
+    expect(
+      (result.calls.voice as Record<string, unknown>).language,
+    ).toBeUndefined();
+  });
+
   test("rejects calls.voice.utteranceEndMs outside the 1000-5000 range", () => {
     for (const utteranceEndMs of [999, 5001]) {
       const result = AssistantConfigSchema.safeParse({
@@ -1409,6 +1416,131 @@ describe("AssistantConfigSchema", () => {
     expect(result.services.tts.providers.deepgram.format).toBe("opus");
   });
 
+  test("accepts services.tts.providers languageVoices maps", () => {
+    const result = AssistantConfigSchema.parse({
+      services: {
+        tts: {
+          providers: {
+            elevenlabs: {
+              languageVoices: { hi: "voice-hindi", ja: "voice-japanese" },
+            },
+            deepgram: { languageVoices: { hi: "aura-2-hi-voice" } },
+            vellum: { languageVoices: { hi: "aura-2-hi-voice" } },
+          },
+        },
+      },
+    });
+    expect(result.services.tts.providers.elevenlabs.languageVoices).toEqual({
+      hi: "voice-hindi",
+      ja: "voice-japanese",
+    });
+    expect(result.services.tts.providers.deepgram.languageVoices).toEqual({
+      hi: "aura-2-hi-voice",
+    });
+    expect(result.services.tts.providers.vellum.languageVoices).toEqual({
+      hi: "aura-2-hi-voice",
+    });
+  });
+
+  test("normalizes user-entered languageVoices keys to lowercase base subtags", () => {
+    // "hi-IN" or "HI" typed into config must still match a spoken "hi"
+    // instead of silently never applying; the schema normalizes on parse.
+    const result = AssistantConfigSchema.parse({
+      services: {
+        tts: {
+          providers: {
+            elevenlabs: {
+              languageVoices: {
+                "hi-IN": "voice-hindi",
+                JA: "voice-japanese",
+                es_419: "voice-spanish",
+                " PT ": "voice-portuguese",
+              },
+            },
+          },
+        },
+      },
+    });
+    expect(result.services.tts.providers.elevenlabs.languageVoices).toEqual({
+      hi: "voice-hindi",
+      ja: "voice-japanese",
+      es: "voice-spanish",
+      pt: "voice-portuguese",
+    });
+  });
+
+  test("keeps the first entry when languageVoices keys collide after normalization", () => {
+    const result = AssistantConfigSchema.parse({
+      services: {
+        tts: {
+          providers: {
+            deepgram: {
+              languageVoices: { "hi-IN": "voice-first", hi: "voice-second" },
+            },
+          },
+        },
+      },
+    });
+    expect(result.services.tts.providers.deepgram.languageVoices).toEqual({
+      hi: "voice-first",
+    });
+  });
+
+  test("drops languageVoices keys that normalize to the empty string", () => {
+    const result = AssistantConfigSchema.parse({
+      services: {
+        tts: {
+          providers: {
+            vellum: {
+              languageVoices: {
+                "": "voice-empty",
+                "  ": "voice-blank",
+                "-IN": "voice-region-only",
+                hi: "voice-hindi",
+              },
+            },
+          },
+        },
+      },
+    });
+    expect(result.services.tts.providers.vellum.languageVoices).toEqual({
+      hi: "voice-hindi",
+    });
+  });
+
+  test("languageVoices defaults to unset", () => {
+    const result = AssistantConfigSchema.parse({});
+    expect(
+      result.services.tts.providers.elevenlabs.languageVoices,
+    ).toBeUndefined();
+    expect(
+      result.services.tts.providers.deepgram.languageVoices,
+    ).toBeUndefined();
+    expect(result.services.tts.providers.vellum.languageVoices).toBeUndefined();
+  });
+
+  test("rejects non-string services.tts.providers languageVoices values", () => {
+    for (const provider of ["elevenlabs", "deepgram", "vellum"]) {
+      const result = AssistantConfigSchema.safeParse({
+        services: {
+          tts: {
+            providers: {
+              [provider]: { languageVoices: { hi: 42 } },
+            },
+          },
+        },
+      });
+      expect(result.success).toBe(false);
+      if (!result.success) {
+        expect(
+          result.error.issues.some((issue) =>
+            issue.path.join(".").includes("languageVoices"),
+          ),
+        ).toBe(true);
+      }
+    }
+  });
+
   // Legacy config objects on disk may carry a `mode` key. Parsing must strip
   // it rather than reject the config — and must not treat it as a managed
   // selection, which is what migration 130 rewrites to `provider: "vellum"`.
@@ -2424,7 +2556,9 @@ describe("loadConfig with schema validation", () => {
     expect(config.calls.userConsultTimeoutSeconds).toBe(120);
     expect(config.calls.disclosure.enabled).toBe(true);
     expect(config.calls.safety.denyCategories).toEqual([]);
-    expect(config.calls.voice.language).toBe("en-US");
+    expect(
+      (config.calls.voice as Record<string, unknown>).language,
+    ).toBeUndefined();
     expect(
       (config.calls.voice as Record<string, unknown>).transcriptionProvider,
     ).toBeUndefined();

package/src/__tests__/events-dev-bypass-actor.test.ts

@@ -27,6 +27,10 @@ let fakeGuardianPrincipalId: string | undefined = undefined;
 // exactly when the SSE self-heal lookup completes.
 const pendingAsyncResolutions: Array<(value: string | undefined) => void> = [];
 
+/** Options each async resolution was called with, in the same order. */
+const asyncResolutionOptions: Array<{ forceRefresh?: boolean } | undefined> =
+  [];
+
 mock.module("../config/env.js", () => ({
   isHttpAuthDisabled: () => fakeHttpAuthDisabled,
   hasUngatedHttpAuthDisabled: () => false,
@@ -42,10 +46,14 @@ mock.module("../runtime/local-actor-identity.js", () => ({
     }
     return fakeGuardianPrincipalId;
   },
-  resolveActorPrincipalIdForLocalGuardian: (rawHeader: string | undefined) => {
+  resolveActorPrincipalIdForLocalGuardian: (
+    rawHeader: string | undefined,
+    options?: { forceRefresh?: boolean },
+  ) => {
     if (rawHeader !== "dev-bypass" || !fakeHttpAuthDisabled) {
       return Promise.resolve(rawHeader);
     }
+    asyncResolutionOptions.push(options);
     return new Promise<string | undefined>((resolve) => {
       pendingAsyncResolutions.push(resolve);
     });
@@ -57,6 +65,23 @@ async function flushMicrotasks(): Promise<void> {
   await new Promise((resolve) => setTimeout(resolve, 0));
 }
 
+/**
+ * Poll until `predicate` holds, for assertions that depend on the heal's
+ * retry backoff timer rather than on promise settling alone.
+ */
+async function waitFor(
+  predicate: () => boolean,
+  timeoutMs = 3_000,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (!predicate()) {
+    if (Date.now() >= deadline) {
+      throw new Error("waitFor timed out");
+    }
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+}
+
 // ── Real imports (after mocks) ────────────────────────────────────────────
 
 import { AssistantEventHub } from "../runtime/assistant-event-hub.js";
@@ -69,6 +94,7 @@ afterAll(() => {
 describe("events SSE registration — dev-bypass actor translation", () => {
   beforeEach(() => {
     pendingAsyncResolutions.length = 0;
+    asyncResolutionOptions.length = 0;
   });
 
   test("translates 'dev-bypass' to the real guardian principalId when auth is disabled", () => {
@@ -261,6 +287,91 @@ describe("events SSE registration — dev-bypass actor translation", () => {
     ac2.abort();
   });
 
+  test("retries the heal after a failed lookup instead of giving up for the connection's lifetime", async () => {
+    // The gateway IPC returns null on any transport failure, so the first
+    // lookup can miss even when a guardian is bound. A subscription left
+    // principal-less rejects every host-proxy result it submits, so the heal
+    // has to outlive one miss rather than stopping there.
+    fakeHttpAuthDisabled = true;
+    fakeGuardianPrincipalId = undefined; // cold cache: sync resolution misses
+
+    const ac = new AbortController();
+    const hub = new AssistantEventHub();
+
+    handleSubscribeAssistantEvents(
+      {
+        headers: {
+          "x-vellum-client-id": "retry-client-001",
+          "x-vellum-interface-id": "chrome-extension",
+          "x-vellum-actor-principal-id": "dev-bypass",
+        },
+        abortSignal: ac.signal,
+      },
+      { hub },
+    );
+
+    // First attempt fails (gateway not reachable yet).
+    expect(pendingAsyncResolutions).toHaveLength(1);
+    pendingAsyncResolutions.shift()!(undefined);
+    await flushMicrotasks();
+    expect(
+      hub.getActorPrincipalIdForClient("retry-client-001"),
+    ).toBeUndefined();
+
+    // The retry fires on the backoff and succeeds this time.
+    await waitFor(() => pendingAsyncResolutions.length === 1);
+    pendingAsyncResolutions.shift()!("guardian-real-id");
+    await flushMicrotasks();
+
+    expect(hub.getActorPrincipalIdForClient("retry-client-001")).toBe(
+      "guardian-real-id",
+    );
+
+    // Every attempt must bypass the guardian-delivery cache. A successful read
+    // that finds no binding is cached for minutes, which outlives the retry
+    // schedule, so a cached read would spend all attempts on the same answer
+    // and never observe a binding created in between.
+    expect(asyncResolutionOptions).toEqual([
+      { forceRefresh: true },
+      { forceRefresh: true },
+    ]);
+
+    ac.abort();
+  });
+
+  test("needsActorPrincipalHeal reports only live, principal-less client subscriptions", () => {
+    const hub = new AssistantEventHub();
+    const bare = hub.subscribe({
+      type: "client",
+      clientId: "needs-heal-001",
+      interfaceId: "chrome-extension",
+      capabilities: [],
+      callback: () => {},
+    });
+    const withPrincipal = hub.subscribe({
+      type: "client",
+      clientId: "needs-heal-002",
+      interfaceId: "macos",
+      capabilities: [],
+      actorPrincipalId: "guardian-real-id",
+      callback: () => {},
+    });
+    const process = hub.subscribe({ type: "process", callback: () => {} });
+
+    expect(hub.needsActorPrincipalHeal(bare.connectionId)).toBe(true);
+    expect(hub.needsActorPrincipalHeal(withPrincipal.connectionId)).toBe(false);
+    expect(hub.needsActorPrincipalHeal(process.connectionId)).toBe(false);
+    expect(hub.needsActorPrincipalHeal("conn-does-not-exist")).toBe(false);
+
+    // A disposed connection stops needing a heal, which is what ends the
+    // retry loop when a client disconnects mid-backoff.
+    bare.dispose();
+    expect(hub.needsActorPrincipalHeal(bare.connectionId)).toBe(false);
+
+    withPrincipal.dispose();
+    process.dispose();
+  });
+
   test("fillClientActorPrincipalId never overwrites a present principal", () => {
     const hub = new AssistantEventHub();
     const sub = hub.subscribe({