@vellumai/assistant 0.11.3-staging.2 → 0.11.3-staging.4

package/src/stt/__tests__/speech-energy.test.ts

@@ -3,6 +3,8 @@ import { describe, expect, test } from "bun:test";
 import {
   DEFAULT_SPEECH_ENERGY_THRESHOLD,
   detectPcm16SpeechActivity,
+  pcm16MaxNormalizedCorrelation,
+  pcm16MeanAmplitude,
 } from "../speech-energy.js";
 
 /** Build a PCM16LE buffer from an array of sample values. */
@@ -66,3 +68,80 @@ describe("detectPcm16SpeechActivity", () => {
     expect(detectPcm16SpeechActivity(quiet, 500)).toBe(false);
   });
 });
+
+describe("pcm16MeanAmplitude", () => {
+  test("returns 0 for an empty buffer", () => {
+    expect(pcm16MeanAmplitude(Buffer.alloc(0))).toBe(0);
+  });
+
+  test("returns the exact mean absolute amplitude", () => {
+    expect(pcm16MeanAmplitude(pcm16([1_000, -2_000, 3_000, -4_000]))).toBe(
+      2_500,
+    );
+  });
+
+  test("ignores a trailing odd byte", () => {
+    const samples = pcm16([3_000, -3_000]);
+    expect(
+      pcm16MeanAmplitude(Buffer.concat([samples, Buffer.from([0x01])])),
+    ).toBe(3_000);
+  });
+
+  test("matches the detector's threshold comparison", () => {
+    const chunks = [
+      Buffer.alloc(0),
+      pcm16([0, 0]),
+      pcm16([500, -500]),
+      pcm16([DEFAULT_SPEECH_ENERGY_THRESHOLD]),
+      pcm16([DEFAULT_SPEECH_ENERGY_THRESHOLD + 1]),
+    ];
+    for (const chunk of chunks) {
+      expect(detectPcm16SpeechActivity(chunk)).toBe(
+        pcm16MeanAmplitude(chunk) > DEFAULT_SPEECH_ENERGY_THRESHOLD,
+      );
+    }
+  });
+});
+
+describe("pcm16MaxNormalizedCorrelation", () => {
+  const wave = (frequency: number, count: number): Buffer =>
+    pcm16(
+      Array.from({ length: count }, (_, index) =>
+        Math.round(
+          8_000 * Math.sin((2 * Math.PI * frequency * index) / 16_000),
+        ),
+      ),
+    );
+
+  test("finds a matching waveform at an arbitrary reference offset", () => {
+    const input = wave(240, 800);
+    const reference = Buffer.concat([wave(410, 400), input, wave(610, 400)]);
+
+    expect(pcm16MaxNormalizedCorrelation(input, reference)).toBeGreaterThan(
+      0.99,
+    );
+  });
+
+  test("is invariant to gain and polarity", () => {
+    const inputSamples = Array.from({ length: 800 }, (_, index) =>
+      Math.round(5_000 * Math.sin((2 * Math.PI * index) / 83)),
+    );
+    const inverted = pcm16(inputSamples.map((sample) => -2 * sample));
+
+    expect(
+      pcm16MaxNormalizedCorrelation(pcm16(inputSamples), inverted),
+    ).toBeGreaterThan(0.99);
+  });
+
+  test("rejects an unrelated waveform and flat power", () => {
+    expect(
+      pcm16MaxNormalizedCorrelation(wave(240, 800), wave(610, 1_600)),
+    ).toBeLessThan(0.3);
+    expect(
+      pcm16MaxNormalizedCorrelation(
+        pcm16(new Array(800).fill(3_000)),
+        pcm16(new Array(1_600).fill(3_000)),
+      ),
+    ).toBe(0);
+  });
+});

package/src/stt/language-metadata.ts

@@ -0,0 +1,65 @@
+/**
+ * Pure helpers for tallying and ranking detected-language tags emitted
+ * by streaming STT providers (e.g. Deepgram nova-3 `multi` per-word tags).
+ *
+ * Tag normalization is `baseLanguageSubtag` (util/language-subtag.ts);
+ * these helpers layer vote-counting on top of it so provider adapters and
+ * event consumers share one notion of a dominant language.
+ */
+
+import { baseLanguageSubtag } from "../util/language-subtag.js";
+
+/**
+ * Cast one vote into a language tally for a transcript chunk's dominant
+ * detected language. `languages` is dominance-ranked, so only the first
+ * entry votes: secondary tags at full weight would let a minority
+ * language outvote the dominant one. Blank and absent tags cast no vote;
+ * regional variants count toward their base subtag.
+ */
+export function voteDominantLanguage(
+  tally: Map<string, number>,
+  languages: readonly string[] | undefined,
+): void {
+  const dominant = baseLanguageSubtag(languages?.[0]);
+  if (dominant !== undefined) {
+    tally.set(dominant, (tally.get(dominant) ?? 0) + 1);
+  }
+}
+
+/**
+ * The dominant tag in a language tally: most votes wins, ties break by
+ * first insertion (Map iteration is insertion order). Undefined for an
+ * empty tally.
+ */
+export function dominantLanguageTag(
+  tally: ReadonlyMap<string, number>,
+): string | undefined {
+  let dominant: string | undefined;
+  let dominantCount = 0;
+  for (const [tag, count] of tally) {
+    if (count > dominantCount) {
+      dominant = tag;
+      dominantCount = count;
+    }
+  }
+  return dominant;
+}
+
+/**
+ * Tally normalized language tags and return them most-frequent-first.
+ * Ties are broken by first appearance in the input. Blank tags are
+ * skipped; regional variants count toward their base subtag.
+ */
+export function rankLanguages(tags: Iterable<string>): string[] {
+  // Map iteration order is insertion order, so a stable sort by count
+  // leaves tied tags in first-appearance order.
+  const counts = new Map<string, number>();
+  for (const tag of tags) {
+    const normalized = baseLanguageSubtag(tag);
+    if (normalized === undefined) {
+      continue;
+    }
+    counts.set(normalized, (counts.get(normalized) ?? 0) + 1);
+  }
+  return [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([tag]) => tag);
+}

package/src/stt/speech-energy.ts

@@ -18,6 +18,120 @@
  */
 export const DEFAULT_SPEECH_ENERGY_THRESHOLD = 800;
 
+/**
+ * Return the mean absolute amplitude of little-endian signed PCM16 audio.
+ * Empty buffers return 0, and a trailing odd byte is ignored.
+ */
+export function pcm16MeanAmplitude(chunk: Buffer): number {
+  const sampleCount = Math.floor(chunk.length / 2);
+  if (sampleCount === 0) {
+    return 0;
+  }
+
+  let totalAmplitude = 0;
+  for (let i = 0; i < sampleCount; i += 1) {
+    totalAmplitude += Math.abs(chunk.readInt16LE(i * 2));
+  }
+  return totalAmplitude / sampleCount;
+}
+
+/**
+ * Find the strongest normalized correlation between a PCM16 input window and
+ * any same-length window in a PCM16 reference. The samples are block-averaged
+ * before matching so this stays cheap enough for live audio and remains
+ * tolerant of the low-pass filtering introduced by speakers and microphones.
+ *
+ * The absolute coefficient makes polarity inversion harmless. A flat input or
+ * reference has no identifying waveform and returns 0 instead of being treated
+ * as a match based on level alone.
+ */
+export function pcm16MaxNormalizedCorrelation(
+  input: Buffer,
+  reference: Buffer,
+  downsampleFactor = 8,
+): number {
+  if (!Number.isInteger(downsampleFactor) || downsampleFactor <= 0) {
+    return 0;
+  }
+
+  let inputSamples = blockAveragePcm16(input, downsampleFactor);
+  const referenceSamples = blockAveragePcm16(reference, downsampleFactor);
+  if (inputSamples.length > referenceSamples.length) {
+    inputSamples = inputSamples.subarray(0, referenceSamples.length);
+  }
+  if (inputSamples.length < 2) {
+    return 0;
+  }
+
+  let inputSum = 0;
+  for (let index = 0; index < inputSamples.length; index += 1) {
+    inputSum += inputSamples[index]!;
+  }
+  const inputMean = inputSum / inputSamples.length;
+  const centeredInput = new Float64Array(inputSamples.length);
+  let inputEnergy = 0;
+  for (let index = 0; index < inputSamples.length; index += 1) {
+    const centered = inputSamples[index]! - inputMean;
+    centeredInput[index] = centered;
+    inputEnergy += centered * centered;
+  }
+  if (inputEnergy === 0) {
+    return 0;
+  }
+
+  const prefixSum = new Float64Array(referenceSamples.length + 1);
+  const prefixSquareSum = new Float64Array(referenceSamples.length + 1);
+  for (let index = 0; index < referenceSamples.length; index += 1) {
+    const sample = referenceSamples[index]!;
+    prefixSum[index + 1] = prefixSum[index]! + sample;
+    prefixSquareSum[index + 1] = prefixSquareSum[index]! + sample * sample;
+  }
+
+  let best = 0;
+  const windowLength = inputSamples.length;
+  for (
+    let offset = 0;
+    offset + windowLength <= referenceSamples.length;
+    offset += 1
+  ) {
+    const referenceSum = prefixSum[offset + windowLength]! - prefixSum[offset]!;
+    const referenceSquareSum =
+      prefixSquareSum[offset + windowLength]! - prefixSquareSum[offset]!;
+    const referenceEnergy =
+      referenceSquareSum - (referenceSum * referenceSum) / windowLength;
+    if (referenceEnergy <= 0) {
+      continue;
+    }
+
+    let dotProduct = 0;
+    for (let index = 0; index < windowLength; index += 1) {
+      dotProduct += centeredInput[index]! * referenceSamples[offset + index]!;
+    }
+    const correlation =
+      Math.abs(dotProduct) / Math.sqrt(inputEnergy * referenceEnergy);
+    best = Math.max(best, Math.min(correlation, 1));
+  }
+  return best;
+}
+
+function blockAveragePcm16(
+  chunk: Buffer,
+  downsampleFactor: number,
+): Float64Array {
+  const sampleCount = Math.floor(chunk.length / 2);
+  const blockCount = Math.floor(sampleCount / downsampleFactor);
+  const result = new Float64Array(blockCount);
+  for (let block = 0; block < blockCount; block += 1) {
+    let sum = 0;
+    const firstSample = block * downsampleFactor;
+    for (let offset = 0; offset < downsampleFactor; offset += 1) {
+      sum += chunk.readInt16LE((firstSample + offset) * 2);
+    }
+    result[block] = sum / downsampleFactor;
+  }
+  return result;
+}
+
 /**
  * Detect speech activity in a chunk of little-endian signed 16-bit mono
  * PCM samples.
@@ -34,16 +148,5 @@ export function detectPcm16SpeechActivity(
   chunk: Buffer,
   threshold = DEFAULT_SPEECH_ENERGY_THRESHOLD,
 ): boolean {
-  const sampleCount = Math.floor(chunk.length / 2);
-  if (sampleCount === 0) {
-    return false;
-  }
-
-  let totalAmplitude = 0;
-  for (let i = 0; i < sampleCount; i++) {
-    totalAmplitude += Math.abs(chunk.readInt16LE(i * 2));
-  }
-  const avgAmplitude = totalAmplitude / sampleCount;
-
-  return avgAmplitude > threshold;
+  return pcm16MeanAmplitude(chunk) > threshold;
 }

package/src/stt/types.ts

@@ -263,6 +263,14 @@ export interface SttStreamServerPartialEvent {
    * provider does not surface confidence on interim chunks.
    */
   readonly confidence?: number;
+  /**
+   * All detected languages of this chunk in dominance order (first entry
+   * dominant), already normalized to lowercase base subtags (e.g. "en",
+   * "hi") by the emitting adapter; consumers must not re-normalize. Only
+   * providers with code-switching metadata (currently Deepgram nova-3
+   * `multi`) populate this. Absence means "unknown", never English.
+   */
+  readonly languages?: readonly string[];
 }
 
 /**
@@ -279,6 +287,14 @@ export interface SttStreamServerFinalEvent {
    * provider does not surface confidence on this chunk.
    */
   readonly confidence?: number;
+  /**
+   * All detected languages of this chunk in dominance order (first entry
+   * dominant), already normalized to lowercase base subtags (e.g. "en",
+   * "hi") by the emitting adapter; consumers must not re-normalize. Only
+   * providers with code-switching metadata (currently Deepgram nova-3
+   * `multi`) populate this. Absence means "unknown", never English.
+   */
+  readonly languages?: readonly string[];
   /**
    * True when this final is the flush response to
    * {@link StreamingTranscriber.finalizeUtterance} — i.e. it commits audio

package/src/tts/__tests__/provider-adapters.test.ts

@@ -549,6 +549,81 @@ describe("ElevenLabs TTS provider adapter", () => {
 
     const body = JSON.parse(capturedBody);
     expect(body.model_id).toBe("eleven_flash_v2_5");
+    expect("language_code" in body).toBe(false);
+  });
+
+  test("synthesizeStream sends language_code when the effective model supports enforcement", async () => {
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(streamOf(new Uint8Array([0x01])), { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createElevenLabsProvider();
+    await provider.synthesizeStream!(makeRequest({ language: "hi" }), () => {});
+
+    const body = JSON.parse(capturedBody);
+    expect(body.model_id).toBe("eleven_flash_v2_5");
+    expect(body.language_code).toBe("hi");
+  });
+
+  test("maps the tl subtag to the fil code the v2.5 roster spells Filipino with", async () => {
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(streamOf(new Uint8Array([0x01])), { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createElevenLabsProvider();
+    await provider.synthesizeStream!(makeRequest({ language: "tl" }), () => {});
+
+    const body = JSON.parse(capturedBody);
+    expect(body.model_id).toBe("eleven_flash_v2_5");
+    expect(body.language_code).toBe("fil");
+  });
+
+  test("omits language_code for a language the v2.5 models do not support", async () => {
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(streamOf(new Uint8Array([0x01])), { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createElevenLabsProvider();
+    await provider.synthesizeStream!(makeRequest({ language: "he" }), () => {});
+
+    const body = JSON.parse(capturedBody);
+    expect(body.model_id).toBe("eleven_flash_v2_5");
+    expect("language_code" in body).toBe(false);
+  });
+
+  test("omits language_code when the configured model does not support enforcement", async () => {
+    mockElevenLabsConfig.voiceModelId = "eleven_multilingual_v2";
+    seedTtsConfig();
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(streamOf(new Uint8Array([0x01])), { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createElevenLabsProvider();
+    await provider.synthesizeStream!(makeRequest({ language: "hi" }), () => {});
+
+    const body = JSON.parse(capturedBody);
+    expect(body.model_id).toBe("eleven_multilingual_v2");
+    expect("language_code" in body).toBe(false);
   });
 
   test("synthesizeStream respects configured voiceModelId over flash default", async () => {
@@ -1179,6 +1254,22 @@ describe("Deepgram TTS provider adapter", () => {
     expect(body.text).toBe("Hello world");
   });
 
+  test("request voiceId overrides the configured Deepgram model", async () => {
+    let capturedUrl = "";
+    globalThis.fetch = mock(async (input: RequestInfo | URL) => {
+      capturedUrl = typeof input === "string" ? input : input.toString();
+      return new Response(new Uint8Array([0x49, 0x44, 0x33]), { status: 200 });
+    }) as unknown as typeof globalThis.fetch;
+
+    const provider = createDeepgramProvider();
+    await provider.synthesize(makeRequest({ voiceId: "aura-2-thalia-en" }));
+    expect(capturedUrl).toContain("model=aura-2-thalia-en");
+
+    // A blank request voiceId falls back to the configured model.
+    await provider.synthesize(makeRequest({ voiceId: "  " }));
+    expect(capturedUrl).toContain("model=aura-asteria-en");
+  });
+
   test("uses linear16 encoding with container=none and sample_rate=16000 when outputFormat is pcm", async () => {
     const audioPayload = new Uint8Array([0x00, 0x01, 0x02]);
     let capturedUrl = "";
@@ -1685,6 +1776,42 @@ describe("xAI TTS provider adapter", () => {
     expect(body.voice_id).toBe("ara");
   });
 
+  test("request language overrides config language", async () => {
+    const audioPayload = new Uint8Array([0x49, 0x44, 0x33]);
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(audioPayload, { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createXaiProvider();
+    await provider.synthesize(makeRequest({ language: "hi" }));
+
+    const body = JSON.parse(capturedBody);
+    expect(body.language).toBe("hi");
+  });
+
+  test("keeps configured language for a request code outside the code-switching roster", async () => {
+    const audioPayload = new Uint8Array([0x49, 0x44, 0x33]);
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(audioPayload, { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createXaiProvider();
+    await provider.synthesize(makeRequest({ language: "th" }));
+
+    const body = JSON.parse(capturedBody);
+    expect(body.language).toBe("auto");
+  });
+
   test("wav config format produces codec=wav without bit_rate", async () => {
     mockXaiConfig.format = "wav";
     seedTtsConfig();
@@ -1902,6 +2029,26 @@ describe("xAI TTS provider adapter", () => {
       expect(capturedSocketOptions().url).toContain("sample_rate=22050");
     });
 
+    test("request language overrides config language in the stream URL", async () => {
+      const provider = createXaiProvider();
+      await provider.synthesizeStream!(
+        makeRequest({ language: "es" }),
+        () => {},
+      );
+
+      expect(capturedSocketOptions().url).toContain("language=es");
+    });
+
+    test("keeps configured language in the stream URL for an out-of-roster request code", async () => {
+      const provider = createXaiProvider();
+      await provider.synthesizeStream!(
+        makeRequest({ language: "th" }),
+        () => {},
+      );
+
+      expect(capturedSocketOptions().url).toContain("language=auto");
+    });
+
     test("mp3 requests carry codec=mp3 and bit_rate and resolve audio/mpeg", async () => {
       const provider = createXaiProvider();
       const result = await provider.synthesizeStream!(makeRequest(), () => {});