@ssml-builder-js/azure-tts-client 2.11.0 → 2.13.0

package/src/safe.ts

@@ -0,0 +1,72 @@
+import { validateAzureSsml, type AzureValidationOptions, type SsmlDiagnostic } from "@ssml-builder-js/ssml-core";
+import { AzureTtsError, createSpeechSdkError } from "./errors.ts";
+import type { AzureTtsClient } from "./client.ts";
+import type { SsmlSynthesisResult } from "./types.ts";
+
+export interface SsmlValidationError {
+  readonly kind: "validation";
+  readonly message: string;
+  readonly diagnostics: readonly SsmlDiagnostic[];
+}
+
+export type Result<T, E> =
+  | { readonly ok: true; readonly success: true; readonly status: "success"; readonly value: T }
+  | {
+      readonly ok: false;
+      readonly success: false;
+      readonly status: "validation-error" | "azure-api-error";
+      readonly error: E;
+    };
+
+export type SynthesisResult<T, E> = Result<T, E>;
+
+export type Success<T> = Extract<Result<T, never>, { readonly ok: true }>;
+export type ValidationErrorResult = Extract<
+  Result<never, SsmlValidationError>,
+  { readonly status: "validation-error" }
+>;
+export type AzureApiErrorResult = Extract<Result<never, AzureTtsError>, { readonly status: "azure-api-error" }>;
+
+export type SsmlSynthesisSafeResult =
+  | Result<SsmlSynthesisResult, never>
+  | Result<never, SsmlValidationError>
+  | Result<never, AzureTtsError>;
+
+export interface SynthesizeSsmlSafeOptions extends AzureValidationOptions {
+  /** Optional nested form for callers that want to keep validation settings grouped. */
+  validation?: AzureValidationOptions;
+}
+
+interface SynthesisClient {
+  synthesizeSsml(ssml: string): Promise<SsmlSynthesisResult>;
+}
+
+/** Validates SSML before invoking Azure and converts validation/API failures to one result shape. */
+export async function synthesizeSsmlSafe(
+  client: Pick<AzureTtsClient, "synthesizeSsml"> | SynthesisClient,
+  ssml: string,
+  options: SynthesizeSsmlSafeOptions = {},
+): Promise<SsmlSynthesisSafeResult> {
+  const validationOptions = options.validation ?? options;
+  const diagnostics = await Promise.resolve(validateAzureSsml(ssml, validationOptions));
+  const errors = diagnostics.filter((diagnostic) => diagnostic.severity === "error");
+  if (errors.length > 0) {
+    return {
+      ok: false,
+      success: false,
+      status: "validation-error",
+      error: {
+        kind: "validation",
+        message: "SSML validation failed; the Azure Speech API was not called.",
+        diagnostics: errors,
+      },
+    };
+  }
+
+  try {
+    return { ok: true, success: true, status: "success", value: await client.synthesizeSsml(ssml) };
+  } catch (error) {
+    const azureError = error instanceof AzureTtsError ? error : createSpeechSdkError(error);
+    return { ok: false, success: false, status: "azure-api-error", error: azureError };
+  }
+}

package/src/synthesis.ts

@@ -1,7 +1,7 @@
 import * as SpeechSDK from "microsoft-cognitiveservices-speech-sdk";
 import { createSpeechSdkError } from "./errors.ts";
 import { createSpeechConfig } from "./speechConfig.ts";
-import type { TtsConfig } from "./types.ts";
+import type { SsmlSynthesisChunk, SsmlSynthesisResult, TtsConfig } from "./types.ts";
 
 function closeSpeechResources(speechConfig: SpeechSDK.SpeechConfig, synthesizer: SpeechSDK.SpeechSynthesizer): void {
   try {
@@ -13,7 +13,9 @@ function closeSpeechResources(speechConfig: SpeechSDK.SpeechConfig, synthesizer:
   } catch {}
 }
 
-export async function synthesizeSpeech(ssml: string, config: TtsConfig): Promise<ArrayBuffer> {
+const ticksToMilliseconds = (ticks: number): number => Math.max(0, ticks) / 10_000;
+
+export async function synthesizeSsml(ssml: string, config: TtsConfig): Promise<SsmlSynthesisResult> {
   if (config.signal?.aborted) {
     throw createSpeechSdkError("Speech synthesis was cancelled.");
   }
@@ -21,7 +23,7 @@ export async function synthesizeSpeech(ssml: string, config: TtsConfig): Promise
   const speechConfig = createSpeechConfig(config);
   const synthesizer = new SpeechSDK.SpeechSynthesizer(speechConfig, null);
 
-  return await new Promise<ArrayBuffer>((resolve, reject) => {
+  return await new Promise<SsmlSynthesisResult>((resolve, reject) => {
     let resourcesClosed = false;
     let settled = false;
     let timeout: ReturnType<typeof setTimeout> | undefined;
@@ -43,6 +45,23 @@ export async function synthesizeSpeech(ssml: string, config: TtsConfig): Promise
       reject(createSpeechSdkError(error));
     };
 
+    const boundaries: SsmlSynthesisResult["boundaries"] = [];
+    const visemes: SsmlSynthesisResult["visemes"] = [];
+    const bookmarks: SsmlSynthesisResult["bookmarks"] = [];
+    synthesizer.wordBoundary = (_sender, event) => {
+      boundaries.push({
+        text: event.text,
+        audioOffsetMs: ticksToMilliseconds(event.audioOffset),
+        durationMs: ticksToMilliseconds(event.duration),
+      });
+    };
+    synthesizer.visemeReceived = (_sender, event) => {
+      visemes.push({ visemeId: event.visemeId, audioOffsetMs: ticksToMilliseconds(event.audioOffset) });
+    };
+    synthesizer.bookmarkReached = (_sender, event) => {
+      bookmarks.push({ name: event.text, audioOffsetMs: ticksToMilliseconds(event.audioOffset) });
+    };
+
     const cb = (result: SpeechSDK.SpeechSynthesisResult) => {
       if (settled) return;
       const { reason, errorDetails } = result;
@@ -54,7 +73,33 @@ export async function synthesizeSpeech(ssml: string, config: TtsConfig): Promise
       settled = true;
       cleanup();
       closeResources();
-      resolve(result.audioData);
+      const eventDurationMs = Math.max(
+        0,
+        ...(boundaries ?? []).map((boundary) => boundary.audioOffsetMs + boundary.durationMs),
+        ...(visemes ?? []).map((viseme) => viseme.audioOffsetMs),
+        ...(bookmarks ?? []).map((bookmark) => bookmark.audioOffsetMs),
+      );
+      const durationMs = result.audioDuration ? ticksToMilliseconds(result.audioDuration) : eventDurationMs;
+      const requestId = (result as SpeechSDK.SpeechSynthesisResult & { resultId?: string }).resultId;
+      const addSourceMetadata = <T extends { audioOffsetMs: number }>(event: T): T => ({
+        ...event,
+        ...(config.sourceTextRange ? { textRange: { ...config.sourceTextRange } } : {}),
+        ...(requestId ? { requestId } : {}),
+      });
+      const sourceBoundaries = boundaries.map((boundary) => addSourceMetadata(boundary));
+      const sourceVisemes = visemes.map((viseme) => addSourceMetadata(viseme));
+      const sourceBookmarks = bookmarks.map((bookmark) => addSourceMetadata(bookmark));
+      resolve({
+        audioData: result.audioData,
+        durationMs,
+        ...(config.sourceTextRange ? { textRange: { ...config.sourceTextRange } } : {}),
+        ...(requestId ? { requestId } : {}),
+        ...(sourceBoundaries.length > 0
+          ? { boundaries: sourceBoundaries, wordBoundary: sourceBoundaries, wordBoundaries: sourceBoundaries }
+          : {}),
+        ...(sourceVisemes.length > 0 ? { visemes: sourceVisemes } : {}),
+        ...(sourceBookmarks.length > 0 ? { bookmarks: sourceBookmarks } : {}),
+      });
     };
 
     try {
@@ -74,3 +119,93 @@ export async function synthesizeSpeech(ssml: string, config: TtsConfig): Promise
     }
   });
 }
+
+/** Synthesizes chunks sequentially, annotates synchronization events, and merges the results. */
+export async function synthesizeSsmlChunks(
+  chunks: readonly (SsmlSynthesisChunk | string)[],
+  config: TtsConfig,
+): Promise<SsmlSynthesisResult> {
+  const results: SsmlSynthesisResult[] = [];
+  const totalChunks = chunks.length;
+  for (const [index, chunk] of chunks.entries()) {
+    const input = typeof chunk === "string" ? { ssml: chunk } : chunk;
+    const result = await synthesizeSsml(input.ssml, {
+      ...config,
+      ...(input.originalTextRange ? { sourceTextRange: input.originalTextRange } : {}),
+      onProgress: undefined,
+    });
+    results.push(result);
+    config.onProgress?.({
+      currentChunk: index + 1,
+      totalChunks,
+      percent: totalChunks === 0 ? 100 : Math.round(((index + 1) / totalChunks) * 100),
+    });
+  }
+  return mergeSynthesisResults(results);
+}
+
+/** Concatenates audio buffers and shifts all synchronization events by prior chunk durations. */
+export function mergeSynthesisResults(results: readonly SsmlSynthesisResult[]): SsmlSynthesisResult {
+  const audioLength = results.reduce((total, result) => total + result.audioData.byteLength, 0);
+  const audioData = new Uint8Array(audioLength);
+  const boundaries: NonNullable<SsmlSynthesisResult["boundaries"]> = [];
+  const visemes: NonNullable<SsmlSynthesisResult["visemes"]> = [];
+  const bookmarks: NonNullable<SsmlSynthesisResult["bookmarks"]> = [];
+  let byteOffset = 0;
+  let durationOffset = 0;
+
+  for (const result of results) {
+    audioData.set(new Uint8Array(result.audioData), byteOffset);
+    byteOffset += result.audioData.byteLength;
+    const chunkBoundaries =
+      result.boundaries && result.boundaries.length > 0
+        ? result.boundaries
+        : (result.wordBoundary ?? result.wordBoundaries ?? []);
+    for (const boundary of chunkBoundaries) {
+      const textRange = boundary.textRange ?? result.textRange;
+      const requestId = boundary.requestId ?? result.requestId;
+      boundaries.push({
+        ...boundary,
+        audioOffsetMs: boundary.audioOffsetMs + durationOffset,
+        ...(textRange ? { textRange: { ...textRange } } : {}),
+        ...(requestId ? { requestId } : {}),
+      });
+    }
+    for (const viseme of result.visemes ?? []) {
+      const textRange = viseme.textRange ?? result.textRange;
+      const requestId = viseme.requestId ?? result.requestId;
+      visemes.push({
+        ...viseme,
+        audioOffsetMs: viseme.audioOffsetMs + durationOffset,
+        ...(textRange ? { textRange: { ...textRange } } : {}),
+        ...(requestId ? { requestId } : {}),
+      });
+    }
+    for (const bookmark of result.bookmarks ?? []) {
+      const textRange = bookmark.textRange ?? result.textRange;
+      const requestId = bookmark.requestId ?? result.requestId;
+      bookmarks.push({
+        ...bookmark,
+        audioOffsetMs: bookmark.audioOffsetMs + durationOffset,
+        ...(textRange ? { textRange: { ...textRange } } : {}),
+        ...(requestId ? { requestId } : {}),
+      });
+    }
+    durationOffset += Math.max(0, result.durationMs);
+  }
+
+  return {
+    audioData: audioData.buffer,
+    durationMs: durationOffset,
+    ...(boundaries.length > 0 ? { boundaries, wordBoundary: boundaries, wordBoundaries: boundaries } : {}),
+    ...(visemes.length > 0 ? { visemes } : {}),
+    ...(bookmarks.length > 0 ? { bookmarks } : {}),
+    ...(results.length === 1 && results[0]?.requestId ? { requestId: results[0].requestId } : {}),
+    ...(results.length === 1 && results[0]?.textRange ? { textRange: { ...results[0].textRange } } : {}),
+  };
+}
+
+/** Backward-compatible audio-only synthesis helper. */
+export async function synthesizeSpeech(ssml: string, config: TtsConfig): Promise<ArrayBuffer> {
+  return (await synthesizeSsml(ssml, config)).audioData;
+}

package/src/types.ts

@@ -5,6 +5,64 @@ export interface TtsConfig {
   subscriptionKey: string;
   region: string;
   outputFormat?: string;
+  /** Original plain-text range represented by this synthesis request. */
+  sourceTextRange?: { start: number; end: number };
+  /** Reports completion of a chunk when using synthesizeSsmlChunks. */
+  onProgress?: (event: { currentChunk: number; totalChunks: number; percent: number }) => void;
+}
+
+export interface SsmlSynthesisBoundary {
+  text: string;
+  audioOffsetMs: number;
+  durationMs: number;
+  textRange?: { start: number; end: number };
+  requestId?: string;
+}
+
+export interface SsmlSynthesisViseme {
+  visemeId: number;
+  audioOffsetMs: number;
+  textRange?: { start: number; end: number };
+  requestId?: string;
+}
+
+export interface SsmlSynthesisBookmark {
+  name: string;
+  audioOffsetMs: number;
+  textRange?: { start: number; end: number };
+  requestId?: string;
+}
+
+/** Audio and Azure Speech synchronization events emitted for one SSML request. */
+export interface SsmlSynthesisResult {
+  audioData: ArrayBuffer;
+  durationMs: number;
+  boundaries?: SsmlSynthesisBoundary[];
+  /** Alias matching the Azure Speech event name. */
+  wordBoundary?: SsmlSynthesisBoundary[];
+  /** Alias for consumers that use Azure's word-boundary terminology. */
+  wordBoundaries?: SsmlSynthesisBoundary[];
+  visemes?: SsmlSynthesisViseme[];
+  bookmarks?: SsmlSynthesisBookmark[];
+  /** Request identifier returned by Azure Speech, when available. */
+  requestId?: string;
+  /** Original plain-text range represented by the result. */
+  textRange?: { start: number; end: number };
+}
+
+export interface SsmlSynthesisChunk {
+  ssml: string;
+  originalTextRange?: { start: number; end: number };
+}
+
+export interface SynthesizeChunksOptions {
+  onProgress?: (event: SynthesisProgressEvent) => void;
+}
+
+export interface SynthesisProgressEvent {
+  currentChunk: number;
+  totalChunks: number;
+  percent: number;
 }
 
 export interface AzureTtsLogger {
@@ -22,4 +80,5 @@ export interface AzureTtsClientOptions {
   endpoint?: string;
   outputFormat?: string;
   logger?: AzureTtsLogger;
+  onProgress?: (event: SynthesisProgressEvent) => void;
 }

package/test/synthesis.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import test, { type TestContext } from "node:test";
 import * as SpeechSDK from "microsoft-cognitiveservices-speech-sdk";
-import { synthesizeSpeech } from "../src/index.ts";
+import { synthesizeSpeech, synthesizeSsml } from "../src/index.ts";
 
 const endpoint = "https://speech.example.test/cognitiveservices/v1";
 const subscriptionKey = "subscription-key";
@@ -51,6 +51,45 @@ test("synthesizeSpeech aborts the SDK request and settles its promise", async (t
   assert.equal(closeCount, 1);
 });
 
+test("synthesizeSsml returns word boundaries, visemes, bookmarks, and duration", async (t) => {
+  const originalFromEndpoint = SpeechSDK.SpeechConfig.fromEndpoint;
+  t.mock.method(SpeechSDK.SpeechConfig, "fromEndpoint", (speechEndpoint, key) =>
+    originalFromEndpoint(speechEndpoint, String(key)),
+  );
+  t.mock.method(
+    SpeechSDK.SpeechSynthesizer.prototype,
+    "speakSsmlAsync",
+    function (this: SpeechSDK.SpeechSynthesizer, _ssml, callback) {
+      this.wordBoundary?.(this, {
+        text: "Hello",
+        audioOffset: 1_000_000,
+        duration: 250_000,
+      } as SpeechSDK.SpeechSynthesisWordBoundaryEventArgs);
+      this.visemeReceived?.(this, { visemeId: 4, audioOffset: 2_000_000 } as SpeechSDK.SpeechSynthesisVisemeEventArgs);
+      this.bookmarkReached?.(this, {
+        text: "chapter-1",
+        audioOffset: 3_000_000,
+      } as SpeechSDK.SpeechSynthesisBookmarkEventArgs);
+      callback?.({
+        audioData: new ArrayBuffer(2),
+        audioDuration: 4_000_000,
+        errorDetails: "",
+        reason: SpeechSDK.ResultReason.SynthesizingAudioCompleted,
+      } as SpeechSDK.SpeechSynthesisResult);
+    },
+  );
+  t.mock.method(SpeechSDK.SpeechSynthesizer.prototype, "close", () => undefined);
+
+  const result = await synthesizeSsml("<speak>Hello</speak>", { endpoint, subscriptionKey, region });
+
+  assert.deepEqual(result.boundaries, [{ text: "Hello", audioOffsetMs: 100, durationMs: 25 }]);
+  assert.deepEqual(result.wordBoundary, result.boundaries);
+  assert.deepEqual(result.wordBoundaries, result.boundaries);
+  assert.deepEqual(result.visemes, [{ visemeId: 4, audioOffsetMs: 200 }]);
+  assert.deepEqual(result.bookmarks, [{ name: "chapter-1", audioOffsetMs: 300 }]);
+  assert.equal(result.durationMs, 400);
+});
+
 test("synthesizeSpeech aborts the SDK request on timeout and settles its promise", async (t) => {
   let closeCount = 0;
   let resultCallback: ((result: SpeechSDK.SpeechSynthesisResult) => void) | undefined;

package/test/v213-pipeline.test.ts

@@ -0,0 +1,68 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { mergeSynthesisResults, synthesizeSsmlSafe } from "../src/index.ts";
+
+const audio = (values: number[]): ArrayBuffer => Uint8Array.from(values).buffer;
+
+test("mergeSynthesisResults concatenates audio and offsets synchronization events", () => {
+  const result = mergeSynthesisResults([
+    {
+      audioData: audio([1, 2]),
+      durationMs: 100,
+      boundaries: [{ text: "one", audioOffsetMs: 20, durationMs: 30, textRange: { start: 0, end: 3 }, requestId: "a" }],
+      visemes: [{ visemeId: 1, audioOffsetMs: 40 }],
+      bookmarks: [{ name: "first", audioOffsetMs: 50 }],
+    },
+    {
+      audioData: audio([3, 4, 5]),
+      durationMs: 250,
+      boundaries: [{ text: "two", audioOffsetMs: 10, durationMs: 20, textRange: { start: 3, end: 6 }, requestId: "b" }],
+      visemes: [{ visemeId: 2, audioOffsetMs: 15 }],
+      bookmarks: [{ name: "second", audioOffsetMs: 25 }],
+    },
+  ]);
+
+  assert.deepEqual([...new Uint8Array(result.audioData)], [1, 2, 3, 4, 5]);
+  assert.equal(result.durationMs, 350);
+  assert.deepEqual(
+    result.boundaries?.map(({ audioOffsetMs }) => audioOffsetMs),
+    [20, 110],
+  );
+  assert.deepEqual(
+    result.visemes?.map(({ audioOffsetMs }) => audioOffsetMs),
+    [40, 115],
+  );
+  assert.deepEqual(
+    result.bookmarks?.map(({ audioOffsetMs }) => audioOffsetMs),
+    [50, 125],
+  );
+  assert.deepEqual(result.boundaries?.[1]?.textRange, { start: 3, end: 6 });
+});
+
+test("synthesizeSsmlSafe blocks invalid SSML without calling the client", async () => {
+  let calls = 0;
+  const result = await synthesizeSsmlSafe(
+    {
+      synthesizeSsml: async () => {
+        calls += 1;
+        return { audioData: new ArrayBuffer(0), durationMs: 0 };
+      },
+    },
+    "<speak>",
+  );
+
+  assert.equal(result.ok, false);
+  assert.equal(result.status, "validation-error");
+  assert.equal(calls, 0);
+});
+
+test("synthesizeSsmlSafe returns a successful result for valid SSML", async () => {
+  const expected = { audioData: audio([1]), durationMs: 10 };
+  const result = await synthesizeSsmlSafe(
+    { synthesizeSsml: async () => expected },
+    '<speak version="1.0" xml:lang="en-US"><voice name="en-US-JennyNeural">Hello</voice></speak>',
+  );
+
+  assert.equal(result.ok, true);
+  if (result.ok) assert.equal(result.value, expected);
+});