@ssml-builder-js/azure-tts-client 2.14.0 → 2.16.0

package/src/types.ts

@@ -1,4 +1,5 @@
-import type { SsmlTextRange } from "@ssml-builder-js/ssml-core";
+import type { SsmlSourceMarker, SsmlSourceTextSegment, SsmlTextRange } from "@ssml-builder-js/ssml-core";
+import type { AzureTtsOutputFormat } from "./outputFormats.ts";
 
 export interface TtsConfig {
   signal?: AbortSignal;
@@ -14,6 +15,29 @@ export interface TtsConfig {
   /** Metadata used to map synchronization events back to the source document. */
   chunkIndex?: number;
   sourceNodePath?: string[];
+  /** Exact source text segments used to map individual Azure events. */
+  sourceTextSegments?: SsmlSourceTextSegment[];
+  sourceMarkers?: SsmlSourceMarker[];
+  concurrency?: number;
+  retryOptions?: RetryOptions;
+}
+
+export type MappingStatus = "exact" | "fallback" | "unmapped";
+
+export interface AudioSpecification {
+  format: string;
+  mimeType: string;
+  codec: "pcm" | "mp3" | "opus" | "silk" | "unknown";
+  sampleRate: number;
+  channels: number;
+  bitrate?: number;
+  isCompressed: boolean;
+}
+
+export interface RetryOptions {
+  maxRetries: number;
+  initialDelayMs: number;
+  maxDelayMs: number;
 }
 
 export type SynthesisChunkStatus = "pending" | "synthesizing" | "success" | "failed";
@@ -32,6 +56,7 @@ export interface SsmlSynthesisBoundary {
   /** Audio offset within the originating chunk before merge. */
   chunkAudioOffsetMs?: number;
   requestId?: string;
+  mappingStatus: MappingStatus;
 }
 
 export interface SsmlSynthesisViseme {
@@ -43,6 +68,7 @@ export interface SsmlSynthesisViseme {
   originalTextRange?: SsmlTextRange;
   chunkAudioOffsetMs?: number;
   requestId?: string;
+  mappingStatus: MappingStatus;
 }
 
 export interface SsmlSynthesisBookmark {
@@ -54,6 +80,7 @@ export interface SsmlSynthesisBookmark {
   originalTextRange?: SsmlTextRange;
   chunkAudioOffsetMs?: number;
   requestId?: string;
+  mappingStatus: MappingStatus;
 }
 
 /** Audio and Azure Speech synchronization events emitted for one SSML request. */
@@ -71,16 +98,31 @@ export interface SsmlSynthesisResult {
   requestId?: string;
   /** Original plain-text range represented by the result. */
   textRange?: { start: number; end: number };
+  /** MIME type of a result produced by an explicit merge operation. */
+  mimeType?: string;
+  audioSpec?: AudioSpecification;
+}
+
+export interface MergedSynthesisResult extends SsmlSynthesisResult {
+  mimeType: string;
 }
 
 export interface SsmlSynthesisChunk {
   ssml: string;
   originalTextRange?: { start: number; end: number };
   sourceNodePath?: string[];
+  sourceTextSegments?: SsmlSourceTextSegment[];
+  sourceMarkers?: SsmlSourceMarker[];
 }
 
 export interface SynthesizeChunksOptions {
   onProgress?: (event: SynthesisProgressEvent) => void;
+  outputFormat?: AzureTtsOutputFormat | string;
+  signal?: AbortSignal;
+  timeoutMs?: number;
+  sourceNodePath?: string[];
+  concurrency?: number;
+  retryOptions?: RetryOptions;
 }
 
 export interface SynthesisProgressEvent {
@@ -93,6 +135,9 @@ export interface SynthesisProgressEvent {
   status: SynthesisChunkStatus;
   durationMs: number;
   error?: unknown;
+  retryAttempt?: number;
+  nextRetryDelayMs?: number;
+  isRetrying?: boolean;
 }
 
 export interface AzureTtsLogger {
@@ -111,4 +156,6 @@ export interface AzureTtsClientOptions {
   outputFormat?: string;
   logger?: AzureTtsLogger;
   onProgress?: (event: SynthesisProgressEvent) => void;
+  concurrency?: number;
+  retryOptions?: RetryOptions;
 }

package/test/synthesis.test.ts

@@ -90,6 +90,70 @@ test("synthesizeSsml returns word boundaries, visemes, bookmarks, and duration",
   assert.equal(result.durationMs, 400);
 });
 
+test("synthesizeSsml maps each synchronization event to its source range and node", async (t) => {
+  const originalFromEndpoint = SpeechSDK.SpeechConfig.fromEndpoint;
+  t.mock.method(SpeechSDK.SpeechConfig, "fromEndpoint", (speechEndpoint, key) =>
+    originalFromEndpoint(speechEndpoint, String(key)),
+  );
+  t.mock.method(
+    SpeechSDK.SpeechSynthesizer.prototype,
+    "speakSsmlAsync",
+    function (this: SpeechSDK.SpeechSynthesizer, _ssml, callback) {
+      this.wordBoundary?.(this, {
+        text: "Hello",
+        textOffset: 7,
+        audioOffset: 0,
+        duration: 100_000,
+      } as SpeechSDK.SpeechSynthesisWordBoundaryEventArgs);
+      this.visemeReceived?.(this, {
+        visemeId: 3,
+        textOffset: 7,
+        audioOffset: 100_000,
+      } as SpeechSDK.SpeechSynthesisVisemeEventArgs);
+      this.bookmarkReached?.(this, {
+        text: "chapter",
+        audioOffset: 200_000,
+      } as SpeechSDK.SpeechSynthesisBookmarkEventArgs);
+      callback?.({
+        audioData: new ArrayBuffer(1),
+        audioDuration: 300_000,
+        errorDetails: "",
+        reason: SpeechSDK.ResultReason.SynthesizingAudioCompleted,
+      } as SpeechSDK.SpeechSynthesisResult);
+    },
+  );
+  t.mock.method(SpeechSDK.SpeechSynthesizer.prototype, "close", () => undefined);
+
+  const result = await synthesizeSsml(
+    '<speak version="1.0" xml:lang="en-US"><voice name="en-US-JennyNeural">prefix <prosody rate="slow">Hello</prosody><bookmark mark="chapter"/></voice></speak>',
+    {
+      endpoint,
+      subscriptionKey,
+      region,
+      sourceTextRange: { start: 100, end: 112 },
+      sourceTextSegments: [
+        { text: "prefix ", range: { start: 100, end: 107 }, sourceNodePath: ["speak", "voice[0]"] },
+        { text: "Hello", range: { start: 107, end: 112 }, sourceNodePath: ["speak", "voice[0]", "prosody[1]"] },
+      ],
+      sourceMarkers: [
+        {
+          kind: "bookmark",
+          name: "chapter",
+          originalTextRange: { start: 112, end: 112 },
+          sourceNodePath: ["speak", "voice[0]", "bookmark[2]"],
+        },
+      ],
+    },
+  );
+
+  assert.deepEqual(result.boundaries?.[0]?.originalTextRange, { start: 107, end: 112 });
+  assert.deepEqual(result.boundaries?.[0]?.sourceNodePath, ["speak", "voice[0]", "prosody[1]"]);
+  assert.deepEqual(result.visemes?.[0]?.originalTextRange, { start: 107, end: 107 });
+  assert.deepEqual(result.visemes?.[0]?.sourceNodePath, ["speak", "voice[0]", "prosody[1]"]);
+  assert.deepEqual(result.bookmarks?.[0]?.originalTextRange, { start: 112, end: 112 });
+  assert.deepEqual(result.bookmarks?.[0]?.sourceNodePath, ["speak", "voice[0]", "bookmark[2]"]);
+});
+
 test("synthesizeSpeech aborts the SDK request on timeout and settles its promise", async (t) => {
   let closeCount = 0;
   let resultCallback: ((result: SpeechSDK.SpeechSynthesisResult) => void) | undefined;

package/test/v213-pipeline.test.ts

@@ -5,22 +5,29 @@ import { mergeSynthesisResults, synthesizeSsmlSafe } from "../src/index.ts";
 const audio = (values: number[]): ArrayBuffer => Uint8Array.from(values).buffer;
 
 test("mergeSynthesisResults concatenates audio and offsets synchronization events", () => {
-  const result = mergeSynthesisResults([
-    {
-      audioData: audio([1, 2]),
-      durationMs: 100,
-      boundaries: [{ text: "one", audioOffsetMs: 20, durationMs: 30, textRange: { start: 0, end: 3 }, requestId: "a" }],
-      visemes: [{ visemeId: 1, audioOffsetMs: 40 }],
-      bookmarks: [{ name: "first", audioOffsetMs: 50 }],
-    },
-    {
-      audioData: audio([3, 4, 5]),
-      durationMs: 250,
-      boundaries: [{ text: "two", audioOffsetMs: 10, durationMs: 20, textRange: { start: 3, end: 6 }, requestId: "b" }],
-      visemes: [{ visemeId: 2, audioOffsetMs: 15 }],
-      bookmarks: [{ name: "second", audioOffsetMs: 25 }],
-    },
-  ]);
+  const result = mergeSynthesisResults(
+    [
+      {
+        audioData: audio([1, 2]),
+        durationMs: 100,
+        boundaries: [
+          { text: "one", audioOffsetMs: 20, durationMs: 30, textRange: { start: 0, end: 3 }, requestId: "a" },
+        ],
+        visemes: [{ visemeId: 1, audioOffsetMs: 40 }],
+        bookmarks: [{ name: "first", audioOffsetMs: 50 }],
+      },
+      {
+        audioData: audio([3, 4, 5]),
+        durationMs: 250,
+        boundaries: [
+          { text: "two", audioOffsetMs: 10, durationMs: 20, textRange: { start: 3, end: 6 }, requestId: "b" },
+        ],
+        visemes: [{ visemeId: 2, audioOffsetMs: 15 }],
+        bookmarks: [{ name: "second", audioOffsetMs: 25 }],
+      },
+    ],
+    { format: "audio-16khz-128kbitrate-mono-mp3" },
+  );
 
   assert.deepEqual([...new Uint8Array(result.audioData)], [1, 2, 3, 4, 5]);
   assert.equal(result.durationMs, 350);

package/test/v214-pipeline.test.ts

@@ -31,7 +31,7 @@ const validSsml = (text: string) =>
   `<speak version="1.0" xml:lang="en-US"><voice name="en-US-JennyNeural">${text}</voice></speak>`;
 
 test("mergeAudioBuffers rebuilds one valid WAV header", () => {
-  const merged = mergeAudioBuffers([wav([1, 2]), wav([3, 4, 5])], "riff-16khz-16bit-mono-pcm");
+  const merged = mergeAudioBuffers([wav([1, 2]), wav([3, 4, 5])], { format: "riff-16khz-16bit-mono-pcm" });
   const bytes = new Uint8Array(merged);
   const view = new DataView(merged);
   assert.equal(new TextDecoder().decode(bytes.slice(0, 4)), "RIFF");
@@ -51,14 +51,18 @@ test("mergeAudioBuffers removes per-buffer ID3 tags from MP3 streams", () => {
   second.set(tag);
   second[tag.length] = 3;
   assert.deepEqual(
-    [...new Uint8Array(mergeAudioBuffers([first.buffer, second.buffer], "audio-16khz-128kbitrate-mono-mp3"))],
+    [
+      ...new Uint8Array(
+        mergeAudioBuffers([first.buffer, second.buffer], { format: "audio-16khz-128kbitrate-mono-mp3" }),
+      ),
+    ],
     [1, 2, 3],
   );
 });
 
 test("mergeAudioBuffers rejects container formats that require remultiplexing", () => {
   assert.throws(
-    () => mergeAudioBuffers([new ArrayBuffer(1)], "webm-24khz-16bit-mono-opus"),
+    () => mergeAudioBuffers([new ArrayBuffer(1)], { format: "webm-24khz-16bit-mono-opus" }),
     (error: unknown) => error instanceof UnsupportedMergeFormatError,
   );
 });

package/test/v215-pipeline.test.ts

@@ -0,0 +1,104 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+  AzureTtsError,
+  SynthesisCancelledError,
+  SynthesisTimeoutError,
+  UnsupportedMergeFormatError,
+  mergeSynthesisResults,
+  synthesizeSsmlChunksSafe,
+  synthesizeSsmlSafe,
+} from "../src/index.ts";
+
+const audio = (values: number[]): ArrayBuffer => Uint8Array.from(values).buffer;
+const validSsml = (text: string) =>
+  `<speak version="1.0" xml:lang="en-US"><voice name="en-US-JennyNeural">${text}</voice></speak>`;
+
+test("merge APIs require a format at runtime and expose MIME metadata", async () => {
+  assert.throws(
+    () => mergeSynthesisResults([{ audioData: audio([1]), durationMs: 1 }], undefined as never),
+    (error: unknown) => error instanceof UnsupportedMergeFormatError && error.kind === "unsupported-format-error",
+  );
+  const merged = mergeSynthesisResults([{ audioData: audio([1]), durationMs: 1 }], {
+    format: "webm-24khz-16bit-mono-opus",
+    customMerger: (buffers) => buffers[0] ?? new ArrayBuffer(0),
+  });
+  assert.equal((await merged).mimeType, "audio/webm");
+});
+
+test("safe synthesis returns each discriminated error kind", async () => {
+  const validation = await synthesizeSsmlSafe(
+    { synthesizeSsml: async () => ({ audioData: new ArrayBuffer(0), durationMs: 0 }) },
+    "<speak>",
+  );
+  assert.equal(validation.ok, false);
+  if (!validation.ok) assert.equal(validation.error.kind, "validation-error");
+
+  const azure = await synthesizeSsmlSafe(
+    {
+      synthesizeSsml: async () => {
+        throw new AzureTtsError(429, "Too Many Requests", "{}", "req");
+      },
+    },
+    validSsml("hello"),
+  );
+  assert.equal(azure.ok, false);
+  if (!azure.ok) {
+    assert.equal(azure.error.kind, "azure-api-error");
+    assert.equal(azure.error.status, 429);
+  }
+
+  for (const error of [new SynthesisTimeoutError("timed out"), new SynthesisCancelledError()]) {
+    const result = await synthesizeSsmlChunksSafe(
+      {
+        synthesizeSsml: async () => {
+          throw error;
+        },
+      },
+      [validSsml("hello")],
+      { outputFormat: "audio-16khz-128kbitrate-mono-mp3" },
+    );
+    assert.equal(result.ok, false);
+    if (!result.ok) assert.equal(result.error.kind, error.kind);
+  }
+});
+
+test("chunk safe synthesis propagates output, cancellation, timeout, and source path settings", async () => {
+  const controller = new AbortController();
+  const calls: Array<{ options?: Record<string, unknown> }> = [];
+  let validatorSignal: AbortSignal | undefined;
+  let validatorPath: readonly string[] | undefined;
+  const result = await synthesizeSsmlChunksSafe(
+    {
+      synthesizeSsml: async (_ssml, options) => {
+        calls.push({ options: options as Record<string, unknown> });
+        return { audioData: new ArrayBuffer(0), durationMs: 0 };
+      },
+    },
+    [
+      {
+        ssml: `${validSsml("hello").replace("</voice>", '<audio src="https://example.test/a.mp3"/></voice>')}`,
+        originalTextRange: { start: 10, end: 15 },
+      },
+    ],
+    {
+      allowExternalAudio: true,
+      outputFormat: "audio-16khz-128kbitrate-mono-mp3",
+      signal: controller.signal,
+      timeoutMs: 1234,
+      sourceNodePath: ["speak", "voice[0]"],
+      urlValidator: async (_url, context, signal) => {
+        validatorPath = context.sourceNodePath;
+        validatorSignal = signal;
+        return true;
+      },
+    },
+  );
+  assert.equal(result.ok, true);
+  assert.equal(calls[0]?.options?.outputFormat, "audio-16khz-128kbitrate-mono-mp3");
+  assert.equal(calls[0]?.options?.signal, controller.signal);
+  assert.equal(calls[0]?.options?.timeoutMs, 1234);
+  assert.deepEqual(calls[0]?.options?.sourceNodePath, ["speak", "voice[0]"]);
+  assert.equal(validatorSignal, controller.signal);
+  assert.deepEqual(validatorPath, ["speak", "voice[0]"]);
+});

package/test/v216-pipeline.test.ts

@@ -0,0 +1,110 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+  AudioFormatMismatchError,
+  AzureTtsError,
+  mergeAudioBuffers,
+  mergeSynthesisResults,
+  synthesizeSsmlChunksSafe,
+} from "../src/index.ts";
+
+const validSsml = (text: string) =>
+  `<speak version="1.0" xml:lang="en-US"><voice name="en-US-JennyNeural">${text}</voice></speak>`;
+
+function wav(data: number[], sampleRate: number): ArrayBuffer {
+  const output = new Uint8Array(44 + data.length + (data.length & 1));
+  const view = new DataView(output.buffer);
+  output.set(new TextEncoder().encode("RIFF"), 0);
+  output.set(new TextEncoder().encode("WAVEfmt "), 8);
+  view.setUint32(4, output.length - 8, true);
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, 1, true);
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * 2, true);
+  view.setUint16(32, 2, true);
+  view.setUint16(34, 16, true);
+  output.set(new TextEncoder().encode("data"), 36);
+  view.setUint32(40, data.length, true);
+  output.set(data, 44);
+  return output.buffer;
+}
+
+test("chunk merging rejects incompatible WAV sample rates", () => {
+  assert.throws(
+    () => mergeAudioBuffers([wav([1, 2], 16_000), wav([3, 4], 24_000)], { format: "riff-16khz-16bit-mono-pcm" }),
+    (error: unknown) => error instanceof AudioFormatMismatchError,
+  );
+});
+
+test("safe chunk synthesis retries transient errors and preserves chunk order", async () => {
+  let attempts = 0;
+  let oneAttempts = 0;
+  let active = 0;
+  let maximumActive = 0;
+  const completed: string[] = [];
+  const retryEvents: number[] = [];
+  const result = await synthesizeSsmlChunksSafe(
+    {
+      synthesizeSsml: async (ssml) => {
+        attempts += 1;
+        active += 1;
+        maximumActive = Math.max(maximumActive, active);
+        await new Promise((resolve) => setTimeout(resolve, ssml.includes("one") ? 5 : 1));
+        active -= 1;
+        if (ssml.includes("one")) {
+          oneAttempts += 1;
+          if (oneAttempts < 3) throw new AzureTtsError(503, "Unavailable", "", null);
+        }
+        completed.push(ssml.includes("one") ? "one" : "two");
+        return { audioData: Uint8Array.of(ssml.includes("one") ? 1 : 2).buffer, durationMs: 10 };
+      },
+    },
+    [validSsml("one"), validSsml("two")],
+    {
+      concurrency: 2,
+      retryOptions: { maxRetries: 2, initialDelayMs: 0, maxDelayMs: 0 },
+      onProgress: (event) => {
+        if (event.isRetrying && event.retryAttempt) retryEvents.push(event.retryAttempt);
+      },
+    },
+  );
+  assert.equal(result.ok, true);
+  assert.equal(maximumActive, 2);
+  assert.deepEqual(retryEvents, [1, 2]);
+  assert.equal(attempts, 4);
+  if (result.ok) assert.deepEqual([...new Uint8Array(result.value.audioData)], [1, 2]);
+});
+
+test("safe chunk synthesis does not retry permanent HTTP errors", async () => {
+  let calls = 0;
+  const result = await synthesizeSsmlChunksSafe(
+    {
+      synthesizeSsml: async () => {
+        calls += 1;
+        throw new AzureTtsError(400, "Bad Request", "", null);
+      },
+    },
+    [validSsml("bad")],
+    { retryOptions: { maxRetries: 3, initialDelayMs: 0, maxDelayMs: 0 } },
+  );
+  assert.equal(result.ok, false);
+  assert.equal(calls, 1);
+});
+
+test("custom mergers receive input specifications and cancellation", async () => {
+  const controller = new AbortController();
+  let received: { format: string; inputSpecs: unknown[]; signal: AbortSignal } | undefined;
+  const result = await mergeSynthesisResults([{ audioData: Uint8Array.of(1).buffer, durationMs: 1 }], {
+    format: "webm-24khz-16bit-mono-opus",
+    signal: controller.signal,
+    customMerger: (buffers, context) => {
+      received = { format: context.format, inputSpecs: context.inputSpecs, signal: context.signal };
+      return buffers[0] ?? new ArrayBuffer(0);
+    },
+  });
+  assert.equal(result.mimeType, "audio/webm");
+  assert.equal(received?.format, "webm-24khz-16bit-mono-opus");
+  assert.equal(received?.inputSpecs.length, 1);
+  assert.equal(received?.signal, controller.signal);
+});