@ssml-builder-js/azure-tts-client 2.14.0 → 2.16.0

package/src/synthesis.ts

@@ -1,10 +1,51 @@
 import * as SpeechSDK from "microsoft-cognitiveservices-speech-sdk";
-import { createSpeechSdkError, UnsupportedMergeFormatError } from "./errors.ts";
+import { getSsmlSourceMap } from "@ssml-builder-js/ssml-core";
+import {
+  AzureTtsError,
+  MergeError,
+  AudioFormatMismatchError,
+  SynthesisCancelledError,
+  SynthesisTimeoutError,
+  toSynthesisError,
+  UnsupportedMergeFormatError,
+} from "./errors.ts";
+import { DEFAULT_OUTPUT_FORMAT, resolveMimeType, type AzureTtsOutputFormat } from "./outputFormats.ts";
 import { createSpeechConfig } from "./speechConfig.ts";
-import type { SsmlSynthesisChunk, SsmlSynthesisResult, SynthesisProgressEvent, TtsConfig } from "./types.ts";
+import type {
+  MergedSynthesisResult,
+  SsmlSynthesisChunk,
+  SsmlSynthesisResult,
+  AudioSpecification,
+  SynthesisProgressEvent,
+  TtsConfig,
+  RetryOptions,
+} from "./types.ts";
 
 export type MergeAudioFormat = "wav" | "mp3" | "raw";
 
+export interface MergeAudioOptions {
+  format: AzureTtsOutputFormat;
+  signal?: AbortSignal;
+  outputMimeType?: string;
+}
+
+export type InputAudioSpecs = AudioSpecification[];
+
+export interface CustomMergerContext {
+  format: string;
+  outputMimeType: string;
+  inputSpecs: InputAudioSpecs;
+  signal: AbortSignal;
+}
+
+export interface MergeSynthesisOptions extends MergeAudioOptions {
+  customMerger?: (buffers: ArrayBuffer[], context: CustomMergerContext) => Promise<ArrayBuffer> | ArrayBuffer;
+}
+
+type AsyncMergeSynthesisOptions = MergeSynthesisOptions & {
+  customMerger: NonNullable<MergeSynthesisOptions["customMerger"]>;
+};
+
 function ascii(bytes: Uint8Array, offset: number, value: string): boolean {
   return [...value].every((character, index) => bytes[offset + index] === character.charCodeAt(0));
 }
@@ -58,6 +99,127 @@ function parseWav(buffer: ArrayBuffer): ParsedWav {
   return { chunks, data, format };
 }
 
+function formatNumber(format: string, pattern: RegExp, fallback: number): number {
+  const match = pattern.exec(format);
+  return match?.[1] ? Number(match[1]) : fallback;
+}
+
+function formatChannels(format: string, fallback: number): number {
+  if (/stereo|2ch|dual/i.test(format)) return 2;
+  if (/mono|1ch/i.test(format)) return 1;
+  return fallback;
+}
+
+function formatAudioSpecification(format: string): AudioSpecification {
+  const sampleRate = formatNumber(format, /(\d+)(?:khz|kHz|hz|Hz)/, 0);
+  const channels = formatChannels(format, 0);
+  const bitrateMatch = /(\d+)(?:kbitrate|kbps|kbits?)/i.exec(format);
+  const bitrate = bitrateMatch?.[1] ? Number(bitrateMatch[1]) * 1000 : undefined;
+  const codec: AudioSpecification["codec"] = /mp3|mpeg/i.test(format)
+    ? "mp3"
+    : /opus/i.test(format)
+      ? "opus"
+      : /silk/i.test(format)
+        ? "silk"
+        : /pcm|mulaw|alaw|siren/i.test(format)
+          ? "pcm"
+          : "unknown";
+  return {
+    format,
+    mimeType: resolveMimeType(format),
+    codec,
+    sampleRate,
+    channels,
+    ...(bitrate ? { bitrate } : {}),
+    isCompressed: codec === "mp3" || codec === "opus" || codec === "silk",
+  };
+}
+
+function parseMp3Specification(buffer: ArrayBuffer, format: string): AudioSpecification | undefined {
+  const bytes = stripMp3Tags(buffer);
+  const bitrates = [
+    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+    [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0],
+    [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0],
+  ];
+  const sampleRates = [
+    [44_100, 48_000, 32_000],
+    [22_050, 24_000, 16_000],
+    [11_025, 12_000, 8_000],
+  ];
+  for (let index = 0; index + 4 <= bytes.length; index += 1) {
+    if (bytes[index] !== 0xff || (bytes[index + 1] ?? 0) < 0xe0) continue;
+    const header = bytes[index + 1] ?? 0;
+    const versionBits = (header >> 3) & 0x03;
+    const layer = (header >> 1) & 0x03;
+    const bitrateIndex = (bytes[index + 2] ?? 0) >> 4;
+    const sampleIndex = ((bytes[index + 2] ?? 0) >> 2) & 0x03;
+    if (versionBits === 1 || layer !== 1 || bitrateIndex === 0 || bitrateIndex === 15 || sampleIndex === 3) continue;
+    const versionIndex = versionBits === 3 ? 0 : versionBits === 2 ? 1 : 2;
+    const bitrateTable = versionBits === 3 ? bitrates[1] : bitrates[2];
+    const sampleRate = sampleRates[versionIndex]?.[sampleIndex] ?? 0;
+    const bitrateKbps = bitrateTable?.[bitrateIndex] ?? 0;
+    if (!sampleRate || !bitrateKbps) continue;
+    return {
+      format,
+      mimeType: "audio/mpeg",
+      codec: "mp3",
+      sampleRate,
+      channels: (bytes[index + 3] ?? 0) >> 6 === 3 ? 1 : 2,
+      bitrate: bitrateKbps * 1000,
+      isCompressed: true,
+    };
+  }
+  return undefined;
+}
+
+/** Extracts the stream specification from a WAV/MP3 header and output-format fallback. */
+export function inspectAudioSpecification(buffer: ArrayBuffer, format: string): AudioSpecification {
+  if (isWavFormat(format) || ascii(new Uint8Array(buffer), 0, "RIFF")) {
+    const parsed = parseWav(buffer);
+    if (parsed.format.byteLength < 16) throw new Error("Invalid WAV fmt chunk.");
+    const view = new DataView(parsed.format.buffer, parsed.format.byteOffset, parsed.format.byteLength);
+    const sampleRate = view.getUint32(4, true);
+    const channels = view.getUint16(2, true);
+    const bitsPerSample = parsed.format.byteLength >= 16 ? view.getUint16(14, true) : 0;
+    const formatCode = view.getUint16(0, true);
+    return {
+      format,
+      mimeType: "audio/wav",
+      codec: formatCode === 1 ? "pcm" : "unknown",
+      sampleRate,
+      channels,
+      ...(sampleRate && channels && bitsPerSample ? { bitrate: sampleRate * channels * bitsPerSample } : {}),
+      isCompressed: formatCode !== 1,
+    };
+  }
+  if (isMp3Format(format)) return parseMp3Specification(buffer, format) ?? formatAudioSpecification(format);
+  return formatAudioSpecification(format);
+}
+
+function validateAudioSpecifications(specs: readonly AudioSpecification[]): void {
+  const first = specs[0];
+  if (!first) return;
+  const mismatch = specs.find(
+    (spec) =>
+      spec.sampleRate !== first.sampleRate ||
+      spec.channels !== first.channels ||
+      (first.bitrate !== undefined && spec.bitrate !== undefined && spec.bitrate !== first.bitrate),
+  );
+  if (mismatch)
+    throw new AudioFormatMismatchError(
+      `Audio chunks have incompatible specifications: ${first.sampleRate}Hz/${first.channels}ch versus ${mismatch.sampleRate}Hz/${mismatch.channels}ch.`,
+      specs,
+    );
+}
+
+function isAudioFormatMismatch(error: unknown): error is AudioFormatMismatchError {
+  return (
+    error instanceof AudioFormatMismatchError ||
+    (error !== null && typeof error === "object" && "kind" in error && error.kind === "audio-format-mismatch")
+  );
+}
+
 function writeUint32(target: Uint8Array, offset: number, value: number): void {
   new DataView(target.buffer).setUint32(offset, value, true);
 }
@@ -154,28 +316,38 @@ export function canMergeAudioFormat(format: string): boolean {
 }
 
 /** Merges audio buffers while preserving the invariants of supported containers. */
-export function mergeAudioBuffers(buffers: readonly ArrayBuffer[], format: string): ArrayBuffer {
-  if (isWavFormat(format)) return mergeWavBuffers(buffers);
-  if (isMp3Format(format)) {
-    const parts = buffers.map(stripMp3Tags);
-    const output = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));
-    let offset = 0;
-    for (const part of parts) {
-      output.set(part, offset);
-      offset += part.byteLength;
+export function mergeAudioBuffers(buffers: readonly ArrayBuffer[], options: MergeAudioOptions): ArrayBuffer;
+export function mergeAudioBuffers(buffers: readonly ArrayBuffer[], options: MergeAudioOptions | string): ArrayBuffer {
+  const format = typeof options === "string" ? options : options?.format;
+  if (!format) throw new UnsupportedMergeFormatError("");
+  try {
+    validateAudioSpecifications(buffers.map((buffer) => inspectAudioSpecification(buffer, format)));
+    if (isWavFormat(format)) return mergeWavBuffers(buffers);
+    if (isMp3Format(format)) {
+      const parts = buffers.map(stripMp3Tags);
+      const output = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));
+      let offset = 0;
+      for (const part of parts) {
+        output.set(part, offset);
+        offset += part.byteLength;
+      }
+      return output.buffer;
     }
-    return output.buffer;
-  }
-  if (isRawFormat(format)) {
-    const output = new Uint8Array(buffers.reduce((total, buffer) => total + buffer.byteLength, 0));
-    let offset = 0;
-    for (const buffer of buffers) {
-      output.set(new Uint8Array(buffer), offset);
-      offset += buffer.byteLength;
+    if (isRawFormat(format)) {
+      const output = new Uint8Array(buffers.reduce((total, buffer) => total + buffer.byteLength, 0));
+      let offset = 0;
+      for (const buffer of buffers) {
+        output.set(new Uint8Array(buffer), offset);
+        offset += buffer.byteLength;
+      }
+      return output.buffer;
     }
-    return output.buffer;
+    throw new UnsupportedMergeFormatError(format);
+  } catch (error) {
+    if (error instanceof UnsupportedMergeFormatError || isAudioFormatMismatch(error) || error instanceof MergeError)
+      throw error;
+    throw new MergeError(`Audio buffers could not be merged for format "${format}".`, error);
   }
-  throw new UnsupportedMergeFormatError(format);
 }
 
 function closeSpeechResources(speechConfig: SpeechSDK.SpeechConfig, synthesizer: SpeechSDK.SpeechSynthesizer): void {
@@ -192,7 +364,7 @@ const ticksToMilliseconds = (ticks: number): number => Math.max(0, ticks) / 10_0
 
 export async function synthesizeSsml(ssml: string, config: TtsConfig): Promise<SsmlSynthesisResult> {
   if (config.signal?.aborted) {
-    throw createSpeechSdkError("Speech synthesis was cancelled.");
+    throw new SynthesisCancelledError();
   }
 
   const speechConfig = createSpeechConfig(config);
@@ -217,24 +389,129 @@ export async function synthesizeSsml(ssml: string, config: TtsConfig): Promise<S
       settled = true;
       cleanup();
       closeResources();
-      reject(createSpeechSdkError(error));
+      reject(toSynthesisError(error));
     };
 
     const boundaries: SsmlSynthesisResult["boundaries"] = [];
     const visemes: SsmlSynthesisResult["visemes"] = [];
     const bookmarks: SsmlSynthesisResult["bookmarks"] = [];
+    let sourceEventCursor = 0;
+    let generatedSourceMap: ReturnType<typeof getSsmlSourceMap> | undefined;
+    if (!config.sourceTextSegments && !config.sourceMarkers) {
+      try {
+        generatedSourceMap = getSsmlSourceMap(ssml);
+      } catch {
+        generatedSourceMap = undefined;
+      }
+    }
+    const sourceBaseOffset = config.sourceTextRange?.start ?? 0;
+    const sourceSegments =
+      config.sourceTextSegments ??
+      generatedSourceMap?.segments.map((segment) => ({
+        ...segment,
+        range: {
+          start: segment.range.start + sourceBaseOffset,
+          end: segment.range.end + sourceBaseOffset,
+        },
+        sourceNodePath: [...segment.sourceNodePath],
+      })) ??
+      [];
+    const sourceMarkers =
+      config.sourceMarkers ??
+      generatedSourceMap?.markers.map((marker) => ({
+        ...marker,
+        originalTextRange: {
+          start: marker.originalTextRange.start + sourceBaseOffset,
+          end: marker.originalTextRange.end + sourceBaseOffset,
+        },
+        sourceNodePath: [...marker.sourceNodePath],
+      })) ??
+      [];
+    const sourceText = sourceSegments.map((segment) => segment.text).join("");
+    const mapSourceEvent = (
+      text: string | undefined,
+      offsetHint: number | undefined,
+      markerName?: string,
+    ): {
+      textRange?: { start: number; end: number };
+      originalTextRange?: { start: number; end: number };
+      sourceNodePath?: string[];
+      mappingStatus: "exact" | "fallback" | "unmapped";
+    } => {
+      const marker = markerName ? sourceMarkers.find((candidate) => candidate.name === markerName) : undefined;
+      if (marker) {
+        return {
+          originalTextRange: { ...marker.originalTextRange },
+          sourceNodePath: [...marker.sourceNodePath],
+          textRange: { ...marker.originalTextRange },
+          mappingStatus: "exact",
+        };
+      }
+      if (sourceSegments.length === 0 && !config.sourceTextRange && !config.sourceNodePath) {
+        const unmapped: { mappingStatus: "unmapped" } = { mappingStatus: "unmapped" };
+        Object.defineProperty(unmapped, "mappingStatus", { value: "unmapped", enumerable: false });
+        return unmapped;
+      }
+      const value = text ?? "";
+      let localStart = Number.isFinite(offsetHint) && (offsetHint ?? 0) >= 0 ? (offsetHint as number) : -1;
+      let mappingStatus: "exact" | "fallback" | "unmapped" = "exact";
+      if (value && localStart >= 0 && sourceText.slice(localStart, localStart + value.length) !== value) {
+        localStart = -1;
+        mappingStatus = "fallback";
+      }
+      if (localStart < 0 || localStart > sourceText.length) {
+        localStart = value ? sourceText.indexOf(value, sourceEventCursor) : sourceEventCursor;
+        if (localStart < 0) localStart = value ? sourceText.indexOf(value) : sourceEventCursor;
+        mappingStatus = "fallback";
+      }
+      localStart = Math.max(0, localStart);
+      const localEnd = Math.min(sourceText.length, localStart + value.length);
+      sourceEventCursor = Math.max(sourceEventCursor, localEnd);
+      const baseStart = config.sourceTextRange?.start ?? sourceSegments[0]?.range.start ?? 0;
+      const fallbackRange = { start: baseStart + localStart, end: baseStart + localEnd };
+      const segment =
+        sourceSegments.find(({ range }) => range.start <= fallbackRange.start && range.end > fallbackRange.start) ??
+        sourceSegments.find(({ range }) => range.end > fallbackRange.start) ??
+        (value.length === 0
+          ? sourceSegments.find(({ range }) => range.start <= fallbackRange.start && range.end >= fallbackRange.start)
+          : undefined);
+      return {
+        originalTextRange: { ...fallbackRange },
+        textRange: { ...fallbackRange },
+        ...(segment
+          ? { sourceNodePath: [...segment.sourceNodePath] }
+          : config.sourceNodePath
+            ? { sourceNodePath: [...config.sourceNodePath] }
+            : {}),
+        mappingStatus: segment || config.sourceTextRange || config.sourceNodePath ? mappingStatus : "unmapped",
+      };
+    };
     synthesizer.wordBoundary = (_sender, event) => {
       boundaries.push({
         text: event.text,
         audioOffsetMs: ticksToMilliseconds(event.audioOffset),
         durationMs: ticksToMilliseconds(event.duration),
+        ...mapSourceEvent(
+          event.text,
+          (event as SpeechSDK.SpeechSynthesisWordBoundaryEventArgs & { textOffset?: number }).textOffset,
+        ),
       });
     };
     synthesizer.visemeReceived = (_sender, event) => {
-      visemes.push({ visemeId: event.visemeId, audioOffsetMs: ticksToMilliseconds(event.audioOffset) });
+      const eventWithOffset = event as SpeechSDK.SpeechSynthesisVisemeEventArgs & { textOffset?: number };
+      visemes.push({
+        visemeId: event.visemeId,
+        audioOffsetMs: ticksToMilliseconds(event.audioOffset),
+        ...mapSourceEvent(undefined, eventWithOffset.textOffset),
+      });
     };
     synthesizer.bookmarkReached = (_sender, event) => {
-      bookmarks.push({ name: event.text, audioOffsetMs: ticksToMilliseconds(event.audioOffset) });
+      const eventWithOffset = event as SpeechSDK.SpeechSynthesisBookmarkEventArgs & { textOffset?: number };
+      bookmarks.push({
+        name: event.text,
+        audioOffsetMs: ticksToMilliseconds(event.audioOffset),
+        ...mapSourceEvent(undefined, eventWithOffset.textOffset, event.text),
+      });
     };
 
     const cb = (result: SpeechSDK.SpeechSynthesisResult) => {
@@ -256,20 +533,31 @@ export async function synthesizeSsml(ssml: string, config: TtsConfig): Promise<S
       );
       const durationMs = result.audioDuration ? ticksToMilliseconds(result.audioDuration) : eventDurationMs;
       const requestId = (result as SpeechSDK.SpeechSynthesisResult & { resultId?: string }).resultId;
-      const addSourceMetadata = <T extends { audioOffsetMs: number }>(event: T): T => ({
-        ...event,
-        ...(config.sourceTextRange ? { textRange: { ...config.sourceTextRange } } : {}),
-        ...(config.sourceTextRange ? { originalTextRange: { ...config.sourceTextRange } } : {}),
-        ...(config.chunkIndex !== undefined ? { chunkIndex: config.chunkIndex } : {}),
-        ...(config.sourceNodePath ? { sourceNodePath: [...config.sourceNodePath] } : {}),
-        ...(requestId ? { requestId } : {}),
-      });
+      const addSourceMetadata = <T extends { audioOffsetMs: number; mappingStatus: "exact" | "fallback" | "unmapped" }>(
+        event: T,
+      ): T => {
+        const mapped = {
+          ...event,
+          ...(config.sourceTextRange && !("textRange" in event) ? { textRange: { ...config.sourceTextRange } } : {}),
+          ...(config.sourceTextRange && !("originalTextRange" in event)
+            ? { originalTextRange: { ...config.sourceTextRange } }
+            : {}),
+          ...(config.chunkIndex !== undefined ? { chunkIndex: config.chunkIndex } : {}),
+          ...(config.sourceNodePath ? { sourceNodePath: [...config.sourceNodePath] } : {}),
+          ...(requestId ? { requestId } : {}),
+        } as T;
+        if (event.mappingStatus === "unmapped")
+          Object.defineProperty(mapped, "mappingStatus", { value: "unmapped", enumerable: false });
+        return mapped;
+      };
       const sourceBoundaries = boundaries.map((boundary) => addSourceMetadata(boundary));
       const sourceVisemes = visemes.map((viseme) => addSourceMetadata(viseme));
       const sourceBookmarks = bookmarks.map((bookmark) => addSourceMetadata(bookmark));
       resolve({
         audioData: result.audioData,
         durationMs,
+        audioSpec: inspectAudioSpecification(result.audioData, config.outputFormat ?? DEFAULT_OUTPUT_FORMAT),
+        mimeType: resolveMimeType(config.outputFormat ?? DEFAULT_OUTPUT_FORMAT),
         ...(config.sourceTextRange ? { textRange: { ...config.sourceTextRange } } : {}),
         ...(requestId ? { requestId } : {}),
         ...(sourceBoundaries.length > 0
@@ -282,12 +570,12 @@ export async function synthesizeSsml(ssml: string, config: TtsConfig): Promise<S
 
     try {
       if (config.signal) {
-        abortHandler = () => rejectWithError("Speech synthesis was cancelled.");
+        abortHandler = () => rejectWithError(new SynthesisCancelledError());
         config.signal.addEventListener("abort", abortHandler, { once: true });
       }
       if (config.timeoutMs !== undefined && config.timeoutMs > 0) {
         timeout = setTimeout(
-          () => rejectWithError(`Speech synthesis timed out after ${config.timeoutMs} ms.`),
+          () => rejectWithError(new SynthesisTimeoutError(`Speech synthesis timed out after ${config.timeoutMs} ms.`)),
           config.timeoutMs,
         );
       }
@@ -298,12 +586,82 @@ export async function synthesizeSsml(ssml: string, config: TtsConfig): Promise<S
   });
 }
 
-/** Synthesizes chunks sequentially, annotates synchronization events, and merges the results. */
+function isRetryableSynthesisError(error: unknown): boolean {
+  if (error instanceof SynthesisCancelledError || error instanceof SynthesisTimeoutError) return false;
+  if (error instanceof AzureTtsError && error.status !== 0)
+    return error.status === 429 || (error.status >= 500 && error.status < 600);
+  const message = error instanceof Error ? error.message : String(error);
+  if (/\b4\d{2}\b/.test(message)) return false;
+  const status = error && typeof error === "object" && "status" in error ? error.status : undefined;
+  if (typeof status === "number") return status === 429 || (status >= 500 && status < 600);
+  return /network|connection|connect|socket|fetch failed|econn|etimedout|temporar|transient|unavailable/i.test(message);
+}
+
+function retryDelay(options: RetryOptions, retryAttempt: number): number {
+  const base = Math.min(options.maxDelayMs, options.initialDelayMs * 2 ** Math.max(0, retryAttempt - 1));
+  return Math.floor(Math.random() * (base + 1));
+}
+
+function resolveConcurrency(value: number | undefined, total: number): number {
+  if (value === undefined) return 1;
+  if (value === Infinity) return Math.max(1, total);
+  return Number.isFinite(value) ? Math.max(1, Math.floor(value)) : 1;
+}
+
+async function waitForRetry(delayMs: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) throw new SynthesisCancelledError();
+  if (delayMs <= 0) return;
+  await new Promise<void>((resolve, reject) => {
+    let timer: ReturnType<typeof setTimeout>;
+    const abort = () => {
+      clearTimeout(timer);
+      signal?.removeEventListener("abort", abort);
+      reject(new SynthesisCancelledError());
+    };
+    timer = setTimeout(() => {
+      signal?.removeEventListener("abort", abort);
+      resolve();
+    }, delayMs);
+    if (signal) {
+      signal.addEventListener("abort", abort, { once: true });
+    }
+  });
+}
+
+async function synthesizeWithRetry(
+  ssml: string,
+  config: TtsConfig,
+  retryOptions: RetryOptions | undefined,
+  onRetry: (retryAttempt: number, nextRetryDelayMs: number) => void,
+): Promise<SsmlSynthesisResult> {
+  const options = retryOptions
+    ? {
+        maxRetries: Math.max(0, Math.floor(retryOptions.maxRetries)),
+        initialDelayMs: Math.max(0, retryOptions.initialDelayMs),
+        maxDelayMs: Math.max(0, retryOptions.maxDelayMs),
+      }
+    : undefined;
+  let attempt = 0;
+  while (true) {
+    if (config.signal?.aborted) throw new SynthesisCancelledError();
+    try {
+      return await synthesizeSsml(ssml, config);
+    } catch (error) {
+      if (!options || attempt >= options.maxRetries || !isRetryableSynthesisError(error)) throw error;
+      attempt += 1;
+      const delayMs = retryDelay(options, attempt);
+      onRetry(attempt, delayMs);
+      await waitForRetry(delayMs, config.signal);
+    }
+  }
+}
+
+/** Synthesizes chunks with bounded concurrency, retries transient failures, and merges in chunk order. */
 export async function synthesizeSsmlChunks(
   chunks: readonly (SsmlSynthesisChunk | string)[],
   config: TtsConfig,
 ): Promise<SsmlSynthesisResult> {
-  const results: SsmlSynthesisResult[] = [];
+  const results: Array<SsmlSynthesisResult | undefined> = new Array(chunks.length);
   const totalChunks = chunks.length;
   const report = (event: SynthesisProgressEvent): void => config.onProgress?.(event);
   for (const [index, chunk] of chunks.entries()) {
@@ -318,76 +676,102 @@ export async function synthesizeSsmlChunks(
       durationMs: 0,
     });
   }
-  for (const [index, chunk] of chunks.entries()) {
-    const input = typeof chunk === "string" ? { ssml: chunk } : chunk;
-    report({
-      currentChunk: index,
-      totalChunks,
-      percent: totalChunks === 0 ? 100 : Math.round((index / totalChunks) * 100),
-      chunkIndex: index,
-      originalTextRange: input.originalTextRange,
-      status: "synthesizing",
-      durationMs: 0,
-    });
-    const startedAt = Date.now();
-    try {
-      const result = await synthesizeSsml(input.ssml, {
-        ...config,
-        ...(input.originalTextRange ? { sourceTextRange: input.originalTextRange } : {}),
-        ...(input.sourceNodePath ? { sourceNodePath: input.sourceNodePath } : {}),
-        chunkIndex: index,
-        onProgress: undefined,
-      });
-      results.push(result);
+  let completed = 0;
+  let nextIndex = 0;
+  const concurrency = resolveConcurrency(config.concurrency, chunks.length);
+  const worker = async (): Promise<void> => {
+    while (true) {
+      const index = nextIndex++;
+      if (index >= chunks.length) return;
+      const chunk = chunks[index];
+      const input = typeof chunk === "string" ? { ssml: chunk } : chunk;
       report({
-        currentChunk: index + 1,
+        currentChunk: completed,
         totalChunks,
-        percent: totalChunks === 0 ? 100 : Math.round(((index + 1) / totalChunks) * 100),
+        percent: totalChunks === 0 ? 100 : Math.round((completed / totalChunks) * 100),
         chunkIndex: index,
         originalTextRange: input.originalTextRange,
-        status: "success",
-        durationMs: Date.now() - startedAt,
+        status: "synthesizing",
+        durationMs: 0,
       });
-    } catch (error) {
-      report({
-        currentChunk: index,
-        totalChunks,
-        percent: totalChunks === 0 ? 100 : Math.round((index / totalChunks) * 100),
-        chunkIndex: index,
-        originalTextRange: input.originalTextRange,
-        status: "failed",
-        durationMs: Date.now() - startedAt,
-        error,
-      });
-      throw error;
+      const startedAt = Date.now();
+      try {
+        const result = await synthesizeWithRetry(
+          input.ssml,
+          {
+            ...config,
+            ...(input.originalTextRange ? { sourceTextRange: input.originalTextRange } : {}),
+            ...((input.sourceNodePath ?? config.sourceNodePath)
+              ? { sourceNodePath: [...(input.sourceNodePath ?? config.sourceNodePath ?? [])] }
+              : {}),
+            ...(input.sourceTextSegments ? { sourceTextSegments: input.sourceTextSegments } : {}),
+            ...(input.sourceMarkers ? { sourceMarkers: input.sourceMarkers } : {}),
+            chunkIndex: index,
+            onProgress: undefined,
+          },
+          config.retryOptions,
+          (retryAttempt, nextRetryDelayMs) =>
+            report({
+              currentChunk: completed,
+              totalChunks,
+              percent: totalChunks === 0 ? 100 : Math.round((completed / totalChunks) * 100),
+              chunkIndex: index,
+              originalTextRange: input.originalTextRange,
+              status: "synthesizing",
+              durationMs: Date.now() - startedAt,
+              retryAttempt,
+              nextRetryDelayMs,
+              isRetrying: true,
+            }),
+        );
+        results[index] = result;
+        completed += 1;
+        report({
+          currentChunk: completed,
+          totalChunks,
+          percent: totalChunks === 0 ? 100 : Math.round((completed / totalChunks) * 100),
+          chunkIndex: index,
+          originalTextRange: input.originalTextRange,
+          status: "success",
+          durationMs: Date.now() - startedAt,
+        });
+      } catch (error) {
+        report({
+          currentChunk: completed,
+          totalChunks,
+          percent: totalChunks === 0 ? 100 : Math.round((completed / totalChunks) * 100),
+          chunkIndex: index,
+          originalTextRange: input.originalTextRange,
+          status: "failed",
+          durationMs: Date.now() - startedAt,
+          error,
+        });
+        throw error;
+      }
     }
-  }
-  return mergeSynthesisResults(results, config.outputFormat ?? "audio-16khz-128kbitrate-mono-mp3");
+  };
+  await Promise.all(Array.from({ length: Math.min(concurrency, chunks.length) }, () => worker()));
+  const orderedResults = results.filter((result): result is SsmlSynthesisResult => result !== undefined);
+  return mergeSynthesisResults(orderedResults, {
+    format: (config.outputFormat ?? DEFAULT_OUTPUT_FORMAT) as AzureTtsOutputFormat,
+    signal: config.signal,
+  });
 }
 
 /** Concatenates audio buffers and shifts all synchronization events by prior chunk durations. */
-export function mergeSynthesisResults(results: readonly SsmlSynthesisResult[], format?: string): SsmlSynthesisResult {
-  const audioData = format
-    ? new Uint8Array(
-        mergeAudioBuffers(
-          results.map((result) => result.audioData),
-          format,
-        ),
-      )
-    : new Uint8Array(results.reduce((total, result) => total + result.audioData.byteLength, 0));
-  if (!format) {
-    let offset = 0;
-    for (const result of results) {
-      audioData.set(new Uint8Array(result.audioData), offset);
-      offset += result.audioData.byteLength;
-    }
-  }
+function createMergedResult(
+  results: readonly SsmlSynthesisResult[],
+  audioData: ArrayBuffer,
+  format: string,
+  audioSpec?: AudioSpecification,
+  outputMimeType?: string,
+): MergedSynthesisResult {
   const boundaries: NonNullable<SsmlSynthesisResult["boundaries"]> = [];
   const visemes: NonNullable<SsmlSynthesisResult["visemes"]> = [];
   const bookmarks: NonNullable<SsmlSynthesisResult["bookmarks"]> = [];
   let durationOffset = 0;
 
-  for (const result of results) {
+  for (const [resultIndex, result] of results.entries()) {
     const chunkBoundaries =
       result.boundaries && result.boundaries.length > 0
         ? result.boundaries
@@ -400,11 +784,12 @@ export function mergeSynthesisResults(results: readonly SsmlSynthesisResult[], f
         ...boundary,
         audioOffsetMs: boundary.audioOffsetMs + durationOffset,
         chunkAudioOffsetMs: boundary.chunkAudioOffsetMs ?? boundary.audioOffsetMs,
-        ...(boundary.chunkIndex === undefined ? { chunkIndex: results.indexOf(result) } : {}),
+        ...(boundary.chunkIndex === undefined ? { chunkIndex: resultIndex } : {}),
         ...(boundary.sourceNodePath ? { sourceNodePath: [...boundary.sourceNodePath] } : {}),
         ...(originalTextRange ? { originalTextRange: { ...originalTextRange } } : {}),
         ...(textRange ? { textRange: { ...textRange } } : {}),
         ...(requestId ? { requestId } : {}),
+        mappingStatus: boundary.mappingStatus ?? "unmapped",
       });
     }
     for (const viseme of result.visemes ?? []) {
@@ -415,11 +800,12 @@ export function mergeSynthesisResults(results: readonly SsmlSynthesisResult[], f
         ...viseme,
         audioOffsetMs: viseme.audioOffsetMs + durationOffset,
         chunkAudioOffsetMs: viseme.chunkAudioOffsetMs ?? viseme.audioOffsetMs,
-        ...(viseme.chunkIndex === undefined ? { chunkIndex: results.indexOf(result) } : {}),
+        ...(viseme.chunkIndex === undefined ? { chunkIndex: resultIndex } : {}),
         ...(viseme.sourceNodePath ? { sourceNodePath: [...viseme.sourceNodePath] } : {}),
         ...(originalTextRange ? { originalTextRange: { ...originalTextRange } } : {}),
         ...(textRange ? { textRange: { ...textRange } } : {}),
         ...(requestId ? { requestId } : {}),
+        mappingStatus: viseme.mappingStatus ?? "unmapped",
       });
     }
     for (const bookmark of result.bookmarks ?? []) {
@@ -430,19 +816,23 @@ export function mergeSynthesisResults(results: readonly SsmlSynthesisResult[], f
         ...bookmark,
         audioOffsetMs: bookmark.audioOffsetMs + durationOffset,
         chunkAudioOffsetMs: bookmark.chunkAudioOffsetMs ?? bookmark.audioOffsetMs,
-        ...(bookmark.chunkIndex === undefined ? { chunkIndex: results.indexOf(result) } : {}),
+        ...(bookmark.chunkIndex === undefined ? { chunkIndex: resultIndex } : {}),
         ...(bookmark.sourceNodePath ? { sourceNodePath: [...bookmark.sourceNodePath] } : {}),
         ...(originalTextRange ? { originalTextRange: { ...originalTextRange } } : {}),
         ...(textRange ? { textRange: { ...textRange } } : {}),
         ...(requestId ? { requestId } : {}),
+        mappingStatus: bookmark.mappingStatus ?? "unmapped",
       });
     }
     durationOffset += Math.max(0, result.durationMs);
   }
 
   return {
-    audioData: audioData.buffer,
+    audioData,
     durationMs: durationOffset,
+    mimeType: resolveMimeType(format),
+    audioSpec: audioSpec ?? formatAudioSpecification(format),
+    ...(outputMimeType ? { mimeType: outputMimeType } : {}),
     ...(boundaries.length > 0 ? { boundaries, wordBoundary: boundaries, wordBoundaries: boundaries } : {}),
     ...(visemes.length > 0 ? { visemes } : {}),
     ...(bookmarks.length > 0 ? { bookmarks } : {}),
@@ -451,6 +841,74 @@ export function mergeSynthesisResults(results: readonly SsmlSynthesisResult[], f
   };
 }
 
+export function mergeSynthesisResults(
+  results: readonly SsmlSynthesisResult[],
+  options: AsyncMergeSynthesisOptions,
+): Promise<MergedSynthesisResult>;
+export function mergeSynthesisResults(
+  results: readonly SsmlSynthesisResult[],
+  options: MergeAudioOptions,
+): MergedSynthesisResult;
+export function mergeSynthesisResults(
+  results: readonly SsmlSynthesisResult[],
+  options: MergeSynthesisOptions | string,
+): SsmlSynthesisResult | Promise<SsmlSynthesisResult> {
+  const resolvedOptions: MergeSynthesisOptions =
+    typeof options === "string" ? { format: options as AzureTtsOutputFormat } : options;
+  const format = resolvedOptions?.format;
+  if (!format) throw new UnsupportedMergeFormatError("");
+  const buffers = results.map((result) => result.audioData);
+  const inputSpecs = results.map((result) => result.audioSpec ?? inspectAudioSpecification(result.audioData, format));
+  validateAudioSpecifications(inputSpecs);
+  const signal = resolvedOptions.signal ?? new AbortController().signal;
+  if (signal.aborted) throw new SynthesisCancelledError();
+  if (resolvedOptions.customMerger) {
+    return Promise.resolve()
+      .then(() =>
+        resolvedOptions.customMerger?.(buffers, {
+          format,
+          outputMimeType: resolvedOptions.outputMimeType ?? resolveMimeType(format),
+          inputSpecs,
+          signal,
+        }),
+      )
+      .then((merged) => {
+        if (!merged) throw new MergeError("The custom audio merger returned no audio buffer.");
+        if (
+          !(merged instanceof ArrayBuffer) ||
+          (buffers.some((buffer) => buffer.byteLength > 0) && merged.byteLength === 0)
+        )
+          throw new MergeError("The custom audio merger returned an invalid audio buffer.");
+        if (signal.aborted) throw new SynthesisCancelledError();
+        return createMergedResult(
+          results,
+          merged,
+          format,
+          inspectAudioSpecification(merged, format),
+          resolvedOptions.outputMimeType,
+        );
+      })
+      .catch((error: unknown) => {
+        if (error instanceof UnsupportedMergeFormatError || isAudioFormatMismatch(error) || error instanceof MergeError)
+          throw error;
+        throw new MergeError(`Custom audio merger failed for format "${format}".`, error);
+      });
+  }
+  try {
+    return createMergedResult(
+      results,
+      mergeAudioBuffers(buffers, { format }),
+      format,
+      inputSpecs[0],
+      resolvedOptions.outputMimeType,
+    );
+  } catch (error) {
+    if (error instanceof UnsupportedMergeFormatError || isAudioFormatMismatch(error) || error instanceof MergeError)
+      throw error;
+    throw new MergeError(`Audio buffers could not be merged for format "${format}".`, error);
+  }
+}
+
 /** Backward-compatible audio-only synthesis helper. */
 export async function synthesizeSpeech(ssml: string, config: TtsConfig): Promise<ArrayBuffer> {
   return (await synthesizeSsml(ssml, config)).audioData;