@ssml-builder-js/azure-tts-client 2.13.0 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/index.d.mts +65 -9
- package/dist/index.d.ts +65 -9
- package/dist/index.js +363 -15
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +357 -15
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
- package/src/client.ts +20 -2
- package/src/errors.ts +11 -0
- package/src/index.ts +13 -3
- package/src/safe.ts +129 -1
- package/src/synthesis.ts +263 -17
- package/src/types.ts +32 -2
- package/src/voiceCatalog.ts +15 -0
- package/test/v213-pipeline.test.ts +4 -0
- package/test/v214-pipeline.test.ts +110 -0
package/src/synthesis.ts
CHANGED
|
@@ -1,7 +1,182 @@
|
|
|
1
1
|
import * as SpeechSDK from "microsoft-cognitiveservices-speech-sdk";
|
|
2
|
-
import { createSpeechSdkError } from "./errors.ts";
|
|
2
|
+
import { createSpeechSdkError, UnsupportedMergeFormatError } from "./errors.ts";
|
|
3
3
|
import { createSpeechConfig } from "./speechConfig.ts";
|
|
4
|
-
import type { SsmlSynthesisChunk, SsmlSynthesisResult, TtsConfig } from "./types.ts";
|
|
4
|
+
import type { SsmlSynthesisChunk, SsmlSynthesisResult, SynthesisProgressEvent, TtsConfig } from "./types.ts";
|
|
5
|
+
|
|
6
|
+
export type MergeAudioFormat = "wav" | "mp3" | "raw";
|
|
7
|
+
|
|
8
|
+
function ascii(bytes: Uint8Array, offset: number, value: string): boolean {
|
|
9
|
+
return [...value].every((character, index) => bytes[offset + index] === character.charCodeAt(0));
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function readUint32(bytes: Uint8Array, offset: number): number {
|
|
13
|
+
return new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(offset, true);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface RiffChunk {
|
|
17
|
+
id: string;
|
|
18
|
+
data: Uint8Array;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
interface ParsedWav {
|
|
22
|
+
chunks: RiffChunk[];
|
|
23
|
+
data: Uint8Array;
|
|
24
|
+
format: Uint8Array;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function parseWav(buffer: ArrayBuffer): ParsedWav {
|
|
28
|
+
const bytes = new Uint8Array(buffer);
|
|
29
|
+
if (bytes.byteLength < 12 || !ascii(bytes, 0, "RIFF") || !ascii(bytes, 8, "WAVE")) {
|
|
30
|
+
throw new Error("Invalid WAV/RIFF audio buffer.");
|
|
31
|
+
}
|
|
32
|
+
const chunks: RiffChunk[] = [];
|
|
33
|
+
const dataParts: Uint8Array[] = [];
|
|
34
|
+
let format: Uint8Array | undefined;
|
|
35
|
+
let offset = 12;
|
|
36
|
+
while (offset < bytes.byteLength) {
|
|
37
|
+
if (offset + 8 > bytes.byteLength) throw new Error("Invalid WAV chunk header.");
|
|
38
|
+
const id = String.fromCharCode(...bytes.slice(offset, offset + 4));
|
|
39
|
+
const size = readUint32(bytes, offset + 4);
|
|
40
|
+
const dataStart = offset + 8;
|
|
41
|
+
const dataEnd = dataStart + size;
|
|
42
|
+
if (dataEnd > bytes.byteLength) throw new Error(`WAV chunk "${id}" exceeds the audio buffer.`);
|
|
43
|
+
const data = bytes.slice(dataStart, dataEnd);
|
|
44
|
+
chunks.push({ id, data });
|
|
45
|
+
if (id === "fmt ") format ??= data;
|
|
46
|
+
if (id === "data") dataParts.push(data);
|
|
47
|
+
offset = dataEnd + (size & 1);
|
|
48
|
+
if (offset > bytes.byteLength) throw new Error("Invalid WAV chunk padding.");
|
|
49
|
+
}
|
|
50
|
+
if (!format || dataParts.length === 0) throw new Error("WAV audio must contain fmt and data chunks.");
|
|
51
|
+
const dataLength = dataParts.reduce((total, part) => total + part.byteLength, 0);
|
|
52
|
+
const data = new Uint8Array(dataLength);
|
|
53
|
+
let dataOffset = 0;
|
|
54
|
+
for (const part of dataParts) {
|
|
55
|
+
data.set(part, dataOffset);
|
|
56
|
+
dataOffset += part.byteLength;
|
|
57
|
+
}
|
|
58
|
+
return { chunks, data, format };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function writeUint32(target: Uint8Array, offset: number, value: number): void {
|
|
62
|
+
new DataView(target.buffer).setUint32(offset, value, true);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function writeChunk(target: Uint8Array, offset: number, id: string, data: Uint8Array): number {
|
|
66
|
+
for (let index = 0; index < 4; index += 1) target[offset + index] = id.charCodeAt(index) ?? 0;
|
|
67
|
+
writeUint32(target, offset + 4, data.byteLength);
|
|
68
|
+
target.set(data, offset + 8);
|
|
69
|
+
const end = offset + 8 + data.byteLength;
|
|
70
|
+
if (data.byteLength & 1) target[end] = 0;
|
|
71
|
+
return end + (data.byteLength & 1);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function mergeWavBuffers(buffers: readonly ArrayBuffer[]): ArrayBuffer {
|
|
75
|
+
if (buffers.length === 0) return new ArrayBuffer(0);
|
|
76
|
+
const parsed = buffers.map(parseWav);
|
|
77
|
+
const first = parsed[0];
|
|
78
|
+
if (!first) throw new Error("At least one WAV buffer is required.");
|
|
79
|
+
if (
|
|
80
|
+
parsed.some(
|
|
81
|
+
(item) => item.format.length !== first.format.length || item.format.some((value, i) => value !== first.format[i]),
|
|
82
|
+
)
|
|
83
|
+
)
|
|
84
|
+
throw new Error("WAV buffers have incompatible fmt chunks.");
|
|
85
|
+
const dataLength = parsed.reduce((total, item) => total + item.data.byteLength, 0);
|
|
86
|
+
const nonDataLength = first.chunks.reduce(
|
|
87
|
+
(total, chunk) => (chunk.id === "data" ? total : total + 8 + chunk.data.byteLength + (chunk.data.byteLength & 1)),
|
|
88
|
+
0,
|
|
89
|
+
);
|
|
90
|
+
const outputLength = 12 + nonDataLength + 8 + dataLength + (dataLength & 1);
|
|
91
|
+
if (outputLength - 8 > 0xffffffff) throw new RangeError("Merged WAV exceeds the RIFF format size limit.");
|
|
92
|
+
const output = new Uint8Array(outputLength);
|
|
93
|
+
output.set(Uint8Array.from([0x52, 0x49, 0x46, 0x46]), 0);
|
|
94
|
+
writeUint32(output, 4, outputLength - 8);
|
|
95
|
+
output.set(Uint8Array.from([0x57, 0x41, 0x56, 0x45]), 8);
|
|
96
|
+
let outputOffset = 12;
|
|
97
|
+
let dataWritten = false;
|
|
98
|
+
for (const chunk of first.chunks) {
|
|
99
|
+
if (chunk.id === "data") {
|
|
100
|
+
if (dataWritten) continue;
|
|
101
|
+
const data = new Uint8Array(dataLength);
|
|
102
|
+
let dataOffset = 0;
|
|
103
|
+
for (const item of parsed) {
|
|
104
|
+
data.set(item.data, dataOffset);
|
|
105
|
+
dataOffset += item.data.byteLength;
|
|
106
|
+
}
|
|
107
|
+
outputOffset = writeChunk(output, outputOffset, "data", data);
|
|
108
|
+
dataWritten = true;
|
|
109
|
+
} else {
|
|
110
|
+
outputOffset = writeChunk(output, outputOffset, chunk.id, chunk.data);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (!dataWritten) throw new Error("WAV audio must contain a data chunk.");
|
|
114
|
+
return output.buffer;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function skipId3v2(bytes: Uint8Array): number {
|
|
118
|
+
if (!ascii(bytes, 0, "ID3") || bytes.byteLength < 10) return 0;
|
|
119
|
+
const size = [bytes[6], bytes[7], bytes[8], bytes[9]].reduce((total, value) => (total << 7) | (value & 0x7f), 0);
|
|
120
|
+
const hasFooter = (bytes[5] & 0x10) !== 0;
|
|
121
|
+
return Math.min(bytes.byteLength, 10 + size + (hasFooter ? 10 : 0));
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function stripMp3Tags(buffer: ArrayBuffer): Uint8Array {
|
|
125
|
+
const bytes = new Uint8Array(buffer);
|
|
126
|
+
const start = skipId3v2(bytes);
|
|
127
|
+
const end =
|
|
128
|
+
bytes.byteLength >= 128 && ascii(bytes, bytes.byteLength - 128, "TAG") ? bytes.byteLength - 128 : bytes.byteLength;
|
|
129
|
+
return bytes.slice(Math.min(start, end), end);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function isMp3Format(format: string): boolean {
|
|
133
|
+
return /(?:mp3|mpeg)/i.test(format);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function isWavFormat(format: string): boolean {
|
|
137
|
+
return /(?:wav|wave|riff)/i.test(format);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function isRawFormat(format: string): boolean {
|
|
141
|
+
return /^raw(?:-|$)/i.test(format);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Returns whether the named output format can be safely concatenated without re-multiplexing. */
|
|
145
|
+
export function resolveMergeAudioFormat(format: string): MergeAudioFormat | undefined {
|
|
146
|
+
if (isWavFormat(format)) return "wav";
|
|
147
|
+
if (isMp3Format(format)) return "mp3";
|
|
148
|
+
if (isRawFormat(format)) return "raw";
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export function canMergeAudioFormat(format: string): boolean {
|
|
153
|
+
return resolveMergeAudioFormat(format) !== undefined;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Merges audio buffers while preserving the invariants of supported containers. */
|
|
157
|
+
export function mergeAudioBuffers(buffers: readonly ArrayBuffer[], format: string): ArrayBuffer {
|
|
158
|
+
if (isWavFormat(format)) return mergeWavBuffers(buffers);
|
|
159
|
+
if (isMp3Format(format)) {
|
|
160
|
+
const parts = buffers.map(stripMp3Tags);
|
|
161
|
+
const output = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));
|
|
162
|
+
let offset = 0;
|
|
163
|
+
for (const part of parts) {
|
|
164
|
+
output.set(part, offset);
|
|
165
|
+
offset += part.byteLength;
|
|
166
|
+
}
|
|
167
|
+
return output.buffer;
|
|
168
|
+
}
|
|
169
|
+
if (isRawFormat(format)) {
|
|
170
|
+
const output = new Uint8Array(buffers.reduce((total, buffer) => total + buffer.byteLength, 0));
|
|
171
|
+
let offset = 0;
|
|
172
|
+
for (const buffer of buffers) {
|
|
173
|
+
output.set(new Uint8Array(buffer), offset);
|
|
174
|
+
offset += buffer.byteLength;
|
|
175
|
+
}
|
|
176
|
+
return output.buffer;
|
|
177
|
+
}
|
|
178
|
+
throw new UnsupportedMergeFormatError(format);
|
|
179
|
+
}
|
|
5
180
|
|
|
6
181
|
function closeSpeechResources(speechConfig: SpeechSDK.SpeechConfig, synthesizer: SpeechSDK.SpeechSynthesizer): void {
|
|
7
182
|
try {
|
|
@@ -84,6 +259,9 @@ export async function synthesizeSsml(ssml: string, config: TtsConfig): Promise<S
|
|
|
84
259
|
const addSourceMetadata = <T extends { audioOffsetMs: number }>(event: T): T => ({
|
|
85
260
|
...event,
|
|
86
261
|
...(config.sourceTextRange ? { textRange: { ...config.sourceTextRange } } : {}),
|
|
262
|
+
...(config.sourceTextRange ? { originalTextRange: { ...config.sourceTextRange } } : {}),
|
|
263
|
+
...(config.chunkIndex !== undefined ? { chunkIndex: config.chunkIndex } : {}),
|
|
264
|
+
...(config.sourceNodePath ? { sourceNodePath: [...config.sourceNodePath] } : {}),
|
|
87
265
|
...(requestId ? { requestId } : {}),
|
|
88
266
|
});
|
|
89
267
|
const sourceBoundaries = boundaries.map((boundary) => addSourceMetadata(boundary));
|
|
@@ -127,66 +305,134 @@ export async function synthesizeSsmlChunks(
|
|
|
127
305
|
): Promise<SsmlSynthesisResult> {
|
|
128
306
|
const results: SsmlSynthesisResult[] = [];
|
|
129
307
|
const totalChunks = chunks.length;
|
|
308
|
+
const report = (event: SynthesisProgressEvent): void => config.onProgress?.(event);
|
|
130
309
|
for (const [index, chunk] of chunks.entries()) {
|
|
131
310
|
const input = typeof chunk === "string" ? { ssml: chunk } : chunk;
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
311
|
+
report({
|
|
312
|
+
currentChunk: index,
|
|
313
|
+
totalChunks,
|
|
314
|
+
percent: totalChunks === 0 ? 100 : Math.round((index / totalChunks) * 100),
|
|
315
|
+
chunkIndex: index,
|
|
316
|
+
originalTextRange: input.originalTextRange,
|
|
317
|
+
status: "pending",
|
|
318
|
+
durationMs: 0,
|
|
136
319
|
});
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
320
|
+
}
|
|
321
|
+
for (const [index, chunk] of chunks.entries()) {
|
|
322
|
+
const input = typeof chunk === "string" ? { ssml: chunk } : chunk;
|
|
323
|
+
report({
|
|
324
|
+
currentChunk: index,
|
|
140
325
|
totalChunks,
|
|
141
|
-
percent: totalChunks === 0 ? 100 : Math.round((
|
|
326
|
+
percent: totalChunks === 0 ? 100 : Math.round((index / totalChunks) * 100),
|
|
327
|
+
chunkIndex: index,
|
|
328
|
+
originalTextRange: input.originalTextRange,
|
|
329
|
+
status: "synthesizing",
|
|
330
|
+
durationMs: 0,
|
|
142
331
|
});
|
|
332
|
+
const startedAt = Date.now();
|
|
333
|
+
try {
|
|
334
|
+
const result = await synthesizeSsml(input.ssml, {
|
|
335
|
+
...config,
|
|
336
|
+
...(input.originalTextRange ? { sourceTextRange: input.originalTextRange } : {}),
|
|
337
|
+
...(input.sourceNodePath ? { sourceNodePath: input.sourceNodePath } : {}),
|
|
338
|
+
chunkIndex: index,
|
|
339
|
+
onProgress: undefined,
|
|
340
|
+
});
|
|
341
|
+
results.push(result);
|
|
342
|
+
report({
|
|
343
|
+
currentChunk: index + 1,
|
|
344
|
+
totalChunks,
|
|
345
|
+
percent: totalChunks === 0 ? 100 : Math.round(((index + 1) / totalChunks) * 100),
|
|
346
|
+
chunkIndex: index,
|
|
347
|
+
originalTextRange: input.originalTextRange,
|
|
348
|
+
status: "success",
|
|
349
|
+
durationMs: Date.now() - startedAt,
|
|
350
|
+
});
|
|
351
|
+
} catch (error) {
|
|
352
|
+
report({
|
|
353
|
+
currentChunk: index,
|
|
354
|
+
totalChunks,
|
|
355
|
+
percent: totalChunks === 0 ? 100 : Math.round((index / totalChunks) * 100),
|
|
356
|
+
chunkIndex: index,
|
|
357
|
+
originalTextRange: input.originalTextRange,
|
|
358
|
+
status: "failed",
|
|
359
|
+
durationMs: Date.now() - startedAt,
|
|
360
|
+
error,
|
|
361
|
+
});
|
|
362
|
+
throw error;
|
|
363
|
+
}
|
|
143
364
|
}
|
|
144
|
-
return mergeSynthesisResults(results);
|
|
365
|
+
return mergeSynthesisResults(results, config.outputFormat ?? "audio-16khz-128kbitrate-mono-mp3");
|
|
145
366
|
}
|
|
146
367
|
|
|
147
368
|
/** Concatenates audio buffers and shifts all synchronization events by prior chunk durations. */
|
|
148
|
-
export function mergeSynthesisResults(results: readonly SsmlSynthesisResult[]): SsmlSynthesisResult {
|
|
149
|
-
const
|
|
150
|
-
|
|
369
|
+
export function mergeSynthesisResults(results: readonly SsmlSynthesisResult[], format?: string): SsmlSynthesisResult {
|
|
370
|
+
const audioData = format
|
|
371
|
+
? new Uint8Array(
|
|
372
|
+
mergeAudioBuffers(
|
|
373
|
+
results.map((result) => result.audioData),
|
|
374
|
+
format,
|
|
375
|
+
),
|
|
376
|
+
)
|
|
377
|
+
: new Uint8Array(results.reduce((total, result) => total + result.audioData.byteLength, 0));
|
|
378
|
+
if (!format) {
|
|
379
|
+
let offset = 0;
|
|
380
|
+
for (const result of results) {
|
|
381
|
+
audioData.set(new Uint8Array(result.audioData), offset);
|
|
382
|
+
offset += result.audioData.byteLength;
|
|
383
|
+
}
|
|
384
|
+
}
|
|
151
385
|
const boundaries: NonNullable<SsmlSynthesisResult["boundaries"]> = [];
|
|
152
386
|
const visemes: NonNullable<SsmlSynthesisResult["visemes"]> = [];
|
|
153
387
|
const bookmarks: NonNullable<SsmlSynthesisResult["bookmarks"]> = [];
|
|
154
|
-
let byteOffset = 0;
|
|
155
388
|
let durationOffset = 0;
|
|
156
389
|
|
|
157
390
|
for (const result of results) {
|
|
158
|
-
audioData.set(new Uint8Array(result.audioData), byteOffset);
|
|
159
|
-
byteOffset += result.audioData.byteLength;
|
|
160
391
|
const chunkBoundaries =
|
|
161
392
|
result.boundaries && result.boundaries.length > 0
|
|
162
393
|
? result.boundaries
|
|
163
394
|
: (result.wordBoundary ?? result.wordBoundaries ?? []);
|
|
164
395
|
for (const boundary of chunkBoundaries) {
|
|
165
396
|
const textRange = boundary.textRange ?? result.textRange;
|
|
397
|
+
const originalTextRange = boundary.originalTextRange ?? textRange;
|
|
166
398
|
const requestId = boundary.requestId ?? result.requestId;
|
|
167
399
|
boundaries.push({
|
|
168
400
|
...boundary,
|
|
169
401
|
audioOffsetMs: boundary.audioOffsetMs + durationOffset,
|
|
402
|
+
chunkAudioOffsetMs: boundary.chunkAudioOffsetMs ?? boundary.audioOffsetMs,
|
|
403
|
+
...(boundary.chunkIndex === undefined ? { chunkIndex: results.indexOf(result) } : {}),
|
|
404
|
+
...(boundary.sourceNodePath ? { sourceNodePath: [...boundary.sourceNodePath] } : {}),
|
|
405
|
+
...(originalTextRange ? { originalTextRange: { ...originalTextRange } } : {}),
|
|
170
406
|
...(textRange ? { textRange: { ...textRange } } : {}),
|
|
171
407
|
...(requestId ? { requestId } : {}),
|
|
172
408
|
});
|
|
173
409
|
}
|
|
174
410
|
for (const viseme of result.visemes ?? []) {
|
|
175
411
|
const textRange = viseme.textRange ?? result.textRange;
|
|
412
|
+
const originalTextRange = viseme.originalTextRange ?? textRange;
|
|
176
413
|
const requestId = viseme.requestId ?? result.requestId;
|
|
177
414
|
visemes.push({
|
|
178
415
|
...viseme,
|
|
179
416
|
audioOffsetMs: viseme.audioOffsetMs + durationOffset,
|
|
417
|
+
chunkAudioOffsetMs: viseme.chunkAudioOffsetMs ?? viseme.audioOffsetMs,
|
|
418
|
+
...(viseme.chunkIndex === undefined ? { chunkIndex: results.indexOf(result) } : {}),
|
|
419
|
+
...(viseme.sourceNodePath ? { sourceNodePath: [...viseme.sourceNodePath] } : {}),
|
|
420
|
+
...(originalTextRange ? { originalTextRange: { ...originalTextRange } } : {}),
|
|
180
421
|
...(textRange ? { textRange: { ...textRange } } : {}),
|
|
181
422
|
...(requestId ? { requestId } : {}),
|
|
182
423
|
});
|
|
183
424
|
}
|
|
184
425
|
for (const bookmark of result.bookmarks ?? []) {
|
|
185
426
|
const textRange = bookmark.textRange ?? result.textRange;
|
|
427
|
+
const originalTextRange = bookmark.originalTextRange ?? textRange;
|
|
186
428
|
const requestId = bookmark.requestId ?? result.requestId;
|
|
187
429
|
bookmarks.push({
|
|
188
430
|
...bookmark,
|
|
189
431
|
audioOffsetMs: bookmark.audioOffsetMs + durationOffset,
|
|
432
|
+
chunkAudioOffsetMs: bookmark.chunkAudioOffsetMs ?? bookmark.audioOffsetMs,
|
|
433
|
+
...(bookmark.chunkIndex === undefined ? { chunkIndex: results.indexOf(result) } : {}),
|
|
434
|
+
...(bookmark.sourceNodePath ? { sourceNodePath: [...bookmark.sourceNodePath] } : {}),
|
|
435
|
+
...(originalTextRange ? { originalTextRange: { ...originalTextRange } } : {}),
|
|
190
436
|
...(textRange ? { textRange: { ...textRange } } : {}),
|
|
191
437
|
...(requestId ? { requestId } : {}),
|
|
192
438
|
});
|
package/src/types.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { SsmlTextRange } from "@ssml-builder-js/ssml-core";
|
|
2
|
+
|
|
1
3
|
export interface TtsConfig {
|
|
2
4
|
signal?: AbortSignal;
|
|
3
5
|
timeoutMs?: number;
|
|
@@ -7,15 +9,28 @@ export interface TtsConfig {
|
|
|
7
9
|
outputFormat?: string;
|
|
8
10
|
/** Original plain-text range represented by this synthesis request. */
|
|
9
11
|
sourceTextRange?: { start: number; end: number };
|
|
10
|
-
/** Reports
|
|
11
|
-
onProgress?: (event:
|
|
12
|
+
/** Reports chunk lifecycle events when using chunk synthesis. */
|
|
13
|
+
onProgress?: (event: SynthesisProgressEvent) => void;
|
|
14
|
+
/** Metadata used to map synchronization events back to the source document. */
|
|
15
|
+
chunkIndex?: number;
|
|
16
|
+
sourceNodePath?: string[];
|
|
12
17
|
}
|
|
13
18
|
|
|
19
|
+
export type SynthesisChunkStatus = "pending" | "synthesizing" | "success" | "failed";
|
|
20
|
+
|
|
14
21
|
export interface SsmlSynthesisBoundary {
|
|
15
22
|
text: string;
|
|
16
23
|
audioOffsetMs: number;
|
|
17
24
|
durationMs: number;
|
|
18
25
|
textRange?: { start: number; end: number };
|
|
26
|
+
/** Chunk that produced this event. */
|
|
27
|
+
chunkIndex?: number;
|
|
28
|
+
/** Path of the source SSML node, when available. */
|
|
29
|
+
sourceNodePath?: string[];
|
|
30
|
+
/** Original text range represented by this event. */
|
|
31
|
+
originalTextRange?: SsmlTextRange;
|
|
32
|
+
/** Audio offset within the originating chunk before merge. */
|
|
33
|
+
chunkAudioOffsetMs?: number;
|
|
19
34
|
requestId?: string;
|
|
20
35
|
}
|
|
21
36
|
|
|
@@ -23,6 +38,10 @@ export interface SsmlSynthesisViseme {
|
|
|
23
38
|
visemeId: number;
|
|
24
39
|
audioOffsetMs: number;
|
|
25
40
|
textRange?: { start: number; end: number };
|
|
41
|
+
chunkIndex?: number;
|
|
42
|
+
sourceNodePath?: string[];
|
|
43
|
+
originalTextRange?: SsmlTextRange;
|
|
44
|
+
chunkAudioOffsetMs?: number;
|
|
26
45
|
requestId?: string;
|
|
27
46
|
}
|
|
28
47
|
|
|
@@ -30,6 +49,10 @@ export interface SsmlSynthesisBookmark {
|
|
|
30
49
|
name: string;
|
|
31
50
|
audioOffsetMs: number;
|
|
32
51
|
textRange?: { start: number; end: number };
|
|
52
|
+
chunkIndex?: number;
|
|
53
|
+
sourceNodePath?: string[];
|
|
54
|
+
originalTextRange?: SsmlTextRange;
|
|
55
|
+
chunkAudioOffsetMs?: number;
|
|
33
56
|
requestId?: string;
|
|
34
57
|
}
|
|
35
58
|
|
|
@@ -53,6 +76,7 @@ export interface SsmlSynthesisResult {
|
|
|
53
76
|
export interface SsmlSynthesisChunk {
|
|
54
77
|
ssml: string;
|
|
55
78
|
originalTextRange?: { start: number; end: number };
|
|
79
|
+
sourceNodePath?: string[];
|
|
56
80
|
}
|
|
57
81
|
|
|
58
82
|
export interface SynthesizeChunksOptions {
|
|
@@ -60,9 +84,15 @@ export interface SynthesizeChunksOptions {
|
|
|
60
84
|
}
|
|
61
85
|
|
|
62
86
|
export interface SynthesisProgressEvent {
|
|
87
|
+
/** 1-based completed chunk count retained for backward compatibility. */
|
|
63
88
|
currentChunk: number;
|
|
64
89
|
totalChunks: number;
|
|
65
90
|
percent: number;
|
|
91
|
+
chunkIndex: number;
|
|
92
|
+
originalTextRange?: SsmlTextRange;
|
|
93
|
+
status: SynthesisChunkStatus;
|
|
94
|
+
durationMs: number;
|
|
95
|
+
error?: unknown;
|
|
66
96
|
}
|
|
67
97
|
|
|
68
98
|
export interface AzureTtsLogger {
|
package/src/voiceCatalog.ts
CHANGED
|
@@ -10,6 +10,9 @@ export interface AzureVoiceCatalogVoice {
|
|
|
10
10
|
locale: string;
|
|
11
11
|
secondaryLocales?: readonly string[];
|
|
12
12
|
styles?: readonly string[];
|
|
13
|
+
supportedTags?: readonly string[];
|
|
14
|
+
unsupportedTags?: readonly string[];
|
|
15
|
+
models?: readonly string[];
|
|
13
16
|
regions: readonly string[];
|
|
14
17
|
status?: "ga" | "preview" | "deprecated";
|
|
15
18
|
}
|
|
@@ -33,6 +36,9 @@ interface AzureVoiceApiRecord {
|
|
|
33
36
|
ShortName?: unknown;
|
|
34
37
|
Status?: unknown;
|
|
35
38
|
StyleList?: unknown;
|
|
39
|
+
SupportedTags?: unknown;
|
|
40
|
+
UnsupportedTags?: unknown;
|
|
41
|
+
Models?: unknown;
|
|
36
42
|
}
|
|
37
43
|
|
|
38
44
|
function stringValue(value: unknown): string | undefined {
|
|
@@ -92,6 +98,9 @@ export async function fetchAzureVoiceCatalog(options: FetchAzureVoiceCatalogOpti
|
|
|
92
98
|
const secondaryLocales = stringList(record.SecondaryLocaleList);
|
|
93
99
|
const styles = stringList(record.StyleList);
|
|
94
100
|
const status = normalizeStatus(record.Status);
|
|
101
|
+
const supportedTags = stringList(record.SupportedTags);
|
|
102
|
+
const unsupportedTags = stringList(record.UnsupportedTags);
|
|
103
|
+
const models = stringList(record.Models);
|
|
95
104
|
const merged: AzureVoiceCatalogVoice = {
|
|
96
105
|
name: existing?.name ?? name,
|
|
97
106
|
locale: existing?.locale ?? locale,
|
|
@@ -101,6 +110,12 @@ export async function fetchAzureVoiceCatalog(options: FetchAzureVoiceCatalogOpti
|
|
|
101
110
|
if (mergedSecondaryLocales.length > 0) merged.secondaryLocales = mergedSecondaryLocales;
|
|
102
111
|
const mergedStyles = [...new Set([...(existing?.styles ?? []), ...styles])];
|
|
103
112
|
if (mergedStyles.length > 0) merged.styles = mergedStyles;
|
|
113
|
+
const mergedSupportedTags = [...new Set([...(existing?.supportedTags ?? []), ...supportedTags])];
|
|
114
|
+
if (mergedSupportedTags.length > 0) merged.supportedTags = mergedSupportedTags;
|
|
115
|
+
const mergedUnsupportedTags = [...new Set([...(existing?.unsupportedTags ?? []), ...unsupportedTags])];
|
|
116
|
+
if (mergedUnsupportedTags.length > 0) merged.unsupportedTags = mergedUnsupportedTags;
|
|
117
|
+
const mergedModels = [...new Set([...(existing?.models ?? []), ...models])];
|
|
118
|
+
if (mergedModels.length > 0) merged.models = mergedModels;
|
|
104
119
|
if (status) merged.status = status;
|
|
105
120
|
else if (existing?.status) merged.status = existing.status;
|
|
106
121
|
voices.set(key, merged);
|
|
@@ -37,6 +37,10 @@ test("mergeSynthesisResults concatenates audio and offsets synchronization event
|
|
|
37
37
|
[50, 125],
|
|
38
38
|
);
|
|
39
39
|
assert.deepEqual(result.boundaries?.[1]?.textRange, { start: 3, end: 6 });
|
|
40
|
+
assert.equal(result.boundaries?.[0]?.chunkIndex, 0);
|
|
41
|
+
assert.equal(result.boundaries?.[1]?.chunkIndex, 1);
|
|
42
|
+
assert.equal(result.boundaries?.[1]?.chunkAudioOffsetMs, 10);
|
|
43
|
+
assert.deepEqual(result.boundaries?.[1]?.originalTextRange, { start: 3, end: 6 });
|
|
40
44
|
});
|
|
41
45
|
|
|
42
46
|
test("synthesizeSsmlSafe blocks invalid SSML without calling the client", async () => {
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import {
|
|
4
|
+
ChunkValidationError,
|
|
5
|
+
UnsupportedMergeFormatError,
|
|
6
|
+
mergeAudioBuffers,
|
|
7
|
+
synthesizeSsmlChunksSafe,
|
|
8
|
+
} from "../src/index.ts";
|
|
9
|
+
|
|
10
|
+
function wav(data: number[], sampleRate = 16_000): ArrayBuffer {
|
|
11
|
+
const pcm = Uint8Array.from(data);
|
|
12
|
+
const output = new Uint8Array(44 + pcm.length + (pcm.length & 1));
|
|
13
|
+
const view = new DataView(output.buffer);
|
|
14
|
+
output.set(new TextEncoder().encode("RIFF"), 0);
|
|
15
|
+
output.set(new TextEncoder().encode("WAVEfmt "), 8);
|
|
16
|
+
view.setUint32(4, output.length - 8, true);
|
|
17
|
+
view.setUint32(16, 16, true);
|
|
18
|
+
view.setUint16(20, 1, true);
|
|
19
|
+
view.setUint16(22, 1, true);
|
|
20
|
+
view.setUint32(24, sampleRate, true);
|
|
21
|
+
view.setUint32(28, sampleRate * 2, true);
|
|
22
|
+
view.setUint16(32, 2, true);
|
|
23
|
+
view.setUint16(34, 16, true);
|
|
24
|
+
output.set(new TextEncoder().encode("data"), 36);
|
|
25
|
+
view.setUint32(40, pcm.length, true);
|
|
26
|
+
output.set(pcm, 44);
|
|
27
|
+
return output.buffer;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const validSsml = (text: string) =>
|
|
31
|
+
`<speak version="1.0" xml:lang="en-US"><voice name="en-US-JennyNeural">${text}</voice></speak>`;
|
|
32
|
+
|
|
33
|
+
test("mergeAudioBuffers rebuilds one valid WAV header", () => {
|
|
34
|
+
const merged = mergeAudioBuffers([wav([1, 2]), wav([3, 4, 5])], "riff-16khz-16bit-mono-pcm");
|
|
35
|
+
const bytes = new Uint8Array(merged);
|
|
36
|
+
const view = new DataView(merged);
|
|
37
|
+
assert.equal(new TextDecoder().decode(bytes.slice(0, 4)), "RIFF");
|
|
38
|
+
assert.equal(new TextDecoder().decode(bytes.slice(8, 12)), "WAVE");
|
|
39
|
+
assert.equal(view.getUint32(4, true), merged.byteLength - 8);
|
|
40
|
+
assert.equal(view.getUint32(40, true), 5);
|
|
41
|
+
assert.deepEqual([...bytes.slice(44, 49)], [1, 2, 3, 4, 5]);
|
|
42
|
+
assert.equal(bytes[49], 0);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("mergeAudioBuffers removes per-buffer ID3 tags from MP3 streams", () => {
|
|
46
|
+
const tag = Uint8Array.from([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0, 0]);
|
|
47
|
+
const first = new Uint8Array(tag.length + 2);
|
|
48
|
+
first.set(tag);
|
|
49
|
+
first.set([1, 2], tag.length);
|
|
50
|
+
const second = new Uint8Array(tag.length + 1);
|
|
51
|
+
second.set(tag);
|
|
52
|
+
second[tag.length] = 3;
|
|
53
|
+
assert.deepEqual(
|
|
54
|
+
[...new Uint8Array(mergeAudioBuffers([first.buffer, second.buffer], "audio-16khz-128kbitrate-mono-mp3"))],
|
|
55
|
+
[1, 2, 3],
|
|
56
|
+
);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("mergeAudioBuffers rejects container formats that require remultiplexing", () => {
|
|
60
|
+
assert.throws(
|
|
61
|
+
() => mergeAudioBuffers([new ArrayBuffer(1)], "webm-24khz-16bit-mono-opus"),
|
|
62
|
+
(error: unknown) => error instanceof UnsupportedMergeFormatError,
|
|
63
|
+
);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("synthesizeSsmlChunksSafe validates all chunks before calling Azure", async () => {
|
|
67
|
+
let calls = 0;
|
|
68
|
+
const progress: string[] = [];
|
|
69
|
+
const result = await synthesizeSsmlChunksSafe(
|
|
70
|
+
{
|
|
71
|
+
synthesizeSsml: async () => {
|
|
72
|
+
calls += 1;
|
|
73
|
+
return { audioData: new ArrayBuffer(0), durationMs: 0 };
|
|
74
|
+
},
|
|
75
|
+
},
|
|
76
|
+
[validSsml("ok"), "<speak>"],
|
|
77
|
+
{ onProgress: (event) => progress.push(`${event.chunkIndex}:${event.status}`) },
|
|
78
|
+
);
|
|
79
|
+
assert.equal(result.ok, false);
|
|
80
|
+
assert.equal(result.status, "validation-error");
|
|
81
|
+
if (!result.ok) {
|
|
82
|
+
assert.ok(result.error instanceof ChunkValidationError);
|
|
83
|
+
assert.equal(result.error.chunkIndex, 1);
|
|
84
|
+
}
|
|
85
|
+
assert.equal(calls, 0);
|
|
86
|
+
assert.deepEqual(progress, ["0:pending", "1:pending", "1:failed"]);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("synthesizeSsmlChunksSafe reports structured lifecycle progress", async () => {
|
|
90
|
+
const progress: Array<{ chunkIndex: number; status: string; durationMs: number }> = [];
|
|
91
|
+
const result = await synthesizeSsmlChunksSafe(
|
|
92
|
+
{
|
|
93
|
+
synthesizeSsml: async () => ({ audioData: Uint8Array.of(1).buffer, durationMs: 20 }),
|
|
94
|
+
},
|
|
95
|
+
[
|
|
96
|
+
{ ssml: validSsml("one"), originalTextRange: { start: 0, end: 3 }, sourceNodePath: ["speak", "voice[0]"] },
|
|
97
|
+
{ ssml: validSsml("two"), originalTextRange: { start: 3, end: 6 }, sourceNodePath: ["speak", "voice[0]"] },
|
|
98
|
+
],
|
|
99
|
+
{
|
|
100
|
+
onProgress: (event) =>
|
|
101
|
+
progress.push({ chunkIndex: event.chunkIndex, status: event.status, durationMs: event.durationMs }),
|
|
102
|
+
},
|
|
103
|
+
);
|
|
104
|
+
assert.equal(result.ok, true);
|
|
105
|
+
assert.deepEqual(
|
|
106
|
+
progress.map(({ chunkIndex, status }) => `${chunkIndex}:${status}`),
|
|
107
|
+
["0:pending", "1:pending", "0:synthesizing", "0:success", "1:synthesizing", "1:success"],
|
|
108
|
+
);
|
|
109
|
+
assert.ok(progress.every(({ durationMs }) => durationMs >= 0));
|
|
110
|
+
});
|