echogarden 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alignment/SpeechAlignment.d.ts +7 -1
- package/dist/alignment/SpeechAlignment.js +66 -55
- package/dist/alignment/SpeechAlignment.js.map +1 -1
- package/dist/api/Alignment.js +10 -6
- package/dist/api/Alignment.js.map +1 -1
- package/docs/Development.md +0 -2
- package/docs/Options.md +1 -1
- package/docs/Tasklist.md +2 -1
- package/package.json +1 -1
- package/src/alignment/SpeechAlignment.ts +88 -64
- package/src/api/Alignment.ts +11 -12
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import * as API from '../api/API.js';
|
|
2
2
|
import { Timeline, TimelineEntry } from '../utilities/Timeline.js';
|
|
3
3
|
import { RawAudio } from '../audio/AudioUtilities.js';
|
|
4
|
+
import { EspeakOptions } from '../synthesis/EspeakTTS.js';
|
|
4
5
|
export declare function alignUsingDtw(sourceRawAudio: RawAudio, referenceRawAudio: RawAudio, referenceTimeline: Timeline, granularities: DtwGranularity[], windowDurations: number[]): Promise<TimelineEntry[]>;
|
|
5
|
-
export declare function
|
|
6
|
+
export declare function alignUsingDtwWithRecognitionReference(sourceRawAudio: RawAudio, referenceRawAudio: RawAudio, referenceTimeline: Timeline, recognitionTimeline: Timeline, granularities: DtwGranularity[], windowDurations: number[], espeakOptions: EspeakOptions, phoneAlignmentMethod?: API.PhoneAlignmentMethod): Promise<Timeline>;
|
|
6
7
|
export declare function interpolatePhoneTimelines(sourceTimeline: Timeline, referenceTimeline: Timeline): Promise<Timeline>;
|
|
7
8
|
export declare function alignPhoneTimelines(sourceRawAudio: RawAudio, sourceWordTimeline: Timeline, referenceRawAudio: RawAudio, referenceTimeline: Timeline, windowDuration: number): Promise<Timeline>;
|
|
9
|
+
export declare function createAlignmentReferenceUsingEspeakForFragments(fragments: string[], espeakOptions: EspeakOptions, insertSeparators?: boolean): Promise<{
|
|
10
|
+
rawAudio: RawAudio;
|
|
11
|
+
timeline: Timeline;
|
|
12
|
+
events: import("../synthesis/EspeakTTS.js").EspeakEvent[];
|
|
13
|
+
}>;
|
|
8
14
|
export declare function createAlignmentReferenceUsingEspeak(transcript: string, language: string, plaintextOptions?: API.PlainTextOptions, customLexiconPaths?: string[], insertSeparators?: boolean): Promise<{
|
|
9
15
|
referenceRawAudio: RawAudio;
|
|
10
16
|
referenceTimeline: TimelineEntry[];
|
|
@@ -96,7 +96,7 @@ export async function alignUsingDtw(sourceRawAudio, referenceRawAudio, reference
|
|
|
96
96
|
logger.end();
|
|
97
97
|
return mappedTimeline;
|
|
98
98
|
}
|
|
99
|
-
export async function
|
|
99
|
+
export async function alignUsingDtwWithRecognitionReference(sourceRawAudio, referenceRawAudio, referenceTimeline, recognitionTimeline, granularities, windowDurations, espeakOptions, phoneAlignmentMethod = 'interpolation') {
|
|
100
100
|
const logger = new Logger();
|
|
101
101
|
if (recognitionTimeline.length == 0) {
|
|
102
102
|
const sourceDuration = getRawAudioDuration(sourceRawAudio);
|
|
@@ -113,6 +113,10 @@ export async function alignUsingDtwWithRecognition(sourceRawAudio, referenceRawA
|
|
|
113
113
|
}
|
|
114
114
|
return interpolatedTimeline;
|
|
115
115
|
}
|
|
116
|
+
// Synthesize the recognized transcript and get its timeline
|
|
117
|
+
logger.start("Synthesize recognized transcript with eSpeak");
|
|
118
|
+
const recognizedWords = recognitionTimeline.map(entry => entry.text);
|
|
119
|
+
const { rawAudio: synthesizedRecognizedTranscriptRawAudio, timeline: synthesizedRecognitionTimeline } = await createAlignmentReferenceUsingEspeakForFragments(recognizedWords, espeakOptions);
|
|
116
120
|
let recognitionTimelineWithPhones;
|
|
117
121
|
if (phoneAlignmentMethod == 'interpolation') {
|
|
118
122
|
// Add phone timelines by interpolating from reference words
|
|
@@ -129,17 +133,19 @@ export async function alignUsingDtwWithRecognition(sourceRawAudio, referenceRawA
|
|
|
129
133
|
throw new Error(`Unknown phone alignment method: ${phoneAlignmentMethod}`);
|
|
130
134
|
}
|
|
131
135
|
// Create a mapping from the synthesized recognized timeline to the recognized timeline
|
|
132
|
-
logger.start(
|
|
136
|
+
logger.start("Map from the synthesized recognized timeline to the recognized timeline");
|
|
133
137
|
const synthesizedToRecognizedTimeMapping = [];
|
|
134
138
|
for (let wordEntryIndex = 0; wordEntryIndex < synthesizedRecognitionTimeline.length; wordEntryIndex++) {
|
|
135
|
-
const
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
139
|
+
const synthesizedTimelineEntry = synthesizedRecognitionTimeline[wordEntryIndex];
|
|
140
|
+
const recognitionTimelineEntry = recognitionTimelineWithPhones[wordEntryIndex];
|
|
141
|
+
synthesizedToRecognizedTimeMapping.push({
|
|
142
|
+
synthesized: synthesizedTimelineEntry.startTime,
|
|
143
|
+
recognized: recognitionTimelineEntry.startTime
|
|
144
|
+
});
|
|
145
|
+
if (synthesizedTimelineEntry.timeline) {
|
|
146
|
+
for (let tokenEntryIndex = 0; tokenEntryIndex < synthesizedTimelineEntry.timeline.length; tokenEntryIndex++) {
|
|
147
|
+
const synthesizedPhoneTimelineEntry = synthesizedTimelineEntry.timeline[tokenEntryIndex];
|
|
148
|
+
const recognitionPhoneTimelineEntry = recognitionTimelineEntry.timeline[tokenEntryIndex];
|
|
143
149
|
synthesizedToRecognizedTimeMapping.push({
|
|
144
150
|
synthesized: synthesizedPhoneTimelineEntry.startTime,
|
|
145
151
|
recognized: recognitionPhoneTimelineEntry.startTime
|
|
@@ -150,37 +156,42 @@ export async function alignUsingDtwWithRecognition(sourceRawAudio, referenceRawA
|
|
|
150
156
|
});
|
|
151
157
|
}
|
|
152
158
|
}
|
|
159
|
+
synthesizedToRecognizedTimeMapping.push({
|
|
160
|
+
synthesized: synthesizedTimelineEntry.endTime,
|
|
161
|
+
recognized: recognitionTimelineEntry.endTime
|
|
162
|
+
});
|
|
153
163
|
}
|
|
154
|
-
logger.start('Align the synthesized recognized transcript with the synthesized ground-truth transcript');
|
|
155
164
|
// Align the synthesized recognized transcript to the synthesized reference transcript
|
|
165
|
+
logger.start("Align the synthesized recognized transcript with the synthesized ground-truth transcript");
|
|
156
166
|
const alignedSynthesizedRecognitionTimeline = await alignUsingDtw(synthesizedRecognizedTranscriptRawAudio, referenceRawAudio, referenceTimeline, granularities, windowDurations);
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
}
|
|
167
|
+
let currentSynthesizedToRecognizedMappingIndex = 0;
|
|
168
|
+
function mapSynthesizedToRecognizedTimeAndAdvance(synthesizedTime) {
|
|
169
|
+
for (;; currentSynthesizedToRecognizedMappingIndex += 1) {
|
|
170
|
+
const left = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].synthesized;
|
|
171
|
+
let right;
|
|
172
|
+
if (currentSynthesizedToRecognizedMappingIndex < synthesizedToRecognizedTimeMapping.length - 1) {
|
|
173
|
+
right = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex + 1].synthesized;
|
|
174
|
+
}
|
|
175
|
+
else {
|
|
176
|
+
right = Infinity;
|
|
177
|
+
}
|
|
178
|
+
if (left > right) {
|
|
179
|
+
throw new Error("Left is larger than right!");
|
|
180
|
+
}
|
|
181
|
+
if (Math.abs(synthesizedTime - left) < Math.abs(synthesizedTime - right)) {
|
|
182
|
+
return synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].recognized;
|
|
174
183
|
}
|
|
175
184
|
}
|
|
185
|
+
}
|
|
186
|
+
function mapTimeline(timeline) {
|
|
176
187
|
const mappedTimeline = [];
|
|
177
188
|
for (const entry of timeline) {
|
|
178
|
-
const mappedEntry =
|
|
179
|
-
mappedEntry.startTime =
|
|
180
|
-
mappedEntry.endTime = mapSynthesizedToRecognizedTime(entry.endTime);
|
|
189
|
+
const mappedEntry = { ...entry };
|
|
190
|
+
mappedEntry.startTime = mapSynthesizedToRecognizedTimeAndAdvance(entry.startTime);
|
|
181
191
|
if (entry.timeline) {
|
|
182
192
|
mappedEntry.timeline = mapTimeline(entry.timeline);
|
|
183
193
|
}
|
|
194
|
+
mappedEntry.endTime = mapSynthesizedToRecognizedTimeAndAdvance(entry.endTime);
|
|
184
195
|
mappedTimeline.push(mappedEntry);
|
|
185
196
|
}
|
|
186
197
|
return mappedTimeline;
|
|
@@ -191,9 +202,9 @@ export async function alignUsingDtwWithRecognition(sourceRawAudio, referenceRawA
|
|
|
191
202
|
}
|
|
192
203
|
export async function interpolatePhoneTimelines(sourceTimeline, referenceTimeline) {
|
|
193
204
|
const interpolatedTimeline = [];
|
|
194
|
-
for (let
|
|
195
|
-
const referenceEntry = referenceTimeline[
|
|
196
|
-
const interpolatedEntry = deepClone(sourceTimeline[
|
|
205
|
+
for (let wordEntryIndex = 0; wordEntryIndex < sourceTimeline.length; wordEntryIndex++) {
|
|
206
|
+
const referenceEntry = referenceTimeline[wordEntryIndex];
|
|
207
|
+
const interpolatedEntry = deepClone(sourceTimeline[wordEntryIndex]);
|
|
197
208
|
interpolatedTimeline.push(interpolatedEntry);
|
|
198
209
|
if (interpolatedEntry.type != 'word') {
|
|
199
210
|
continue;
|
|
@@ -211,17 +222,11 @@ export async function interpolatePhoneTimelines(sourceTimeline, referenceTimelin
|
|
|
211
222
|
endTime: interpolatedEndTime
|
|
212
223
|
};
|
|
213
224
|
}
|
|
214
|
-
const
|
|
215
|
-
for (const
|
|
216
|
-
|
|
217
|
-
const interpolatedPhoneEntries = [];
|
|
218
|
-
for (const phoneEntry of (tokenEntry.timeline || [])) {
|
|
219
|
-
interpolatedPhoneEntries.push(mapEntry(phoneEntry));
|
|
220
|
-
}
|
|
221
|
-
interpolatedTokenEntry.timeline = interpolatedPhoneEntries;
|
|
222
|
-
interpolatedTokenEntries.push(interpolatedTokenEntry);
|
|
225
|
+
const interpolatedPhoneEntries = [];
|
|
226
|
+
for (const phoneEntry of (referenceEntry.timeline || [])) {
|
|
227
|
+
interpolatedPhoneEntries.push(mapEntry(phoneEntry));
|
|
223
228
|
}
|
|
224
|
-
interpolatedEntry.timeline =
|
|
229
|
+
interpolatedEntry.timeline = interpolatedPhoneEntries;
|
|
225
230
|
}
|
|
226
231
|
return interpolatedTimeline;
|
|
227
232
|
}
|
|
@@ -268,22 +273,28 @@ export async function alignPhoneTimelines(sourceRawAudio, sourceWordTimeline, re
|
|
|
268
273
|
};
|
|
269
274
|
}
|
|
270
275
|
// Add phone timeline using the mapped time information
|
|
271
|
-
const
|
|
272
|
-
for (const
|
|
273
|
-
|
|
274
|
-
for (const referencePhoneEntry of (referenceTokenEntry.timeline || [])) {
|
|
275
|
-
alignedPhoneTimeline.push(mapEntry(referencePhoneEntry));
|
|
276
|
-
}
|
|
277
|
-
alignedTokenTimeline.push({
|
|
278
|
-
...mapEntry(referenceTokenEntry),
|
|
279
|
-
timeline: alignedPhoneTimeline
|
|
280
|
-
});
|
|
276
|
+
const alignedPhoneTimeline = [];
|
|
277
|
+
for (const referencePhoneEntry of (referenceWordEntry.timeline || [])) {
|
|
278
|
+
alignedPhoneTimeline.push(mapEntry(referencePhoneEntry));
|
|
281
279
|
}
|
|
282
|
-
alignedWordEntry.timeline =
|
|
280
|
+
alignedWordEntry.timeline = alignedPhoneTimeline;
|
|
283
281
|
alignedWordTimeline.push(alignedWordEntry);
|
|
284
282
|
}
|
|
285
283
|
return alignedWordTimeline;
|
|
286
284
|
}
|
|
285
|
+
export async function createAlignmentReferenceUsingEspeakForFragments(fragments, espeakOptions, insertSeparators = true) {
|
|
286
|
+
const progressLogger = new Logger();
|
|
287
|
+
progressLogger.start("Load espeak module");
|
|
288
|
+
const Espeak = await import("../synthesis/EspeakTTS.js");
|
|
289
|
+
progressLogger.start("Create alignment reference with eSpeak");
|
|
290
|
+
const result = await Espeak.synthesizeFragments(fragments, espeakOptions);
|
|
291
|
+
result.timeline = result.timeline.flatMap(clause => clause.timeline);
|
|
292
|
+
for (const wordEntry of result.timeline) {
|
|
293
|
+
wordEntry.timeline = wordEntry.timeline.flatMap(tokenEntry => tokenEntry.timeline);
|
|
294
|
+
}
|
|
295
|
+
progressLogger.end();
|
|
296
|
+
return result;
|
|
297
|
+
}
|
|
287
298
|
export async function createAlignmentReferenceUsingEspeak(transcript, language, plaintextOptions, customLexiconPaths, insertSeparators) {
|
|
288
299
|
const logger = new Logger();
|
|
289
300
|
logger.start('Create alignment reference with eSpeak');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SpeechAlignment.js","sourceRoot":"","sources":["../../src/alignment/SpeechAlignment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,2BAA2B,CAAA;AAIhD,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAe,MAAM,gBAAgB,CAAA;AACpF,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AACxF,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,4BAA4B,EAAY,MAAM,4BAA4B,CAAA;AAC/J,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAA;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAEjD,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,cAAwB,EACxB,iBAA2B,EAC3B,iBAA2B,EAC3B,aAA+B,EAC/B,eAAyB;IAEzB,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,IAAI,eAAe,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;IACxD,CAAC;IAED,IAAI,eAAe,CAAC,MAAM,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAA;IAC/E,CAAC;IAED,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;IAE5D,IAAI,eAAuB,CAAA;IAC3B,IAAI,aAA4B,CAAA;IAChC,IAAI,eAAqC,CAAA;IAEzC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACzE,MAAM,cAAc,GAAG,eAAe,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,WAAW,GAAG,8BAA8B,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC,CAAA;QAE9F,MAAM,CAAC,gBAAgB,CAAC,6BAA6B,SAAS,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,EAAE,EAAE,wBAAwB,cAAc,mBAAmB,WAAW,EAAE,EAAE,KAAK,CAAC,aAAa,CAAC,CAAA;QAE5L,MAAM,WAAW,GAAG,wBAAwB,CAAC,EAAE,GAAG,4BAA4B,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAgB,CAAA;QAE3J,eAAe,GAAG,CAAC,GAAG,WAAW,CAAC,WAAY,CAAA;QAE9C,0BAA0B;QAC1B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAA;QAC/C,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAA;QAEzE,uBAAuB;QACvB,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;QACnE,MAAM,CAAC,GAAG,EAAE,CAAA;QAEZ,eAAe;QACf,MAAM,CAAC,gBAAgB,CAAC,6BAA6B,EAAE,GAAG,yBAAyB,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEhL,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,4BAA4B,GAAG,GAAG,GAAG,gBAAgB,CAAA;YAE3D,IAAI,cAAc,GAAG,4BAA4B,EAAE,CAAC;gBACnD,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,yCAAyC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,gEAAgE,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,sGAAsG,EAAE,KAAK,CAAC,YAAY,CAAC,CAAA;YAC5T,CAAC;QACF,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;QAC7C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,eAAe,CAAC,CAAA;QAEpE,IAAI,aAAmC,CAAA;QAEvC,IAAI,eAAe,EAAE,CAAC;YACrB,aAAa,GAAG,EAAE,CAAA;YAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,yBAAyB,GAAG,CAAC,GAAG,cAAc,CAAC,MAAM,CAAA;gBAE3D,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,eAAgB,CAAC,MAAM,CAAC,CAAA;gBAC3F,MAAM,cAAc,GAAG,eAAe,CAAC,mBAAmB,CAAC,CAAA;gBAC3D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,CAAA;gBAEnE,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,CAAC,CAAA;QAE3G,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;QAEpC,eAAe,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAA;QAEjG,MAAM,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAA;IAE1C,SAAS,sBAAsB,CAAC,aAA4B,EAAE,OAAO,GAAG,IAAI;QAC3E,MAAM,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,GAAG,eAAe,CAAC,CAAA;QACtF,MAAM,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,GAAG,eAAe,CAAC,CAAA;QAElF,IAAI,wBAAwB,GAAG,CAAC,IAAI,sBAAsB,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;QAC5E,CAAC;QAED,MAAM,qBAAqB,GAAG,0BAA0B,CAAC,wBAAwB,EAAE,aAAa,EAAE,OAAO,CAAC,CAAA;QAC1G,MAAM,mBAAmB,GAAG,0BAA0B,CAAC,sBAAsB,EAAE,aAAa,EAAE,OAAO,CAAC,CAAA;QAEtG,IAAI,aAAmC,CAAA;QAEvC,IAAI,OAAO,IAAI,aAAa,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;YAC/C,aAAa,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAA;QACrF,CAAC;QAED,+DAA+D;QAC/D,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,GAAG,eAAe,CAAC,CAAA;QAErF,IAAI,gBAAgB,GAAG,qBAAqB,GAAG,qBAAqB,CAAA;QACpE,IAAI,cAAc,GAAG,mBAAmB,GAAG,qBAAqB,CAAA;QAEhE,MAAM,YAAY,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAA;QAE/F,MAAM,wBAAwB,GAAG,CAAC,EAAE,CAAA;QAEpC,gBAAgB,IAAI,4BAA4B,CAAC,YAAY,EAAE,wBAAwB,CAAC,CAAA;QACxF,cAAc,IAAI,0BAA0B,CAAC,YAAY,EAAE,wBAAwB,CAAC,CAAA;QAEpF,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAA;QAE3D,8BAA8B;QAC9B,MAAM,SAAS,GAAG,gBAAgB,GAAG,cAAc,CAAC,UAAU,CAAA;QAC9D,MAAM,OAAO,GAAG,cAAc,GAAG,cAAc,CAAC,UAAU,CAAA;QAE1D,OAAO;YACN,IAAI,EAAE,aAAa,CAAC,IAAI;YACxB,IAAI,EAAE,aAAa,CAAC,IAAI;YAExB,SAAS;YACT,OAAO;YAEP,QAAQ,EAAE,aAAa;SACvB,CAAA;IACF,CAAC;IAED,MAAM,cAAc,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC,CAAA;IAEtG,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,cAAc,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CACjD,cAAwB,EACxB,iBAA2B,EAC3B,iBAA2B,EAC3B,mBAA6B,EAC7B,uCAAiD,EACjD,8BAAwC,EACxC,aAA+B,EAC/B,eAAyB,EACzB,uBAAiD,eAAe;IAEhE,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,IAAI,mBAAmB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,cAAc,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;QAC1D,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAC,CAAA;QAChE,MAAM,KAAK,GAAG,cAAc,GAAG,iBAAiB,CAAA;QAEhD,MAAM,oBAAoB,GAAa,EAAE,CAAA;QAEzC,KAAK,MAAM,KAAK,IAAI,iBAAiB,EAAE,CAAC;YACvC,oBAAoB,CAAC,IAAI,CAAC;gBACzB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK;gBAClC,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,KAAK;aAC9B,CAAC,CAAA;QACH,CAAC;QAED,OAAO,oBAAoB,CAAA;IAC5B,CAAC;IAED,IAAI,6BAAuC,CAAA;IAE3C,IAAI,oBAAoB,IAAI,eAAe,EAAE,CAAC;QAC7C,4DAA4D;QAC5D,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAExC,6BAA6B,GAAG,MAAM,yBAAyB,CAAC,mBAAmB,EAAE,8BAA8B,CAAC,CAAA;IACrH,CAAC;SAAM,IAAI,oBAAoB,IAAI,KAAK,EAAE,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAA;QAElC,8FAA8F;QAC9F,4BAA4B;QAC5B,6BAA6B,GAAG,MAAM,mBAAmB,CACxD,cAAc,EACd,mBAAmB,EACnB,uCAAuC,EACvC,8BAA8B,EAC9B,EAAE,CAAC,CAAA;IACL,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,mCAAmC,oBAAoB,EAAE,CAAC,CAAA;IAC3E,CAAC;IAED,uFAAuF;IACvF,MAAM,CAAC,KAAK,CAAC,yEAAyE,CAAC,CAAA;IASvF,MAAM,kCAAkC,GAAuC,EAAE,CAAA;IAEjF,KAAK,IAAI,cAAc,GAAG,CAAC,EAAE,cAAc,GAAG,8BAA8B,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,CAAC;QACvG,MAAM,4BAA4B,GAAG,8BAA8B,CAAC,cAAc,CAAC,CAAA;QACnF,MAAM,4BAA4B,GAAG,6BAA6B,CAAC,cAAc,CAAC,CAAA;QAElF,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,4BAA4B,CAAC,QAAS,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE,CAAC;YAClH,MAAM,6BAA6B,GAAG,4BAA4B,CAAC,QAAS,CAAC,eAAe,CAAC,CAAA;YAC7F,MAAM,6BAA6B,GAAG,4BAA4B,CAAC,QAAS,CAAC,eAAe,CAAC,CAAA;YAE7F,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,6BAA6B,CAAC,QAAS,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE,CAAC;gBACnH,MAAM,6BAA6B,GAAG,6BAA6B,CAAC,QAAS,CAAC,eAAe,CAAC,CAAA;gBAC9F,MAAM,6BAA6B,GAAG,6BAA6B,CAAC,QAAS,CAAC,eAAe,CAAC,CAAA;gBAE9F,kCAAkC,CAAC,IAAI,CAAC;oBACvC,WAAW,EAAE,6BAA6B,CAAC,SAAS;oBACpD,UAAU,EAAE,6BAA6B,CAAC,SAAS;iBACnD,CAAC,CAAA;gBAEF,kCAAkC,CAAC,IAAI,CAAC;oBACvC,WAAW,EAAE,6BAA6B,CAAC,OAAO;oBAClD,UAAU,EAAE,6BAA6B,CAAC,OAAO;iBACjD,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,0FAA0F,CAAC,CAAA;IACxG,sFAAsF;IACtF,MAAM,qCAAqC,GAAG,MAAM,aAAa,CAChE,uCAAuC,EACvC,iBAAiB,EACjB,iBAAiB,EACjB,aAAa,EACb,eAAe,CAAC,CAAA;IAEjB,SAAS,WAAW,CAAC,QAAkB;QACtC,SAAS,8BAA8B,CAAC,eAAuB;YAC9D,KAAK,IAAI,YAAY,GAAG,CAAC,GAAI,YAAY,IAAI,CAAC,EAAE,CAAC;gBAChD,MAAM,IAAI,GAAG,kCAAkC,CAAC,YAAY,CAAC,CAAC,WAAW,CAAA;gBAEzE,IAAI,KAAa,CAAA;gBAEjB,IAAI,YAAY,GAAG,kCAAkC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClE,KAAK,GAAG,kCAAkC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,WAAW,CAAA;gBACzE,CAAC;qBAAM,CAAC;oBACP,KAAK,GAAG,QAAQ,CAAA;gBACjB,CAAC;gBAED,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;gBAC/C,CAAC;gBAED,IAAI,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,KAAK,CAAC,EAAE,CAAC;oBAC1E,OAAO,kCAAkC,CAAC,YAAY,CAAC,CAAC,UAAU,CAAA;gBACnE,CAAC;YACF,CAAC;QACF,CAAC;QAED,MAAM,cAAc,GAAa,EAAE,CAAA;QAEnC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC9B,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,CAAC,CAAA;YAEpC,WAAW,CAAC,SAAS,GAAG,8BAA8B,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YACvE,WAAW,CAAC,OAAO,GAAG,8BAA8B,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YAEnE,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACpB,WAAW,CAAC,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YACnD,CAAC;YAED,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;QACjC,CAAC;QAED,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,qCAAqC,CAAC,CAAA;IAEjE,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,cAAwB,EAAE,iBAA2B;IACpG,MAAM,oBAAoB,GAAa,EAAE,CAAA;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,MAAM,cAAc,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAA;QAE3C,MAAM,iBAAiB,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAA;QACtD,oBAAoB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QAE5C,IAAI,iBAAiB,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;YACtC,SAAQ;QACT,CAAC;QAED,MAAM,yBAAyB,GAAG,iBAAiB,CAAC,OAAO,GAAG,iBAAiB,CAAC,SAAS,CAAA;QACzF,MAAM,sBAAsB,GAAG,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,SAAS,CAAA;QAEhF,SAAS,QAAQ,CAAC,WAA0B;YAC3C,MAAM,uCAAuC,GAC5C,CAAC,WAAW,CAAC,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,sBAAsB,CAAA;YAE5E,MAAM,qCAAqC,GAC1C,CAAC,WAAW,CAAC,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,sBAAsB,CAAA;YAE1E,MAAM,qBAAqB,GAC1B,iBAAiB,CAAC,SAAS,GAAG,CAAC,SAAS,CAAC,uCAAuC,CAAC,GAAG,yBAAyB,CAAC,CAAA;YAE/G,MAAM,mBAAmB,GACxB,iBAAiB,CAAC,SAAS,GAAG,CAAC,SAAS,CAAC,qCAAqC,CAAC,GAAG,yBAAyB,CAAC,CAAA;YAE7G,OAAO;gBACN,GAAG,WAAW;gBAEd,SAAS,EAAE,qBAAqB;gBAChC,OAAO,EAAE,mBAAmB;aAC5B,CAAA;QACF,CAAC;QAED,MAAM,wBAAwB,GAAa,EAAE,CAAA;QAE7C,KAAK,MAAM,UAAU,IAAI,CAAC,cAAc,CAAC,QAAQ,IAAI,EAAE,CAAE,EAAE,CAAC;YAC3D,MAAM,sBAAsB,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAA;YAEnD,MAAM,wBAAwB,GAAa,EAAE,CAAA;YAE7C,KAAK,MAAM,UAAU,IAAI,CAAC,UAAU,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC;gBACtD,wBAAwB,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAA;YACpD,CAAC;YAED,sBAAsB,CAAC,QAAQ,GAAG,wBAAwB,CAAA;YAE1D,wBAAwB,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAA;QACtD,CAAC;QAED,iBAAiB,CAAC,QAAQ,GAAG,wBAAwB,CAAA;IACtD,CAAC;IAED,OAAO,oBAAoB,CAAA;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,cAAwB,EACxB,kBAA4B,EAC5B,iBAA2B,EAC3B,iBAA2B,EAC3B,cAAsB;IAEtB,MAAM,WAAW,GAAgB,wBAAwB,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAA;IAEzF,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,CAAC,WAAY,CAAA;IAEpD,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAA;IACzE,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;IAEnE,MAAM,mBAAmB,GAAa,EAAE,CAAA;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnD,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAA;QAE/C,MAAM,gBAAgB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAA;QAEzD,IAAI,gBAAgB,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;YACrC,SAAQ;QACT,CAAC;QAED,MAAM,4BAA4B,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,SAAS,GAAG,eAAe,CAAC,CAAA;QAC/F,IAAI,0BAA0B,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,OAAO,GAAG,eAAe,CAAC,CAAA;QAEzF,8CAA8C;QAC9C,IAAI,0BAA0B,IAAI,4BAA4B,EAAE,CAAC;YAChE,0BAA0B,GAAG,0BAA0B,GAAG,CAAC,CAAA;QAC5D,CAAC;QAED,MAAM,kBAAkB,GAAG,cAAc,CAAC,KAAK,CAAC,4BAA4B,EAAE,0BAA0B,CAAC,CAAA;QAEzG,MAAM,0BAA0B,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,CAAA;QAC3F,IAAI,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,GAAG,eAAe,CAAC,CAAA;QAErF,8CAA8C;QAC9C,IAAI,wBAAwB,IAAI,0BAA0B,EAAE,CAAC;YAC5D,wBAAwB,GAAG,0BAA0B,GAAG,CAAC,CAAA;QAC1D,CAAC;QAED,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,0BAA0B,EAAE,wBAAwB,CAAC,CAAA;QAE/F,mBAAmB;QACnB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,kBAAkB,EAAE,eAAe,EAAE,cAAc,GAAG,eAAe,CAAC,CAAA;QAC1G,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;QAE1C,SAAS,QAAQ,CAAC,cAA6B;YAC9C,MAAM,yBAAyB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,SAAS,GAAG,kBAAkB,CAAC,SAAS,CAAC,GAAG,eAAe,CAAC,CAAA;YACzH,MAAM,uBAAuB,GAAG,0BAA0B,CAAC,yBAAyB,EAAE,aAAa,CAAC,CAAA;YACpG,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,SAAS,GAAG,CAAC,uBAAuB,GAAG,eAAe,CAAC,CAAA;YAEjG,MAAM,uBAAuB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,OAAO,GAAG,kBAAkB,CAAC,SAAS,CAAC,GAAG,eAAe,CAAC,CAAA;YACrH,MAAM,qBAAqB,GAAG,0BAA0B,CAAC,uBAAuB,EAAE,aAAa,CAAC,CAAA;YAChG,MAAM,cAAc,GAAG,gBAAgB,CAAC,SAAS,GAAG,CAAC,qBAAqB,GAAG,eAAe,CAAC,CAAA;YAE7F,OAAO;gBACN,GAAG,cAAc;gBAEjB,SAAS,EAAE,gBAAgB;gBAC3B,OAAO,EAAE,cAAc;aACvB,CAAA;QACF,CAAC;QAED,uDAAuD;QACvD,MAAM,oBAAoB,GAAa,EAAE,CAAA;QAEzC,KAAK,MAAM,mBAAmB,IAAI,CAAC,kBAAkB,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC;YACvE,MAAM,oBAAoB,GAAa,EAAE,CAAA;YAEzC,KAAK,MAAM,mBAAmB,IAAI,CAAC,mBAAmB,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC;gBACxE,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAA;YACzD,CAAC;YAED,oBAAoB,CAAC,IAAI,CAAC;gBACzB,GAAG,QAAQ,CAAC,mBAAmB,CAAC;gBAEhC,QAAQ,EAAE,oBAAoB;aAC9B,CAAC,CAAA;QACH,CAAC;QAED,gBAAgB,CAAC,QAAQ,GAAG,oBAAoB,CAAA;QAEhD,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAC3C,CAAC;IAED,OAAO,mBAAmB,CAAA;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,UAAkB,EAAE,QAAgB,EAAE,gBAAuC,EAAE,kBAA6B,EAAE,gBAA0B;IACjM,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAEtD,MAAM,gBAAgB,GAAyB;QAC9C,MAAM,EAAE,QAAQ;QAChB,QAAQ;QAER,SAAS,EAAE,gBAAgB;QAC3B,kBAAkB,EAAE,kBAAkB;QAEtC,MAAM,EAAE;YACP,QAAQ,EAAE,KAAK;YACf,gBAAgB;SAChB;KACD,CAAA;IAED,IAAI,EACH,KAAK,EAAE,iBAAiB,EACxB,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,WAAW,EAClB,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAA;IAElD,MAAM,gBAAgB,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAS,CAAC,CAAA;IAC1E,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAS,CAAC,CAAA;IAEvE,iBAAiB,GAAG,MAAM,kBAAkB,CAAC,iBAA6B,EAAE,KAAK,CAAC,CAAA;IAClF,iBAAiB,GAAG,yBAAyB,CAAC,iBAAiB,CAAC,CAAA;IAEhE,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,YAAY,EAAE,WAAW,EAAE,CAAA;AAC3E,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACvC,MAAM,aAAa,GAAkB,EAAE,CAAA;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAEzB,IAAI,aAAa,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;YAC9C,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;QACpE,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAA;AACrB,CAAC;AAED,SAAS,0BAA0B,CAAC,mBAA2B,EAAE,aAA4B,EAAE,cAAgC,OAAO;IACrI,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,CAAA;IACT,CAAC;IAED,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAE5E,MAAM,kBAAkB,GAAG,aAAa,CAAC,mBAAmB,CAAC,CAAA;IAE7D,IAAI,gBAAwB,CAAA;IAE5B,IAAI,WAAW,IAAI,OAAO,EAAE,CAAC;QAC5B,gBAAgB,GAAG,kBAAkB,CAAC,KAAK,CAAA;IAC5C,CAAC;SAAM,CAAC;QACP,gBAAgB,GAAG,kBAAkB,CAAC,IAAI,CAAA;IAC3C,CAAC;IAED,OAAO,gBAAgB,CAAA;AACxB,CAAC;AAED,SAAS,8BAA8B,CAAC,WAA2B,EAAE,aAAqB;IACzF,IAAI,WAAW,IAAI,MAAM,EAAE,CAAC;QAC3B,OAAO,WAAW,CAAA;IACnB,CAAC;IAED,IAAI,aAAa,GAAG,EAAE,EAAE,CAAC;QACxB,OAAO,MAAM,CAAA;IACd,CAAC;SAAM,IAAI,aAAa,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;QACpC,OAAO,QAAQ,CAAA;IAChB,CAAC;SAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACb,CAAC;AACF,CAAC;AAED,SAAS,4BAA4B,CAAC,WAA2B,EAAE,aAAqB;IACvF,IAAI,WAAwB,CAAA;IAE5B,WAAW,GAAG,8BAA8B,CAAC,WAAW,EAAE,aAAa,CAAC,CAAA;IAExE,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC;QAC7B,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IAC5E,CAAC;SAAM,IAAI,WAAW,IAAI,OAAO,EAAE,CAAC;QACnC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IAC5E,CAAC;SAAM,IAAI,WAAW,IAAI,KAAK,EAAE,CAAC;QACjC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IAC5E,CAAC;SAAM,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC;QACpC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IAC5E,CAAC;SAAM,IAAI,WAAW,IAAI,MAAM,EAAE,CAAC;QAClC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAA;IAC3E,CAAC;SAAM,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC;QACpC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAA;IAC3E,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,GAAG,CAAC,CAAA;IACjE,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC"}
|
|
1
|
+
{"version":3,"file":"SpeechAlignment.js","sourceRoot":"","sources":["../../src/alignment/SpeechAlignment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,2BAA2B,CAAA;AAIhD,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAe,MAAM,gBAAgB,CAAA;AACpF,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAA;AACxF,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,4BAA4B,EAAY,MAAM,4BAA4B,CAAA;AAC/J,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAA;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAGjD,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,cAAwB,EACxB,iBAA2B,EAC3B,iBAA2B,EAC3B,aAA+B,EAC/B,eAAyB;IAEzB,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,IAAI,eAAe,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;IACxD,CAAC;IAED,IAAI,eAAe,CAAC,MAAM,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAA;IAC/E,CAAC;IAED,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;IAE5D,IAAI,eAAuB,CAAA;IAC3B,IAAI,aAA4B,CAAA;IAChC,IAAI,eAAqC,CAAA;IAEzC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACzE,MAAM,cAAc,GAAG,eAAe,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,WAAW,GAAG,8BAA8B,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC,CAAA;QAE9F,MAAM,CAAC,gBAAgB,CAAC,6BAA6B,SAAS,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,EAAE,EAAE,wBAAwB,cAAc,mBAAmB,WAAW,EAAE,EAAE,KAAK,CAAC,aAAa,CAAC,CAAA;QAE5L,MAAM,WAAW,GAAG,wBAAwB,CAAC,EAAE,GAAG,4BAA4B,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAgB,CAAA;QAE3J,eAAe,GAAG,CAAC,GAAG,WAAW,CAAC,WAAY,CAAA;QAE9C,0BAA0B;QAC1B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAA;QAC/C,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAA;QAEzE,uBAAuB;QACvB,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;QACnE,MAAM,CAAC,GAAG,EAAE,CAAA;QAEZ,eAAe;QACf,MAAM,CAAC,gBAAgB,CAAC,6BAA6B,EAAE,GAAG,yBAAyB,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEhL,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,4BAA4B,GAAG,GAAG,GAAG,gBAAgB,CAAA;YAE3D,IAAI,cAAc,GAAG,4BAA4B,EAAE,CAAC;gBACnD,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,yCAAyC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,gEAAgE,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,sGAAsG,EAAE,KAAK,CAAC,YAAY,CAAC,CAAA;YAC5T,CAAC;QACF,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;QAC7C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,eAAe,CAAC,CAAA;QAEpE,IAAI,aAAmC,CAAA;QAEvC,IAAI,eAAe,EAAE,CAAC;YACrB,aAAa,GAAG,EAAE,CAAA;YAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,yBAAyB,GAAG,CAAC,GAAG,cAAc,CAAC,MAAM,CAAA;gBAE3D,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,eAAgB,CAAC,MAAM,CAAC,CAAA;gBAC3F,MAAM,cAAc,GAAG,eAAe,CAAC,mBAAmB,CAAC,CAAA;gBAC3D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,CAAA;gBAEnE,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,CAAC,CAAA;QAE3G,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;QAEpC,eAAe,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAA;QAEjG,MAAM,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAA;IAE1C,SAAS,sBAAsB,CAAC,aAA4B,EAAE,OAAO,GAAG,IAAI;QAC3E,MAAM,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,GAAG,eAAe,CAAC,CAAA;QACtF,MAAM,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,GAAG,eAAe,CAAC,CAAA;QAElF,IAAI,wBAAwB,GAAG,CAAC,IAAI,sBAAsB,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;QAC5E,CAAC;QAED,MAAM,qBAAqB,GAAG,0BAA0B,CAAC,wBAAwB,EAAE,aAAa,EAAE,OAAO,CAAC,CAAA;QAC1G,MAAM,mBAAmB,GAAG,0BAA0B,CAAC,sBAAsB,EAAE,aAAa,EAAE,OAAO,CAAC,CAAA;QAEtG,IAAI,aAAmC,CAAA;QAEvC,IAAI,OAAO,IAAI,aAAa,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;YAC/C,aAAa,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAA;QACrF,CAAC;QAED,+DAA+D;QAC/D,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,GAAG,eAAe,CAAC,CAAA;QAErF,IAAI,gBAAgB,GAAG,qBAAqB,GAAG,qBAAqB,CAAA;QACpE,IAAI,cAAc,GAAG,mBAAmB,GAAG,qBAAqB,CAAA;QAEhE,MAAM,YAAY,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAA;QAE/F,MAAM,wBAAwB,GAAG,CAAC,EAAE,CAAA;QAEpC,gBAAgB,IAAI,4BAA4B,CAAC,YAAY,EAAE,wBAAwB,CAAC,CAAA;QACxF,cAAc,IAAI,0BAA0B,CAAC,YAAY,EAAE,wBAAwB,CAAC,CAAA;QAEpF,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAA;QAE3D,8BAA8B;QAC9B,MAAM,SAAS,GAAG,gBAAgB,GAAG,cAAc,CAAC,UAAU,CAAA;QAC9D,MAAM,OAAO,GAAG,cAAc,GAAG,cAAc,CAAC,UAAU,CAAA;QAE1D,OAAO;YACN,IAAI,EAAE,aAAa,CAAC,IAAI;YACxB,IAAI,EAAE,aAAa,CAAC,IAAI;YAExB,SAAS;YACT,OAAO;YAEP,QAAQ,EAAE,aAAa;SACvB,CAAA;IACF,CAAC;IAED,MAAM,cAAc,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC,CAAA;IAEtG,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,cAAc,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qCAAqC,CAC1D,cAAwB,EACxB,iBAA2B,EAC3B,iBAA2B,EAC3B,mBAA6B,EAC7B,aAA+B,EAC/B,eAAyB,EACzB,aAA4B,EAC5B,uBAAiD,eAAe;IAEhE,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,IAAI,mBAAmB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,cAAc,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;QAC1D,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAC,CAAA;QAChE,MAAM,KAAK,GAAG,cAAc,GAAG,iBAAiB,CAAA;QAEhD,MAAM,oBAAoB,GAAa,EAAE,CAAA;QAEzC,KAAK,MAAM,KAAK,IAAI,iBAAiB,EAAE,CAAC;YACvC,oBAAoB,CAAC,IAAI,CAAC;gBACzB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK;gBAClC,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,KAAK;aAC9B,CAAC,CAAA;QACH,CAAC;QAED,OAAO,oBAAoB,CAAA;IAC5B,CAAC;IAED,4DAA4D;IAC5D,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAA;IAC5D,MAAM,eAAe,GAAG,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEpE,MAAM,EACL,QAAQ,EAAE,uCAAuC,EACjD,QAAQ,EAAE,8BAA8B,EACxC,GAAG,MAAM,+CAA+C,CAAC,eAAe,EAAE,aAAa,CAAC,CAAA;IAEzF,IAAI,6BAAuC,CAAA;IAE3C,IAAI,oBAAoB,IAAI,eAAe,EAAE,CAAC;QAC7C,4DAA4D;QAC5D,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAExC,6BAA6B,GAAG,MAAM,yBAAyB,CAC9D,mBAAmB,EACnB,8BAA8B,CAC9B,CAAA;IACF,CAAC;SAAM,IAAI,oBAAoB,IAAI,KAAK,EAAE,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAA;QAElC,8FAA8F;QAC9F,4BAA4B;QAC5B,6BAA6B,GAAG,MAAM,mBAAmB,CACxD,cAAc,EACd,mBAAmB,EACnB,uCAAuC,EACvC,8BAA8B,EAC9B,EAAE,CAAC,CAAA;IACL,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,mCAAmC,oBAAoB,EAAE,CAAC,CAAA;IAC3E,CAAC;IAED,uFAAuF;IACvF,MAAM,CAAC,KAAK,CAAC,yEAAyE,CAAC,CAAA;IASvF,MAAM,kCAAkC,GAAuC,EAAE,CAAA;IAEjF,KAAK,IAAI,cAAc,GAAG,CAAC,EAAE,cAAc,GAAG,8BAA8B,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,CAAC;QACvG,MAAM,wBAAwB,GAAG,8BAA8B,CAAC,cAAc,CAAC,CAAA;QAC/E,MAAM,wBAAwB,GAAG,6BAA6B,CAAC,cAAc,CAAC,CAAA;QAE9E,kCAAkC,CAAC,IAAI,CAAC;YACvC,WAAW,EAAE,wBAAwB,CAAC,SAAS;YAC/C,UAAU,EAAE,wBAAwB,CAAC,SAAS;SAC9C,CAAC,CAAA;QAEF,IAAI,wBAAwB,CAAC,QAAQ,EAAE,CAAC;YACvC,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,wBAAwB,CAAC,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE,CAAC;gBAC7G,MAAM,6BAA6B,GAAG,wBAAwB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;gBACxF,MAAM,6BAA6B,GAAG,wBAAwB,CAAC,QAAS,CAAC,eAAe,CAAC,CAAA;gBAEzF,kCAAkC,CAAC,IAAI,CAAC;oBACvC,WAAW,EAAE,6BAA6B,CAAC,SAAS;oBACpD,UAAU,EAAE,6BAA6B,CAAC,SAAS;iBACnD,CAAC,CAAA;gBAEF,kCAAkC,CAAC,IAAI,CAAC;oBACvC,WAAW,EAAE,6BAA6B,CAAC,OAAO;oBAClD,UAAU,EAAE,6BAA6B,CAAC,OAAO;iBACjD,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,kCAAkC,CAAC,IAAI,CAAC;YACvC,WAAW,EAAE,wBAAwB,CAAC,OAAO;YAC7C,UAAU,EAAE,wBAAwB,CAAC,OAAO;SAC5C,CAAC,CAAA;IACH,CAAC;IAED,sFAAsF;IACtF,MAAM,CAAC,KAAK,CAAC,0FAA0F,CAAC,CAAA;IAExG,MAAM,qCAAqC,GAAG,MAAM,aAAa,CAChE,uCAAuC,EACvC,iBAAiB,EACjB,iBAAiB,EACjB,aAAa,EACb,eAAe,CAAC,CAAA;IAEjB,IAAI,0CAA0C,GAAG,CAAC,CAAA;IAElD,SAAS,wCAAwC,CAAC,eAAuB;QACxE,QAAS,0CAA0C,IAAI,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,GAAG,kCAAkC,CAAC,0CAA0C,CAAC,CAAC,WAAW,CAAA;YAEvG,IAAI,KAAa,CAAA;YAEjB,IAAI,0CAA0C,GAAG,kCAAkC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChG,KAAK,GAAG,kCAAkC,CAAC,0CAA0C,GAAG,CAAC,CAAC,CAAC,WAAW,CAAA;YACvG,CAAC;iBAAM,CAAC;gBACP,KAAK,GAAG,QAAQ,CAAA;YACjB,CAAC;YAED,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;YAC9C,CAAC;YAED,IAAI,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,KAAK,CAAC,EAAE,CAAC;gBAC1E,OAAO,kCAAkC,CAAC,0CAA0C,CAAC,CAAC,UAAU,CAAA;YACjG,CAAC;QACF,CAAC;IACF,CAAC;IAED,SAAS,WAAW,CAAC,QAAkB;QACtC,MAAM,cAAc,GAAa,EAAE,CAAA;QAEnC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC9B,MAAM,WAAW,GAAG,EAAE,GAAG,KAAK,EAAE,CAAA;YAEhC,WAAW,CAAC,SAAS,GAAG,wCAAwC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAEjF,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACpB,WAAW,CAAC,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YACnD,CAAC;YAED,WAAW,CAAC,OAAO,GAAG,wCAAwC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YAE7E,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;QACjC,CAAC;QAED,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,qCAAqC,CAAC,CAAA;IAEjE,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,cAAwB,EAAE,iBAA2B;IACpG,MAAM,oBAAoB,GAAa,EAAE,CAAA;IAEzC,KAAK,IAAI,cAAc,GAAG,CAAC,EAAE,cAAc,GAAG,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,CAAC;QACvF,MAAM,cAAc,GAAG,iBAAiB,CAAC,cAAc,CAAC,CAAA;QAExD,MAAM,iBAAiB,GAAG,SAAS,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC,CAAA;QACnE,oBAAoB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QAE5C,IAAI,iBAAiB,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;YACtC,SAAQ;QACT,CAAC;QAED,MAAM,yBAAyB,GAAG,iBAAiB,CAAC,OAAO,GAAG,iBAAiB,CAAC,SAAS,CAAA;QACzF,MAAM,sBAAsB,GAAG,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,SAAS,CAAA;QAEhF,SAAS,QAAQ,CAAC,WAA0B;YAC3C,MAAM,uCAAuC,GAC5C,CAAC,WAAW,CAAC,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,sBAAsB,CAAA;YAE5E,MAAM,qCAAqC,GAC1C,CAAC,WAAW,CAAC,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,sBAAsB,CAAA;YAE1E,MAAM,qBAAqB,GAC1B,iBAAiB,CAAC,SAAS,GAAG,CAAC,SAAS,CAAC,uCAAuC,CAAC,GAAG,yBAAyB,CAAC,CAAA;YAE/G,MAAM,mBAAmB,GACxB,iBAAiB,CAAC,SAAS,GAAG,CAAC,SAAS,CAAC,qCAAqC,CAAC,GAAG,yBAAyB,CAAC,CAAA;YAE7G,OAAO;gBACN,GAAG,WAAW;gBAEd,SAAS,EAAE,qBAAqB;gBAChC,OAAO,EAAE,mBAAmB;aAC5B,CAAA;QACF,CAAC;QAED,MAAM,wBAAwB,GAAa,EAAE,CAAA;QAE7C,KAAK,MAAM,UAAU,IAAI,CAAC,cAAc,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1D,wBAAwB,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAA;QACpD,CAAC;QAED,iBAAiB,CAAC,QAAQ,GAAG,wBAAwB,CAAA;IACtD,CAAC;IAED,OAAO,oBAAoB,CAAA;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,cAAwB,EACxB,kBAA4B,EAC5B,iBAA2B,EAC3B,iBAA2B,EAC3B,cAAsB;IAEtB,MAAM,WAAW,GAAgB,wBAAwB,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAA;IAEzF,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,CAAC,WAAY,CAAA;IAEpD,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAA;IACzE,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;IAEnE,MAAM,mBAAmB,GAAa,EAAE,CAAA;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnD,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAA;QAE/C,MAAM,gBAAgB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAA;QAEzD,IAAI,gBAAgB,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;YACrC,SAAQ;QACT,CAAC;QAED,MAAM,4BAA4B,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,SAAS,GAAG,eAAe,CAAC,CAAA;QAC/F,IAAI,0BAA0B,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,OAAO,GAAG,eAAe,CAAC,CAAA;QAEzF,8CAA8C;QAC9C,IAAI,0BAA0B,IAAI,4BAA4B,EAAE,CAAC;YAChE,0BAA0B,GAAG,0BAA0B,GAAG,CAAC,CAAA;QAC5D,CAAC;QAED,MAAM,kBAAkB,GAAG,cAAc,CAAC,KAAK,CAAC,4BAA4B,EAAE,0BAA0B,CAAC,CAAA;QAEzG,MAAM,0BAA0B,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,CAAA;QAC3F,IAAI,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,GAAG,eAAe,CAAC,CAAA;QAErF,8CAA8C;QAC9C,IAAI,wBAAwB,IAAI,0BAA0B,EAAE,CAAC;YAC5D,wBAAwB,GAAG,0BAA0B,GAAG,CAAC,CAAA;QAC1D,CAAC;QAED,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,0BAA0B,EAAE,wBAAwB,CAAC,CAAA;QAE/F,mBAAmB;QACnB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,kBAAkB,EAAE,eAAe,EAAE,cAAc,GAAG,eAAe,CAAC,CAAA;QAC1G,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;QAE1C,SAAS,QAAQ,CAAC,cAA6B;YAC9C,MAAM,yBAAyB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,SAAS,GAAG,kBAAkB,CAAC,SAAS,CAAC,GAAG,eAAe,CAAC,CAAA;YACzH,MAAM,uBAAuB,GAAG,0BAA0B,CAAC,yBAAyB,EAAE,aAAa,CAAC,CAAA;YACpG,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,SAAS,GAAG,CAAC,uBAAuB,GAAG,eAAe,CAAC,CAAA;YAEjG,MAAM,uBAAuB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,OAAO,GAAG,kBAAkB,CAAC,SAAS,CAAC,GAAG,eAAe,CAAC,CAAA;YACrH,MAAM,qBAAqB,GAAG,0BAA0B,CAAC,uBAAuB,EAAE,aAAa,CAAC,CAAA;YAChG,MAAM,cAAc,GAAG,gBAAgB,CAAC,SAAS,GAAG,CAAC,qBAAqB,GAAG,eAAe,CAAC,CAAA;YAE7F,OAAO;gBACN,GAAG,cAAc;gBAEjB,SAAS,EAAE,gBAAgB;gBAC3B,OAAO,EAAE,cAAc;aACvB,CAAA;QACF,CAAC;QAED,uDAAuD;QACvD,MAAM,oBAAoB,GAAa,EAAE,CAAA;QAEzC,KAAK,MAAM,mBAAmB,IAAI,CAAC,kBAAkB,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC;YACvE,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAA;QACzD,CAAC;QAED,gBAAgB,CAAC,QAAQ,GAAG,oBAAoB,CAAA;QAEhD,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAC3C,CAAC;IAED,OAAO,mBAAmB,CAAA;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+CAA+C,CAAC,SAAmB,EAAE,aAA4B,EAAE,gBAAgB,GAAG,IAAI;IAC/I,MAAM,cAAc,GAAG,IAAI,MAAM,EAAE,CAAA;IAEnC,cAAc,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;IAExD,cAAc,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAE9D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;IAEzE,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAS,CAAC,CAAA;IAErE,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACzC,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,UAAU,CAAC,QAAS,CAAC,CAAA;IACrF,CAAC;IAED,cAAc,CAAC,GAAG,EAAE,CAAA;IAEpB,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,UAAkB,EAAE,QAAgB,EAAE,gBAAuC,EAAE,kBAA6B,EAAE,gBAA0B;IACjM,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAEtD,MAAM,gBAAgB,GAAyB;QAC9C,MAAM,EAAE,QAAQ;QAChB,QAAQ;QAER,SAAS,EAAE,gBAAgB;QAC3B,kBAAkB,EAAE,kBAAkB;QAEtC,MAAM,EAAE;YACP,QAAQ,EAAE,KAAK;YACf,gBAAgB;SAChB;KACD,CAAA;IAED,IAAI,EACH,KAAK,EAAE,iBAAiB,EACxB,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,WAAW,EAClB,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAA;IAElD,MAAM,gBAAgB,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAS,CAAC,CAAA;IAC1E,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAS,CAAC,CAAA;IAEvE,iBAAiB,GAAG,MAAM,kBAAkB,CAAC,iBAA6B,EAAE,KAAK,CAAC,CAAA;IAClF,iBAAiB,GAAG,yBAAyB,CAAC,iBAAiB,CAAC,CAAA;IAEhE,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,YAAY,EAAE,WAAW,EAAE,CAAA;AAC3E,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACvC,MAAM,aAAa,GAAkB,EAAE,CAAA;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAEzB,IAAI,aAAa,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;YAC9C,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;QACpE,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAA;AACrB,CAAC;AAED,SAAS,0BAA0B,CAAC,mBAA2B,EAAE,aAA4B,EAAE,cAAgC,OAAO;IACrI,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,CAAA;IACT,CAAC;IAED,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAE5E,MAAM,kBAAkB,GAAG,aAAa,CAAC,mBAAmB,CAAC,CAAA;IAE7D,IAAI,gBAAwB,CAAA;IAE5B,IAAI,WAAW,IAAI,OAAO,EAAE,CAAC;QAC5B,gBAAgB,GAAG,kBAAkB,CAAC,KAAK,CAAA;IAC5C,CAAC;SAAM,CAAC;QACP,gBAAgB,GAAG,kBAAkB,CAAC,IAAI,CAAA;IAC3C,CAAC;IAED,OAAO,gBAAgB,CAAA;AACxB,CAAC;AAED,SAAS,8BAA8B,CAAC,WAA2B,EAAE,aAAqB;IACzF,IAAI,WAAW,IAAI,MAAM,EAAE,CAAC;QAC3B,OAAO,WAAW,CAAA;IACnB,CAAC;IAED,IAAI,aAAa,GAAG,EAAE,EAAE,CAAC;QACxB,OAAO,MAAM,CAAA;IACd,CAAC;SAAM,IAAI,aAAa,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;QACpC,OAAO,QAAQ,CAAA;IAChB,CAAC;SAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACb,CAAC;AACF,CAAC;AAED,SAAS,4BAA4B,CAAC,WAA2B,EAAE,aAAqB;IACvF,IAAI,WAAwB,CAAA;IAE5B,WAAW,GAAG,8BAA8B,CAAC,WAAW,EAAE,aAAa,CAAC,CAAA;IAExE,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC;QAC7B,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IAC5E,CAAC;SAAM,IAAI,WAAW,IAAI,OAAO,EAAE,CAAC;QACnC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IAC5E,CAAC;SAAM,IAAI,WAAW,IAAI,KAAK,EAAE,CAAC;QACjC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IAC5E,CAAC;SAAM,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC;QACpC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IAC5E,CAAC;SAAM,IAAI,WAAW,IAAI,MAAM,EAAE,CAAC;QAClC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAA;IAC3E,CAAC;SAAM,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC;QACpC,WAAW,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAA;IAC3E,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,iCAAiC,WAAW,GAAG,CAAC,CAAA;IACjE,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC"}
|
package/dist/api/Alignment.js
CHANGED
|
@@ -8,6 +8,7 @@ import { formatLanguageCodeWithName, getDefaultDialectForLanguageCodeIfPossible,
|
|
|
8
8
|
import chalk from 'chalk';
|
|
9
9
|
import { createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js';
|
|
10
10
|
import { defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js';
|
|
11
|
+
import { defaultEspeakOptions } from '../synthesis/EspeakTTS.js';
|
|
11
12
|
const log = logToStderr;
|
|
12
13
|
export async function align(input, transcript, options) {
|
|
13
14
|
const logger = new Logger();
|
|
@@ -62,7 +63,7 @@ export async function align(input, transcript, options) {
|
|
|
62
63
|
}
|
|
63
64
|
language = getDefaultDialectForLanguageCodeIfPossible(language);
|
|
64
65
|
logger.start('Load alignment module');
|
|
65
|
-
const {
|
|
66
|
+
const { alignUsingDtwWithRecognitionReference, alignUsingDtw } = await import('../alignment/SpeechAlignment.js');
|
|
66
67
|
function getDtwWindowDurationsAndGranularities() {
|
|
67
68
|
let granularities;
|
|
68
69
|
let windowDurations;
|
|
@@ -113,15 +114,18 @@ export async function align(input, transcript, options) {
|
|
|
113
114
|
const { transcript: recognizedTranscript, wordTimeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions);
|
|
114
115
|
// Synthesize the ground-truth transcript and get its timeline
|
|
115
116
|
logger.start('Synthesize ground-truth transcript with eSpeak');
|
|
116
|
-
const { referenceRawAudio, referenceTimeline, } = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, true);
|
|
117
|
-
// Synthesize the recognized transcript and get its timeline
|
|
118
|
-
logger.start('Synthesize recognized transcript with eSpeak');
|
|
119
|
-
const { referenceRawAudio: synthesizedRecognizedTranscriptRawAudio, referenceTimeline: synthesizedRecognitionTimeline } = await createAlignmentReferenceUsingEspeak(recognizedTranscript, language, undefined, undefined, true);
|
|
117
|
+
const { referenceRawAudio, referenceTimeline, espeakVoice, } = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, true);
|
|
120
118
|
logger.end();
|
|
121
119
|
const { windowDurations, granularities } = getDtwWindowDurationsAndGranularities();
|
|
122
120
|
const phoneAlignmentMethod = options.dtw.phoneAlignmentMethod;
|
|
121
|
+
const espeakOptions = {
|
|
122
|
+
...defaultEspeakOptions,
|
|
123
|
+
voice: espeakVoice,
|
|
124
|
+
useKlatt: false,
|
|
125
|
+
insertSeparators: true
|
|
126
|
+
};
|
|
123
127
|
// Align the ground-truth transcript and the recognized transcript
|
|
124
|
-
mappedTimeline = await alignUsingDtwWithRecognitionReference(sourceRawAudio, referenceRawAudio, referenceTimeline, recognitionTimeline,
|
|
128
|
+
mappedTimeline = await alignUsingDtwWithRecognitionReference(sourceRawAudio, referenceRawAudio, referenceTimeline, recognitionTimeline, granularities, windowDurations, espeakOptions, phoneAlignmentMethod);
|
|
125
129
|
break;
|
|
126
130
|
}
|
|
127
131
|
case 'whisper': {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Alignment.js","sourceRoot":"","sources":["../../src/api/Alignment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAA8B,cAAc,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAC/I,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,EAAY,uBAAuB,EAAE,4BAA4B,EAAE,qCAAqC,EAAE,MAAM,0BAA0B,CAAA;AACjJ,OAAO,EAAE,0BAA0B,EAAE,0CAA0C,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAE5J,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,EAAkB,mCAAmC,EAAE,MAAM,iCAAiC,CAAA;AACrG,OAAO,EAAmB,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;
|
|
1
|
+
{"version":3,"file":"Alignment.js","sourceRoot":"","sources":["../../src/api/Alignment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAA8B,cAAc,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAC/I,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,EAAY,uBAAuB,EAAE,4BAA4B,EAAE,qCAAqC,EAAE,MAAM,0BAA0B,CAAA;AACjJ,OAAO,EAAE,0BAA0B,EAAE,0CAA0C,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAE5J,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,EAAkB,mCAAmC,EAAE,MAAM,iCAAiC,CAAA;AACrG,OAAO,EAAmB,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;AACvF,OAAO,EAAiB,oBAAoB,EAAE,MAAM,2BAA2B,CAAA;AAE/E,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,KAAuB,EAAE,UAAkB,EAAE,OAAyB;IACjG,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;IAE5C,OAAO,GAAG,UAAU,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAA;IAEtD,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAA;IAEjD,IAAI,cAAwB,CAAA;IAC5B,IAAI,gBAAsC,CAAA;IAC1C,IAAI,kBAAwC,CAAA;IAE5C,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACd,MAAM,CAAC,GAAG,EAAE,CAAC;QAEb,CAAC,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,gBAAiB,CAAC,CAAC,CAAA;QAExG,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEd,cAAc,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAClE,CAAC;SAAM,CAAC;QACP,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED,IAAI,oBAA0C,CAAA;IAE9C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACpD,CAAC,EAAE,QAAQ,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,GAAG,MAAM,GAAG,CAAC,mBAAmB,CAAC,cAAc,EAAE,OAAO,CAAC,GAAI,CAAC,CAAC,CAAA;QAEnI,MAAM,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAA;IAErC,cAAc,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;IACpD,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IAE/E,IAAI,OAAO,CAAC,GAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;QACzC,MAAM,mBAAmB,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;QAE/D,IAAI,mBAAmB,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,+CAA+C;YAClF,OAAO,CAAC,GAAI,CAAC,cAAc,GAAG,EAAE,CAAA;QACjC,CAAC;aAAM,IAAI,mBAAmB,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,uDAAuD;YAClG,OAAO,CAAC,GAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC,CAAA;QACnE,CAAC;aAAM,CAAC,CAAC,8CAA8C;YACtD,OAAO,CAAC,GAAI,CAAC,cAAc,GAAG,EAAE,GAAG,EAAE,CAAA;QACtC,CAAC;IACF,CAAC;IAED,IAAI,QAAgB,CAAA;IAEpB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAS,CAAC,CAAA;IACpD,CAAC;SAAM,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAA;QACzD,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,GAAG,CAAC,kBAAkB,CAAC,UAAU,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAA;QACtG,QAAQ,GAAG,gBAAgB,CAAA;QAE3B,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,CAAA;IAC3F,CAAC;IAED,QAAQ,GAAG,0CAA0C,CAAC,QAAQ,CAAC,CAAA;IAE/D,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAA;IAErC,MAAM,EAAE,qCAAqC,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAA;IAEhH,SAAS,qCAAqC;QAC7C,IAAI,aAA+B,CAAA;QACnC,IAAI,eAAyB,CAAA;QAE7B,IAAI,OAAO,OAAO,CAAC,GAAI,CAAC,WAAW,IAAI,QAAQ,EAAE,CAAC;YACjD,aAAa,GAAG,CAAC,OAAO,CAAC,GAAI,CAAC,WAAW,CAAC,CAAA;QAC3C,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,GAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACpD,aAAa,GAAG,OAAO,CAAC,GAAI,CAAC,WAAW,CAAA;QACzC,CAAC;aAAM,CAAC;YACP,aAAa,GAAG,CAAC,MAAM,CAAC,CAAA;QACzB,CAAC;QAED,IAAI,OAAO,OAAO,CAAC,GAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,CAAC;YACpD,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAC/B,eAAe,GAAG,CAAC,OAAO,CAAC,GAAI,CAAC,cAAc,CAAC,CAAA;YAChD,CAAC;iBAAM,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtC,eAAe,GAAG,CAAC,OAAO,CAAC,GAAI,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;YACpD,CAAC;iBAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,wJAAwJ,CAAC,CAAA;YAC1K,CAAC;QACF,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,GAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACvD,eAAe,GAAG,OAAO,CAAC,GAAI,CAAC,cAAc,CAAA;QAC9C,CAAC;aAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAC5C,CAAC;QAED,IAAI,aAAa,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,4DAA4D,aAAa,CAAC,MAAM,wCAAwC,eAAe,CAAC,MAAM,8DAA8D,CAAC,CAAA;QAC9N,CAAC;QAED,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,CAAA;IAC1C,CAAC;IAED,IAAI,cAAwB,CAAA;IAE5B,QAAQ,OAAO,CAAC,MAAM,EAAE,CAAC;QACxB,KAAK,KAAK,CAAC,CAAC,CAAC;YACZ,MAAM,EACL,iBAAiB,EACjB,iBAAiB,EACjB,GAAG,MAAM,mCAAmC,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAA;YAElH,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,qCAAqC,EAAE,CAAA;YAElF,cAAc,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,aAAa,EAAE,eAAe,CAAC,CAAA;YAE1H,MAAK;QACN,CAAC;QAED,KAAK,QAAQ,CAAC,CAAC,CAAC;YACf,MAAM,kBAAkB,GACvB,UAAU,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,WAAW,CAAC,CAAA;YAElE,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,yBAAyB;YACzB,MAAM,EAAE,UAAU,EAAE,oBAAoB,EAAE,YAAY,EAAE,mBAAmB,EAAE,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAA;YAEvI,8DAA8D;YAC9D,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAA;YAE9D,MAAM,EACL,iBAAiB,EACjB,iBAAiB,EACjB,WAAW,GACX,GAAG,MAAM,mCAAmC,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAA;YAExH,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,qCAAqC,EAAE,CAAA;YAElF,MAAM,oBAAoB,GAAG,OAAO,CAAC,GAAI,CAAC,oBAAqB,CAAA;YAE/D,MAAM,aAAa,GAAkB;gBACpC,GAAG,oBAAoB;gBACvB,KAAK,EAAE,WAAW;gBAClB,QAAQ,EAAE,KAAK;gBACf,gBAAgB,EAAE,IAAI;aACtB,CAAA;YAED,kEAAkE;YAClE,cAAc,GAAG,MAAM,qCAAqC,CAC3D,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EAEjB,mBAAmB,EAEnB,aAAa,EACb,eAAe,EACf,aAAa,EACb,oBAAoB,CAAC,CAAA;YAEtB,MAAK;QACN,CAAC;QAED,KAAK,SAAS,CAAC,CAAC,CAAC;YAChB,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAA;YAE/D,MAAM,cAAc,GAAG,OAAO,CAAC,OAAQ,CAAA;YAEvC,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAA;YAExD,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,uBAAuB,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;YAExG,IAAI,mBAAmB,CAAC,cAAc,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC9C,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAA;YACrG,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,cAAc,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAA;YAE3G,MAAK;QACN,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,6FAA6F;IAC7F,IAAI,oBAAoB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,GAAG,CAAC,iCAAiC,CAAC,cAAc,EAAE,oBAAoB,CAAC,CAAA;IAC5E,CAAC;IAED,mBAAmB;IACnB,4BAA4B,CAAC,cAAc,EAAE,UAAU,CAAC,CAAA;IAExD,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,qCAAqC,CAAC,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,eAAe,EAAE,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;IAEhL,MAAM,CAAC,GAAG,EAAE,CAAA;IACZ,MAAM,CAAC,WAAW,CAAC,sBAAsB,EAAE,cAAc,EAAE,KAAK,CAAC,aAAa,CAAC,CAAA;IAE/E,OAAO;QACN,QAAQ,EAAE,eAAe;QACzB,YAAY,EAAE,cAAc;QAE5B,UAAU;QACV,QAAQ;QAER,aAAa;QACb,gBAAgB;QAChB,kBAAkB;KAClB,CAAA;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,cAAwB,EAAE,eAAyB,EAAE,gBAAkC;IAC1H,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,KAAK,MAAM,YAAY,IAAI,eAAe,EAAE,CAAC;QAC5C,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAA;QAErC,MAAM,gBAAgB,GAAG,YAAY,CAAC,SAAS,CAAA;QAC/C,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAA;QAE3C,MAAM,uBAAuB,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;QACxF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;QAEpF,MAAM,mBAAmB,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,uBAAuB,EAAE,qBAAqB,CAAC,CAAA;QACjH,MAAM,eAAe,GAAa;YACjC,aAAa,EAAE,CAAC,mBAAmB,CAAC;YACpC,UAAU,EAAE,cAAc,CAAC,UAAU;SACrC,CAAA;QAED,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,MAAM,KAAK,CAAC,eAAe,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAA;QAEpG,MAAM,yBAAyB,GAAG,uBAAuB,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAA;QAE3F,QAAQ,CAAC,IAAI,CAAC,GAAG,yBAAyB,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAiDD,MAAM,CAAC,MAAM,uBAAuB,GAAqB;IACxD,MAAM,EAAE,KAAK;IAEb,QAAQ,EAAE,SAAS;IAEnB,OAAO,EAAE,KAAK;IAEd,IAAI,EAAE,IAAI;IAEV,kBAAkB,EAAE,SAAS;IAE7B,iBAAiB,EAAE,EAClB;IAED,SAAS,EAAE;QACV,eAAe,EAAE,QAAQ;QACzB,UAAU,EAAE,UAAU;KACtB;IAED,SAAS,EAAE,0BAA0B;IAErC,GAAG,EAAE;QACJ,WAAW,EAAE,MAAM;QACnB,cAAc,EAAE,SAAS;QACzB,oBAAoB,EAAE,KAAK;KAC3B;IAED,WAAW,EAAE;QACZ,OAAO,EAAE;YACR,WAAW,EAAE,IAAI;YACjB,iBAAiB,EAAE,CAAC;YACpB,oBAAoB,EAAE,GAAG;YACzB,gBAAgB,EAAE,GAAG;YACrB,eAAe,EAAE,IAAI;YACrB,kBAAkB,EAAE,IAAI;YACxB,IAAI,EAAE,SAAS;YACf,qBAAqB,EAAE,KAAK;SAC5B;KACD;IAED,GAAG,EAAE;QACJ,MAAM,EAAE,eAAe;KACvB;IAED,gBAAgB,EAAE,EACjB;IAED,OAAO,EAAE,EACR;CACD,CAAA;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAyB;IACrD;QACC,EAAE,EAAE,KAAK;QACT,IAAI,EAAE,sBAAsB;QAC5B,WAAW,EAAE,4GAA4G;QACzH,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,8CAA8C;QACpD,WAAW,EAAE,wJAAwJ;QACrK,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,SAAS;QACb,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,mJAAmJ;QAChK,IAAI,EAAE,OAAO;KACb;CACD,CAAA"}
|
package/docs/Development.md
CHANGED
|
@@ -22,8 +22,6 @@ Create the `.vscode` subdirectory at the project base directory
|
|
|
22
22
|
Add a `.vscode/tasks.json` file to run the TypeScript compiler in watch mode:
|
|
23
23
|
```json
|
|
24
24
|
{
|
|
25
|
-
// See http://go.microsoft.com/fwlink/?LinkId=733558
|
|
26
|
-
// for the documentation about the tasks.json format
|
|
27
25
|
"version": "2.0.0",
|
|
28
26
|
"command": "tsc.cmd",
|
|
29
27
|
"args": [
|
package/docs/Options.md
CHANGED
|
@@ -221,7 +221,7 @@ Applies to CLI operation: `align`, API method: `align`
|
|
|
221
221
|
|
|
222
222
|
**DTW-RA only**:
|
|
223
223
|
* `recognition`: prefix to provide recognition options when using `dtw-ra` method, for example: setting `recognition.engine = whisper` and `recognition.whisper.model = base.en`
|
|
224
|
-
* `dtw.phoneAlignmentMethod`: algorithm to use when aligning phones: can either be set to `dtw` or `
|
|
224
|
+
* `dtw.phoneAlignmentMethod`: algorithm to use when aligning phones: can either be set to `dtw` or `interpolation`. Defaults to `dtw`
|
|
225
225
|
|
|
226
226
|
**Whisper alignment only**:
|
|
227
227
|
* `whisper`: prefix to provide Whisper options when the `whisper` alignment engine is used (does not apply to `dtw-ra` when `whisper` engine is used, for that use `recognition.whisper` prefix instead).
|
package/docs/Tasklist.md
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
## Bugs
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
* In DTW-RA, recognition transcript including something like "Question 2.What does Juan", where "2.What" has a point in the middle, is breaking playback of the timeline.
|
|
6
6
|
|
|
7
|
+
### Synthesis
|
|
7
8
|
|
|
8
9
|
### Phoneme processing
|
|
9
10
|
* IPA -> Kirshenbaum translation is still not completely similar to what is output by eSpeak. Also, in rare situations, it outputs characters that are not accepted by eSpeak and eSpeak errors. Investigate when that happens and how to improve on this.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "echogarden",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"description": "An integrated speech toolset, providing a variety of synthesis, recognition, alignment, and other processing tools, designed to be directly accessible to end-users.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "GPL-3.0",
|
|
@@ -12,6 +12,7 @@ import { synthesize } from '../api/API.js'
|
|
|
12
12
|
import { resampleAudioSpeex } from '../dsp/SpeexResampler.js'
|
|
13
13
|
import { deepClone } from '../utilities/ObjectUtilities.js'
|
|
14
14
|
import { zeroIfNaN } from '../math/VectorMath.js'
|
|
15
|
+
import { EspeakOptions } from '../synthesis/EspeakTTS.js'
|
|
15
16
|
|
|
16
17
|
export async function alignUsingDtw(
|
|
17
18
|
sourceRawAudio: RawAudio,
|
|
@@ -150,15 +151,14 @@ export async function alignUsingDtw(
|
|
|
150
151
|
return mappedTimeline
|
|
151
152
|
}
|
|
152
153
|
|
|
153
|
-
export async function
|
|
154
|
+
export async function alignUsingDtwWithRecognitionReference(
|
|
154
155
|
sourceRawAudio: RawAudio,
|
|
155
156
|
referenceRawAudio: RawAudio,
|
|
156
157
|
referenceTimeline: Timeline,
|
|
157
158
|
recognitionTimeline: Timeline,
|
|
158
|
-
synthesizedRecognizedTranscriptRawAudio: RawAudio,
|
|
159
|
-
synthesizedRecognitionTimeline: Timeline,
|
|
160
159
|
granularities: DtwGranularity[],
|
|
161
160
|
windowDurations: number[],
|
|
161
|
+
espeakOptions: EspeakOptions,
|
|
162
162
|
phoneAlignmentMethod: API.PhoneAlignmentMethod = 'interpolation') {
|
|
163
163
|
|
|
164
164
|
const logger = new Logger()
|
|
@@ -182,13 +182,25 @@ export async function alignUsingDtwWithRecognition(
|
|
|
182
182
|
return interpolatedTimeline
|
|
183
183
|
}
|
|
184
184
|
|
|
185
|
+
// Synthesize the recognized transcript and get its timeline
|
|
186
|
+
logger.start("Synthesize recognized transcript with eSpeak")
|
|
187
|
+
const recognizedWords = recognitionTimeline.map(entry => entry.text)
|
|
188
|
+
|
|
189
|
+
const {
|
|
190
|
+
rawAudio: synthesizedRecognizedTranscriptRawAudio,
|
|
191
|
+
timeline: synthesizedRecognitionTimeline
|
|
192
|
+
} = await createAlignmentReferenceUsingEspeakForFragments(recognizedWords, espeakOptions)
|
|
193
|
+
|
|
185
194
|
let recognitionTimelineWithPhones: Timeline
|
|
186
195
|
|
|
187
196
|
if (phoneAlignmentMethod == 'interpolation') {
|
|
188
197
|
// Add phone timelines by interpolating from reference words
|
|
189
198
|
logger.start('Interpolate phone timing')
|
|
190
199
|
|
|
191
|
-
recognitionTimelineWithPhones = await interpolatePhoneTimelines(
|
|
200
|
+
recognitionTimelineWithPhones = await interpolatePhoneTimelines(
|
|
201
|
+
recognitionTimeline,
|
|
202
|
+
synthesizedRecognitionTimeline
|
|
203
|
+
)
|
|
192
204
|
} else if (phoneAlignmentMethod == 'dtw') {
|
|
193
205
|
logger.start('Align phone timing')
|
|
194
206
|
|
|
@@ -205,7 +217,7 @@ export async function alignUsingDtwWithRecognition(
|
|
|
205
217
|
}
|
|
206
218
|
|
|
207
219
|
// Create a mapping from the synthesized recognized timeline to the recognized timeline
|
|
208
|
-
logger.start(
|
|
220
|
+
logger.start("Map from the synthesized recognized timeline to the recognized timeline")
|
|
209
221
|
|
|
210
222
|
type SynthesizedToRecognizedTimeMappingEntry = {
|
|
211
223
|
synthesized: number
|
|
@@ -217,16 +229,18 @@ export async function alignUsingDtwWithRecognition(
|
|
|
217
229
|
const synthesizedToRecognizedTimeMapping: SynthesizedToRecognizedTimeMapping = []
|
|
218
230
|
|
|
219
231
|
for (let wordEntryIndex = 0; wordEntryIndex < synthesizedRecognitionTimeline.length; wordEntryIndex++) {
|
|
220
|
-
const
|
|
221
|
-
const
|
|
232
|
+
const synthesizedTimelineEntry = synthesizedRecognitionTimeline[wordEntryIndex]
|
|
233
|
+
const recognitionTimelineEntry = recognitionTimelineWithPhones[wordEntryIndex]
|
|
222
234
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
235
|
+
synthesizedToRecognizedTimeMapping.push({
|
|
236
|
+
synthesized: synthesizedTimelineEntry.startTime,
|
|
237
|
+
recognized: recognitionTimelineEntry.startTime
|
|
238
|
+
})
|
|
226
239
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
const
|
|
240
|
+
if (synthesizedTimelineEntry.timeline) {
|
|
241
|
+
for (let tokenEntryIndex = 0; tokenEntryIndex < synthesizedTimelineEntry.timeline.length; tokenEntryIndex++) {
|
|
242
|
+
const synthesizedPhoneTimelineEntry = synthesizedTimelineEntry.timeline[tokenEntryIndex]
|
|
243
|
+
const recognitionPhoneTimelineEntry = recognitionTimelineEntry.timeline![tokenEntryIndex]
|
|
230
244
|
|
|
231
245
|
synthesizedToRecognizedTimeMapping.push({
|
|
232
246
|
synthesized: synthesizedPhoneTimelineEntry.startTime,
|
|
@@ -239,10 +253,16 @@ export async function alignUsingDtwWithRecognition(
|
|
|
239
253
|
})
|
|
240
254
|
}
|
|
241
255
|
}
|
|
256
|
+
|
|
257
|
+
synthesizedToRecognizedTimeMapping.push({
|
|
258
|
+
synthesized: synthesizedTimelineEntry.endTime,
|
|
259
|
+
recognized: recognitionTimelineEntry.endTime
|
|
260
|
+
})
|
|
242
261
|
}
|
|
243
262
|
|
|
244
|
-
logger.start('Align the synthesized recognized transcript with the synthesized ground-truth transcript')
|
|
245
263
|
// Align the synthesized recognized transcript to the synthesized reference transcript
|
|
264
|
+
logger.start("Align the synthesized recognized transcript with the synthesized ground-truth transcript")
|
|
265
|
+
|
|
246
266
|
const alignedSynthesizedRecognitionTimeline = await alignUsingDtw(
|
|
247
267
|
synthesizedRecognizedTranscriptRawAudio,
|
|
248
268
|
referenceRawAudio,
|
|
@@ -250,41 +270,44 @@ export async function alignUsingDtwWithRecognition(
|
|
|
250
270
|
granularities,
|
|
251
271
|
windowDurations)
|
|
252
272
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
273
|
+
let currentSynthesizedToRecognizedMappingIndex = 0
|
|
274
|
+
|
|
275
|
+
function mapSynthesizedToRecognizedTimeAndAdvance(synthesizedTime: number) {
|
|
276
|
+
for (; ; currentSynthesizedToRecognizedMappingIndex += 1) {
|
|
277
|
+
const left = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].synthesized
|
|
257
278
|
|
|
258
|
-
|
|
279
|
+
let right: number
|
|
259
280
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
281
|
+
if (currentSynthesizedToRecognizedMappingIndex < synthesizedToRecognizedTimeMapping.length - 1) {
|
|
282
|
+
right = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex + 1].synthesized
|
|
283
|
+
} else {
|
|
284
|
+
right = Infinity
|
|
285
|
+
}
|
|
265
286
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
287
|
+
if (left > right) {
|
|
288
|
+
throw new Error("Left is larger than right!")
|
|
289
|
+
}
|
|
269
290
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
}
|
|
291
|
+
if (Math.abs(synthesizedTime - left) < Math.abs(synthesizedTime - right)) {
|
|
292
|
+
return synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].recognized
|
|
273
293
|
}
|
|
274
294
|
}
|
|
295
|
+
}
|
|
275
296
|
|
|
297
|
+
function mapTimeline(timeline: Timeline) {
|
|
276
298
|
const mappedTimeline: Timeline = []
|
|
277
299
|
|
|
278
300
|
for (const entry of timeline) {
|
|
279
|
-
const mappedEntry =
|
|
301
|
+
const mappedEntry = { ...entry }
|
|
280
302
|
|
|
281
|
-
mappedEntry.startTime =
|
|
282
|
-
mappedEntry.endTime = mapSynthesizedToRecognizedTime(entry.endTime)
|
|
303
|
+
mappedEntry.startTime = mapSynthesizedToRecognizedTimeAndAdvance(entry.startTime)
|
|
283
304
|
|
|
284
305
|
if (entry.timeline) {
|
|
285
306
|
mappedEntry.timeline = mapTimeline(entry.timeline)
|
|
286
307
|
}
|
|
287
308
|
|
|
309
|
+
mappedEntry.endTime = mapSynthesizedToRecognizedTimeAndAdvance(entry.endTime)
|
|
310
|
+
|
|
288
311
|
mappedTimeline.push(mappedEntry)
|
|
289
312
|
}
|
|
290
313
|
|
|
@@ -301,10 +324,10 @@ export async function alignUsingDtwWithRecognition(
|
|
|
301
324
|
export async function interpolatePhoneTimelines(sourceTimeline: Timeline, referenceTimeline: Timeline) {
|
|
302
325
|
const interpolatedTimeline: Timeline = []
|
|
303
326
|
|
|
304
|
-
for (let
|
|
305
|
-
const referenceEntry = referenceTimeline[
|
|
327
|
+
for (let wordEntryIndex = 0; wordEntryIndex < sourceTimeline.length; wordEntryIndex++) {
|
|
328
|
+
const referenceEntry = referenceTimeline[wordEntryIndex]
|
|
306
329
|
|
|
307
|
-
const interpolatedEntry = deepClone(sourceTimeline[
|
|
330
|
+
const interpolatedEntry = deepClone(sourceTimeline[wordEntryIndex])
|
|
308
331
|
interpolatedTimeline.push(interpolatedEntry)
|
|
309
332
|
|
|
310
333
|
if (interpolatedEntry.type != 'word') {
|
|
@@ -335,23 +358,13 @@ export async function interpolatePhoneTimelines(sourceTimeline: Timeline, refere
|
|
|
335
358
|
}
|
|
336
359
|
}
|
|
337
360
|
|
|
338
|
-
const
|
|
339
|
-
|
|
340
|
-
for (const tokenEntry of (referenceEntry.timeline || [])!) {
|
|
341
|
-
const interpolatedTokenEntry = mapEntry(tokenEntry)
|
|
342
|
-
|
|
343
|
-
const interpolatedPhoneEntries: Timeline = []
|
|
344
|
-
|
|
345
|
-
for (const phoneEntry of (tokenEntry.timeline || [])) {
|
|
346
|
-
interpolatedPhoneEntries.push(mapEntry(phoneEntry))
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
interpolatedTokenEntry.timeline = interpolatedPhoneEntries
|
|
361
|
+
const interpolatedPhoneEntries: Timeline = []
|
|
350
362
|
|
|
351
|
-
|
|
363
|
+
for (const phoneEntry of (referenceEntry.timeline || [])) {
|
|
364
|
+
interpolatedPhoneEntries.push(mapEntry(phoneEntry))
|
|
352
365
|
}
|
|
353
366
|
|
|
354
|
-
interpolatedEntry.timeline =
|
|
367
|
+
interpolatedEntry.timeline = interpolatedPhoneEntries
|
|
355
368
|
}
|
|
356
369
|
|
|
357
370
|
return interpolatedTimeline
|
|
@@ -424,23 +437,13 @@ export async function alignPhoneTimelines(
|
|
|
424
437
|
}
|
|
425
438
|
|
|
426
439
|
// Add phone timeline using the mapped time information
|
|
427
|
-
const
|
|
440
|
+
const alignedPhoneTimeline: Timeline = []
|
|
428
441
|
|
|
429
|
-
for (const
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
for (const referencePhoneEntry of (referenceTokenEntry.timeline || [])) {
|
|
433
|
-
alignedPhoneTimeline.push(mapEntry(referencePhoneEntry))
|
|
434
|
-
}
|
|
435
|
-
|
|
436
|
-
alignedTokenTimeline.push({
|
|
437
|
-
...mapEntry(referenceTokenEntry),
|
|
438
|
-
|
|
439
|
-
timeline: alignedPhoneTimeline
|
|
440
|
-
})
|
|
442
|
+
for (const referencePhoneEntry of (referenceWordEntry.timeline || [])) {
|
|
443
|
+
alignedPhoneTimeline.push(mapEntry(referencePhoneEntry))
|
|
441
444
|
}
|
|
442
445
|
|
|
443
|
-
alignedWordEntry.timeline =
|
|
446
|
+
alignedWordEntry.timeline = alignedPhoneTimeline
|
|
444
447
|
|
|
445
448
|
alignedWordTimeline.push(alignedWordEntry)
|
|
446
449
|
}
|
|
@@ -448,6 +451,27 @@ export async function alignPhoneTimelines(
|
|
|
448
451
|
return alignedWordTimeline
|
|
449
452
|
}
|
|
450
453
|
|
|
454
|
+
export async function createAlignmentReferenceUsingEspeakForFragments(fragments: string[], espeakOptions: EspeakOptions, insertSeparators = true) {
|
|
455
|
+
const progressLogger = new Logger()
|
|
456
|
+
|
|
457
|
+
progressLogger.start("Load espeak module")
|
|
458
|
+
const Espeak = await import("../synthesis/EspeakTTS.js")
|
|
459
|
+
|
|
460
|
+
progressLogger.start("Create alignment reference with eSpeak")
|
|
461
|
+
|
|
462
|
+
const result = await Espeak.synthesizeFragments(fragments, espeakOptions)
|
|
463
|
+
|
|
464
|
+
result.timeline = result.timeline.flatMap(clause => clause.timeline!)
|
|
465
|
+
|
|
466
|
+
for (const wordEntry of result.timeline) {
|
|
467
|
+
wordEntry.timeline = wordEntry.timeline!.flatMap(tokenEntry => tokenEntry.timeline!)
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
progressLogger.end()
|
|
471
|
+
|
|
472
|
+
return result
|
|
473
|
+
}
|
|
474
|
+
|
|
451
475
|
export async function createAlignmentReferenceUsingEspeak(transcript: string, language: string, plaintextOptions?: API.PlainTextOptions, customLexiconPaths?: string[], insertSeparators?: boolean) {
|
|
452
476
|
const logger = new Logger()
|
|
453
477
|
|
package/src/api/Alignment.ts
CHANGED
|
@@ -11,7 +11,7 @@ import { WhisperOptions } from '../recognition/WhisperSTT.js'
|
|
|
11
11
|
import chalk from 'chalk'
|
|
12
12
|
import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js'
|
|
13
13
|
import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
|
|
14
|
-
|
|
14
|
+
import { EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
|
|
15
15
|
|
|
16
16
|
const log = logToStderr
|
|
17
17
|
|
|
@@ -85,7 +85,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
|
|
|
85
85
|
|
|
86
86
|
logger.start('Load alignment module')
|
|
87
87
|
|
|
88
|
-
const {
|
|
88
|
+
const { alignUsingDtwWithRecognitionReference, alignUsingDtw } = await import('../alignment/SpeechAlignment.js')
|
|
89
89
|
|
|
90
90
|
function getDtwWindowDurationsAndGranularities() {
|
|
91
91
|
let granularities: DtwGranularity[]
|
|
@@ -153,21 +153,21 @@ export async function align(input: AudioSourceParam, transcript: string, options
|
|
|
153
153
|
const {
|
|
154
154
|
referenceRawAudio,
|
|
155
155
|
referenceTimeline,
|
|
156
|
+
espeakVoice,
|
|
156
157
|
} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, true)
|
|
157
158
|
|
|
158
|
-
// Synthesize the recognized transcript and get its timeline
|
|
159
|
-
logger.start('Synthesize recognized transcript with eSpeak')
|
|
160
|
-
|
|
161
|
-
const {
|
|
162
|
-
referenceRawAudio: synthesizedRecognizedTranscriptRawAudio,
|
|
163
|
-
referenceTimeline: synthesizedRecognitionTimeline
|
|
164
|
-
} = await createAlignmentReferenceUsingEspeak(recognizedTranscript, language, undefined, undefined, true)
|
|
165
|
-
|
|
166
159
|
logger.end()
|
|
167
160
|
|
|
168
161
|
const { windowDurations, granularities } = getDtwWindowDurationsAndGranularities()
|
|
169
162
|
|
|
170
163
|
const phoneAlignmentMethod = options.dtw!.phoneAlignmentMethod!
|
|
164
|
+
|
|
165
|
+
const espeakOptions: EspeakOptions = {
|
|
166
|
+
...defaultEspeakOptions,
|
|
167
|
+
voice: espeakVoice,
|
|
168
|
+
useKlatt: false,
|
|
169
|
+
insertSeparators: true
|
|
170
|
+
}
|
|
171
171
|
|
|
172
172
|
// Align the ground-truth transcript and the recognized transcript
|
|
173
173
|
mappedTimeline = await alignUsingDtwWithRecognitionReference(
|
|
@@ -176,11 +176,10 @@ export async function align(input: AudioSourceParam, transcript: string, options
|
|
|
176
176
|
referenceTimeline,
|
|
177
177
|
|
|
178
178
|
recognitionTimeline,
|
|
179
|
-
synthesizedRecognizedTranscriptRawAudio,
|
|
180
|
-
synthesizedRecognitionTimeline,
|
|
181
179
|
|
|
182
180
|
granularities,
|
|
183
181
|
windowDurations,
|
|
182
|
+
espeakOptions,
|
|
184
183
|
phoneAlignmentMethod)
|
|
185
184
|
|
|
186
185
|
break
|