echogarden 2.8.7 → 2.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/lexicons/words.en.json +1 -1
- package/dist/api/Recognition.d.ts +3 -3
- package/dist/api/Recognition.js +5 -5
- package/dist/api/Recognition.js.map +1 -1
- package/dist/api/SpeechTranslation.d.ts +4 -4
- package/dist/api/SpeechTranslation.js +4 -4
- package/dist/api/SpeechTranslation.js.map +1 -1
- package/dist/denoising/NSNet2.js +8 -8
- package/dist/denoising/NSNet2.js.map +1 -1
- package/dist/denoising/RNNoise.js.map +1 -1
- package/dist/dsp/FFT.d.ts +2 -2
- package/dist/dsp/FFT.js +19 -15
- package/dist/dsp/FFT.js.map +1 -1
- package/dist/dsp/MFCC.d.ts +2 -2
- package/dist/dsp/MFCC.js +8 -8
- package/dist/dsp/MFCC.js.map +1 -1
- package/dist/dsp/{MelSpectogram.d.ts → MelSpectrogram.d.ts} +5 -5
- package/dist/dsp/{MelSpectogram.js → MelSpectrogram.js} +9 -9
- package/dist/dsp/MelSpectrogram.js.map +1 -0
- package/dist/encodings/Utf8.d.ts +1 -1
- package/dist/math/VectorMath.d.ts +8 -4
- package/dist/math/VectorMath.js +12 -1
- package/dist/math/VectorMath.js.map +1 -1
- package/dist/recognition/AzureCognitiveServicesSTT.js +1 -1
- package/dist/recognition/AzureCognitiveServicesSTT.js.map +1 -1
- package/dist/recognition/WhisperSTT.d.ts +5 -5
- package/dist/recognition/WhisperSTT.js +38 -32
- package/dist/recognition/WhisperSTT.js.map +1 -1
- package/dist/source-separation/MDXNetSourceSeparation.js +5 -5
- package/dist/source-separation/MDXNetSourceSeparation.js.map +1 -1
- package/dist/speech-embeddings/WavToVec2BertFeatureEmbeddings.js +14 -14
- package/dist/speech-embeddings/WavToVec2BertFeatureEmbeddings.js.map +1 -1
- package/dist/subtitles/Subtitles.js +1 -1
- package/dist/subtitles/Subtitles.js.map +1 -1
- package/dist/synthesis/AzureCognitiveServicesTTS.js +1 -1
- package/dist/synthesis/AzureCognitiveServicesTTS.js.map +1 -1
- package/dist/synthesis/FliteTTS.js +2 -1
- package/dist/synthesis/FliteTTS.js.map +1 -1
- package/dist/utilities/PackageManager.js +1 -1
- package/docs/Tasklist.md +8 -0
- package/package.json +13 -13
- package/src/api/Recognition.ts +10 -7
- package/src/api/SpeechTranslation.ts +16 -6
- package/src/denoising/NSNet2.ts +8 -8
- package/src/denoising/RNNoise.ts +1 -1
- package/src/dsp/FFT.ts +23 -17
- package/src/dsp/MFCC.ts +8 -8
- package/src/dsp/{MelSpectogram.ts → MelSpectrogram.ts} +9 -8
- package/src/math/VectorMath.ts +17 -1
- package/src/recognition/AzureCognitiveServicesSTT.ts +1 -1
- package/src/recognition/WhisperSTT.ts +61 -35
- package/src/source-separation/MDXNetSourceSeparation.ts +5 -5
- package/src/speech-embeddings/WavToVec2BertFeatureEmbeddings.ts +14 -14
- package/src/subtitles/Subtitles.ts +1 -1
- package/src/synthesis/AzureCognitiveServicesTTS.ts +1 -2
- package/src/synthesis/FliteTTS.ts +2 -1
- package/src/utilities/PackageManager.ts +1 -1
- package/dist/dsp/MelSpectogram.js.map +0 -1
|
@@ -46,7 +46,7 @@ async function requestRecognition(rawAudio: RawAudio, subscriptionKey: string, s
|
|
|
46
46
|
|
|
47
47
|
const inputStream = SpeechSDK.AudioInputStream.createPushStream(audioFormat)
|
|
48
48
|
|
|
49
|
-
inputStream.write(encodedAudio)
|
|
49
|
+
inputStream.write(encodedAudio.buffer as ArrayBuffer)
|
|
50
50
|
inputStream.close()
|
|
51
51
|
|
|
52
52
|
const audioConfig = SpeechSDK.AudioConfig.fromStreamInput(inputStream)
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import chalk from 'chalk'
|
|
1
2
|
import type * as Onnx from 'onnxruntime-node'
|
|
2
3
|
|
|
3
4
|
import { Logger } from '../utilities/Logger.js'
|
|
4
|
-
import {
|
|
5
|
+
import { computeMelSpectrogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectrogram.js'
|
|
5
6
|
import { clip, getIntegerRange, getTopKIndexes, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
|
|
6
|
-
import { indexOfMax, logOfVector, logSumExp, meanOfVector, medianOfVector, softmax,
|
|
7
|
+
import { indexOfMax, logOfVector, logSumExp, meanOfVector, medianOfVector, softmax, sumAndSumOfSquaresOfVector, sumOfSquaresOfVector, sumVector } from '../math/VectorMath.js'
|
|
7
8
|
|
|
8
9
|
import { alignDTWWindowed } from '../alignment/DTWSequenceAlignmentWindowed.js'
|
|
9
10
|
import { extendDeep } from '../utilities/ObjectUtilities.js'
|
|
@@ -14,7 +15,6 @@ import { readFileAsUtf8 } from '../utilities/FileSystem.js'
|
|
|
14
15
|
import { logLevelGreaterOrEqualTo, type LanguageDetectionResults } from '../api/API.js'
|
|
15
16
|
import { formatLanguageCodeWithName, getShortLanguageCode, languageCodeToName } from '../utilities/Locale.js'
|
|
16
17
|
import { loadPackage } from '../utilities/PackageManager.js'
|
|
17
|
-
import chalk from 'chalk'
|
|
18
18
|
import { XorShift32PRNG } from '../utilities/RandomGenerator.js'
|
|
19
19
|
import { detectSpeechLanguageByParts } from '../api/SpeechLanguageDetection.js'
|
|
20
20
|
import { type Tiktoken } from 'tiktoken/lite'
|
|
@@ -33,7 +33,8 @@ export async function recognize(
|
|
|
33
33
|
modelDir: string,
|
|
34
34
|
task: WhisperTask,
|
|
35
35
|
sourceLanguage: string,
|
|
36
|
-
options: WhisperOptions
|
|
36
|
+
options: WhisperOptions,
|
|
37
|
+
onPart?: WhisperPartCallback) {
|
|
37
38
|
|
|
38
39
|
options = extendDeep(defaultWhisperOptions, options)
|
|
39
40
|
|
|
@@ -82,7 +83,7 @@ export async function recognize(
|
|
|
82
83
|
decoderProviders,
|
|
83
84
|
seed)
|
|
84
85
|
|
|
85
|
-
const result = await whisper.recognize(sourceRawAudio, task, sourceLanguage, options)
|
|
86
|
+
const result = await whisper.recognize(sourceRawAudio, task, sourceLanguage, options, undefined, onPart)
|
|
86
87
|
|
|
87
88
|
return result
|
|
88
89
|
}
|
|
@@ -366,6 +367,7 @@ export class Whisper {
|
|
|
366
367
|
language: string,
|
|
367
368
|
options: WhisperOptions,
|
|
368
369
|
logitFilter?: WhisperLogitFilter,
|
|
370
|
+
onPart?: WhisperPartCallback,
|
|
369
371
|
) {
|
|
370
372
|
await this.initializeIfNeeded()
|
|
371
373
|
|
|
@@ -500,6 +502,13 @@ export class Whisper {
|
|
|
500
502
|
// Generate timeline from alignment path
|
|
501
503
|
const partTimeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, partTokens, segmentStartTime, segmentEndTime, partTokensConfidence)
|
|
502
504
|
|
|
505
|
+
if (onPart) {
|
|
506
|
+
const partWordTimeline = this.tokenTimelineToWordTimeline(partTimeline, language)
|
|
507
|
+
const partTranscript = this.tokensToText(partTokens)
|
|
508
|
+
|
|
509
|
+
onPart(partTranscript, partTimeline, partWordTimeline)
|
|
510
|
+
}
|
|
511
|
+
|
|
503
512
|
// Add tokens to output
|
|
504
513
|
allDecodedTokens.push(...partTokens)
|
|
505
514
|
timeline.push(...partTimeline)
|
|
@@ -1234,8 +1243,7 @@ export class Whisper {
|
|
|
1234
1243
|
throw new Error(`Audio part is longer than 30 seconds`)
|
|
1235
1244
|
}
|
|
1236
1245
|
|
|
1237
|
-
|
|
1238
|
-
await logger.startAsync('Extract mel spectogram from audio part')
|
|
1246
|
+
await logger.startAsync('Extract Mel spectrogram from audio part')
|
|
1239
1247
|
|
|
1240
1248
|
// Pad audio samples to ensure that have a duration of 30 seconds
|
|
1241
1249
|
const paddedAudioSamples = new Float32Array(maxAudioSamples)
|
|
@@ -1243,40 +1251,40 @@ export class Whisper {
|
|
|
1243
1251
|
|
|
1244
1252
|
const rawAudioPart: RawAudio = { audioChannels: [paddedAudioSamples], sampleRate }
|
|
1245
1253
|
|
|
1246
|
-
|
|
1254
|
+
// Compute Mel spectrogram
|
|
1255
|
+
const { melSpectrogram } = await computeMelSpectrogramUsingFilterbanks(rawAudioPart, fftOrder, fftWindowSize, fftHopLength, filterbanks)
|
|
1247
1256
|
|
|
1248
|
-
|
|
1257
|
+
// Flatten, transpose, apply logarithm and normalize Mel spectrogram
|
|
1258
|
+
await logger.startAsync('Process Mel spectrogram')
|
|
1249
1259
|
|
|
1250
|
-
const
|
|
1260
|
+
const flattenedLogMelSpectrogram = new Float32Array(maxAudioFrames * filterbankCount)
|
|
1251
1261
|
|
|
1252
|
-
// Find maximum log mel value in the spectrum
|
|
1253
1262
|
let maxLogMel = -Infinity
|
|
1254
1263
|
|
|
1255
|
-
for (
|
|
1256
|
-
for (
|
|
1257
|
-
|
|
1258
|
-
|
|
1264
|
+
for (let i = 0; i < filterbankCount; i++) {
|
|
1265
|
+
for (let j = 0; j < maxAudioFrames; j++) {
|
|
1266
|
+
const mel = melSpectrogram[j][i]
|
|
1267
|
+
const logMel = Math.log10(Math.max(mel, 1e-10))
|
|
1268
|
+
|
|
1269
|
+
if (logMel > maxLogMel) {
|
|
1270
|
+
maxLogMel = logMel
|
|
1259
1271
|
}
|
|
1272
|
+
|
|
1273
|
+
flattenedLogMelSpectrogram[(i * maxAudioFrames) + j] = logMel
|
|
1260
1274
|
}
|
|
1261
1275
|
}
|
|
1262
1276
|
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1277
|
+
for (let i = 0; i < flattenedLogMelSpectrogram.length; i++) {
|
|
1278
|
+
const logMel = flattenedLogMelSpectrogram[i]
|
|
1279
|
+
const normalizedLogMel = (Math.max(logMel, maxLogMel - 8) + 4) / 4
|
|
1266
1280
|
|
|
1267
|
-
|
|
1268
|
-
const flattenedNormalizedLogMelSpectogram = new Float32Array(maxAudioFrames * filterbankCount)
|
|
1269
|
-
|
|
1270
|
-
for (let i = 0; i < filterbankCount; i++) {
|
|
1271
|
-
for (let j = 0; j < maxAudioFrames; j++) {
|
|
1272
|
-
flattenedNormalizedLogMelSpectogram[(i * maxAudioFrames) + j] = normalizedLogMelSpectogram[j][i]
|
|
1273
|
-
}
|
|
1281
|
+
flattenedLogMelSpectrogram[i] = normalizedLogMel
|
|
1274
1282
|
}
|
|
1275
1283
|
|
|
1276
1284
|
// Run the encoder model
|
|
1277
|
-
await logger.startAsync('Encode
|
|
1285
|
+
await logger.startAsync('Encode Mel spectrogram with Whisper encoder model')
|
|
1278
1286
|
|
|
1279
|
-
const inputTensor = new Onnx.Tensor('float32',
|
|
1287
|
+
const inputTensor = new Onnx.Tensor('float32', flattenedLogMelSpectrogram, [1, filterbankCount, maxAudioFrames])
|
|
1280
1288
|
|
|
1281
1289
|
const encoderInputs = { mel: inputTensor }
|
|
1282
1290
|
|
|
@@ -1488,8 +1496,10 @@ export class Whisper {
|
|
|
1488
1496
|
let countOfAllWeightsForHead = 0
|
|
1489
1497
|
|
|
1490
1498
|
for (const tokenFrames of head) {
|
|
1491
|
-
|
|
1492
|
-
|
|
1499
|
+
const { sum, sumOfSquares } = sumAndSumOfSquaresOfVector(tokenFrames)
|
|
1500
|
+
|
|
1501
|
+
sumOfAllWeightsForHead += sum
|
|
1502
|
+
sumOfAllSquaredWeightsForHead += sumOfSquares
|
|
1493
1503
|
countOfAllWeightsForHead += tokenFrames.length
|
|
1494
1504
|
}
|
|
1495
1505
|
|
|
@@ -2004,7 +2014,20 @@ export type WhisperTokenData = {
|
|
|
2004
2014
|
|
|
2005
2015
|
export type WhisperLogitFilter = (logits: number[], decodedTokens: number[], isFirstPart: boolean, isFinalPart: boolean) => number[]
|
|
2006
2016
|
|
|
2007
|
-
export type WhisperModelName =
|
|
2017
|
+
export type WhisperModelName =
|
|
2018
|
+
'tiny' |
|
|
2019
|
+
'tiny.en' |
|
|
2020
|
+
'base' |
|
|
2021
|
+
'base.en' |
|
|
2022
|
+
'small' |
|
|
2023
|
+
'small.en' |
|
|
2024
|
+
'medium' |
|
|
2025
|
+
'medium.en' |
|
|
2026
|
+
'large-v1' |
|
|
2027
|
+
'large-v2' |
|
|
2028
|
+
'large-v3' |
|
|
2029
|
+
'large-v3-turbo'
|
|
2030
|
+
|
|
2008
2031
|
export type WhisperTask = 'transcribe' | 'translate' | 'detect-language'
|
|
2009
2032
|
|
|
2010
2033
|
export const modelNameToPackageName: { [modelName in WhisperModelName]: string } = {
|
|
@@ -2124,17 +2147,18 @@ const languageIdLookup: { [s: string]: number } = {
|
|
|
2124
2147
|
'ba': 96,
|
|
2125
2148
|
'jw': 97,
|
|
2126
2149
|
'su': 98,
|
|
2150
|
+
//'yue': 99
|
|
2127
2151
|
}
|
|
2128
2152
|
|
|
2129
2153
|
const alignmentHeadsIndexes: { [name in WhisperModelName]: number[] } = {
|
|
2130
|
-
'tiny.en': [6, 12, 17, 18, 19, 20, 21, 22,],
|
|
2131
2154
|
'tiny': [14, 18, 20, 21, 22, 23,],
|
|
2132
|
-
'
|
|
2155
|
+
'tiny.en': [6, 12, 17, 18, 19, 20, 21, 22,],
|
|
2133
2156
|
'base': [25, 34, 35, 39, 41, 42, 44, 46,],
|
|
2134
|
-
'
|
|
2157
|
+
'base.en': [27, 39, 41, 45, 47,],
|
|
2135
2158
|
'small': [63, 69, 96, 100, 103, 104, 108, 115, 117, 125,],
|
|
2136
|
-
'
|
|
2159
|
+
'small.en': [78, 84, 87, 92, 98, 101, 103, 108, 112, 116, 118, 120, 121, 122, 123, 126, 131, 134, 136,],
|
|
2137
2160
|
'medium': [223, 244, 255, 257, 320, 372,],
|
|
2161
|
+
'medium.en': [180, 225, 236, 238, 244, 256, 260, 265, 284, 286, 295, 298, 303, 320, 323, 329, 334, 348,],
|
|
2138
2162
|
'large-v1': [199, 222, 224, 237, 447, 451, 457, 462, 475,],
|
|
2139
2163
|
'large-v2': [212, 277, 331, 332, 333, 355, 356, 364, 371, 379, 391, 422, 423, 443, 449, 452, 465, 467, 473, 505, 521, 532, 555,],
|
|
2140
2164
|
'large-v3': [140, 217, 258, 272, 321, 354, 391, 424, 481, 506,],
|
|
@@ -2653,4 +2677,6 @@ export const defaultWhisperVADOptions: WhisperVADOptions = {
|
|
|
2653
2677
|
decoderProvider: undefined,
|
|
2654
2678
|
}
|
|
2655
2679
|
|
|
2656
|
-
type WhisperTimestampAccuracy = 'medium' | 'high'
|
|
2680
|
+
export type WhisperTimestampAccuracy = 'medium' | 'high'
|
|
2681
|
+
|
|
2682
|
+
export type WhisperPartCallback = (partTranscript: string, partTokenTimeline: Timeline, partWordTimeline: Timeline) => void
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type * as Onnx from 'onnxruntime-node'
|
|
2
2
|
import { getEmptyRawAudio, RawAudio } from '../audio/AudioUtilities.js'
|
|
3
|
-
import { getWindowWeights,
|
|
3
|
+
import { getWindowWeights, createStftrGenerator, stiftr, WindowType } from '../dsp/FFT.js'
|
|
4
4
|
import { logToStderr } from '../utilities/Utilities.js'
|
|
5
5
|
import { Logger } from '../utilities/Logger.js'
|
|
6
6
|
import { OnnxExecutionProvider, dmlProviderAvailable, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js'
|
|
@@ -74,8 +74,8 @@ export class MDXNet {
|
|
|
74
74
|
const fftSizeReciprocal = 1 / fftSize
|
|
75
75
|
|
|
76
76
|
// Initialize generators for STFT frames for each channel
|
|
77
|
-
const fftFramesLeftGenerator =
|
|
78
|
-
const fftFramesRightGenerator =
|
|
77
|
+
const fftFramesLeftGenerator = await createStftrGenerator(rawAudio.audioChannels[0], fftSize, fftWindowSize, fftHopSize, fftWindowType)
|
|
78
|
+
const fftFramesRightGenerator = await createStftrGenerator(rawAudio.audioChannels[1], fftSize, fftWindowSize, fftHopSize, fftWindowType)
|
|
79
79
|
|
|
80
80
|
// Initial windowed lists to store recently computed STFT frames
|
|
81
81
|
const fftFramesLeftWindowedList = new WindowedList<Float32Array>(segmentSize)
|
|
@@ -95,13 +95,13 @@ export class MDXNet {
|
|
|
95
95
|
}
|
|
96
96
|
|
|
97
97
|
while (fftFramesLeftWindowedList.endOffset < segmentEndFrameOffset) {
|
|
98
|
-
const nextLeftFrameResult =
|
|
98
|
+
const nextLeftFrameResult = fftFramesLeftGenerator.next()
|
|
99
99
|
|
|
100
100
|
if (nextLeftFrameResult.done) {
|
|
101
101
|
break
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
-
const nextRightFrameResult =
|
|
104
|
+
const nextRightFrameResult = fftFramesRightGenerator.next()
|
|
105
105
|
|
|
106
106
|
if (nextRightFrameResult.done) {
|
|
107
107
|
break
|
|
@@ -2,7 +2,7 @@ import type * as Onnx from 'onnxruntime-node'
|
|
|
2
2
|
import { OnnxExecutionProvider, getOnnxSessionOptions } from '../utilities/OnnxUtilities.js';
|
|
3
3
|
|
|
4
4
|
import { RawAudio } from "../audio/AudioUtilities.js";
|
|
5
|
-
import {
|
|
5
|
+
import { computeMelSpectrogram } from "../dsp/MelSpectrogram.js";
|
|
6
6
|
import { Logger } from '../utilities/Logger.js';
|
|
7
7
|
import { concatFloat32Arrays, splitFloat32Array } from '../utilities/Utilities.js';
|
|
8
8
|
import { applyEmphasis } from '../dsp/MFCC.js';
|
|
@@ -31,7 +31,7 @@ export class Wav2Vec2BertFeatureEmbeddings {
|
|
|
31
31
|
|
|
32
32
|
rawAudio.audioChannels[0] = applyEmphasis(rawAudio.audioChannels[0], 0.97)
|
|
33
33
|
|
|
34
|
-
const {
|
|
34
|
+
const { melSpectrogram } = await computeMelSpectrogram(
|
|
35
35
|
rawAudio,
|
|
36
36
|
512,
|
|
37
37
|
400,
|
|
@@ -42,8 +42,8 @@ export class Wav2Vec2BertFeatureEmbeddings {
|
|
|
42
42
|
'povey')
|
|
43
43
|
|
|
44
44
|
// Ensure even length
|
|
45
|
-
if (
|
|
46
|
-
|
|
45
|
+
if (melSpectrogram.length % 2 != 0) {
|
|
46
|
+
melSpectrogram.push(new Float32Array(80))
|
|
47
47
|
}
|
|
48
48
|
|
|
49
49
|
// Normalize filterbanks
|
|
@@ -51,24 +51,24 @@ export class Wav2Vec2BertFeatureEmbeddings {
|
|
|
51
51
|
let sum = 0
|
|
52
52
|
let sumOfSquares = 0
|
|
53
53
|
|
|
54
|
-
for (let i = 0; i <
|
|
55
|
-
const value =
|
|
54
|
+
for (let i = 0; i < melSpectrogram.length; i++) {
|
|
55
|
+
const value = melSpectrogram[i][filterbankIndex]
|
|
56
56
|
|
|
57
57
|
sum += value
|
|
58
58
|
sumOfSquares += value ** 2
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
const mean = sum /
|
|
62
|
-
const normalizationFactor = 1 / (Math.sqrt(sumOfSquares /
|
|
61
|
+
const mean = sum / melSpectrogram.length
|
|
62
|
+
const normalizationFactor = 1 / (Math.sqrt(sumOfSquares / melSpectrogram.length) + 1e-40)
|
|
63
63
|
|
|
64
|
-
for (let i = 0; i <
|
|
65
|
-
|
|
66
|
-
|
|
64
|
+
for (let i = 0; i < melSpectrogram.length; i++) {
|
|
65
|
+
melSpectrogram[i][filterbankIndex] -= mean
|
|
66
|
+
melSpectrogram[i][filterbankIndex] *= normalizationFactor
|
|
67
67
|
}
|
|
68
68
|
}
|
|
69
69
|
|
|
70
70
|
// Flatten
|
|
71
|
-
const
|
|
71
|
+
const flattenedMelSpectrogram = concatFloat32Arrays(melSpectrogram)
|
|
72
72
|
|
|
73
73
|
// Initialize session
|
|
74
74
|
await this.initializeSessionIfNeeded()
|
|
@@ -77,9 +77,9 @@ export class Wav2Vec2BertFeatureEmbeddings {
|
|
|
77
77
|
|
|
78
78
|
const Onnx = await import('onnxruntime-node')
|
|
79
79
|
|
|
80
|
-
const inputTensor = new Onnx.Tensor('float32',
|
|
80
|
+
const inputTensor = new Onnx.Tensor('float32', flattenedMelSpectrogram, [1, melSpectrogram.length / 2, 80 * 2])
|
|
81
81
|
|
|
82
|
-
const attentionMask = new Int32Array(
|
|
82
|
+
const attentionMask = new Int32Array(melSpectrogram.length / 2).fill(1)
|
|
83
83
|
const attentionMaskTensor = new Onnx.Tensor('int32', attentionMask, [1, attentionMask.length])
|
|
84
84
|
|
|
85
85
|
// Run inference
|
|
@@ -174,7 +174,7 @@ function getCuesFromTimeline_IsolateSegmentSentence(timeline: Timeline, config:
|
|
|
174
174
|
|
|
175
175
|
// Generate one or more cues from each segment or sentence in the timeline.
|
|
176
176
|
for (let entry of timeline) {
|
|
177
|
-
if (entry.type == 'segment' && entry.timeline?.[0]
|
|
177
|
+
if (entry.type == 'segment' && entry.timeline?.[0]?.type == 'sentence') {
|
|
178
178
|
if (config.mode == 'segment') {
|
|
179
179
|
// If the mode is 'segment', flatten all sentences to a single word timeline
|
|
180
180
|
entry.timeline = entry.timeline!.flatMap(t => t.timeline!)
|
|
@@ -53,9 +53,8 @@ export async function synthesize(
|
|
|
53
53
|
const buffers: Uint8Array[] = []
|
|
54
54
|
|
|
55
55
|
while (true) {
|
|
56
|
-
|
|
57
56
|
const buffer = new Uint8Array(bufferSize)
|
|
58
|
-
const amountRead = await audioOutputStream.read(buffer)
|
|
57
|
+
const amountRead = await audioOutputStream.read(buffer.buffer)
|
|
59
58
|
|
|
60
59
|
if (amountRead == 0) {
|
|
61
60
|
audioOutputStream.close()
|
|
@@ -116,7 +116,8 @@ async function getModuleObject() {
|
|
|
116
116
|
if (!fliteModuleObject) {
|
|
117
117
|
const fliteWasiPath = await resolveModuleMainPath('@echogarden/flite-wasi')
|
|
118
118
|
|
|
119
|
-
|
|
119
|
+
const wasiFileContent = await readFileAsBinary(fliteWasiPath) as Uint8Array<ArrayBuffer>
|
|
120
|
+
fliteModuleObject = await WebAssembly.compile(wasiFileContent)
|
|
120
121
|
}
|
|
121
122
|
|
|
122
123
|
return fliteModuleObject
|
|
@@ -148,7 +148,7 @@ const packageVersionTagResolutionLookup: { [packageName: string]: string } = {
|
|
|
148
148
|
// FFMpeg binaries
|
|
149
149
|
'ffmpeg-6.0-win32-x64': '20240316',
|
|
150
150
|
'ffmpeg-6.0-win32-ia32': '20240316',
|
|
151
|
-
'ffmpeg-6.1-win32-arm64': '
|
|
151
|
+
'ffmpeg-6.1-win32-arm64': '20250819',
|
|
152
152
|
'ffmpeg-6.0-darwin-x64': '20240316',
|
|
153
153
|
'ffmpeg-6.0-darwin-arm64': '20240316',
|
|
154
154
|
'ffmpeg-6.0-linux-x64': '20240316',
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"MelSpectogram.js","sourceRoot":"","sources":["../../src/dsp/MelSpectogram.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAE/B,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,QAAkB,EAAE,QAAgB,EAAE,UAAkB,EAAE,SAAiB,EAAE,eAAuB,EAAE,gBAAwB,EAAE,gBAAwB,EAAE,aAA6B,MAAM;IACvO,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAA;IACtC,MAAM,QAAQ,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;IACnC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAA;IAChD,MAAM,cAAc,GAAG,GAAG,CAAC,iBAAiB,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAA;IAExE,MAAM,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAA;IACtD,MAAM,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAA;IAEtD,MAAM,4BAA4B,GAAG,kCAAkC,CAAC,eAAe,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,CAAA;IAC9H,MAAM,cAAc,GAAG,iBAAiB,CAAC,cAAc,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,CAAA;IAE5H,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,oCAAoC,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,cAAc,EAAE,UAAU,CAAC,CAAA;AACnH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oCAAoC,CAAC,QAAkB,EAAE,QAAgB,EAAE,UAAkB,EAAE,SAAiB,EAAE,WAAyB,EAAE,aAA6B,MAAM;IACrM,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,CAAA;IAE5F,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;IACtD,MAAM,aAAa,GAAG,wBAAwB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;IAEtE,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,CAAA;AACpC,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,SAAyB,EAAE,cAA4B;IAC/F,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;QAC/B,MAAM,aAAa,GAAG,GAAG,CAAC,uBAAuB,CAAC,QAAQ,CAAC,CAAA;QAC3D,OAAO,0BAA0B,CAAC,aAAa,EAAE,cAAc,CAAC,CAAA;IACjE,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,aAA2B,EAAE,WAAyB;IAChG,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAA;IAC1C,MAAM,WAAW,GAAG,IAAI,YAAY,CAAC,eAAe,CAAC,CAAA;IAErD,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,eAAe,EAAE,YAAY,EAAE,EAAE,CAAC;QAC3E,MAAM,UAAU,GAAG,WAAW,CAAC,YAAY,CAAC,CAAA;QAC5C,MAAM,oBAAoB,GAAG,UAAU,CAAC,UAAU,CAAA;QAClD,MAAM,iBAAiB,GAAG,UAAU,CAAC,OAAO,CAAA;QAE5C,IAAI,oBAAoB,KAAK,CAAC,CAAC,EAAE,CAAC;YACjC,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAA;QAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,MAAM,kBAAkB,GAAG,oBAAoB,GAAG,CAAC,CAAA;YAEnD,IAAI,kBAAkB,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;gBAChD,MAAK;YACN,CAAC;YAED,MAAM,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,kBAAkB,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAA;YAE5D,YAAY,IAAI,MAAM,GAAG,kBAAkB,CAAA;QAC5C,CAAC;QAED,WAAW,CAAC,YAAY,CAAC,GAAG,YAAY,CAAA;IACzC,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,0BAAwC,EAAE,oBAAkC,EAAE,iBAAyB,EAAE,iBAAyB;IACnK,MAAM,eAAe,GAAG,oBAAoB,CAAC,MAAM,CAAA;IACnD,MAAM,2BAA2B,GAAG,0BAA0B,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAA;IAE1G,MAAM,WAAW,GAAiB,EAAE,CAAA;IAEpC,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,eAAe,EAAE,eAAe,EAAE,EAAE,CAAC;QACpF,MAAM,eAAe,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAA;QAE7D,MAAM,aAAa,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAA;QACzG,MAAM,cAAc,GAAG,eAAe,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAA;QAE5H,MAAM,KAAK,GAAG,cAAc,GAAG,aAAa,CAAA;QAC5C,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,CAAA;QAE3B,IAAI,UAAU,GAAG,CAAC,CAAC,CAAA;QACnB,IAAI,OAAO,GAAa,EAAE,CAAA;QAE1B,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,KAAK,IAAI,sBAAsB,GAAG,CAAC,EAAE,sBAAsB,GAAG,2BAA2B,CAAC,MAAM,EAAE,sBAAsB,EAAE,EAAE,CAAC;YAC5H,MAAM,6BAA6B,GAAG,2BAA2B,CAAC,sBAAsB,CAAC,CAAA;YAEzF,IAAI,MAAM,GAAG,CAAC,CAAA;YAEd,IAAI,6BAA6B,IAAI,aAAa,IAAI,6BAA6B,IAAI,eAAe,EAAE,CAAC;gBACxG,MAAM,GAAG,CAAC,6BAA6B,GAAG,aAAa,CAAC,GAAG,SAAS,CAAA;YACrE,CAAC;iBAAM,IAAI,6BAA6B,GAAG,eAAe,IAAI,6BAA6B,IAAI,cAAc,EAAE,CAAC;gBAC/G,MAAM,GAAG,CAAC,cAAc,GAAG,6BAA6B,CAAC,GAAG,SAAS,CAAA;YACtE,CAAC;YAED,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChB,IAAI,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC;oBACtB,UAAU,GAAG,sBAAsB,CAAA;gBACpC,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACpB,SAAS,IAAI,MAAM,CAAA;YACpB,CAAC;iBAAM,IAAI,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC7B,MAAK;YACN,CAAC;QACF,CAAC;QAED,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;QAEnD,WAAW,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAA;IAC1C,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,kCAAkC,CAAC,YAAoB,EAAE,iBAAyB,EAAE,iBAAyB;IAC5H,MAAM,WAAW,GAAG,CAAC,iBAAiB,GAAG,iBAAiB,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,CAAA;IAEhF,MAAM,iBAAiB,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAA;IAExD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,iBAAiB,CAAC,CAAC,CAAC,GAAG,iBAAiB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,iBAAiB,CAAA;AACzB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC3C,OAAO,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAW;IACrC,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAA;AACpD,CAAC"}
|