echogarden 2.8.6 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/Recognition.d.ts +3 -3
- package/dist/api/Recognition.js +5 -5
- package/dist/api/Recognition.js.map +1 -1
- package/dist/api/SpeechTranslation.d.ts +4 -4
- package/dist/api/SpeechTranslation.js +4 -4
- package/dist/api/SpeechTranslation.js.map +1 -1
- package/dist/denoising/NSNet2.js +8 -8
- package/dist/denoising/NSNet2.js.map +1 -1
- package/dist/dsp/FFT.d.ts +2 -2
- package/dist/dsp/FFT.js +19 -15
- package/dist/dsp/FFT.js.map +1 -1
- package/dist/dsp/MFCC.d.ts +1 -1
- package/dist/dsp/MFCC.js +8 -8
- package/dist/dsp/MFCC.js.map +1 -1
- package/dist/dsp/{MelSpectogram.d.ts → MelSpectrogram.d.ts} +5 -5
- package/dist/dsp/{MelSpectogram.js → MelSpectrogram.js} +9 -9
- package/dist/dsp/MelSpectrogram.js.map +1 -0
- package/dist/math/VectorMath.d.ts +5 -1
- package/dist/math/VectorMath.js +15 -2
- package/dist/math/VectorMath.js.map +1 -1
- package/dist/nlp/IPA.d.ts +3 -3
- package/dist/recognition/WhisperSTT.d.ts +9 -6
- package/dist/recognition/WhisperSTT.js +92 -60
- package/dist/recognition/WhisperSTT.js.map +1 -1
- package/dist/source-separation/MDXNetSourceSeparation.js +5 -5
- package/dist/source-separation/MDXNetSourceSeparation.js.map +1 -1
- package/dist/speech-embeddings/WavToVec2BertFeatureEmbeddings.js +14 -14
- package/dist/speech-embeddings/WavToVec2BertFeatureEmbeddings.js.map +1 -1
- package/dist/utilities/Utilities.d.ts +1 -1
- package/dist/utilities/Utilities.js +4 -2
- package/dist/utilities/Utilities.js.map +1 -1
- package/docs/Licenses.md +0 -2
- package/docs/Tasklist.md +18 -5
- package/package.json +14 -14
- package/src/api/Recognition.ts +10 -7
- package/src/api/SpeechTranslation.ts +16 -6
- package/src/denoising/NSNet2.ts +8 -8
- package/src/dsp/FFT.ts +23 -17
- package/src/dsp/MFCC.ts +8 -8
- package/src/dsp/{MelSpectogram.ts → MelSpectrogram.ts} +9 -8
- package/src/math/VectorMath.ts +22 -2
- package/src/recognition/WhisperSTT.ts +131 -65
- package/src/source-separation/MDXNetSourceSeparation.ts +5 -5
- package/src/speech-embeddings/WavToVec2BertFeatureEmbeddings.ts +14 -14
- package/src/utilities/Utilities.ts +4 -2
- package/dist/dsp/MelSpectogram.js.map +0 -1
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { AudioSourceParam, RawAudio } from '../audio/AudioUtilities.js';
|
|
2
1
|
import * as API from './API.js';
|
|
2
|
+
import { AudioSourceParam, RawAudio } from '../audio/AudioUtilities.js';
|
|
3
3
|
import { Timeline } from '../utilities/Timeline.js';
|
|
4
|
-
import { type WhisperOptions } from '../recognition/WhisperSTT.js';
|
|
4
|
+
import { type WhisperPartCallback, type WhisperOptions } from '../recognition/WhisperSTT.js';
|
|
5
5
|
import { type SubtitlesConfig } from '../subtitles/Subtitles.js';
|
|
6
6
|
import { type OpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js';
|
|
7
7
|
import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js';
|
|
8
8
|
import { type SileroRecognitionOptions } from '../recognition/SileroSTT.js';
|
|
9
9
|
import { type DeepgramSTTOptions } from '../recognition/DeepgramSTT.js';
|
|
10
|
-
export declare function recognize(input: AudioSourceParam, options: RecognitionOptions): Promise<RecognitionResult>;
|
|
10
|
+
export declare function recognize(input: AudioSourceParam, options: RecognitionOptions, onPart?: WhisperPartCallback): Promise<RecognitionResult>;
|
|
11
11
|
export interface RecognitionResult {
|
|
12
12
|
transcript: string;
|
|
13
13
|
timeline: Timeline;
|
package/dist/api/Recognition.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import * as API from './API.js';
|
|
1
3
|
import { extendDeep } from '../utilities/ObjectUtilities.js';
|
|
2
4
|
import { logToStderr } from '../utilities/Utilities.js';
|
|
3
5
|
import { ensureRawAudio, normalizeAudioLevelInPlace, trimAudioEnd } from '../audio/AudioUtilities.js';
|
|
4
6
|
import { Logger } from '../utilities/Logger.js';
|
|
5
|
-
import * as API from './API.js';
|
|
6
7
|
import { addWordTextOffsetsToTimelineInPlace, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js';
|
|
7
8
|
import { formatLanguageCodeWithName, parseLangIdentifier } from '../utilities/Locale.js';
|
|
8
9
|
import { loadPackage } from '../utilities/PackageManager.js';
|
|
9
|
-
import chalk from 'chalk';
|
|
10
10
|
const log = logToStderr;
|
|
11
|
-
export async function recognize(input, options) {
|
|
11
|
+
export async function recognize(input, options, onPart) {
|
|
12
12
|
const logger = new Logger();
|
|
13
13
|
const startTimestamp = logger.getTimestamp();
|
|
14
14
|
options = extendDeep(defaultRecognitionOptions, options);
|
|
@@ -65,7 +65,7 @@ export async function recognize(input, options) {
|
|
|
65
65
|
logger.end();
|
|
66
66
|
const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, shortLanguageCode);
|
|
67
67
|
logger.end();
|
|
68
|
-
({ transcript, timeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, 'transcribe', shortLanguageCode, whisperOptions));
|
|
68
|
+
({ transcript, timeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, 'transcribe', shortLanguageCode, whisperOptions, onPart));
|
|
69
69
|
break;
|
|
70
70
|
}
|
|
71
71
|
case 'whisper.cpp': {
|
|
@@ -193,7 +193,7 @@ export async function recognize(input, options) {
|
|
|
193
193
|
// Make segment timeline
|
|
194
194
|
const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(timeline, transcript, languageCode, 'single', 'preserve');
|
|
195
195
|
logger.end();
|
|
196
|
-
logger.logDuration('
|
|
196
|
+
logger.logDuration('\nTotal recognition time', startTimestamp, chalk.magentaBright);
|
|
197
197
|
return {
|
|
198
198
|
transcript,
|
|
199
199
|
timeline: segmentTimeline,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Recognition.js","sourceRoot":"","sources":["../../src/api/Recognition.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAA8B,cAAc,EAAE,0BAA0B,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACjI,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,
|
|
1
|
+
{"version":3,"file":"Recognition.js","sourceRoot":"","sources":["../../src/api/Recognition.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAA;AAEzB,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAE/B,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAA8B,cAAc,EAAE,0BAA0B,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACjI,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,EAAY,mCAAmC,EAAE,qCAAqC,EAAE,MAAM,0BAA0B,CAAA;AAC/H,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAA;AACxF,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AAU5D,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAuB,EAAE,OAA2B,EAAE,MAA4B;IACjH,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;IAE5C,OAAO,GAAG,UAAU,CAAC,yBAAyB,EAAE,OAAO,CAAC,CAAA;IAExD,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAA;IAEjD,IAAI,cAAwB,CAAA;IAC5B,IAAI,gBAAsC,CAAA;IAC1C,IAAI,kBAAwC,CAAA;IAE5C,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACd,MAAM,CAAC,GAAG,EAAE,CAAC;QAEb,CAAC,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,gBAAiB,CAAC,CAAC,CAAA;QAExG,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEd,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAC5C,cAAc,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAClE,CAAC;SAAM,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAC5C,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED,IAAI,oBAA0C,CAAA;IAE9C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACpD,CAAC,EAAE,QAAQ,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,GAAG,MAAM,GAAG,CAAC,mBAAmB,CAAC,cAAc,EAAE,OAAO,CAAC,GAAI,CAAC,CAAC,CAAA;QAEnI,MAAM,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;IAExC,0BAA0B,CAAC,cAAc,CAAC,CAAA;IAC1C,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IAE/E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAO,CAAA;IAE9B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QAEhE,OAAO,CAAC,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAA;QAEpC,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,EAAE,0BAA0B,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAA;IAC5F,CAAC;SAAM,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAA;QAC7D,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,GAAG,CAAC,oBAAoB,CAAC,cAAc,EAAE,OAAO,CAAC,iBAAkB,CAAC,CAAA;QAEvG,OAAO,CAAC,QAAQ,GAAG,gBAAgB,CAAA;QAEnC,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,CAAA;IAC3F,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAEhE,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAA;IACtC,MAAM,iBAAiB,GAAG,YAAY,CAAC,wBAAwB,CAAA;IAE/D,IAAI,UAAkB,CAAA;IACtB,IAAI,QAA8B,CAAA;IAElC,MAAM,CAAC,KAAK,CAAC,QAAQ,MAAM,SAAS,CAAC,CAAA;IAErC,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,SAAS,CAAC,CAAC,CAAC;YAChB,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAA;YAE/D,MAAM,cAAc,GAAG,OAAO,CAAC,OAAQ,CAAA;YAEvC,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,uBAAuB,CAAC,cAAc,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAA;YAEjH,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,SAAS,CACrD,cAAc,EACd,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,MAAM,CACN,CAAC,CAAA;YAEF,MAAK;QACN,CAAC;QAED,KAAK,aAAa,CAAC,CAAC,CAAC;YACpB,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAA;YAErE,MAAM,iBAAiB,GAAG,OAAO,CAAC,UAAW,CAAA;YAE7C,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAA;YAEjH,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,aAAa,CAAC,SAAS,CACxD,cAAc,EACd,YAAY,EACZ,iBAAiB,EACjB,SAAS,EACT,SAAS,EACT,iBAAiB,CACjB,CAAC,CAAA;YAEF,MAAK;QACN,CAAC;QAED,KAAK,MAAM,CAAC,CAAC,CAAC;YACb,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;YAEzD,IAAI,CAAC;gBACJ,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;YACjC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACZ,GAAG,CAAC,CAAC,CAAC,CAAA;gBACN,MAAM,IAAI,KAAK,CAAC,gLAAgL,CAAC,CAAA;YAClM,CAAC;YAED,MAAM,WAAW,GAAG,OAAO,CAAC,IAAK,CAAA;YAEjC,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAA;YAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,mIAAmI,CAAC,CAAA;YACrJ,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,CAAA;YAErF,MAAK;QACN,CAAC;QAED,KAAK,QAAQ,CAAC,CAAC,CAAC;YACf,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAA;YAE7D,MAAM,aAAa,GAAG,OAAO,CAAC,MAAO,CAAA;YAErC,IAAI,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;YAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,SAAS,CAAC,yBAAyB,CAAC,iBAAiB,CAAC,CAAA;gBAE1E,IAAI,CAAC,WAAW,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,aAAa,iBAAiB,8BAA8B,CAAC,CAAA;gBAC9E,CAAC;gBAED,SAAS,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,CAAA;YAC3C,CAAC;YAED,MAAM,sBAAsB,GAA4B,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;YAE9G,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,SAAS,CAAC,SAAS,CACpD,cAAc,EACd,SAAS,EACT,sBAAsB,CAAC,CAAC,CAAA;YAEzB,MAAK;QACN,CAAC;QAED,KAAK,cAAc,CAAC,CAAC,CAAC;YACrB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAA;YAEvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAY,CAAC,MAAM,CAAA;YAE1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAA;YACpC,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAA;YAEtG,MAAK;QACN,CAAC;QAED,KAAK,iBAAiB,CAAC,CAAC,CAAC;YACxB,MAAM,yBAAyB,GAAG,MAAM,MAAM,CAAC,6CAA6C,CAAC,CAAA;YAE7F,MAAM,eAAe,GAAG,OAAO,CAAC,cAAe,CAAC,eAAe,CAAA;YAE/D,IAAI,CAAC,eAAe,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;YAChE,CAAC;YAED,MAAM,aAAa,GAAG,OAAO,CAAC,cAAe,CAAC,aAAa,CAAA;YAE3D,IAAI,CAAC,aAAa,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,yBAAyB,CAAC,SAAS,CAAC,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAA;YAEzI,MAAK;QACN,CAAC;QAED,KAAK,mBAAmB,CAAC,CAAC,CAAC;YAC1B,MAAM,mBAAmB,GAAG,MAAM,MAAM,CAAC,uCAAuC,CAAC,CAAA;YAEjF,MAAM,MAAM,GAAG,OAAO,CAAC,gBAAiB,CAAC,MAAM,CAAA;YAE/C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;YACxD,CAAC;YAED,MAAM,WAAW,GAAG,OAAO,CAAC,gBAAiB,CAAC,WAAW,CAAA;YAEzD,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAA;YAC/D,CAAC;YAED,MAAM,eAAe,GAAG,OAAO,CAAC,gBAAiB,CAAC,eAAe,CAAA;YAEjE,IAAI,CAAC,eAAe,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;YACnE,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,CAAC,CAAC,CAAA;YAEpI,MAAK;QACN,CAAC;QAED,KAAK,cAAc,CAAC,CAAC,CAAC;YACrB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAA;YAEvE,MAAM,qBAAqB,GAAG,OAAO,CAAC,WAAY,CAAA;YAElD,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;YACpD,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,cAAc,EAAE,iBAAiB,EAAE,qBAAqB,CAAC,CAAC,CAAA;YAErH,MAAK;QACN,CAAC;QAED,KAAK,UAAU,CAAC,CAAC,CAAC;YACjB,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAA;YAEjE,MAAM,eAAe,GAAG,OAAO,CAAC,QAAS,CAAA;YAEzC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAA;YAChD,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC,CAAA;YAE3I,MAAK;QACN,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,0DAA0D;IAC1D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAA;QACzC,MAAM,gBAAgB,GAAyB,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAA;QAExG,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAA;QAEtF,QAAQ,GAAG,YAAY,CAAA;IACxB,CAAC;IAED,6FAA6F;IAC7F,IAAI,oBAAoB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,GAAG,CAAC,iCAAiC,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAA;IACtE,CAAC;IAED,mBAAmB;IACnB,mCAAmC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IAEzD,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,qCAAqC,CAAC,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAA;IAEjI,MAAM,CAAC,GAAG,EAAE,CAAA;IACZ,MAAM,CAAC,WAAW,CAAC,0BAA0B,EAAE,cAAc,EAAE,KAAK,CAAC,aAAa,CAAC,CAAA;IAEnF,OAAO;QACN,UAAU;QAEV,QAAQ,EAAE,eAAe;QACzB,YAAY,EAAE,QAAQ;QAEtB,QAAQ,EAAE,YAAY;QAEtB,aAAa;QACb,gBAAgB;QAChB,kBAAkB;KAClB,CAAA;AACF,CAAC;AAwED,MAAM,CAAC,MAAM,yBAAyB,GAAuB;IAC5D,MAAM,EAAE,SAAS;IAEjB,QAAQ,EAAE,SAAS;IAEnB,eAAe,EAAE,CAAC;IAElB,OAAO,EAAE,KAAK;IAEd,IAAI,EAAE,IAAI;IAEV,SAAS,EAAE,EACV;IAED,iBAAiB,EAAE,EAClB;IAED,SAAS,EAAE,EACV;IAED,GAAG,EAAE;QACJ,MAAM,EAAE,eAAe;KACvB;IAED,OAAO,EAAE,EACR;IAED,UAAU,EAAE,EACX;IAED,IAAI,EAAE;QACL,SAAS,EAAE,SAAS;KACpB;IAED,MAAM,EAAE,EACP;IAED,WAAW,EAAE;QACZ,MAAM,EAAE,SAAS;QACjB,wBAAwB,EAAE,EAAE;QAC5B,eAAe,EAAE,KAAK;QACtB,eAAe,EAAE,IAAI;QACrB,gBAAgB,EAAE,IAAI;KACtB;IAED,cAAc,EAAE;QACf,eAAe,EAAE,SAAS;QAC1B,aAAa,EAAE,SAAS;KACxB;IAED,gBAAgB,EAAE;QACjB,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,SAAS;QACtB,eAAe,EAAE,SAAS;KAC1B;IAED,WAAW,EAAE,EACZ;IAED,QAAQ,EAAE,EACT;CACD,CAAA;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAyB;IACvD;QACC,EAAE,EAAE,SAAS;QACb,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,8EAA8E;QAC3F,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,aAAa;QACjB,IAAI,EAAE,2BAA2B;QACjC,WAAW,EAAE,4DAA4D;QACzE,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,+BAA+B;QAC5C,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE,4BAA4B;QACzC,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,cAAc;QAClB,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE,sCAAsC;QACnD,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,iBAAiB;QACrB,IAAI,EAAE,0BAA0B;QAChC,WAAW,EAAE,yCAAyC;QACtD,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,mBAAmB;QACvB,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,sCAAsC;QACnD,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,cAAc;QAClB,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE,sCAAsC;QACnD,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,wCAAwC;QACrD,IAAI,EAAE,OAAO;KACb;CACD,CAAA"}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
+
import * as API from './API.js';
|
|
1
2
|
import { AudioSourceParam, RawAudio } from '../audio/AudioUtilities.js';
|
|
2
3
|
import { Timeline } from '../utilities/Timeline.js';
|
|
3
|
-
import { type WhisperOptions } from '../recognition/WhisperSTT.js';
|
|
4
|
-
import { EngineMetadata } from './Common.js';
|
|
4
|
+
import { WhisperPartCallback, type WhisperOptions } from '../recognition/WhisperSTT.js';
|
|
5
|
+
import { type EngineMetadata } from './Common.js';
|
|
5
6
|
import { type SpeechLanguageDetectionOptions } from './API.js';
|
|
6
7
|
import { type SubtitlesConfig } from '../subtitles/Subtitles.js';
|
|
7
|
-
import * as API from './API.js';
|
|
8
8
|
import { type OpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js';
|
|
9
9
|
import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js';
|
|
10
|
-
export declare function translateSpeech(input: AudioSourceParam, options: SpeechTranslationOptions): Promise<SpeechTranslationResult>;
|
|
10
|
+
export declare function translateSpeech(input: AudioSourceParam, options: SpeechTranslationOptions, onPart?: WhisperPartCallback): Promise<SpeechTranslationResult>;
|
|
11
11
|
export interface SpeechTranslationResult {
|
|
12
12
|
transcript: string;
|
|
13
13
|
timeline: Timeline;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import * as API from './API.js';
|
|
1
3
|
import { extendDeep } from '../utilities/ObjectUtilities.js';
|
|
2
4
|
import { logToStderr } from '../utilities/Utilities.js';
|
|
3
5
|
import { ensureRawAudio, normalizeAudioLevelInPlace, trimAudioEnd } from '../audio/AudioUtilities.js';
|
|
@@ -5,13 +7,11 @@ import { Logger } from '../utilities/Logger.js';
|
|
|
5
7
|
import { addWordTextOffsetsToTimelineInPlace, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js';
|
|
6
8
|
import { formatLanguageCodeWithName, getShortLanguageCode, normalizeIdentifierToLanguageCode, parseLangIdentifier } from '../utilities/Locale.js';
|
|
7
9
|
import { detectSpeechLanguage } from './API.js';
|
|
8
|
-
import chalk from 'chalk';
|
|
9
|
-
import * as API from './API.js';
|
|
10
10
|
const log = logToStderr;
|
|
11
11
|
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
12
12
|
// Speech translation
|
|
13
13
|
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
14
|
-
export async function translateSpeech(input, options) {
|
|
14
|
+
export async function translateSpeech(input, options, onPart) {
|
|
15
15
|
const logger = new Logger();
|
|
16
16
|
const startTimestamp = logger.getTimestamp();
|
|
17
17
|
options = extendDeep(defaultSpeechTranslationOptions, options);
|
|
@@ -81,7 +81,7 @@ export async function translateSpeech(input, options) {
|
|
|
81
81
|
throw new Error('Both translation source and target languages are English');
|
|
82
82
|
}
|
|
83
83
|
logger.end();
|
|
84
|
-
({ transcript, timeline: wordTimeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, 'translate', sourceLanguage, whisperOptions));
|
|
84
|
+
({ transcript, timeline: wordTimeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, 'translate', sourceLanguage, whisperOptions, onPart));
|
|
85
85
|
break;
|
|
86
86
|
}
|
|
87
87
|
case 'whisper.cpp': {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SpeechTranslation.js","sourceRoot":"","sources":["../../src/api/SpeechTranslation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAA8B,cAAc,EAAE,0BAA0B,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACjI,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,EAAY,mCAAmC,EAAE,qCAAqC,EAAE,MAAM,0BAA0B,CAAA;AAE/H,OAAO,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAA;AAEjJ,OAAO,EAAuC,oBAAoB,EAAE,MAAM,UAAU,CAAA;
|
|
1
|
+
{"version":3,"file":"SpeechTranslation.js","sourceRoot":"","sources":["../../src/api/SpeechTranslation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAA;AAEzB,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAE/B,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAA8B,cAAc,EAAE,0BAA0B,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACjI,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,EAAY,mCAAmC,EAAE,qCAAqC,EAAE,MAAM,0BAA0B,CAAA;AAE/H,OAAO,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAA;AAEjJ,OAAO,EAAuC,oBAAoB,EAAE,MAAM,UAAU,CAAA;AAMpF,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,6FAA6F;AAC7F,qBAAqB;AACrB,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAuB,EAAE,OAAiC,EAAE,MAA4B;IAC7H,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;IAE5C,OAAO,GAAG,UAAU,CAAC,+BAA+B,EAAE,OAAO,CAAC,CAAA;IAE9D,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAA;IAEjD,IAAI,cAAwB,CAAA;IAC5B,IAAI,gBAAsC,CAAA;IAC1C,IAAI,kBAAwC,CAAA;IAE5C,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACd,MAAM,CAAC,GAAG,EAAE,CAAC;QAEb,CAAC,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,gBAAiB,CAAC,CAAC,CAAA;QAExG,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEd,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAC5C,cAAc,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAClE,CAAC;SAAM,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAC5C,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED,IAAI,oBAA0C,CAAA;IAE9C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACpD,CAAC,EAAE,QAAQ,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,GAAG,MAAM,GAAG,CAAC,mBAAmB,CAAC,cAAc,EAAE,OAAO,CAAC,GAAI,CAAC,CAAC,CAAA;QAEnI,MAAM,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;IAExC,0BAA0B,CAAC,cAAc,CAAC,CAAA;IAC1C,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IAE/E,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;QAEtE,OAAO,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAA;QAE1C,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,gBAAgB,CAAC,2BAA2B,EAAE,0BAA0B,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IACzG,CAAC;SAAM,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAA;QACpE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,oBAAoB,CAAC,cAAc,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAA;QAExG,OAAO,CAAC,cAAc,GAAG,gBAAgB,CAAA;QAEzC,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,gBAAgB,CAAC,0BAA0B,EAAE,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,CAAA;IAClG,CAAC;IAED,OAAO,CAAC,cAAc,GAAG,MAAM,iCAAiC,CAAC,OAAO,CAAC,cAAe,CAAC,CAAA;IAEzF,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,0BAA0B,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IAE9F,MAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAA;IAEhD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAO,CAAA;IAC9B,MAAM,cAAc,GAAG,OAAO,CAAC,cAAe,CAAA;IAC9C,MAAM,cAAc,GAAG,OAAO,CAAC,cAAe,CAAA;IAE9C,IAAI,UAAkB,CAAA;IACtB,IAAI,YAAkC,CAAA;IACtC,IAAI,eAAqC,CAAA;IAEzC,MAAM,CAAC,KAAK,CAAC,QAAQ,MAAM,SAAS,CAAC,CAAA;IAErC,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,SAAS,CAAC,CAAC,CAAC;YAChB,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAA;YAE/D,MAAM,cAAc,GAAG,OAAO,CAAC,OAAQ,CAAA;YAEvC,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YACpE,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YAEpE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,uBAAuB,CAAC,cAAc,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAA;YAEvH,IAAI,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAA;YAChF,CAAC;YAED,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAA;YACzF,CAAC;YAED,IAAI,uBAAuB,IAAI,IAAI,IAAI,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBACxE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;YAC5E,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,UAAU,CAAC,SAAS,CACnE,cAAc,EACd,SAAS,EACT,QAAQ,EACR,WAAW,EACX,cAAc,EACd,cAAc,EACd,MAAM,CACN,CAAC,CAAA;YAEF,MAAK;QACN,CAAC;QAED,KAAK,aAAa,CAAC,CAAC,CAAC;YACpB,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAA;YAErE,MAAM,iBAAiB,GAAG,OAAO,CAAC,UAAW,CAAA;YAE7C,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YACpE,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YAEpE,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAA;YAEvH,IAAI,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAA;YACpF,CAAC;YAED,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAA;YAC7F,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,aAAa,CAAC,SAAS,CACtE,cAAc,EACd,WAAW,EACX,uBAAuB,EACvB,SAAS,EACT,SAAS,EACT,iBAAiB,CACjB,CAAC,CAAA;YAEF,MAAK;QACN,CAAC;QAED,KAAK,cAAc,CAAC,CAAC,CAAC;YACrB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAA;YAEvE,MAAM,qBAAqB,GAAG,OAAO,CAAC,WAAY,CAAA;YAElD,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;YACpD,CAAC;YAED,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YACpE,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YAEpE,IAAI,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAA;YAC5F,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,cAAc,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,WAAW,CAAC,CAAC,CAAA;YAEzJ,MAAK;QACN,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,6FAA6F;IAC7F,IAAI,oBAAoB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,IAAI,YAAY,EAAE,CAAC;YAClB,GAAG,CAAC,iCAAiC,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAA;QAC1E,CAAC;aAAM,IAAI,eAAe,EAAE,CAAC;YAC5B,GAAG,CAAC,iCAAiC,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAA;QAC7E,CAAC;IACF,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QAClB,mCAAmC,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;IAC9D,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACtB,CAAC,EAAE,eAAe,EAAE,GAAG,MAAM,qCAAqC,CAAC,YAAa,EAAE,UAAU,EAAE,cAAc,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAA;IACrI,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACd,MAAM,CAAC,WAAW,CAAC,+BAA+B,EAAE,cAAc,EAAE,KAAK,CAAC,aAAa,CAAC,CAAA;IAExF,OAAO;QACN,UAAU;QACV,QAAQ,EAAE,eAAe;QACzB,YAAY;QAEZ,cAAc;QACd,cAAc;QAEd,aAAa;QACb,gBAAgB;QAChB,kBAAkB;KAClB,CAAA;AACF,CAAC;AAoCD,MAAM,CAAC,MAAM,+BAA+B,GAA6B;IACxE,MAAM,EAAE,SAAS;IAEjB,cAAc,EAAE,SAAS;IACzB,cAAc,EAAE,IAAI;IAEpB,IAAI,EAAE,IAAI;IACV,OAAO,EAAE,KAAK;IAEd,iBAAiB,EAAE,SAAS;IAE5B,SAAS,EAAE,EACV;IAED,GAAG,EAAE;QACJ,MAAM,EAAE,eAAe;KACvB;IAED,OAAO,EAAE,EACR;IAED,UAAU,EAAE,EACX;IAED,WAAW,EAAE,EACZ;CACD,CAAA;AAED,MAAM,CAAC,MAAM,wBAAwB,GAAqB;IACzD;QACC,EAAE,EAAE,SAAS;QACb,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,oHAAoH;QACjI,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,aAAa;QACjB,IAAI,EAAE,2BAA2B;QACjC,WAAW,EAAE,oHAAoH;QACjI,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,cAAc;QAClB,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE,+FAA+F;QAC5G,IAAI,EAAE,OAAO;KACb;CACD,CAAA"}
|
package/dist/denoising/NSNet2.js
CHANGED
|
@@ -47,8 +47,8 @@ export class NSNet2 {
|
|
|
47
47
|
const fftRealBinCount = (fftSize / 2) + 1;
|
|
48
48
|
logger.start('Compute STFT frames');
|
|
49
49
|
const stftrFrames = await stftr(rawAudio.audioChannels[0], fftSize, fftSize, fftHopSize, 'hann');
|
|
50
|
-
logger.start('Compute log-power
|
|
51
|
-
let
|
|
50
|
+
logger.start('Compute log-power spectrogram');
|
|
51
|
+
let logPowerSpectrogram = [];
|
|
52
52
|
{
|
|
53
53
|
for (const frame of stftrFrames) {
|
|
54
54
|
const logPowerSpectrum = new Float32Array(frame.length / 2);
|
|
@@ -62,15 +62,15 @@ export class NSNet2 {
|
|
|
62
62
|
const logPowerValue = Math.log10(clampedPowerValue);
|
|
63
63
|
logPowerSpectrum[writeOffset++] = logPowerValue;
|
|
64
64
|
}
|
|
65
|
-
|
|
65
|
+
logPowerSpectrogram.push(logPowerSpectrum);
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
|
-
logger.start('Process log-power
|
|
69
|
-
const frameCount =
|
|
68
|
+
logger.start('Process log-power spectrogram using ONNX model');
|
|
69
|
+
const frameCount = logPowerSpectrogram.length;
|
|
70
70
|
let flattenedOutputTensor;
|
|
71
71
|
{
|
|
72
72
|
const Onnx = await import('onnxruntime-node');
|
|
73
|
-
const flattenedFeatures = concatFloat32Arrays(
|
|
73
|
+
const flattenedFeatures = concatFloat32Arrays(logPowerSpectrogram);
|
|
74
74
|
const inputTensor = new Onnx.Tensor('float32', flattenedFeatures, [1, frameCount, fftRealBinCount]);
|
|
75
75
|
const inputs = { input: inputTensor };
|
|
76
76
|
const result = await this.session.run(inputs);
|
|
@@ -93,8 +93,8 @@ export class NSNet2 {
|
|
|
93
93
|
}
|
|
94
94
|
}
|
|
95
95
|
}
|
|
96
|
-
// Allow
|
|
97
|
-
|
|
96
|
+
// Allow logPowerSpectrogram to be garbage collected
|
|
97
|
+
logPowerSpectrogram = undefined;
|
|
98
98
|
logger.start('Reconstruct filtered signal using inverse STFT');
|
|
99
99
|
const filteredSignal = await stiftr(stftrFrames, fftSize, fftSize, fftHopSize, 'hann');
|
|
100
100
|
const denoisedAudio = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"NSNet2.js","sourceRoot":"","sources":["../../src/denoising/NSNet2.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAY,MAAM,4BAA4B,CAAA;AAG3E,OAAO,EAAE,qBAAqB,EAAyB,MAAM,+BAA+B,CAAA;AAC5F,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAA;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAA;AACrE,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAkB,EAAE,OAAsB;IAC5E,MAAM,sBAAsB,GAA4B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA,CAAA,gBAAgB;IAElH,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,KAAM,EAAE,OAAO,CAAC,kBAAmB,EAAE,sBAAsB,EAAE,OAAO,CAAC,cAAe,CAAC,CAAA;IAEzH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;IAEpD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,OAAO,MAAM;IAID;IACA;IACA;IACA;IANjB,OAAO,CAAwB;IAE/B,YACiB,SAA0B,EAC1B,kBAA0B,EAC1B,kBAA2C,EAC3C,cAAsB;QAHtB,cAAS,GAAT,SAAS,CAAiB;QAC1B,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,uBAAkB,GAAlB,kBAAkB,CAAyB;QAC3C,mBAAc,GAAd,cAAc,CAAQ;IACvC,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,QAAkB;QACpC,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;QAE3B,MAAM,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;QACvD,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAE/B,IAAI,OAAe,CAAA;QAEnB,IAAI,IAAI,CAAC,SAAS,KAAK,gBAAgB,EAAE,CAAC;YACzC,OAAO,GAAG,IAAI,CAAA;YAEd,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,KAAK,gBAAgB,EAAE,CAAC;YAChD,OAAO,GAAG,GAAG,CAAA;YAEb,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,CAAA;QAC9B,MAAM,eAAe,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;QAEzC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;QACnC,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;QAEhG,MAAM,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"NSNet2.js","sourceRoot":"","sources":["../../src/denoising/NSNet2.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAY,MAAM,4BAA4B,CAAA;AAG3E,OAAO,EAAE,qBAAqB,EAAyB,MAAM,+BAA+B,CAAA;AAC5F,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAA;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAA;AACrE,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAkB,EAAE,OAAsB;IAC5E,MAAM,sBAAsB,GAA4B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA,CAAA,gBAAgB;IAElH,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,KAAM,EAAE,OAAO,CAAC,kBAAmB,EAAE,sBAAsB,EAAE,OAAO,CAAC,cAAe,CAAC,CAAA;IAEzH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;IAEpD,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,OAAO,MAAM;IAID;IACA;IACA;IACA;IANjB,OAAO,CAAwB;IAE/B,YACiB,SAA0B,EAC1B,kBAA0B,EAC1B,kBAA2C,EAC3C,cAAsB;QAHtB,cAAS,GAAT,SAAS,CAAiB;QAC1B,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,uBAAkB,GAAlB,kBAAkB,CAAyB;QAC3C,mBAAc,GAAd,cAAc,CAAQ;IACvC,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,QAAkB;QACpC,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;QAE3B,MAAM,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;QACvD,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAE/B,IAAI,OAAe,CAAA;QAEnB,IAAI,IAAI,CAAC,SAAS,KAAK,gBAAgB,EAAE,CAAC;YACzC,OAAO,GAAG,IAAI,CAAA;YAEd,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,KAAK,gBAAgB,EAAE,CAAC;YAChD,OAAO,GAAG,GAAG,CAAA;YAEb,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,CAAA;QAC9B,MAAM,eAAe,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;QAEzC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;QACnC,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;QAEhG,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;QAC7C,IAAI,mBAAmB,GAAmB,EAAE,CAAA;QAE5C,CAAC;YACA,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;gBACjC,MAAM,gBAAgB,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;gBAE3D,IAAI,UAAU,GAAG,CAAC,CAAA;gBAClB,IAAI,WAAW,GAAG,CAAC,CAAA;gBAEnB,OAAO,UAAU,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBAClC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,CAAA;oBAChC,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,CAAA;oBAErC,MAAM,UAAU,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC,CAAA;oBACjD,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;oBACrD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;oBAEnD,gBAAgB,CAAC,WAAW,EAAE,CAAC,GAAG,aAAa,CAAA;gBAChD,CAAC;gBAED,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAC3C,CAAC;QACF,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAA;QAE9D,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,CAAA;QAC7C,IAAI,qBAAmC,CAAA;QAEvC,CAAC;YACA,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;YAE7C,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,mBAAmB,CAAC,CAAA;YAElE,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,iBAAiB,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC,CAAA;YACnG,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,CAAA;YAErC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAE9C,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,IAAoB,CAAA;QAC3D,CAAC;QAED,CAAC;YACA,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAA;YAEtE,MAAM,iBAAiB,GAAG,CAAC,GAAG,OAAO,CAAA;YAErC,MAAM,YAAY,GAAG,oBAAoB,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;YAC/D,MAAM,YAAY,GAAG,GAAG,CAAA;YAExB,IAAI,gCAAgC,GAAG,CAAC,CAAA;YAExC,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,UAAU,EAAE,UAAU,EAAE,EAAE,CAAC;gBAChE,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,CAAA;gBAErC,IAAI,cAAc,GAAG,CAAC,CAAA;gBAEtB,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,eAAe,EAAE,QAAQ,EAAE,EAAE,CAAC;oBAC/D,IAAI,SAAS,GAAG,qBAAqB,CAAC,gCAAgC,EAAE,CAAC,CAAA;oBACzE,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,YAAY,CAAC,CAAA;oBAEvD,KAAK,CAAC,cAAc,EAAE,CAAC,IAAI,SAAS,GAAG,iBAAiB,CAAA;oBACxD,KAAK,CAAC,cAAc,EAAE,CAAC,IAAI,SAAS,GAAG,iBAAiB,CAAA;gBACzD,CAAC;YACF,CAAC;QACF,CAAC;QAED,oDAAoD;QACpD,mBAAmB,GAAG,SAAgB,CAAA;QAEtC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAA;QAC9D,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;QAEtF,MAAM,aAAa,GAAa;YAC/B,aAAa,EAAE,CAAC,cAAc,CAAC;YAC/B,UAAU,EAAE,QAAQ,CAAC,UAAU;SAC/B,CAAA;QAED,MAAM,CAAC,GAAG,EAAE,CAAA;QAEZ,OAAO,EAAE,aAAa,EAAE,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAM;QACP,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;QAC/D,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;QAEvF,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAA;QACnF,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAA;QAE1E,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QAE7C,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,EAAE,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAA;QAEjG,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACrF,CAAC;CACD;AAKD,MAAM,CAAC,MAAM,oBAAoB,GAAkB;IAClD,KAAK,EAAE,gBAAgB;IACvB,kBAAkB,EAAE,SAAS;IAC7B,QAAQ,EAAE,SAAS;IACnB,cAAc,EAAE,EAAE;CAClB,CAAA"}
|
package/dist/dsp/FFT.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { ComplexNumber } from '../math/VectorMath.js';
|
|
2
2
|
export declare function stftr(samples: Float32Array, fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType): Promise<Float32Array<ArrayBufferLike>[]>;
|
|
3
|
-
export declare function
|
|
3
|
+
export declare function createStftrGenerator(samples: Float32Array, fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType): Promise<Generator<Float32Array<ArrayBuffer>, void, unknown>>;
|
|
4
4
|
export declare function stiftr(binsForFrames: Float32Array[], fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType, expectedOutputLength?: number): Promise<Float32Array<ArrayBuffer>>;
|
|
5
5
|
export declare function getBinFrequencies(binCount: number, maxFrequency: number): Float32Array<ArrayBuffer>;
|
|
6
|
-
export declare function
|
|
6
|
+
export declare function fftFramesToPowerSpectrogram(fftFrames: Float32Array[]): Float32Array<ArrayBuffer>[];
|
|
7
7
|
export declare function fftFrameToPowerSpectrum(fftFrame: Float32Array): Float32Array<ArrayBuffer>;
|
|
8
8
|
export declare function binBufferToComplex(bins: Float32Array, extendAndMirror?: boolean): ComplexNumber[];
|
|
9
9
|
export declare function complexToBinBuffer(complexBins: ComplexNumber[]): Float32Array<ArrayBuffer>;
|
package/dist/dsp/FFT.js
CHANGED
|
@@ -3,13 +3,14 @@ import { createWasmHeapManager } from 'wasm-heap-manager';
|
|
|
3
3
|
// Compute short-term Fourier transform (real-valued)
|
|
4
4
|
export async function stftr(samples, fftOrder, windowSize, hopSize, windowType) {
|
|
5
5
|
const frames = [];
|
|
6
|
-
|
|
6
|
+
const stftrGenerator = await createStftrGenerator(samples, fftOrder, windowSize, hopSize, windowType);
|
|
7
|
+
for (const frame of stftrGenerator) {
|
|
7
8
|
frames.push(frame);
|
|
8
9
|
}
|
|
9
10
|
return frames;
|
|
10
11
|
}
|
|
11
12
|
// Incrementally generate short-term Fourier transform frames (real-valued)
|
|
12
|
-
export async function
|
|
13
|
+
export async function createStftrGenerator(samples, fftOrder, windowSize, hopSize, windowType) {
|
|
13
14
|
if (fftOrder % 2 !== 0 || windowSize % 2 !== 0) {
|
|
14
15
|
throw new Error('FFT order and window size must be multiples of 2');
|
|
15
16
|
}
|
|
@@ -30,20 +31,23 @@ export async function* stftrGenerator(samples, fftOrder, windowSize, hopSize, wi
|
|
|
30
31
|
const frameBufferRef = wasmHeap.allocFloat32Array(fftOrder);
|
|
31
32
|
const binsBufferRef = wasmHeap.allocFloat32Array(fftOrder * 2);
|
|
32
33
|
const workBufferRef = wasmHeap.allocFloat32Array(fftOrder * 2);
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
34
|
+
function* stftrGenerator() {
|
|
35
|
+
for (let offset = 0; offset < sampleCount; offset += hopSize) {
|
|
36
|
+
const windowSamples = samples.subarray(offset, offset + windowSize);
|
|
37
|
+
frameBufferRef.clear();
|
|
38
|
+
const frameBufferView = frameBufferRef.view;
|
|
39
|
+
for (let i = 0; i < windowSamples.length; i++) {
|
|
40
|
+
frameBufferView[i] = windowSamples[i] * windowWeights[i];
|
|
41
|
+
}
|
|
42
|
+
binsBufferRef.clear();
|
|
43
|
+
m._pffft_transform_ordered(statePtr, frameBufferRef.address, binsBufferRef.address, workBufferRef.address, 0);
|
|
44
|
+
const bins = binsBufferRef.view.slice(0, fftOrder + 2);
|
|
45
|
+
yield bins;
|
|
39
46
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
const bins = binsBufferRef.view.slice(0, fftOrder + 2);
|
|
43
|
-
yield bins;
|
|
47
|
+
m._pffft_destroy_setup(statePtr);
|
|
48
|
+
wasmHeap.freeAll();
|
|
44
49
|
}
|
|
45
|
-
|
|
46
|
-
wasmHeap.freeAll();
|
|
50
|
+
return stftrGenerator();
|
|
47
51
|
}
|
|
48
52
|
// Compute short-term inverse Fourier transform (real-valued)
|
|
49
53
|
export async function stiftr(binsForFrames, fftOrder, windowSize, hopSize, windowType, expectedOutputLength) {
|
|
@@ -106,7 +110,7 @@ export function getBinFrequencies(binCount, maxFrequency) {
|
|
|
106
110
|
return binFrequencies;
|
|
107
111
|
}
|
|
108
112
|
// Convert an array of raw FFT frames to a power spectrum
|
|
109
|
-
export function
|
|
113
|
+
export function fftFramesToPowerSpectrogram(fftFrames) {
|
|
110
114
|
return fftFrames.map(fftFrame => fftFrameToPowerSpectrum(fftFrame));
|
|
111
115
|
}
|
|
112
116
|
// Convert a raw FFT frame to a power spectrum
|
package/dist/dsp/FFT.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FFT.js","sourceRoot":"","sources":["../../src/dsp/FFT.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAA;AACpF,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAA;AAEzD,qDAAqD;AACrD,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,OAAqB,EAAE,QAAgB,EAAE,UAAkB,EAAE,OAAe,EAAE,UAAsB;IAC/H,MAAM,MAAM,GAAmB,EAAE,CAAA;IAEjC,
|
|
1
|
+
{"version":3,"file":"FFT.js","sourceRoot":"","sources":["../../src/dsp/FFT.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAA;AACpF,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAA;AAEzD,qDAAqD;AACrD,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,OAAqB,EAAE,QAAgB,EAAE,UAAkB,EAAE,OAAe,EAAE,UAAsB;IAC/H,MAAM,MAAM,GAAmB,EAAE,CAAA;IAEjC,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAA;IAErG,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,OAAqB,EAAE,QAAgB,EAAE,UAAkB,EAAE,OAAe,EAAE,UAAsB;IAC9I,IAAI,QAAQ,GAAG,CAAC,KAAK,CAAC,IAAI,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAA;IACpE,CAAC;IAED,IAAI,UAAU,GAAG,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACvE,CAAC;IAED,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,CAAA;IAErC,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,cAAc,CAAC,CAAA;IAChD,OAAO,GAAG,mBAAmB,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAA;IAE1D,MAAM,aAAa,GAAG,gBAAgB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;IAE9D,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,+BAA+B,CAAC,QAAQ,CAAC,CAAC,CAAA;IAEjF,MAAM,QAAQ,GAAG,qBAAqB,CACrC,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EACrB,CAAC,CAAC,qBAAqB,EACvB,CAAC,CAAC,mBAAmB,CACrB,CAAA;IAED,MAAM,QAAQ,GAAG,CAAC,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAEhD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAA;IAClC,MAAM,cAAc,GAAG,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAC3D,MAAM,aAAa,GAAG,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAA;IAC9D,MAAM,aAAa,GAAG,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAA;IAE9D,QAAQ,CAAC,CAAC,cAAc;QACvB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,EAAE,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9D,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAAC,CAAA;YACnE,cAAc,CAAC,KAAK,EAAE,CAAA;YAEtB,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAA;YAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/C,eAAe,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;YACzD,CAAC;YAED,aAAa,CAAC,KAAK,EAAE,CAAA;YAErB,CAAC,CAAC,wBAAwB,CAAC,QAAQ,EAAE,cAAc,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;YAE7G,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAA;YAEtD,MAAM,IAAI,CAAA;QACX,CAAC;QAED,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;QAEhC,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnB,CAAC;IAED,OAAO,cAAc,EAAE,CAAA;AACxB,CAAC;AAED,6DAA6D;AAC7D,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,aAA6B,EAAE,QAAgB,EAAE,UAAkB,EAAE,OAAe,EAAE,UAAsB,EAAE,oBAA6B;IACvK,IAAI,QAAQ,GAAG,CAAC,KAAK,CAAC,IAAI,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,UAAU,GAAG,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACvE,CAAC;IAED,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAA;IAEvC,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,CAAA;IAErC,MAAM,aAAa,GAAG,gBAAgB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;IAE9D,MAAM,cAAc,GAAG,OAAO,GAAG,UAAU,CAAA;IAE3C,IAAI,oBAAoB,IAAI,IAAI,EAAE,CAAC;QAClC,oBAAoB,GAAG,cAAc,CAAA;IACtC,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,YAAY,CAAC,cAAc,CAAC,CAAA;IAEnD,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,+BAA+B,CAAC,QAAQ,CAAC,CAAC,CAAA;IAEjF,MAAM,QAAQ,GAAG,qBAAqB,CACrC,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EACrB,CAAC,CAAC,qBAAqB,EACvB,CAAC,CAAC,mBAAmB,CACrB,CAAA;IAED,MAAM,QAAQ,GAAG,CAAC,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAEhD,MAAM,cAAc,GAAG,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAC3D,MAAM,OAAO,GAAG,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAA;IACxD,MAAM,aAAa,GAAG,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAA;IAE9D,MAAM,4BAA4B,GAAG,IAAI,YAAY,CAAC,cAAc,CAAC,CAAA;IAErE,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,UAAU,EAAE,UAAU,EAAE,EAAE,CAAC;QAChE,MAAM,YAAY,GAAG,aAAa,CAAC,UAAU,CAAC,CAAA;QAC9C,OAAO,CAAC,KAAK,EAAE,CAAA;QACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;QAE9B,cAAc,CAAC,KAAK,EAAE,CAAA;QAEtB,CAAC,CAAC,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,cAAc,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;QAEvG,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,CAAA;QAExC,MAAM,gBAAgB,GAAG,UAAU,GAAG,OAAO,CAAA;QAE7C,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,UAAU,EAAE,YAAY,EAAE,EAAE,CAAC;YACtE,MAAM,WAAW,GAAG,YAAY,CAAC,YAAY,CAAC,CAAA;YAC9C,MAAM,MAAM,GAAG,aAAa,CAAC,YAAY,CAAC,CAAA;YAE1C,MAAM,aAAa,GAAG,gBAAgB,GAAG,YAAY,CAAA;YAErD,UAAU,CAAC,aAAa,CAAC,IAAI,WAAW,GAAG,MAAM,CAAA;YAEjD,4BAA4B,CAAC,aAAa,CAAC,IAAI,MAAM,IAAI,CAAC,CAAA;QAC3D,CAAC;IACF,CAAC;IAED,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IAChC,QAAQ,CAAC,OAAO,EAAE,CAAA;IAElB,0DAA0D;IAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,UAAU,CAAC,CAAC,CAAC,IAAI,4BAA4B,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;IACxD,CAAC;IAED,MAAM,iBAAiB,GAAG,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,cAAc,GAAG,oBAAoB,CAAC,CAAA;IAEjG,OAAO,iBAAiB,CAAA;AACzB,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,YAAoB;IACvE,MAAM,cAAc,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,CAAA;IACjD,MAAM,aAAa,GAAG,YAAY,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAA;IAEnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,SAAS,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,SAAS,IAAI,aAAa,EAAE,CAAC;QAC3F,cAAc,CAAC,CAAC,CAAC,GAAG,SAAS,CAAA;IAC9B,CAAC;IAED,OAAO,cAAc,CAAA;AACtB,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,2BAA2B,CAAC,SAAyB;IACpE,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,uBAAuB,CAAC,QAAQ,CAAC,CAAC,CAAA;AACpE,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,uBAAuB,CAAC,QAAsB;IAC7D,MAAM,aAAa,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAE3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,CAAA;QACvB,MAAM,sBAAsB,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,2BAA2B,GAAG,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;QAC3D,MAAM,QAAQ,GAAG,CAAC,sBAAsB,IAAI,CAAC,CAAC,GAAG,CAAC,2BAA2B,IAAI,CAAC,CAAC,CAAA;QAEnF,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAA;IAC5B,CAAC;IAED,OAAO,aAAa,CAAA;AACrB,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,kBAAkB,CAAC,IAAkB,EAAE,eAAe,GAAG,KAAK;IAC7E,MAAM,WAAW,GAAoB,EAAE,CAAA;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,WAAW,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YACb,SAAS,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;SACtB,CAAC,CAAA;IACH,CAAC;IAED,IAAI,eAAe,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QAE/B,KAAK,IAAI,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,MAAM,kBAAkB,GAAG,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC,CAAA;YAEnD,WAAW,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,kBAAkB,CAAC,IAAI;gBAC7B,SAAS,EAAE,CAAC,kBAAkB,CAAC,SAAS;aACxC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,kBAAkB,CAAC,WAA4B;IAC9D,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,QAAQ,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3D,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;QAEjC,SAAS,CAAC,QAAQ,EAAE,CAAC,GAAG,UAAU,CAAC,IAAI,CAAA;QACvC,SAAS,CAAC,QAAQ,EAAE,CAAC,GAAG,UAAU,CAAC,SAAS,CAAA;IAC7C,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,0BAA0B,CAAC,IAAY,EAAE,SAAiB;IACzE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAEzC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;AAC5B,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,0BAA0B,CAAC,SAAiB,EAAE,KAAa;IAC1E,MAAM,IAAI,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACxC,MAAM,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAE7C,OAAO,EAAE,IAAI,EAAE,SAAS,EAAmB,CAAA;AAC5C,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,gBAAgB,CAAC,UAAsB,EAAE,UAAkB;IAC1E,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,UAAU,CAAC,CAAA;IAE5C,MAAM,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAA;IAEpD,IAAI,UAAU,IAAI,MAAM,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,yEAAyE;YACzE,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;SAAM,IAAI,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QAC9D,CAAC;IACF,CAAC;SAAM,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,6EAA6E;YAC7E,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;SAAM,IAAI,UAAU,IAAI,OAAO,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;QAExD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,OAAO,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;QACpC,CAAC;IACF,CAAC;SAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qCAAqC,UAAU,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CAAC,QAAgB;IACrE,MAAM,aAAa,GAAG,MAAM,mBAAmB,EAAE,CAAA;IAEjD,IAAI,aAAa,KAAK,KAAK,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAA;IACb,CAAC;IAED,OAAO,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED,IAAI,oBAAyB,CAAA;AAC7B,IAAI,iBAAsB,CAAA;AAE1B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,UAAmB;IACzD,OAAO,UAAU,CAAC,CAAC,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,uBAAuB,EAAE,CAAA;AACvE,CAAC;AAED,6DAA6D;AAC7D,KAAK,UAAU,uBAAuB;IACrC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAA;IAEvE,oBAAoB,GAAG,MAAM,WAAW,EAAE,CAAA;IAE1C,OAAO,oBAAoB,CAAA;AAC5B,CAAC;AAED,yDAAyD;AACzD,KAAK,UAAU,oBAAoB;IAClC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAA;IAE5E,iBAAiB,GAAG,MAAM,WAAW,EAAE,CAAA;IAEvC,OAAO,iBAAiB,CAAA;AACzB,CAAC"}
|
package/dist/dsp/MFCC.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { RawAudio } from '../audio/AudioUtilities.js';
|
|
2
2
|
export declare function computeMFCCs(monoAudio: RawAudio, options?: MfccOptions): Promise<Float32Array<ArrayBuffer>[]>;
|
|
3
|
-
export declare function
|
|
3
|
+
export declare function melSpectrogramToMFCCs(melSpectrogram: ArrayLike<number>[], mfccFeatureCount: number): Float32Array<ArrayBuffer>[];
|
|
4
4
|
export declare function melSpectrumToMFCC(melSpectrum: ArrayLike<number>, mfccFeatureCount: number, dctMatrix: ArrayLike<number>[], normalization?: 'none' | 'orthonormal'): Float32Array<ArrayBuffer>;
|
|
5
5
|
export declare function createDCTType2CoefficientMatrix(mfccFeatureCount: number, melBandCount: number): Float32Array<ArrayBufferLike>[];
|
|
6
6
|
export declare function applyEmphasis(samples: ArrayLike<number>, emphasisFactor?: number, initialState?: number): Float32Array<ArrayBuffer>;
|
package/dist/dsp/MFCC.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { extendDeep } from '../utilities/ObjectUtilities.js';
|
|
2
2
|
import { Logger } from '../utilities/Logger.js';
|
|
3
3
|
import { resampleAudioSpeex } from './SpeexResampler.js';
|
|
4
|
-
import {
|
|
4
|
+
import { computeMelSpectrogram } from './MelSpectrogram.js';
|
|
5
5
|
import { powerToDecibels } from '../audio/AudioUtilities.js';
|
|
6
6
|
import { normalizeVectors } from '../math/VectorMath.js';
|
|
7
7
|
export async function computeMFCCs(monoAudio, options = {}) {
|
|
@@ -30,10 +30,10 @@ export async function computeMFCCs(monoAudio, options = {}) {
|
|
|
30
30
|
logger.start('Apply emphasis');
|
|
31
31
|
resampledAudio.audioChannels[0] = applyEmphasis(resampledAudio.audioChannels[0], emphasisFactor);
|
|
32
32
|
}
|
|
33
|
-
logger.start('Compute Mel
|
|
34
|
-
const {
|
|
35
|
-
logger.start('Extract MFCCs from Mel
|
|
36
|
-
let mfccs =
|
|
33
|
+
logger.start('Compute Mel spectrogram');
|
|
34
|
+
const { melSpectrogram } = await computeMelSpectrogram(resampledAudio, fftOrder, windowSize, hopLength, filterbankCount, lowerFrequencyHz, upperFrequencyHz);
|
|
35
|
+
logger.start('Extract MFCCs from Mel spectrogram');
|
|
36
|
+
let mfccs = melSpectrogramToMFCCs(melSpectrogram, featureCount);
|
|
37
37
|
if (options.normalize) {
|
|
38
38
|
logger.start('Normalize MFCCs');
|
|
39
39
|
const { normalizedVectors, mean, stdDeviation } = normalizeVectors(mfccs);
|
|
@@ -51,10 +51,10 @@ export async function computeMFCCs(monoAudio, options = {}) {
|
|
|
51
51
|
logger.end();
|
|
52
52
|
return mfccs;
|
|
53
53
|
}
|
|
54
|
-
export function
|
|
55
|
-
const melBandCount =
|
|
54
|
+
export function melSpectrogramToMFCCs(melSpectrogram, mfccFeatureCount) {
|
|
55
|
+
const melBandCount = melSpectrogram[0].length;
|
|
56
56
|
const dctMatrix = createDCTType2CoefficientMatrix(mfccFeatureCount, melBandCount);
|
|
57
|
-
const mfccs =
|
|
57
|
+
const mfccs = melSpectrogram.map(frame => melSpectrumToMFCC(frame, mfccFeatureCount, dctMatrix));
|
|
58
58
|
return mfccs;
|
|
59
59
|
}
|
|
60
60
|
export function melSpectrumToMFCC(melSpectrum, mfccFeatureCount, dctMatrix, normalization = 'orthonormal') {
|
package/dist/dsp/MFCC.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MFCC.js","sourceRoot":"","sources":["../../src/dsp/MFCC.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"MFCC.js","sourceRoot":"","sources":["../../src/dsp/MFCC.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAC3D,OAAO,EAAY,eAAe,EAAE,MAAM,4BAA4B,CAAA;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AAExD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,SAAmB,EAAE,UAAuB,EAAE;IAChF,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAC3B,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAA;IAElC,IAAI,SAAS,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAA;IACtC,CAAC;IAED,OAAO,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAA;IAE3C,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAmB,CAAA;IACtD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAa,CAAA;IAE1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAS,CAAA;IAElC,MAAM,cAAc,GAAG,OAAO,CAAC,cAAe,CAAA;IAC9C,MAAM,UAAU,GAAG,cAAc,GAAG,kBAAkB,CAAA;IACtD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAY,CAAA;IACxC,MAAM,SAAS,GAAG,WAAW,GAAG,kBAAkB,CAAA;IAElD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAgB,CAAA;IAChD,MAAM,gBAAgB,GAAG,OAAO,CAAC,SAAU,CAAA;IAC3C,MAAM,gBAAgB,GAAG,OAAO,CAAC,SAAU,CAAA;IAE3C,MAAM,cAAc,GAAG,OAAO,CAAC,cAAe,CAAA;IAC9C,MAAM,eAAe,GAAG,OAAO,CAAC,eAAgB,CAAA;IAChD,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAqB,CAAA;IAE1D,MAAM,CAAC,KAAK,CAAC,2CAA2C,kBAAkB,KAAK,CAAC,CAAA;IAChF,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAA;IAE9E,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;QAC9B,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;IACjG,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAA;IACvC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,qBAAqB,CAAC,cAAc,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAA;IAE5J,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAA;IAClD,IAAI,KAAK,GAAG,qBAAqB,CAAC,cAAc,EAAE,YAAY,CAAC,CAAA;IAE/D,IAAI,OAAO,CAAC,SAAU,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;QAE/B,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAA;QAEzE,KAAK,GAAG,iBAAiB,CAAA;IAC1B,CAAC;IAED,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAExC,KAAK,GAAG,cAAc,CAAC,KAAK,EAAE,eAAe,CAAC,CAAA;IAC/C,CAAC;IAED,IAAI,oBAAoB,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;QACZ,CAAC;IACF,CAAC;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,KAAK,CAAA;AACb,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,cAAmC,EAAE,gBAAwB;IAClG,MAAM,YAAY,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IAC7C,MAAM,SAAS,GAAG,+BAA+B,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAA;IAEjF,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC,CAAA;IAEhG,OAAO,KAAK,CAAA;AACb,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,WAA8B,EAAE,gBAAwB,EAAE,SAA8B,EAAE,gBAAwC,aAAa;IAChL,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAA;IAEvC,IAAI,+BAAuC,CAAA;IAC3C,IAAI,kCAA0C,CAAA;IAE9C,IAAI,aAAa,IAAI,aAAa,EAAE,CAAC;QACpC,+BAA+B,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAA;QACvE,kCAAkC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAA;IAC3E,CAAC;SAAM,CAAC;QACP,+BAA+B,GAAG,CAAC,CAAA;QACnC,kCAAkC,GAAG,CAAC,CAAA;IACvC,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,gBAAgB,CAAC,CAAA;IAE/C,KAAK,IAAI,gBAAgB,GAAG,CAAC,EAAE,gBAAgB,GAAG,gBAAgB,EAAE,gBAAgB,EAAE,EAAE,CAAC;QACxF,MAAM,YAAY,GAAG,SAAS,CAAC,gBAAgB,CAAC,CAAA;QAEhD,IAAI,GAAG,GAAG,CAAC,CAAA;QAEX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,MAAM,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAA;YAE9C,GAAG,IAAI,cAAc,GAAG,MAAM,CAAA;QAC/B,CAAC;QAED,MAAM,mBAAmB,GAAG,gBAAgB,IAAI,CAAC,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC,CAAC,kCAAkC,CAAA;QAExH,oDAAoD;QACpD,IAAI,CAAC,gBAAgB,CAAC,GAAG,mBAAmB,GAAG,CAAC,GAAG,GAAG,CAAA,CAAC,4CAA4C;IACpG,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,gBAAwB,EAAE,YAAoB;IAC7F,MAAM,SAAS,GAAmB,EAAE,CAAA;IAEpC,KAAK,IAAI,gBAAgB,GAAG,CAAC,EAAE,gBAAgB,GAAG,gBAAgB,EAAE,gBAAgB,EAAE,EAAE,CAAC;QACxF,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAA;QAE1C,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,GAAG,gBAAgB,GAAG,YAAY,CAAA;QAEjE,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE,YAAY,EAAE,EAAE,CAAC;YACxE,GAAG,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,CAAA;QACrE,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACpB,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAA0B,EAAE,cAAc,GAAG,IAAI,EAAE,YAAY,GAAG,CAAC;IAChG,MAAM,gBAAgB,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAEzD,gBAAgB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,YAAY,CAAC,CAAA;IAElE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClD,gBAAgB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IACrE,CAAC;IAED,OAAO,gBAAgB,CAAA;AACxB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAA0B,EAAE,eAAuB;IACjF,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IAEpC,MAAM,iBAAiB,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAA;IAExD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,CAAA;IACjG,CAAC;IAED,MAAM,aAAa,GAAmB,EAAE,CAAA;IAExC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAA;QAEnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAA;QACjD,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IACjC,CAAC;IAED,OAAO,aAAa,CAAA;AACrB,CAAC;AAiBD,MAAM,CAAC,MAAM,kBAAkB,GAAgB;IAC9C,eAAe,EAAE,EAAE;IACnB,YAAY,EAAE,EAAE;IAChB,QAAQ,EAAE,GAAG;IACb,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,KAAK;IACrB,WAAW,EAAE,KAAK;IAClB,cAAc,EAAE,IAAI;IACpB,kBAAkB,EAAE,KAAK;IACzB,eAAe,EAAE,CAAC;IAClB,SAAS,EAAE,KAAK;IAChB,oBAAoB,EAAE,KAAK;CAC3B,CAAA;AAED,MAAM,UAAU,wBAAwB,CAAC,OAAoB;IAC5D,OAAO,UAAU,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAA;AAC/C,CAAC"}
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { RawAudio } from '../audio/AudioUtilities.js';
|
|
2
2
|
import * as FFT from './FFT.js';
|
|
3
|
-
export declare function
|
|
4
|
-
|
|
3
|
+
export declare function computeMelSpectrogram(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbankCount: number, lowerFrequencyHz: number, upperFrequencyHz: number, windowType?: FFT.WindowType): Promise<{
|
|
4
|
+
melSpectrogram: Float32Array<ArrayBuffer>[];
|
|
5
5
|
fftFrames: Float32Array<ArrayBufferLike>[];
|
|
6
6
|
}>;
|
|
7
|
-
export declare function
|
|
8
|
-
|
|
7
|
+
export declare function computeMelSpectrogramUsingFilterbanks(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbanks: Filterbank[], windowType?: FFT.WindowType): Promise<{
|
|
8
|
+
melSpectrogram: Float32Array<ArrayBuffer>[];
|
|
9
9
|
fftFrames: Float32Array<ArrayBufferLike>[];
|
|
10
10
|
}>;
|
|
11
|
-
export declare function
|
|
11
|
+
export declare function fftFramesToMelSpectrogram(fftFrames: Float32Array[], melFilterbanks: Filterbank[]): Float32Array<ArrayBuffer>[];
|
|
12
12
|
export declare function powerSpectrumToMelSpectrum(powerSpectrum: Float32Array, filterbanks: Filterbank[]): Float32Array<ArrayBuffer>;
|
|
13
13
|
export declare function getMelFilterbanks(powerSpectrumFrequenciesHz: Float32Array, centerFrequenciesMel: Float32Array, lowerFrequencyMel: number, upperFrequencyMel: number): Filterbank[];
|
|
14
14
|
export declare function getMelFilterbanksCenterFrequencies(melBandCount: number, lowerFrequencyMel: number, upperFrequencyMel: number): Float32Array<ArrayBuffer>;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Logger } from '../utilities/Logger.js';
|
|
2
2
|
import * as FFT from './FFT.js';
|
|
3
|
-
export async function
|
|
3
|
+
export async function computeMelSpectrogram(rawAudio, fftOrder, windowSize, hopLength, filterbankCount, lowerFrequencyHz, upperFrequencyHz, windowType = 'hann') {
|
|
4
4
|
const logger = new Logger();
|
|
5
|
-
logger.start('Compute
|
|
5
|
+
logger.start('Compute Mel filterbank');
|
|
6
6
|
const binCount = (fftOrder / 2) + 2;
|
|
7
7
|
const nyquistFrequency = rawAudio.sampleRate / 2;
|
|
8
8
|
const binFrequencies = FFT.getBinFrequencies(binCount, nyquistFrequency);
|
|
@@ -11,19 +11,19 @@ export async function computeMelSpectogram(rawAudio, fftOrder, windowSize, hopLe
|
|
|
11
11
|
const filterbanksCenterFrequencies = getMelFilterbanksCenterFrequencies(filterbankCount, lowerFrequencyMel, upperFrequencyMel);
|
|
12
12
|
const melFilterbanks = getMelFilterbanks(binFrequencies, filterbanksCenterFrequencies, lowerFrequencyMel, upperFrequencyMel);
|
|
13
13
|
logger.end();
|
|
14
|
-
return
|
|
14
|
+
return computeMelSpectrogramUsingFilterbanks(rawAudio, fftOrder, windowSize, hopLength, melFilterbanks, windowType);
|
|
15
15
|
}
|
|
16
|
-
export async function
|
|
16
|
+
export async function computeMelSpectrogramUsingFilterbanks(rawAudio, fftOrder, windowSize, hopLength, filterbanks, windowType = 'hann') {
|
|
17
17
|
const logger = new Logger();
|
|
18
18
|
logger.start('Compute short-time FFTs');
|
|
19
19
|
const audioSamples = rawAudio.audioChannels[0];
|
|
20
20
|
const fftFrames = await FFT.stftr(audioSamples, fftOrder, windowSize, hopLength, windowType);
|
|
21
|
-
logger.start('Convert FFT frames to
|
|
22
|
-
const
|
|
21
|
+
logger.start('Convert FFT frames to Mel spectrogram');
|
|
22
|
+
const melSpectrogram = fftFramesToMelSpectrogram(fftFrames, filterbanks);
|
|
23
23
|
logger.end();
|
|
24
|
-
return {
|
|
24
|
+
return { melSpectrogram, fftFrames };
|
|
25
25
|
}
|
|
26
|
-
export function
|
|
26
|
+
export function fftFramesToMelSpectrogram(fftFrames, melFilterbanks) {
|
|
27
27
|
return fftFrames.map(fftFrame => {
|
|
28
28
|
const powerSpectrum = FFT.fftFrameToPowerSpectrum(fftFrame);
|
|
29
29
|
return powerSpectrumToMelSpectrum(powerSpectrum, melFilterbanks);
|
|
@@ -105,4 +105,4 @@ export function hertzToMel(frequency) {
|
|
|
105
105
|
export function melToHertz(mel) {
|
|
106
106
|
return 700.0 * (Math.pow(10.0, mel / 2595.0) - 1.0);
|
|
107
107
|
}
|
|
108
|
-
//# sourceMappingURL=
|
|
108
|
+
//# sourceMappingURL=MelSpectrogram.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MelSpectrogram.js","sourceRoot":"","sources":["../../src/dsp/MelSpectrogram.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAE/B,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,QAAkB,EAAE,QAAgB,EAAE,UAAkB,EAAE,SAAiB,EAAE,eAAuB,EAAE,gBAAwB,EAAE,gBAAwB,EAAE,aAA6B,MAAM;IACxO,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAA;IACtC,MAAM,QAAQ,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;IACnC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAA;IAChD,MAAM,cAAc,GAAG,GAAG,CAAC,iBAAiB,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAA;IAExE,MAAM,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAA;IACtD,MAAM,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAA;IAEtD,MAAM,4BAA4B,GAAG,kCAAkC,CAAC,eAAe,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,CAAA;IAC9H,MAAM,cAAc,GAAG,iBAAiB,CAAC,cAAc,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,CAAA;IAE5H,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,qCAAqC,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,cAAc,EAAE,UAAU,CAAC,CAAA;AACpH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qCAAqC,CAAC,QAAkB,EAAE,QAAgB,EAAE,UAAkB,EAAE,SAAiB,EAAE,WAAyB,EAAE,aAA6B,MAAM;IACtM,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,CAAA;IAE5F,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAA;IACrD,MAAM,cAAc,GAAG,yBAAyB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;IAExE,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,SAAyB,EAAE,cAA4B;IAChG,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;QAC/B,MAAM,aAAa,GAAG,GAAG,CAAC,uBAAuB,CAAC,QAAQ,CAAC,CAAA;QAE3D,OAAO,0BAA0B,CAAC,aAAa,EAAE,cAAc,CAAC,CAAA;IACjE,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,aAA2B,EAAE,WAAyB;IAChG,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAA;IAC1C,MAAM,WAAW,GAAG,IAAI,YAAY,CAAC,eAAe,CAAC,CAAA;IAErD,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,eAAe,EAAE,YAAY,EAAE,EAAE,CAAC;QAC3E,MAAM,UAAU,GAAG,WAAW,CAAC,YAAY,CAAC,CAAA;QAC5C,MAAM,oBAAoB,GAAG,UAAU,CAAC,UAAU,CAAA;QAClD,MAAM,iBAAiB,GAAG,UAAU,CAAC,OAAO,CAAA;QAE5C,IAAI,oBAAoB,KAAK,CAAC,CAAC,EAAE,CAAC;YACjC,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAA;QAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,MAAM,kBAAkB,GAAG,oBAAoB,GAAG,CAAC,CAAA;YAEnD,IAAI,kBAAkB,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;gBAChD,MAAK;YACN,CAAC;YAED,MAAM,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,kBAAkB,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAA;YAE5D,YAAY,IAAI,MAAM,GAAG,kBAAkB,CAAA;QAC5C,CAAC;QAED,WAAW,CAAC,YAAY,CAAC,GAAG,YAAY,CAAA;IACzC,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,0BAAwC,EAAE,oBAAkC,EAAE,iBAAyB,EAAE,iBAAyB;IACnK,MAAM,eAAe,GAAG,oBAAoB,CAAC,MAAM,CAAA;IACnD,MAAM,2BAA2B,GAAG,0BAA0B,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAA;IAE1G,MAAM,WAAW,GAAiB,EAAE,CAAA;IAEpC,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,eAAe,EAAE,eAAe,EAAE,EAAE,CAAC;QACpF,MAAM,eAAe,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAA;QAE7D,MAAM,aAAa,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAA;QACzG,MAAM,cAAc,GAAG,eAAe,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAA;QAE5H,MAAM,KAAK,GAAG,cAAc,GAAG,aAAa,CAAA;QAC5C,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,CAAA;QAE3B,IAAI,UAAU,GAAG,CAAC,CAAC,CAAA;QACnB,IAAI,OAAO,GAAa,EAAE,CAAA;QAE1B,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,KAAK,IAAI,sBAAsB,GAAG,CAAC,EAAE,sBAAsB,GAAG,2BAA2B,CAAC,MAAM,EAAE,sBAAsB,EAAE,EAAE,CAAC;YAC5H,MAAM,6BAA6B,GAAG,2BAA2B,CAAC,sBAAsB,CAAC,CAAA;YAEzF,IAAI,MAAM,GAAG,CAAC,CAAA;YAEd,IAAI,6BAA6B,IAAI,aAAa,IAAI,6BAA6B,IAAI,eAAe,EAAE,CAAC;gBACxG,MAAM,GAAG,CAAC,6BAA6B,GAAG,aAAa,CAAC,GAAG,SAAS,CAAA;YACrE,CAAC;iBAAM,IAAI,6BAA6B,GAAG,eAAe,IAAI,6BAA6B,IAAI,cAAc,EAAE,CAAC;gBAC/G,MAAM,GAAG,CAAC,cAAc,GAAG,6BAA6B,CAAC,GAAG,SAAS,CAAA;YACtE,CAAC;YAED,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChB,IAAI,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC;oBACtB,UAAU,GAAG,sBAAsB,CAAA;gBACpC,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACpB,SAAS,IAAI,MAAM,CAAA;YACpB,CAAC;iBAAM,IAAI,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC7B,MAAK;YACN,CAAC;QACF,CAAC;QAED,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;QAEnD,WAAW,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAA;IAC1C,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,kCAAkC,CAAC,YAAoB,EAAE,iBAAyB,EAAE,iBAAyB;IAC5H,MAAM,WAAW,GAAG,CAAC,iBAAiB,GAAG,iBAAiB,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,CAAA;IAEhF,MAAM,iBAAiB,GAAG,IAAI,YAAY,CAAC,YAAY,CAAC,CAAA;IAExD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,iBAAiB,CAAC,CAAC,CAAC,GAAG,iBAAiB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,iBAAiB,CAAA;AACzB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC3C,OAAO,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAW;IACrC,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAA;AACpD,CAAC"}
|
|
@@ -46,7 +46,11 @@ export declare function cosineSimilarityPrecomputedMagnitudes(vector1: ArrayLike
|
|
|
46
46
|
export declare function minkowskiDistance(vector1: ArrayLike<number>, vector2: ArrayLike<number>, power: number): number;
|
|
47
47
|
export declare function subtractVectors(vector1: ArrayLike<number>, vector2: ArrayLike<number>): Float32Array<ArrayBuffer>;
|
|
48
48
|
export declare function sumVector(vector: ArrayLike<number>): number;
|
|
49
|
-
export declare function
|
|
49
|
+
export declare function sumOfSquaresOfVector(vector: ArrayLike<number>): number;
|
|
50
|
+
export declare function sumAndSumOfSquaresOfVector(vector: ArrayLike<number>): {
|
|
51
|
+
sum: number;
|
|
52
|
+
sumOfSquares: number;
|
|
53
|
+
};
|
|
50
54
|
export declare function dotProduct(vector1: ArrayLike<number>, vector2: ArrayLike<number>): number;
|
|
51
55
|
export declare function magnitude(vector: ArrayLike<number>): number;
|
|
52
56
|
export declare function maxValue(vector: ArrayLike<number>): number;
|
package/dist/math/VectorMath.js
CHANGED
|
@@ -243,7 +243,9 @@ export function medianOfVector(vector) {
|
|
|
243
243
|
if (vector.length == 0) {
|
|
244
244
|
throw new Error('Vector is empty');
|
|
245
245
|
}
|
|
246
|
-
|
|
246
|
+
const sortedArray = Array.from(vector).sort((a, b) => a - b);
|
|
247
|
+
const median = sortedArray[Math.floor(sortedArray.length / 2)];
|
|
248
|
+
return median;
|
|
247
249
|
}
|
|
248
250
|
export function stdDeviationOfVector(vector, kind = 'population', mean) {
|
|
249
251
|
return Math.sqrt(varianceOfVector(vector, kind, mean));
|
|
@@ -449,7 +451,7 @@ export function sumVector(vector) {
|
|
|
449
451
|
}
|
|
450
452
|
return result;
|
|
451
453
|
}
|
|
452
|
-
export function
|
|
454
|
+
export function sumOfSquaresOfVector(vector) {
|
|
453
455
|
const elementCount = vector.length;
|
|
454
456
|
let result = 0.0;
|
|
455
457
|
for (let i = 0; i < elementCount; i++) {
|
|
@@ -457,6 +459,17 @@ export function sumOfSquaresForVector(vector) {
|
|
|
457
459
|
}
|
|
458
460
|
return result;
|
|
459
461
|
}
|
|
462
|
+
export function sumAndSumOfSquaresOfVector(vector) {
|
|
463
|
+
const elementCount = vector.length;
|
|
464
|
+
let sum = 0.0;
|
|
465
|
+
let sumOfSquares = 0.0;
|
|
466
|
+
for (let i = 0; i < elementCount; i++) {
|
|
467
|
+
const value = vector[i];
|
|
468
|
+
sum += value;
|
|
469
|
+
sumOfSquares += value ** 2;
|
|
470
|
+
}
|
|
471
|
+
return { sum, sumOfSquares };
|
|
472
|
+
}
|
|
460
473
|
export function dotProduct(vector1, vector2) {
|
|
461
474
|
if (vector1.length != vector2.length) {
|
|
462
475
|
throw new Error('Vectors are not the same length');
|