echogarden 0.0.1 → 0.1.0

package/dist/recognition/WhisperSTT.d.ts

@@ -0,0 +1,69 @@
+import Onnx from 'onnxruntime-node';
+import { Timeline } from '../utilities/Timeline.js';
+import { AlignmentPath } from '../alignment/SpeechAlignment.js';
+import { RawAudio } from '../audio/AudioUtilities.js';
+import type { LanguageDetectionResults } from '../api/API.js';
+export declare function recognize(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, tokenizerDir: string, task: WhisperTask, sourceLanguage: string): Promise<{
+    transcript: string;
+    timeline: Timeline;
+}>;
+export declare function align(sourceRawAudio: RawAudio, referenceText: string, modelName: WhisperModelName, modelDir: string, tokenizerDir: string, language: string): Promise<Timeline>;
+export declare function detectLanguage(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, tokenizerDir: string): Promise<LanguageDetectionResults>;
+export declare class Whisper {
+    modelName: WhisperModelName;
+    modelDir: string;
+    tokenizerDir: string;
+    isMultiligualModel: boolean;
+    audioEncoder?: Onnx.InferenceSession;
+    textDecoder?: Onnx.InferenceSession;
+    textToTokenLookup: Map<string, number>;
+    tokenToTextLookup: Map<number, string>;
+    merges: [string, string][];
+    onnxOptions: Onnx.InferenceSession.SessionOptions;
+    tokenConfig: {
+        suppressedTokens: number[];
+        sotToken: number;
+        sotPrevToken: number;
+        eotToken: number;
+        noTimestampsToken: number;
+        noSpeechToken: number;
+        timestampTokensStart: number;
+    };
+    constructor(modelName: WhisperModelName, modelDir: string, tokenizerDir: string);
+    initialize(): Promise<void>;
+    recognize(rawAudio: RawAudio, task: WhisperTask, language: string): Promise<{
+        transcript: string;
+        timeline: Timeline;
+    }>;
+    align(rawAudio: RawAudio, referenceText: string, language: string): Promise<Timeline>;
+    detectLanguage(audioFeatures: Onnx.Tensor): Promise<LanguageDetectionResults>;
+    decodeTokens(audioFeatures: Onnx.Tensor, initialTokens: number[], audioDuration: number, isFinalPart: boolean): Promise<{
+        decodedTokens: number[];
+        decodedTokensTimestampLogits: number[][];
+        crossAttentionQKs: Onnx.Tensor[];
+    }>;
+    inferCrossAttentionQKs(tokens: number[], audioFeatures: Onnx.Tensor): Promise<Onnx.Tensor[]>;
+    encodeAudio(rawAudio: RawAudio): Promise<Onnx.Tensor>;
+    addSegmentsToTimeline(timeline: Timeline, tokens: number[], initialTimeOffset: number, audioDuration: number): void;
+    addWordsToTimeline(timeline: Timeline, tokens: number[], rawAudio: RawAudio, crossAttentionQKs: Onnx.Tensor[], initialAudioTimeOffset: number, duration: number): Promise<void>;
+    mergeSuccessiveWordFragmentsInTimeline(timeline: Timeline): Timeline;
+    getWordTimelineFromAlignmentPath(alignmentPath: AlignmentPath, tokens: number[], startTimeOffset: number, endTimeOffset: number, correctionAmount?: number): Promise<Timeline>;
+    findAlignmentPathFromQKs(qksTensors: Onnx.Tensor[], tokens: number[], segmentStartFrame: number, segmentEndFrame: number, headIndexes?: number[]): Promise<AlignmentPath>;
+    getKvDimensions(groupCount: number, length: number): number[];
+    getInitialTokens(language: string, task: WhisperTask, disableTimestamps?: boolean): number[];
+    getAlignmentHeadIndexes(): number[];
+    tokensToText(tokens: number[]): string;
+    textToTokens(text: string, language: string): Promise<number[]>;
+}
+export declare function loadPackagesAndGetPaths(modelName: WhisperModelName | undefined, languageCode: string | undefined): Promise<{
+    modelName: WhisperModelName;
+    modelDir: string;
+    tokenizerDir: string;
+}>;
+export declare function isMultiligualModel(modelName: WhisperModelName): boolean;
+export type WhisperModelName = "tiny" | "tiny.en" | "base" | "base.en" | "small" | "small.en" | "medium" | "medium.en" | "large" | "large-v1" | "large-v2";
+export type WhisperTask = "transcribe" | "translate";
+export declare const modelNameToPackageName: {
+    [modelName in WhisperModelName]: string;
+};
+export declare const tokenizerPackageName = "whisper-tokenizer";