echogarden 0.12.2 → 1.0.0

package/dist/recognition/WhisperSTT.d.ts

@@ -5,41 +5,47 @@ import { AlignmentPath } from '../alignment/SpeechAlignment.js';
 import { RawAudio } from '../audio/AudioUtilities.js';
 import type { LanguageDetectionResults } from '../api/API.js';
 import { XorShift32RNG } from '../utilities/RandomGenerator.js';
-export declare function recognize(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, tokenizerDir: string, task: WhisperTask, sourceLanguage: string, options: WhisperOptions): Promise<{
+import { type Tiktoken } from 'tiktoken/lite';
+export declare function recognize(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, task: WhisperTask, sourceLanguage: string, options: WhisperOptions): Promise<{
     transcript: string;
     timeline: Timeline;
 }>;
-export declare function align(sourceRawAudio: RawAudio, referenceText: string, modelName: WhisperModelName, modelDir: string, tokenizerDir: string, sourceLanguage: string): Promise<Timeline>;
-export declare function detectLanguage(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, tokenizerDir: string): Promise<LanguageDetectionResults>;
+export declare function align(sourceRawAudio: RawAudio, referenceText: string, modelName: WhisperModelName, modelDir: string, sourceLanguage: string): Promise<Timeline>;
+export declare function detectLanguage(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, temperature: number): Promise<LanguageDetectionResults>;
 export declare class Whisper {
     modelName: WhisperModelName;
     modelDir: string;
-    tokenizerDir: string;
     isMultiligualModel: boolean;
     audioEncoder?: Onnx.InferenceSession;
     textDecoder?: Onnx.InferenceSession;
-    textToTokenLookup: Map<string, number>;
-    tokenToTextLookup: Map<number, string>;
-    merges: [string, string][];
+    tiktoken?: Tiktoken;
     onnxOptions: Onnx.InferenceSession.SessionOptions;
     tokenConfig: {
-        suppressedTokens: number[];
-        sotToken: number;
-        sotPrevToken: number;
-        eotToken: number;
+        endOfTextToken: number;
+        startOfTextToken: number;
+        languageTokensStart: number;
+        languageTokensEnd: number;
+        translateTaskToken: number;
+        transcribeTaskToken: number;
+        startOfPromptToken: number;
+        nonSpeechToken: number;
         noTimestampsToken: number;
-        noSpeechToken: number;
         timestampTokensStart: number;
+        timestampTokensEnd: number;
+        suppressedTokens: Set<number>;
     };
     randomGen: XorShift32RNG;
-    constructor(modelName: WhisperModelName, modelDir: string, tokenizerDir: string);
-    initialize(): Promise<void>;
+    constructor(modelName: WhisperModelName, modelDir: string, rngSeed?: number);
+    initializeIfNeeded(): Promise<void>;
+    initializeTokenizerIfNeeded(): Promise<void>;
+    initializeEncoderSessionIfNeeded(): Promise<void>;
+    initializeDecoderSessionIfNeeded(): Promise<void>;
     recognize(rawAudio: RawAudio, task: WhisperTask, language: string, options: WhisperOptions): Promise<{
         transcript: string;
         timeline: Timeline;
     }>;
     align(rawAudio: RawAudio, referenceText: string, language: string): Promise<Timeline>;
-    detectLanguage(audioFeatures: Onnx.Tensor): Promise<LanguageDetectionResults>;
+    detectLanguage(audioFeatures: Onnx.Tensor, temperature: number): Promise<LanguageDetectionResults>;
     decodeTokens(audioFeatures: Onnx.Tensor, initialTokens: number[], audioDuration: number, isFirstPart: boolean, isFinalPart: boolean, options: WhisperOptions): Promise<{
         decodedTokens: number[];
         decodedTokensTimestampLogits: number[][];
@@ -50,23 +56,37 @@ export declare class Whisper {
     encodeAudio(rawAudio: RawAudio): Promise<Onnx.Tensor>;
     addSegmentsToTimeline(timeline: Timeline, tokens: number[], initialTimeOffset: number, audioDuration: number): void;
     addWordsToTimeline(timeline: Timeline, tokens: number[], rawAudio: RawAudio, crossAttentionQKs: Onnx.Tensor[], initialAudioTimeOffset: number, duration: number): Promise<void>;
-    mergeSuccessiveWordFragmentsInTimeline(timeline: Timeline): Timeline;
-    getWordTimelineFromAlignmentPath(alignmentPath: AlignmentPath, tokens: number[], startTimeOffset: number, endTimeOffset: number, tokensConfidence?: number[], correctionAmount?: number): Promise<Timeline>;
+    tokenTimelineToWordTimeline(tokenTimeline: Timeline): Timeline;
+    getTokenTimelineFromAlignmentPath(alignmentPath: AlignmentPath, tokens: number[], startTimeOffset: number, endTimeOffset: number, tokensConfidence?: number[], correctionAmount?: number): Promise<Timeline>;
     findAlignmentPathFromQKs(qksTensors: Onnx.Tensor[], tokens: number[], segmentStartFrame: number, segmentEndFrame: number, headIndexes?: number[]): Promise<AlignmentPath>;
     getKvDimensions(groupCount: number, length: number): number[];
-    getInitialTokens(language: string, task: WhisperTask, disableTimestamps?: boolean): number[];
+    getTextStartTokens(language: string, task: WhisperTask, disableTimestamps?: boolean): number[];
+    tokenToText(token: number, includeMetadataTokens?: boolean): string;
+    tokensToText(tokens: number[], includeMetadataTokens?: boolean): string;
+    textToTokens(text: string): number[];
+    isTextToken(token: number): boolean;
+    isMetadataToken(token: number): boolean;
+    isLanguageToken(token: number): boolean;
+    isTimestampToken(token: number): boolean;
+    isNonTimestampToken(token: number): boolean;
+    timestampTokenToSeconds(timestampToken: number): number;
+    isValidToken(token: number): boolean;
+    assertIsValidToken(token: number): void;
+    secondsToFrame(seconds: number): number;
+    secondsRangeToFrameCount(startSeconds: number, endSeconds: number): number;
+    languageTokenToLanguageIndex(languageToken: number): void;
+    get isEnglishOnlyModel(): boolean;
     getAlignmentHeadIndexes(): number[];
-    tokensToText(tokens: number[]): string;
-    textToTokens(text: string, language: string): Promise<number[]>;
 }
 export declare function loadPackagesAndGetPaths(modelName: WhisperModelName | undefined, languageCode: string | undefined): Promise<{
     modelName: WhisperModelName;
     modelDir: string;
-    tokenizerDir: string;
 }>;
-export declare function isMultiligualModel(modelName: WhisperModelName): boolean;
-export type WhisperModelName = "tiny" | "tiny.en" | "base" | "base.en" | "small" | "small.en" | "medium" | "medium.en" | "large" | "large-v1" | "large-v2" | "large-v3";
-export type WhisperTask = "transcribe" | "translate";
+export declare function normalizeWhisperModelName(modelName: WhisperModelName, languageCode: string | undefined): WhisperModelName;
+export declare function isMultilingualModel(modelName: WhisperModelName): boolean;
+export declare function isEnglishOnlyModel(modelName: WhisperModelName): boolean;
+export type WhisperModelName = 'tiny' | 'tiny.en' | 'base' | 'base.en' | 'small' | 'small.en' | 'medium' | 'medium.en' | 'large' | 'large-v1' | 'large-v2' | 'large-v3';
+export type WhisperTask = 'transcribe' | 'translate' | 'detect-language';
 export declare const modelNameToPackageName: {
     [modelName in WhisperModelName]: string;
 };
@@ -78,5 +98,9 @@ export interface WhisperOptions {
     topCandidateCount?: number;
     punctuationThreshold?: number;
     autoPromptParts?: boolean;
+    maxTokensPerPart?: number;
+    suppressRepetition?: boolean;
+    seed?: number;
+    decodeTimestampTokens?: boolean;
 }
-export declare const whisperOptionsDefaults: WhisperOptions;
+export declare const defaultWhisperOptions: WhisperOptions;