mlx-bun 0.3.0 → 0.5.0

package/src/audio/whisper-tokenizer.ts

@@ -0,0 +1,239 @@
+// Whisper tokenizer — mlx-whisper's tokenizer.py semantics over the stock
+// HF tokenizer.json (openai/whisper-large-v3-turbo ships the multilingual
+// tiktoken vocab in HF form; ids are identical, checked in
+// tests/unit/whisper-tokenizer.test.ts against the oracle constants).
+//
+// Special-token layout (multilingual v3, n_vocab 51866): 50256 BPE ranks +
+// "<|endoftext|>"(50257) "<|startoftranscript|>"(50258) + 100 language
+// tokens + "<|translate|>" "<|transcribe|>" "<|startoflm|>" "<|startofprev|>"
+// "<|nospeech|>" "<|notimestamps|>" + 1501 timestamp tokens "<|0.00|>" …
+// "<|30.00|>". The mlx-community/whisper-* repos ship no tokenizer, so the
+// loader resolves the openai/whisper-* tokenizer files (see
+// resolveWhisperTokenizerDir).
+
+import { existsSync, readdirSync } from "node:fs";
+import { loadTokenizer, type LoadedTokenizer } from "../tokenizer";
+
+export const WHISPER_LANGUAGES: Record<string, string> = {
+  en: "english", zh: "chinese", de: "german", es: "spanish", ru: "russian", ko: "korean",
+  fr: "french", ja: "japanese", pt: "portuguese", tr: "turkish", pl: "polish", ca: "catalan",
+  nl: "dutch", ar: "arabic", sv: "swedish", it: "italian", id: "indonesian", hi: "hindi",
+  fi: "finnish", vi: "vietnamese", he: "hebrew", uk: "ukrainian", el: "greek", ms: "malay",
+  cs: "czech", ro: "romanian", da: "danish", hu: "hungarian", ta: "tamil", no: "norwegian",
+  th: "thai", ur: "urdu", hr: "croatian", bg: "bulgarian", lt: "lithuanian", la: "latin",
+  mi: "maori", ml: "malayalam", cy: "welsh", sk: "slovak", te: "telugu", fa: "persian",
+  lv: "latvian", bn: "bengali", sr: "serbian", az: "azerbaijani", sl: "slovenian", kn: "kannada",
+  et: "estonian", mk: "macedonian", br: "breton", eu: "basque", is: "icelandic", hy: "armenian",
+  ne: "nepali", mn: "mongolian", bs: "bosnian", kk: "kazakh", sq: "albanian", sw: "swahili",
+  gl: "galician", mr: "marathi", pa: "punjabi", si: "sinhala", km: "khmer", sn: "shona",
+  yo: "yoruba", so: "somali", af: "afrikaans", oc: "occitan", ka: "georgian", be: "belarusian",
+  tg: "tajik", sd: "sindhi", gu: "gujarati", am: "amharic", yi: "yiddish", lo: "lao",
+  uz: "uzbek", fo: "faroese", ht: "haitian creole", ps: "pashto", tk: "turkmen", nn: "nynorsk",
+  mt: "maltese", sa: "sanskrit", lb: "luxembourgish", my: "myanmar", bo: "tibetan", tl: "tagalog",
+  mg: "malagasy", as: "assamese", tt: "tatar", haw: "hawaiian", ln: "lingala", ha: "hausa",
+  ba: "bashkir", jw: "javanese", su: "sundanese", yue: "cantonese",
+};
+
+const TO_LANGUAGE_CODE: Record<string, string> = {
+  ...Object.fromEntries(Object.entries(WHISPER_LANGUAGES).map(([c, n]) => [n, c])),
+  burmese: "my", valencian: "ca", flemish: "nl", haitian: "ht", letzeburgesch: "lb",
+  pushto: "ps", panjabi: "pa", moldavian: "ro", moldovan: "ro", sinhalese: "si", castilian: "es",
+  mandarin: "zh",
+};
+
+/** "English" / "en" → "en"; throws on unknown. */
+export function normalizeWhisperLanguage(language: string): string {
+  const l = language.toLowerCase();
+  if (l in WHISPER_LANGUAGES) return l;
+  const code = TO_LANGUAGE_CODE[l];
+  if (code) return code;
+  throw new Error(`Unsupported language: ${language}`);
+}
+
+export type WhisperTask = "transcribe" | "translate";
+
+const ASCII_PUNCTUATION = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
+
+export class WhisperTokenizer {
+  readonly eot: number;
+  readonly sot: number;
+  readonly translate: number;
+  readonly transcribe: number;
+  readonly sotLm: number;
+  readonly sotPrev: number;
+  readonly noSpeech: number;
+  readonly noTimestamps: number;
+  readonly timestampBegin: number;
+  readonly numLanguages: number;
+  /** Language token ids in id order (sot+1 …). */
+  readonly allLanguageTokens: number[];
+  readonly allLanguageCodes: string[];
+  readonly #tok: LoadedTokenizer;
+  readonly #encodeCache = new Map<string, number[]>();
+
+  constructor(tok: LoadedTokenizer, numLanguages: number) {
+    this.#tok = tok;
+    this.numLanguages = numLanguages;
+    const one = (text: string): number => {
+      const ids = tok.encode(text, false);
+      if (ids.length !== 1) throw new Error(`whisper tokenizer: ${text} is not a single token`);
+      return ids[0]!;
+    };
+    this.eot = one("<|endoftext|>");
+    this.sot = one("<|startoftranscript|>");
+    this.translate = one("<|translate|>");
+    this.transcribe = one("<|transcribe|>");
+    this.sotLm = one("<|startoflm|>");
+    this.sotPrev = one("<|startofprev|>");
+    this.noSpeech = one("<|nospeech|>");
+    this.noTimestamps = one("<|notimestamps|>");
+    this.timestampBegin = one("<|0.00|>");
+    // language tokens: the oracle's order is by (insertion) special-token
+    // order == ascending id, capped at num_languages
+    const codes = Object.keys(WHISPER_LANGUAGES).slice(0, numLanguages);
+    this.allLanguageTokens = codes.map((c) => one(`<|${c}|>`));
+    this.allLanguageCodes = codes;
+  }
+
+  encode(text: string): number[] {
+    let ids = this.#encodeCache.get(text);
+    if (!ids) {
+      ids = this.#tok.encode(text, false);
+      if (this.#encodeCache.size < 4096) this.#encodeCache.set(text, ids);
+    }
+    return ids;
+  }
+
+  /** decode() drops timestamp tokens (ids ≥ timestampBegin) like the oracle. */
+  decode(ids: number[]): string {
+    return this.#tok.decode(ids.filter((t) => t < this.timestampBegin), false);
+  }
+
+  /** Text tokens only (id < eot). */
+  decodeText(ids: number[]): string {
+    return this.#tok.decode(ids.filter((t) => t < this.eot), false);
+  }
+
+  decodeWithTimestamps(ids: number[]): string {
+    return this.#tok.decode(ids, false);
+  }
+
+  languageToken(language: string): number {
+    const i = this.allLanguageCodes.indexOf(language);
+    if (i < 0) throw new Error(`Language ${language} not found in tokenizer.`);
+    return this.allLanguageTokens[i]!;
+  }
+
+  /** [sot, <|lang|>, <|task|>] */
+  sotSequence(language: string | null, task: WhisperTask): number[] {
+    const seq = [this.sot];
+    if (language !== null) seq.push(this.languageToken(language));
+    seq.push(task === "transcribe" ? this.transcribe : this.translate);
+    return seq;
+  }
+
+  /** tokenizer.non_speech_tokens — punctuation/symbol tokens suppressed at
+   *  every step (mlx-whisper tokenizer.py). */
+  get nonSpeechTokens(): number[] {
+    const symbols = "\"#()*+/:;<=>@[\\]^_`{|}~「」『』".split("");
+    symbols.push(...["<<", ">>", "<<<", ">>>", "--", "---", "-(", "-[", "('", "(\"", "((", "))", "(((", ")))", "[[", "]]", "{{", "}}", "♪♪", "♪♪♪"]);
+    const miscellaneous = new Set("♩♪♫♬♭♮♯".split(""));
+    const result = new Set<number>([this.encode(" -")[0]!, this.encode(" '")[0]!]);
+    for (const symbol of [...symbols, ...miscellaneous]) {
+      for (const tokens of [this.encode(symbol), this.encode(" " + symbol)]) {
+        if (tokens.length === 1 || miscellaneous.has(symbol)) result.add(tokens[0]!);
+      }
+    }
+    return [...result].sort((a, b) => a - b);
+  }
+
+  /** Split text tokens into words (split_to_word_tokens): unicode-boundary
+   *  split for CJK/Thai/Lao/Myanmar, space split otherwise. */
+  splitToWordTokens(tokens: number[], language: string): { words: string[]; wordTokens: number[][] } {
+    if (["zh", "ja", "th", "lo", "my", "yue"].includes(language)) return this.#splitOnUnicode(tokens);
+    return this.#splitOnSpaces(tokens);
+  }
+
+  #splitOnUnicode(tokens: number[]): { words: string[]; wordTokens: number[][] } {
+    const decodedFull = this.decodeWithTimestamps(tokens);
+    const replacement = "�";
+    const words: string[] = [];
+    const wordTokens: number[][] = [];
+    let current: number[] = [];
+    let unicodeOffset = 0;
+    for (const token of tokens) {
+      current.push(token);
+      const decoded = this.decodeWithTimestamps(current);
+      const idx = decoded.indexOf(replacement);
+      if (idx < 0 || decodedFull[unicodeOffset + idx] === replacement) {
+        words.push(decoded);
+        wordTokens.push(current);
+        current = [];
+        unicodeOffset += decoded.length;
+      }
+    }
+    return { words, wordTokens };
+  }
+
+  #splitOnSpaces(tokens: number[]): { words: string[]; wordTokens: number[][] } {
+    const sub = this.#splitOnUnicode(tokens);
+    const words: string[] = [];
+    const wordTokens: number[][] = [];
+    for (let i = 0; i < sub.words.length; i++) {
+      const subword = sub.words[i]!;
+      const subwordTokens = sub.wordTokens[i]!;
+      const special = subwordTokens[0]! >= this.eot;
+      const withSpace = subword.startsWith(" ");
+      // python: `subword.strip() in string.punctuation` — a SUBSTRING test.
+      const punctuation = ASCII_PUNCTUATION.includes(subword.trim());
+      if (special || withSpace || punctuation || words.length === 0) {
+        words.push(subword);
+        wordTokens.push(subwordTokens);
+      } else {
+        words[words.length - 1] += subword;
+        wordTokens[wordTokens.length - 1]!.push(...subwordTokens);
+      }
+    }
+    return { words, wordTokens };
+  }
+}
+
+/** Tokenizer directory for a Whisper checkpoint: the model dir itself when
+ *  it ships tokenizer.json, else the matching openai/whisper-* HF snapshot
+ *  (multilingual v3 vocab → large-v3-turbo; v2 → large-v2; english-only →
+ *  base.en). Returns null when nothing is on disk. */
+export function resolveWhisperTokenizerDir(modelDir: string, nVocab: number): string | null {
+  if (existsSync(`${modelDir}/tokenizer.json`) && existsSync(`${modelDir}/tokenizer_config.json`))
+    return modelDir;
+  const home = process.env.HOME ?? "";
+  const hub = `${home}/.cache/huggingface/hub`;
+  for (const repo of whisperTokenizerRepos(nVocab)) {
+    const base = `${hub}/models--${repo.replace("/", "--")}/snapshots`;
+    try {
+      for (const snap of readdirSync(base)) {
+        const dir = `${base}/${snap}`;
+        if (existsSync(`${dir}/tokenizer.json`) && existsSync(`${dir}/tokenizer_config.json`)) return dir;
+      }
+    } catch { /* not downloaded */ }
+  }
+  return null;
+}
+
+/** Candidate HF repos whose tokenizer matches an n_vocab. */
+export function whisperTokenizerRepos(nVocab: number): string[] {
+  if (nVocab === 51866) return ["openai/whisper-large-v3-turbo", "openai/whisper-large-v3"];
+  if (nVocab === 51865) return ["openai/whisper-large-v2", "openai/whisper-small", "openai/whisper-tiny"];
+  return ["openai/whisper-tiny.en", "openai/whisper-base.en", "openai/whisper-small.en", "openai/whisper-medium.en"];
+}
+
+export async function loadWhisperTokenizer(modelDir: string, nVocab: number): Promise<WhisperTokenizer> {
+  const dir = resolveWhisperTokenizerDir(modelDir, nVocab);
+  if (!dir)
+    throw new Error(
+      `no Whisper tokenizer for ${modelDir}: place tokenizer.json + tokenizer_config.json ` +
+      `beside the weights or download ${whisperTokenizerRepos(nVocab)[0]} (tokenizer files only)`,
+    );
+  const isMultilingual = nVocab >= 51865;
+  const numLanguages = nVocab - 51765 - (isMultilingual ? 1 : 0);
+  return new WhisperTokenizer(await loadTokenizer(dir), numLanguages);
+}

package/src/audio/whisper-transcribe.ts

@@ -0,0 +1,410 @@
+// Whisper long-form transcription — a port of mlx-whisper's transcribe.py
+// seek loop (30 s windows, temperature fallback, no-speech skipping,
+// timestamp-driven seeking, previous-text conditioning), restructured as a
+// RESUMABLE run: `feed()` appends audio and transcribes every window that
+// is already complete (≥ 30 s past the seek point); `finish()` drains the
+// remainder. Batch transcription is the same run fed once, so the oracle
+// parity gate covers both. Streaming caveat: a window fed early sees a
+// log-mel normalized against the audio received so far (the 8-decade clamp
+// is relative to the clip maximum), which can differ from the batch result
+// in near-silent bins. Word timestamps live in whisper-timing.ts.
+
+import { MlxArray } from "../mlx/array";
+import { Dtype } from "../mlx/ffi";
+import * as ops from "../mlx/ops";
+import type { WhisperModel } from "../model/whisper";
+import {
+  WhisperDecodingTask, type WhisperDecodingOptions, type WhisperDecodingResult,
+} from "./whisper-decode";
+import {
+  WHISPER_FRAMES_PER_SECOND, WHISPER_HOP, WHISPER_N_FRAMES, WHISPER_N_SAMPLES,
+  WHISPER_SAMPLE_RATE, WhisperMelExtractor, padOrTrimFrames,
+} from "./whisper-mel";
+import { addWordTimestamps } from "./whisper-timing";
+import { normalizeWhisperLanguage, type WhisperTokenizer } from "./whisper-tokenizer";
+
+export interface WhisperSegment {
+  id: number;
+  seek: number;
+  start: number;
+  end: number;
+  text: string;
+  tokens: number[];
+  temperature: number;
+  avgLogprob: number;
+  compressionRatio: number;
+  noSpeechProb: number;
+  words?: WhisperWord[];
+}
+
+export interface WhisperWord {
+  word: string;
+  start: number;
+  end: number;
+  probability: number;
+}
+
+export interface WhisperTranscribeOptions extends WhisperDecodingOptions {
+  /** Single temperature or the fallback ladder (oracle default: 0, .2 … 1). */
+  temperatures?: number[];
+  compressionRatioThreshold?: number | null;
+  logprobThreshold?: number | null;
+  noSpeechThreshold?: number | null;
+  conditionOnPreviousText?: boolean;
+  initialPrompt?: string | null;
+  /** [start, end, start, end, …] seconds; last end defaults to the clip end.
+   *  Batch only: a run with clips never transcribes eagerly. */
+  clipTimestamps?: number[];
+  /** Word-level timestamps via cross-attention DTW (whisper-timing.ts);
+   *  one extra faithful decoder pass per window. */
+  wordTimestamps?: boolean;
+  /** With wordTimestamps: skip silent gaps longer than this many seconds
+   *  around a likely hallucination (transcribe.py hallucination_silence_threshold). */
+  hallucinationSilenceThreshold?: number | null;
+  /** Progress callback: seeked frames out of content frames. */
+  onProgress?: (done: number, total: number) => void;
+  /** Called with each finalized segment as the loop emits it. */
+  onSegment?: (segment: WhisperSegment) => void;
+  /** Checked between windows; an aborted signal throws "transcription cancelled". */
+  signal?: AbortSignal;
+}
+
+export interface WhisperTranscription {
+  text: string;
+  segments: WhisperSegment[];
+  language: string;
+  /** Detected-language probabilities when language was auto-detected. */
+  languageProbs?: Record<string, number>;
+}
+
+const DEFAULT_TEMPERATURES = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0];
+
+export class WhisperTranscriber {
+  readonly mel: WhisperMelExtractor;
+  constructor(readonly model: WhisperModel, readonly tokenizer: WhisperTokenizer) {
+    this.mel = new WhisperMelExtractor(model.dims.nMels);
+  }
+
+  /** Transcribe a complete 16 kHz mono float32 waveform. */
+  async transcribe(samples: Float32Array, options: WhisperTranscribeOptions = {}): Promise<WhisperTranscription> {
+    const run = new WhisperStreamingRun(this, options);
+    await run.feed(samples);
+    return run.finish();
+  }
+
+  /** Start a resumable run (see WhisperStreamingRun). */
+  start(options: WhisperTranscribeOptions = {}): WhisperStreamingRun {
+    return new WhisperStreamingRun(this, options);
+  }
+
+  dispose(): void {
+    this.mel.dispose();
+  }
+}
+
+/** A transcription in progress over audio that is still arriving. */
+export class WhisperStreamingRun {
+  readonly #tr: WhisperTranscriber;
+  readonly #o: WhisperTranscribeOptions;
+  #buf = new Float32Array(WHISPER_SAMPLE_RATE * 60);
+  #len = 0;
+  #seek = 0;
+  #language: string | null;
+  #languageProbs: Record<string, number> | undefined;
+  #allTokens: number[] = [];
+  #allSegments: WhisperSegment[] = [];
+  #promptResetSince = 0;
+  #initialPromptTokens: number[];
+  #lastSpeechTimestamp = 0;
+  #finished = false;
+  readonly #temperatures: number[];
+  readonly #crThreshold: number | null;
+  readonly #lpThreshold: number | null;
+  readonly #nsThreshold: number | null;
+  readonly #condition: boolean;
+
+  constructor(transcriber: WhisperTranscriber, options: WhisperTranscribeOptions) {
+    this.#tr = transcriber;
+    this.#o = options;
+    this.#language = options.language == null ? null : normalizeWhisperLanguage(options.language);
+    this.#temperatures = options.temperatures ?? (options.temperature !== undefined ? [options.temperature] : DEFAULT_TEMPERATURES);
+    this.#crThreshold = options.compressionRatioThreshold === undefined ? 2.4 : options.compressionRatioThreshold;
+    this.#lpThreshold = options.logprobThreshold === undefined ? -1.0 : options.logprobThreshold;
+    this.#nsThreshold = options.noSpeechThreshold === undefined ? 0.6 : options.noSpeechThreshold;
+    this.#condition = options.conditionOnPreviousText ?? true;
+    this.#initialPromptTokens = options.initialPrompt != null ? transcriber.tokenizer.encode(" " + options.initialPrompt.trim()) : [];
+    this.#allTokens.push(...this.#initialPromptTokens);
+  }
+
+  /** Samples received so far. */
+  get sampleCount(): number {
+    return this.#len;
+  }
+
+  /** Segments finalized so far. */
+  get segments(): readonly WhisperSegment[] {
+    return this.#allSegments;
+  }
+
+  /** Append audio WITHOUT transcribing yet (e.g. no speech detected so far). */
+  feedSilent(samples: Float32Array): void {
+    if (this.#finished) throw new Error("transcription run already finished");
+    if (this.#len + samples.length > this.#buf.length) {
+      const next = new Float32Array(Math.max(this.#buf.length * 2, this.#len + samples.length));
+      next.set(this.#buf.subarray(0, this.#len));
+      this.#buf = next;
+    }
+    this.#buf.set(samples, this.#len);
+    this.#len += samples.length;
+  }
+
+  /** Append audio; transcribes every complete 30 s window. Returns the
+   *  segments finalized by this call. */
+  async feed(samples: Float32Array): Promise<WhisperSegment[]> {
+    this.feedSilent(samples);
+    if (this.#o.clipTimestamps) return [];
+    const before = this.#allSegments.length;
+    await this.#drain(false);
+    return this.#allSegments.slice(before);
+  }
+
+  /** Transcribe the remaining audio and return the whole transcription. */
+  async finish(): Promise<WhisperTranscription> {
+    if (this.#finished) throw new Error("transcription run already finished");
+    this.#finished = true;
+    await this.#drain(true);
+    const t = this.#tr.tokenizer;
+    return {
+      text: t.decodeText(this.#allTokens.slice(this.#initialPromptTokens.length)),
+      segments: this.#allSegments,
+      language: this.#language ?? "en",
+      languageProbs: this.#languageProbs,
+    };
+  }
+
+  async #drain(final: boolean): Promise<void> {
+    const { model, tokenizer: t } = this.#tr;
+    const samples = this.#buf.subarray(0, this.#len);
+    if (samples.length <= 200) return; // below one mel frame of real audio
+    const mel = this.#tr.mel.logMel(samples, WHISPER_N_SAMPLES);
+    try {
+      const contentFrames = mel.shape[0]! - WHISPER_N_FRAMES;
+      // language detection on the first window (batch: same as the oracle)
+      if (this.#language === null) {
+        if (!final && contentFrames - this.#seek < WHISPER_N_FRAMES) return;
+        if (!model.isMultilingual) this.#language = "en";
+        else {
+          const seg = padOrTrimFrames(mel, WHISPER_N_FRAMES, model.dtype);
+          const seg3 = reshape3(seg);
+          const probe = new WhisperDecodingTask(model, t, { ...this.#o, language: null });
+          const feats = probe.encode(seg3);
+          const det = probe.detectLanguage(feats);
+          feats.dispose(); seg3.dispose(); seg.dispose();
+          this.#language = det.code;
+          this.#languageProbs = det.probs;
+        }
+      }
+      // clip windows: batch only
+      let clips: [number, number][];
+      if (this.#o.clipTimestamps) {
+        const pts = this.#o.clipTimestamps.map((ts) => Math.round(ts * WHISPER_FRAMES_PER_SECOND));
+        if (pts.length === 0) pts.push(0);
+        if (pts.length % 2 === 1) pts.push(contentFrames);
+        else pts[pts.length - 1] = Math.min(contentFrames, pts[pts.length - 1]!);
+        clips = [];
+        for (let i = 0; i < pts.length; i += 2) clips.push([pts[i]!, pts[i + 1]!]);
+        this.#seek = clips[0]![0];
+      } else clips = [[0, contentFrames]];
+      for (const [, clipEnd] of clips) {
+        while (this.#seek < clipEnd) {
+          // during feed, only windows with 30 s of audio past `seek` are final
+          if (!final && contentFrames - this.#seek < WHISPER_N_FRAMES) return;
+          if (this.#o.signal?.aborted) throw new Error("transcription cancelled");
+          if (this.#allSegments.length > 0) await Bun.sleep(0);
+          await this.#window(mel, contentFrames, clipEnd);
+          this.#o.onProgress?.(Math.min(contentFrames, this.#seek), final ? contentFrames : Math.max(contentFrames, this.#seek));
+        }
+      }
+    } finally {
+      mel.dispose();
+    }
+  }
+
+  #decodeWithFallback(segment: MlxArray, prompt: number[]): WhisperDecodingResult {
+    const { model, tokenizer: t } = this.#tr;
+    let result: WhisperDecodingResult | null = null;
+    for (const temp of this.#temperatures) {
+      const opts: WhisperDecodingOptions = { ...this.#o, language: this.#language, task: this.#o.task ?? "transcribe", prompt, temperature: temp };
+      if (temp > 0) { opts.beamSize = null; opts.patience = null; }
+      else opts.bestOf = null;
+      result = new WhisperDecodingTask(model, t, opts).run(segment);
+      let needsFallback = false;
+      if (this.#crThreshold !== null && result.compressionRatio > this.#crThreshold) needsFallback = true;
+      if (this.#lpThreshold !== null && result.avgLogprob < this.#lpThreshold) needsFallback = true;
+      if (this.#nsThreshold !== null && result.noSpeechProb > this.#nsThreshold) needsFallback = false;
+      if (!needsFallback) break;
+    }
+    return result!;
+  }
+
+  /** One window at `#seek` (transcribe.py loop body). */
+  async #window(mel: MlxArray, contentFrames: number, clipEnd: number): Promise<void> {
+    const { model, tokenizer: t } = this.#tr;
+    const o = this.#o;
+    const seek = this.#seek;
+    const timeOffset = (seek * WHISPER_HOP) / WHISPER_SAMPLE_RATE;
+    const segmentSize = Math.min(WHISPER_N_FRAMES, contentFrames - seek, clipEnd - seek);
+    const segmentDuration = (segmentSize * WHISPER_HOP) / WHISPER_SAMPLE_RATE;
+    const melSeg = mel.slice([seek, 0], [seek + segmentSize, model.dims.nMels]);
+    const padded = padOrTrimFrames(melSeg, WHISPER_N_FRAMES, model.dtype);
+    melSeg.dispose();
+    const seg3 = reshape3(padded);
+    padded.dispose();
+    const prompt = this.#allTokens.slice(this.#promptResetSince);
+    let result: WhisperDecodingResult;
+    try {
+      result = this.#decodeWithFallback(seg3, prompt);
+    } catch (e) {
+      seg3.dispose();
+      throw e;
+    }
+    const tokens = result.tokens;
+    if (this.#nsThreshold !== null) {
+      let shouldSkip = result.noSpeechProb > this.#nsThreshold;
+      if (this.#lpThreshold !== null && result.avgLogprob > this.#lpThreshold) shouldSkip = false;
+      if (shouldSkip) {
+        seg3.dispose();
+        this.#seek += segmentSize;
+        return;
+      }
+    }
+    const previousSeek = seek;
+    const current: WhisperSegment[] = [];
+    const newSegment = (start: number, end: number, toks: number[]): WhisperSegment => ({
+      id: 0, seek, start, end,
+      text: t.decodeText(toks), tokens: toks,
+      temperature: result.temperature, avgLogprob: result.avgLogprob,
+      compressionRatio: result.compressionRatio, noSpeechProb: result.noSpeechProb,
+    });
+    const isTs = tokens.map((x) => x >= t.timestampBegin);
+    const singleTimestampEnding = tokens.length >= 2 && !isTs[tokens.length - 2] && isTs[tokens.length - 1];
+    const consecutive: number[] = [];
+    for (let i = 0; i + 1 < tokens.length; i++) if (isTs[i] && isTs[i + 1]) consecutive.push(i + 1);
+    const inputStride = WHISPER_N_FRAMES / model.dims.nAudioCtx; // 2
+    const timePrecision = (inputStride * WHISPER_HOP) / WHISPER_SAMPLE_RATE; // 0.02
+    if (consecutive.length > 0) {
+      const slices = [...consecutive];
+      if (singleTimestampEnding) slices.push(tokens.length);
+      let lastSlice = 0;
+      for (const cur of slices) {
+        const sliced = tokens.slice(lastSlice, cur);
+        const startPos = sliced[0]! - t.timestampBegin;
+        const endPos = sliced[sliced.length - 1]! - t.timestampBegin;
+        current.push(newSegment(timeOffset + startPos * timePrecision, timeOffset + endPos * timePrecision, sliced));
+        lastSlice = cur;
+      }
+      if (singleTimestampEnding) this.#seek += segmentSize;
+      else {
+        const lastPos = tokens[lastSlice - 1]! - t.timestampBegin;
+        this.#seek += lastPos * inputStride;
+      }
+    } else {
+      let duration = segmentDuration;
+      const timestamps = tokens.filter((x) => x >= t.timestampBegin);
+      if (timestamps.length > 0 && timestamps[timestamps.length - 1] !== t.timestampBegin) {
+        const lastPos = timestamps[timestamps.length - 1]! - t.timestampBegin;
+        duration = lastPos * timePrecision;
+      }
+      current.push(newSegment(timeOffset, timeOffset + duration, tokens));
+      this.#seek += segmentSize;
+    }
+
+    if (o.wordTimestamps) {
+      this.#lastSpeechTimestamp = addWordTimestamps(current, model, t, this.#language!, seg3, segmentSize, this.#lastSpeechTimestamp);
+      if (!singleTimestampEnding) {
+        const lastWordEnd = lastWordEndOf(current);
+        if (lastWordEnd !== null && lastWordEnd > timeOffset) this.#seek = Math.round(lastWordEnd * WHISPER_FRAMES_PER_SECOND);
+      }
+      const threshold = o.hallucinationSilenceThreshold ?? null;
+      if (threshold !== null) {
+        const windowEndTime = ((previousSeek + WHISPER_N_FRAMES) * WHISPER_HOP) / WHISPER_SAMPLE_RATE;
+        if (!singleTimestampEnding) {
+          const lastWordEnd = lastWordEndOf(current);
+          if (lastWordEnd !== null && lastWordEnd > timeOffset) {
+            const remaining = windowEndTime - lastWordEnd;
+            this.#seek = remaining > threshold ? Math.round(lastWordEnd * WHISPER_FRAMES_PER_SECOND) : previousSeek + segmentSize;
+          }
+        }
+        const first = current.find((s) => s.words && s.words.length);
+        if (first && isSegmentAnomaly(first)) {
+          const gap = first.start - timeOffset;
+          if (gap > threshold) { this.#seek = previousSeek + Math.round(gap * WHISPER_FRAMES_PER_SECOND); seg3.dispose(); return; }
+        }
+        let halLastEnd = this.#lastSpeechTimestamp;
+        for (let si = 0; si < current.length; si++) {
+          const segment = current[si]!;
+          if (!segment.words?.length) continue;
+          if (isSegmentAnomaly(segment)) {
+            const nextSeg = current.slice(si + 1).find((s) => s.words && s.words.length);
+            const halNextStart = nextSeg ? nextSeg.words![0]!.start : timeOffset + segmentDuration;
+            const silenceBefore = segment.start - halLastEnd > threshold || segment.start < threshold || segment.start - timeOffset < 2.0;
+            const silenceAfter = halNextStart - segment.end > threshold || isSegmentAnomaly(nextSeg) || windowEndTime - segment.end < 2.0;
+            if (silenceBefore && silenceAfter) {
+              this.#seek = Math.round(Math.max(timeOffset + 1, segment.start) * WHISPER_FRAMES_PER_SECOND);
+              if ((contentFrames * WHISPER_HOP) / WHISPER_SAMPLE_RATE - segment.end < threshold) this.#seek = contentFrames;
+              current.splice(si);
+              break;
+            }
+          }
+          halLastEnd = segment.end;
+        }
+      }
+      const lastWordEnd = lastWordEndOf(current);
+      if (lastWordEnd !== null) this.#lastSpeechTimestamp = lastWordEnd;
+    }
+    seg3.dispose();
+    for (const s of current) {
+      if (s.start === s.end || s.text.trim() === "") { s.text = ""; s.tokens = []; s.words = []; }
+    }
+    for (const s of current) {
+      s.id = this.#allSegments.length;
+      this.#allSegments.push(s);
+      o.onSegment?.(s);
+    }
+    for (const s of current) this.#allTokens.push(...s.tokens);
+    if (!this.#condition || result.temperature > 0.5) this.#promptResetSince = this.#allTokens.length;
+  }
+}
+
+/** [T, M] → [1, T, M]. */
+function reshape3(a: MlxArray): MlxArray {
+  const [T, M] = a.shape as [number, number];
+  return ops.reshape(a, [1, T, M]);
+}
+export { Dtype };
+
+const PUNCTUATION = "\"'“¿([{-\"'.。,，!！?？:：”)]}、";
+
+function lastWordEndOf(segments: WhisperSegment[]): number | null {
+  for (let i = segments.length - 1; i >= 0; i--) {
+    const w = segments[i]!.words;
+    if (w && w.length) return w[w.length - 1]!.end;
+  }
+  return segments.length ? segments[segments.length - 1]!.end : null;
+}
+
+function wordAnomalyScore(word: WhisperWord): number {
+  const duration = word.end - word.start;
+  let score = 0;
+  if (word.probability < 0.15) score += 1;
+  if (duration < 0.133) score += (0.133 - duration) * 15;
+  if (duration > 2.0) score += duration - 2.0;
+  return score;
+}
+
+function isSegmentAnomaly(segment: WhisperSegment | undefined): boolean {
+  if (!segment || !segment.words?.length) return false;
+  const words = segment.words.filter((w) => !PUNCTUATION.includes(w.word)).slice(0, 8);
+  const score = words.reduce((a, w) => a + wordAnomalyScore(w), 0);
+  return score >= 3 || score + 0.01 >= words.length;
+}