mlx-bun 0.4.0 → 0.5.0

package/src/audio/whisper-format.ts

@@ -0,0 +1,73 @@
+// Transcription output formats shared by the CLI and /v1/audio/transcriptions:
+// OpenAI `json` / `verbose_json` / `text` / `srt` / `vtt`.
+
+import type { WhisperSegment, WhisperTranscription } from "./whisper-transcribe";
+
+export type TranscriptionFormat = "json" | "verbose_json" | "text" | "srt" | "vtt";
+
+export const TRANSCRIPTION_FORMATS: readonly TranscriptionFormat[] =
+  ["json", "verbose_json", "text", "srt", "vtt"];
+
+function pad2(n: number): string {
+  return n < 10 ? `0${n}` : String(n);
+}
+
+/** HH:MM:SS,mmm (srt) or HH:MM:SS.mmm (vtt). */
+export function formatTimestamp(seconds: number, separator: "," | "." = ","): string {
+  let ms = Math.round(seconds * 1000);
+  const hours = Math.floor(ms / 3_600_000);
+  ms -= hours * 3_600_000;
+  const minutes = Math.floor(ms / 60_000);
+  ms -= minutes * 60_000;
+  const secs = Math.floor(ms / 1000);
+  ms -= secs * 1000;
+  return `${pad2(hours)}:${pad2(minutes)}:${pad2(secs)}${separator}${String(ms).padStart(3, "0")}`;
+}
+
+export function toSrt(segments: WhisperSegment[]): string {
+  let out = "";
+  let n = 1;
+  for (const s of segments) {
+    if (!s.text.trim()) continue;
+    out += `${n++}\n${formatTimestamp(s.start, ",")} --> ${formatTimestamp(s.end, ",")}\n${s.text.trim()}\n\n`;
+  }
+  return out;
+}
+
+export function toVtt(segments: WhisperSegment[]): string {
+  let out = "WEBVTT\n\n";
+  for (const s of segments) {
+    if (!s.text.trim()) continue;
+    out += `${formatTimestamp(s.start, ".")} --> ${formatTimestamp(s.end, ".")}\n${s.text.trim()}\n\n`;
+  }
+  return out;
+}
+
+/** OpenAI verbose_json body (segments carry the openai-whisper fields). */
+export function toVerboseJson(r: WhisperTranscription, durationSeconds: number, task: string): Record<string, unknown> {
+  return {
+    task,
+    language: r.language,
+    duration: durationSeconds,
+    text: r.text.trim(),
+    segments: r.segments.map((s) => ({
+      id: s.id, seek: s.seek, start: s.start, end: s.end, text: s.text, tokens: s.tokens,
+      temperature: s.temperature, avg_logprob: s.avgLogprob, compression_ratio: s.compressionRatio,
+      no_speech_prob: s.noSpeechProb,
+      ...(s.words ? { words: s.words } : {}),
+    })),
+    ...(r.segments.some((s) => s.words) ? { words: r.segments.flatMap((s) => s.words ?? []) } : {}),
+  };
+}
+
+export function formatTranscription(
+  r: WhisperTranscription, format: TranscriptionFormat, durationSeconds: number, task: string,
+): string {
+  switch (format) {
+    case "text": return r.text.trim() + "\n";
+    case "json": return JSON.stringify({ text: r.text.trim() }) + "\n";
+    case "verbose_json": return JSON.stringify(toVerboseJson(r, durationSeconds, task)) + "\n";
+    case "srt": return toSrt(r.segments);
+    case "vtt": return toVtt(r.segments);
+  }
+}

package/src/audio/whisper-mel.ts

@@ -0,0 +1,185 @@
+// Whisper log-mel spectrogram — an op-for-op port of mlx-whisper's
+// audio.py (log_mel_spectrogram / stft / mel_filters), gated bit-exact
+// against the oracle in tests/parity/whisper.test.ts.
+//
+// Pipeline (single 16 kHz mono float32 waveform):
+//   [optional zero pad of `padding` samples on the right]
+//   → reflect pad N_FFT/2 = 200 samples each side (done host-side; the
+//     values are a copy so the device graph is identical)
+//   → frames [T, 400] via as_strided (stride HOP = 160)
+//   → × periodic Hann window (np.hanning(401)[:-1], float32)
+//   → rfft n=400 → drop the LAST frame → |·|² (float32)
+//   → @ melᵀ [201, 128]   (librosa slaney filters, float32 — built below)
+//   → log10(max(·, 1e-10)) → max(·, global_max − 8) → (· + 4) / 4
+// Output: [T, n_mels] float32 where T = (len + 400 − 400 + 160) // 160 − 1
+// (the oracle's `freqs[:-1]`), i.e. exactly len/160 frames for a multiple
+// of the hop.
+//
+// The mel filter bank reproduces librosa.filters.mel(sr=16000, n_fft=400,
+// n_mels, htk=False, norm="slaney") in float64 and rounds to float32 — the
+// same values mlx-whisper ships pre-computed in assets/mel_filters.npz
+// (checked bit-exact by the parity test against the oracle dump).
+
+import { MlxArray } from "../mlx/array";
+import { Dtype } from "../mlx/ffi";
+import * as ops from "../mlx/ops";
+
+export const WHISPER_SAMPLE_RATE = 16_000;
+export const WHISPER_N_FFT = 400;
+export const WHISPER_HOP = 160;
+export const WHISPER_CHUNK_SECONDS = 30;
+/** 480_000 samples per 30 s window. */
+export const WHISPER_N_SAMPLES = WHISPER_CHUNK_SECONDS * WHISPER_SAMPLE_RATE;
+/** 3000 mel frames per window. */
+export const WHISPER_N_FRAMES = WHISPER_N_SAMPLES / WHISPER_HOP;
+/** 100 mel frames per second. */
+export const WHISPER_FRAMES_PER_SECOND = WHISPER_SAMPLE_RATE / WHISPER_HOP;
+/** 50 encoder positions per second (conv2 stride 2). */
+export const WHISPER_TOKENS_PER_SECOND = WHISPER_SAMPLE_RATE / (WHISPER_HOP * 2);
+
+const dispose = (old: MlxArray, next: MlxArray): MlxArray => {
+  old.dispose();
+  return next;
+};
+
+// --- librosa slaney mel filter bank ----------------------------------------
+const F_SP = 200.0 / 3;
+const MIN_LOG_HZ = 1000.0;
+const MIN_LOG_MEL = MIN_LOG_HZ / F_SP; // 15
+const LOGSTEP = Math.log(6.4) / 27.0;
+
+function hzToMel(f: number): number {
+  const m = f / F_SP;
+  return f >= MIN_LOG_HZ ? MIN_LOG_MEL + Math.log(f / MIN_LOG_HZ) / LOGSTEP : m;
+}
+function melToHz(m: number): number {
+  return m >= MIN_LOG_MEL ? MIN_LOG_HZ * Math.exp(LOGSTEP * (m - MIN_LOG_MEL)) : F_SP * m;
+}
+
+/** Row-major [nMels, 201] float32 — librosa.filters.mel semantics. */
+export function whisperMelFilters(nMels: number): Float32Array {
+  const nBins = WHISPER_N_FFT / 2 + 1; // 201
+  const fmax = WHISPER_SAMPLE_RATE / 2;
+  // np.fft.rfftfreq(400, 1/16000) = arange(201) * 16000/400
+  // np.fft.rfftfreq: val = 1/(n·d) with d = 1/sr, then k·val — keep numpy's
+  // float64 rounding (1/16000 is inexact in binary) for bit-exact filters.
+  const val = 1.0 / (WHISPER_N_FFT * (1.0 / WHISPER_SAMPLE_RATE));
+  const fftFreqs = new Float64Array(nBins);
+  for (let k = 0; k < nBins; k++) fftFreqs[k] = k * val;
+  // mel_frequencies(n_mels + 2): np.linspace in mel space, mapped to Hz
+  const nPts = nMels + 2;
+  const melMin = hzToMel(0);
+  const melMax = hzToMel(fmax);
+  const melF = new Float64Array(nPts);
+  const step = (melMax - melMin) / (nPts - 1);
+  for (let i = 0; i < nPts; i++) melF[i] = melToHz(melMin + step * i);
+  const out = new Float32Array(nMels * nBins);
+  for (let i = 0; i < nMels; i++) {
+    const fd0 = melF[i + 1]! - melF[i]!;
+    const fd1 = melF[i + 2]! - melF[i + 1]!;
+    const enorm = 2.0 / (melF[i + 2]! - melF[i]!);
+    for (let k = 0; k < nBins; k++) {
+      const lower = -(melF[i]! - fftFreqs[k]!) / fd0;
+      const upper = (melF[i + 2]! - fftFreqs[k]!) / fd1;
+      // librosa stores the un-normalized weight into a float32 array, then
+      // `weights *= enorm[:, None]` (float64 multiply, float32 store).
+      const w = Math.fround(Math.max(0, Math.min(lower, upper)));
+      out[i * nBins + k] = Math.fround(w * enorm);
+    }
+  }
+  return out;
+}
+
+/** np.hanning(N+1)[:-1] as float32 (periodic Hann). */
+export function whisperHannWindow(): Float32Array {
+  const n = WHISPER_N_FFT;
+  const w = new Float32Array(n);
+  for (let i = 0; i < n; i++) w[i] = Math.fround(0.5 - 0.5 * Math.cos((2 * Math.PI * i) / n));
+  return w;
+}
+
+/** Device-resident constants for one mel configuration (built once per model). */
+export class WhisperMelExtractor {
+  readonly nMels: number;
+  /** [201, nMels] float32 — filters transposed for `magnitudes @ filtersᵀ`. */
+  readonly #filtersT: MlxArray;
+  /** [400] float32. */
+  readonly #window: MlxArray;
+
+  constructor(nMels: number) {
+    this.nMels = nMels;
+    const f = whisperMelFilters(nMels);
+    const nBins = WHISPER_N_FFT / 2 + 1;
+    const t = new Float32Array(nBins * nMels);
+    for (let i = 0; i < nMels; i++)
+      for (let k = 0; k < nBins; k++) t[k * nMels + i] = f[i * nBins + k]!;
+    this.#filtersT = MlxArray.fromFloat32(t, [nBins, nMels]);
+    this.#window = MlxArray.fromFloat32(whisperHannWindow(), [WHISPER_N_FFT]);
+  }
+
+  /** Number of mel frames the oracle produces for `numSamples` (+ padding). */
+  static frameCount(numSamples: number, padding = 0): number {
+    const len = numSamples + padding + WHISPER_N_FFT; // reflect-padded length
+    return Math.floor((len - WHISPER_N_FFT + WHISPER_HOP) / WHISPER_HOP) - 1;
+  }
+
+  /** log_mel_spectrogram(audio, n_mels, padding) → [T, nMels] float32 (lazy). */
+  logMel(samples: Float32Array, padding = 0): MlxArray {
+    const half = WHISPER_N_FFT / 2;
+    const n = samples.length + padding;
+    if (samples.length <= half) throw new Error(`whisper mel: clip too short (${samples.length} samples)`);
+    // host-side: zero pad right, then reflect pad `half` on both sides
+    const padded = new Float32Array(n + 2 * half);
+    // prefix = x[1:half+1][::-1]
+    for (let i = 0; i < half; i++) padded[i] = samples[half - i] ?? 0;
+    padded.set(samples, half);
+    // (zero padding region already 0)
+    // suffix = x[-(half+1):-1][::-1] over the ZERO-PADDED signal
+    const src = (idx: number): number => (idx < samples.length ? samples[idx]! : 0);
+    for (let i = 0; i < half; i++) padded[half + n + i] = src(n - 2 - i);
+    const x = MlxArray.fromFloat32(padded, [padded.length]);
+    const t = Math.floor((padded.length - WHISPER_N_FFT + WHISPER_HOP) / WHISPER_HOP);
+    let frames = ops.asStrided(x, [t, WHISPER_N_FFT], [WHISPER_HOP, 1]);
+    frames = dispose(frames, ops.mul(frames, this.#window));
+    let spec = ops.rfft(frames, WHISPER_N_FFT, -1);
+    frames.dispose();
+    x.dispose();
+    spec = dispose(spec, spec.slice([0, 0], [t - 1, WHISPER_N_FFT / 2 + 1]));
+    spec = dispose(spec, ops.abs(spec));
+    spec = dispose(spec, ops.square(spec));
+    let mel = ops.matmul(spec, this.#filtersT);
+    spec.dispose();
+    const floor = ops.scalarLike(1e-10, mel);
+    mel = dispose(mel, ops.maximum(mel, floor));
+    floor.dispose();
+    mel = dispose(mel, ops.log10(mel));
+    const mx = ops.maxAll(mel);
+    const eight = ops.scalarLike(8.0, mel);
+    const lo = ops.sub(mx, eight);
+    mel = dispose(mel, ops.maximum(mel, lo));
+    mx.dispose();
+    eight.dispose();
+    lo.dispose();
+    const four = ops.scalarLike(4.0, mel);
+    mel = dispose(mel, ops.add(mel, four));
+    mel = dispose(mel, ops.div(mel, four));
+    four.dispose();
+    return mel;
+  }
+
+  dispose(): void {
+    this.#filtersT.dispose();
+    this.#window.dispose();
+  }
+}
+
+/** pad_or_trim along the frame axis to `length` frames (zeros on the right). */
+export function padOrTrimFrames(mel: MlxArray, length: number, dtype: Dtype): MlxArray {
+  const [t, m] = mel.shape as [number, number];
+  let out: MlxArray;
+  if (t > length) out = mel.slice([0, 0], [length, m]);
+  else if (t < length) out = ops.pad(mel, [[0, length - t], [0, 0]]);
+  else out = ops.copyOf(mel);
+  if (out.dtype !== dtype) out = dispose(out, out.astype(dtype));
+  return out;
+}

package/src/audio/whisper-timing.ts

@@ -0,0 +1,274 @@
+// Word-level timestamps — a port of mlx-whisper's timing.py: cross-attention
+// weights of the checkpoint's `alignment_heads` over the text tokens →
+// z-normalize per head → median filter (width 7) along time → mean over
+// heads → DTW (numba dtw_cpu / backtrace, ported to plain JS) → word
+// boundaries from the tokenizer's word split; then add_word_timestamps'
+// duration heuristics and punctuation merging. Runs the FAITHFUL decoder
+// once per window with `wantCrossQk` (the alignment pass of the reference).
+
+import { MlxArray } from "../mlx/array";
+import { Dtype } from "../mlx/ffi";
+import * as ops from "../mlx/ops";
+import { WhisperKvCache, type WhisperModel } from "../model/whisper";
+import { WHISPER_HOP, WHISPER_SAMPLE_RATE, WHISPER_TOKENS_PER_SECOND } from "./whisper-mel";
+import type { WhisperTokenizer } from "./whisper-tokenizer";
+import type { WhisperSegment, WhisperWord } from "./whisper-transcribe";
+
+export interface WordTiming {
+  word: string;
+  tokens: number[];
+  start: number;
+  end: number;
+  probability: number;
+}
+
+/** Reflect-padded median filter along the last axis (scipy.signal.medfilt semantics
+ *  on the padded row; odd width). */
+export function medianFilter(x: Float32Array, rows: number, cols: number, width: number): Float32Array {
+  const pad = Math.floor(width / 2);
+  if (cols <= pad) return x;
+  const out = new Float32Array(rows * cols);
+  const buf = new Float32Array(width);
+  for (let r = 0; r < rows; r++) {
+    const row = x.subarray(r * cols, (r + 1) * cols);
+    for (let c = 0; c < cols; c++) {
+      for (let k = 0; k < width; k++) {
+        let j = c - pad + k;
+        if (j < 0) j = -j; // reflect
+        else if (j >= cols) j = 2 * cols - j - 2;
+        buf[k] = row[j]!;
+      }
+      const sorted = buf.slice().sort();
+      out[r * cols + c] = sorted[pad]!;
+    }
+  }
+  return out;
+}
+
+/** DTW over cost matrix x [N, M] (row-major); returns the aligned index
+ *  pairs (text_indices, time_indices) like timing.dtw_cpu + backtrace. */
+export function dtw(x: Float32Array, N: number, M: number): { text: Int32Array; time: Int32Array } {
+  const cost = new Float32Array((N + 1) * (M + 1)).fill(Infinity);
+  const trace = new Int8Array((N + 1) * (M + 1)).fill(-1);
+  const W = M + 1;
+  cost[0] = 0;
+  for (let j = 1; j <= M; j++) {
+    for (let i = 1; i <= N; i++) {
+      const c0 = cost[(i - 1) * W + (j - 1)]!;
+      const c1 = cost[(i - 1) * W + j]!;
+      const c2 = cost[i * W + (j - 1)]!;
+      let c: number;
+      let t: number;
+      if (c0 < c1 && c0 < c2) { c = c0; t = 0; }
+      else if (c1 < c0 && c1 < c2) { c = c1; t = 1; }
+      else { c = c2; t = 2; }
+      cost[i * W + j] = x[(i - 1) * M + (j - 1)]! + c;
+      trace[i * W + j] = t;
+    }
+  }
+  // backtrace
+  let i = N;
+  let j = M;
+  for (let jj = 0; jj <= M; jj++) trace[jj] = 2;
+  for (let ii = 0; ii <= N; ii++) trace[ii * W] = 1;
+  const text: number[] = [];
+  const time: number[] = [];
+  while (i > 0 || j > 0) {
+    text.push(i - 1);
+    time.push(j - 1);
+    const t = trace[i * W + j];
+    if (t === 0) { i--; j--; }
+    else if (t === 1) i--;
+    else if (t === 2) j--;
+    else throw new Error("Unexpected trace value");
+  }
+  text.reverse();
+  time.reverse();
+  return { text: Int32Array.from(text), time: Int32Array.from(time) };
+}
+
+/** find_alignment: word timings for `textTokens` within one 30 s window
+ *  (`mel` [1, 3000, n_mels] model dtype, `numFrames` = content mel frames). */
+export function findAlignment(
+  model: WhisperModel, tokenizer: WhisperTokenizer, language: string,
+  textTokens: number[], mel: MlxArray, numFrames: number,
+  medfiltWidth = 7, qkScale = 1.0,
+): WordTiming[] {
+  if (textTokens.length === 0) return [];
+  const sot = tokenizer.sotSequence(language, "transcribe");
+  const tokens = [...sot, tokenizer.noTimestamps, ...textTokens, tokenizer.eot];
+  const feats = model.encode(mel);
+  const cache = new WhisperKvCache(model.dims.nTextLayer);
+  const tokArr = ops.fromInt32(tokens, [1, tokens.length]);
+  const { logits, crossQk } = model.decode(tokArr, feats, cache, { wantCrossQk: true });
+  tokArr.dispose();
+  feats.dispose();
+  cache.dispose();
+  const V = model.dims.nVocab;
+  const T = tokens.length;
+  // text-token probabilities from the logits predicting each text token
+  let sampled = logits.slice([0, sot.length, 0], [1, T - 2, tokenizer.eot]);
+  logits.dispose();
+  sampled = (() => { const r = ops.reshape(sampled, [T - 2 - sot.length, tokenizer.eot]); sampled.dispose(); return r; })();
+  const probs = ops.softmaxAxis(sampled, -1, true);
+  sampled.dispose();
+  const idx = ops.fromInt32(textTokens, [textTokens.length, 1]);
+  const picked = ops.takeAlongAxis(probs, idx, 1);
+  idx.dispose();
+  probs.dispose();
+  const textTokenProbs = picked.toFloat32();
+  picked.dispose();
+  // alignment heads: [heads, T, frames]
+  const heads = model.alignmentHeads;
+  const frames = Math.floor(numFrames / 2);
+  const stacked: MlxArray[] = [];
+  for (const [l, h] of heads) {
+    const qk = crossQk[l];
+    if (!qk) continue;
+    const w = qk.slice([0, h, 0, 0], [1, h + 1, T, frames]); // [1,1,T,frames]
+    stacked.push(ops.reshape(w, [T, frames]));
+    w.dispose();
+  }
+  for (const q of crossQk) q?.dispose();
+  let weights = ops.stackAxis(stacked, 0); // [heads, T, frames]
+  for (const a of stacked) a.dispose();
+  if (qkScale !== 1.0) weights = (() => { const s = ops.scalarLike(qkScale, weights); const r = ops.mul(weights, s); s.dispose(); weights.dispose(); return r; })();
+  weights = (() => { const r = ops.softmaxAxis(weights, -1, true); weights.dispose(); return r; })();
+  weights = (() => { const r = weights.astype(Dtype.float32); weights.dispose(); return r; })();
+  const mean = ops.meanAxis(weights, -2, true);
+  let std = ops.varAxis(weights, -2, true, 0);
+  std = (() => { const r = ops.sqrt(std); std.dispose(); return r; })();
+  let z = ops.sub(weights, mean);
+  z = (() => { const r = ops.div(z, std); z.dispose(); return r; })();
+  weights.dispose(); mean.dispose(); std.dispose();
+  const nHeads = stacked.length;
+  const zHost = z.toFloat32();
+  z.dispose();
+  const filtered = medianFilter(zHost, nHeads * T, frames, medfiltWidth);
+  // mean over heads → [T, frames], then rows sot.length … T-2
+  const rows = T - 1 - sot.length;
+  const matrix = new Float32Array(rows * frames);
+  for (let r = 0; r < rows; r++)
+    for (let f = 0; f < frames; f++) {
+      let acc = 0;
+      for (let h = 0; h < nHeads; h++) acc += filtered[(h * T + sot.length + r) * frames + f]!;
+      matrix[r * frames + f] = -(acc / nHeads);
+    }
+  const { text: textIndices, time: timeIndices } = dtw(matrix, rows, frames);
+  const { words, wordTokens } = tokenizer.splitToWordTokens([...textTokens, tokenizer.eot], language);
+  if (wordTokens.length <= 1) return [];
+  const boundaries = [0];
+  for (let i = 0; i < wordTokens.length - 1; i++) boundaries.push(boundaries[i]! + wordTokens[i]!.length);
+  const jumps: boolean[] = [true];
+  for (let i = 1; i < textIndices.length; i++) jumps.push(textIndices[i]! - textIndices[i - 1]! !== 0);
+  const jumpTimes: number[] = [];
+  for (let i = 0; i < timeIndices.length; i++) if (jumps[i]) jumpTimes.push(timeIndices[i]! / WHISPER_TOKENS_PER_SECOND);
+  const out: WordTiming[] = [];
+  for (let w = 0; w < words.length; w++) {
+    const a = boundaries[w]!;
+    const b = boundaries[w + 1] ?? a;
+    let p = 0;
+    for (let i = a; i < b; i++) p += textTokenProbs[i] ?? 0;
+    out.push({
+      word: words[w]!, tokens: wordTokens[w]!,
+      start: jumpTimes[a] ?? jumpTimes[jumpTimes.length - 1] ?? 0,
+      end: jumpTimes[b] ?? jumpTimes[jumpTimes.length - 1] ?? 0,
+      probability: b > a ? p / (b - a) : 0,
+    });
+  }
+  return out;
+}
+
+const SENTENCE_END = ".。!！?？";
+
+function mergePunctuations(alignment: WordTiming[], prepended: string, appended: string): void {
+  let i = alignment.length - 2;
+  let j = alignment.length - 1;
+  while (i >= 0) {
+    const previous = alignment[i]!;
+    const following = alignment[j]!;
+    if (previous.word.startsWith(" ") && prepended.includes(previous.word.trim())) {
+      following.word = previous.word + following.word;
+      following.tokens = [...previous.tokens, ...following.tokens];
+      previous.word = "";
+      previous.tokens = [];
+    } else j = i;
+    i--;
+  }
+  i = 0;
+  j = 1;
+  while (j < alignment.length) {
+    const previous = alignment[i]!;
+    const following = alignment[j]!;
+    if (!previous.word.endsWith(" ") && appended.includes(following.word)) {
+      previous.word = previous.word + following.word;
+      previous.tokens = [...previous.tokens, ...following.tokens];
+      following.word = "";
+      following.tokens = [];
+    } else i = j;
+    j++;
+  }
+}
+
+/** add_word_timestamps: annotates `segments` (one window) with words. */
+export function addWordTimestamps(
+  segments: WhisperSegment[], model: WhisperModel, tokenizer: WhisperTokenizer, language: string,
+  mel: MlxArray, numFrames: number, lastSpeechTimestamp: number,
+  prependPunctuations = "\"'“¿([{-", appendPunctuations = "\"'.。,，!！?？:：”)]}、",
+): number {
+  if (segments.length === 0) return lastSpeechTimestamp;
+  const perSegment = segments.map((s) => s.tokens.filter((t) => t < tokenizer.eot));
+  const textTokens = perSegment.flat();
+  const alignment = findAlignment(model, tokenizer, language, textTokens, mel, numFrames);
+  const durations = alignment.map((t) => t.end - t.start).filter((d) => d !== 0);
+  const sortedD = [...durations].sort((a, b) => a - b);
+  let medianDuration = sortedD.length ? (sortedD.length % 2 ? sortedD[(sortedD.length - 1) / 2]! : (sortedD[sortedD.length / 2 - 1]! + sortedD[sortedD.length / 2]!) / 2) : 0;
+  medianDuration = Math.min(0.7, medianDuration);
+  const maxDuration = medianDuration * 2;
+  if (durations.length > 0) {
+    for (let i = 1; i < alignment.length; i++) {
+      const a = alignment[i]!;
+      if (a.end - a.start > maxDuration) {
+        if (SENTENCE_END.includes(a.word)) a.end = a.start + maxDuration;
+        else if (SENTENCE_END.includes(alignment[i - 1]!.word)) a.start = a.end - maxDuration;
+      }
+    }
+  }
+  mergePunctuations(alignment, prependPunctuations, appendPunctuations);
+  const timeOffset = (segments[0]!.seek * WHISPER_HOP) / WHISPER_SAMPLE_RATE;
+  let wordIndex = 0;
+  for (let si = 0; si < segments.length; si++) {
+    const segment = segments[si]!;
+    const tt = perSegment[si]!;
+    let saved = 0;
+    const words: WhisperWord[] = [];
+    while (wordIndex < alignment.length && saved < tt.length) {
+      const timing = alignment[wordIndex]!;
+      if (timing.word)
+        words.push({ word: timing.word, start: Math.round((timeOffset + timing.start) * 100) / 100, end: Math.round((timeOffset + timing.end) * 100) / 100, probability: timing.probability });
+      saved += timing.tokens.length;
+      wordIndex++;
+    }
+    if (words.length > 0) {
+      const w0 = words[0]!;
+      if (w0.end - lastSpeechTimestamp > medianDuration * 4 &&
+          (w0.end - w0.start > maxDuration || (words.length > 1 && words[1]!.end - w0.start > maxDuration * 2))) {
+        if (words.length > 1 && words[1]!.end - words[1]!.start > maxDuration) {
+          const boundary = Math.max(words[1]!.end / 2, words[1]!.end - maxDuration);
+          w0.end = words[1]!.start = boundary;
+        }
+        w0.start = Math.max(0, w0.end - maxDuration);
+      }
+      if (segment.start < w0.end && segment.start - 0.5 > w0.start)
+        w0.start = Math.max(0, Math.min(w0.end - medianDuration, segment.start));
+      else segment.start = w0.start;
+      const wl = words[words.length - 1]!;
+      if (segment.end > wl.start && segment.end + 0.5 < wl.end)
+        wl.end = Math.max(wl.start + medianDuration, segment.end);
+      else segment.end = wl.end;
+      lastSpeechTimestamp = segment.end;
+    }
+    segment.words = words;
+  }
+  return lastSpeechTimestamp;
+}