@gotcos/glasses-server 6.71.14 → 6.72.0

package/server/lib/meeting-batch-transcribe.ts

@@ -6,14 +6,23 @@ import { createHash } from 'node:crypto'
 import { existsSync, readFileSync, readdirSync, utimesSync, writeFileSync } from 'node:fs'
 import { availableParallelism } from 'node:os'
 import { basename, join, resolve } from 'node:path'
-import { enhanceAudio } from './audio-enhance.js'
 import {
+  applyCorrections,
   getHighQualityCheckpointFingerprint,
   getHighQualityTranscriptionCapability,
   getWhisperCommitCapability,
   transcribeHighQuality,
+  type HighQualityTranscriptionResult,
   type WhisperWord,
 } from './whisper-local.js'
+import { asrFrontEnd } from './stt-live.js'
+import { measureSpeechSeconds } from './vad-silero.js'
+import { batchTimestampsEnabled } from './whisper-batch-timing.js'
+import { readSttSettings, recordLastSttJob, type FrontEndId, type HqId, type SecondOpinionId } from './stt-settings.js'
+import { currentEffectiveStt } from './stt-runtime.js'
+import { transcribeParakeet, type ParakeetResult, type ParakeetRunOptions } from './parakeet.js'
+import { isDeletionProneEngine, resolveHypotheses, RESOLVER_VERSION, type Hypothesis } from './stt-resolver.js'
+import { getVocabulary } from './profile.js'
 import { durableAtomicWriteFileSync } from './atomic-fs.js'
 import { isMetalBatchPreempted } from './whisper-metal-gate.js'
 import {
@@ -27,7 +36,9 @@ import {
   evaluateBatchQuality,
   type BatchResult,
   type BatchSegment,
+  type BatchSttSummary,
   type BatchTranscription,
+  type SegmentStt,
 } from './batch-transcript-quality.js'
 
 const WAV_HEADER_SIZE = 44
@@ -223,15 +234,22 @@ function segmentSourceHash(audioDir: string, segment: BatchSegment): string | nu
   return hash.digest('hex')
 }
 
+/** The decoder identity a checkpoint was made under: timestamps and front end (6.72.0). */
+export interface CheckpointIdentity { timestamps: boolean; frontEnd: FrontEndId }
+function currentCheckpointIdentity(): CheckpointIdentity {
+  return { timestamps: batchTimestampsEnabled(), frontEnd: readSttSettings().frontEnd }
+}
+
 function progressiveFailureIdentity(
   audioDir: string,
   segment: BatchSegment,
   previousText?: string,
+  identity: CheckpointIdentity = currentCheckpointIdentity(),
 ): string | null {
   const sourceHash = segmentSourceHash(audioDir, segment)
   if (!sourceHash) return null
   return createHash('sha256').update([
-    checkpointKey(segment), sourceHash, contextHash(previousText), getHighQualityCheckpointFingerprint(),
+    checkpointKey(segment), sourceHash, contextHash(previousText), getHighQualityCheckpointFingerprint(identity),
   ].join('\0')).digest('hex')
 }
 
@@ -250,8 +268,9 @@ function readProgressiveCheckpoint(
   segment: BatchSegment,
   previousText?: string,
   expectedSessionId?: string,
+  identity: CheckpointIdentity = currentCheckpointIdentity(),
 ): BatchResult | null {
-  const cached = readProgressiveCheckpointMetadata(audioDir, segment, previousText, undefined, expectedSessionId)
+  const cached = readProgressiveCheckpointMetadata(audioDir, segment, previousText, undefined, expectedSessionId, identity)
   if (!cached) return null
   const manifest = loadProgressiveManifest(audioDir)
   const checkpoint = manifest?.checkpoints?.[checkpointKey(segment)]
@@ -270,13 +289,14 @@ function readProgressiveCheckpointMetadata(
   previousText?: string,
   manifest = loadProgressiveManifest(audioDir),
   expectedSessionId?: string,
+  identity: CheckpointIdentity = currentCheckpointIdentity(),
 ): BatchResult | null {
   if (expectedSessionId && manifest?.sessionId !== expectedSessionId) return null
   const checkpoint = manifest?.checkpoints?.[checkpointKey(segment)]
   if (
     !checkpoint
     || checkpoint.contextHash !== contextHash(previousText)
-    || checkpoint.configFingerprint !== getHighQualityCheckpointFingerprint()
+    || checkpoint.configFingerprint !== getHighQualityCheckpointFingerprint(identity)
   ) return null
   const cached = checkpoint.result
   const validWord = (word: unknown, speakerRequired: boolean): boolean => {
@@ -310,7 +330,12 @@ function readProgressiveCheckpointMetadata(
     || !Array.isArray(cached.speakerWords)
     || cached.speakerWords.some(word => !validWord(word, true))
   ) return null
-  return cached
+  // Optional 6.72.0 fields: dropped (never trusted) when malformed, so reuse falls back to record-only.
+  const spokenOk = Array.isArray(cached.spokenWords) && cached.spokenWords.every(word => validWord(word, false))
+  const spansOk = Array.isArray(cached.speechSpans) && cached.speechSpans.every(span => Array.isArray(span) && span.length === 2
+    && span.every(x => typeof x === 'number' && Number.isFinite(x) && x >= 0) && span[1] >= span[0])
+  const { spokenWords: _s, speechSpans: _p, ...rest } = cached
+  return { ...rest, ...(spokenOk ? { spokenWords: cached.spokenWords } : {}), ...(spansOk ? { speechSpans: cached.speechSpans } : {}) }
 }
 
 function writeProgressiveCheckpoint(
@@ -392,14 +417,337 @@ export function readProgressiveAskRefinements(
   return results
 }
 
+/** The STT plan one HQ job runs with, frozen when the job starts (6.72.0). A
+ *  settings change applies to the next saved meeting, never mid-job. */
+export interface HqJobPlan {
+  engine: HqId
+  secondOpinion: SecondOpinionId
+  resolver: boolean
+  frontEnd: FrontEndId
+}
+
+export function currentHqJobPlan(): HqJobPlan {
+  const eff = currentEffectiveStt()
+  return { engine: eff.hq.engine, secondOpinion: eff.secondOpinion.engine, resolver: eff.resolver, frontEnd: eff.frontEnd }
+}
+
+/** Legacy pre-6.72 behaviour, for callers and tests that pass no plan. */
+export const LEGACY_HQ_PLAN: Readonly<HqJobPlan> = Object.freeze({ engine: 'large-v3-legacy', secondOpinion: 'off', resolver: false, frontEnd: 'full' })
+
+export interface HqDecodeDeps {
+  whisper: typeof transcribeHighQuality
+  parakeet: (wav: Buffer, options: ParakeetRunOptions) => Promise<ParakeetResult>
+  frontEnd: (audio: Buffer, frontEnd: FrontEndId) => Promise<Buffer>
+  vocabulary: () => string[]
+  /** Silero speech seconds of a window (null when unmeasured); judges an empty Parakeet primary. */
+  speechSeconds?: (wav: Buffer) => number | null
+}
+
+// Arrow wrappers, not references: a test that mocks one of these modules
+// without every export must still be able to import this one.
+const defaultHqDeps: HqDecodeDeps = {
+  whisper: (...args) => transcribeHighQuality(...args),
+  parakeet: (wav, options) => transcribeParakeet(wav, options),
+  frontEnd: (audio, frontEnd) => asrFrontEnd(audio, frontEnd),
+  vocabulary: () => getVocabulary(),
+  speechSeconds: wav => measureSpeechSeconds(wav),
+}
+
+/** A first Parakeet run in a process may compile CoreML (12 to 25 s); later runs are well under a second. */
+export const PARAKEET_FIRST_WINDOW_TIMEOUT_MS = 180_000
+export const PARAKEET_WINDOW_TIMEOUT_MS = 30_000
+
+interface WholeDecode {
+  engine: 'large-v3' | 'large-v3-legacy' | 'parakeet-ultra'
+  /** Name-corrected text. */
+  text: string
+  /** What BatchResult.words carries (Whisper tokens, or Parakeet words), name-corrected. */
+  words: WhisperWord[]
+  /** Uncorrected spoken words for the resolver; absent when the clocks are not trustworthy. */
+  spoken?: WhisperWord[]
+  /** Where the primary's own VAD heard speech (window clock); gap fill needs it. */
+  speechSpans?: Array<[number, number]>
+  timestamps: boolean
+}
+
+function errorReason(error: unknown): string {
+  return error instanceof Error ? `${error.name}: ${error.message}`.slice(0, 200) : String(error).slice(0, 200)
+}
+
+/** Parakeet failures in a row before the job stops calling it (each can cost its deadline). */
+export const PARAKEET_JOB_BREAKER_FAILURES = 2
+
+interface JobState {
+  parakeetRan: boolean
+  parakeetFailures: number
+  parakeetStopped: boolean
+}
+
+async function decodeWhisperOnce(
+  deps: HqDecodeDeps,
+  input: Buffer,
+  previousText: string | undefined,
+  timestamps: boolean,
+  segment: BatchSegment,
+  audioDir: string,
+): Promise<HighQualityTranscriptionResult> {
+  try {
+    return await deps.whisper(input, previousText?.slice(-250), { priority: 'batch', timestamps })
+  } catch (error) {
+    // A live meeting took the GPU mid-segment. The truncated Metal output
+    // was already discarded upstream (BLOCKER contract) — it is never
+    // accepted. Retry this SAME segment once on CPU, which cannot itself be
+    // preempted, so a busy day degrades to slow rather than to a silently
+    // missing stretch of transcript.
+    if (!isMetalBatchPreempted(error)) throw error
+    console.log(
+      `[meeting-batch] Segment ${segment.startChunkIdx}-${segment.endChunkIdx} preempted off Metal; `
+      + 'retrying once on CPU',
+    )
+    refreshPendingLease(audioDir)
+    return deps.whisper(input, previousText?.slice(-250), { priority: 'batch', forceCpu: true, forceCpuReason: 'preempt_retry', timestamps })
+  }
+}
+
+function asWholeWhisper(result: HighQualityTranscriptionResult, engine: 'large-v3' | 'large-v3-legacy', timestamps: boolean): WholeDecode {
+  const ran = result.timestamps ?? timestamps
+  return {
+    engine,
+    text: result.text,
+    words: result.words ?? [],
+    ...(ran && result.spokenWords ? { spoken: result.spokenWords } : {}),
+    ...(ran && result.speechSpans ? { speechSpans: result.speechSpans } : {}),
+    timestamps: ran,
+  }
+}
+
+/**
+ * Whisper for one window. A timestamp decode that fails (a VAD-table or clock
+ * check, or anything else) is retried once with the legacy decoder, so a
+ * timing problem never costs the window's text.
+ */
+async function decodeWhisperWindow(
+  deps: HqDecodeDeps,
+  input: Buffer,
+  previousText: string | undefined,
+  engine: 'large-v3' | 'large-v3-legacy',
+  segment: BatchSegment,
+  audioDir: string,
+): Promise<{ decode: WholeDecode; fallbackReason?: string }> {
+  const timestamps = batchTimestampsEnabled(engine)
+  try {
+    return { decode: asWholeWhisper(await decodeWhisperOnce(deps, input, previousText, timestamps, segment, audioDir), engine, timestamps) }
+  } catch (error) {
+    if (!timestamps) throw error
+    console.warn(`[meeting-batch] Timestamp decode failed for ${segment.startChunkIdx}-${segment.endChunkIdx}; retrying with the legacy decoder: ${errorReason(error)}`)
+    const legacy = await decodeWhisperOnce(deps, input, previousText, false, segment, audioDir)
+    return { decode: asWholeWhisper(legacy, 'large-v3-legacy', false), fallbackReason: `timestamps_failed: ${errorReason(error)}` }
+  }
+}
+
+/** A Parakeet window with no words over audio the speech detector says holds speech is a failed decode. */
+export const PARAKEET_EMPTY_SPEECH_SEC = 0.5
+
+async function decodeParakeetWindow(
+  deps: HqDecodeDeps,
+  input: Buffer,
+  timeoutMs: number,
+  model: 'parakeet-ultra',
+  requireWordsOverSpeech: boolean,
+): Promise<WholeDecode> {
+  const r = await deps.parakeet(input, { model, lane: 'batch', timeoutMs })
+  if (requireWordsOverSpeech && !r.text.trim()) {
+    const speech = deps.speechSeconds?.(input) ?? null
+    if (speech !== null && speech >= PARAKEET_EMPTY_SPEECH_SEC) {
+      throw Object.assign(new Error(`Parakeet returned no words over ${speech.toFixed(1)} s of speech`), { reason: 'bad_output' })
+    }
+  }
+  const spoken = r.words.map(w => ({ word: w.word, start: w.start, end: w.end, probability: w.probability ?? 0 }))
+  return {
+    engine: model,
+    text: applyCorrections(r.text),
+    words: spoken.map(w => ({ ...w, word: applyCorrections(w.word) })),
+    spoken,
+    timestamps: true,
+  }
+}
+
+const isParakeet = (engine: string): boolean => engine.startsWith('parakeet')
+
+function asHypothesis(d: WholeDecode): Hypothesis {
+  return { engine: d.engine, text: d.text, ...(d.spoken ? { words: d.spoken } : {}) }
+}
+
+function hypothesisRecord(d: WholeDecode): SegmentStt['primary'] {
+  return { engine: d.engine, text: d.text, ...(d.spoken ? { words: d.spoken.map(w => ({ word: w.word, start: w.start, end: w.end, probability: w.probability })) } : {}) }
+}
+
+function parakeetTimeout(job: JobState): number {
+  const t = job.parakeetRan ? PARAKEET_WINDOW_TIMEOUT_MS : PARAKEET_FIRST_WINDOW_TIMEOUT_MS
+  job.parakeetRan = true
+  return t
+}
+
+/** Run Parakeet under the job breaker: after PARAKEET_JOB_BREAKER_FAILURES in a row it is not called again this job. */
+async function parakeetGuarded(job: JobState, run: () => Promise<WholeDecode>): Promise<WholeDecode> {
+  if (job.parakeetStopped) throw Object.assign(new Error('Parakeet stopped for this job after repeated failures'), { reason: 'job_breaker' })
+  try {
+    const d = await run()
+    job.parakeetFailures = 0
+    return d
+  } catch (error) {
+    job.parakeetFailures++
+    if (job.parakeetFailures >= PARAKEET_JOB_BREAKER_FAILURES) {
+      job.parakeetStopped = true
+      console.warn(`[meeting-batch] Parakeet failed ${job.parakeetFailures} windows in a row; not calling it again this job`)
+    }
+    throw error
+  }
+}
+
+/**
+ * One HQ window under the job's plan: the primary engine (or a reused
+ * progressive checkpoint), then (if planned) the second opinion on the SAME
+ * front-end audio, then the resolver. No failure costs the window its text:
+ * Parakeet primary failing falls back to Large-v3; Whisper failing outright
+ * falls back to a planned Parakeet second opinion as the primary; a failed
+ * second opinion leaves the primary exactly as heard.
+ */
+async function decodeHqWindow(
+  input: Buffer,
+  previousText: string | undefined,
+  segment: BatchSegment,
+  entries: IndexedTranscriptChunk[],
+  plan: HqJobPlan,
+  deps: HqDecodeDeps,
+  job: JobState,
+  audioDir: string,
+  reused?: BatchResult,
+): Promise<BatchResult> {
+  let primary: WholeDecode
+  let fallbackReason: string | undefined
+  let usedSecondAsPrimary = false
+  const secondModel = plan.secondOpinion === 'parakeet-ultra' ? plan.secondOpinion : null
+  if (reused) {
+    const ts = batchTimestampsEnabled(plan.engine === 'large-v3-legacy' ? 'large-v3-legacy' : 'large-v3')
+    primary = {
+      engine: plan.engine === 'large-v3-legacy' ? 'large-v3-legacy' : 'large-v3',
+      text: reused.text,
+      words: reused.words,
+      ...(ts && reused.spokenWords ? { spoken: reused.spokenWords } : {}),
+      ...(ts && reused.speechSpans ? { speechSpans: reused.speechSpans } : {}),
+      timestamps: ts,
+    }
+  } else if (plan.engine === 'parakeet-ultra') {
+    try {
+      primary = await parakeetGuarded(job, () => decodeParakeetWindow(deps, input, parakeetTimeout(job), 'parakeet-ultra', true))
+    } catch (error) {
+      fallbackReason = `parakeet_failed: ${errorReason(error)}`
+      console.warn(`[meeting-batch] Parakeet HQ failed for ${segment.startChunkIdx}-${segment.endChunkIdx}; using Large-v3: ${errorReason(error)}`)
+      const w = await decodeWhisperWindow(deps, input, previousText, 'large-v3', segment, audioDir)
+      primary = w.decode
+      if (w.fallbackReason) fallbackReason += `; ${w.fallbackReason}`
+    }
+  } else {
+    try {
+      const w = await decodeWhisperWindow(deps, input, previousText, plan.engine, segment, audioDir)
+      primary = w.decode
+      fallbackReason = w.fallbackReason
+    } catch (error) {
+      if (!secondModel || job.parakeetStopped) throw error
+      // Whisper failed both ways: the planned Parakeet second opinion keeps the window's text.
+      primary = await parakeetGuarded(job, () => decodeParakeetWindow(deps, input, parakeetTimeout(job), secondModel, true))
+      fallbackReason = `whisper_failed: ${errorReason(error)}`
+      usedSecondAsPrimary = true
+      console.warn(`[meeting-batch] Whisper failed for ${segment.startChunkIdx}-${segment.endChunkIdx}; Parakeet (${secondModel}) is this window's primary`)
+    }
+  }
+
+  let second: WholeDecode | null = null
+  let secondError: string | undefined
+  const secondEngine = plan.secondOpinion
+  const sameFamily = secondEngine !== 'off' && isParakeet(secondEngine) === isParakeet(primary.engine)
+  if (secondEngine !== 'off' && !sameFamily && !usedSecondAsPrimary) {
+    try {
+      if (secondEngine === 'large-v3') {
+        second = (await decodeWhisperWindow(deps, input, previousText, 'large-v3', segment, audioDir)).decode
+      } else {
+        second = await parakeetGuarded(job, () => decodeParakeetWindow(deps, input, parakeetTimeout(job), secondEngine, false))
+      }
+    } catch (error) {
+      secondError = errorReason(error)
+      console.warn(`[meeting-batch] Second opinion (${secondEngine}) failed for ${segment.startChunkIdx}-${segment.endChunkIdx}; keeping the primary: ${secondError}`)
+    }
+  }
+
+  let text = primary.text
+  let words = primary.words
+  let applied = false
+  let wordsFromSecond = 0
+  let skipped: string | undefined
+  let disagreements: SegmentStt['disagreements'] = []
+  if (second) {
+    // The resolver may change words only when the primary's clocks are real
+    // (timestamps on, or Parakeet); otherwise it only records disagreements.
+    const apply = plan.resolver && primary.spoken !== undefined
+    const resolved = resolveHypotheses(asHypothesis(primary), asHypothesis(second), {
+      vocabulary: deps.vocabulary(),
+      primaryDeletionProne: isDeletionProneEngine(primary.engine),
+      speechSpans: primary.speechSpans ?? null,
+      apply,
+    })
+    skipped = resolved.skipped
+    disagreements = resolved.disagreements
+    if (apply && resolved.wordsFromSecond > 0 && resolved.timed) {
+      applied = true
+      wordsFromSecond = resolved.wordsFromSecond
+      // Spoken words are uncorrected: join, then correct the text exactly once.
+      text = applyCorrections(resolved.text)
+      words = resolved.words.map(w => ({ word: applyCorrections(w.word), start: w.start, end: w.end, probability: w.probability ?? 0 }))
+    } else {
+      // What is saved must say what was applied.
+      disagreements = disagreements.map(d => ({ ...d, taken: 'primary' as const }))
+    }
+  }
+  // Only a prompted Whisper decode can echo the previous window's text.
+  if (previousText && !isParakeet(primary.engine)) text = stripOverlap(text, previousText)
+
+  const stt: SegmentStt = {
+    engine: primary.engine,
+    requestedEngine: plan.engine,
+    ...(fallbackReason ? { fallbackReason } : {}),
+    ...(reused ? { reusedCheckpoint: true } : {}),
+    frontEnd: plan.frontEnd,
+    timestamps: primary.timestamps,
+    primary: hypothesisRecord(primary),
+    ...(second ? { secondOpinion: hypothesisRecord(second) } : secondError ? { secondOpinion: { engine: secondEngine, text: '', error: secondError } } : {}),
+    resolver: { version: RESOLVER_VERSION, enabled: plan.resolver, applied, wordsFromSecond, ...(skipped ? { skipped } : {}) },
+    disagreements,
+  }
+  return {
+    segment, text, words, speakerWords: mapWordsToSpeakers(words, segment, entries),
+    ...(primary.spoken ? { spokenWords: primary.spoken } : {}),
+    ...(primary.speechSpans ? { speechSpans: primary.speechSpans } : {}),
+    stt,
+  }
+}
+
 export async function transcribeSegments(
   audioDir: string,
   segments: BatchSegment[],
   entries: IndexedTranscriptChunk[],
-  expectedSessionId?: string,
+  expectedSessionId: string | undefined,
+  plan: HqJobPlan,
+  deps: HqDecodeDeps = defaultHqDeps,
 ): Promise<BatchResult[]> {
   const results: BatchResult[] = []
   const meetingId = basename(audioDir)
+  const job: JobState = { parakeetRan: false, parakeetFailures: 0, parakeetStopped: false }
+  // A progressive checkpoint holds one Whisper decode made under the same
+  // timing mode and front end. It is reused as the primary for a Whisper plan;
+  // with a second opinion planned, only the second opinion is decoded.
+  const identity: CheckpointIdentity = { timestamps: batchTimestampsEnabled(plan.engine === 'parakeet-ultra' ? 'large-v3' : plan.engine), frontEnd: plan.frontEnd }
+  const checkpointsUsable = plan.engine !== 'parakeet-ultra'
   writeMeetingBatchProgress(audioDir, {
     phase: 'hq_polish',
     segmentsDone: 0,
@@ -410,11 +758,12 @@ export async function transcribeSegments(
     try {
       refreshPendingLease(audioDir)
       const previousText = results.at(-1)?.text
-      const cached = progressiveMeetingHqEnabled()
-        ? readProgressiveCheckpoint(audioDir, segment, previousText, expectedSessionId)
+      const cached = checkpointsUsable && progressiveMeetingHqEnabled()
+        ? readProgressiveCheckpoint(audioDir, segment, previousText, expectedSessionId, identity)
         : null
-      if (cached) {
-        results.push(cached)
+      if (cached && plan.secondOpinion === 'off') {
+        results.push({ ...cached, stt: { engine: plan.engine, requestedEngine: plan.engine, reusedCheckpoint: true, frontEnd: plan.frontEnd, timestamps: identity.timestamps,
+          primary: { engine: plan.engine, text: cached.text }, resolver: { version: RESOLVER_VERSION, enabled: plan.resolver, applied: false, wordsFromSecond: 0 }, disagreements: [] } })
         writeMeetingBatchProgress(audioDir, {
           phase: 'hq_polish',
           segmentsDone: results.length,
@@ -424,36 +773,8 @@ export async function transcribeSegments(
         continue
       }
       const combined = concatenateWavChunks(audioDir, segment.startChunkIdx, segment.endChunkIdx)
-      const enhanced = await enhanceAudio(combined)
-      let result
-      try {
-        result = await transcribeHighQuality(enhanced, previousText?.slice(-250), { priority: 'batch' })
-      } catch (error) {
-        // A live meeting took the GPU mid-segment. The truncated Metal output
-        // was already discarded upstream (BLOCKER contract) — it is never
-        // accepted. Retry this SAME segment once on CPU, which cannot itself be
-        // preempted, so a busy day degrades to slow rather than to a silently
-        // missing stretch of transcript.
-        if (!isMetalBatchPreempted(error)) throw error
-        console.log(
-          `[meeting-batch] Segment ${segment.startChunkIdx}-${segment.endChunkIdx} preempted off Metal; `
-          + 'retrying once on CPU',
-        )
-        refreshPendingLease(audioDir)
-        result = await transcribeHighQuality(enhanced, previousText?.slice(-250), {
-          priority: 'batch',
-          forceCpu: true,
-          forceCpuReason: 'preempt_retry',
-        })
-      }
-      const text = previousText ? stripOverlap(result.text, previousText) : result.text
-      const words = result.words ?? []
-      results.push({
-        segment,
-        text,
-        words,
-        speakerWords: mapWordsToSpeakers(words, segment, entries),
-      })
+      const input = await deps.frontEnd(combined, plan.frontEnd)
+      results.push(await decodeHqWindow(input, previousText, segment, entries, plan, deps, job, audioDir, cached ?? undefined))
       refreshPendingLease(audioDir)
       writeMeetingBatchProgress(audioDir, {
         phase: 'hq_polish',
@@ -477,6 +798,35 @@ export async function transcribeSegments(
   return results
 }
 
+/** Orphan recovery's HQ work: the plan is taken when the recovery is accepted
+ *  (like a save), and the same plan both decodes and summarizes. */
+export function recoveryHqJob(
+  quarantineDir: string,
+  segments: BatchSegment[],
+  entries: IndexedTranscriptChunk[],
+  deps: HqDecodeDeps = defaultHqDeps,
+): { plan: HqJobPlan; run: () => Promise<BatchResult[]> } {
+  const plan = currentHqJobPlan()
+  return { plan, run: () => transcribeSegments(quarantineDir, segments, entries, undefined, plan, deps) }
+}
+
+export function summarizeStt(plan: HqJobPlan, results: readonly BatchResult[], segmentsPlanned = results.length): BatchSttSummary {
+  return {
+    plan: { engine: plan.engine, secondOpinion: plan.secondOpinion, resolver: plan.resolver, frontEnd: plan.frontEnd,
+      timestamps: batchTimestampsEnabled(plan.engine === 'parakeet-ultra' ? 'large-v3' : plan.engine) },
+    resolverVersion: RESOLVER_VERSION,
+    windows: results.length,
+    windowsLost: Math.max(0, segmentsPlanned - results.length),
+    disagreements: results.reduce((n, r) => n + (r.stt?.disagreements.length ?? 0), 0),
+    wordsFromSecond: results.reduce((n, r) => n + (r.stt?.resolver.wordsFromSecond ?? 0), 0),
+    secondOpinionFailures: results.filter(r => r.stt?.secondOpinion?.error).length,
+    primaryFallbacks: results.filter(r => r.stt?.fallbackReason).length,
+    timestampFallbacks: results.filter(r => r.stt?.fallbackReason?.includes('timestamps_failed')).length,
+    checkpointsReused: results.filter(r => r.stt?.reusedCheckpoint).length,
+    resolverSkipped: results.filter(r => r.stt?.resolver.skipped).length,
+  }
+}
+
 let batchQueueTail: Promise<void> = Promise.resolve()
 
 /** Chain arbitrary HQ-decoder work onto the same serialization tail the batch
@@ -650,7 +1000,8 @@ function queueProgressiveWork(sessionId: string, state: ProgressiveSessionState)
         pending.segment.startChunkIdx,
         pending.segment.endChunkIdx,
       )
-      const enhanced = await enhanceAudio(combined)
+      const identity = currentCheckpointIdentity()
+      const enhanced = await asrFrontEnd(combined, identity.frontEnd)
       if (controller.signal.aborted || state.closed) return
       const decoded = await transcribeHighQuality(enhanced, pending.previousText?.slice(-250), {
         priority: 'batch',
@@ -659,6 +1010,7 @@ function queueProgressiveWork(sessionId: string, state: ProgressiveSessionState)
         threads: progressiveThreads(),
         backgroundCpu: true,
         signal: controller.signal,
+        timestamps: identity.timestamps,
       })
       if (controller.signal.aborted || state.closed) return
       const text = pending.previousText
@@ -670,12 +1022,15 @@ function queueProgressiveWork(sessionId: string, state: ProgressiveSessionState)
         text,
         words,
         speakerWords: mapWordsToSpeakers(words, pending.segment, state.entries),
+        // Kept so the save-time job can reuse this decode as a resolver primary.
+        ...(decoded.timestamps && decoded.spokenWords ? { spokenWords: decoded.spokenWords } : {}),
+        ...(decoded.timestamps && decoded.speechSpans ? { speechSpans: decoded.speechSpans } : {}),
       }
       writeProgressiveCheckpoint(state.audioDir, sessionId, {
         key: checkpointKey(pending.segment),
         sourceHash,
         contextHash: contextHash(pending.previousText),
-        configFingerprint: getHighQualityCheckpointFingerprint(),
+        configFingerprint: getHighQualityCheckpointFingerprint(identity),
         completedAt: new Date().toISOString(),
         wallTimeMs: Date.now() - started,
         result,
@@ -846,11 +1201,15 @@ export function runMeetingBatchPipeline(
   })
   const lease = setInterval(() => refreshPendingLease(audioDir), 60_000)
   lease.unref()
+  // Frozen at SAVE time: a settings change applies to the next saved meeting,
+  // never to one already queued behind another HQ job.
+  const plan = currentHqJobPlan()
   const job = batchQueueTail.then(() => runMeetingBatchPipelineNow(
     audioDir,
     entries,
     streamingWordCount,
     sessionId,
+    plan,
   )).finally(() => {
     clearInterval(lease)
     clearMeetingBatchProgress(audioDir)
@@ -863,7 +1222,8 @@ async function runMeetingBatchPipelineNow(
   audioDir: string,
   entries: IndexedTranscriptChunk[],
   streamingWordCount: number,
-  sessionId?: string,
+  sessionId: string | undefined,
+  plan: HqJobPlan,
 ): Promise<BatchTranscription> {
   try {
     refreshPendingLease(audioDir)
@@ -885,7 +1245,8 @@ async function runMeetingBatchPipelineNow(
       segmentsTotal: segments.length,
       meetingId: basename(audioDir),
     })
-    const batchSegments = await transcribeSegments(audioDir, segments, entries, sessionId)
+    console.log(`[meeting-batch] STT plan: ${JSON.stringify(plan)}`)
+    const batchSegments = await transcribeSegments(audioDir, segments, entries, sessionId, plan)
     writeMeetingBatchProgress(audioDir, {
       phase: 'quality_check',
       segmentsDone: segments.length,
@@ -894,6 +1255,8 @@ async function runMeetingBatchPipelineNow(
     })
     const batchTranscript = batchSegments.map(result => result.text).join(' ')
     const qualityReport = evaluateBatchQuality(batchSegments, streamingWordCount)
+    const sttSummary = summarizeStt(plan, batchSegments, segments.length)
+    recordLastSttJob({ at: new Date().toISOString(), meeting: basename(audioDir), outcome: qualityReport.accepted ? 'batch' : 'rejected', summary: sttSummary })
     if (!qualityReport.accepted) {
       console.warn(
         `[meeting-batch] Candidate rejected (${qualityReport.reason}): `
@@ -908,7 +1271,7 @@ async function runMeetingBatchPipelineNow(
       return { transcriptionQuality: 'streaming', qualityReport }
     }
 
-    return { transcriptionQuality: 'batch', batchTranscript, batchSegments, qualityReport }
+    return { transcriptionQuality: 'batch', batchTranscript, batchSegments, qualityReport, stt: sttSummary }
   } catch (error) {
     console.error(`[meeting-batch] Pipeline failed: ${error instanceof Error ? error.message : String(error)}`)
     writeMeetingBatchTerminal(audioDir, {

package/server/lib/parakeet-manifest.ts

@@ -0,0 +1,41 @@
+// GENERATED from the 2026-10-10 bake-off model manifest (private corpus MODEL_MANIFEST.json).
+// Every file of each pinned Hugging Face revision, with its size and SHA-256. The installer
+// refuses any byte that does not match. Do not edit by hand: a changed hash is a new model.
+
+export interface PinnedFile { path: string; size: number; sha256: string }
+export interface PinnedModel { id: string; repo: string; revision: string; modelVersion: string; files: PinnedFile[] }
+
+/** fluidaudiocli 0.17.4, the same binary the Nemotron diarizer pins (nemotron-live.ts KNOWN_CLI_SHA256). */
+export const PARAKEET_CLI = { version: '0.17.4', sha256: '4714154623b59c40404563828cb0fbeb4da6878e90f582ad776c1c2cd061d35a', size: 22039856 } as const
+
+export const PARAKEET_ULTRA: PinnedModel = {
+  id: 'parakeet-ultra', repo: 'FluidInference/parakeet-ultra-coreml', revision: '95eaa59a39d4394f047a4dc5cce480388a60d1b6', modelVersion: 'ultra',
+  files: [
+    { path: 'Decoder.mlmodelc/analytics/coremldata.bin', size: 243, sha256: 'fe92b6cfaa012abd5248c0bc877832f19807015abffc60d87b8ccc8ccb48b3b5' },
+    { path: 'Decoder.mlmodelc/coremldata.bin', size: 560, sha256: '3b06e66768f0df7e21795f50e2b29300e33eeb1a2579dc42c695279c2d308497' },
+    { path: 'Decoder.mlmodelc/model.mil', size: 13110, sha256: '956f600207f88396017ca5c96cfa3acd5bfee60835a5b44b762e033a8fb28955' },
+    { path: 'Decoder.mlmodelc/weights/weight.bin', size: 23604992, sha256: '02a0d219f281b9665bc10c8768649403b2eebbbf4b44c627041d948e0de11bb4' },
+    { path: 'Encoder.mlmodelc/analytics/coremldata.bin', size: 243, sha256: 'd87101d824d6723cf95304da33755c3c60e762663b9ef2b0c4bd0aa166a09a0d' },
+    { path: 'Encoder.mlmodelc/coremldata.bin', size: 514, sha256: '397a84a4062f563cbc5f56077c674f09a61d85be5090f61d2f1932afb92ac0fe' },
+    { path: 'Encoder.mlmodelc/model.mil', size: 1002653, sha256: 'f5d601568a4171d99a314c0fe3f6bc67715da2623732a3fb566e050ea83e5848' },
+    { path: 'Encoder.mlmodelc/weights/weight.bin', size: 594211328, sha256: '315ba01f33cadbf601d43ac7f5c86208b7aa75fdaa34705c9869d5abe3521c9b' },
+    { path: 'JointDecisionv3.mlmodelc/analytics/coremldata.bin', size: 243, sha256: '68d38ca646aebafa7a9329e2efda50f5767c49e89fdb5f77f212072bb66f97c4' },
+    { path: 'JointDecisionv3.mlmodelc/coremldata.bin', size: 592, sha256: '5e3af5a4ce686f6c237cadbd9284e10d333bc0e1546633cd0e430e6194044bc4' },
+    { path: 'JointDecisionv3.mlmodelc/model.mil', size: 11777, sha256: '791b3c3cf3eb2079c84623fc880f6bba008d1366e5f8b03e9a8ed8bd4d7194a0' },
+    { path: 'JointDecisionv3.mlmodelc/weights/weight.bin', size: 12642764, sha256: '3f310b85b82341c53ec383025ab094a4e462ee1c592e1ad7c6bfe39cff66ca25' },
+    { path: 'Preprocessor.mlmodelc/analytics/coremldata.bin', size: 243, sha256: 'c9beeb989c8d66f8be11df59bc6df277ec76cee404f6865b46243835ef562f6d' },
+    { path: 'Preprocessor.mlmodelc/coremldata.bin', size: 486, sha256: 'dbde3f2300842c1fd51ef3ff948a0bcffe65ffd2dca10707f2509f32c1d65b1d' },
+    { path: 'Preprocessor.mlmodelc/metadata.json', size: 2841, sha256: '2a98699e22d279dd37fa1d238aeb1c6db1df0d6fad687775324157689d8f3acf' },
+    { path: 'Preprocessor.mlmodelc/model.mil', size: 28181, sha256: '4b8518a956450fec57f06c2a21bdffc26973f7f1fa6842fb38fe917f896b6b93' },
+    { path: 'Preprocessor.mlmodelc/weights/weight.bin', size: 491072, sha256: '129b76e3aeafa8afa3ea76d995b964b145fe83700d579f6ff42c4c38fa0968ea' },
+    { path: 'README.md', size: 2235, sha256: '9e9e26b6da628a0dc34b9c55f2029e486fc02304ccd7f83825333572ae11c158' },
+    { path: 'config.json', size: 414, sha256: '965c69bf500c7be18958df6438dc1ba897721690ae412937ca7c91e28cec1870' },
+    { path: 'parakeet_v3_vocab.json', size: 151122, sha256: '7ec60e05f1b24480736ec0eed40900f4626bce1fa9a60fd700ec7e2a59198735' },
+    { path: 'parakeet_vocab.json', size: 151122, sha256: '7ec60e05f1b24480736ec0eed40900f4626bce1fa9a60fd700ec7e2a59198735' },
+  ],
+}
+
+// Parakeet TDT v2 (FluidInference/parakeet-tdt-0.6b-v2-coreml@ee09c569, 464 MB) was
+// evaluated and removed in 6.72.0: on the shipped path it gave no benefit Ultra
+// cannot (docs/stt-672-evaluation.md), so it is never downloaded or installed.
+export const PARAKEET_MODELS: readonly PinnedModel[] = [PARAKEET_ULTRA]