@gotcos/glasses-server 6.71.14 → 6.73.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,6 +22,11 @@ export interface PromptDraftTranscriptRecord {
22
22
  degraded: boolean
23
23
  degradationReason?: string
24
24
  acceptedDegraded?: boolean
25
+ /** 6.73.0: the setting this decode followed ("fast:parakeet", "final:large-v3").
26
+ * Absent on older records, which were all made under the 6.72.0 defaults. */
27
+ sttKey?: string
28
+ engine?: 'whisper' | 'parakeet-ultra'
29
+ engineReason?: string
25
30
  }
26
31
 
27
32
  /** Higher wins. Fast warm must never replace an HQ/cloud decode for the same audio. */
@@ -0,0 +1,60 @@
1
+ // The live chunk's quiet gate and ASR front end (6.72.0). Kept apart from the
2
+ // decoders so the live route imports only audio code.
3
+
4
+ import { enhanceAudio } from './audio-enhance.js'
5
+ import { isSileroAvailable, measureSpeechSeconds } from './vad-silero.js'
6
+ import type { FrontEndId, QuietGateId } from './stt-settings.js'
7
+
8
+ /** Under this many seconds of Silero speech a chunk is quiet (bake-off canary 5). */
9
+ export const QUIET_SPEECH_SEC = 0.5
10
+ export const QUIET_RMS = 150
11
+
12
+ export interface QuietDecision {
13
+ isQuiet: boolean
14
+ rms: number
15
+ /** Silero speech seconds; null when the gate fell back to RMS. */
16
+ speechSec: number | null
17
+ gate: 'silero' | 'rms' | 'rms_fallback'
18
+ }
19
+
20
+ export function chunkRms(wav: Buffer): number {
21
+ const pcm = wav.subarray(44)
22
+ const n = Math.floor(pcm.length / 2)
23
+ let sumSq = 0
24
+ for (let i = 0; i < n; i++) { const s = pcm.readInt16LE(i * 2); sumSq += s * s }
25
+ return Math.sqrt(sumSq / Math.max(1, n))
26
+ }
27
+
28
+ /**
29
+ * Quiet means "do not prompt, do not enhance, drop silence echoes". RMS 150
30
+ * fired on 1% of G2 chunks while Silero found no speech in 9.3%, so 6.72.0
31
+ * gates on Silero speech seconds (the in-process sherpa Silero, about 17 ms a
32
+ * 6 s chunk). Silero unavailable or the clip too short: the old RMS gate.
33
+ */
34
+ export function decideQuiet(wav: Buffer, gate: QuietGateId, vad: { available: () => boolean; speechSeconds: (wav: Buffer) => number | null } = { available: () => isSileroAvailable(), speechSeconds: w => measureSpeechSeconds(w) }): QuietDecision {
35
+ const rms = chunkRms(wav)
36
+ if (gate === 'silero' && vad.available()) {
37
+ const speechSec = vad.speechSeconds(wav)
38
+ if (speechSec !== null && Number.isFinite(speechSec)) {
39
+ return { isQuiet: speechSec < QUIET_SPEECH_SEC, rms, speechSec, gate: 'silero' }
40
+ }
41
+ return { isQuiet: rms < QUIET_RMS, rms, speechSec: null, gate: 'rms_fallback' }
42
+ }
43
+ return { isQuiet: rms < QUIET_RMS, rms, speechSec: null, gate: 'rms' }
44
+ }
45
+
46
+ /**
47
+ * Whether the silence-hallucination TEXT drops (URL-only, repeated thank-you,
48
+ * quiet vocabulary echo) may fire. They stay on the pre-6.72 RMS test: the
49
+ * Silero gate also calls a chunk holding one short real name quiet, and those
50
+ * drops would delete it.
51
+ */
52
+ export function silenceDropsAllowed(decision: QuietDecision): boolean {
53
+ return decision.rms < QUIET_RMS
54
+ }
55
+
56
+ /** ASR input for a live chunk or an HQ window. Playback audio is never touched here. */
57
+ export function asrFrontEnd(audio: Buffer, frontEnd: FrontEndId): Promise<Buffer> {
58
+ return frontEnd === 'full' ? enhanceAudio(audio) : enhanceAudio(audio, { profile: 'light' })
59
+ }
60
+
@@ -0,0 +1,327 @@
1
+ // Second-opinion resolver for meeting HQ (6.72.0). Algorithmic only: no model
2
+ // call, no network, no LLM. Transcription must never call `claude -p`.
3
+ //
4
+ // Two decoders hear the same window. Their words are aligned by edit distance
5
+ // over normalized words (an optional time-skew limit exists but is off: it
6
+ // measured worse), and every place they disagree is recorded. The resolver
7
+ // then applies two measured rules, and nothing else:
8
+ //
9
+ // 1. Gap fill. Large-v3 is deletion-prone on G2 audio (bake-off silver
10
+ // deletions 6.1% for the fixed Large-v3, 6.6% on the shipped path, against
11
+ // 2.1% for Parakeet Ultra). Where the primary is a Whisper engine and the
12
+ // second opinion heard words the primary has nothing for, those words are
13
+ // taken, but only inside a span the primary's own VAD marked as speech and
14
+ // only when the second opinion's mean word confidence is at least 0.5.
15
+ // 2. Vocabulary rescue. Where the second opinion's side of a short
16
+ // disagreement (at most 4 words, and the primary's side at most 2 longer)
17
+ // holds a profile vocabulary term the primary's side does not, the second
18
+ // opinion's words are taken.
19
+ //
20
+ // Everything else stays as the primary heard it. No region whose two sides
21
+ // differ in a negation, a number or a unit is ever changed, whatever its length
22
+ // (the 2026-10-13 diagnostic saw "born lacking" become "born having" and
23
+ // "18 months" become "18 minutes"); it is listed as a protected disagreement.
24
+
25
+ export interface HypWord {
26
+ word: string
27
+ start: number
28
+ end: number
29
+ probability?: number
30
+ }
31
+
32
+ export interface Hypothesis {
33
+ engine: string
34
+ text: string
35
+ /** Spoken words (not sub-word tokens), seconds from the start of the window. */
36
+ words?: HypWord[]
37
+ }
38
+
39
+ export type DisagreementKind = 'substitution' | 'primary_only' | 'second_only'
40
+ export type ResolutionReason = 'gap_fill' | 'vocabulary' | 'protected' | 'not_speech' | 'low_confidence' | 'too_wide' | null
41
+
42
+ export interface Disagreement {
43
+ kind: DisagreementKind
44
+ primary: string
45
+ second: string
46
+ start: number | null
47
+ end: number | null
48
+ taken: 'primary' | 'second'
49
+ reason: ResolutionReason
50
+ }
51
+
52
+ export interface ResolveOptions {
53
+ vocabulary: readonly string[]
54
+ /** Gap fill only for a deletion-prone primary (the Whisper engines). */
55
+ primaryDeletionProne: boolean
56
+ /** Where the primary's own VAD heard speech, seconds on the window clock.
57
+ * Gap fill needs it: a run outside every span is background the primary's VAD
58
+ * rejected, and is not taken. Absent: no gap fill at all. */
59
+ speechSpans?: ReadonlyArray<readonly [number, number]> | null
60
+ /** Mean word confidence a gap-fill run needs (second opinion's own scores). */
61
+ minGapConfidence?: number
62
+ /** false: report disagreements, change nothing. */
63
+ apply: boolean
64
+ /** When set, two words more than this far apart in time never align. Off by
65
+ * default: on the production-path silver decode every limit tried (2, 3, 5,
66
+ * 10 s) made the resolver worse than plain text alignment, because Large-v3's
67
+ * remapped token clocks have a heavy tail (median skew 0.23 s, p90 1.0 s). */
68
+ maxSkewSec?: number | null
69
+ }
70
+
71
+ export interface ResolveResult {
72
+ text: string
73
+ words: HypWord[]
74
+ /** True when the resolver's words carry clocks for every word. */
75
+ timed: boolean
76
+ disagreements: Disagreement[]
77
+ wordsFromSecond: number
78
+ applied: boolean
79
+ skipped?: 'too_long'
80
+ }
81
+
82
+ export const RESOLVER_VERSION = 'stt-resolver-v1'
83
+ /** Alignment is quadratic; a 30 s window is about 100 words a side. */
84
+ export const MAX_ALIGNMENT_CELLS = 2_000_000
85
+ export const DEFAULT_MAX_SKEW_SEC: number | null = null
86
+ /** Agreed words each side of a region that a vocabulary match may span. */
87
+ export const CONTEXT_WORDS = 3
88
+ /** A vocabulary rescue replaces at most this many second-opinion words... */
89
+ export const MAX_VOCAB_REGION_WORDS = 4
90
+ /** ...and at most this many more primary words than it inserts. */
91
+ export const MAX_VOCAB_REGION_SLACK = 2
92
+ export const DEFAULT_MIN_GAP_CONFIDENCE = 0.5
93
+ /** Share of a gap-fill run's words whose midpoint must lie in a speech span. */
94
+ export const MIN_SPEECH_SHARE = 0.5
95
+
96
+ const NEGATIONS = new Set([
97
+ 'not', 'no', 'never', 'none', 'nor', 'nothing', 'nobody', 'neither', 'without', 'cannot',
98
+ 'dont', 'doesnt', 'didnt', 'isnt', 'arent', 'wasnt', 'werent', 'wont', 'wouldnt', 'cant',
99
+ 'couldnt', 'shouldnt', 'havent', 'hasnt', 'hadnt', 'aint', 'lacking', 'lack', 'lacks',
100
+ ])
101
+ // Unambiguous units and number words only. "am", "one", "second" and "first"
102
+ // are everyday words far more often than quantities, and protecting them would
103
+ // block most real gap fills.
104
+ const UNITS = new Set([
105
+ 'seconds', 'minute', 'minutes', 'hour', 'hours', 'day', 'days', 'week', 'weeks',
106
+ 'month', 'months', 'year', 'years', 'percent', 'dollar', 'dollars', 'cents',
107
+ 'thousand', 'million', 'billion',
108
+ ])
109
+ const NUMBER_WORDS = new Set([
110
+ 'zero', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',
111
+ 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen',
112
+ 'nineteen', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety',
113
+ 'hundred', 'half', 'quarter', 'twice',
114
+ ])
115
+
116
+
117
+ /** Lowercase letters and digits only. "Don't," and "dont" are the same word. */
118
+ export function wordKey(word: string): string {
119
+ return word.normalize('NFKD').replace(/\p{M}+/gu, '').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
120
+ }
121
+
122
+ export function isProtectedKey(key: string): boolean {
123
+ return NEGATIONS.has(key) || UNITS.has(key) || NUMBER_WORDS.has(key) || /\d/.test(key)
124
+ }
125
+
126
+ interface Token { display: string; key: string; start: number | null; end: number | null; probability?: number }
127
+
128
+ function tokens(h: Hypothesis): Token[] {
129
+ const timedWords = Array.isArray(h.words) && h.words.length > 0 ? h.words : null
130
+ const source = timedWords
131
+ ? timedWords.map(w => ({
132
+ display: String(w.word ?? '').trim(),
133
+ start: Number.isFinite(w.start) ? w.start : null,
134
+ end: Number.isFinite(w.end) ? w.end : null,
135
+ probability: w.probability,
136
+ }))
137
+ : String(h.text ?? '').split(/\s+/).map(display => ({ display, start: null, end: null, probability: undefined }))
138
+ return source
139
+ .filter(t => t.display.length > 0)
140
+ .map(t => ({ ...t, key: wordKey(t.display) }))
141
+ .filter(t => t.key.length > 0)
142
+ }
143
+
144
+ function vocabSequences(vocabulary: readonly string[]): string[][] {
145
+ const out: string[][] = []
146
+ const seen = new Set<string>()
147
+ for (const term of vocabulary) {
148
+ const seq = String(term).split(/\s+/).map(wordKey).filter(Boolean)
149
+ if (!seq.length) continue
150
+ const id = seq.join(' ')
151
+ if (seen.has(id)) continue
152
+ seen.add(id)
153
+ out.push(seq)
154
+ // "ThriftCart" is also heard as "thrift cart": both spellings count.
155
+ const joined = seq.join('')
156
+ if (seq.length > 1 && !seen.has(joined)) { seen.add(joined); out.push([joined]) }
157
+ }
158
+ return out
159
+ }
160
+
161
+ function termsIn(keys: string[], sequences: string[][]): Set<string> {
162
+ const found = new Set<string>()
163
+ for (const seq of sequences) {
164
+ for (let i = 0; i + seq.length <= keys.length; i++) {
165
+ let hit = true
166
+ for (let k = 0; k < seq.length; k++) if (keys[i + k] !== seq[k]) { hit = false; break }
167
+ if (hit) { found.add(seq.join('')); break }
168
+ }
169
+ }
170
+ return found
171
+ }
172
+
173
+ function mid(t: Token): number | null {
174
+ if (t.start === null) return null
175
+ return t.end === null ? t.start : (t.start + t.end) / 2
176
+ }
177
+
178
+ type Op = { kind: '=' | 's' | 'p' | 'q'; i: number | null; j: number | null }
179
+
180
+ /** Edit-distance alignment. Two words far apart in time cannot pair. */
181
+ function align(a: Token[], b: Token[], maxSkew: number | null): Op[] {
182
+ const n = a.length, m = b.length
183
+ const INF = 1e9
184
+ const D = new Float64Array((n + 1) * (m + 1))
185
+ const W = m + 1
186
+ for (let i = 0; i <= n; i++) D[i * W] = i
187
+ for (let j = 0; j <= m; j++) D[j] = j
188
+ const pairCost = (i: number, j: number): number => {
189
+ const ta = a[i], tb = b[j]
190
+ const ma = mid(ta), mb = mid(tb)
191
+ if (maxSkew !== null && ma !== null && mb !== null && Math.abs(ma - mb) > maxSkew) return INF
192
+ return ta.key === tb.key ? 0 : 1
193
+ }
194
+ for (let i = 1; i <= n; i++) {
195
+ for (let j = 1; j <= m; j++) {
196
+ const diag = D[(i - 1) * W + j - 1] + pairCost(i - 1, j - 1)
197
+ const up = D[(i - 1) * W + j] + 1
198
+ const left = D[i * W + j - 1] + 1
199
+ D[i * W + j] = Math.min(diag, up, left)
200
+ }
201
+ }
202
+ const ops: Op[] = []
203
+ let i = n, j = m
204
+ while (i > 0 || j > 0) {
205
+ if (i > 0 && j > 0) {
206
+ const c = pairCost(i - 1, j - 1)
207
+ if (c < INF && D[i * W + j] === D[(i - 1) * W + j - 1] + c) {
208
+ ops.push({ kind: c === 0 ? '=' : 's', i: i - 1, j: j - 1 }); i--; j--; continue
209
+ }
210
+ }
211
+ if (i > 0 && D[i * W + j] === D[(i - 1) * W + j] + 1) { ops.push({ kind: 'p', i: i - 1, j: null }); i--; continue }
212
+ ops.push({ kind: 'q', i: null, j: j - 1 }); j--
213
+ }
214
+ return ops.reverse()
215
+ }
216
+
217
+ function span(ts: Token[]): { start: number | null; end: number | null } {
218
+ const starts = ts.map(t => t.start).filter((x): x is number => x !== null)
219
+ const ends = ts.map(t => t.end ?? t.start).filter((x): x is number => x !== null)
220
+ return { start: starts.length ? Math.min(...starts) : null, end: ends.length ? Math.max(...ends) : null }
221
+ }
222
+
223
+ function sameProtected(p: Token[], q: Token[]): boolean {
224
+ const pk = p.map(t => t.key).filter(isProtectedKey).join(' ')
225
+ const qk = q.map(t => t.key).filter(isProtectedKey).join(' ')
226
+ return pk === qk
227
+ }
228
+
229
+ /** A second-opinion-only run is taken only where the primary's VAD heard
230
+ * speech and the second opinion was confident. */
231
+ function gapVerdict(q: Token[], options: ResolveOptions): ResolutionReason {
232
+ const spans = options.speechSpans
233
+ if (!spans || !spans.length) return 'not_speech'
234
+ const timed = q.filter(t => t.start !== null)
235
+ if (!timed.length) return 'not_speech'
236
+ const inside = timed.filter(t => { const m = mid(t)!; return spans.some(([a, b]) => m >= a && m <= b) }).length
237
+ if (inside / timed.length < MIN_SPEECH_SHARE) return 'not_speech'
238
+ const scored = q.filter(t => typeof t.probability === 'number' && Number.isFinite(t.probability))
239
+ const floor = options.minGapConfidence ?? DEFAULT_MIN_GAP_CONFIDENCE
240
+ if (scored.length && scored.reduce((n, t) => n + (t.probability as number), 0) / scored.length < floor) return 'low_confidence'
241
+ return 'gap_fill'
242
+ }
243
+
244
+ export function resolveHypotheses(primary: Hypothesis, second: Hypothesis, options: ResolveOptions): ResolveResult {
245
+ const P = tokens(primary), Q = tokens(second)
246
+ const toWords = (ts: Token[]): HypWord[] => ts.map(t => ({
247
+ word: t.display,
248
+ start: t.start ?? 0,
249
+ end: Math.max(t.start ?? 0, t.end ?? t.start ?? 0),
250
+ ...(t.probability !== undefined ? { probability: t.probability } : {}),
251
+ }))
252
+ const primaryOnly = (): ResolveResult => ({
253
+ text: P.map(t => t.display).join(' '),
254
+ words: toWords(P),
255
+ timed: P.length > 0 && P.every(t => t.start !== null),
256
+ disagreements: [],
257
+ wordsFromSecond: 0,
258
+ applied: false,
259
+ })
260
+ if ((P.length + 1) * (Q.length + 1) > MAX_ALIGNMENT_CELLS) return { ...primaryOnly(), skipped: 'too_long' }
261
+
262
+ const sequences = vocabSequences(options.vocabulary)
263
+ const ops = align(P, Q, options.maxSkewSec ?? DEFAULT_MAX_SKEW_SEC)
264
+ const out: Token[] = []
265
+ const disagreements: Disagreement[] = []
266
+ let wordsFromSecond = 0
267
+ let k = 0
268
+ while (k < ops.length) {
269
+ if (ops[k].kind === '=') { out.push(P[ops[k].i!]); k++; continue }
270
+ const p: Token[] = [], q: Token[] = []
271
+ const regionStart = k
272
+ while (k < ops.length && ops[k].kind !== '=') {
273
+ if (ops[k].i !== null) p.push(P[ops[k].i!])
274
+ if (ops[k].j !== null) q.push(Q[ops[k].j!])
275
+ k++
276
+ }
277
+ // Agreed words on both sides, so a term split across the edge of the
278
+ // region ("drift cart" vs "thrift cart") is still seen whole.
279
+ const before: string[] = [], after: string[] = []
280
+ for (let b = regionStart - 1; b >= 0 && before.length < CONTEXT_WORDS && ops[b].kind === '='; b--) before.unshift(P[ops[b].i!].key)
281
+ for (let a = k; a < ops.length && after.length < CONTEXT_WORDS && ops[a].kind === '='; a++) after.push(P[ops[a].i!].key)
282
+ const kind: DisagreementKind = !p.length ? 'second_only' : !q.length ? 'primary_only' : 'substitution'
283
+ let take: 'primary' | 'second' = 'primary'
284
+ let reason: ResolutionReason = null
285
+ if (q.length) {
286
+ const withContext = (side: Token[]) => [...before, ...side.map(t => t.key), ...after]
287
+ const primaryTerms = termsIn(withContext(p), sequences)
288
+ const newTerms = [...termsIn(withContext(q), sequences)].filter(t => !primaryTerms.has(t))
289
+ const wantGap = kind === 'second_only' && options.primaryDeletionProne
290
+ const wantVocab = newTerms.length > 0
291
+ if (wantGap || wantVocab) {
292
+ if (!sameProtected(p, q)) reason = 'protected'
293
+ else if (wantVocab && (q.length > MAX_VOCAB_REGION_WORDS || p.length > q.length + MAX_VOCAB_REGION_SLACK)) {
294
+ // A term inside a long disagreement is not a reason to swap the whole region.
295
+ reason = wantGap ? gapVerdict(q, options) : 'too_wide'
296
+ if (reason === 'gap_fill') take = 'second'
297
+ } else if (wantVocab) { take = 'second'; reason = 'vocabulary' }
298
+ else { reason = gapVerdict(q, options); if (reason === 'gap_fill') take = 'second' }
299
+ }
300
+ }
301
+ const where = span([...p, ...q])
302
+ disagreements.push({
303
+ kind,
304
+ primary: p.map(t => t.display).join(' '),
305
+ second: q.map(t => t.display).join(' '),
306
+ start: where.start,
307
+ end: where.end,
308
+ taken: options.apply ? take : 'primary',
309
+ reason,
310
+ })
311
+ if (options.apply && take === 'second') { out.push(...q); wordsFromSecond += q.length }
312
+ else out.push(...p)
313
+ }
314
+ return {
315
+ text: out.map(t => t.display).join(' '),
316
+ words: toWords(out),
317
+ timed: out.length > 0 && out.every(t => t.start !== null),
318
+ disagreements,
319
+ wordsFromSecond,
320
+ applied: options.apply,
321
+ }
322
+ }
323
+
324
+ /** Whisper's deletion-prone profile is the reason gap fill exists. */
325
+ export function isDeletionProneEngine(engine: string): boolean {
326
+ return engine === 'large-v3' || engine === 'large-v3-legacy'
327
+ }
@@ -0,0 +1,233 @@
1
+ // Where the STT settings meet the decoders (6.72.0): what this Mac can run,
2
+ // the live quiet gate and ASR front end, and the meeting preview dispatch.
3
+
4
+ import { existsSync } from 'node:fs'
5
+ import {
6
+ applyCorrections,
7
+ getHighQualityTranscriptionCapability,
8
+ getWhisperCommitCapability,
9
+ WHISPER_TURBO_MODEL_PATH,
10
+ } from './whisper-local.js'
11
+ import { transcribeWhisperMeetingPreview } from './whisper-preview.js'
12
+ import { asrFrontEnd } from './stt-live.js'
13
+ import {
14
+ getInstallProgress, parakeetInstalled, parakeetInstallRunning, parakeetWarmState, pinnedCliKnown,
15
+ PARAKEET_VERSION_LABEL, transcribeParakeet, warmParakeet, type ParakeetModelId,
16
+ } from './parakeet.js'
17
+ import {
18
+ effectiveStt, readSttSettings, type EffectiveStt, type FrontEndId, type ParakeetStatus,
19
+ type SttCapabilities,
20
+ } from './stt-settings.js'
21
+
22
+ /** The live preview's Parakeet model. Ultra over v2 on the live-lane evidence
23
+ * (docs/stt-672-evaluation.md): lower WER, and v2 put text on 3 of 40 clean
24
+ * non-speech units where Ultra put none. */
25
+ export const PREVIEW_PARAKEET_MODEL: ParakeetModelId = 'parakeet-ultra'
26
+ /** The phone gives a preview 1.8 s end to end. Parakeet gets this long... */
27
+ export const PARAKEET_PREVIEW_TIMEOUT_MS = 900
28
+ /** ...and a miss falls back to Turbo only while this much of the budget is left unused. */
29
+ export const PREVIEW_TURBO_FALLBACK_BEFORE_MS = 900
30
+ /** Consecutive Parakeet preview misses (not busy lanes) that open the breaker. */
31
+ export const PREVIEW_BREAKER_MISSES = 3
32
+ export const PREVIEW_BREAKER_BASE_MS = 60_000
33
+ export const PREVIEW_BREAKER_MAX_MS = 10 * 60_000
34
+
35
+ // ── Recording probe (registered by the server; avoids an import cycle) ───────
36
+ let recordingLiveProbe: () => boolean = () => false
37
+ export function setRecordingLiveProbe(probe: () => boolean): void { recordingLiveProbe = probe }
38
+ function recordingLive(): boolean { try { return recordingLiveProbe() } catch { return true } }
39
+
40
+ // ── Preview breaker and counters ─────────────────────────────────────────────
41
+ const previewStats = { parakeet: 0, parakeetMisses: 0, turboFallbacks: 0, turbo: 0, nonTurboWithheld: 0, empty: 0 }
42
+ let breaker = { misses: 0, openUntil: 0, backoffMs: PREVIEW_BREAKER_BASE_MS }
43
+
44
+ export function previewBreakerState(now = Date.now()): { open: boolean; retryAt: string | null; misses: number } {
45
+ const open = now < breaker.openUntil
46
+ return { open, retryAt: open ? new Date(breaker.openUntil).toISOString() : null, misses: breaker.misses }
47
+ }
48
+ export function previewCounters(): typeof previewStats { return { ...previewStats } }
49
+
50
+ function notePreviewMiss(now = Date.now()): void {
51
+ previewStats.parakeetMisses++
52
+ breaker.misses++
53
+ if (breaker.misses >= PREVIEW_BREAKER_MISSES) {
54
+ breaker.openUntil = now + breaker.backoffMs
55
+ console.warn(`[stt] Parakeet preview missed ${breaker.misses} in a row; Turbo previews for ${Math.round(breaker.backoffMs / 1000)} s`)
56
+ breaker.backoffMs = Math.min(breaker.backoffMs * 2, PREVIEW_BREAKER_MAX_MS)
57
+ breaker.misses = 0
58
+ }
59
+ }
60
+ function notePreviewHit(): void { breaker = { misses: 0, openUntil: 0, backoffMs: PREVIEW_BREAKER_BASE_MS } }
61
+
62
+ /** 6.73.0: dictation Parakeet decodes share the preview breaker, so a Parakeet that keeps
63
+ * failing stops being tried for prompts and meetings alike. A busy lane is not a miss. */
64
+ export function noteParakeetDictationMiss(now = Date.now()): void { notePreviewMiss(now) }
65
+ export function noteParakeetDictationHit(): void { notePreviewHit() }
66
+
67
+ export function __resetPreviewForTests(): void {
68
+ for (const k of Object.keys(previewStats) as Array<keyof typeof previewStats>) previewStats[k] = 0
69
+ breaker = { misses: 0, openUntil: 0, backoffMs: PREVIEW_BREAKER_BASE_MS }
70
+ recordingLiveProbe = () => false
71
+ }
72
+
73
+ export function currentSttCapabilities(): SttCapabilities {
74
+ const commit = getWhisperCommitCapability()
75
+ const hq = getHighQualityTranscriptionCapability()
76
+ const installed = parakeetInstalled().installed
77
+ const warm = parakeetWarmState(PREVIEW_PARAKEET_MODEL).state
78
+ return {
79
+ turboModel: existsSync(WHISPER_TURBO_MODEL_PATH),
80
+ liveLargeV3: commit.effectiveModel === 'large-v3',
81
+ hqLargeV3: hq.hqAvailable && hq.model === 'large-v3',
82
+ parakeetInstalled: installed,
83
+ parakeetPreviewWarm: installed && warm === 'warm',
84
+ parakeetPreviewWarming: installed && warm === 'warming',
85
+ parakeetPreviewWarmFailed: installed && warm === 'failed',
86
+ parakeetPreviewBreakerOpen: installed && previewBreakerState().open,
87
+ previewDisabledByEnv: process.env.COS_WHISPER_MEETING_PREVIEW === '0',
88
+ dictationDisabledByEnv: process.env.COS_DICTATION_FOLLOWS_STT === '0',
89
+ }
90
+ }
91
+
92
+ export function currentEffectiveStt(): EffectiveStt {
93
+ return effectiveStt(readSttSettings(), currentSttCapabilities())
94
+ }
95
+
96
+ export function parakeetStatus(live: boolean): ParakeetStatus {
97
+ const installed = parakeetInstalled()
98
+ const progress = getInstallProgress()
99
+ const running = parakeetInstallRunning()
100
+ const sandbox = existsSync('/usr/bin/sandbox-exec')
101
+ const appleSilicon = process.platform === 'darwin' && process.arch === 'arm64'
102
+ // A button that can never succeed is never offered: the pinned CLI must be on this Mac.
103
+ const cli = appleSilicon ? pinnedCliKnown() : false
104
+ const installable = !installed.installed && !running && sandbox && appleSilicon && cli === true
105
+ const warm = parakeetWarmState(PREVIEW_PARAKEET_MODEL)
106
+ const note = installed.installed
107
+ ? 'Installed. Runs offline on the Neural Engine.'
108
+ : running
109
+ ? 'Installing.'
110
+ : !sandbox || !appleSilicon
111
+ ? 'Parakeet needs an Apple silicon Mac.'
112
+ : cli === false
113
+ ? 'Parakeet needs FluidAudio fluidaudiocli 0.17.4 at ~/.cos-glasses/bin (the speaker-separation tool); this Mac does not have that build.'
114
+ : cli === null
115
+ ? 'Checking this Mac for the FluidAudio CLI.'
116
+ : live
117
+ ? 'Install after the live recording is saved.'
118
+ : 'About 654 MB. Installs the pinned model and checks every file.'
119
+ return {
120
+ installed: installed.installed,
121
+ installable,
122
+ version: PARAKEET_VERSION_LABEL,
123
+ bytes: installed.installed ? installed.bytes : progress.bytesDone,
124
+ progress: { ...progress, state: installed.installed && !running ? 'installed' : progress.state },
125
+ warm: { preview: warm.state, ...(warm.error ? { error: warm.error } : {}), ...(warm.at ? { at: new Date(warm.at).toISOString() } : {}) },
126
+ preview: { breaker: previewBreakerState(), counters: previewCounters() },
127
+ note,
128
+ }
129
+ }
130
+
131
+ /** A failed warm-up is retried at most this often, never per chunk. */
132
+ export const WARM_RETRY_MS = 120_000
133
+
134
+ /**
135
+ * Start warming the preview model when it is selected; never awaited by a
136
+ * chunk. Deferred while a recording is live (the first CoreML compile is 12 to
137
+ * 25 s of Neural Engine work next to the live diarizer); a timer retries.
138
+ */
139
+ export function ensurePreviewWarm(now = Date.now()): 'started' | 'skipped' | 'deferred' {
140
+ const settings = readSttSettings()
141
+ // 6.73.0: a Parakeet HQ model also warms, because dictation final text uses it only warm.
142
+ const wanted = settings.preview === 'parakeet' || settings.hq === 'parakeet-ultra'
143
+ if (!wanted || !parakeetInstalled().installed) return 'skipped'
144
+ const w = parakeetWarmState(PREVIEW_PARAKEET_MODEL)
145
+ if (w.state === 'warm' || w.state === 'warming') return 'skipped'
146
+ if (w.state === 'failed' && now - w.at < WARM_RETRY_MS) return 'skipped'
147
+ if (recordingLive()) return 'deferred'
148
+ void warmParakeet(PREVIEW_PARAKEET_MODEL).then(ok => {
149
+ console.log(`[stt] Parakeet preview ${ok ? 'warm' : 'not warm'}: ${JSON.stringify(parakeetWarmState(PREVIEW_PARAKEET_MODEL))}`)
150
+ })
151
+ return 'started'
152
+ }
153
+
154
+ let warmTimer: ReturnType<typeof setInterval> | null = null
155
+ /** Server start: one try now, then every minute (a no-op unless a Parakeet preview or HQ model is selected and cold). */
156
+ export function startPreviewWarmLoop(): void {
157
+ ensurePreviewWarm()
158
+ if (warmTimer) return
159
+ warmTimer = setInterval(() => { try { ensurePreviewWarm() } catch { /* next tick */ } }, 60_000)
160
+ warmTimer.unref?.()
161
+ }
162
+ export function stopPreviewWarmLoop(): void { if (warmTimer) clearInterval(warmTimer); warmTimer = null }
163
+
164
+ // ── Meeting preview ──────────────────────────────────────────────────────────
165
+
166
+ export interface PreviewResult {
167
+ text: string
168
+ model: string
169
+ backend: string
170
+ }
171
+
172
+ export interface PreviewDeps {
173
+ effective: () => EffectiveStt
174
+ turbo: (audio: Buffer) => Promise<{ text: string; model: string; backend: string } | null>
175
+ parakeet: (audio: Buffer) => Promise<{ text: string } | null>
176
+ frontEnd: (audio: Buffer, frontEnd: FrontEndId) => Promise<Buffer>
177
+ now?: () => number
178
+ }
179
+
180
+ export const defaultPreviewDeps: PreviewDeps = {
181
+ effective: () => currentEffectiveStt(),
182
+ turbo: audio => transcribeWhisperMeetingPreview(audio),
183
+ parakeet: async audio => {
184
+ const result = await transcribeParakeet(audio, { model: PREVIEW_PARAKEET_MODEL, lane: 'preview', timeoutMs: PARAKEET_PREVIEW_TIMEOUT_MS })
185
+ return { text: applyCorrections(result.text) }
186
+ },
187
+ frontEnd: (audio, fe) => asrFrontEnd(audio, fe),
188
+ }
189
+
190
+ /**
191
+ * Cosmetic preview for the still-open phrase. Never durable, never attributed.
192
+ *
193
+ * `acceptedModels` is what the client says it accepts (`previewModels`). A
194
+ * client that lists nothing (every phone before 6.10.639) gets Whisper Turbo
195
+ * whatever the setting, so a preview it would reject is never sent. A
196
+ * Parakeet miss falls back to Turbo only while the phone's 1.8 s budget allows
197
+ * (PREVIEW_TURBO_FALLBACK_BEFORE_MS); otherwise it is a quiet miss (204).
198
+ * The Parakeet preview hears the selected ASR front end (highpass by default);
199
+ * the Whisper previews keep raw audio, as before 6.72 (highpass measured
200
+ * slightly worse for Turbo).
201
+ */
202
+ export async function transcribeMeetingPreview(audio: Buffer, acceptedModels: readonly string[] = [], deps: PreviewDeps = defaultPreviewDeps): Promise<PreviewResult | null> {
203
+ const now = deps.now ?? (() => performance.now())
204
+ const started = now()
205
+ let eff: EffectiveStt | null = null
206
+ try { eff = deps.effective() } catch (error) {
207
+ console.warn(`[stt] preview settings unreadable, using Turbo: ${error instanceof Error ? error.message : error}`)
208
+ }
209
+ const turbo = async () => {
210
+ previewStats.turbo++
211
+ const t = await deps.turbo(audio)
212
+ return t ? { text: t.text, model: t.model, backend: t.backend } : null
213
+ }
214
+ if (!eff) return turbo()
215
+ if (eff.settings.preview === 'parakeet' && eff.preview.engine !== 'parakeet') ensurePreviewWarm()
216
+ if (eff.preview.engine === 'parakeet') {
217
+ if (!acceptedModels.includes(PREVIEW_PARAKEET_MODEL)) {
218
+ previewStats.nonTurboWithheld++
219
+ return turbo()
220
+ }
221
+ try {
222
+ const r = await deps.parakeet(await deps.frontEnd(audio, eff.frontEnd))
223
+ if (r) { notePreviewHit(); previewStats.parakeet++; return { text: r.text, model: PREVIEW_PARAKEET_MODEL, backend: 'fluidaudio-coreml' } }
224
+ } catch (error) {
225
+ const busy = (error as { reason?: string })?.reason === 'busy'
226
+ if (!busy) notePreviewMiss()
227
+ console.warn(`[stt] Parakeet preview missed${busy ? ' (busy)' : ''}: ${error instanceof Error ? error.message : error}`)
228
+ }
229
+ if (now() - started >= PREVIEW_TURBO_FALLBACK_BEFORE_MS) { previewStats.empty++; return null }
230
+ previewStats.turboFallbacks++
231
+ }
232
+ return turbo()
233
+ }