@gotcos/glasses-server 6.71.14 → 6.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +16 -6
- package/CHANGELOG.md +32 -0
- package/README.md +188 -3
- package/docs/stt-672-evaluation.md +113 -0
- package/package.json +4 -1
- package/server/index.ts +18 -1
- package/server/lib/batch-transcript-quality.ts +46 -0
- package/server/lib/dictation-stt.ts +117 -0
- package/server/lib/meeting-batch-persistence.ts +23 -1
- package/server/lib/meeting-batch-transcribe.ts +408 -45
- package/server/lib/parakeet-manifest.ts +41 -0
- package/server/lib/parakeet.ts +675 -0
- package/server/lib/prompt-draft-store.ts +5 -0
- package/server/lib/stt-live.ts +60 -0
- package/server/lib/stt-resolver.ts +327 -0
- package/server/lib/stt-runtime.ts +233 -0
- package/server/lib/stt-settings.ts +431 -0
- package/server/lib/transcribe-audio.ts +53 -4
- package/server/lib/vad-silero.ts +52 -0
- package/server/lib/whisper-batch-timing.ts +13 -3
- package/server/lib/whisper-local.ts +58 -17
- package/server/routes/meeting.ts +9 -3
- package/server/routes/prompt-drafts.ts +56 -17
- package/server/routes/stt-settings.ts +85 -0
- package/server/routes/transcribe-stream.ts +34 -22
- package/server/routes/transcribe.ts +6 -2
|
@@ -22,6 +22,11 @@ export interface PromptDraftTranscriptRecord {
|
|
|
22
22
|
degraded: boolean
|
|
23
23
|
degradationReason?: string
|
|
24
24
|
acceptedDegraded?: boolean
|
|
25
|
+
/** 6.73.0: the setting this decode followed ("fast:parakeet", "final:large-v3").
|
|
26
|
+
* Absent on older records, which were all made under the 6.72.0 defaults. */
|
|
27
|
+
sttKey?: string
|
|
28
|
+
engine?: 'whisper' | 'parakeet-ultra'
|
|
29
|
+
engineReason?: string
|
|
25
30
|
}
|
|
26
31
|
|
|
27
32
|
/** Higher wins. Fast warm must never replace an HQ/cloud decode for the same audio. */
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// The live chunk's quiet gate and ASR front end (6.72.0). Kept apart from the
|
|
2
|
+
// decoders so the live route imports only audio code.
|
|
3
|
+
|
|
4
|
+
import { enhanceAudio } from './audio-enhance.js'
|
|
5
|
+
import { isSileroAvailable, measureSpeechSeconds } from './vad-silero.js'
|
|
6
|
+
import type { FrontEndId, QuietGateId } from './stt-settings.js'
|
|
7
|
+
|
|
8
|
+
/** Under this many seconds of Silero speech a chunk is quiet (bake-off canary 5). */
|
|
9
|
+
export const QUIET_SPEECH_SEC = 0.5
|
|
10
|
+
export const QUIET_RMS = 150
|
|
11
|
+
|
|
12
|
+
export interface QuietDecision {
|
|
13
|
+
isQuiet: boolean
|
|
14
|
+
rms: number
|
|
15
|
+
/** Silero speech seconds; null when the gate fell back to RMS. */
|
|
16
|
+
speechSec: number | null
|
|
17
|
+
gate: 'silero' | 'rms' | 'rms_fallback'
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function chunkRms(wav: Buffer): number {
|
|
21
|
+
const pcm = wav.subarray(44)
|
|
22
|
+
const n = Math.floor(pcm.length / 2)
|
|
23
|
+
let sumSq = 0
|
|
24
|
+
for (let i = 0; i < n; i++) { const s = pcm.readInt16LE(i * 2); sumSq += s * s }
|
|
25
|
+
return Math.sqrt(sumSq / Math.max(1, n))
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Quiet means "do not prompt, do not enhance, drop silence echoes". RMS 150
|
|
30
|
+
* fired on 1% of G2 chunks while Silero found no speech in 9.3%, so 6.72.0
|
|
31
|
+
* gates on Silero speech seconds (the in-process sherpa Silero, about 17 ms a
|
|
32
|
+
* 6 s chunk). Silero unavailable or the clip too short: the old RMS gate.
|
|
33
|
+
*/
|
|
34
|
+
export function decideQuiet(wav: Buffer, gate: QuietGateId, vad: { available: () => boolean; speechSeconds: (wav: Buffer) => number | null } = { available: () => isSileroAvailable(), speechSeconds: w => measureSpeechSeconds(w) }): QuietDecision {
|
|
35
|
+
const rms = chunkRms(wav)
|
|
36
|
+
if (gate === 'silero' && vad.available()) {
|
|
37
|
+
const speechSec = vad.speechSeconds(wav)
|
|
38
|
+
if (speechSec !== null && Number.isFinite(speechSec)) {
|
|
39
|
+
return { isQuiet: speechSec < QUIET_SPEECH_SEC, rms, speechSec, gate: 'silero' }
|
|
40
|
+
}
|
|
41
|
+
return { isQuiet: rms < QUIET_RMS, rms, speechSec: null, gate: 'rms_fallback' }
|
|
42
|
+
}
|
|
43
|
+
return { isQuiet: rms < QUIET_RMS, rms, speechSec: null, gate: 'rms' }
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Whether the silence-hallucination TEXT drops (URL-only, repeated thank-you,
|
|
48
|
+
* quiet vocabulary echo) may fire. They stay on the pre-6.72 RMS test: the
|
|
49
|
+
* Silero gate also calls a chunk holding one short real name quiet, and those
|
|
50
|
+
* drops would delete it.
|
|
51
|
+
*/
|
|
52
|
+
export function silenceDropsAllowed(decision: QuietDecision): boolean {
|
|
53
|
+
return decision.rms < QUIET_RMS
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** ASR input for a live chunk or an HQ window. Playback audio is never touched here. */
|
|
57
|
+
export function asrFrontEnd(audio: Buffer, frontEnd: FrontEndId): Promise<Buffer> {
|
|
58
|
+
return frontEnd === 'full' ? enhanceAudio(audio) : enhanceAudio(audio, { profile: 'light' })
|
|
59
|
+
}
|
|
60
|
+
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
// Second-opinion resolver for meeting HQ (6.72.0). Algorithmic only: no model
|
|
2
|
+
// call, no network, no LLM. Transcription must never call `claude -p`.
|
|
3
|
+
//
|
|
4
|
+
// Two decoders hear the same window. Their words are aligned by edit distance
|
|
5
|
+
// over normalized words (an optional time-skew limit exists but is off: it
|
|
6
|
+
// measured worse), and every place they disagree is recorded. The resolver
|
|
7
|
+
// then applies two measured rules, and nothing else:
|
|
8
|
+
//
|
|
9
|
+
// 1. Gap fill. Large-v3 is deletion-prone on G2 audio (bake-off silver
|
|
10
|
+
// deletions 6.1% for the fixed Large-v3, 6.6% on the shipped path, against
|
|
11
|
+
// 2.1% for Parakeet Ultra). Where the primary is a Whisper engine and the
|
|
12
|
+
// second opinion heard words the primary has nothing for, those words are
|
|
13
|
+
// taken, but only inside a span the primary's own VAD marked as speech and
|
|
14
|
+
// only when the second opinion's mean word confidence is at least 0.5.
|
|
15
|
+
// 2. Vocabulary rescue. Where the second opinion's side of a short
|
|
16
|
+
// disagreement (at most 4 words, and the primary's side at most 2 longer)
|
|
17
|
+
// holds a profile vocabulary term the primary's side does not, the second
|
|
18
|
+
// opinion's words are taken.
|
|
19
|
+
//
|
|
20
|
+
// Everything else stays as the primary heard it. No region whose two sides
|
|
21
|
+
// differ in a negation, a number or a unit is ever changed, whatever its length
|
|
22
|
+
// (the 2026-10-13 diagnostic saw "born lacking" become "born having" and
|
|
23
|
+
// "18 months" become "18 minutes"); it is listed as a protected disagreement.
|
|
24
|
+
|
|
25
|
+
export interface HypWord {
|
|
26
|
+
word: string
|
|
27
|
+
start: number
|
|
28
|
+
end: number
|
|
29
|
+
probability?: number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface Hypothesis {
|
|
33
|
+
engine: string
|
|
34
|
+
text: string
|
|
35
|
+
/** Spoken words (not sub-word tokens), seconds from the start of the window. */
|
|
36
|
+
words?: HypWord[]
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type DisagreementKind = 'substitution' | 'primary_only' | 'second_only'
|
|
40
|
+
export type ResolutionReason = 'gap_fill' | 'vocabulary' | 'protected' | 'not_speech' | 'low_confidence' | 'too_wide' | null
|
|
41
|
+
|
|
42
|
+
export interface Disagreement {
|
|
43
|
+
kind: DisagreementKind
|
|
44
|
+
primary: string
|
|
45
|
+
second: string
|
|
46
|
+
start: number | null
|
|
47
|
+
end: number | null
|
|
48
|
+
taken: 'primary' | 'second'
|
|
49
|
+
reason: ResolutionReason
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface ResolveOptions {
|
|
53
|
+
vocabulary: readonly string[]
|
|
54
|
+
/** Gap fill only for a deletion-prone primary (the Whisper engines). */
|
|
55
|
+
primaryDeletionProne: boolean
|
|
56
|
+
/** Where the primary's own VAD heard speech, seconds on the window clock.
|
|
57
|
+
* Gap fill needs it: a run outside every span is background the primary's VAD
|
|
58
|
+
* rejected, and is not taken. Absent: no gap fill at all. */
|
|
59
|
+
speechSpans?: ReadonlyArray<readonly [number, number]> | null
|
|
60
|
+
/** Mean word confidence a gap-fill run needs (second opinion's own scores). */
|
|
61
|
+
minGapConfidence?: number
|
|
62
|
+
/** false: report disagreements, change nothing. */
|
|
63
|
+
apply: boolean
|
|
64
|
+
/** When set, two words more than this far apart in time never align. Off by
|
|
65
|
+
* default: on the production-path silver decode every limit tried (2, 3, 5,
|
|
66
|
+
* 10 s) made the resolver worse than plain text alignment, because Large-v3's
|
|
67
|
+
* remapped token clocks have a heavy tail (median skew 0.23 s, p90 1.0 s). */
|
|
68
|
+
maxSkewSec?: number | null
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface ResolveResult {
|
|
72
|
+
text: string
|
|
73
|
+
words: HypWord[]
|
|
74
|
+
/** True when the resolver's words carry clocks for every word. */
|
|
75
|
+
timed: boolean
|
|
76
|
+
disagreements: Disagreement[]
|
|
77
|
+
wordsFromSecond: number
|
|
78
|
+
applied: boolean
|
|
79
|
+
skipped?: 'too_long'
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export const RESOLVER_VERSION = 'stt-resolver-v1'
|
|
83
|
+
/** Alignment is quadratic; a 30 s window is about 100 words a side. */
|
|
84
|
+
export const MAX_ALIGNMENT_CELLS = 2_000_000
|
|
85
|
+
export const DEFAULT_MAX_SKEW_SEC: number | null = null
|
|
86
|
+
/** Agreed words each side of a region that a vocabulary match may span. */
|
|
87
|
+
export const CONTEXT_WORDS = 3
|
|
88
|
+
/** A vocabulary rescue replaces at most this many second-opinion words... */
|
|
89
|
+
export const MAX_VOCAB_REGION_WORDS = 4
|
|
90
|
+
/** ...and at most this many more primary words than it inserts. */
|
|
91
|
+
export const MAX_VOCAB_REGION_SLACK = 2
|
|
92
|
+
export const DEFAULT_MIN_GAP_CONFIDENCE = 0.5
|
|
93
|
+
/** Share of a gap-fill run's words whose midpoint must lie in a speech span. */
|
|
94
|
+
export const MIN_SPEECH_SHARE = 0.5
|
|
95
|
+
|
|
96
|
+
const NEGATIONS = new Set([
|
|
97
|
+
'not', 'no', 'never', 'none', 'nor', 'nothing', 'nobody', 'neither', 'without', 'cannot',
|
|
98
|
+
'dont', 'doesnt', 'didnt', 'isnt', 'arent', 'wasnt', 'werent', 'wont', 'wouldnt', 'cant',
|
|
99
|
+
'couldnt', 'shouldnt', 'havent', 'hasnt', 'hadnt', 'aint', 'lacking', 'lack', 'lacks',
|
|
100
|
+
])
|
|
101
|
+
// Unambiguous units and number words only. "am", "one", "second" and "first"
|
|
102
|
+
// are everyday words far more often than quantities, and protecting them would
|
|
103
|
+
// block most real gap fills.
|
|
104
|
+
const UNITS = new Set([
|
|
105
|
+
'seconds', 'minute', 'minutes', 'hour', 'hours', 'day', 'days', 'week', 'weeks',
|
|
106
|
+
'month', 'months', 'year', 'years', 'percent', 'dollar', 'dollars', 'cents',
|
|
107
|
+
'thousand', 'million', 'billion',
|
|
108
|
+
])
|
|
109
|
+
const NUMBER_WORDS = new Set([
|
|
110
|
+
'zero', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',
|
|
111
|
+
'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen',
|
|
112
|
+
'nineteen', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety',
|
|
113
|
+
'hundred', 'half', 'quarter', 'twice',
|
|
114
|
+
])
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
/** Lowercase letters and digits only. "Don't," and "dont" are the same word. */
|
|
118
|
+
export function wordKey(word: string): string {
|
|
119
|
+
return word.normalize('NFKD').replace(/\p{M}+/gu, '').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function isProtectedKey(key: string): boolean {
|
|
123
|
+
return NEGATIONS.has(key) || UNITS.has(key) || NUMBER_WORDS.has(key) || /\d/.test(key)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
interface Token { display: string; key: string; start: number | null; end: number | null; probability?: number }
|
|
127
|
+
|
|
128
|
+
function tokens(h: Hypothesis): Token[] {
|
|
129
|
+
const timedWords = Array.isArray(h.words) && h.words.length > 0 ? h.words : null
|
|
130
|
+
const source = timedWords
|
|
131
|
+
? timedWords.map(w => ({
|
|
132
|
+
display: String(w.word ?? '').trim(),
|
|
133
|
+
start: Number.isFinite(w.start) ? w.start : null,
|
|
134
|
+
end: Number.isFinite(w.end) ? w.end : null,
|
|
135
|
+
probability: w.probability,
|
|
136
|
+
}))
|
|
137
|
+
: String(h.text ?? '').split(/\s+/).map(display => ({ display, start: null, end: null, probability: undefined }))
|
|
138
|
+
return source
|
|
139
|
+
.filter(t => t.display.length > 0)
|
|
140
|
+
.map(t => ({ ...t, key: wordKey(t.display) }))
|
|
141
|
+
.filter(t => t.key.length > 0)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function vocabSequences(vocabulary: readonly string[]): string[][] {
|
|
145
|
+
const out: string[][] = []
|
|
146
|
+
const seen = new Set<string>()
|
|
147
|
+
for (const term of vocabulary) {
|
|
148
|
+
const seq = String(term).split(/\s+/).map(wordKey).filter(Boolean)
|
|
149
|
+
if (!seq.length) continue
|
|
150
|
+
const id = seq.join(' ')
|
|
151
|
+
if (seen.has(id)) continue
|
|
152
|
+
seen.add(id)
|
|
153
|
+
out.push(seq)
|
|
154
|
+
// "ThriftCart" is also heard as "thrift cart": both spellings count.
|
|
155
|
+
const joined = seq.join('')
|
|
156
|
+
if (seq.length > 1 && !seen.has(joined)) { seen.add(joined); out.push([joined]) }
|
|
157
|
+
}
|
|
158
|
+
return out
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function termsIn(keys: string[], sequences: string[][]): Set<string> {
|
|
162
|
+
const found = new Set<string>()
|
|
163
|
+
for (const seq of sequences) {
|
|
164
|
+
for (let i = 0; i + seq.length <= keys.length; i++) {
|
|
165
|
+
let hit = true
|
|
166
|
+
for (let k = 0; k < seq.length; k++) if (keys[i + k] !== seq[k]) { hit = false; break }
|
|
167
|
+
if (hit) { found.add(seq.join('')); break }
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return found
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function mid(t: Token): number | null {
|
|
174
|
+
if (t.start === null) return null
|
|
175
|
+
return t.end === null ? t.start : (t.start + t.end) / 2
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
type Op = { kind: '=' | 's' | 'p' | 'q'; i: number | null; j: number | null }
|
|
179
|
+
|
|
180
|
+
/** Edit-distance alignment. Two words far apart in time cannot pair. */
|
|
181
|
+
function align(a: Token[], b: Token[], maxSkew: number | null): Op[] {
|
|
182
|
+
const n = a.length, m = b.length
|
|
183
|
+
const INF = 1e9
|
|
184
|
+
const D = new Float64Array((n + 1) * (m + 1))
|
|
185
|
+
const W = m + 1
|
|
186
|
+
for (let i = 0; i <= n; i++) D[i * W] = i
|
|
187
|
+
for (let j = 0; j <= m; j++) D[j] = j
|
|
188
|
+
const pairCost = (i: number, j: number): number => {
|
|
189
|
+
const ta = a[i], tb = b[j]
|
|
190
|
+
const ma = mid(ta), mb = mid(tb)
|
|
191
|
+
if (maxSkew !== null && ma !== null && mb !== null && Math.abs(ma - mb) > maxSkew) return INF
|
|
192
|
+
return ta.key === tb.key ? 0 : 1
|
|
193
|
+
}
|
|
194
|
+
for (let i = 1; i <= n; i++) {
|
|
195
|
+
for (let j = 1; j <= m; j++) {
|
|
196
|
+
const diag = D[(i - 1) * W + j - 1] + pairCost(i - 1, j - 1)
|
|
197
|
+
const up = D[(i - 1) * W + j] + 1
|
|
198
|
+
const left = D[i * W + j - 1] + 1
|
|
199
|
+
D[i * W + j] = Math.min(diag, up, left)
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
const ops: Op[] = []
|
|
203
|
+
let i = n, j = m
|
|
204
|
+
while (i > 0 || j > 0) {
|
|
205
|
+
if (i > 0 && j > 0) {
|
|
206
|
+
const c = pairCost(i - 1, j - 1)
|
|
207
|
+
if (c < INF && D[i * W + j] === D[(i - 1) * W + j - 1] + c) {
|
|
208
|
+
ops.push({ kind: c === 0 ? '=' : 's', i: i - 1, j: j - 1 }); i--; j--; continue
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
if (i > 0 && D[i * W + j] === D[(i - 1) * W + j] + 1) { ops.push({ kind: 'p', i: i - 1, j: null }); i--; continue }
|
|
212
|
+
ops.push({ kind: 'q', i: null, j: j - 1 }); j--
|
|
213
|
+
}
|
|
214
|
+
return ops.reverse()
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function span(ts: Token[]): { start: number | null; end: number | null } {
|
|
218
|
+
const starts = ts.map(t => t.start).filter((x): x is number => x !== null)
|
|
219
|
+
const ends = ts.map(t => t.end ?? t.start).filter((x): x is number => x !== null)
|
|
220
|
+
return { start: starts.length ? Math.min(...starts) : null, end: ends.length ? Math.max(...ends) : null }
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function sameProtected(p: Token[], q: Token[]): boolean {
|
|
224
|
+
const pk = p.map(t => t.key).filter(isProtectedKey).join(' ')
|
|
225
|
+
const qk = q.map(t => t.key).filter(isProtectedKey).join(' ')
|
|
226
|
+
return pk === qk
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** A second-opinion-only run is taken only where the primary's VAD heard
|
|
230
|
+
* speech and the second opinion was confident. */
|
|
231
|
+
function gapVerdict(q: Token[], options: ResolveOptions): ResolutionReason {
|
|
232
|
+
const spans = options.speechSpans
|
|
233
|
+
if (!spans || !spans.length) return 'not_speech'
|
|
234
|
+
const timed = q.filter(t => t.start !== null)
|
|
235
|
+
if (!timed.length) return 'not_speech'
|
|
236
|
+
const inside = timed.filter(t => { const m = mid(t)!; return spans.some(([a, b]) => m >= a && m <= b) }).length
|
|
237
|
+
if (inside / timed.length < MIN_SPEECH_SHARE) return 'not_speech'
|
|
238
|
+
const scored = q.filter(t => typeof t.probability === 'number' && Number.isFinite(t.probability))
|
|
239
|
+
const floor = options.minGapConfidence ?? DEFAULT_MIN_GAP_CONFIDENCE
|
|
240
|
+
if (scored.length && scored.reduce((n, t) => n + (t.probability as number), 0) / scored.length < floor) return 'low_confidence'
|
|
241
|
+
return 'gap_fill'
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export function resolveHypotheses(primary: Hypothesis, second: Hypothesis, options: ResolveOptions): ResolveResult {
|
|
245
|
+
const P = tokens(primary), Q = tokens(second)
|
|
246
|
+
const toWords = (ts: Token[]): HypWord[] => ts.map(t => ({
|
|
247
|
+
word: t.display,
|
|
248
|
+
start: t.start ?? 0,
|
|
249
|
+
end: Math.max(t.start ?? 0, t.end ?? t.start ?? 0),
|
|
250
|
+
...(t.probability !== undefined ? { probability: t.probability } : {}),
|
|
251
|
+
}))
|
|
252
|
+
const primaryOnly = (): ResolveResult => ({
|
|
253
|
+
text: P.map(t => t.display).join(' '),
|
|
254
|
+
words: toWords(P),
|
|
255
|
+
timed: P.length > 0 && P.every(t => t.start !== null),
|
|
256
|
+
disagreements: [],
|
|
257
|
+
wordsFromSecond: 0,
|
|
258
|
+
applied: false,
|
|
259
|
+
})
|
|
260
|
+
if ((P.length + 1) * (Q.length + 1) > MAX_ALIGNMENT_CELLS) return { ...primaryOnly(), skipped: 'too_long' }
|
|
261
|
+
|
|
262
|
+
const sequences = vocabSequences(options.vocabulary)
|
|
263
|
+
const ops = align(P, Q, options.maxSkewSec ?? DEFAULT_MAX_SKEW_SEC)
|
|
264
|
+
const out: Token[] = []
|
|
265
|
+
const disagreements: Disagreement[] = []
|
|
266
|
+
let wordsFromSecond = 0
|
|
267
|
+
let k = 0
|
|
268
|
+
while (k < ops.length) {
|
|
269
|
+
if (ops[k].kind === '=') { out.push(P[ops[k].i!]); k++; continue }
|
|
270
|
+
const p: Token[] = [], q: Token[] = []
|
|
271
|
+
const regionStart = k
|
|
272
|
+
while (k < ops.length && ops[k].kind !== '=') {
|
|
273
|
+
if (ops[k].i !== null) p.push(P[ops[k].i!])
|
|
274
|
+
if (ops[k].j !== null) q.push(Q[ops[k].j!])
|
|
275
|
+
k++
|
|
276
|
+
}
|
|
277
|
+
// Agreed words on both sides, so a term split across the edge of the
|
|
278
|
+
// region ("drift cart" vs "thrift cart") is still seen whole.
|
|
279
|
+
const before: string[] = [], after: string[] = []
|
|
280
|
+
for (let b = regionStart - 1; b >= 0 && before.length < CONTEXT_WORDS && ops[b].kind === '='; b--) before.unshift(P[ops[b].i!].key)
|
|
281
|
+
for (let a = k; a < ops.length && after.length < CONTEXT_WORDS && ops[a].kind === '='; a++) after.push(P[ops[a].i!].key)
|
|
282
|
+
const kind: DisagreementKind = !p.length ? 'second_only' : !q.length ? 'primary_only' : 'substitution'
|
|
283
|
+
let take: 'primary' | 'second' = 'primary'
|
|
284
|
+
let reason: ResolutionReason = null
|
|
285
|
+
if (q.length) {
|
|
286
|
+
const withContext = (side: Token[]) => [...before, ...side.map(t => t.key), ...after]
|
|
287
|
+
const primaryTerms = termsIn(withContext(p), sequences)
|
|
288
|
+
const newTerms = [...termsIn(withContext(q), sequences)].filter(t => !primaryTerms.has(t))
|
|
289
|
+
const wantGap = kind === 'second_only' && options.primaryDeletionProne
|
|
290
|
+
const wantVocab = newTerms.length > 0
|
|
291
|
+
if (wantGap || wantVocab) {
|
|
292
|
+
if (!sameProtected(p, q)) reason = 'protected'
|
|
293
|
+
else if (wantVocab && (q.length > MAX_VOCAB_REGION_WORDS || p.length > q.length + MAX_VOCAB_REGION_SLACK)) {
|
|
294
|
+
// A term inside a long disagreement is not a reason to swap the whole region.
|
|
295
|
+
reason = wantGap ? gapVerdict(q, options) : 'too_wide'
|
|
296
|
+
if (reason === 'gap_fill') take = 'second'
|
|
297
|
+
} else if (wantVocab) { take = 'second'; reason = 'vocabulary' }
|
|
298
|
+
else { reason = gapVerdict(q, options); if (reason === 'gap_fill') take = 'second' }
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
const where = span([...p, ...q])
|
|
302
|
+
disagreements.push({
|
|
303
|
+
kind,
|
|
304
|
+
primary: p.map(t => t.display).join(' '),
|
|
305
|
+
second: q.map(t => t.display).join(' '),
|
|
306
|
+
start: where.start,
|
|
307
|
+
end: where.end,
|
|
308
|
+
taken: options.apply ? take : 'primary',
|
|
309
|
+
reason,
|
|
310
|
+
})
|
|
311
|
+
if (options.apply && take === 'second') { out.push(...q); wordsFromSecond += q.length }
|
|
312
|
+
else out.push(...p)
|
|
313
|
+
}
|
|
314
|
+
return {
|
|
315
|
+
text: out.map(t => t.display).join(' '),
|
|
316
|
+
words: toWords(out),
|
|
317
|
+
timed: out.length > 0 && out.every(t => t.start !== null),
|
|
318
|
+
disagreements,
|
|
319
|
+
wordsFromSecond,
|
|
320
|
+
applied: options.apply,
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** Whisper's deletion-prone profile is the reason gap fill exists. */
|
|
325
|
+
export function isDeletionProneEngine(engine: string): boolean {
|
|
326
|
+
return engine === 'large-v3' || engine === 'large-v3-legacy'
|
|
327
|
+
}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
// Where the STT settings meet the decoders (6.72.0): what this Mac can run,
|
|
2
|
+
// the live quiet gate and ASR front end, and the meeting preview dispatch.
|
|
3
|
+
|
|
4
|
+
import { existsSync } from 'node:fs'
|
|
5
|
+
import {
|
|
6
|
+
applyCorrections,
|
|
7
|
+
getHighQualityTranscriptionCapability,
|
|
8
|
+
getWhisperCommitCapability,
|
|
9
|
+
WHISPER_TURBO_MODEL_PATH,
|
|
10
|
+
} from './whisper-local.js'
|
|
11
|
+
import { transcribeWhisperMeetingPreview } from './whisper-preview.js'
|
|
12
|
+
import { asrFrontEnd } from './stt-live.js'
|
|
13
|
+
import {
|
|
14
|
+
getInstallProgress, parakeetInstalled, parakeetInstallRunning, parakeetWarmState, pinnedCliKnown,
|
|
15
|
+
PARAKEET_VERSION_LABEL, transcribeParakeet, warmParakeet, type ParakeetModelId,
|
|
16
|
+
} from './parakeet.js'
|
|
17
|
+
import {
|
|
18
|
+
effectiveStt, readSttSettings, type EffectiveStt, type FrontEndId, type ParakeetStatus,
|
|
19
|
+
type SttCapabilities,
|
|
20
|
+
} from './stt-settings.js'
|
|
21
|
+
|
|
22
|
+
/** The live preview's Parakeet model. Ultra over v2 on the live-lane evidence
|
|
23
|
+
* (docs/stt-672-evaluation.md): lower WER, and v2 put text on 3 of 40 clean
|
|
24
|
+
* non-speech units where Ultra put none. */
|
|
25
|
+
export const PREVIEW_PARAKEET_MODEL: ParakeetModelId = 'parakeet-ultra'
|
|
26
|
+
/** The phone gives a preview 1.8 s end to end. Parakeet gets this long... */
|
|
27
|
+
export const PARAKEET_PREVIEW_TIMEOUT_MS = 900
|
|
28
|
+
/** ...and a miss falls back to Turbo only while this much of the budget is left unused. */
|
|
29
|
+
export const PREVIEW_TURBO_FALLBACK_BEFORE_MS = 900
|
|
30
|
+
/** Consecutive Parakeet preview misses (not busy lanes) that open the breaker. */
|
|
31
|
+
export const PREVIEW_BREAKER_MISSES = 3
|
|
32
|
+
export const PREVIEW_BREAKER_BASE_MS = 60_000
|
|
33
|
+
export const PREVIEW_BREAKER_MAX_MS = 10 * 60_000
|
|
34
|
+
|
|
35
|
+
// ── Recording probe (registered by the server; avoids an import cycle) ───────
|
|
36
|
+
let recordingLiveProbe: () => boolean = () => false
|
|
37
|
+
export function setRecordingLiveProbe(probe: () => boolean): void { recordingLiveProbe = probe }
|
|
38
|
+
function recordingLive(): boolean { try { return recordingLiveProbe() } catch { return true } }
|
|
39
|
+
|
|
40
|
+
// ── Preview breaker and counters ─────────────────────────────────────────────
|
|
41
|
+
const previewStats = { parakeet: 0, parakeetMisses: 0, turboFallbacks: 0, turbo: 0, nonTurboWithheld: 0, empty: 0 }
|
|
42
|
+
let breaker = { misses: 0, openUntil: 0, backoffMs: PREVIEW_BREAKER_BASE_MS }
|
|
43
|
+
|
|
44
|
+
export function previewBreakerState(now = Date.now()): { open: boolean; retryAt: string | null; misses: number } {
|
|
45
|
+
const open = now < breaker.openUntil
|
|
46
|
+
return { open, retryAt: open ? new Date(breaker.openUntil).toISOString() : null, misses: breaker.misses }
|
|
47
|
+
}
|
|
48
|
+
export function previewCounters(): typeof previewStats { return { ...previewStats } }
|
|
49
|
+
|
|
50
|
+
function notePreviewMiss(now = Date.now()): void {
|
|
51
|
+
previewStats.parakeetMisses++
|
|
52
|
+
breaker.misses++
|
|
53
|
+
if (breaker.misses >= PREVIEW_BREAKER_MISSES) {
|
|
54
|
+
breaker.openUntil = now + breaker.backoffMs
|
|
55
|
+
console.warn(`[stt] Parakeet preview missed ${breaker.misses} in a row; Turbo previews for ${Math.round(breaker.backoffMs / 1000)} s`)
|
|
56
|
+
breaker.backoffMs = Math.min(breaker.backoffMs * 2, PREVIEW_BREAKER_MAX_MS)
|
|
57
|
+
breaker.misses = 0
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function notePreviewHit(): void { breaker = { misses: 0, openUntil: 0, backoffMs: PREVIEW_BREAKER_BASE_MS } }
|
|
61
|
+
|
|
62
|
+
/** 6.73.0: dictation Parakeet decodes share the preview breaker, so a Parakeet that keeps
|
|
63
|
+
* failing stops being tried for prompts and meetings alike. A busy lane is not a miss. */
|
|
64
|
+
export function noteParakeetDictationMiss(now = Date.now()): void { notePreviewMiss(now) }
|
|
65
|
+
export function noteParakeetDictationHit(): void { notePreviewHit() }
|
|
66
|
+
|
|
67
|
+
export function __resetPreviewForTests(): void {
|
|
68
|
+
for (const k of Object.keys(previewStats) as Array<keyof typeof previewStats>) previewStats[k] = 0
|
|
69
|
+
breaker = { misses: 0, openUntil: 0, backoffMs: PREVIEW_BREAKER_BASE_MS }
|
|
70
|
+
recordingLiveProbe = () => false
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function currentSttCapabilities(): SttCapabilities {
|
|
74
|
+
const commit = getWhisperCommitCapability()
|
|
75
|
+
const hq = getHighQualityTranscriptionCapability()
|
|
76
|
+
const installed = parakeetInstalled().installed
|
|
77
|
+
const warm = parakeetWarmState(PREVIEW_PARAKEET_MODEL).state
|
|
78
|
+
return {
|
|
79
|
+
turboModel: existsSync(WHISPER_TURBO_MODEL_PATH),
|
|
80
|
+
liveLargeV3: commit.effectiveModel === 'large-v3',
|
|
81
|
+
hqLargeV3: hq.hqAvailable && hq.model === 'large-v3',
|
|
82
|
+
parakeetInstalled: installed,
|
|
83
|
+
parakeetPreviewWarm: installed && warm === 'warm',
|
|
84
|
+
parakeetPreviewWarming: installed && warm === 'warming',
|
|
85
|
+
parakeetPreviewWarmFailed: installed && warm === 'failed',
|
|
86
|
+
parakeetPreviewBreakerOpen: installed && previewBreakerState().open,
|
|
87
|
+
previewDisabledByEnv: process.env.COS_WHISPER_MEETING_PREVIEW === '0',
|
|
88
|
+
dictationDisabledByEnv: process.env.COS_DICTATION_FOLLOWS_STT === '0',
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function currentEffectiveStt(): EffectiveStt {
|
|
93
|
+
return effectiveStt(readSttSettings(), currentSttCapabilities())
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function parakeetStatus(live: boolean): ParakeetStatus {
|
|
97
|
+
const installed = parakeetInstalled()
|
|
98
|
+
const progress = getInstallProgress()
|
|
99
|
+
const running = parakeetInstallRunning()
|
|
100
|
+
const sandbox = existsSync('/usr/bin/sandbox-exec')
|
|
101
|
+
const appleSilicon = process.platform === 'darwin' && process.arch === 'arm64'
|
|
102
|
+
// A button that can never succeed is never offered: the pinned CLI must be on this Mac.
|
|
103
|
+
const cli = appleSilicon ? pinnedCliKnown() : false
|
|
104
|
+
const installable = !installed.installed && !running && sandbox && appleSilicon && cli === true
|
|
105
|
+
const warm = parakeetWarmState(PREVIEW_PARAKEET_MODEL)
|
|
106
|
+
const note = installed.installed
|
|
107
|
+
? 'Installed. Runs offline on the Neural Engine.'
|
|
108
|
+
: running
|
|
109
|
+
? 'Installing.'
|
|
110
|
+
: !sandbox || !appleSilicon
|
|
111
|
+
? 'Parakeet needs an Apple silicon Mac.'
|
|
112
|
+
: cli === false
|
|
113
|
+
? 'Parakeet needs FluidAudio fluidaudiocli 0.17.4 at ~/.cos-glasses/bin (the speaker-separation tool); this Mac does not have that build.'
|
|
114
|
+
: cli === null
|
|
115
|
+
? 'Checking this Mac for the FluidAudio CLI.'
|
|
116
|
+
: live
|
|
117
|
+
? 'Install after the live recording is saved.'
|
|
118
|
+
: 'About 654 MB. Installs the pinned model and checks every file.'
|
|
119
|
+
return {
|
|
120
|
+
installed: installed.installed,
|
|
121
|
+
installable,
|
|
122
|
+
version: PARAKEET_VERSION_LABEL,
|
|
123
|
+
bytes: installed.installed ? installed.bytes : progress.bytesDone,
|
|
124
|
+
progress: { ...progress, state: installed.installed && !running ? 'installed' : progress.state },
|
|
125
|
+
warm: { preview: warm.state, ...(warm.error ? { error: warm.error } : {}), ...(warm.at ? { at: new Date(warm.at).toISOString() } : {}) },
|
|
126
|
+
preview: { breaker: previewBreakerState(), counters: previewCounters() },
|
|
127
|
+
note,
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** A failed warm-up is retried at most this often, never per chunk. */
|
|
132
|
+
export const WARM_RETRY_MS = 120_000
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Start warming the preview model when it is selected; never awaited by a
|
|
136
|
+
* chunk. Deferred while a recording is live (the first CoreML compile is 12 to
|
|
137
|
+
* 25 s of Neural Engine work next to the live diarizer); a timer retries.
|
|
138
|
+
*/
|
|
139
|
+
export function ensurePreviewWarm(now = Date.now()): 'started' | 'skipped' | 'deferred' {
|
|
140
|
+
const settings = readSttSettings()
|
|
141
|
+
// 6.73.0: a Parakeet HQ model also warms, because dictation final text uses it only warm.
|
|
142
|
+
const wanted = settings.preview === 'parakeet' || settings.hq === 'parakeet-ultra'
|
|
143
|
+
if (!wanted || !parakeetInstalled().installed) return 'skipped'
|
|
144
|
+
const w = parakeetWarmState(PREVIEW_PARAKEET_MODEL)
|
|
145
|
+
if (w.state === 'warm' || w.state === 'warming') return 'skipped'
|
|
146
|
+
if (w.state === 'failed' && now - w.at < WARM_RETRY_MS) return 'skipped'
|
|
147
|
+
if (recordingLive()) return 'deferred'
|
|
148
|
+
void warmParakeet(PREVIEW_PARAKEET_MODEL).then(ok => {
|
|
149
|
+
console.log(`[stt] Parakeet preview ${ok ? 'warm' : 'not warm'}: ${JSON.stringify(parakeetWarmState(PREVIEW_PARAKEET_MODEL))}`)
|
|
150
|
+
})
|
|
151
|
+
return 'started'
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
let warmTimer: ReturnType<typeof setInterval> | null = null
|
|
155
|
+
/** Server start: one try now, then every minute (a no-op unless a Parakeet preview or HQ model is selected and cold). */
|
|
156
|
+
export function startPreviewWarmLoop(): void {
|
|
157
|
+
ensurePreviewWarm()
|
|
158
|
+
if (warmTimer) return
|
|
159
|
+
warmTimer = setInterval(() => { try { ensurePreviewWarm() } catch { /* next tick */ } }, 60_000)
|
|
160
|
+
warmTimer.unref?.()
|
|
161
|
+
}
|
|
162
|
+
export function stopPreviewWarmLoop(): void { if (warmTimer) clearInterval(warmTimer); warmTimer = null }
|
|
163
|
+
|
|
164
|
+
// ── Meeting preview ──────────────────────────────────────────────────────────
|
|
165
|
+
|
|
166
|
+
export interface PreviewResult {
|
|
167
|
+
text: string
|
|
168
|
+
model: string
|
|
169
|
+
backend: string
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export interface PreviewDeps {
|
|
173
|
+
effective: () => EffectiveStt
|
|
174
|
+
turbo: (audio: Buffer) => Promise<{ text: string; model: string; backend: string } | null>
|
|
175
|
+
parakeet: (audio: Buffer) => Promise<{ text: string } | null>
|
|
176
|
+
frontEnd: (audio: Buffer, frontEnd: FrontEndId) => Promise<Buffer>
|
|
177
|
+
now?: () => number
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
export const defaultPreviewDeps: PreviewDeps = {
|
|
181
|
+
effective: () => currentEffectiveStt(),
|
|
182
|
+
turbo: audio => transcribeWhisperMeetingPreview(audio),
|
|
183
|
+
parakeet: async audio => {
|
|
184
|
+
const result = await transcribeParakeet(audio, { model: PREVIEW_PARAKEET_MODEL, lane: 'preview', timeoutMs: PARAKEET_PREVIEW_TIMEOUT_MS })
|
|
185
|
+
return { text: applyCorrections(result.text) }
|
|
186
|
+
},
|
|
187
|
+
frontEnd: (audio, fe) => asrFrontEnd(audio, fe),
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Cosmetic preview for the still-open phrase. Never durable, never attributed.
|
|
192
|
+
*
|
|
193
|
+
* `acceptedModels` is what the client says it accepts (`previewModels`). A
|
|
194
|
+
* client that lists nothing (every phone before 6.10.639) gets Whisper Turbo
|
|
195
|
+
* whatever the setting, so a preview it would reject is never sent. A
|
|
196
|
+
* Parakeet miss falls back to Turbo only while the phone's 1.8 s budget allows
|
|
197
|
+
* (PREVIEW_TURBO_FALLBACK_BEFORE_MS); otherwise it is a quiet miss (204).
|
|
198
|
+
* The Parakeet preview hears the selected ASR front end (highpass by default);
|
|
199
|
+
* the Whisper previews keep raw audio, as before 6.72 (highpass measured
|
|
200
|
+
* slightly worse for Turbo).
|
|
201
|
+
*/
|
|
202
|
+
export async function transcribeMeetingPreview(audio: Buffer, acceptedModels: readonly string[] = [], deps: PreviewDeps = defaultPreviewDeps): Promise<PreviewResult | null> {
|
|
203
|
+
const now = deps.now ?? (() => performance.now())
|
|
204
|
+
const started = now()
|
|
205
|
+
let eff: EffectiveStt | null = null
|
|
206
|
+
try { eff = deps.effective() } catch (error) {
|
|
207
|
+
console.warn(`[stt] preview settings unreadable, using Turbo: ${error instanceof Error ? error.message : error}`)
|
|
208
|
+
}
|
|
209
|
+
const turbo = async () => {
|
|
210
|
+
previewStats.turbo++
|
|
211
|
+
const t = await deps.turbo(audio)
|
|
212
|
+
return t ? { text: t.text, model: t.model, backend: t.backend } : null
|
|
213
|
+
}
|
|
214
|
+
if (!eff) return turbo()
|
|
215
|
+
if (eff.settings.preview === 'parakeet' && eff.preview.engine !== 'parakeet') ensurePreviewWarm()
|
|
216
|
+
if (eff.preview.engine === 'parakeet') {
|
|
217
|
+
if (!acceptedModels.includes(PREVIEW_PARAKEET_MODEL)) {
|
|
218
|
+
previewStats.nonTurboWithheld++
|
|
219
|
+
return turbo()
|
|
220
|
+
}
|
|
221
|
+
try {
|
|
222
|
+
const r = await deps.parakeet(await deps.frontEnd(audio, eff.frontEnd))
|
|
223
|
+
if (r) { notePreviewHit(); previewStats.parakeet++; return { text: r.text, model: PREVIEW_PARAKEET_MODEL, backend: 'fluidaudio-coreml' } }
|
|
224
|
+
} catch (error) {
|
|
225
|
+
const busy = (error as { reason?: string })?.reason === 'busy'
|
|
226
|
+
if (!busy) notePreviewMiss()
|
|
227
|
+
console.warn(`[stt] Parakeet preview missed${busy ? ' (busy)' : ''}: ${error instanceof Error ? error.message : error}`)
|
|
228
|
+
}
|
|
229
|
+
if (now() - started >= PREVIEW_TURBO_FALLBACK_BEFORE_MS) { previewStats.empty++; return null }
|
|
230
|
+
previewStats.turboFallbacks++
|
|
231
|
+
}
|
|
232
|
+
return turbo()
|
|
233
|
+
}
|