@gotcos/glasses-server 6.45.3 → 6.45.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/package.json +1 -1
- package/server/lib/embedding-eviction.ts +19 -0
- package/server/lib/held-voice-groups.ts +752 -0
- package/server/lib/prompt-tail-guard.ts +257 -0
- package/server/lib/speaker-embeddings.ts +8 -1
- package/server/lib/training-audio-provenance.ts +1 -1
- package/server/lib/transcribe-audio.ts +9 -1
- package/server/lib/vad-silero.ts +8 -3
- package/server/lib/voice-enrolment-selection.ts +31 -5
- package/server/lib/whisper-local.ts +12 -6
- package/server/routes/prompt-drafts.ts +13 -2
- package/server/routes/transcribe-stream.ts +5 -3
- package/server/routes/voice.ts +117 -1
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
// The tail of a dictated prompt: keep what was said, drop what the decoder made up.
|
|
2
|
+
//
|
|
3
|
+
// Chris, 2026-09-12: a prompt closed with "I'm going to go to the bathroom" after
|
|
4
|
+
// he had stopped talking. The interactive path runs whisper without VAD on
|
|
5
|
+
// purpose (a short prompt must never be trimmed), and a decoder handed trailing
|
|
6
|
+
// silence plus a vocabulary prompt finishes the sentence it thinks it heard.
|
|
7
|
+
//
|
|
8
|
+
// Miles's constraint governs everything here: "as long as we don't end up
|
|
9
|
+
// cutting the final portion of a user's prompt. It's only going to be when we
|
|
10
|
+
// see things like 'Thanks for watching,' 'I need to go to the bathroom,' and
|
|
11
|
+
// random shit like that that we want to reject it."
|
|
12
|
+
//
|
|
13
|
+
// Two rules, each on the LAST sentence, at most `TAIL_MAX_DROPS` times:
|
|
14
|
+
//
|
|
15
|
+
// (a) lexicon the whole sentence is a known filler — anchored start to end,
|
|
16
|
+
// so "put the risks at the end" and "the webinar about thanks
|
|
17
|
+
// for watching screens" never match (QA 2026-09-12 found the
|
|
18
|
+
// unanchored form eating both). Applies with or without a VAD.
|
|
19
|
+
// (b) evidence the VAD MEASURED the audio, the sentence's tokens start at
|
|
20
|
+
// least `TAIL_SILENCE_MIN_SEC` after the last speech it heard,
|
|
21
|
+
// AND whisper's own mean token probability for the sentence is
|
|
22
|
+
// under `TAIL_LOW_CONFIDENCE`. Two independent signals must
|
|
23
|
+
// agree; either alone keeps the text.
|
|
24
|
+
//
|
|
25
|
+
// Rule (b) never empties a chunk in which the VAD heard any speech: if the only
|
|
26
|
+
// sentence left is low-confidence, it stays. A chunk empties only when every
|
|
27
|
+
// sentence was a filler, or the VAD measured the whole chunk as silence and the
|
|
28
|
+
// decoder was unsure of what it wrote there. A VAD that did not run (model
|
|
29
|
+
// absent, audio too short, internal error) reports UNKNOWN, never silence —
|
|
30
|
+
// `TrimResult.measured` is required, because every fallback shape is otherwise
|
|
31
|
+
// identical to real silence.
|
|
32
|
+
//
|
|
33
|
+
// Token alignment is by characters, not by whitespace words: whisper-cli emits
|
|
34
|
+
// one entry per TOKEN, and "bathroom." can be three of them.
|
|
35
|
+
|
|
36
|
+
import { isSileroAvailable, trimSilence } from './vad-silero.js'
|
|
37
|
+
import type { WhisperWord } from './whisper-local.js'
|
|
38
|
+
|
|
39
|
+
/** A sentence that starts this long after the last measured speech was decoded from silence. */
|
|
40
|
+
export const TAIL_SILENCE_MIN_SEC = 1.0
|
|
41
|
+
/** Below this mean token probability whisper is guessing. Calibrated on ten of
|
|
42
|
+
* Miles's real prompt chunks (2026-09-12, large-v3): trailing sentences scored
|
|
43
|
+
* 0.72 to 1.00, one real sentence in a noisy room 0.43 — which is why this
|
|
44
|
+
* number never acts alone. */
|
|
45
|
+
export const TAIL_LOW_CONFIDENCE = 0.45
|
|
46
|
+
/** Never more than this many trailing sentences. */
|
|
47
|
+
export const TAIL_MAX_DROPS = 2
|
|
48
|
+
/** Only the last window of a long chunk is measured. The shared Silero instance
|
|
49
|
+
* is built with a 10 s buffer, and real prompt chunks are 2 to 7 s; a longer
|
|
50
|
+
* chunk's earlier speech is treated as unknown, which can only KEEP text. */
|
|
51
|
+
export const TAIL_WINDOW_SEC = 10
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Whole-sentence fillers. Each pattern is anchored at both ends AND closed —
|
|
55
|
+
* no free-text tail — so it matches a sentence that IS the filler and nothing
|
|
56
|
+
* that merely starts with its words. The re-validation of 2026-09-12 caught
|
|
57
|
+
* `.{0,30}` tails letting "Thanks for watching the demo.", "Transcript from the
|
|
58
|
+
* Silas call is in the folder." and "Subscribe me to the newsletter." through.
|
|
59
|
+
* "See you later", "the end", "transcribed by" and "translated by" are
|
|
60
|
+
* deliberately absent: each occurs in real dictation ("Translated by Friday."),
|
|
61
|
+
* and whisper's inventions are the caption-credit shapes below. A filler this
|
|
62
|
+
* list misses fails SAFE: the text stays.
|
|
63
|
+
*/
|
|
64
|
+
export const TAIL_FILLER_SENTENCES: readonly RegExp[] = [
|
|
65
|
+
/^(?:thanks?|thank you)(?: (?:all|everyone|guys|so much|very much))? for (?:watching|listening)(?:,? (?:everyone|everybody|guys|all))?[.!]?$/i,
|
|
66
|
+
/^(?:subtitles?|captions?) (?:by|provided by|created by) (?:the )?[\w.'’-]+(?: [\w.'’-]+){0,2}(?: community)?[.!]?$/i,
|
|
67
|
+
/^transcript(?:ion)? by [\w'’-]+\.(?:com|org|net|io|ai)[.!]?$/i,
|
|
68
|
+
/^(?:please )?(?:like(?:,)? (?:and )?)?(?:share(?:,)? (?:and )?)?subscribe(?: to (?:my|the|our) channel)?[.!]?$/i,
|
|
69
|
+
/^(?:don['’]?t|do not) forget to (?:like(?: and)? )?subscribe(?: to (?:my|the|our) channel)?[.!]?$/i,
|
|
70
|
+
/^(?:i(?:['’]ll| will)? )?see you(?: all| guys)? (?:next time|in the next (?:one|video|episode))[.!]?$/i,
|
|
71
|
+
/^(?:i(?:['’]m| am) (?:going to|gonna) |i (?:need|have|got) to |i gotta |let me |gonna )?(?:go (?:to )?)?(?:the )?(?:bathroom|restroom|toilet)(?: (?:real quick|really quick|right quick|quick|now))?[.!]?$/i,
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
export interface SpeechWindows {
|
|
75
|
+
/** True only when the VAD RAN over the audio (`TrimResult.measured`). */
|
|
76
|
+
available: boolean
|
|
77
|
+
/** Seconds from the start of the chunk to the end of the last speech the VAD
|
|
78
|
+
* heard. 0 when it measured the whole chunk and heard nothing; the window
|
|
79
|
+
* offset when only the tail was measured and it was silent; null when
|
|
80
|
+
* unavailable. */
|
|
81
|
+
lastSpeechEndSec: number | null
|
|
82
|
+
/** Share of the MEASURED audio that was speech. */
|
|
83
|
+
speechRatio: number
|
|
84
|
+
/** True when the whole chunk was measured, false when only its last
|
|
85
|
+
* `TAIL_WINDOW_SEC`. */
|
|
86
|
+
whole: boolean
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export const SPEECH_UNKNOWN: SpeechWindows = { available: false, lastSpeechEndSec: null, speechRatio: 0, whole: false }
|
|
90
|
+
|
|
91
|
+
const WAV_HEADER = 44
|
|
92
|
+
const VAD_SAMPLE_RATE = 16_000
|
|
93
|
+
|
|
94
|
+
/** The last `TAIL_WINDOW_SEC` of a 16 kHz mono 16-bit WAV, with a fresh header. */
|
|
95
|
+
function tailWindow(wav: Buffer): { wav: Buffer; offsetSec: number; whole: boolean } {
|
|
96
|
+
if (wav.length <= WAV_HEADER) return { wav, offsetSec: 0, whole: true }
|
|
97
|
+
const channels = wav.readUInt16LE(22)
|
|
98
|
+
const rate = wav.readUInt32LE(24)
|
|
99
|
+
const bits = wav.readUInt16LE(34)
|
|
100
|
+
const blockAlign = Math.max(1, (channels * bits) / 8)
|
|
101
|
+
if (rate !== VAD_SAMPLE_RATE || channels !== 1 || bits !== 16) return { wav, offsetSec: 0, whole: true }
|
|
102
|
+
const bytesPerSec = rate * blockAlign
|
|
103
|
+
const pcmBytes = wav.length - WAV_HEADER
|
|
104
|
+
// A body that is not whole samples cannot be windowed on a sample boundary;
|
|
105
|
+
// hand it over untouched, where the VAD refuses it as unmeasured.
|
|
106
|
+
if (pcmBytes % blockAlign !== 0) return { wav, offsetSec: 0, whole: true }
|
|
107
|
+
if (pcmBytes <= TAIL_WINDOW_SEC * bytesPerSec) return { wav, offsetSec: 0, whole: true }
|
|
108
|
+
const tailBytes = Math.floor((TAIL_WINDOW_SEC * bytesPerSec) / blockAlign) * blockAlign
|
|
109
|
+
const start = wav.length - tailBytes
|
|
110
|
+
const out = Buffer.alloc(WAV_HEADER + tailBytes)
|
|
111
|
+
wav.copy(out, 0, 0, WAV_HEADER)
|
|
112
|
+
wav.copy(out, WAV_HEADER, start)
|
|
113
|
+
out.writeUInt32LE(36 + tailBytes, 4)
|
|
114
|
+
out.writeUInt32LE(tailBytes, 40)
|
|
115
|
+
return { wav: out, offsetSec: (start - WAV_HEADER) / bytesPerSec, whole: false }
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Where the VAD heard speech in (the tail of) this chunk. */
|
|
119
|
+
export function speechWindowsFromWav(wav: Buffer): SpeechWindows {
|
|
120
|
+
if (!isSileroAvailable()) return SPEECH_UNKNOWN
|
|
121
|
+
const window = tailWindow(wav)
|
|
122
|
+
let result: ReturnType<typeof trimSilence>
|
|
123
|
+
try {
|
|
124
|
+
result = trimSilence(window.wav)
|
|
125
|
+
} catch {
|
|
126
|
+
return SPEECH_UNKNOWN
|
|
127
|
+
}
|
|
128
|
+
if (result.measured !== true) return SPEECH_UNKNOWN
|
|
129
|
+
let lastEnd: number | null = null
|
|
130
|
+
for (const seg of result.segments) {
|
|
131
|
+
const end = (seg.startSample + seg.sampleCount) / VAD_SAMPLE_RATE + window.offsetSec
|
|
132
|
+
if (lastEnd === null || end > lastEnd) lastEnd = end
|
|
133
|
+
}
|
|
134
|
+
return {
|
|
135
|
+
available: true,
|
|
136
|
+
lastSpeechEndSec: lastEnd ?? window.offsetSec,
|
|
137
|
+
speechRatio: result.speechRatio,
|
|
138
|
+
whole: window.whole,
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Sentence boundaries: a terminator followed by whitespace, except after an
|
|
143
|
+
* abbreviation or an initial. Decimals never split (no whitespace inside). */
|
|
144
|
+
export function splitSentences(text: string): string[] {
|
|
145
|
+
const raw = text.trim().split(/(?<=[.!?])\s+(?=\S)/)
|
|
146
|
+
const out: string[] = []
|
|
147
|
+
for (const piece of raw) {
|
|
148
|
+
const prev = out[out.length - 1]
|
|
149
|
+
if (prev !== undefined && /(?:^|\s)(?:dr|mr|mrs|ms|jr|sr|st|vs|etc|e\.g|i\.e|a\.m|p\.m|no|[a-z])\.$/i.test(prev)) {
|
|
150
|
+
out[out.length - 1] = `${prev} ${piece}`
|
|
151
|
+
} else {
|
|
152
|
+
out.push(piece)
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return out.filter(s => s.length > 0)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function normalizeSentence(sentence: string): string {
|
|
159
|
+
return sentence.trim().replace(/^["'“”‘’(\[]+|["'“”‘’)\]]+$/g, '').replace(/\s+/g, ' ').trim()
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Is this sentence, whole, a known filler? */
|
|
163
|
+
export function matchesTailLexicon(sentence: string): boolean {
|
|
164
|
+
const s = normalizeSentence(sentence)
|
|
165
|
+
return s.length > 0 && TAIL_FILLER_SENTENCES.some(re => re.test(s))
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export interface TailTokenStats {
|
|
169
|
+
startSec: number | null
|
|
170
|
+
meanProbability: number | null
|
|
171
|
+
/** Index of the sentence's first token, so a caller that drops the sentence
|
|
172
|
+
* can trim its tokens before judging the one before it. */
|
|
173
|
+
tokenStart: number | null
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* The tokens that spell the last sentence, found by walking back from the end
|
|
178
|
+
* until the accumulated non-space characters cover it. Returns nulls when the
|
|
179
|
+
* tokens do not cover the text at all (alignment unknown): then rule (b) is
|
|
180
|
+
* inert and the sentence stays.
|
|
181
|
+
*/
|
|
182
|
+
export function tailTokenStats(words: WhisperWord[] | undefined, sentence: string): TailTokenStats {
|
|
183
|
+
const none = { startSec: null, meanProbability: null, tokenStart: null }
|
|
184
|
+
if (!words || words.length === 0) return none
|
|
185
|
+
const target = sentence.replace(/\s+/g, '').length
|
|
186
|
+
if (target === 0) return none
|
|
187
|
+
let acc = 0
|
|
188
|
+
let i = words.length
|
|
189
|
+
while (i > 0 && acc < target) {
|
|
190
|
+
i--
|
|
191
|
+
acc += String(words[i].word ?? '').replace(/\s+/g, '').length
|
|
192
|
+
}
|
|
193
|
+
if (acc < target) return none
|
|
194
|
+
const slice = words.slice(i)
|
|
195
|
+
// The tail must SPELL the sentence, not merely be as long as it: a
|
|
196
|
+
// correction that changed a token's length ("Carrot IQ" -> "CaratIQ") would
|
|
197
|
+
// otherwise shift the window and its start time. Mismatch fails safe.
|
|
198
|
+
const letters = (s: string) => s.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '')
|
|
199
|
+
if (letters(slice.map(w => String(w.word ?? '')).join('')) !== letters(sentence)) return none
|
|
200
|
+
let sum = 0
|
|
201
|
+
let n = 0
|
|
202
|
+
for (const w of slice) {
|
|
203
|
+
if (typeof w.probability === 'number' && Number.isFinite(w.probability)) { sum += w.probability; n++ }
|
|
204
|
+
}
|
|
205
|
+
const start = typeof slice[0]?.start === 'number' && Number.isFinite(slice[0].start) ? slice[0].start : null
|
|
206
|
+
return { startSec: start, meanProbability: n > 0 ? sum / n : null, tokenStart: i }
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export type TailDropReason = 'lexicon' | 'low_confidence_after_speech'
|
|
210
|
+
|
|
211
|
+
export interface TailDrop {
|
|
212
|
+
text: string
|
|
213
|
+
reason: TailDropReason
|
|
214
|
+
startSec: number | null
|
|
215
|
+
meanProbability: number | null
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export interface GuardedPromptTail {
|
|
219
|
+
text: string
|
|
220
|
+
dropped: TailDrop[]
|
|
221
|
+
lastSpeechEndSec: number | null
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
export function guardPromptTail(input: { text: string; words?: WhisperWord[]; speech: SpeechWindows }): GuardedPromptTail {
|
|
225
|
+
const sentences = splitSentences(input.text)
|
|
226
|
+
const dropped: TailDrop[] = []
|
|
227
|
+
const lastSpeechEnd = input.speech.available ? input.speech.lastSpeechEndSec : null
|
|
228
|
+
// Tokens still standing: a dropped sentence's tokens leave with it, so the
|
|
229
|
+
// sentence before it is judged against its own tokens.
|
|
230
|
+
let remaining = input.words
|
|
231
|
+
while (sentences.length > 0 && dropped.length < TAIL_MAX_DROPS) {
|
|
232
|
+
const last = sentences[sentences.length - 1]
|
|
233
|
+
let reason: TailDropReason | null = null
|
|
234
|
+
const stats = tailTokenStats(remaining, last)
|
|
235
|
+
if (matchesTailLexicon(last)) {
|
|
236
|
+
reason = 'lexicon'
|
|
237
|
+
} else if (lastSpeechEnd !== null) {
|
|
238
|
+
if (
|
|
239
|
+
stats.startSec !== null && stats.meanProbability !== null
|
|
240
|
+
&& stats.startSec >= lastSpeechEnd + TAIL_SILENCE_MIN_SEC
|
|
241
|
+
&& stats.meanProbability < TAIL_LOW_CONFIDENCE
|
|
242
|
+
) {
|
|
243
|
+
// The last sentence standing is kept unless the VAD measured the WHOLE
|
|
244
|
+
// chunk and heard nothing in it: real speech somewhere means the
|
|
245
|
+
// decoder's low confidence here is not enough to empty the prompt.
|
|
246
|
+
const measuredSilentChunk = input.speech.whole && input.speech.speechRatio === 0
|
|
247
|
+
if (sentences.length === 1 && !measuredSilentChunk) break
|
|
248
|
+
reason = 'low_confidence_after_speech'
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (!reason) break
|
|
252
|
+
dropped.push({ text: last, reason, startSec: stats.startSec, meanProbability: stats.meanProbability })
|
|
253
|
+
sentences.pop()
|
|
254
|
+
if (remaining && stats.tokenStart !== null) remaining = remaining.slice(0, stats.tokenStart)
|
|
255
|
+
}
|
|
256
|
+
return { text: sentences.join(' '), dropped, lastSpeechEndSec: lastSpeechEnd }
|
|
257
|
+
}
|
|
@@ -88,7 +88,14 @@ const CALIBRATION_LOG = resolve(DATA_DIR, 'speaker-calibration.jsonl')
|
|
|
88
88
|
// Thresholds
|
|
89
89
|
const VERIFY_THRESHOLD = 0.65
|
|
90
90
|
const SEARCH_THRESHOLD = 0.55
|
|
91
|
-
|
|
91
|
+
/** The bar `autoEnroll` enrols at. From 6.45.4 also the bar at which a held
|
|
92
|
+
* group is offered as a person with one click ("high"): the identifier would
|
|
93
|
+
* have written that sample itself. */
|
|
94
|
+
export const AUTO_ENROLL_THRESHOLD = 0.88 // High bar — must be very confident before auto-enrolling
|
|
95
|
+
/** The live path only ASKS autoEnroll about a chunk above this; autoEnroll
|
|
96
|
+
* then applies AUTO_ENROLL_THRESHOLD. Named so transcribe-stream and the
|
|
97
|
+
* held-voice panel read the same number (6.45.4). */
|
|
98
|
+
export const AUTO_ENROLL_CANDIDATE_SIMILARITY = 0.72
|
|
92
99
|
const AUTO_ENROLL_CONSENSUS = 2 // Must match N times in same session before enrolling
|
|
93
100
|
// Raised 20 -> 40 on 2026-08-06. Measured, not guessed: search latency is 1 us
|
|
94
101
|
// at 20, 40 AND 80 samples per speaker (77 speakers, sherpa SpeakerEmbeddingManager),
|
|
@@ -49,7 +49,7 @@ export function trainingSourceFor(filename: string): string {
|
|
|
49
49
|
export function isSampleFromSession(source: string | undefined | null, sessionId: string): boolean {
|
|
50
50
|
const s = String(source ?? '')
|
|
51
51
|
if (!sessionId) return false
|
|
52
|
-
for (const prefix of ['auto:', 'correction:', 'g2-training:']) {
|
|
52
|
+
for (const prefix of ['auto:', 'correction:', 'g2-training:', 'ext-group:']) {
|
|
53
53
|
if (s.startsWith(prefix) && s.slice(prefix.length) === sessionId) return true
|
|
54
54
|
}
|
|
55
55
|
return false
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { WhisperWord } from './whisper-local.js'
|
|
1
2
|
import {
|
|
2
3
|
transcribeLocal,
|
|
3
4
|
transcribeHighQuality,
|
|
@@ -28,6 +29,8 @@ export type TranscribeMode = 'hq' | 'fast'
|
|
|
28
29
|
|
|
29
30
|
export interface TranscribeAudioResult {
|
|
30
31
|
text: string
|
|
32
|
+
/** 6.45.4 — the decoder's timed words with token probability, when the backend produced them. */
|
|
33
|
+
words?: WhisperWord[]
|
|
31
34
|
backend: string
|
|
32
35
|
mode: TranscribeMode
|
|
33
36
|
requestedMode: TranscribeMode
|
|
@@ -158,6 +161,7 @@ export async function transcribeAudioBuffer(
|
|
|
158
161
|
}
|
|
159
162
|
|
|
160
163
|
let text: string
|
|
164
|
+
let words: WhisperWord[] | undefined
|
|
161
165
|
let backend: string
|
|
162
166
|
let actualQuality: 'hq' | 'fast' | 'cloud'
|
|
163
167
|
let degradationReason: string | undefined = effectiveMode !== requestedMode ? 'audio_too_long' : undefined
|
|
@@ -168,8 +172,9 @@ export async function transcribeAudioBuffer(
|
|
|
168
172
|
// A0 (2026-07-30): ffmpeg enhance light (highpass=f=80) was measured dropping
|
|
169
173
|
// leading speech on compose ("device just for your awareness"). Meeting batch
|
|
170
174
|
// still enhances in meeting-batch-transcribe.ts — this path is prompt/interactive only.
|
|
171
|
-
const result = await transcribeHighQuality(audioBuffer, undefined, { priority: 'interactive' })
|
|
175
|
+
const result = await transcribeHighQuality(audioBuffer, undefined, { priority: 'interactive', words: true })
|
|
172
176
|
text = result.text
|
|
177
|
+
words = result.words
|
|
173
178
|
actualQuality = result.actualQuality
|
|
174
179
|
if (result.actualQuality === 'hq') {
|
|
175
180
|
backend = 'hq-large-v3'
|
|
@@ -183,6 +188,7 @@ export async function transcribeAudioBuffer(
|
|
|
183
188
|
try {
|
|
184
189
|
const result = await transcribeLocal(audioBuffer, undefined, undefined, { affectsCircuit })
|
|
185
190
|
text = result.text
|
|
191
|
+
words = result.words
|
|
186
192
|
backend = `fast-local-${result.backend}`
|
|
187
193
|
actualQuality = 'fast'
|
|
188
194
|
} catch (localErr: any) {
|
|
@@ -203,6 +209,7 @@ export async function transcribeAudioBuffer(
|
|
|
203
209
|
try {
|
|
204
210
|
const result = await transcribeLocal(audioBuffer, undefined, undefined, { affectsCircuit })
|
|
205
211
|
text = result.text
|
|
212
|
+
words = result.words
|
|
206
213
|
backend = `fast-local-${result.backend}`
|
|
207
214
|
actualQuality = 'fast'
|
|
208
215
|
} catch (localErr: any) {
|
|
@@ -270,6 +277,7 @@ export async function transcribeAudioBuffer(
|
|
|
270
277
|
|
|
271
278
|
return {
|
|
272
279
|
text: text.trim(),
|
|
280
|
+
...(words && words.length > 0 ? { words } : {}),
|
|
273
281
|
backend,
|
|
274
282
|
mode: effectiveMode,
|
|
275
283
|
requestedMode,
|
package/server/lib/vad-silero.ts
CHANGED
|
@@ -32,6 +32,11 @@ export interface TrimResult {
|
|
|
32
32
|
trimmedWav: Buffer
|
|
33
33
|
speechRatio: number
|
|
34
34
|
segments: Array<{ startSample: number; sampleCount: number }>
|
|
35
|
+
/** True when the VAD ran over the audio and `segments` is its verdict. Absent
|
|
36
|
+
* on every fallback (model not loaded, audio too short, internal error), which
|
|
37
|
+
* otherwise look identical to measured silence. A reader deciding to drop
|
|
38
|
+
* text on "no speech" must require this. (6.45.4: prompt-tail-guard) */
|
|
39
|
+
measured?: true
|
|
35
40
|
}
|
|
36
41
|
|
|
37
42
|
/** Initialize Silero VAD. Returns true if model loaded, false otherwise. */
|
|
@@ -124,7 +129,7 @@ export function trimSilence(wavBuffer: Buffer): TrimResult {
|
|
|
124
129
|
|
|
125
130
|
if (speechSegments.length === 0 || totalSpeechSamples === 0) {
|
|
126
131
|
// No speech detected — return original unchanged
|
|
127
|
-
return { trimmedWav: wavBuffer, speechRatio: 0.0, segments: [] }
|
|
132
|
+
return { trimmedWav: wavBuffer, speechRatio: 0.0, segments: [], measured: true }
|
|
128
133
|
}
|
|
129
134
|
|
|
130
135
|
// Concatenate speech segments into a single Float32Array
|
|
@@ -168,10 +173,10 @@ export function trimSilence(wavBuffer: Buffer): TrimResult {
|
|
|
168
173
|
|
|
169
174
|
// Guard: if trimmed result is too small for Whisper, fall back to original
|
|
170
175
|
if (trimmedWav.length < 100) {
|
|
171
|
-
return { trimmedWav: wavBuffer, speechRatio, segments: segmentInfo }
|
|
176
|
+
return { trimmedWav: wavBuffer, speechRatio, segments: segmentInfo, measured: true }
|
|
172
177
|
}
|
|
173
178
|
|
|
174
|
-
return { trimmedWav, speechRatio, segments: segmentInfo }
|
|
179
|
+
return { trimmedWav, speechRatio, segments: segmentInfo, measured: true }
|
|
175
180
|
} catch (err: any) {
|
|
176
181
|
console.error('[silero-vad] trimSilence error:', err.message)
|
|
177
182
|
return fallback
|
|
@@ -82,10 +82,19 @@ export function dominantCoherentCluster(
|
|
|
82
82
|
const n = embeddings.length
|
|
83
83
|
if (n === 0) return { members: [], seed: -1 }
|
|
84
84
|
if (n === 1) return { members: [0], seed: 0 }
|
|
85
|
+
return dominantCoherentClusterFromMatrix(pairwiseSimilarityMatrix(embeddings), embeddings.map((_, i) => i), floor)
|
|
86
|
+
}
|
|
85
87
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
88
|
+
/**
|
|
89
|
+
* Full pairwise cosine matrix, computed once. The refinement in
|
|
90
|
+
* `dominantCoherentClusterFromMatrix` reads it repeatedly, and recomputing a
|
|
91
|
+
* 192-dim cosine inside that loop is the difference between microseconds and
|
|
92
|
+
* seconds on a long meeting. Held-voice grouping (6.45.4) also builds it once
|
|
93
|
+
* for every sample in the retention window and then carves cluster after
|
|
94
|
+
* cluster out of the same matrix.
|
|
95
|
+
*/
|
|
96
|
+
export function pairwiseSimilarityMatrix(embeddings: Float32Array[]): number[][] {
|
|
97
|
+
const n = embeddings.length
|
|
89
98
|
const sim: number[][] = Array.from({ length: n }, () => new Array<number>(n).fill(0))
|
|
90
99
|
for (let i = 0; i < n; i++) {
|
|
91
100
|
sim[i][i] = 1
|
|
@@ -95,14 +104,31 @@ export function dominantCoherentCluster(
|
|
|
95
104
|
sim[j][i] = s
|
|
96
105
|
}
|
|
97
106
|
}
|
|
107
|
+
return sim
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* `dominantCoherentCluster` over a precomputed matrix and an explicit candidate
|
|
112
|
+
* set. `members` and `seed` are indices into the matrix (the same index space
|
|
113
|
+
* as `candidates`), so a caller carving several clusters out of one pool passes
|
|
114
|
+
* the survivors back in without renumbering anything. Same contract otherwise:
|
|
115
|
+
* a lone candidate is its own cluster; a crowd that agrees on nothing is EMPTY.
|
|
116
|
+
*/
|
|
117
|
+
export function dominantCoherentClusterFromMatrix(
|
|
118
|
+
sim: number[][],
|
|
119
|
+
candidates: number[],
|
|
120
|
+
floor: number = VOICE_COHERENCE_FLOOR,
|
|
121
|
+
): CoherentCluster {
|
|
122
|
+
if (candidates.length === 0) return { members: [], seed: -1 }
|
|
123
|
+
if (candidates.length === 1) return { members: [candidates[0]], seed: candidates[0] }
|
|
98
124
|
|
|
99
125
|
let bestSeed = -1
|
|
100
126
|
let bestAgree: number[] = []
|
|
101
127
|
let bestMean = -Infinity
|
|
102
|
-
for (
|
|
128
|
+
for (const i of candidates) {
|
|
103
129
|
const agree: number[] = []
|
|
104
130
|
let sum = 0
|
|
105
|
-
for (
|
|
131
|
+
for (const j of candidates) {
|
|
106
132
|
if (i !== j && sim[i][j] >= floor) { agree.push(j); sum += sim[i][j] }
|
|
107
133
|
}
|
|
108
134
|
const meanSim = agree.length > 0 ? sum / agree.length : -Infinity
|
|
@@ -940,6 +940,11 @@ export async function transcribeHighQuality(
|
|
|
940
940
|
* an infinite loop. */
|
|
941
941
|
opts: {
|
|
942
942
|
priority?: 'interactive' | 'batch'
|
|
943
|
+
/** Ask whisper-cli for per-token timing and probability (`-ojf`) on an
|
|
944
|
+
* interactive decode too. Batch always captures them. The prompt path
|
|
945
|
+
* needs them for the tail guard (6.45.4): without this flag the guard's
|
|
946
|
+
* confidence rule could never fire, which QA caught on 2026-09-12. */
|
|
947
|
+
words?: boolean
|
|
943
948
|
forceCpu?: boolean
|
|
944
949
|
forceCpuReason?: string
|
|
945
950
|
threads?: number
|
|
@@ -965,8 +970,9 @@ export async function transcribeHighQuality(
|
|
|
965
970
|
const tmpWav = join('/tmp', `cos-whisper-hq-${id}.wav`)
|
|
966
971
|
const outBase = join('/tmp', `cos-whisper-hq-${id}`)
|
|
967
972
|
const jsonPath = `${outBase}.json`
|
|
968
|
-
// Word clocks
|
|
969
|
-
|
|
973
|
+
// Word clocks on post-meeting CPU polish, and on any decode that asks for
|
|
974
|
+
// them (the prompt tail guard). Other live decodes stay compact JSON.
|
|
975
|
+
const captureWords = opts.priority === 'batch' || opts.words === true
|
|
970
976
|
|
|
971
977
|
const modelPath = resolveBatchModel()
|
|
972
978
|
const useLargeV3 = modelPath === BATCH_MODEL_LARGE_V3
|
|
@@ -1013,7 +1019,7 @@ export async function transcribeHighQuality(
|
|
|
1013
1019
|
'-np',
|
|
1014
1020
|
'--prompt', buildPrompt(context),
|
|
1015
1021
|
]
|
|
1016
|
-
if (
|
|
1022
|
+
if (captureWords) {
|
|
1017
1023
|
args.push('-ojf', '-of', outBase)
|
|
1018
1024
|
}
|
|
1019
1025
|
if (useVad) {
|
|
@@ -1120,7 +1126,7 @@ export async function transcribeHighQuality(
|
|
|
1120
1126
|
|
|
1121
1127
|
let finalText = text
|
|
1122
1128
|
let words: WhisperWord[] | undefined
|
|
1123
|
-
if (
|
|
1129
|
+
if (captureWords) {
|
|
1124
1130
|
try {
|
|
1125
1131
|
if (existsSync(jsonPath)) {
|
|
1126
1132
|
const parsed = parseWhisperCliFullJson(readFileSync(jsonPath, 'utf8'))
|
|
@@ -1142,7 +1148,7 @@ export async function transcribeHighQuality(
|
|
|
1142
1148
|
const modelTag = useLargeV3 ? 'large-v3' : 'turbo'
|
|
1143
1149
|
console.log(
|
|
1144
1150
|
`[whisper-hq] Batch transcribed in ${elapsed}ms ` +
|
|
1145
|
-
`(${modelTag}${useVad ? '+vad' : ''}${
|
|
1151
|
+
`(${modelTag}${useVad ? '+vad' : ''}${captureWords ? '+words' : ''}` +
|
|
1146
1152
|
`${words ? `, ${words.length} words` : ''}` +
|
|
1147
1153
|
// Device forensics: without these, "why is polish slow today" is
|
|
1148
1154
|
// unanswerable after the fact.
|
|
@@ -1162,7 +1168,7 @@ export async function transcribeHighQuality(
|
|
|
1162
1168
|
return words ? { text: corrected, words, ...metadata } : { text: corrected, ...metadata }
|
|
1163
1169
|
} finally {
|
|
1164
1170
|
try { unlinkSync(tmpWav) } catch { /* cleanup */ }
|
|
1165
|
-
if (
|
|
1171
|
+
if (captureWords) {
|
|
1166
1172
|
try { unlinkSync(jsonPath) } catch { /* cleanup */ }
|
|
1167
1173
|
}
|
|
1168
1174
|
}
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
transcriptQualityRank,
|
|
17
17
|
type PromptDraftTranscriptRecord,
|
|
18
18
|
} from '../lib/prompt-draft-store.js'
|
|
19
|
+
import { SPEECH_UNKNOWN, guardPromptTail, speechWindowsFromWav } from '../lib/prompt-tail-guard.js'
|
|
19
20
|
import {
|
|
20
21
|
transcribeAudioBuffer,
|
|
21
22
|
resolveTranscribeMode,
|
|
@@ -291,7 +292,15 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
|
|
|
291
292
|
const policy = purpose === 'warm' ? 'local-only' as const : 'automatic' as const
|
|
292
293
|
const result = await transcribeAudioBuffer(audio, { mode, policy })
|
|
293
294
|
if (!isCurrentChunk(draftId, chunkIndex, audio)) return ''
|
|
294
|
-
|
|
295
|
+
// 6.45.4 — the sentence Whisper invents after the speaker stops. Never trims
|
|
296
|
+
// audio; drops a trailing sentence only on the junk lexicon or on silence
|
|
297
|
+
// plus the decoder's own low confidence (prompt-tail-guard.ts).
|
|
298
|
+
const guarded = guardPromptTail({ text: result.text, words: result.words, speech: speechWindowsFromWav(audio) })
|
|
299
|
+
for (const drop of guarded.dropped) {
|
|
300
|
+
console.warn(`[prompt-draft] tail_drop ${draftId}/${chunkIndex} ${purpose}/${mode} (${drop.reason}, start=${drop.startSec ?? '?'}s, lastSpeech=${guarded.lastSpeechEndSec ?? '?'}s, p=${drop.meanProbability?.toFixed(2) ?? '?'}): "${drop.text.slice(0, 100)}"`)
|
|
301
|
+
}
|
|
302
|
+
if (guarded.dropped.length > 0 && !guarded.text.trim()) throw new NoSpeechDetectedError(result.text)
|
|
303
|
+
const text = sanitizeTranscript(draftId, guarded.text)
|
|
295
304
|
const record: PromptDraftTranscriptRecord = {
|
|
296
305
|
text, hash, requestedMode: result.requestedMode, actualQuality: result.actualQuality,
|
|
297
306
|
backend: result.backend, degraded: result.degraded,
|
|
@@ -497,7 +506,9 @@ promptDraftsRouter.post('/prompt-drafts/:draftId/peek', async (req, res) => {
|
|
|
497
506
|
lease.setPhase('active')
|
|
498
507
|
try {
|
|
499
508
|
const result = await transcribeWhisperPreview(audio)
|
|
500
|
-
|
|
509
|
+
// 6.45.4: the same whole-sentence filler rule as the commit path, so an
|
|
510
|
+
// invented closing line never paints on the lens only to vanish later.
|
|
511
|
+
const text = sanitizeTranscript(draftId, guardPromptTail({ text: result.text, speech: SPEECH_UNKNOWN }).text, false)
|
|
501
512
|
if (!text) return
|
|
502
513
|
if (!loadPromptDraftMeta(draftId)) return
|
|
503
514
|
emitDisplay({
|
|
@@ -39,7 +39,7 @@ import { errMsg } from '../lib/utils.js'
|
|
|
39
39
|
import { transcribeLocal, applyCorrections, type WhisperWord } from '../lib/whisper-local.js'
|
|
40
40
|
import { enhanceAudio } from '../lib/audio-enhance.js'
|
|
41
41
|
import { trimSilence, isSileroAvailable } from '../lib/vad-silero.js'
|
|
42
|
-
import { identifySpeaker, isEmbeddingAvailable, autoEnroll, getEmbeddingCount } from '../lib/speaker-embeddings.js'
|
|
42
|
+
import { identifySpeaker, isEmbeddingAvailable, autoEnroll, getEmbeddingCount, AUTO_ENROLL_CANDIDATE_SIMILARITY } from '../lib/speaker-embeddings.js'
|
|
43
43
|
import {
|
|
44
44
|
assertOpenAIWhisperBudget,
|
|
45
45
|
recordOpenAIWhisperUsage,
|
|
@@ -1157,7 +1157,9 @@ setInterval(() => {
|
|
|
1157
1157
|
const dirPath = resolve(EXT_AUDIO_DIR, dir)
|
|
1158
1158
|
try {
|
|
1159
1159
|
const files = readdirSync(dirPath)
|
|
1160
|
-
|
|
1160
|
+
// 6.45.4: an emptied session (every held sample named or discarded)
|
|
1161
|
+
// frees its per-session cap too, so the next stranger can be held.
|
|
1162
|
+
if (files.length === 0) { rmSync(dirPath, { recursive: true, force: true }); extAudioCounts.delete(dir); continue }
|
|
1161
1163
|
const { mtimeMs } = statSync(resolve(dirPath, files[0]))
|
|
1162
1164
|
if (Date.now() - mtimeMs > EXT_AUDIO_TTL_MS) {
|
|
1163
1165
|
rmSync(dirPath, { recursive: true, force: true })
|
|
@@ -1897,7 +1899,7 @@ function identifyChunkSpeaker(audioBuffer: Buffer, sessionId: string, chunkIndex
|
|
|
1897
1899
|
console.log(`[speaker] Embedding: ${speaker} vs Amplitude: ${clientSpeaker} (sim: ${embeddingResult.similarity.toFixed(2)})`)
|
|
1898
1900
|
}
|
|
1899
1901
|
|
|
1900
|
-
if (embeddingResult.similarity >=
|
|
1902
|
+
if (embeddingResult.similarity >= AUTO_ENROLL_CANDIDATE_SIMILARITY && speaker !== 'Ext') {
|
|
1901
1903
|
const enrollResult = autoEnroll(speaker, audioBuffer, embeddingResult.similarity, sessionId)
|
|
1902
1904
|
if (enrollResult.enrolled) {
|
|
1903
1905
|
console.log(`[speaker] Auto-enrolled ${speaker} from G2 mic (sim: ${embeddingResult.similarity.toFixed(3)})`)
|