@gotcos/glasses-server 6.21.7 → 6.21.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/CHANGELOG.md +48 -0
- package/README.md +9 -0
- package/package.json +1 -1
- package/server/index.ts +5 -1
- package/server/lib/audio-retention.ts +50 -0
- package/server/lib/batch-transcript-quality.ts +1 -1
- package/server/lib/g2-enrichment-runner.ts +23 -6
- package/server/lib/g2-ops-handoff.ts +193 -32
- package/server/lib/meeting-batch-transcribe.ts +554 -10
- package/server/lib/meeting-finalization-jobs.ts +235 -0
- package/server/lib/meeting-store.ts +9 -0
- package/server/lib/speaker-calibration-log.ts +86 -0
- package/server/lib/speaker-embeddings.ts +162 -36
- package/server/lib/voice-profile-store.ts +365 -0
- package/server/lib/whisper-local.ts +77 -6
- package/server/routes/health.ts +26 -3
- package/server/routes/meeting.ts +268 -52
- package/server/routes/transcribe-stream.ts +113 -2
- package/server/routes/voice.ts +268 -22
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
// Durability and integrity for voice-profiles.json.
|
|
2
|
+
//
|
|
3
|
+
// This file is the single most irreplaceable artifact the server owns: on this
|
|
4
|
+
// box it holds 79 profiles / 1,317 embeddings / ~7.8 MB, accumulated over months
|
|
5
|
+
// of training that cannot be re-derived once the source audio has aged out. It
|
|
6
|
+
// was being written with three raw `writeFileSync` calls and read with an
|
|
7
|
+
// unguarded `JSON.parse`, so one crash, disk-full, or iCloud-sync mid-write
|
|
8
|
+
// would take all of it and every subsequent read would throw.
|
|
9
|
+
//
|
|
10
|
+
// Everything here is a pure function over an explicit path so it can be tested
|
|
11
|
+
// by execution rather than by asserting on source text. speaker-embeddings.ts
|
|
12
|
+
// delegates to it and keeps the sherpa-onnx manager in sync.
|
|
13
|
+
|
|
14
|
+
import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from 'node:fs'
|
|
15
|
+
import { basename, dirname, join, resolve } from 'node:path'
|
|
16
|
+
import { durableAtomicWriteFileSync, loadJsonOrQuarantine } from './atomic-fs.js'
|
|
17
|
+
|
|
18
|
+
export interface VoiceProfile {
|
|
19
|
+
name: string
|
|
20
|
+
/** One row per enrolled sample. Parallel to `sources`, index for index. */
|
|
21
|
+
embeddings: number[][]
|
|
22
|
+
/** Provenance: 'manual' | 'fireflies' | 'g2-training' | 'auto:<sessionId>' | … */
|
|
23
|
+
sources?: string[]
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface ProfileStore {
|
|
27
|
+
profiles: VoiceProfile[]
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** What normalization had to fix. All non-zero counts are worth logging: they
|
|
31
|
+
* mean the file on disk had drifted from its own invariants. */
|
|
32
|
+
export interface StoreRepairs {
|
|
33
|
+
/** `sources` was missing, short, or long relative to `embeddings`. */
|
|
34
|
+
sourcesRealigned: number
|
|
35
|
+
/** A null/non-string slot inside `sources` (observed in the live store). */
|
|
36
|
+
sourcesCoerced: number
|
|
37
|
+
/** Embedding rows that were not arrays of finite numbers — unusable. */
|
|
38
|
+
embeddingsDropped: number
|
|
39
|
+
/** Rows whose length differs from the profile's modal dimension. Kept on
|
|
40
|
+
* disk, but excluded from centroids: sherpa compares by cosine, and a
|
|
41
|
+
* short row silently produces NaN across the whole averaged vector. */
|
|
42
|
+
dimensionMismatch: number
|
|
43
|
+
/** Entries with no usable name, or duplicate names collapsed. */
|
|
44
|
+
profilesDropped: number
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function emptyRepairs(): StoreRepairs {
|
|
48
|
+
return {
|
|
49
|
+
sourcesRealigned: 0,
|
|
50
|
+
sourcesCoerced: 0,
|
|
51
|
+
embeddingsDropped: 0,
|
|
52
|
+
dimensionMismatch: 0,
|
|
53
|
+
profilesDropped: 0,
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function hasRepairs(r: StoreRepairs): boolean {
|
|
58
|
+
return r.sourcesRealigned > 0 || r.sourcesCoerced > 0 || r.embeddingsDropped > 0
|
|
59
|
+
|| r.dimensionMismatch > 0 || r.profilesDropped > 0
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function describeRepairs(r: StoreRepairs): string {
|
|
63
|
+
const parts: string[] = []
|
|
64
|
+
if (r.profilesDropped) parts.push(`${r.profilesDropped} unusable profile(s)`)
|
|
65
|
+
if (r.embeddingsDropped) parts.push(`${r.embeddingsDropped} unusable embedding row(s)`)
|
|
66
|
+
if (r.sourcesRealigned) parts.push(`${r.sourcesRealigned} profile(s) with misaligned sources[]`)
|
|
67
|
+
if (r.sourcesCoerced) parts.push(`${r.sourcesCoerced} null/non-string source slot(s)`)
|
|
68
|
+
if (r.dimensionMismatch) parts.push(`${r.dimensionMismatch} wrong-dimension embedding(s) excluded from centroids`)
|
|
69
|
+
return parts.join(', ')
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const UNKNOWN_SOURCE = 'unknown'
|
|
73
|
+
|
|
74
|
+
function isUsableEmbedding(row: unknown): row is number[] {
|
|
75
|
+
return Array.isArray(row) && row.length > 0 && row.every(v => typeof v === 'number' && Number.isFinite(v))
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The dimension the majority of a profile's rows agree on.
|
|
79
|
+
*
|
|
80
|
+
* Not `embeddings[0].length`: if row 0 happens to be the corrupt one, every
|
|
81
|
+
* other row becomes the "mismatch" and the profile is emptied. */
|
|
82
|
+
export function modalDimension(embeddings: number[][]): number {
|
|
83
|
+
const counts = new Map<number, number>()
|
|
84
|
+
for (const row of embeddings) counts.set(row.length, (counts.get(row.length) ?? 0) + 1)
|
|
85
|
+
let best = 0, bestCount = -1
|
|
86
|
+
for (const [dim, count] of counts) {
|
|
87
|
+
// Tie-break toward the larger dimension: a truncated row is the likelier
|
|
88
|
+
// corruption than a systematically longer one.
|
|
89
|
+
if (count > bestCount || (count === bestCount && dim > best)) { best = dim; bestCount = count }
|
|
90
|
+
}
|
|
91
|
+
return best
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Bring a parsed store back to its invariants, reporting every change.
|
|
95
|
+
*
|
|
96
|
+
* Deliberately conservative: rows that are merely the wrong DIMENSION are kept
|
|
97
|
+
* on disk and only excluded from centroid math, because they may be a
|
|
98
|
+
* recoverable model change rather than corruption. Only rows that cannot be
|
|
99
|
+
* numbers at all are dropped. */
|
|
100
|
+
export function normalizeProfileStore(raw: unknown): { store: ProfileStore; repairs: StoreRepairs } {
|
|
101
|
+
const repairs = emptyRepairs()
|
|
102
|
+
const rawProfiles = (raw as ProfileStore | null)?.profiles
|
|
103
|
+
if (!Array.isArray(rawProfiles)) return { store: { profiles: [] }, repairs }
|
|
104
|
+
|
|
105
|
+
const byName = new Map<string, VoiceProfile>()
|
|
106
|
+
for (const candidate of rawProfiles) {
|
|
107
|
+
const name = typeof (candidate as VoiceProfile)?.name === 'string'
|
|
108
|
+
? (candidate as VoiceProfile).name.trim()
|
|
109
|
+
: ''
|
|
110
|
+
if (!name) { repairs.profilesDropped++; continue }
|
|
111
|
+
|
|
112
|
+
const rawEmbeddings = Array.isArray((candidate as VoiceProfile).embeddings)
|
|
113
|
+
? (candidate as VoiceProfile).embeddings as unknown[]
|
|
114
|
+
: []
|
|
115
|
+
const rawSources = Array.isArray((candidate as VoiceProfile).sources)
|
|
116
|
+
? (candidate as VoiceProfile).sources as unknown[]
|
|
117
|
+
: null
|
|
118
|
+
|
|
119
|
+
// Walk both arrays together so dropping row i also drops source i — the
|
|
120
|
+
// exact coupling `embeddings.shift()` + `sources?.shift()` used to break.
|
|
121
|
+
const embeddings: number[][] = []
|
|
122
|
+
const sources: string[] = []
|
|
123
|
+
for (let i = 0; i < rawEmbeddings.length; i++) {
|
|
124
|
+
const row = rawEmbeddings[i]
|
|
125
|
+
if (!isUsableEmbedding(row)) { repairs.embeddingsDropped++; continue }
|
|
126
|
+
embeddings.push(row)
|
|
127
|
+
const source = rawSources?.[i]
|
|
128
|
+
if (typeof source === 'string' && source.length > 0) {
|
|
129
|
+
sources.push(source)
|
|
130
|
+
} else {
|
|
131
|
+
// Missing (sources shorter than embeddings) or a null/non-string slot.
|
|
132
|
+
if (rawSources && i < rawSources.length) repairs.sourcesCoerced++
|
|
133
|
+
sources.push(UNKNOWN_SOURCE)
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const wasMisaligned = rawSources === null
|
|
138
|
+
? embeddings.length > 0
|
|
139
|
+
: rawSources.length !== rawEmbeddings.length
|
|
140
|
+
if (wasMisaligned) repairs.sourcesRealigned++
|
|
141
|
+
|
|
142
|
+
const dim = modalDimension(embeddings)
|
|
143
|
+
for (const row of embeddings) if (row.length !== dim) repairs.dimensionMismatch++
|
|
144
|
+
|
|
145
|
+
const existing = byName.get(name)
|
|
146
|
+
if (existing) {
|
|
147
|
+
// Duplicate profile names cannot both be registered in the manager (one
|
|
148
|
+
// vector per name), so the later rows would be invisible. Merge instead.
|
|
149
|
+
repairs.profilesDropped++
|
|
150
|
+
existing.embeddings.push(...embeddings)
|
|
151
|
+
existing.sources!.push(...sources)
|
|
152
|
+
continue
|
|
153
|
+
}
|
|
154
|
+
byName.set(name, { name, embeddings, sources })
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return { store: { profiles: [...byName.values()] }, repairs }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export type StoreLoad = {
|
|
161
|
+
store: ProfileStore
|
|
162
|
+
repairs: StoreRepairs
|
|
163
|
+
/** 'missing' is normal on a fresh install. 'corrupt' means the previous file
|
|
164
|
+
* was quarantined and the training in it is GONE unless a backup is used. */
|
|
165
|
+
status: 'ok' | 'missing' | 'corrupt'
|
|
166
|
+
quarantinedAs?: string
|
|
167
|
+
/** Set when `status: 'corrupt'` and a backup was successfully substituted. */
|
|
168
|
+
recoveredFromBackup?: string
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function backupDirFor(storePath: string): string {
|
|
172
|
+
return join(dirname(storePath), `${basename(storePath, '.json')}.backups`)
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Newest first. */
|
|
176
|
+
export function listProfileStoreBackups(storePath: string): string[] {
|
|
177
|
+
const dir = backupDirFor(storePath)
|
|
178
|
+
if (!existsSync(dir)) return []
|
|
179
|
+
try {
|
|
180
|
+
return readdirSync(dir)
|
|
181
|
+
.filter(f => f.endsWith('.json'))
|
|
182
|
+
.map(f => resolve(dir, f))
|
|
183
|
+
.map(p => ({ p, mtime: safeMtime(p) }))
|
|
184
|
+
.sort((a, b) => b.mtime - a.mtime)
|
|
185
|
+
.map(entry => entry.p)
|
|
186
|
+
} catch { return [] }
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function safeMtime(path: string): number {
|
|
190
|
+
try { return statSync(path).mtimeMs } catch { return 0 }
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function safeSize(path: string): number {
|
|
194
|
+
try { return statSync(path).size } catch { return 0 }
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export const PROFILE_STORE_BACKUP_KEEP = 5
|
|
198
|
+
/** Don't copy 7.8 MB on every enroll; one snapshot per hour bounds the churn
|
|
199
|
+
* while still guaranteeing a recent restore point. */
|
|
200
|
+
export const PROFILE_STORE_BACKUP_MIN_INTERVAL_MS = 60 * 60 * 1000
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Load the store, quarantining a corrupt file and falling back to the newest
|
|
204
|
+
* usable backup rather than silently starting from zero profiles.
|
|
205
|
+
*
|
|
206
|
+
* Starting empty is the dangerous default here: the very next `saveProfileStore`
|
|
207
|
+
* would durably persist that empty store over the only remaining copy.
|
|
208
|
+
*/
|
|
209
|
+
export function loadVoiceProfileStore(storePath: string): StoreLoad {
|
|
210
|
+
const result = loadJsonOrQuarantine<unknown>(storePath)
|
|
211
|
+
if (result.status === 'missing') {
|
|
212
|
+
return { store: { profiles: [] }, repairs: emptyRepairs(), status: 'missing' }
|
|
213
|
+
}
|
|
214
|
+
if (result.status === 'ok') {
|
|
215
|
+
const { store, repairs } = normalizeProfileStore(result.data)
|
|
216
|
+
return { store, repairs, status: 'ok' }
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
for (const backup of listProfileStoreBackups(storePath)) {
|
|
220
|
+
const restored = loadJsonOrQuarantine<unknown>(backup)
|
|
221
|
+
if (restored.status !== 'ok') continue
|
|
222
|
+
const { store, repairs } = normalizeProfileStore(restored.data)
|
|
223
|
+
if (store.profiles.length === 0) continue
|
|
224
|
+
return {
|
|
225
|
+
store,
|
|
226
|
+
repairs,
|
|
227
|
+
status: 'corrupt',
|
|
228
|
+
quarantinedAs: result.quarantinedAs,
|
|
229
|
+
recoveredFromBackup: backup,
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return {
|
|
233
|
+
store: { profiles: [] },
|
|
234
|
+
repairs: emptyRepairs(),
|
|
235
|
+
status: 'corrupt',
|
|
236
|
+
quarantinedAs: result.quarantinedAs,
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Snapshot the current file before it is overwritten, then prune old copies. */
|
|
241
|
+
export function backupProfileStore(
|
|
242
|
+
storePath: string,
|
|
243
|
+
options: { nowMs?: number; force?: boolean; keep?: number; minIntervalMs?: number } = {},
|
|
244
|
+
): string | null {
|
|
245
|
+
if (!existsSync(storePath)) return null
|
|
246
|
+
if (safeSize(storePath) === 0) return null
|
|
247
|
+
|
|
248
|
+
const now = options.nowMs ?? Date.now()
|
|
249
|
+
const keep = options.keep ?? PROFILE_STORE_BACKUP_KEEP
|
|
250
|
+
const minInterval = options.minIntervalMs ?? PROFILE_STORE_BACKUP_MIN_INTERVAL_MS
|
|
251
|
+
const existing = listProfileStoreBackups(storePath)
|
|
252
|
+
if (!options.force && existing.length > 0 && now - safeMtime(existing[0]) < minInterval) return null
|
|
253
|
+
|
|
254
|
+
const dir = backupDirFor(storePath)
|
|
255
|
+
try {
|
|
256
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 })
|
|
257
|
+
const stamp = new Date(now).toISOString().replace(/[:.]/g, '-')
|
|
258
|
+
const target = join(dir, `${basename(storePath, '.json')}.${stamp}.json`)
|
|
259
|
+
// Copy through the durable writer so a backup can never itself be torn.
|
|
260
|
+
const raw = loadJsonOrQuarantine<unknown>(storePath)
|
|
261
|
+
if (raw.status !== 'ok') return null
|
|
262
|
+
durableAtomicWriteFileSync(target, JSON.stringify(raw.data), { mode: 0o600 })
|
|
263
|
+
for (const stale of listProfileStoreBackups(storePath).slice(keep)) {
|
|
264
|
+
try { unlinkSync(stale) } catch { /* pruning is best effort */ }
|
|
265
|
+
}
|
|
266
|
+
return target
|
|
267
|
+
} catch {
|
|
268
|
+
// A failed backup must never block the write it precedes.
|
|
269
|
+
return null
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Persist the store atomically, after taking a rotating backup.
|
|
275
|
+
*
|
|
276
|
+
* Refuses to write an empty store over a populated file. That is the shape of
|
|
277
|
+
* every catastrophic loss here — a failed load returning `{profiles: []}`, then
|
|
278
|
+
* a routine save committing it — and no legitimate caller needs it. Callers
|
|
279
|
+
* that really mean it (a full reset) pass `allowEmpty`.
|
|
280
|
+
*/
|
|
281
|
+
export function saveVoiceProfileStore(
|
|
282
|
+
storePath: string,
|
|
283
|
+
store: ProfileStore,
|
|
284
|
+
options: { allowEmpty?: boolean; nowMs?: number } = {},
|
|
285
|
+
): { written: boolean; backup: string | null; refusedReason?: string } {
|
|
286
|
+
if (store.profiles.length === 0 && !options.allowEmpty) {
|
|
287
|
+
const current = loadJsonOrQuarantine<ProfileStore>(storePath)
|
|
288
|
+
if (current.status === 'ok' && (current.data?.profiles?.length ?? 0) > 0) {
|
|
289
|
+
return {
|
|
290
|
+
written: false,
|
|
291
|
+
backup: null,
|
|
292
|
+
refusedReason: `refusing to overwrite ${current.data.profiles.length} stored profile(s) with an empty store`,
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const backup = backupProfileStore(storePath, { nowMs: options.nowMs })
|
|
298
|
+
mkdirSync(dirname(storePath), { recursive: true, mode: 0o700 })
|
|
299
|
+
durableAtomicWriteFileSync(storePath, JSON.stringify(store, null, 2), { mode: 0o600 })
|
|
300
|
+
return { written: true, backup }
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** Drop the oldest sample, keeping `embeddings` and `sources` in lockstep.
|
|
304
|
+
*
|
|
305
|
+
* The old inline form was `embeddings.shift()` followed by `sources?.shift()`,
|
|
306
|
+
* which silently skipped the second half whenever `sources` was undefined and
|
|
307
|
+
* desynchronised provenance from that point on — permanently, since every
|
|
308
|
+
* later push appended to both. */
|
|
309
|
+
export function dropOldestEmbedding(profile: VoiceProfile): { droppedSource: string | null } {
|
|
310
|
+
if (profile.embeddings.length === 0) return { droppedSource: null }
|
|
311
|
+
profile.embeddings.shift()
|
|
312
|
+
if (!Array.isArray(profile.sources)) profile.sources = []
|
|
313
|
+
// Re-align first so a short sources[] cannot shift the wrong provenance off.
|
|
314
|
+
while (profile.sources.length < profile.embeddings.length + 1) profile.sources.push(UNKNOWN_SOURCE)
|
|
315
|
+
const droppedSource = profile.sources.shift() ?? null
|
|
316
|
+
profile.sources.length = profile.embeddings.length
|
|
317
|
+
return { droppedSource }
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** Append a sample to both arrays together. */
|
|
321
|
+
export function appendEmbedding(profile: VoiceProfile, embedding: number[], source: string): void {
|
|
322
|
+
if (!Array.isArray(profile.sources)) profile.sources = []
|
|
323
|
+
while (profile.sources.length < profile.embeddings.length) profile.sources.push(UNKNOWN_SOURCE)
|
|
324
|
+
profile.embeddings.push(embedding)
|
|
325
|
+
profile.sources.push(source)
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Remove every sample whose provenance matches, e.g. one poisoned session.
|
|
329
|
+
* Returns the number removed so a retraction can be reported rather than
|
|
330
|
+
* assumed. */
|
|
331
|
+
export function removeEmbeddingsBySource(
|
|
332
|
+
profile: VoiceProfile,
|
|
333
|
+
predicate: (source: string) => boolean,
|
|
334
|
+
): number {
|
|
335
|
+
if (!Array.isArray(profile.sources)) profile.sources = []
|
|
336
|
+
while (profile.sources.length < profile.embeddings.length) profile.sources.push(UNKNOWN_SOURCE)
|
|
337
|
+
const keptEmbeddings: number[][] = []
|
|
338
|
+
const keptSources: string[] = []
|
|
339
|
+
let removed = 0
|
|
340
|
+
for (let i = 0; i < profile.embeddings.length; i++) {
|
|
341
|
+
if (predicate(profile.sources[i] ?? UNKNOWN_SOURCE)) { removed++; continue }
|
|
342
|
+
keptEmbeddings.push(profile.embeddings[i])
|
|
343
|
+
keptSources.push(profile.sources[i] ?? UNKNOWN_SOURCE)
|
|
344
|
+
}
|
|
345
|
+
profile.embeddings = keptEmbeddings
|
|
346
|
+
profile.sources = keptSources
|
|
347
|
+
return removed
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/** Delete a person's profile outright. Returns what was removed so the caller
|
|
351
|
+
* can report a per-store count instead of a bare success. */
|
|
352
|
+
export function deleteProfileFromStore(
|
|
353
|
+
store: ProfileStore,
|
|
354
|
+
name: string,
|
|
355
|
+
): { removedProfiles: number; removedEmbeddings: number } {
|
|
356
|
+
let removedProfiles = 0
|
|
357
|
+
let removedEmbeddings = 0
|
|
358
|
+
store.profiles = store.profiles.filter(profile => {
|
|
359
|
+
if (profile.name !== name) return true
|
|
360
|
+
removedProfiles++
|
|
361
|
+
removedEmbeddings += profile.embeddings.length
|
|
362
|
+
return false
|
|
363
|
+
})
|
|
364
|
+
return { removedProfiles, removedEmbeddings }
|
|
365
|
+
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import { spawn, execFile } from 'node:child_process'
|
|
10
10
|
import type { ChildProcess } from 'node:child_process'
|
|
11
|
-
import { writeFileSync, unlinkSync, existsSync, readFileSync } from 'node:fs'
|
|
11
|
+
import { writeFileSync, unlinkSync, existsSync, readFileSync, statSync } from 'node:fs'
|
|
12
12
|
import { basename, join } from 'node:path'
|
|
13
13
|
import { homedir } from 'node:os'
|
|
14
14
|
import crypto from 'node:crypto'
|
|
@@ -74,6 +74,37 @@ export interface HighQualityTranscriptionResult {
|
|
|
74
74
|
degradationReason?: HighQualityUnavailableReason
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
let highQualityCheckpointFingerprint: string | null = null
|
|
78
|
+
|
|
79
|
+
/** Stable, path-free cache identity for progressive meeting HQ checkpoints.
|
|
80
|
+
* Model/configuration is immutable for the lifetime of one server process, so
|
|
81
|
+
* cache this instead of stat'ing multi-GB model files on every health poll. */
|
|
82
|
+
export function getHighQualityCheckpointFingerprint(): string {
|
|
83
|
+
if (highQualityCheckpointFingerprint) return highQualityCheckpointFingerprint
|
|
84
|
+
const identity = (path: string): Record<string, number | boolean> => {
|
|
85
|
+
try {
|
|
86
|
+
const stat = statSync(path)
|
|
87
|
+
return { present: true, size: stat.size, mtimeMs: Math.floor(stat.mtimeMs) }
|
|
88
|
+
} catch {
|
|
89
|
+
return { present: false, size: 0, mtimeMs: 0 }
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
highQualityCheckpointFingerprint = crypto.createHash('sha256').update(JSON.stringify({
|
|
93
|
+
schema: 2,
|
|
94
|
+
serverVersion: process.env.COS_SERVER_VERSION?.trim() || 'development',
|
|
95
|
+
whisperCli: identity(WHISPER_CLI),
|
|
96
|
+
model: identity(resolveBatchModel()),
|
|
97
|
+
vad: identity(VAD_MODEL_PATH),
|
|
98
|
+
beam: 5,
|
|
99
|
+
bestOf: 5,
|
|
100
|
+
vadEnabled: hqCliVadEnabled('batch'),
|
|
101
|
+
enhancement: 'audio-enhance-v1',
|
|
102
|
+
prompt: buildWhisperPrompt(),
|
|
103
|
+
corrections: getWhisperCorrections(),
|
|
104
|
+
})).digest('hex')
|
|
105
|
+
return highQualityCheckpointFingerprint
|
|
106
|
+
}
|
|
107
|
+
|
|
77
108
|
export function classifyHighQualityTranscriptionCapability(input: {
|
|
78
109
|
enabled: boolean
|
|
79
110
|
cliPresent: boolean
|
|
@@ -806,7 +837,14 @@ export async function transcribeHighQuality(
|
|
|
806
837
|
/** forceCpu: the batch pipeline's one CPU retry after a Metal preempt. It
|
|
807
838
|
* bypasses the gate entirely so the retry cannot itself be preempted into
|
|
808
839
|
* an infinite loop. */
|
|
809
|
-
opts: {
|
|
840
|
+
opts: {
|
|
841
|
+
priority?: 'interactive' | 'batch'
|
|
842
|
+
forceCpu?: boolean
|
|
843
|
+
forceCpuReason?: string
|
|
844
|
+
threads?: number
|
|
845
|
+
backgroundCpu?: boolean
|
|
846
|
+
signal?: AbortSignal
|
|
847
|
+
} = {},
|
|
810
848
|
): Promise<HighQualityTranscriptionResult> {
|
|
811
849
|
if (!cliAvailable) {
|
|
812
850
|
// Fall back to server (no beam search available via HTTP API)
|
|
@@ -844,7 +882,7 @@ export async function transcribeHighQuality(
|
|
|
844
882
|
const useVad = hqCliVadEnabled(opts.priority)
|
|
845
883
|
const decision: { device: 'metal' | 'cpu'; reason: string; metalEnabled: boolean } = isBatch
|
|
846
884
|
? (opts.forceCpu
|
|
847
|
-
? { device: 'cpu', reason: 'preempt_retry', metalEnabled: batchHqMetalEnabled() }
|
|
885
|
+
? { device: 'cpu', reason: opts.forceCpuReason || 'preempt_retry', metalEnabled: batchHqMetalEnabled() }
|
|
848
886
|
: chooseBatchDevice())
|
|
849
887
|
: { device: 'metal', reason: 'interactive', metalEnabled: batchHqMetalEnabled() }
|
|
850
888
|
const useMetal = decision.device === 'metal'
|
|
@@ -858,11 +896,14 @@ export async function transcribeHighQuality(
|
|
|
858
896
|
: 2
|
|
859
897
|
const beam = isBatch ? 5 : interactiveBeam
|
|
860
898
|
const bestOf = beam
|
|
899
|
+
const requestedThreads = Number.isFinite(opts.threads)
|
|
900
|
+
? Math.max(1, Math.min(16, Math.floor(opts.threads!)))
|
|
901
|
+
: 8
|
|
861
902
|
const args = [
|
|
862
903
|
'-m', modelPath,
|
|
863
904
|
'-f', tmpWav,
|
|
864
905
|
// CPU batch stays at 8 threads so it cannot starve live work of cores.
|
|
865
|
-
'-t', (isBatch && !useMetal) ?
|
|
906
|
+
'-t', (isBatch && !useMetal) ? String(requestedThreads) : '16',
|
|
866
907
|
'-l', 'en',
|
|
867
908
|
...(useMetal ? ['-fa'] : ['-ng']),
|
|
868
909
|
'-bs', String(beam),
|
|
@@ -879,9 +920,18 @@ export async function transcribeHighQuality(
|
|
|
879
920
|
// omits this — VAD was measured dropping real leading speech on compose.
|
|
880
921
|
args.push('--vad', '--vad-model', VAD_MODEL_PATH)
|
|
881
922
|
}
|
|
882
|
-
const
|
|
923
|
+
const useBackgroundTaskPolicy = Boolean(
|
|
924
|
+
isBatch && !useMetal && opts.backgroundCpu
|
|
925
|
+
&& process.platform === 'darwin'
|
|
926
|
+
&& existsSync('/usr/sbin/taskpolicy'),
|
|
927
|
+
)
|
|
928
|
+
const proc = spawn(
|
|
929
|
+
useBackgroundTaskPolicy ? '/usr/sbin/taskpolicy' : WHISPER_CLI,
|
|
930
|
+
useBackgroundTaskPolicy ? ['-b', WHISPER_CLI, ...args] : args,
|
|
931
|
+
{
|
|
883
932
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
884
|
-
|
|
933
|
+
},
|
|
934
|
+
)
|
|
885
935
|
ownedHqChildren.add(proc)
|
|
886
936
|
|
|
887
937
|
// BLOCKER contract: a preempted Metal child is a HARD FAIL. Its stdout
|
|
@@ -896,6 +946,17 @@ export async function transcribeHighQuality(
|
|
|
896
946
|
|
|
897
947
|
let stdout = ''
|
|
898
948
|
let stderr = ''
|
|
949
|
+
let aborted = false
|
|
950
|
+
let abortForceKill: ReturnType<typeof setTimeout> | null = null
|
|
951
|
+
const onAbort = (): void => {
|
|
952
|
+
aborted = true
|
|
953
|
+
try { proc.kill('SIGTERM') } catch { /* already exited */ }
|
|
954
|
+
abortForceKill = setTimeout(() => {
|
|
955
|
+
try { proc.kill('SIGKILL') } catch { /* already exited */ }
|
|
956
|
+
}, 2_000)
|
|
957
|
+
}
|
|
958
|
+
if (opts.signal?.aborted) onAbort()
|
|
959
|
+
else opts.signal?.addEventListener('abort', onAbort, { once: true })
|
|
899
960
|
proc.stdout.on('data', (chunk: Buffer) => { stdout += chunk.toString() })
|
|
900
961
|
proc.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString() })
|
|
901
962
|
|
|
@@ -915,9 +976,11 @@ export async function transcribeHighQuality(
|
|
|
915
976
|
}, timeoutMs)
|
|
916
977
|
|
|
917
978
|
proc.on('close', (code) => {
|
|
979
|
+
opts.signal?.removeEventListener('abort', onAbort)
|
|
918
980
|
ownedHqChildren.delete(proc)
|
|
919
981
|
unregisterMetalBatchChild(proc)
|
|
920
982
|
clearTimeout(timeout)
|
|
983
|
+
if (abortForceKill) clearTimeout(abortForceKill)
|
|
921
984
|
if (forceKill) clearTimeout(forceKill)
|
|
922
985
|
// Preempt is checked FIRST and ignores the exit code: SIGTERM often
|
|
923
986
|
// yields a non-zero code, but a race could also let the child exit 0
|
|
@@ -926,6 +989,12 @@ export async function transcribeHighQuality(
|
|
|
926
989
|
reject(new MetalBatchPreemptedError(preemptedReason))
|
|
927
990
|
return
|
|
928
991
|
}
|
|
992
|
+
if (aborted) {
|
|
993
|
+
const error = new Error('Progressive HQ checkpoint aborted')
|
|
994
|
+
error.name = 'AbortError'
|
|
995
|
+
reject(error)
|
|
996
|
+
return
|
|
997
|
+
}
|
|
929
998
|
if (timedOut) {
|
|
930
999
|
reject(new Error(`whisper-cli HQ timeout (${timeoutMs / 1000}s, model=${useLargeV3 ? 'large-v3' : 'turbo'})`))
|
|
931
1000
|
return
|
|
@@ -938,9 +1007,11 @@ export async function transcribeHighQuality(
|
|
|
938
1007
|
})
|
|
939
1008
|
|
|
940
1009
|
proc.on('error', (err) => {
|
|
1010
|
+
opts.signal?.removeEventListener('abort', onAbort)
|
|
941
1011
|
ownedHqChildren.delete(proc)
|
|
942
1012
|
unregisterMetalBatchChild(proc)
|
|
943
1013
|
clearTimeout(timeout)
|
|
1014
|
+
if (abortForceKill) clearTimeout(abortForceKill)
|
|
944
1015
|
if (forceKill) clearTimeout(forceKill)
|
|
945
1016
|
reject(new Error(`whisper-cli HQ spawn error: ${err.message}`))
|
|
946
1017
|
})
|
package/server/routes/health.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { serverMetrics } from '../lib/server-metrics.js'
|
|
|
6
6
|
import { getServerInstanceId } from '../lib/server-instance-id.js'
|
|
7
7
|
import { localFirstMeetingsCapability } from '../lib/local-first-meetings-contract.js'
|
|
8
8
|
import { isSileroAvailable } from '../lib/vad-silero.js'
|
|
9
|
-
import { speakerModelState } from '../lib/speaker-embeddings.js'
|
|
9
|
+
import { speakerModelState, speakerReadiness } from '../lib/speaker-embeddings.js'
|
|
10
10
|
import { getAvailableCliSessionId } from '../lib/claude-bridge.js'
|
|
11
11
|
import {
|
|
12
12
|
isWhisperLocalAvailable,
|
|
@@ -40,6 +40,9 @@ import { listUnsavedCaptures } from '../lib/unsaved-audio-quarantine.js'
|
|
|
40
40
|
import { getWhisperPreviewCapability } from '../lib/whisper-preview.js'
|
|
41
41
|
import { getTranscriptionProfileStatus } from '../lib/profile.js'
|
|
42
42
|
import { getHealthStaticProbes } from '../lib/health-static-probes.js'
|
|
43
|
+
import { getEarlyMeetingSyncSnapshot } from '../lib/g2-ops-handoff.js'
|
|
44
|
+
import { getProgressiveHqSnapshot } from '../lib/meeting-batch-transcribe.js'
|
|
45
|
+
import { getMeetingFinalizationSnapshot } from '../lib/meeting-finalization-jobs.js'
|
|
43
46
|
|
|
44
47
|
export const healthRouter = Router()
|
|
45
48
|
|
|
@@ -110,7 +113,8 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
110
113
|
// npm tarball), so publish its state rather than letting the amplitude
|
|
111
114
|
// fallback masquerade as working diarization. Availability only — the resolved
|
|
112
115
|
// path is a local filesystem detail and health is unauthenticated.
|
|
113
|
-
|
|
116
|
+
const speakerId = speakerModelState()
|
|
117
|
+
checks.speaker_id = speakerId.state
|
|
114
118
|
|
|
115
119
|
// Health is unauthenticated. Publish only availability; the actual CLI
|
|
116
120
|
// session id is a resumable runtime handle and belongs on authenticated
|
|
@@ -173,15 +177,22 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
173
177
|
: whisper_health.startupState === 'preflight' || whisper_health.startupState === 'loading'
|
|
174
178
|
? 'starting'
|
|
175
179
|
: 'degraded'
|
|
180
|
+
// A configured-but-broken voiceprint model is a fault, not a preference. The
|
|
181
|
+
// full reasoning, including why 'unavailable' must NOT degrade, lives with
|
|
182
|
+
// speakerReadiness() so it is testable rather than an inline ternary here.
|
|
183
|
+
const speakerReadinessState = speakerReadiness(speakerId.state)
|
|
176
184
|
const readiness = {
|
|
177
185
|
// /api/health remains a liveness endpoint and intentionally returns HTTP
|
|
178
186
|
// 200 while the server can answer. This separate field prevents an HTTP-
|
|
179
187
|
// green response from hiding a configured local subsystem failure.
|
|
180
|
-
status: whisperReadiness === 'degraded' ? 'degraded' : 'ready',
|
|
188
|
+
status: whisperReadiness === 'degraded' || speakerReadinessState === 'degraded' ? 'degraded' : 'ready',
|
|
181
189
|
admissions: maintenance.admissionsOpen ? 'open' : 'maintenance',
|
|
182
190
|
whisper: whisperReadiness,
|
|
183
191
|
whisperError: whisper_health.lastError,
|
|
184
192
|
localTts: tts_local.ready ? 'ready' : 'unavailable',
|
|
193
|
+
// Asserted, not merely reported: `speaker_id` above says what the state IS,
|
|
194
|
+
// this says whether that state is acceptable.
|
|
195
|
+
speakerId: speakerReadinessState,
|
|
185
196
|
}
|
|
186
197
|
const openai_whisper_budget = getOpenAIWhisperBudgetState()
|
|
187
198
|
const codex_models = getCodexModelCatalogSnapshot()
|
|
@@ -190,6 +201,7 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
190
201
|
const cursorSnapshot = getCursorModelCatalogSnapshot()
|
|
191
202
|
const { agentBinary: _cursorAgentBinary, ...cursor_models } = cursorSnapshot
|
|
192
203
|
const meeting_sync = getMeetingSyncSnapshot()
|
|
204
|
+
const progressiveHq = getProgressiveHqSnapshot()
|
|
193
205
|
// Quarantined unsaved captures (6.19.0). Compact on this unauthenticated
|
|
194
206
|
// surface — same exposure level as meeting_sync's meetingIds. Full detail
|
|
195
207
|
// plus the recover action live on the authenticated /api/meeting/orphans.
|
|
@@ -235,6 +247,11 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
235
247
|
},
|
|
236
248
|
cliDebug: CLI_DEBUG_CAPABILITY,
|
|
237
249
|
liveCues,
|
|
250
|
+
meetingLifecycle: {
|
|
251
|
+
earlySyncClaim: getEarlyMeetingSyncSnapshot(),
|
|
252
|
+
progressiveHq,
|
|
253
|
+
finalization: getMeetingFinalizationSnapshot(),
|
|
254
|
+
},
|
|
238
255
|
...(localFirstMeetings ? { localFirstMeetings } : {}),
|
|
239
256
|
},
|
|
240
257
|
// /api/health is intentionally unauthenticated for setup diagnostics.
|
|
@@ -261,6 +278,7 @@ healthRouter.get('/models', async (req, res) => {
|
|
|
261
278
|
const transcriptionHq = getHighQualityTranscriptionCapability()
|
|
262
279
|
const transcriptionLive = getWhisperPreviewCapability()
|
|
263
280
|
const transcriptionProfile = getTranscriptionProfileStatus()
|
|
281
|
+
const progressiveHq = getProgressiveHqSnapshot()
|
|
264
282
|
const cursorOptions = cursorCatalog.options.filter(option => !!option.id)
|
|
265
283
|
res.json({
|
|
266
284
|
...catalog,
|
|
@@ -282,6 +300,11 @@ healthRouter.get('/models', async (req, res) => {
|
|
|
282
300
|
hq: transcriptionHq,
|
|
283
301
|
profile: transcriptionProfile,
|
|
284
302
|
},
|
|
303
|
+
meetingLifecycle: {
|
|
304
|
+
earlySyncClaim: getEarlyMeetingSyncSnapshot(),
|
|
305
|
+
progressiveHq,
|
|
306
|
+
finalization: getMeetingFinalizationSnapshot(),
|
|
307
|
+
},
|
|
285
308
|
cliDebug: CLI_DEBUG_CAPABILITY,
|
|
286
309
|
recovery: managedRuntimeCapability(),
|
|
287
310
|
// Same helper as /api/health — the companion's 15s liveness poll reads
|