@gotcos/glasses-server 6.21.9 → 6.21.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +77 -0
- package/package.json +1 -1
- package/server/lib/audio-retention.ts +50 -0
- package/server/lib/meeting-speaker-review.ts +268 -0
- package/server/lib/meeting-store.ts +57 -2
- package/server/lib/speaker-calibration-log.ts +146 -0
- package/server/lib/speaker-embeddings.ts +266 -36
- package/server/lib/voice-profile-store.ts +507 -0
- package/server/routes/health.ts +11 -3
- package/server/routes/meeting.ts +56 -1
- package/server/routes/transcribe-stream.ts +46 -0
- package/server/routes/voice.ts +347 -22
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
// Durability and integrity for voice-profiles.json.
|
|
2
|
+
//
|
|
3
|
+
// This file is the single most irreplaceable artifact the server owns: on this
|
|
4
|
+
// box it holds 79 profiles / 1,317 embeddings / ~7.8 MB, accumulated over months
|
|
5
|
+
// of training that cannot be re-derived once the source audio has aged out. It
|
|
6
|
+
// was being written with three raw `writeFileSync` calls and read with an
|
|
7
|
+
// unguarded `JSON.parse`, so one crash, disk-full, or iCloud-sync mid-write
|
|
8
|
+
// would take all of it and every subsequent read would throw.
|
|
9
|
+
//
|
|
10
|
+
// Everything here is a pure function over an explicit path so it can be tested
|
|
11
|
+
// by execution rather than by asserting on source text. speaker-embeddings.ts
|
|
12
|
+
// delegates to it and keeps the sherpa-onnx manager in sync.
|
|
13
|
+
|
|
14
|
+
import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from 'node:fs'
|
|
15
|
+
import { basename, dirname, join, resolve } from 'node:path'
|
|
16
|
+
import { durableAtomicWriteFileSync, loadJsonOrQuarantine } from './atomic-fs.js'
|
|
17
|
+
|
|
18
|
+
export interface VoiceProfile {
|
|
19
|
+
name: string
|
|
20
|
+
/** One row per enrolled sample. Parallel to `sources`, index for index. */
|
|
21
|
+
embeddings: number[][]
|
|
22
|
+
/** Provenance: 'manual' | 'fireflies' | 'g2-training' | 'auto:<sessionId>' | … */
|
|
23
|
+
sources?: string[]
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface ProfileStore {
|
|
27
|
+
profiles: VoiceProfile[]
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** What normalization had to fix. All non-zero counts are worth logging: they
|
|
31
|
+
* mean the file on disk had drifted from its own invariants. */
|
|
32
|
+
export interface StoreRepairs {
|
|
33
|
+
/** `sources` was missing, short, or long relative to `embeddings`. */
|
|
34
|
+
sourcesRealigned: number
|
|
35
|
+
/** A null/non-string slot inside `sources` (observed in the live store). */
|
|
36
|
+
sourcesCoerced: number
|
|
37
|
+
/** Embedding rows that were not arrays of finite numbers — unusable. */
|
|
38
|
+
embeddingsDropped: number
|
|
39
|
+
/** Rows whose length differs from the profile's modal dimension. Kept on
|
|
40
|
+
* disk, but excluded from centroids: sherpa compares by cosine, and a
|
|
41
|
+
* short row silently produces NaN across the whole averaged vector. */
|
|
42
|
+
dimensionMismatch: number
|
|
43
|
+
/** Entries with no usable name, or duplicate names collapsed. */
|
|
44
|
+
profilesDropped: number
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function emptyRepairs(): StoreRepairs {
|
|
48
|
+
return {
|
|
49
|
+
sourcesRealigned: 0,
|
|
50
|
+
sourcesCoerced: 0,
|
|
51
|
+
embeddingsDropped: 0,
|
|
52
|
+
dimensionMismatch: 0,
|
|
53
|
+
profilesDropped: 0,
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function hasRepairs(r: StoreRepairs): boolean {
|
|
58
|
+
return r.sourcesRealigned > 0 || r.sourcesCoerced > 0 || r.embeddingsDropped > 0
|
|
59
|
+
|| r.dimensionMismatch > 0 || r.profilesDropped > 0
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function describeRepairs(r: StoreRepairs): string {
|
|
63
|
+
const parts: string[] = []
|
|
64
|
+
if (r.profilesDropped) parts.push(`${r.profilesDropped} unusable profile(s)`)
|
|
65
|
+
if (r.embeddingsDropped) parts.push(`${r.embeddingsDropped} unusable embedding row(s)`)
|
|
66
|
+
if (r.sourcesRealigned) parts.push(`${r.sourcesRealigned} profile(s) with misaligned sources[]`)
|
|
67
|
+
if (r.sourcesCoerced) parts.push(`${r.sourcesCoerced} null/non-string source slot(s)`)
|
|
68
|
+
if (r.dimensionMismatch) parts.push(`${r.dimensionMismatch} wrong-dimension embedding(s) excluded from centroids`)
|
|
69
|
+
return parts.join(', ')
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const UNKNOWN_SOURCE = 'unknown'
|
|
73
|
+
|
|
74
|
+
function isUsableEmbedding(row: unknown): row is number[] {
|
|
75
|
+
return Array.isArray(row) && row.length > 0 && row.every(v => typeof v === 'number' && Number.isFinite(v))
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The dimension the majority of a profile's rows agree on.
|
|
79
|
+
*
|
|
80
|
+
* Not `embeddings[0].length`: if row 0 happens to be the corrupt one, every
|
|
81
|
+
* other row becomes the "mismatch" and the profile is emptied. */
|
|
82
|
+
export function modalDimension(embeddings: number[][]): number {
|
|
83
|
+
const counts = new Map<number, number>()
|
|
84
|
+
for (const row of embeddings) counts.set(row.length, (counts.get(row.length) ?? 0) + 1)
|
|
85
|
+
let best = 0, bestCount = -1
|
|
86
|
+
for (const [dim, count] of counts) {
|
|
87
|
+
// Tie-break toward the larger dimension: a truncated row is the likelier
|
|
88
|
+
// corruption than a systematically longer one.
|
|
89
|
+
if (count > bestCount || (count === bestCount && dim > best)) { best = dim; bestCount = count }
|
|
90
|
+
}
|
|
91
|
+
return best
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Bring a parsed store back to its invariants, reporting every change.
|
|
95
|
+
*
|
|
96
|
+
* Deliberately conservative: rows that are merely the wrong DIMENSION are kept
|
|
97
|
+
* on disk and only excluded from centroid math, because they may be a
|
|
98
|
+
* recoverable model change rather than corruption. Only rows that cannot be
|
|
99
|
+
* numbers at all are dropped. */
|
|
100
|
+
export function normalizeProfileStore(raw: unknown): { store: ProfileStore; repairs: StoreRepairs } {
|
|
101
|
+
const repairs = emptyRepairs()
|
|
102
|
+
const rawProfiles = (raw as ProfileStore | null)?.profiles
|
|
103
|
+
if (!Array.isArray(rawProfiles)) return { store: { profiles: [] }, repairs }
|
|
104
|
+
|
|
105
|
+
const byName = new Map<string, VoiceProfile>()
|
|
106
|
+
for (const candidate of rawProfiles) {
|
|
107
|
+
const name = typeof (candidate as VoiceProfile)?.name === 'string'
|
|
108
|
+
? (candidate as VoiceProfile).name.trim()
|
|
109
|
+
: ''
|
|
110
|
+
if (!name) { repairs.profilesDropped++; continue }
|
|
111
|
+
|
|
112
|
+
const rawEmbeddings = Array.isArray((candidate as VoiceProfile).embeddings)
|
|
113
|
+
? (candidate as VoiceProfile).embeddings as unknown[]
|
|
114
|
+
: []
|
|
115
|
+
const rawSources = Array.isArray((candidate as VoiceProfile).sources)
|
|
116
|
+
? (candidate as VoiceProfile).sources as unknown[]
|
|
117
|
+
: null
|
|
118
|
+
|
|
119
|
+
// Walk both arrays together so dropping row i also drops source i — the
|
|
120
|
+
// exact coupling `embeddings.shift()` + `sources?.shift()` used to break.
|
|
121
|
+
const embeddings: number[][] = []
|
|
122
|
+
const sources: string[] = []
|
|
123
|
+
for (let i = 0; i < rawEmbeddings.length; i++) {
|
|
124
|
+
const row = rawEmbeddings[i]
|
|
125
|
+
if (!isUsableEmbedding(row)) { repairs.embeddingsDropped++; continue }
|
|
126
|
+
embeddings.push(row)
|
|
127
|
+
const source = rawSources?.[i]
|
|
128
|
+
if (typeof source === 'string' && source.length > 0) {
|
|
129
|
+
sources.push(source)
|
|
130
|
+
} else {
|
|
131
|
+
// Missing (sources shorter than embeddings) or a null/non-string slot.
|
|
132
|
+
if (rawSources && i < rawSources.length) repairs.sourcesCoerced++
|
|
133
|
+
sources.push(UNKNOWN_SOURCE)
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const wasMisaligned = rawSources === null
|
|
138
|
+
? embeddings.length > 0
|
|
139
|
+
: rawSources.length !== rawEmbeddings.length
|
|
140
|
+
if (wasMisaligned) repairs.sourcesRealigned++
|
|
141
|
+
|
|
142
|
+
const dim = modalDimension(embeddings)
|
|
143
|
+
for (const row of embeddings) if (row.length !== dim) repairs.dimensionMismatch++
|
|
144
|
+
|
|
145
|
+
const existing = byName.get(name)
|
|
146
|
+
if (existing) {
|
|
147
|
+
// Duplicate profile names cannot both be registered in the manager (one
|
|
148
|
+
// vector per name), so the later rows would be invisible. Merge instead.
|
|
149
|
+
repairs.profilesDropped++
|
|
150
|
+
existing.embeddings.push(...embeddings)
|
|
151
|
+
existing.sources!.push(...sources)
|
|
152
|
+
continue
|
|
153
|
+
}
|
|
154
|
+
byName.set(name, { name, embeddings, sources })
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return { store: { profiles: [...byName.values()] }, repairs }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export type StoreLoad = {
|
|
161
|
+
store: ProfileStore
|
|
162
|
+
repairs: StoreRepairs
|
|
163
|
+
/** 'missing' is normal on a fresh install. 'corrupt' means the previous file
|
|
164
|
+
* was quarantined and the training in it is GONE unless a backup is used. */
|
|
165
|
+
status: 'ok' | 'missing' | 'corrupt'
|
|
166
|
+
quarantinedAs?: string
|
|
167
|
+
/** Set when `status: 'corrupt'` and a backup was successfully substituted. */
|
|
168
|
+
recoveredFromBackup?: string
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function backupDirFor(storePath: string): string {
|
|
172
|
+
return join(dirname(storePath), `${basename(storePath, '.json')}.backups`)
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Newest first. */
|
|
176
|
+
export function listProfileStoreBackups(storePath: string): string[] {
|
|
177
|
+
const dir = backupDirFor(storePath)
|
|
178
|
+
if (!existsSync(dir)) return []
|
|
179
|
+
try {
|
|
180
|
+
return readdirSync(dir)
|
|
181
|
+
.filter(f => f.endsWith('.json'))
|
|
182
|
+
.map(f => resolve(dir, f))
|
|
183
|
+
.map(p => ({ p, mtime: safeMtime(p) }))
|
|
184
|
+
.sort((a, b) => b.mtime - a.mtime)
|
|
185
|
+
.map(entry => entry.p)
|
|
186
|
+
} catch { return [] }
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function safeMtime(path: string): number {
|
|
190
|
+
try { return statSync(path).mtimeMs } catch { return 0 }
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function safeSize(path: string): number {
|
|
194
|
+
try { return statSync(path).size } catch { return 0 }
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export const PROFILE_STORE_BACKUP_KEEP = 5
|
|
198
|
+
/** Don't copy 7.8 MB on every enroll; one snapshot per hour bounds the churn
|
|
199
|
+
* while still guaranteeing a recent restore point. */
|
|
200
|
+
export const PROFILE_STORE_BACKUP_MIN_INTERVAL_MS = 60 * 60 * 1000
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Load the store, quarantining a corrupt file and falling back to the newest
|
|
204
|
+
* usable backup rather than silently starting from zero profiles.
|
|
205
|
+
*
|
|
206
|
+
* Starting empty is the dangerous default here: the very next `saveProfileStore`
|
|
207
|
+
* would durably persist that empty store over the only remaining copy.
|
|
208
|
+
*/
|
|
209
|
+
export function loadVoiceProfileStore(storePath: string): StoreLoad {
|
|
210
|
+
const result = loadJsonOrQuarantine<unknown>(storePath)
|
|
211
|
+
if (result.status === 'missing') {
|
|
212
|
+
return { store: { profiles: [] }, repairs: emptyRepairs(), status: 'missing' }
|
|
213
|
+
}
|
|
214
|
+
if (result.status === 'ok') {
|
|
215
|
+
const { store, repairs } = normalizeProfileStore(result.data)
|
|
216
|
+
return { store, repairs, status: 'ok' }
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
for (const backup of listProfileStoreBackups(storePath)) {
|
|
220
|
+
const restored = loadJsonOrQuarantine<unknown>(backup)
|
|
221
|
+
if (restored.status !== 'ok') continue
|
|
222
|
+
const { store, repairs } = normalizeProfileStore(restored.data)
|
|
223
|
+
if (store.profiles.length === 0) continue
|
|
224
|
+
return {
|
|
225
|
+
store,
|
|
226
|
+
repairs,
|
|
227
|
+
status: 'corrupt',
|
|
228
|
+
quarantinedAs: result.quarantinedAs,
|
|
229
|
+
recoveredFromBackup: backup,
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return {
|
|
233
|
+
store: { profiles: [] },
|
|
234
|
+
repairs: emptyRepairs(),
|
|
235
|
+
status: 'corrupt',
|
|
236
|
+
quarantinedAs: result.quarantinedAs,
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Snapshot the current file before it is overwritten, then prune old copies. */
|
|
241
|
+
export function backupProfileStore(
|
|
242
|
+
storePath: string,
|
|
243
|
+
options: { nowMs?: number; force?: boolean; keep?: number; minIntervalMs?: number } = {},
|
|
244
|
+
): string | null {
|
|
245
|
+
if (!existsSync(storePath)) return null
|
|
246
|
+
if (safeSize(storePath) === 0) return null
|
|
247
|
+
|
|
248
|
+
const now = options.nowMs ?? Date.now()
|
|
249
|
+
const keep = options.keep ?? PROFILE_STORE_BACKUP_KEEP
|
|
250
|
+
const minInterval = options.minIntervalMs ?? PROFILE_STORE_BACKUP_MIN_INTERVAL_MS
|
|
251
|
+
const existing = listProfileStoreBackups(storePath)
|
|
252
|
+
if (!options.force && existing.length > 0 && now - safeMtime(existing[0]) < minInterval) return null
|
|
253
|
+
|
|
254
|
+
const dir = backupDirFor(storePath)
|
|
255
|
+
try {
|
|
256
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 })
|
|
257
|
+
const stamp = new Date(now).toISOString().replace(/[:.]/g, '-')
|
|
258
|
+
const target = join(dir, `${basename(storePath, '.json')}.${stamp}.json`)
|
|
259
|
+
// Copy through the durable writer so a backup can never itself be torn.
|
|
260
|
+
const raw = loadJsonOrQuarantine<unknown>(storePath)
|
|
261
|
+
if (raw.status !== 'ok') return null
|
|
262
|
+
durableAtomicWriteFileSync(target, JSON.stringify(raw.data), { mode: 0o600 })
|
|
263
|
+
for (const stale of listProfileStoreBackups(storePath).slice(keep)) {
|
|
264
|
+
try { unlinkSync(stale) } catch { /* pruning is best effort */ }
|
|
265
|
+
}
|
|
266
|
+
return target
|
|
267
|
+
} catch {
|
|
268
|
+
// A failed backup must never block the write it precedes.
|
|
269
|
+
return null
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Persist the store atomically, after taking a rotating backup.
|
|
275
|
+
*
|
|
276
|
+
* Refuses to write an empty store over a populated file. That is the shape of
|
|
277
|
+
* every catastrophic loss here — a failed load returning `{profiles: []}`, then
|
|
278
|
+
* a routine save committing it — and no legitimate caller needs it. Callers
|
|
279
|
+
* that really mean it (a full reset) pass `allowEmpty`.
|
|
280
|
+
*/
|
|
281
|
+
export function saveVoiceProfileStore(
|
|
282
|
+
storePath: string,
|
|
283
|
+
store: ProfileStore,
|
|
284
|
+
options: { allowEmpty?: boolean; nowMs?: number } = {},
|
|
285
|
+
): { written: boolean; backup: string | null; refusedReason?: string } {
|
|
286
|
+
if (store.profiles.length === 0 && !options.allowEmpty) {
|
|
287
|
+
const current = loadJsonOrQuarantine<ProfileStore>(storePath)
|
|
288
|
+
if (current.status === 'ok' && (current.data?.profiles?.length ?? 0) > 0) {
|
|
289
|
+
return {
|
|
290
|
+
written: false,
|
|
291
|
+
backup: null,
|
|
292
|
+
refusedReason: `refusing to overwrite ${current.data.profiles.length} stored profile(s) with an empty store`,
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const backup = backupProfileStore(storePath, { nowMs: options.nowMs })
|
|
298
|
+
mkdirSync(dirname(storePath), { recursive: true, mode: 0o700 })
|
|
299
|
+
durableAtomicWriteFileSync(storePath, JSON.stringify(store, null, 2), { mode: 0o600 })
|
|
300
|
+
return { written: true, backup }
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** Drop the oldest sample, keeping `embeddings` and `sources` in lockstep.
|
|
304
|
+
*
|
|
305
|
+
* The old inline form was `embeddings.shift()` followed by `sources?.shift()`,
|
|
306
|
+
* which silently skipped the second half whenever `sources` was undefined and
|
|
307
|
+
* desynchronised provenance from that point on — permanently, since every
|
|
308
|
+
* later push appended to both. */
|
|
309
|
+
export function dropOldestEmbedding(profile: VoiceProfile): { droppedSource: string | null } {
|
|
310
|
+
if (profile.embeddings.length === 0) return { droppedSource: null }
|
|
311
|
+
profile.embeddings.shift()
|
|
312
|
+
if (!Array.isArray(profile.sources)) profile.sources = []
|
|
313
|
+
// Re-align first so a short sources[] cannot shift the wrong provenance off.
|
|
314
|
+
while (profile.sources.length < profile.embeddings.length + 1) profile.sources.push(UNKNOWN_SOURCE)
|
|
315
|
+
const droppedSource = profile.sources.shift() ?? null
|
|
316
|
+
profile.sources.length = profile.embeddings.length
|
|
317
|
+
return { droppedSource }
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** Append a sample to both arrays together. */
|
|
321
|
+
export function appendEmbedding(profile: VoiceProfile, embedding: number[], source: string): void {
|
|
322
|
+
if (!Array.isArray(profile.sources)) profile.sources = []
|
|
323
|
+
while (profile.sources.length < profile.embeddings.length) profile.sources.push(UNKNOWN_SOURCE)
|
|
324
|
+
profile.embeddings.push(embedding)
|
|
325
|
+
profile.sources.push(source)
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Remove every sample whose provenance matches, e.g. one poisoned session.
|
|
329
|
+
* Returns the number removed so a retraction can be reported rather than
|
|
330
|
+
* assumed. */
|
|
331
|
+
export function removeEmbeddingsBySource(
|
|
332
|
+
profile: VoiceProfile,
|
|
333
|
+
predicate: (source: string) => boolean,
|
|
334
|
+
): number {
|
|
335
|
+
if (!Array.isArray(profile.sources)) profile.sources = []
|
|
336
|
+
while (profile.sources.length < profile.embeddings.length) profile.sources.push(UNKNOWN_SOURCE)
|
|
337
|
+
const keptEmbeddings: number[][] = []
|
|
338
|
+
const keptSources: string[] = []
|
|
339
|
+
let removed = 0
|
|
340
|
+
for (let i = 0; i < profile.embeddings.length; i++) {
|
|
341
|
+
if (predicate(profile.sources[i] ?? UNKNOWN_SOURCE)) { removed++; continue }
|
|
342
|
+
keptEmbeddings.push(profile.embeddings[i])
|
|
343
|
+
keptSources.push(profile.sources[i] ?? UNKNOWN_SOURCE)
|
|
344
|
+
}
|
|
345
|
+
profile.embeddings = keptEmbeddings
|
|
346
|
+
profile.sources = keptSources
|
|
347
|
+
return removed
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/** Cosine similarity between two raw rows. Local so this module stays free of
|
|
351
|
+
* the sherpa runtime and can be unit-tested without a 26 MB model. */
|
|
352
|
+
export function rowCosine(a: number[], b: number[]): number {
|
|
353
|
+
if (a.length !== b.length || a.length === 0) return 0
|
|
354
|
+
let dot = 0, na = 0, nb = 0
|
|
355
|
+
for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i] }
|
|
356
|
+
const denom = Math.sqrt(na) * Math.sqrt(nb)
|
|
357
|
+
return denom > 0 ? dot / denom : 0
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/** L2-normalized mean of the rows that share the modal dimension. */
|
|
361
|
+
export function profileCentroid(embeddings: number[][]): number[] {
|
|
362
|
+
const dim = modalDimension(embeddings)
|
|
363
|
+
if (dim === 0) return []
|
|
364
|
+
const usable = embeddings.filter(row => row.length === dim)
|
|
365
|
+
if (usable.length === 0) return []
|
|
366
|
+
const c = new Array<number>(dim).fill(0)
|
|
367
|
+
for (const row of usable) for (let i = 0; i < dim; i++) c[i] += row[i]
|
|
368
|
+
for (let i = 0; i < dim; i++) c[i] /= usable.length
|
|
369
|
+
const norm = Math.sqrt(c.reduce((sum, v) => sum + v * v, 0))
|
|
370
|
+
return norm > 0 ? c.map(v => v / norm) : c
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** How close two profiles' centroids are. This is the number a merge must be
|
|
374
|
+
* justified by: names are not evidence, and `Miles Mallard` / `Manoj Kumar`
|
|
375
|
+
* are different humans who would merge happily on a name heuristic. */
|
|
376
|
+
export function profileSimilarity(a: VoiceProfile, b: VoiceProfile): number {
|
|
377
|
+
return rowCosine(profileCentroid(a.embeddings), profileCentroid(b.embeddings))
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** Pick the N most acoustically diverse samples, carrying provenance along.
|
|
381
|
+
*
|
|
382
|
+
* A merge routinely produces more samples than the per-speaker cap (two
|
|
383
|
+
* capped profiles make 40 against a cap of 20), and which 20 survive matters:
|
|
384
|
+
* taking the first N would keep one profile's acoustic conditions and discard
|
|
385
|
+
* the other's, which is the opposite of what merging is for. Greedy
|
|
386
|
+
* max-min-distance keeps the spread.
|
|
387
|
+
*
|
|
388
|
+
* Returns indices so the caller can slice embeddings and sources together. */
|
|
389
|
+
export function selectDiverseIndices(embeddings: number[][], maxN: number): number[] {
|
|
390
|
+
if (embeddings.length <= maxN) return embeddings.map((_, i) => i)
|
|
391
|
+
|
|
392
|
+
// Seed with the two most dissimilar rows.
|
|
393
|
+
let worstPair = Number.POSITIVE_INFINITY, seedA = 0, seedB = 1
|
|
394
|
+
for (let i = 0; i < embeddings.length; i++) {
|
|
395
|
+
for (let j = i + 1; j < embeddings.length; j++) {
|
|
396
|
+
const sim = rowCosine(embeddings[i], embeddings[j])
|
|
397
|
+
if (sim < worstPair) { worstPair = sim; seedA = i; seedB = j }
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
const chosen = [seedA, seedB]
|
|
401
|
+
const taken = new Set(chosen)
|
|
402
|
+
while (chosen.length < maxN) {
|
|
403
|
+
let best = -1, bestMinDist = -Infinity
|
|
404
|
+
for (let i = 0; i < embeddings.length; i++) {
|
|
405
|
+
if (taken.has(i)) continue
|
|
406
|
+
let minDist = Infinity
|
|
407
|
+
for (const c of chosen) {
|
|
408
|
+
const dist = 1 - rowCosine(embeddings[i], embeddings[c])
|
|
409
|
+
if (dist < minDist) minDist = dist
|
|
410
|
+
}
|
|
411
|
+
if (minDist > bestMinDist) { bestMinDist = minDist; best = i }
|
|
412
|
+
}
|
|
413
|
+
if (best === -1) break
|
|
414
|
+
chosen.push(best)
|
|
415
|
+
taken.add(best)
|
|
416
|
+
}
|
|
417
|
+
// Ascending so the surviving order still reflects enrollment order.
|
|
418
|
+
return chosen.sort((x, y) => x - y)
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
export interface MergeOutcome {
|
|
422
|
+
/** Centroid cosine between the target and each source, before merging. */
|
|
423
|
+
similarity: Record<string, number>
|
|
424
|
+
samplesBefore: number
|
|
425
|
+
samplesAfter: number
|
|
426
|
+
/** Samples discarded by the cap, not by the merge itself. */
|
|
427
|
+
droppedToCap: number
|
|
428
|
+
mergedFrom: string[]
|
|
429
|
+
missing: string[]
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* Fold one or more profiles into another.
|
|
434
|
+
*
|
|
435
|
+
* Provenance is deliberately PRESERVED rather than restamped `merged:*`: the
|
|
436
|
+
* per-source retraction path is what makes a poisoned `auto:<sessionId>` sample
|
|
437
|
+
* removable later, and overwriting it to record a bookkeeping event would trade
|
|
438
|
+
* a useful fact for a useless one.
|
|
439
|
+
*/
|
|
440
|
+
export function mergeProfilesInStore(
|
|
441
|
+
store: ProfileStore,
|
|
442
|
+
into: string,
|
|
443
|
+
from: string[],
|
|
444
|
+
options: { cap?: number } = {},
|
|
445
|
+
): MergeOutcome {
|
|
446
|
+
const cap = options.cap ?? 20
|
|
447
|
+
const target = store.profiles.find(p => p.name === into)
|
|
448
|
+
const outcome: MergeOutcome = {
|
|
449
|
+
similarity: {},
|
|
450
|
+
samplesBefore: target?.embeddings.length ?? 0,
|
|
451
|
+
samplesAfter: target?.embeddings.length ?? 0,
|
|
452
|
+
droppedToCap: 0,
|
|
453
|
+
mergedFrom: [],
|
|
454
|
+
missing: [],
|
|
455
|
+
}
|
|
456
|
+
if (!target) {
|
|
457
|
+
outcome.missing = [into, ...from]
|
|
458
|
+
return outcome
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
const embeddings = [...target.embeddings]
|
|
462
|
+
const sources = [...(target.sources ?? [])]
|
|
463
|
+
while (sources.length < embeddings.length) sources.push(UNKNOWN_SOURCE)
|
|
464
|
+
|
|
465
|
+
for (const name of from) {
|
|
466
|
+
if (name === into) continue
|
|
467
|
+
const source = store.profiles.find(p => p.name === name)
|
|
468
|
+
if (!source) { outcome.missing.push(name); continue }
|
|
469
|
+
outcome.similarity[name] = profileSimilarity(target, source)
|
|
470
|
+
const sourceSources = [...(source.sources ?? [])]
|
|
471
|
+
while (sourceSources.length < source.embeddings.length) sourceSources.push(UNKNOWN_SOURCE)
|
|
472
|
+
for (let i = 0; i < source.embeddings.length; i++) {
|
|
473
|
+
embeddings.push(source.embeddings[i])
|
|
474
|
+
sources.push(sourceSources[i] ?? UNKNOWN_SOURCE)
|
|
475
|
+
}
|
|
476
|
+
outcome.mergedFrom.push(name)
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
if (outcome.mergedFrom.length === 0) return outcome
|
|
480
|
+
|
|
481
|
+
const keep = selectDiverseIndices(embeddings, cap)
|
|
482
|
+
outcome.droppedToCap = embeddings.length - keep.length
|
|
483
|
+
target.embeddings = keep.map(i => embeddings[i])
|
|
484
|
+
target.sources = keep.map(i => sources[i])
|
|
485
|
+
outcome.samplesAfter = target.embeddings.length
|
|
486
|
+
|
|
487
|
+
const removed = new Set(outcome.mergedFrom)
|
|
488
|
+
store.profiles = store.profiles.filter(p => !removed.has(p.name))
|
|
489
|
+
return outcome
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/** Delete a person's profile outright. Returns what was removed so the caller
|
|
493
|
+
* can report a per-store count instead of a bare success. */
|
|
494
|
+
export function deleteProfileFromStore(
|
|
495
|
+
store: ProfileStore,
|
|
496
|
+
name: string,
|
|
497
|
+
): { removedProfiles: number; removedEmbeddings: number } {
|
|
498
|
+
let removedProfiles = 0
|
|
499
|
+
let removedEmbeddings = 0
|
|
500
|
+
store.profiles = store.profiles.filter(profile => {
|
|
501
|
+
if (profile.name !== name) return true
|
|
502
|
+
removedProfiles++
|
|
503
|
+
removedEmbeddings += profile.embeddings.length
|
|
504
|
+
return false
|
|
505
|
+
})
|
|
506
|
+
return { removedProfiles, removedEmbeddings }
|
|
507
|
+
}
|
package/server/routes/health.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { serverMetrics } from '../lib/server-metrics.js'
|
|
|
6
6
|
import { getServerInstanceId } from '../lib/server-instance-id.js'
|
|
7
7
|
import { localFirstMeetingsCapability } from '../lib/local-first-meetings-contract.js'
|
|
8
8
|
import { isSileroAvailable } from '../lib/vad-silero.js'
|
|
9
|
-
import { speakerModelState } from '../lib/speaker-embeddings.js'
|
|
9
|
+
import { speakerModelState, speakerReadiness } from '../lib/speaker-embeddings.js'
|
|
10
10
|
import { getAvailableCliSessionId } from '../lib/claude-bridge.js'
|
|
11
11
|
import {
|
|
12
12
|
isWhisperLocalAvailable,
|
|
@@ -113,7 +113,8 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
113
113
|
// npm tarball), so publish its state rather than letting the amplitude
|
|
114
114
|
// fallback masquerade as working diarization. Availability only — the resolved
|
|
115
115
|
// path is a local filesystem detail and health is unauthenticated.
|
|
116
|
-
|
|
116
|
+
const speakerId = speakerModelState()
|
|
117
|
+
checks.speaker_id = speakerId.state
|
|
117
118
|
|
|
118
119
|
// Health is unauthenticated. Publish only availability; the actual CLI
|
|
119
120
|
// session id is a resumable runtime handle and belongs on authenticated
|
|
@@ -176,15 +177,22 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
176
177
|
: whisper_health.startupState === 'preflight' || whisper_health.startupState === 'loading'
|
|
177
178
|
? 'starting'
|
|
178
179
|
: 'degraded'
|
|
180
|
+
// A configured-but-broken voiceprint model is a fault, not a preference. The
|
|
181
|
+
// full reasoning, including why 'unavailable' must NOT degrade, lives with
|
|
182
|
+
// speakerReadiness() so it is testable rather than an inline ternary here.
|
|
183
|
+
const speakerReadinessState = speakerReadiness(speakerId.state)
|
|
179
184
|
const readiness = {
|
|
180
185
|
// /api/health remains a liveness endpoint and intentionally returns HTTP
|
|
181
186
|
// 200 while the server can answer. This separate field prevents an HTTP-
|
|
182
187
|
// green response from hiding a configured local subsystem failure.
|
|
183
|
-
status: whisperReadiness === 'degraded' ? 'degraded' : 'ready',
|
|
188
|
+
status: whisperReadiness === 'degraded' || speakerReadinessState === 'degraded' ? 'degraded' : 'ready',
|
|
184
189
|
admissions: maintenance.admissionsOpen ? 'open' : 'maintenance',
|
|
185
190
|
whisper: whisperReadiness,
|
|
186
191
|
whisperError: whisper_health.lastError,
|
|
187
192
|
localTts: tts_local.ready ? 'ready' : 'unavailable',
|
|
193
|
+
// Asserted, not merely reported: `speaker_id` above says what the state IS,
|
|
194
|
+
// this says whether that state is acceptable.
|
|
195
|
+
speakerId: speakerReadinessState,
|
|
188
196
|
}
|
|
189
197
|
const openai_whisper_budget = getOpenAIWhisperBudgetState()
|
|
190
198
|
const codex_models = getCodexModelCatalogSnapshot()
|
package/server/routes/meeting.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// the standalone public meeting store. The live transcript and chunk metadata
|
|
3
3
|
// are durable before the session is closed; batch improvement runs afterward.
|
|
4
4
|
|
|
5
|
-
import { existsSync, readdirSync, rmSync, statSync, unlinkSync } from 'node:fs'
|
|
5
|
+
import { existsSync, readdirSync, readFileSync, rmSync, statSync, unlinkSync } from 'node:fs'
|
|
6
6
|
import { resolve } from 'node:path'
|
|
7
7
|
import { Router } from 'express'
|
|
8
8
|
import { emitDisplay } from '../lib/display-bus.js'
|
|
@@ -64,6 +64,8 @@ import {
|
|
|
64
64
|
type TranscriptGapReport,
|
|
65
65
|
} from './transcribe-stream.js'
|
|
66
66
|
import { getServerInstanceId } from '../lib/server-instance-id.js'
|
|
67
|
+
import { getOwnerSpeakerLabel } from '../lib/profile.js'
|
|
68
|
+
import { reviewMeetingSpeakers, type ReviewChunk } from '../lib/meeting-speaker-review.js'
|
|
67
69
|
import {
|
|
68
70
|
acquireMaintenanceWork,
|
|
69
71
|
maintenanceAdmissionsOpen,
|
|
@@ -613,6 +615,59 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
|
|
|
613
615
|
}
|
|
614
616
|
})
|
|
615
617
|
|
|
618
|
+
// ── Speaker review (6.21.12) ──────────────────────────────────────────
|
|
619
|
+
// Backs COS Control's naming panel. Read-only: it reports what a saved
|
|
620
|
+
// meeting's sidecar already contains and never writes. Naming, merging, and
|
|
621
|
+
// rebuilding are the /api/voice/* routes, each with its own confirmation.
|
|
622
|
+
//
|
|
623
|
+
// Keyed on sessionId so it can reuse the store's traversal-hardened readers
|
|
624
|
+
// (safeDirectoryRealpath / safeReadFile) instead of reassembling a path from
|
|
625
|
+
// client-supplied domain and filename components.
|
|
626
|
+
router.get('/meeting/:sessionId/speakers', (req, res) => {
|
|
627
|
+
const sessionId = String(req.params.sessionId ?? '')
|
|
628
|
+
if (!/^[A-Za-z0-9:_-]{3,96}$/.test(sessionId)) {
|
|
629
|
+
res.status(400).json({ error: 'Invalid sessionId', reason: 'invalid_session_id' })
|
|
630
|
+
return
|
|
631
|
+
}
|
|
632
|
+
const saved = store.findBySessionId(sessionId)
|
|
633
|
+
if (!saved) {
|
|
634
|
+
res.status(404).json({ error: 'No saved meeting for this session', reason: 'meeting_not_found' })
|
|
635
|
+
return
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
let chunks: unknown
|
|
639
|
+
try {
|
|
640
|
+
const raw = JSON.parse(readFileSync(saved.sidecarPath, 'utf-8')) as Record<string, unknown>
|
|
641
|
+
chunks = Array.isArray(raw) ? raw : raw.chunks
|
|
642
|
+
} catch {
|
|
643
|
+
// Defensive: findBySessionId already parsed this sidecar to match the
|
|
644
|
+
// session, so a corrupt file 404s above and never reaches here. This
|
|
645
|
+
// covers the narrow race where it becomes unreadable in between. Either
|
|
646
|
+
// way the answer is never 200-with-no-voices, which would read as
|
|
647
|
+
// "nobody spoke" and invite naming voices that were never analysed.
|
|
648
|
+
res.status(422).json({ error: 'Chunk sidecar is missing or unreadable', reason: 'sidecar_unreadable' })
|
|
649
|
+
return
|
|
650
|
+
}
|
|
651
|
+
if (!Array.isArray(chunks)) {
|
|
652
|
+
res.status(422).json({ error: 'Chunk sidecar holds no chunk array', reason: 'sidecar_empty' })
|
|
653
|
+
return
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
const review = reviewMeetingSpeakers(chunks as ReviewChunk[], {
|
|
657
|
+
owner: getOwnerSpeakerLabel(),
|
|
658
|
+
phrasesPerVoice: Math.max(1, Math.min(6, Number(req.query.phrases) || 3)),
|
|
659
|
+
})
|
|
660
|
+
res.set('Cache-Control', 'private, no-store')
|
|
661
|
+
res.json({
|
|
662
|
+
sessionId,
|
|
663
|
+
title: saved.title,
|
|
664
|
+
domain: saved.domain,
|
|
665
|
+
filename: saved.filename,
|
|
666
|
+
durationMin: saved.durationMin,
|
|
667
|
+
...review,
|
|
668
|
+
})
|
|
669
|
+
})
|
|
670
|
+
|
|
616
671
|
// ── Unsaved-capture recovery (6.19.0) ─────────────────────────────────
|
|
617
672
|
// Surface-only by decision (Miles, 2026-08-02): the server NEVER drives
|
|
618
673
|
// recovery on its own. It lists what the quarantine holds, and one
|