@gotcos/glasses-server 6.59.0 → 6.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +8 -0
- package/CHANGELOG.md +66 -0
- package/README.md +142 -7
- package/bin/hooks/cos-session-hook +24 -4
- package/managed-runtime-contract.json +6 -1
- package/package.json +4 -1
- package/server/index.ts +34 -2
- package/server/lib/claude-hooks-installer.ts +120 -17
- package/server/lib/client-liveness.ts +92 -4
- package/server/lib/diarizer-backend.ts +163 -0
- package/server/lib/meeting-relabel-enrolment.ts +44 -7
- package/server/lib/meeting-store.ts +3 -1
- package/server/lib/nemotron-cli.ts +256 -0
- package/server/lib/nemotron-final.ts +979 -0
- package/server/lib/nemotron-live.ts +693 -0
- package/server/lib/nemotron-segments.ts +358 -0
- package/server/lib/permission-broker.ts +560 -31
- package/server/lib/session-hooks-runtime.ts +23 -0
- package/server/lib/speaker-embeddings.ts +7 -4
- package/server/lib/voiceprint-floor.ts +4 -0
- package/server/routes/health.ts +10 -0
- package/server/routes/meeting.ts +55 -2
- package/server/routes/permission-broker.ts +37 -3
- package/server/routes/transcribe-stream.ts +68 -5
|
@@ -36,8 +36,24 @@ export interface HookSubscription {
|
|
|
36
36
|
matcher?: string
|
|
37
37
|
async: boolean
|
|
38
38
|
timeout: number
|
|
39
|
+
/**
|
|
40
|
+
* 6.60.0: the command passes the block's own `timeout` to the script as its second
|
|
41
|
+
* argument. The script stops waiting 5 s before it and stamps it on the request it posts
|
|
42
|
+
* (`hookWaitS`), so the broker knows the wait of THIS invocation. The command and the
|
|
43
|
+
* timeout sit in one block, so whether or not a running Claude session re-reads its hook
|
|
44
|
+
* settings, the stamp it sends and the timeout it enforces come from the same block.
|
|
45
|
+
*/
|
|
46
|
+
passTimeout?: boolean
|
|
39
47
|
}
|
|
40
48
|
|
|
49
|
+
/**
|
|
50
|
+
* 6.60.0: Claude Code's wait on the PermissionRequest hook: the away hold (default 600 s)
|
|
51
|
+
* plus slack. The script's curl stops at 625 and the broker answers by 620 at the latest.
|
|
52
|
+
*/
|
|
53
|
+
export const PERMISSION_HOOK_TIMEOUT_S = 630
|
|
54
|
+
/** What every install before 6.60.0 carries for PermissionRequest, with no wait argument. */
|
|
55
|
+
export const LEGACY_PERMISSION_HOOK_TIMEOUT_S = 130
|
|
56
|
+
|
|
41
57
|
/**
|
|
42
58
|
* Every event the reducer understands, with the timing each one needs. SessionStart,
|
|
43
59
|
* UserPromptSubmit, Stop and SessionEnd are synchronous: measured 2026-09-15, an async
|
|
@@ -55,6 +71,10 @@ export interface HookSubscription {
|
|
|
55
71
|
* because the merge keeps exactly one of ours per event and status requires it: the script
|
|
56
72
|
* spools only AskUserQuestion and ExitPlanMode, which is what the old async matcher block
|
|
57
73
|
* did. Changing this row makes every existing install read `drift` until Install hooks.
|
|
74
|
+
*
|
|
75
|
+
* 6.60.0: PermissionRequest waits 630 s (was 130) and passes that number to the script, for
|
|
76
|
+
* the away hold. An install that differs ONLY by still carrying the earlier block (130 s, no
|
|
77
|
+
* argument) reads `drift` with `priorWaitOnly`, and stays halt-ready (`hookHaltReady`).
|
|
58
78
|
*/
|
|
59
79
|
export const HOOK_SUBSCRIPTIONS: readonly HookSubscription[] = [
|
|
60
80
|
{ event: 'SessionStart', async: false, timeout: 5 },
|
|
@@ -62,7 +82,7 @@ export const HOOK_SUBSCRIPTIONS: readonly HookSubscription[] = [
|
|
|
62
82
|
{ event: 'UserPromptSubmit', async: false, timeout: 5 },
|
|
63
83
|
{ event: 'Stop', async: false, timeout: 5 },
|
|
64
84
|
{ event: 'StopFailure', async: true, timeout: 10 },
|
|
65
|
-
{ event: 'PermissionRequest', async: false, timeout:
|
|
85
|
+
{ event: 'PermissionRequest', async: false, timeout: PERMISSION_HOOK_TIMEOUT_S, passTimeout: true },
|
|
66
86
|
{ event: 'PermissionDenied', async: true, timeout: 10 },
|
|
67
87
|
{ event: 'PreToolUse', async: false, timeout: 5 },
|
|
68
88
|
{ event: 'PostToolUse', async: true, timeout: 10 },
|
|
@@ -117,9 +137,13 @@ export function shellQuote(value: string): string {
|
|
|
117
137
|
/**
|
|
118
138
|
* The installed command carries the two paths the script needs as environment, so the
|
|
119
139
|
* server and the script can never disagree about where the spool or the token is.
|
|
140
|
+
* 6.60.0: `waitS`, for a subscription with `passTimeout`, follows the event name as the
|
|
141
|
+
* script's second argument (an argument, never an environment variable: a variable could
|
|
142
|
+
* be inherited from the shell that started Claude, an argument only comes from this block).
|
|
120
143
|
*/
|
|
121
|
-
export function hookCommand(scriptPath: string, event: string, paths: HookPaths = currentHookPaths()): string {
|
|
122
|
-
|
|
144
|
+
export function hookCommand(scriptPath: string, event: string, paths: HookPaths = currentHookPaths(), waitS?: number): string {
|
|
145
|
+
const base = `COS_GLASSES_HOME=${shellQuote(paths.home)} COS_HOOK_SPOOL=${shellQuote(paths.spoolDir)} ${shellQuote(scriptPath)} ${event}`
|
|
146
|
+
return waitS === undefined ? base : `${base} ${Math.trunc(waitS)}`
|
|
123
147
|
}
|
|
124
148
|
|
|
125
149
|
export interface HookPaths {
|
|
@@ -133,20 +157,57 @@ export function currentHookPaths(): HookPaths {
|
|
|
133
157
|
|
|
134
158
|
type HookBlock = { matcher?: unknown; hooks?: unknown }
|
|
135
159
|
|
|
160
|
+
function hookIsOurs(h: unknown): boolean {
|
|
161
|
+
return !!h && typeof h === 'object' && typeof (h as { command?: unknown }).command === 'string'
|
|
162
|
+
&& ((h as { command: string }).command.includes(HOOK_SCRIPT_MARKER) || (h as { command: string }).command.includes(HOOK_SCRIPT_NAME + ' '))
|
|
163
|
+
}
|
|
164
|
+
|
|
136
165
|
function blockIsOurs(block: unknown): boolean {
|
|
137
166
|
if (!block || typeof block !== 'object') return false
|
|
138
167
|
const hooks = (block as HookBlock).hooks
|
|
139
168
|
if (!Array.isArray(hooks)) return false
|
|
140
|
-
return hooks.some(
|
|
141
|
-
&& ((h as { command: string }).command.includes(HOOK_SCRIPT_MARKER) || (h as { command: string }).command.includes(HOOK_SCRIPT_NAME + ' ')))
|
|
169
|
+
return hooks.some(hookIsOurs)
|
|
142
170
|
}
|
|
143
171
|
|
|
144
172
|
function canonicalBlock(scriptPath: string, sub: HookSubscription, paths: HookPaths): Record<string, unknown> {
|
|
145
|
-
const
|
|
173
|
+
const command = hookCommand(scriptPath, sub.event, paths, sub.passTimeout ? sub.timeout : undefined)
|
|
174
|
+
const hook: Record<string, unknown> = { type: 'command', command, timeout: sub.timeout }
|
|
146
175
|
if (sub.async) hook.async = true
|
|
147
176
|
return sub.matcher ? { matcher: sub.matcher, hooks: [hook] } : { hooks: [hook] }
|
|
148
177
|
}
|
|
149
178
|
|
|
179
|
+
/**
|
|
180
|
+
* 6.60.0: the block an install before 6.60.0 wrote for this event, when this release changed
|
|
181
|
+
* it only by the wait: PermissionRequest at 130 s with no wait argument. Null for any other.
|
|
182
|
+
*/
|
|
183
|
+
function priorWaitBlock(scriptPath: string, sub: HookSubscription, paths: HookPaths): Record<string, unknown> | null {
|
|
184
|
+
if (sub.event !== 'PermissionRequest') return null
|
|
185
|
+
return canonicalBlock(scriptPath, { event: sub.event, async: sub.async, timeout: LEGACY_PERMISSION_HOOK_TIMEOUT_S }, paths)
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* 6.60.0: how long Claude Code waits on OUR PermissionRequest hook, read from a settings
|
|
190
|
+
* object: the smallest `timeout` among our entries (Claude runs each, so the first to give up
|
|
191
|
+
* bounds the wait). Null when none of ours is there, or one of ours has no usable number:
|
|
192
|
+
* the broker then keeps the old safe clamp (`lib/permission-broker.ts` `holdCeilingS`).
|
|
193
|
+
*/
|
|
194
|
+
export function permissionHookTimeoutS(settings: unknown): number | null {
|
|
195
|
+
const hooks = settings && typeof settings === 'object' ? (settings as Record<string, unknown>).hooks : null
|
|
196
|
+
const blocks = hooks && typeof hooks === 'object' ? (hooks as Record<string, unknown>).PermissionRequest : null
|
|
197
|
+
if (!Array.isArray(blocks)) return null
|
|
198
|
+
let least: number | null = null
|
|
199
|
+
for (const block of blocks) {
|
|
200
|
+
if (!blockIsOurs(block)) continue
|
|
201
|
+
for (const h of (block as HookBlock).hooks as unknown[]) {
|
|
202
|
+
if (!hookIsOurs(h)) continue
|
|
203
|
+
const timeout = (h as { timeout?: unknown }).timeout
|
|
204
|
+
if (typeof timeout !== 'number' || !Number.isFinite(timeout) || timeout <= 0) return null
|
|
205
|
+
least = least === null ? timeout : Math.min(least, timeout)
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return least
|
|
209
|
+
}
|
|
210
|
+
|
|
150
211
|
export type MergeResult =
|
|
151
212
|
| { ok: true; settings: Record<string, unknown>; changed: boolean }
|
|
152
213
|
/** The file holds a `hooks` shape this merge would have to destroy to proceed. */
|
|
@@ -190,22 +251,29 @@ export function stripHookSettings(current: unknown): MergeResult {
|
|
|
190
251
|
return { ok: true, settings, changed: JSON.stringify(settings) !== before }
|
|
191
252
|
}
|
|
192
253
|
|
|
193
|
-
/**
|
|
194
|
-
|
|
254
|
+
/**
|
|
255
|
+
* Which of our events a settings object carries, by exact canonical command. `priorWait`
|
|
256
|
+
* (6.60.0) names the drifted events whose one block of ours is exactly the block an earlier
|
|
257
|
+
* release wrote, differing only by the wait (`priorWaitBlock`).
|
|
258
|
+
*/
|
|
259
|
+
export function subscribedEvents(current: unknown, scriptPath: string, paths: HookPaths = currentHookPaths()): { subscribed: string[]; missing: string[]; drifted: string[]; priorWait: string[] } {
|
|
195
260
|
const hooks = current && typeof current === 'object' && (current as Record<string, unknown>).hooks
|
|
196
261
|
const table = hooks && typeof hooks === 'object' ? hooks as Record<string, unknown> : {}
|
|
197
262
|
const subscribed: string[] = []
|
|
198
263
|
const missing: string[] = []
|
|
199
264
|
const drifted: string[] = []
|
|
265
|
+
const priorWait: string[] = []
|
|
200
266
|
for (const sub of HOOK_SUBSCRIPTIONS) {
|
|
201
267
|
const blocks = Array.isArray(table[sub.event]) ? table[sub.event] as unknown[] : []
|
|
202
268
|
const ours = blocks.filter(blockIsOurs)
|
|
203
269
|
if (ours.length === 0) { missing.push(sub.event); continue }
|
|
204
270
|
const canonical = JSON.stringify(canonicalBlock(scriptPath, sub, paths))
|
|
205
|
-
if (ours.length === 1 && JSON.stringify(ours[0]) === canonical) subscribed.push(sub.event)
|
|
206
|
-
|
|
271
|
+
if (ours.length === 1 && JSON.stringify(ours[0]) === canonical) { subscribed.push(sub.event); continue }
|
|
272
|
+
drifted.push(sub.event)
|
|
273
|
+
const prior = priorWaitBlock(scriptPath, sub, paths)
|
|
274
|
+
if (prior && ours.length === 1 && JSON.stringify(ours[0]) === JSON.stringify(prior)) priorWait.push(sub.event)
|
|
207
275
|
}
|
|
208
|
-
return { subscribed, missing, drifted }
|
|
276
|
+
return { subscribed, missing, drifted, priorWait }
|
|
209
277
|
}
|
|
210
278
|
|
|
211
279
|
export type HookInstallState =
|
|
@@ -231,6 +299,16 @@ export interface HookStatus {
|
|
|
231
299
|
subscribed: string[]
|
|
232
300
|
missing: string[]
|
|
233
301
|
drifted: string[]
|
|
302
|
+
/** 6.60.0: drifted events whose block is exactly an earlier release's, but for the wait. */
|
|
303
|
+
priorWait: string[]
|
|
304
|
+
/**
|
|
305
|
+
* 6.60.0: `drift`, and the ONLY difference is that the settings still carry the earlier
|
|
306
|
+
* PermissionRequest block (130 s, no wait argument): an install from before 6.60.0 that
|
|
307
|
+
* has not run Install hooks since. Questions are then held at most 120 s, as before.
|
|
308
|
+
*/
|
|
309
|
+
priorWaitOnly: boolean
|
|
310
|
+
/** 6.60.0: our PermissionRequest hook's `timeout` in the settings file, or null (`permissionHookTimeoutS`). */
|
|
311
|
+
permissionHookTimeoutS: number | null
|
|
234
312
|
tokenPresent: boolean
|
|
235
313
|
}
|
|
236
314
|
|
|
@@ -272,16 +350,19 @@ export function hookStatus(paths: { settingsPath?: string; scriptPath?: string;
|
|
|
272
350
|
const tokenPresent = existsSync(hookTokenPath())
|
|
273
351
|
const read = readSettings(settingsPath)
|
|
274
352
|
if (!read.ok) {
|
|
275
|
-
return { state: read.reason, installed: false, settingsPath, scriptPath, scriptSha, packageScriptSha, subscribed: [], missing: HOOK_SUBSCRIPTIONS.map(s => s.event), drifted: [], tokenPresent }
|
|
353
|
+
return { state: read.reason, installed: false, settingsPath, scriptPath, scriptSha, packageScriptSha, subscribed: [], missing: HOOK_SUBSCRIPTIONS.map(s => s.event), drifted: [], priorWait: [], priorWaitOnly: false, permissionHookTimeoutS: null, tokenPresent }
|
|
276
354
|
}
|
|
277
355
|
const events = subscribedEvents(read.settings, scriptPath, hookPaths)
|
|
356
|
+
const permissionWaitS = permissionHookTimeoutS(read.settings)
|
|
278
357
|
let state: HookInstallState
|
|
279
358
|
if (hooksDisabled(read.settings)) state = 'disabled_by_settings'
|
|
280
359
|
else if (events.subscribed.length === 0 && events.drifted.length === 0) state = 'missing'
|
|
281
360
|
else if (events.missing.length > 0 || events.drifted.length > 0) state = 'drift'
|
|
282
361
|
else if (!scriptSha || (packageScriptSha && scriptSha !== packageScriptSha)) state = 'script_outdated'
|
|
283
362
|
else state = 'installed'
|
|
284
|
-
|
|
363
|
+
const priorWaitOnly = state === 'drift' && events.missing.length === 0 && events.drifted.length > 0
|
|
364
|
+
&& events.drifted.every(event => events.priorWait.includes(event))
|
|
365
|
+
return { state, installed: state === 'installed', settingsPath, scriptPath, scriptSha, packageScriptSha, ...events, priorWaitOnly, permissionHookTimeoutS: permissionWaitS, tokenPresent }
|
|
285
366
|
}
|
|
286
367
|
|
|
287
368
|
/**
|
|
@@ -295,22 +376,35 @@ export function hookStatus(paths: { settingsPath?: string; scriptPath?: string;
|
|
|
295
376
|
* - c0b41bf8...: the 6.53.3 script as first built, before its /qa round made every event
|
|
296
377
|
* drain stdin. Never published, but installable from a local build of that commit
|
|
297
378
|
* (7433b7f), and it carries the same halt check.
|
|
379
|
+
* - df54677f...: the script 6.53.3 through 6.59.0 shipped. 6.60.0 changed it only for the
|
|
380
|
+
* away hold (the PermissionRequest wait); its halt check is the same.
|
|
298
381
|
* Each is pinned against its bytes in `server/lib/__fixtures__/`.
|
|
299
382
|
*/
|
|
300
383
|
export const HALT_CAPABLE_PRIOR_SCRIPT_SHAS: readonly string[] = [
|
|
301
384
|
'1158bb06297550128f01fc47caa68806a5287d6da2d05b0d1c812e8ae20764e7',
|
|
302
385
|
'c0b41bf8581cfea498e1e1ac5220fe4e148bd97448d13b60a88879ee5992cc51',
|
|
386
|
+
'df54677f79908893509ee87608cfe3f4a53ced03f88f337f16a63741d6b3d547',
|
|
303
387
|
]
|
|
304
388
|
|
|
305
389
|
/**
|
|
306
390
|
* 6.53.3: can the hooks on this Mac stop a desk Claude run now? `installed`, or the only
|
|
307
391
|
* difference is a halt-capable earlier script (above). What the cancel route's
|
|
308
392
|
* `hooksReady` and `/api/health` `features.sessionCancel.deskClaude` read.
|
|
393
|
+
*
|
|
394
|
+
* 6.60.0: or `drift` with `priorWaitOnly` (the settings still carry the earlier
|
|
395
|
+
* PermissionRequest wait, every other block is current) on this package's script or a
|
|
396
|
+
* halt-capable earlier one. The wait changes how long a question is held, not the halt
|
|
397
|
+
* check, so updating the server never turns cancel from the lens off. Any other drift
|
|
398
|
+
* still is not halt-ready.
|
|
309
399
|
*/
|
|
310
|
-
export function hookHaltReady(status: Pick<HookStatus, 'installed' | 'state' | 'scriptSha'>): boolean {
|
|
400
|
+
export function hookHaltReady(status: Pick<HookStatus, 'installed' | 'state' | 'scriptSha'> & Partial<Pick<HookStatus, 'priorWaitOnly' | 'packageScriptSha'>>): boolean {
|
|
311
401
|
if (status.installed === true) return true
|
|
312
|
-
|
|
313
|
-
|
|
402
|
+
const priorScript = typeof status.scriptSha === 'string' && HALT_CAPABLE_PRIOR_SCRIPT_SHAS.includes(status.scriptSha)
|
|
403
|
+
if (status.state === 'script_outdated') return priorScript
|
|
404
|
+
if (status.state === 'drift' && status.priorWaitOnly === true) {
|
|
405
|
+
return priorScript || (typeof status.scriptSha === 'string' && status.scriptSha === status.packageScriptSha)
|
|
406
|
+
}
|
|
407
|
+
return false
|
|
314
408
|
}
|
|
315
409
|
|
|
316
410
|
const INSTALL_COMMAND = 'npx --yes @gotcos/glasses-server@latest --hooks install'
|
|
@@ -319,8 +413,17 @@ const INSTALL_COMMAND = 'npx --yes @gotcos/glasses-server@latest --hooks install
|
|
|
319
413
|
* 6.53.3: one plain sentence for `--hooks status` saying what the state means and what fixes
|
|
320
414
|
* it. Null when installed. Carries no path: the status JSON beside it already names them.
|
|
321
415
|
*/
|
|
322
|
-
export function hookStatusAdvice(status: Pick<HookStatus, 'installed' | 'state' | 'scriptSha' | 'packageScriptSha'>): string | null {
|
|
416
|
+
export function hookStatusAdvice(status: Pick<HookStatus, 'installed' | 'state' | 'scriptSha' | 'packageScriptSha'> & Partial<Pick<HookStatus, 'priorWaitOnly'>>): string | null {
|
|
323
417
|
const short = (sha: string | null): string => (sha ? sha.slice(0, 12) : 'none')
|
|
418
|
+
// 6.60.0: the one drift an update itself causes. Say what it costs and that nothing broke.
|
|
419
|
+
if (status.state === 'drift' && status.priorWaitOnly === true) {
|
|
420
|
+
return 'The Claude settings file still carries the earlier permission hook wait (130 s), so a question that starts while you are away '
|
|
421
|
+
+ 'is held for the glasses for at most 2 minutes instead of 10. '
|
|
422
|
+
+ `Run \`${INSTALL_COMMAND}\`, or press Install hooks in COS Control. `
|
|
423
|
+
+ (hookHaltReady(status)
|
|
424
|
+
? 'Cancel from the lens still stops desk runs meanwhile.'
|
|
425
|
+
: 'Until then a desk run cannot be cancelled from the lens.')
|
|
426
|
+
}
|
|
324
427
|
switch (status.state) {
|
|
325
428
|
case 'installed':
|
|
326
429
|
return null
|
|
@@ -23,28 +23,116 @@
|
|
|
23
23
|
//
|
|
24
24
|
// In memory: a restart forgets it, and the next poll restores it. Until then nothing is
|
|
25
25
|
// held, which is the native dialog, the safe side.
|
|
26
|
+
//
|
|
27
|
+
// 6.60.0 (QA round 1, the contract shared with COS Glasses 6.9.563): the away hold needs
|
|
28
|
+
// more than a poll. A poll QUALIFIES only with `hold=1` (this client shows held questions;
|
|
29
|
+
// 6.9.563 and later send it) AND a finite `presenceAgeMs` within [0, 24 h]: the ms since
|
|
30
|
+
// the latest real wearer evidence (glasses: a ring or temple input, or a device status with
|
|
31
|
+
// wearing true; phone: a foreground touch). A connected lens nobody wears sends no
|
|
32
|
+
// `presenceAgeMs`, and an older client sends no `hold=1`, so neither can make the Mac hold a
|
|
33
|
+
// question for 10 minutes.
|
|
34
|
+
//
|
|
35
|
+
// Kept apart, all in memory (a restart forgets them, which fails toward 6.59.0):
|
|
36
|
+
// - the newest counting poll of any kind (`lastQuestionsPollAt`, for /api/health);
|
|
37
|
+
// - the newest poll WITHOUT `hold=1` (`lastLegacyQuestionsPollAt`): the 6.59.0 liveness,
|
|
38
|
+
// unchanged for every older client;
|
|
39
|
+
// - the newest poll WITH `hold=1` (`lastHoldQuestionsPollAt`);
|
|
40
|
+
// - the newest qualifying presence, on BOTH clocks, each forward-only: monotonic
|
|
41
|
+
// (`performance.now()`, immune to the wall clock stepping) and wall. Its age is the
|
|
42
|
+
// larger of the two (`presenceAgeMs`): on macOS the monotonic clock does not advance
|
|
43
|
+
// while the Mac sleeps, so on its own it would read a pre-sleep presence as younger
|
|
44
|
+
// than it is; the wall clock does advance. A wall clock stepped forward reads older
|
|
45
|
+
// (no hold), never younger.
|
|
46
|
+
|
|
47
|
+
/** 6.60.0: the most a client may report as `presenceAgeMs`. Anything else is no presence. */
|
|
48
|
+
export const PRESENCE_AGE_MAX_MS = 86_400_000
|
|
26
49
|
|
|
27
50
|
let lastPollAt: number | null = null
|
|
51
|
+
let lastLegacyPollAt: number | null = null
|
|
52
|
+
let lastHoldPollAt: number | null = null
|
|
53
|
+
let lastPresenceMono: number | null = null
|
|
54
|
+
let lastPresenceWall: number | null = null
|
|
55
|
+
|
|
56
|
+
/** 6.60.0: the monotonic clock. Milliseconds since this process started; paused while the Mac sleeps. */
|
|
57
|
+
export function monotonicNowMs(): number {
|
|
58
|
+
return performance.now()
|
|
59
|
+
}
|
|
28
60
|
|
|
29
61
|
/** A stored stamp this far ahead of a new poll means the clock stepped back. */
|
|
30
62
|
const CLOCK_STEP_BACK_MS = 5_000
|
|
31
63
|
|
|
64
|
+
/** Newest-wins for a poll stamp, following a clock that stepped back (see below). */
|
|
65
|
+
function newerStamp(stored: number | null, at: number): number {
|
|
66
|
+
return stored === null || at > stored || stored - at > CLOCK_STEP_BACK_MS ? at : stored
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** 6.60.0: what a counting poll said about the client, read from its query. */
|
|
70
|
+
export interface QuestionsPollFacts {
|
|
71
|
+
/** `hold=1`: this client can show held questions. */
|
|
72
|
+
hold?: boolean
|
|
73
|
+
/** ms since the latest real wearer evidence; null or absent when there is none since boot. */
|
|
74
|
+
presenceAgeMs?: number | null
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** 6.60.0: a `presenceAgeMs` the contract accepts: a finite whole number within [0, 24 h]. */
|
|
78
|
+
export function validPresenceAgeMs(value: unknown): value is number {
|
|
79
|
+
return typeof value === 'number' && Number.isInteger(value) && value >= 0 && value <= PRESENCE_AGE_MAX_MS
|
|
80
|
+
}
|
|
81
|
+
|
|
32
82
|
/**
|
|
33
83
|
* A counting questions poll answered at `at` (server clock). Never moves backwards for a
|
|
34
84
|
* slower request answered later, but a clock that stepped back is followed: a stamp left in
|
|
35
85
|
* the future would read stale to the broker, and ignoring every poll until the clock caught
|
|
36
|
-
* up would leave no live client for that long.
|
|
86
|
+
* up would leave no live client for that long. 6.60.0: `poll` says whether it was a `hold=1`
|
|
87
|
+
* poll and carried qualifying presence; `mono` is the monotonic time it was answered.
|
|
37
88
|
*/
|
|
38
|
-
export function noteQuestionsPoll(at: number): void {
|
|
89
|
+
export function noteQuestionsPoll(at: number, poll: QuestionsPollFacts = {}, mono: number = monotonicNowMs()): void {
|
|
39
90
|
if (!Number.isFinite(at)) return
|
|
40
|
-
|
|
91
|
+
lastPollAt = newerStamp(lastPollAt, at)
|
|
92
|
+
if (poll.hold !== true) {
|
|
93
|
+
lastLegacyPollAt = newerStamp(lastLegacyPollAt, at)
|
|
94
|
+
return
|
|
95
|
+
}
|
|
96
|
+
lastHoldPollAt = newerStamp(lastHoldPollAt, at)
|
|
97
|
+
if (!validPresenceAgeMs(poll.presenceAgeMs) || !Number.isFinite(mono)) return
|
|
98
|
+
// Forward-only on each clock: a slower poll carrying older evidence never moves it back.
|
|
99
|
+
const presenceMono = mono - poll.presenceAgeMs
|
|
100
|
+
const presenceWall = at - poll.presenceAgeMs
|
|
101
|
+
if (lastPresenceMono === null || presenceMono > lastPresenceMono) lastPresenceMono = presenceMono
|
|
102
|
+
if (lastPresenceWall === null || presenceWall > lastPresenceWall) lastPresenceWall = presenceWall
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* 6.60.0: how long ago the newest qualifying wearer presence was, or null when no
|
|
107
|
+
* qualifying poll has arrived since this process started. The larger of the monotonic age
|
|
108
|
+
* and the wall-clock age, so a Mac that slept never makes it read younger.
|
|
109
|
+
*/
|
|
110
|
+
export function presenceAgeMs(mono: number = monotonicNowMs(), wall: number = Date.now()): number | null {
|
|
111
|
+
if (lastPresenceMono === null || !Number.isFinite(mono)) return null
|
|
112
|
+
const monoAge = Math.max(0, mono - lastPresenceMono)
|
|
113
|
+
const wallAge = lastPresenceWall !== null && Number.isFinite(wall) ? wall - lastPresenceWall : 0
|
|
114
|
+
return Math.max(monoAge, wallAge)
|
|
41
115
|
}
|
|
42
116
|
|
|
43
|
-
/** When the newest authenticated questions poll arrived, or null
|
|
117
|
+
/** When the newest authenticated questions poll of any kind arrived, or null since boot (/api/health). */
|
|
44
118
|
export function lastQuestionsPollAt(): number | null {
|
|
45
119
|
return lastPollAt
|
|
46
120
|
}
|
|
47
121
|
|
|
122
|
+
/** 6.60.0: the newest poll WITHOUT `hold=1`: the 6.59.0 liveness, for every older client. */
|
|
123
|
+
export function lastLegacyQuestionsPollAt(): number | null {
|
|
124
|
+
return lastLegacyPollAt
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** 6.60.0: the newest poll WITH `hold=1`. */
|
|
128
|
+
export function lastHoldQuestionsPollAt(): number | null {
|
|
129
|
+
return lastHoldPollAt
|
|
130
|
+
}
|
|
131
|
+
|
|
48
132
|
export function __resetClientLivenessForTests(): void {
|
|
49
133
|
lastPollAt = null
|
|
134
|
+
lastLegacyPollAt = null
|
|
135
|
+
lastHoldPollAt = null
|
|
136
|
+
lastPresenceMono = null
|
|
137
|
+
lastPresenceWall = null
|
|
50
138
|
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// Which model assigns who-spoke-when.
|
|
2
|
+
//
|
|
3
|
+
// Voiceprints (ERes2Net) name a person. They do not separate two people inside
|
|
4
|
+
// one chunk. Nemotron 3 Diarization does that, then the voiceprint names the
|
|
5
|
+
// track. COS_DIARIZER selects the track model. The default is Nemotron. If its
|
|
6
|
+
// CLI or its local models are missing, the server stays on the voiceprint path
|
|
7
|
+
// and says so.
|
|
8
|
+
//
|
|
9
|
+
// OFFLINE BY CONSTRUCTION. The FluidAudio CLI downloads its models from Hugging
|
|
10
|
+
// Face whenever it is run WITHOUT `--models`, and it deletes and re-downloads a
|
|
11
|
+
// cache whose weights marker it does not recognise. A meeting must never wait on
|
|
12
|
+
// that, so the server always passes `--models <dir>`, a local directory holding
|
|
13
|
+
// the compiled bundle and the silence embedding. That code path in FluidAudio
|
|
14
|
+
// (Nemotron3Models.load(directory:)) has no network call at all. When no complete
|
|
15
|
+
// local directory exists the server does not run the CLI.
|
|
16
|
+
|
|
17
|
+
import { execFileSync } from 'node:child_process'
|
|
18
|
+
import { chmodSync, existsSync, lstatSync, mkdirSync, readlinkSync, statSync, symlinkSync, unlinkSync } from 'node:fs'
|
|
19
|
+
import { homedir } from 'node:os'
|
|
20
|
+
import { join } from 'node:path'
|
|
21
|
+
|
|
22
|
+
export type DiarizerMode = 'nemotron' | 'embedding'
|
|
23
|
+
|
|
24
|
+
/** Why Nemotron cannot run at all on this install. Per-chunk reasons live in nemotron-live.ts. */
|
|
25
|
+
export type DiarizerStaticFallback = 'cli_missing' | 'models_missing'
|
|
26
|
+
|
|
27
|
+
export interface DiarizerChoice {
|
|
28
|
+
requested: DiarizerMode
|
|
29
|
+
active: DiarizerMode
|
|
30
|
+
/** Why Nemotron was not used. Null when the request is what is running. */
|
|
31
|
+
fallback: DiarizerStaticFallback | null
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** The canary's preset: 32-frame chunks, about 155x realtime warm on an M3 Ultra. */
|
|
35
|
+
export const NEMOTRON_VARIANT = 'fast32'
|
|
36
|
+
/** The FluidAudio v0.17.4 cache keeps the compiled preset under this subdirectory. */
|
|
37
|
+
export const NEMOTRON_BUNDLE = 'monolithic/v2'
|
|
38
|
+
export const NEMOTRON_MODEL_DIRNAME = `Nemotron3Diarizer_${NEMOTRON_VARIANT}.mlmodelc`
|
|
39
|
+
export const NEMOTRON_SILENCE_FILE = 'learnable_sil_emb.bin'
|
|
40
|
+
|
|
41
|
+
const EMBEDDING_ALIASES = new Set(['embedding', 'eres2net', 'voiceprint', 'legacy'])
|
|
42
|
+
|
|
43
|
+
export function requestedDiarizer(env: NodeJS.ProcessEnv = process.env): DiarizerMode {
|
|
44
|
+
const raw = (env.COS_DIARIZER ?? 'nemotron').trim().toLowerCase()
|
|
45
|
+
return EMBEDDING_ALIASES.has(raw) ? 'embedding' : 'nemotron'
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** First existing binary. Never returns a path that is not on disk. */
|
|
49
|
+
export function nemotronCliPath(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): string | null {
|
|
50
|
+
if (typeof env.COS_NEMOTRON_CLI === 'string') {
|
|
51
|
+
return env.COS_NEMOTRON_CLI.length > 0 && existsSync(env.COS_NEMOTRON_CLI) ? env.COS_NEMOTRON_CLI : null
|
|
52
|
+
}
|
|
53
|
+
const installed = join(home, '.cos-glasses', 'bin', 'fluidaudiocli')
|
|
54
|
+
if (existsSync(installed)) return installed
|
|
55
|
+
// Health reads this on every request; on an install with no CLI that was a
|
|
56
|
+
// synchronous `which` per request (QA 6.61.0 N5). Cached for a minute per PATH.
|
|
57
|
+
const key = env.PATH ?? ''
|
|
58
|
+
const now = Date.now()
|
|
59
|
+
if (whichCache && whichCache.key === key && now - whichCache.at < WHICH_CACHE_MS) {
|
|
60
|
+
return whichCache.value && existsSync(whichCache.value) ? whichCache.value : null
|
|
61
|
+
}
|
|
62
|
+
let value: string | null = null
|
|
63
|
+
try {
|
|
64
|
+
const found = execFileSync('which', ['fluidaudiocli'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 2_000 }).trim()
|
|
65
|
+
value = found && existsSync(found) ? found : null
|
|
66
|
+
} catch {
|
|
67
|
+
value = null
|
|
68
|
+
}
|
|
69
|
+
whichCache = { key, at: now, value }
|
|
70
|
+
return value
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const WHICH_CACHE_MS = 60_000
|
|
74
|
+
let whichCache: { key: string; at: number; value: string | null } | null = null
|
|
75
|
+
|
|
76
|
+
/** The directory passed to `--models`. COS_NEMOTRON_MODELS overrides it. */
|
|
77
|
+
export function nemotronModelsDir(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): string {
|
|
78
|
+
const override = env.COS_NEMOTRON_MODELS?.trim()
|
|
79
|
+
return override ? override : join(home, '.cos-glasses', 'models', 'nemotron-3-diarization')
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Where FluidAudio's own downloader leaves the models (`fluidaudiocli` run once without `--models`). */
|
|
83
|
+
export function fluidAudioCacheDir(home: string = homedir()): string {
|
|
84
|
+
return join(home, 'Library', 'Application Support', 'FluidAudio', 'Models', 'nemotron-3-diarization')
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function isFile(path: string): boolean {
|
|
88
|
+
try { return statSync(path).isFile() } catch { return false }
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** A compiled bundle is complete once its coremldata.bin is on disk (FluidAudio's own test). */
|
|
92
|
+
export function modelsDirComplete(dir: string): boolean {
|
|
93
|
+
return isFile(join(dir, NEMOTRON_MODEL_DIRNAME, 'coremldata.bin')) && isFile(join(dir, NEMOTRON_SILENCE_FILE))
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function cacheComplete(home: string): boolean {
|
|
97
|
+
const cache = fluidAudioCacheDir(home)
|
|
98
|
+
return isFile(join(cache, NEMOTRON_BUNDLE, NEMOTRON_MODEL_DIRNAME, 'coremldata.bin'))
|
|
99
|
+
&& isFile(join(cache, NEMOTRON_SILENCE_FILE))
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface NemotronModelSource {
|
|
103
|
+
dir: string
|
|
104
|
+
/** env: COS_NEMOTRON_MODELS. cos: ~/.cos-glasses/models already complete.
|
|
105
|
+
* fluidaudio-cache: complete only after prepareNemotronModelsDir links it. */
|
|
106
|
+
source: 'env' | 'cos' | 'fluidaudio-cache'
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Read-only: where the models are, or null. Never creates anything (health calls this). */
|
|
110
|
+
export function nemotronModelSource(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): NemotronModelSource | null {
|
|
111
|
+
const dir = nemotronModelsDir(env, home)
|
|
112
|
+
if (env.COS_NEMOTRON_MODELS?.trim()) {
|
|
113
|
+
// An explicit override is the answer, complete or not. Falling through to a
|
|
114
|
+
// different copy would run weights the operator did not choose.
|
|
115
|
+
return modelsDirComplete(dir) ? { dir, source: 'env' } : null
|
|
116
|
+
}
|
|
117
|
+
if (modelsDirComplete(dir)) return { dir, source: 'cos' }
|
|
118
|
+
if (cacheComplete(home)) return { dir, source: 'fluidaudio-cache' }
|
|
119
|
+
return null
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Make `--models` point at a complete local copy, or return null.
|
|
124
|
+
*
|
|
125
|
+
* FluidAudio keeps the bundle one level below the silence embedding, while
|
|
126
|
+
* `--models` wants both in one directory, so the COS directory holds two symlinks
|
|
127
|
+
* into the FluidAudio cache. Called by the warm-up, never by a health read.
|
|
128
|
+
*/
|
|
129
|
+
export function prepareNemotronModelsDir(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): string | null {
|
|
130
|
+
const found = nemotronModelSource(env, home)
|
|
131
|
+
if (!found) return null
|
|
132
|
+
if (found.source !== 'fluidaudio-cache') return found.dir
|
|
133
|
+
const cache = fluidAudioCacheDir(home)
|
|
134
|
+
const links: Array<[string, string]> = [
|
|
135
|
+
[join(found.dir, NEMOTRON_MODEL_DIRNAME), join(cache, NEMOTRON_BUNDLE, NEMOTRON_MODEL_DIRNAME)],
|
|
136
|
+
[join(found.dir, NEMOTRON_SILENCE_FILE), join(cache, NEMOTRON_SILENCE_FILE)],
|
|
137
|
+
]
|
|
138
|
+
try {
|
|
139
|
+
mkdirSync(found.dir, { recursive: true, mode: 0o700 })
|
|
140
|
+
try { chmodSync(found.dir, 0o700) } catch { /* best effort */ }
|
|
141
|
+
for (const [link, target] of links) {
|
|
142
|
+
let current: string | null = null
|
|
143
|
+
try {
|
|
144
|
+
if (lstatSync(link).isSymbolicLink()) current = readlinkSync(link)
|
|
145
|
+
else continue // a real file or directory someone placed there wins
|
|
146
|
+
} catch { /* absent */ }
|
|
147
|
+
if (current === target) continue
|
|
148
|
+
if (current !== null) unlinkSync(link)
|
|
149
|
+
symlinkSync(target, link)
|
|
150
|
+
}
|
|
151
|
+
} catch {
|
|
152
|
+
return null
|
|
153
|
+
}
|
|
154
|
+
return modelsDirComplete(found.dir) ? found.dir : null
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function activeDiarizer(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): DiarizerChoice {
|
|
158
|
+
const requested = requestedDiarizer(env)
|
|
159
|
+
if (requested === 'embedding') return { requested, active: 'embedding', fallback: null }
|
|
160
|
+
if (!nemotronCliPath(env, home)) return { requested, active: 'embedding', fallback: 'cli_missing' }
|
|
161
|
+
if (!nemotronModelSource(env, home)) return { requested, active: 'embedding', fallback: 'models_missing' }
|
|
162
|
+
return { requested, active: 'nemotron', fallback: null }
|
|
163
|
+
}
|
|
@@ -96,6 +96,10 @@ export interface EnrolmentReport {
|
|
|
96
96
|
/** Candidates rejected as a different voice. `clusterSkipped === attempted`
|
|
97
97
|
* means the bucket had no dominant voice at all and nothing was written. */
|
|
98
98
|
clusterSkipped: number
|
|
99
|
+
/** 6.61.0: candidates refused before clustering because the chunk is not one
|
|
100
|
+
* voice: its own capture-time voiceprint named a third person, or Nemotron
|
|
101
|
+
* split it into more than one turn. */
|
|
102
|
+
otherVoiceSkipped?: number
|
|
99
103
|
skipped: EnrolmentSkipReason
|
|
100
104
|
}
|
|
101
105
|
|
|
@@ -230,16 +234,35 @@ export function enrolNamedVoice(input: EnrolNamedVoiceInput): EnrolmentReport {
|
|
|
230
234
|
if (read.missing) return { ...IDLE, skipped: missingReason(sidecar) }
|
|
231
235
|
|
|
232
236
|
const wanted = new Set(rawIndices)
|
|
233
|
-
const
|
|
234
|
-
if (
|
|
237
|
+
const found = read.rows.filter(r => wanted.has(r.i) && r.embedding && r.embedding.length > 0)
|
|
238
|
+
if (found.length === 0) return { ...IDLE, skipped: 'no_embeddings' }
|
|
235
239
|
|
|
236
|
-
|
|
240
|
+
// 6.61.0 (QA W3). Since Nemotron, a chunk's sidecar `speaker` is the dominant
|
|
241
|
+
// TURN's name, while the embedding row holds the WHOLE chunk's voice and the
|
|
242
|
+
// label the voiceprint gave that whole chunk. A rename of the turn's label must
|
|
243
|
+
// not teach the profile a chunk whose own voiceprint named someone else, or a
|
|
244
|
+
// chunk Nemotron heard as more than one voice: either would enrol the wrong
|
|
245
|
+
// person. `from` stays allowed (a wrong label is how a new person first shows up).
|
|
246
|
+
const chunksByIndex = new Map<number, Record<string, unknown>>()
|
|
247
|
+
for (const entry of Array.isArray(sidecar.chunkEntries) ? sidecar.chunkEntries as Array<Record<string, unknown>> : []) {
|
|
248
|
+
if (typeof entry?.chunkIndex === 'number' && entry.chunk && typeof entry.chunk === 'object') {
|
|
249
|
+
chunksByIndex.set(entry.chunkIndex, entry.chunk as Record<string, unknown>)
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
const candidates = found.filter(r => !isAnotherVoice(r.speaker, input.from, to) && !isMultiVoiceChunk(chunksByIndex.get(r.i)))
|
|
253
|
+
// Reported only when it happened, so the response is the 6.60 shape otherwise.
|
|
254
|
+
const otherVoice = found.length - candidates.length > 0 ? { otherVoiceSkipped: found.length - candidates.length } : {}
|
|
255
|
+
if (candidates.length === 0) {
|
|
256
|
+
return { enrolled: 0, attempted: found.length, created: false, clusterSkipped: 0, ...otherVoice, skipped: null }
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
const attempted = found.length
|
|
237
260
|
const cluster = dominantCoherentCluster(candidates.map(r => r.embedding))
|
|
238
|
-
const clusterSkipped =
|
|
261
|
+
const clusterSkipped = candidates.length - cluster.members.length
|
|
239
262
|
// Every candidate disagreed with every other one. There is no voice here to
|
|
240
263
|
// learn, only a bucket of strangers sharing one placeholder label.
|
|
241
264
|
if (cluster.members.length === 0) {
|
|
242
|
-
return { enrolled: 0, attempted, created: false, clusterSkipped, skipped: null }
|
|
265
|
+
return { enrolled: 0, attempted, created: false, clusterSkipped, ...otherVoice, skipped: null }
|
|
243
266
|
}
|
|
244
267
|
|
|
245
268
|
// Diversity trims the coherent group to a bounded, spread-out sample. Those
|
|
@@ -261,12 +284,26 @@ export function enrolNamedVoice(input: EnrolNamedVoiceInput): EnrolmentReport {
|
|
|
261
284
|
}
|
|
262
285
|
} catch {
|
|
263
286
|
return {
|
|
264
|
-
enrolled, attempted, created: !existedBefore && enrolled > 0, clusterSkipped,
|
|
287
|
+
enrolled, attempted, created: !existedBefore && enrolled > 0, clusterSkipped, ...otherVoice,
|
|
265
288
|
skipped: 'store_unavailable',
|
|
266
289
|
}
|
|
267
290
|
}
|
|
268
291
|
|
|
269
292
|
return {
|
|
270
|
-
enrolled, attempted, created: !existedBefore && enrolled > 0, clusterSkipped, skipped: null,
|
|
293
|
+
enrolled, attempted, created: !existedBefore && enrolled > 0, clusterSkipped, ...otherVoice, skipped: null,
|
|
271
294
|
}
|
|
272
295
|
}
|
|
296
|
+
|
|
297
|
+
/** The row's own capture-time voiceprint named a real person other than the two in this rename. */
|
|
298
|
+
export function isAnotherVoice(rowSpeaker: unknown, from: string, to: string): boolean {
|
|
299
|
+
if (typeof rowSpeaker !== 'string' || !rowSpeaker.trim()) return false
|
|
300
|
+
if (isPlaceholderLabel(rowSpeaker)) return false
|
|
301
|
+
return rowSpeaker !== from && rowSpeaker !== to
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** Nemotron heard more than one turn in this chunk, live or in the final pass. */
|
|
305
|
+
export function isMultiVoiceChunk(chunk: Record<string, unknown> | undefined): boolean {
|
|
306
|
+
if (!chunk) return false
|
|
307
|
+
const turns = (value: unknown): number => Array.isArray(value) ? value.length : 0
|
|
308
|
+
return turns(chunk.segments) > 1 || turns(chunk.finalSegments) > 1
|
|
309
|
+
}
|
|
@@ -379,7 +379,9 @@ export class MeetingStore {
|
|
|
379
379
|
canonicalProvider: canonicalProvider(input.chunks),
|
|
380
380
|
providers,
|
|
381
381
|
providerCandidates: input.providerCandidates ?? {},
|
|
382
|
-
|
|
382
|
+
// 6.61.0: a chunk can hold two named voices; its segments count too.
|
|
383
|
+
speakers: [...new Set(input.chunks.flatMap(chunk => [chunk.speaker, ...(chunk.segments ?? []).map(segment => segment.speaker)])
|
|
384
|
+
.filter(speaker => speaker && speaker !== 'Ext'))],
|
|
383
385
|
chunks: input.chunks,
|
|
384
386
|
chunkEntries: input.chunkEntries ?? input.chunks.map((chunk, chunkIndex) => ({ chunkIndex, chunk })),
|
|
385
387
|
transferIntegrity: input.transferIntegrity ?? null,
|