@gotcos/glasses-server 6.19.0 → 6.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cos-profile.example.json +3 -3
- package/.env.example +7 -0
- package/CHANGELOG.md +37 -0
- package/README.md +17 -0
- package/bin/cli.cjs +156 -4
- package/managed-runtime-contract.json +2 -0
- package/package.json +1 -1
- package/server/index.ts +16 -1
- package/server/lib/hallucination-filter.ts +5 -8
- package/server/lib/profile.ts +81 -2
- package/server/lib/whisper-local.ts +7 -14
- package/server/lib/whisper-preview.ts +243 -0
- package/server/routes/glossary.ts +2 -11
- package/server/routes/health.ts +18 -2
- package/server/routes/prompt-drafts.ts +2 -5
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"owner_name": "
|
|
2
|
+
"owner_name": "User",
|
|
3
3
|
"owner_speaker_label": "Me",
|
|
4
|
-
"vocabulary": [
|
|
5
|
-
"whisper_corrections": "{
|
|
4
|
+
"vocabulary": [],
|
|
5
|
+
"whisper_corrections": "{}",
|
|
6
6
|
"negative_rules": []
|
|
7
7
|
}
|
package/.env.example
CHANGED
|
@@ -83,6 +83,13 @@ BIND_HOST=0.0.0.0
|
|
|
83
83
|
# COS_HQ_SPECULATIVE_WARM=0 # disable background HQ warm
|
|
84
84
|
# COS_BATCH_LARGE_V3=0 # explicitly use turbo instead of full HQ
|
|
85
85
|
# COS_HQ_BEAM_INTERACTIVE=2 # interactive only; meetings stay at beam 5
|
|
86
|
+
# Adaptive prompt transcription keeps three quality lanes separate:
|
|
87
|
+
# Small.en provisional preview -> Large-v3-Turbo committed live text -> Large-v3 HQ polish.
|
|
88
|
+
# Unset preserves the pre-6.20 Turbo behavior. `auto` uses Small.en when it has
|
|
89
|
+
# been provisioned, otherwise Turbo. Guided Setup writes small.en explicitly
|
|
90
|
+
# and downloads its ~466 MB model. `off` hides provisional peeks without
|
|
91
|
+
# changing committed transcription or meeting audio.
|
|
92
|
+
# COS_WHISPER_PREVIEW_MODEL=auto # auto | small.en | turbo | off
|
|
86
93
|
|
|
87
94
|
# Spoken reply playback defaults to local Kokoro on Apple silicon Macs. The
|
|
88
95
|
# first run creates a private venv and downloads the model. Local mode fails
|
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,40 @@
|
|
|
1
|
+
## 6.20.0
|
|
2
|
+
|
|
3
|
+
Adaptive transcription makes fast feedback additive instead of a quality
|
|
4
|
+
trade-off. The preview lane can be fast without changing the transcript that is
|
|
5
|
+
saved, searched, or sent.
|
|
6
|
+
|
|
7
|
+
- **Three explicit local transcription lanes.** Prompt previews can run on a
|
|
8
|
+
dedicated Small.en sidecar, committed live text remains on
|
|
9
|
+
Large-v3-Turbo, and HQ polish remains on Large-v3. The existing phone
|
|
10
|
+
`provisional` contract is unchanged. A late or failed preview cannot replace
|
|
11
|
+
committed text or advance the recovery ledger.
|
|
12
|
+
- **Safe adaptive fallback.** `COS_WHISPER_PREVIEW_MODEL` accepts `auto`,
|
|
13
|
+
`small.en`, `turbo`, or `off`. `auto` uses Small.en only when provisioned;
|
|
14
|
+
otherwise it preserves the prior Turbo behavior. A Small.en failure falls
|
|
15
|
+
back to the non-circuit Turbo path and cannot trip or success-reset the
|
|
16
|
+
authoritative Whisper breaker. The early private
|
|
17
|
+
`COS_WHISPER_REALTIME_MODEL` name remains a migration alias.
|
|
18
|
+
- **No surprise on update.** An unset preview setting keeps the pre-6.20 Turbo
|
|
19
|
+
behavior. Guided Setup explicitly opts users into Small.en after provisioning
|
|
20
|
+
its weights.
|
|
21
|
+
- **One setup command.** `--setup-transcription` records the adaptive choice
|
|
22
|
+
and provisions Small.en preview, Turbo commit, and Large-v3 HQ weights.
|
|
23
|
+
Pair with `--prepare-only` for COS Control so provisioning exits before the
|
|
24
|
+
managed LaunchAgent takes ownership.
|
|
25
|
+
- **Truthful health.** `/api/health` and `/api/models` add path-free
|
|
26
|
+
`capabilities.transcription.live` and `.profile` blocks alongside the
|
|
27
|
+
existing `.hq` block. COS Control can report the effective preview, commit,
|
|
28
|
+
and HQ models independently.
|
|
29
|
+
- **Factory vocabulary can no longer reduce accuracy.** The shipped profile is
|
|
30
|
+
empty and safe. Existing `Your Name`, `NameOne`, `NameTwo`, `YourCompany`,
|
|
31
|
+
`ProductName`, and `Soundalike -> YourName` examples are ignored everywhere
|
|
32
|
+
decoder bias or correction data is consumed. Startup warns once, and health
|
|
33
|
+
reports the ignored count. Real terms are trimmed and deduplicated.
|
|
34
|
+
- **Correction keys are literal.** User correction keys are escaped before
|
|
35
|
+
regular-expression construction, so punctuation in names cannot alter the
|
|
36
|
+
matcher.
|
|
37
|
+
|
|
1
38
|
## 6.19.0
|
|
2
39
|
|
|
3
40
|
Meeting audio is evidence. This release stops the server from ever deleting an
|
package/README.md
CHANGED
|
@@ -174,6 +174,9 @@ in the managed CLI working directory),
|
|
|
174
174
|
server-owned query recovery), and `COS_MEDIA_ROOT` (optional image-store
|
|
175
175
|
location; default `~/.cos-glasses/data/media`). Your name + transcription vocabulary live in
|
|
176
176
|
`~/.cos-glasses/.cos-profile.json` (see `.cos-profile.example.json`).
|
|
177
|
+
Factory example values are ignored; add the real names, companies, acronyms,
|
|
178
|
+
and specialist terms you say often. Guided Setup writes a safe empty profile
|
|
179
|
+
instead of biasing Whisper toward placeholder text.
|
|
177
180
|
Telegram activity export is disabled by default even when a private COS
|
|
178
181
|
pipeline contains `.telegram_config.json`; enable it only with the explicit
|
|
179
182
|
`COS_TELEGRAM_NOTIFICATIONS=1` opt-in.
|
|
@@ -233,6 +236,20 @@ Prompt dictation defaults to HQ. The phone owns the preference: **Fast mode
|
|
|
233
236
|
OFF** requests HQ, and **Fast mode ON** requests turbo. The Mac performs all
|
|
234
237
|
decoding; the phone does not run Whisper.
|
|
235
238
|
|
|
239
|
+
For the recommended adaptive setup, run:
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
npx --yes @gotcos/glasses-server@latest --setup-transcription
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
That keeps three jobs separate: Small.en supplies provisional words on the
|
|
246
|
+
lens, Large-v3-Turbo commits the authoritative live transcript, and Large-v3
|
|
247
|
+
polishes saved prompts and meetings. Small.en never writes the recovery ledger.
|
|
248
|
+
If its sidecar is missing or unhealthy, preview falls back to Turbo without
|
|
249
|
+
changing final quality. Set `COS_WHISPER_PREVIEW_MODEL=turbo` to keep one live
|
|
250
|
+
model, or `off` to disable provisional peeks. Existing installs that only
|
|
251
|
+
update the server remain on Turbo until Guided Setup opts them into Small.en.
|
|
252
|
+
|
|
236
253
|
The first server start downloads the real-time turbo model. True HQ additionally
|
|
237
254
|
requires the full `ggml-large-v3.bin` model (about 3.1 GB):
|
|
238
255
|
|
package/bin/cli.cjs
CHANGED
|
@@ -15,6 +15,8 @@ const {
|
|
|
15
15
|
unlinkSync,
|
|
16
16
|
renameSync,
|
|
17
17
|
chmodSync,
|
|
18
|
+
writeFileSync,
|
|
19
|
+
statfsSync,
|
|
18
20
|
} = require('fs')
|
|
19
21
|
const { delimiter, join, resolve } = require('path')
|
|
20
22
|
const { homedir } = require('os')
|
|
@@ -22,6 +24,8 @@ const { homedir } = require('os')
|
|
|
22
24
|
// bin/cli.cjs -> package root is one level up. The server lives at <root>/server.
|
|
23
25
|
const PKG_ROOT = resolve(__dirname, '..')
|
|
24
26
|
const CONFIG_DIR = join(homedir(), '.cos-glasses')
|
|
27
|
+
const PREPARE_ONLY = process.argv.includes('--prepare-only')
|
|
28
|
+
const SETUP_TRANSCRIPTION = process.argv.includes('--setup-transcription')
|
|
25
29
|
|
|
26
30
|
// Record where the user ran `npx @gotcos/glasses-server` from. The server spawns
|
|
27
31
|
// with cwd = PKG_ROOT (the npx cache), so without this the user's Starter-Kit COS
|
|
@@ -41,6 +45,7 @@ if (process.argv.includes('--help') || process.argv.includes('-h')) {
|
|
|
41
45
|
console.log('')
|
|
42
46
|
console.log(' Usage:')
|
|
43
47
|
console.log(' npx --yes @gotcos/glasses-server@latest')
|
|
48
|
+
console.log(' npx --yes @gotcos/glasses-server@latest --setup-transcription')
|
|
44
49
|
console.log(' npx --yes @gotcos/glasses-server@latest --prepare-only')
|
|
45
50
|
console.log('')
|
|
46
51
|
console.log(' Requirements:')
|
|
@@ -261,7 +266,7 @@ try {
|
|
|
261
266
|
// and the packaged runtime without creating COS config, downloading Kokoro
|
|
262
267
|
// models, installing Kokoro packages, changing COS permissions, or starting a
|
|
263
268
|
// listener. An invoked agent CLI may still maintain its own user cache.
|
|
264
|
-
if (
|
|
269
|
+
if (PREPARE_ONLY && !SETUP_TRANSCRIPTION) {
|
|
265
270
|
if (process.platform === 'darwin' && process.arch === 'arm64') {
|
|
266
271
|
const python = compatibleKokoroPython()
|
|
267
272
|
if (python) {
|
|
@@ -329,6 +334,31 @@ if (existsSync(ENV_FILE)) {
|
|
|
329
334
|
}
|
|
330
335
|
} catch { /* config is optional */ }
|
|
331
336
|
}
|
|
337
|
+
|
|
338
|
+
function upsertEnvValue(file, key, value) {
|
|
339
|
+
const current = existsSync(file) ? readFileSync(file, 'utf8') : ''
|
|
340
|
+
const lines = current.split(/\r?\n/)
|
|
341
|
+
const prefix = `${key}=`
|
|
342
|
+
let replaced = false
|
|
343
|
+
const next = lines.map((line) => {
|
|
344
|
+
if (line.trimStart().startsWith('#')) return line
|
|
345
|
+
if (!line.startsWith(prefix)) return line
|
|
346
|
+
replaced = true
|
|
347
|
+
return `${prefix}${value}`
|
|
348
|
+
})
|
|
349
|
+
if (!replaced) {
|
|
350
|
+
if (next.length && next.at(-1) !== '') next.push('')
|
|
351
|
+
next.push(`${prefix}${value}`)
|
|
352
|
+
}
|
|
353
|
+
writeFileSync(file, next.join('\n').replace(/\n*$/, '\n'), { mode: 0o600 })
|
|
354
|
+
chmodSync(file, 0o600)
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
if (SETUP_TRANSCRIPTION) {
|
|
358
|
+
upsertEnvValue(ENV_FILE, 'COS_WHISPER_PREVIEW_MODEL', 'small.en')
|
|
359
|
+
process.env.COS_WHISPER_PREVIEW_MODEL = 'small.en'
|
|
360
|
+
console.log(green(' ✓') + ' Adaptive transcription selected ' + dim('— Small.en preview · Turbo commit · Large-v3 HQ'))
|
|
361
|
+
}
|
|
332
362
|
// Persistent profile (identity + transcription vocabulary)
|
|
333
363
|
const PROFILE_FILE = join(CONFIG_DIR, '.cos-profile.json')
|
|
334
364
|
const PROFILE_EXAMPLE = join(PKG_ROOT, '.cos-profile.example.json')
|
|
@@ -349,11 +379,23 @@ if (!process.env.COS_PROFILE_PATH) process.env.COS_PROFILE_PATH = PROFILE_FILE
|
|
|
349
379
|
// Step 5: local Whisper detection + model download. Voice stays local-only by
|
|
350
380
|
// default; cloud fallback requires an explicit flag plus a configured key.
|
|
351
381
|
const WHISPER_KNOWN_PATHS = ['/opt/homebrew/bin/whisper-cli', '/usr/local/bin/whisper-cli']
|
|
382
|
+
const WHISPER_SERVER_KNOWN_PATHS = ['/opt/homebrew/bin/whisper-server', '/usr/local/bin/whisper-server']
|
|
352
383
|
const WHISPER_MODEL_DIR = join(homedir(), '.local/share/whisper-models')
|
|
353
384
|
const WHISPER_MODEL_PATH = join(WHISPER_MODEL_DIR, 'ggml-large-v3-turbo.bin')
|
|
354
385
|
const WHISPER_MODEL_PARTIAL = WHISPER_MODEL_PATH + '.partial'
|
|
355
386
|
const WHISPER_MODEL_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin'
|
|
356
387
|
const WHISPER_MODEL_MIN_BYTES = 800_000_000
|
|
388
|
+
const WHISPER_SMALL_MODEL_PATH = join(WHISPER_MODEL_DIR, 'ggml-small.en.bin')
|
|
389
|
+
const WHISPER_SMALL_MODEL_PARTIAL = WHISPER_SMALL_MODEL_PATH + '.partial'
|
|
390
|
+
const WHISPER_SMALL_MODEL_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin'
|
|
391
|
+
const WHISPER_SMALL_MODEL_MIN_BYTES = 400_000_000
|
|
392
|
+
const WHISPER_LARGE_MODEL_PATH = join(WHISPER_MODEL_DIR, 'ggml-large-v3.bin')
|
|
393
|
+
const WHISPER_LARGE_MODEL_PARTIAL = WHISPER_LARGE_MODEL_PATH + '.partial'
|
|
394
|
+
const WHISPER_LARGE_MODEL_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin'
|
|
395
|
+
const WHISPER_LARGE_MODEL_MIN_BYTES = 2_800_000_000
|
|
396
|
+
const WHISPER_MODEL_EXPECTED_BYTES = 1_620_000_000
|
|
397
|
+
const WHISPER_SMALL_MODEL_EXPECTED_BYTES = 466_000_000
|
|
398
|
+
const WHISPER_LARGE_MODEL_EXPECTED_BYTES = 3_100_000_000
|
|
357
399
|
function findWhisperCli() {
|
|
358
400
|
for (const p of WHISPER_KNOWN_PATHS) { if (existsSync(p)) return p }
|
|
359
401
|
try {
|
|
@@ -363,12 +405,58 @@ function findWhisperCli() {
|
|
|
363
405
|
return null
|
|
364
406
|
}
|
|
365
407
|
}
|
|
366
|
-
function
|
|
408
|
+
function findWhisperServer() {
|
|
409
|
+
for (const p of WHISPER_SERVER_KNOWN_PATHS) { if (existsSync(p)) return p }
|
|
410
|
+
try {
|
|
411
|
+
const found = execSync('command -v whisper-server 2>/dev/null', { shell: '/bin/sh', stdio: 'pipe', timeout: 2000 }).toString().trim()
|
|
412
|
+
return found || null
|
|
413
|
+
} catch {
|
|
414
|
+
return null
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
function isValidWhisperModel(p, minBytes = WHISPER_MODEL_MIN_BYTES) {
|
|
367
418
|
if (!existsSync(p)) return false
|
|
368
|
-
try { return statSync(p).size >=
|
|
419
|
+
try { return statSync(p).size >= minBytes } catch { return false }
|
|
369
420
|
}
|
|
370
421
|
const whisperCliPath = findWhisperCli()
|
|
422
|
+
const whisperServerPath = findWhisperServer()
|
|
371
423
|
const hasValidModel = isValidWhisperModel(WHISPER_MODEL_PATH)
|
|
424
|
+
|
|
425
|
+
if (SETUP_TRANSCRIPTION && (!whisperCliPath || !whisperServerPath)) {
|
|
426
|
+
console.log('')
|
|
427
|
+
console.log(red(' ✗ Adaptive transcription needs whisper.cpp'))
|
|
428
|
+
console.log(' Install it first: ' + bold('brew install whisper-cpp'))
|
|
429
|
+
console.log(' Then rerun this setup command. No model download was started.')
|
|
430
|
+
console.log('')
|
|
431
|
+
process.exit(1)
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
if (SETUP_TRANSCRIPTION && process.env.SKIP_WHISPER_DOWNLOAD !== '1') {
|
|
435
|
+
mkdirSync(WHISPER_MODEL_DIR, { recursive: true })
|
|
436
|
+
const targets = [
|
|
437
|
+
[WHISPER_MODEL_PATH, WHISPER_MODEL_PARTIAL, WHISPER_MODEL_MIN_BYTES, WHISPER_MODEL_EXPECTED_BYTES],
|
|
438
|
+
[WHISPER_SMALL_MODEL_PATH, WHISPER_SMALL_MODEL_PARTIAL, WHISPER_SMALL_MODEL_MIN_BYTES, WHISPER_SMALL_MODEL_EXPECTED_BYTES],
|
|
439
|
+
[WHISPER_LARGE_MODEL_PATH, WHISPER_LARGE_MODEL_PARTIAL, WHISPER_LARGE_MODEL_MIN_BYTES, WHISPER_LARGE_MODEL_EXPECTED_BYTES],
|
|
440
|
+
]
|
|
441
|
+
const missingBytes = targets.reduce((sum, [path, , minimum, expected]) =>
|
|
442
|
+
sum + (isValidWhisperModel(path, minimum) ? 0 : expected), 0)
|
|
443
|
+
const reclaimableBytes = targets.reduce((sum, [, partial]) => {
|
|
444
|
+
try { return sum + statSync(partial).size } catch { return sum }
|
|
445
|
+
}, 0)
|
|
446
|
+
const fsStats = statfsSync(WHISPER_MODEL_DIR)
|
|
447
|
+
const availableBytes = (fsStats.bavail * fsStats.bsize) + reclaimableBytes
|
|
448
|
+
const safetyMarginBytes = 750_000_000
|
|
449
|
+
if (missingBytes > 0 && availableBytes < missingBytes + safetyMarginBytes) {
|
|
450
|
+
const requiredGB = ((missingBytes + safetyMarginBytes) / 1_000_000_000).toFixed(1)
|
|
451
|
+
const availableGB = (availableBytes / 1_000_000_000).toFixed(1)
|
|
452
|
+
console.log('')
|
|
453
|
+
console.log(red(' ✗ Not enough free disk space for adaptive transcription'))
|
|
454
|
+
console.log(` Need about ${requiredGB} GB; ${availableGB} GB is available after reusable partial downloads.`)
|
|
455
|
+
console.log(' Free space, then rerun this setup command. Existing models were not removed.')
|
|
456
|
+
console.log('')
|
|
457
|
+
process.exit(1)
|
|
458
|
+
}
|
|
459
|
+
}
|
|
372
460
|
let localVoiceReady = Boolean(whisperCliPath && hasValidModel)
|
|
373
461
|
if (whisperCliPath && hasValidModel) {
|
|
374
462
|
console.log(green(' ✓') + ' whisper.cpp + model ready ' + dim('— voice = local (FREE)'))
|
|
@@ -383,7 +471,7 @@ if (whisperCliPath && hasValidModel) {
|
|
|
383
471
|
} else {
|
|
384
472
|
try {
|
|
385
473
|
mkdirSync(WHISPER_MODEL_DIR, { recursive: true })
|
|
386
|
-
|
|
474
|
+
execFileSync('curl', ['-fL', '--progress-bar', WHISPER_MODEL_URL, '-o', WHISPER_MODEL_PARTIAL], { stdio: 'inherit', timeout: 900000 })
|
|
387
475
|
const stats = statSync(WHISPER_MODEL_PARTIAL)
|
|
388
476
|
if (stats.size < WHISPER_MODEL_MIN_BYTES) throw new Error(`Downloaded file too small: ${stats.size} bytes`)
|
|
389
477
|
renameSync(WHISPER_MODEL_PARTIAL, WHISPER_MODEL_PATH)
|
|
@@ -405,6 +493,70 @@ if (process.env.COS_OPENAI_WHISPER_FALLBACK === '1') {
|
|
|
405
493
|
console.log(green(' ✓') + ' Transcription policy: local-only ' + dim('— a key alone never uploads audio'))
|
|
406
494
|
}
|
|
407
495
|
|
|
496
|
+
const previewChoice = (process.env.COS_WHISPER_PREVIEW_MODEL || process.env.COS_WHISPER_REALTIME_MODEL || 'auto').trim().toLowerCase()
|
|
497
|
+
const wantsSmallPreview = ['small', 'small.en', 'ggml-small.en.bin'].includes(previewChoice)
|
|
498
|
+
if (whisperCliPath && wantsSmallPreview) {
|
|
499
|
+
if (isValidWhisperModel(WHISPER_SMALL_MODEL_PATH, WHISPER_SMALL_MODEL_MIN_BYTES)) {
|
|
500
|
+
console.log(green(' ✓') + ' Small.en preview model ready ' + dim('— Turbo remains authoritative'))
|
|
501
|
+
} else {
|
|
502
|
+
if (existsSync(WHISPER_SMALL_MODEL_PATH)) { try { unlinkSync(WHISPER_SMALL_MODEL_PATH) } catch {} }
|
|
503
|
+
if (existsSync(WHISPER_SMALL_MODEL_PARTIAL)) { try { unlinkSync(WHISPER_SMALL_MODEL_PARTIAL) } catch {} }
|
|
504
|
+
console.log(yellow(' ⚠') + ' Adaptive preview model missing')
|
|
505
|
+
console.log(' ' + dim('Downloading ggml-small.en (~466 MB).'))
|
|
506
|
+
if (process.env.SKIP_WHISPER_DOWNLOAD === '1') {
|
|
507
|
+
console.log(yellow(' ⚠') + ' SKIP_WHISPER_DOWNLOAD=1 — previews use Turbo until Small.en is provisioned')
|
|
508
|
+
} else {
|
|
509
|
+
try {
|
|
510
|
+
mkdirSync(WHISPER_MODEL_DIR, { recursive: true })
|
|
511
|
+
execFileSync('curl', ['-fL', '--progress-bar', WHISPER_SMALL_MODEL_URL, '-o', WHISPER_SMALL_MODEL_PARTIAL], { stdio: 'inherit', timeout: 900000 })
|
|
512
|
+
const stats = statSync(WHISPER_SMALL_MODEL_PARTIAL)
|
|
513
|
+
if (stats.size < WHISPER_SMALL_MODEL_MIN_BYTES) throw new Error(`Downloaded file too small: ${stats.size} bytes`)
|
|
514
|
+
renameSync(WHISPER_SMALL_MODEL_PARTIAL, WHISPER_SMALL_MODEL_PATH)
|
|
515
|
+
console.log(green(' ✓') + ' Small.en preview model downloaded ' + dim('— Turbo commit unchanged'))
|
|
516
|
+
} catch (err) {
|
|
517
|
+
try { unlinkSync(WHISPER_SMALL_MODEL_PARTIAL) } catch {}
|
|
518
|
+
console.log(red(' ✗') + ' Small.en download failed ' + dim('— previews safely fall back to Turbo'))
|
|
519
|
+
console.log(' ' + dim('Error: ' + (err.message || err).toString().slice(0, 120)))
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
if (whisperCliPath && SETUP_TRANSCRIPTION) {
|
|
526
|
+
if (isValidWhisperModel(WHISPER_LARGE_MODEL_PATH, WHISPER_LARGE_MODEL_MIN_BYTES)) {
|
|
527
|
+
console.log(green(' ✓') + ' Large-v3 HQ model ready ' + dim('— saved meetings use full polish'))
|
|
528
|
+
} else {
|
|
529
|
+
if (existsSync(WHISPER_LARGE_MODEL_PATH)) { try { unlinkSync(WHISPER_LARGE_MODEL_PATH) } catch {} }
|
|
530
|
+
if (existsSync(WHISPER_LARGE_MODEL_PARTIAL)) { try { unlinkSync(WHISPER_LARGE_MODEL_PARTIAL) } catch {} }
|
|
531
|
+
console.log(yellow(' ⚠') + ' HQ polish model missing')
|
|
532
|
+
console.log(' ' + dim('Downloading ggml-large-v3 (~3.1 GB).'))
|
|
533
|
+
if (process.env.SKIP_WHISPER_DOWNLOAD === '1') {
|
|
534
|
+
console.log(yellow(' ⚠') + ' SKIP_WHISPER_DOWNLOAD=1 — HQ safely reports unavailable and live Turbo continues')
|
|
535
|
+
} else {
|
|
536
|
+
try {
|
|
537
|
+
mkdirSync(WHISPER_MODEL_DIR, { recursive: true })
|
|
538
|
+
execFileSync('curl', ['-fL', '--progress-bar', WHISPER_LARGE_MODEL_URL, '-o', WHISPER_LARGE_MODEL_PARTIAL], { stdio: 'inherit', timeout: 1800000 })
|
|
539
|
+
const stats = statSync(WHISPER_LARGE_MODEL_PARTIAL)
|
|
540
|
+
if (stats.size < WHISPER_LARGE_MODEL_MIN_BYTES) throw new Error(`Downloaded file too small: ${stats.size} bytes`)
|
|
541
|
+
renameSync(WHISPER_LARGE_MODEL_PARTIAL, WHISPER_LARGE_MODEL_PATH)
|
|
542
|
+
console.log(green(' ✓') + ' Large-v3 HQ model downloaded')
|
|
543
|
+
} catch (err) {
|
|
544
|
+
try { unlinkSync(WHISPER_LARGE_MODEL_PARTIAL) } catch {}
|
|
545
|
+
console.log(red(' ✗') + ' Large-v3 download failed ' + dim('— live Turbo remains available'))
|
|
546
|
+
console.log(' ' + dim('Error: ' + (err.message || err).toString().slice(0, 120)))
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
if (PREPARE_ONLY && SETUP_TRANSCRIPTION) {
|
|
553
|
+
console.log('')
|
|
554
|
+
console.log(green(' ✓ Adaptive transcription setup complete'))
|
|
555
|
+
console.log(' Return to COS Control and Restart, install, or update the managed server.')
|
|
556
|
+
console.log('')
|
|
557
|
+
process.exit(0)
|
|
558
|
+
}
|
|
559
|
+
|
|
408
560
|
// Step 6: image capability — ffmpeg validates, strips metadata, normalizes,
|
|
409
561
|
// and builds the exact 288x144 G2 variant. It is optional so text/voice remain
|
|
410
562
|
// useful on a minimal install, but the launcher should make the gap visible.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gotcos/glasses-server",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.20.0",
|
|
4
4
|
"description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/server/index.ts
CHANGED
|
@@ -48,6 +48,7 @@ import {
|
|
|
48
48
|
stopCodexModelCatalogRefresh,
|
|
49
49
|
} from './lib/codex-model-catalog.js'
|
|
50
50
|
import { startWhisperServer, stopWhisperServer } from './lib/whisper-local.js'
|
|
51
|
+
import { startWhisperPreviewServer, stopWhisperPreviewServer } from './lib/whisper-preview.js'
|
|
51
52
|
import { startLocalTtsServer, stopLocalTtsServer } from './lib/tts-local.js'
|
|
52
53
|
import { initSileroVAD } from './lib/vad-silero.js'
|
|
53
54
|
import { initSessionCache } from './lib/session-cache-writer.js'
|
|
@@ -58,6 +59,7 @@ import { listenRequiredServers, type RequiredListener } from './lib/listener-sta
|
|
|
58
59
|
import { serverMetrics } from './lib/server-metrics.js'
|
|
59
60
|
import { initializeServerInstanceId } from './lib/server-instance-id.js'
|
|
60
61
|
import { appendPrivateEnvBlock, UnsafeUserConfigPathError } from './lib/secure-user-config.js'
|
|
62
|
+
import { getTranscriptionProfileStatus } from './lib/profile.js'
|
|
61
63
|
import { createQueryJobsRouter } from './routes/query-jobs.js'
|
|
62
64
|
import {
|
|
63
65
|
initQueryJobRuntime,
|
|
@@ -296,6 +298,7 @@ async function gracefulShutdown(): Promise<void> {
|
|
|
296
298
|
try { logActiveSessionsOnShutdown() } catch { /* best-effort flush */ }
|
|
297
299
|
stopCodexModelCatalogRefresh()
|
|
298
300
|
stopWhisperServer()
|
|
301
|
+
stopWhisperPreviewServer()
|
|
299
302
|
stopLocalTtsServer()
|
|
300
303
|
clearTimeout(forceExit)
|
|
301
304
|
process.exit(0)
|
|
@@ -407,7 +410,19 @@ listenRequiredServers(listeners).then(() => {
|
|
|
407
410
|
// known-good snapshot if Codex is temporarily unavailable.
|
|
408
411
|
startCodexModelCatalogRefresh()
|
|
409
412
|
// Start local whisper-server (model stays in RAM for ~50ms transcription)
|
|
410
|
-
|
|
413
|
+
const transcriptionProfile = getTranscriptionProfileStatus()
|
|
414
|
+
if (transcriptionProfile.ignoredPlaceholderTerms > 0 || transcriptionProfile.ignoredPlaceholderCorrection) {
|
|
415
|
+
console.warn(
|
|
416
|
+
'[whisper] .cos-profile.json still contains factory vocabulary. ' +
|
|
417
|
+
'Placeholder terms are ignored; add real names and terminology in Guided Setup for better accuracy.',
|
|
418
|
+
)
|
|
419
|
+
}
|
|
420
|
+
// Load the authoritative Turbo worker first. Starting both model loads at
|
|
421
|
+
// once made first boot slower on smaller Macs and could cause Control's
|
|
422
|
+
// readiness proof to time out even though both models were healthy.
|
|
423
|
+
startWhisperServer()
|
|
424
|
+
.then(() => startWhisperPreviewServer())
|
|
425
|
+
.catch(err => console.error('[startup] Whisper server/preview error:', err))
|
|
411
426
|
startLocalTtsServer().catch(err => console.error('[startup] Local TTS server error:', err))
|
|
412
427
|
// Initialize speaker embeddings (voiceprint-based diarization) — fails soft if model absent
|
|
413
428
|
const embeddingOk = initSpeakerEmbeddings()
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// isFullHallucination(text) — returns true if the text IS a hallucination in its
|
|
12
12
|
// entirety (silence artifacts, caption training, foreign script, filler-only).
|
|
13
13
|
|
|
14
|
-
import { getNegativeRules, getVocabulary, getOwnerName,
|
|
14
|
+
import { getNegativeRules, getVocabulary, getOwnerName, getWhisperCorrections } from './profile.js'
|
|
15
15
|
|
|
16
16
|
// ── Whole-chunk silence hallucinations ─────────────────────────────────────
|
|
17
17
|
const KNOWN_HALLUCINATIONS = [
|
|
@@ -333,13 +333,10 @@ function getVocabEchoMatcher(): RegExp {
|
|
|
333
333
|
for (const v of getVocabulary()) if (v && v.trim()) raw.add(v.trim())
|
|
334
334
|
// Include whisper_corrections key/value variants so the echo matches whatever
|
|
335
335
|
// spelling whisper emits ("POS Nation" ↔ "POSNation", "Jewel 360" ↔ "Jewel360").
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
for (const [k, val] of Object.entries(map)) { if (k) raw.add(k); if (val) raw.add(val) }
|
|
341
|
-
}
|
|
342
|
-
} catch { /* malformed corrections — ignore */ }
|
|
336
|
+
for (const [key, value] of Object.entries(getWhisperCorrections())) {
|
|
337
|
+
raw.add(key)
|
|
338
|
+
raw.add(value)
|
|
339
|
+
}
|
|
343
340
|
// Only UNAMBIGUOUS terms trigger an echo drop: multi-word phrases ("POS Nation",
|
|
344
341
|
// "IT Retail", "Jeremy Sokolic") and brand-shaped single tokens with an internal
|
|
345
342
|
// capital or digit ("POSNation", "CaratIQ", "Jewel360"). Plain single-word tokens
|
package/server/lib/profile.ts
CHANGED
|
@@ -8,6 +8,10 @@ import { atomicWriteFileSync } from './atomic-fs.js'
|
|
|
8
8
|
|
|
9
9
|
const APP_ROOT = resolve(import.meta.dirname, '../..')
|
|
10
10
|
|
|
11
|
+
const PLACEHOLDER_OWNER_NAMES = new Set(['your name', 'user'])
|
|
12
|
+
const PLACEHOLDER_VOCABULARY = new Set(['nameone', 'nametwo', 'yourcompany', 'productname'])
|
|
13
|
+
const PLACEHOLDER_CORRECTIONS = new Set(['soundalike\u0000yourname'])
|
|
14
|
+
|
|
11
15
|
/** The profile in the data home. Survives updates; the APP_ROOT copy does not. */
|
|
12
16
|
export function homeProfilePath(): string {
|
|
13
17
|
return resolve(homedir(), '.cos-glasses', '.cos-profile.json')
|
|
@@ -81,7 +85,8 @@ export function loadProfileField(field: string, fallback: string): string {
|
|
|
81
85
|
}
|
|
82
86
|
|
|
83
87
|
export function getOwnerName(): string {
|
|
84
|
-
|
|
88
|
+
const value = loadProfileField('owner_name', 'User').trim()
|
|
89
|
+
return !value || PLACEHOLDER_OWNER_NAMES.has(value.toLowerCase()) ? 'User' : value
|
|
85
90
|
}
|
|
86
91
|
|
|
87
92
|
/** Short speaker label for the glasses wearer, used by diarization to fast-path
|
|
@@ -92,7 +97,81 @@ export function getOwnerSpeakerLabel(): string {
|
|
|
92
97
|
|
|
93
98
|
export function getVocabulary(): string[] {
|
|
94
99
|
const profile = loadProfile()
|
|
95
|
-
|
|
100
|
+
if (!Array.isArray(profile.vocabulary)) return []
|
|
101
|
+
const seen = new Set<string>()
|
|
102
|
+
return (profile.vocabulary as unknown[]).flatMap(value => {
|
|
103
|
+
if (typeof value !== 'string') return []
|
|
104
|
+
const term = value.trim()
|
|
105
|
+
const key = term.toLowerCase()
|
|
106
|
+
if (!term || PLACEHOLDER_VOCABULARY.has(key) || seen.has(key)) return []
|
|
107
|
+
seen.add(key)
|
|
108
|
+
return [term]
|
|
109
|
+
})
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Typed correction map shared by every decoder caller. The legacy profile
|
|
113
|
+
* stores this field as a JSON string, while hand-authored profiles sometimes
|
|
114
|
+
* use an object; accept both and ignore the factory example pair. */
|
|
115
|
+
export function getWhisperCorrections(): Record<string, string> {
|
|
116
|
+
const raw = loadProfile().whisper_corrections
|
|
117
|
+
let parsed: unknown = raw
|
|
118
|
+
if (typeof raw === 'string') {
|
|
119
|
+
try { parsed = JSON.parse(raw) } catch { return {} }
|
|
120
|
+
}
|
|
121
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {}
|
|
122
|
+
|
|
123
|
+
const corrections: Record<string, string> = {}
|
|
124
|
+
for (const [sourceRaw, targetRaw] of Object.entries(parsed as Record<string, unknown>)) {
|
|
125
|
+
if (typeof targetRaw !== 'string') continue
|
|
126
|
+
const source = sourceRaw.trim()
|
|
127
|
+
const target = targetRaw.trim()
|
|
128
|
+
if (!source || !target) continue
|
|
129
|
+
if (PLACEHOLDER_CORRECTIONS.has(`${source.toLowerCase()}\u0000${target.toLowerCase()}`)) continue
|
|
130
|
+
corrections[source] = target
|
|
131
|
+
}
|
|
132
|
+
return corrections
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export interface TranscriptionProfileStatus {
|
|
136
|
+
configured: boolean
|
|
137
|
+
ownerConfigured: boolean
|
|
138
|
+
vocabularyTerms: number
|
|
139
|
+
ignoredPlaceholderTerms: number
|
|
140
|
+
ignoredPlaceholderCorrection: boolean
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Path-free setup truth for startup warnings, health, and COS Control. */
|
|
144
|
+
export function getTranscriptionProfileStatus(): TranscriptionProfileStatus {
|
|
145
|
+
const profile = loadProfile()
|
|
146
|
+
const rawOwner = typeof profile.owner_name === 'string' ? profile.owner_name.trim() : ''
|
|
147
|
+
const rawVocabulary = Array.isArray(profile.vocabulary)
|
|
148
|
+
? (profile.vocabulary as unknown[]).filter((value): value is string => typeof value === 'string')
|
|
149
|
+
: []
|
|
150
|
+
const ignoredPlaceholderTerms = rawVocabulary.filter(term => PLACEHOLDER_VOCABULARY.has(term.trim().toLowerCase())).length
|
|
151
|
+
const rawCorrections = (() => {
|
|
152
|
+
const value = profile.whisper_corrections
|
|
153
|
+
if (typeof value === 'string') {
|
|
154
|
+
try { return JSON.parse(value) as unknown } catch { return null }
|
|
155
|
+
}
|
|
156
|
+
return value
|
|
157
|
+
})()
|
|
158
|
+
const ignoredPlaceholderCorrection = Boolean(
|
|
159
|
+
rawCorrections
|
|
160
|
+
&& typeof rawCorrections === 'object'
|
|
161
|
+
&& !Array.isArray(rawCorrections)
|
|
162
|
+
&& Object.entries(rawCorrections as Record<string, unknown>).some(([source, target]) =>
|
|
163
|
+
typeof target === 'string'
|
|
164
|
+
&& PLACEHOLDER_CORRECTIONS.has(`${source.trim().toLowerCase()}\u0000${target.trim().toLowerCase()}`)),
|
|
165
|
+
)
|
|
166
|
+
const ownerConfigured = Boolean(rawOwner) && !PLACEHOLDER_OWNER_NAMES.has(rawOwner.toLowerCase())
|
|
167
|
+
const vocabularyTerms = getVocabulary().length
|
|
168
|
+
return {
|
|
169
|
+
configured: ownerConfigured || vocabularyTerms > 0 || Object.keys(getWhisperCorrections()).length > 0,
|
|
170
|
+
ownerConfigured,
|
|
171
|
+
vocabularyTerms,
|
|
172
|
+
ignoredPlaceholderTerms,
|
|
173
|
+
ignoredPlaceholderCorrection,
|
|
174
|
+
}
|
|
96
175
|
}
|
|
97
176
|
|
|
98
177
|
export function getSystemContext(): string {
|
|
@@ -12,7 +12,7 @@ import { writeFileSync, unlinkSync, existsSync, readFileSync } from 'node:fs'
|
|
|
12
12
|
import { basename, join } from 'node:path'
|
|
13
13
|
import { homedir } from 'node:os'
|
|
14
14
|
import crypto from 'node:crypto'
|
|
15
|
-
import { getVocabulary, getOwnerName } from './profile.js'
|
|
15
|
+
import { getVocabulary, getOwnerName, getWhisperCorrections } from './profile.js'
|
|
16
16
|
import { stripBrandUrls } from './hallucination-filter.js'
|
|
17
17
|
import {
|
|
18
18
|
batchHqMetalEnabled,
|
|
@@ -1022,22 +1022,15 @@ async function transcribeViaCLI(audioBuffer: Buffer, context?: string, isQuiet?:
|
|
|
1022
1022
|
// Post-processing correction dictionary — deterministic fixes for names Whisper garbles.
|
|
1023
1023
|
// Prompt biasing is probabilistic; regex replacement is guaranteed.
|
|
1024
1024
|
// User-specific corrections loaded from .cos-profile.json "whisper_corrections" field.
|
|
1025
|
-
import { loadProfileField } from './profile.js'
|
|
1026
|
-
|
|
1027
1025
|
function buildCorrections(): Array<[RegExp, string]> {
|
|
1028
1026
|
const corrections: Array<[RegExp, string]> = []
|
|
1029
1027
|
|
|
1030
|
-
//
|
|
1031
|
-
//
|
|
1032
|
-
|
|
1033
|
-
const
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
for (const [pattern, replacement] of Object.entries(map)) {
|
|
1037
|
-
corrections.push([new RegExp(`\\b${pattern}\\b`, 'gi'), replacement])
|
|
1038
|
-
}
|
|
1039
|
-
}
|
|
1040
|
-
} catch { /* invalid JSON — skip */ }
|
|
1028
|
+
// Escape correction keys before interpolation. Names such as "A.C.M.E."
|
|
1029
|
+
// are literal vocabulary, never regular-expression programs.
|
|
1030
|
+
for (const [pattern, replacement] of Object.entries(getWhisperCorrections())) {
|
|
1031
|
+
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
1032
|
+
corrections.push([new RegExp(`\\b${escaped}\\b`, 'gi'), replacement])
|
|
1033
|
+
}
|
|
1041
1034
|
|
|
1042
1035
|
return corrections
|
|
1043
1036
|
}
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
// Adaptive provisional transcription. This sidecar is deliberately isolated
|
|
2
|
+
// from the authoritative large-v3-turbo server in whisper-local.ts:
|
|
3
|
+
// small.en -> cosmetic prompt preview only
|
|
4
|
+
// turbo -> committed live transcript (unchanged)
|
|
5
|
+
// large-v3 -> HQ save/polish (unchanged)
|
|
6
|
+
|
|
7
|
+
import { spawn } from 'node:child_process'
|
|
8
|
+
import type { ChildProcess } from 'node:child_process'
|
|
9
|
+
import { existsSync } from 'node:fs'
|
|
10
|
+
import { homedir } from 'node:os'
|
|
11
|
+
import { join } from 'node:path'
|
|
12
|
+
import { applyCorrections, getWhisperHealth, transcribeLocal } from './whisper-local.js'
|
|
13
|
+
import { getOwnerName, getVocabulary } from './profile.js'
|
|
14
|
+
|
|
15
|
+
export type WhisperPreviewRequest = 'auto' | 'small.en' | 'turbo' | 'off'
|
|
16
|
+
export type WhisperPreviewModel = 'small.en' | 'large-v3-turbo' | null
|
|
17
|
+
export type WhisperPreviewReason =
|
|
18
|
+
| 'disabled'
|
|
19
|
+
| 'small_model_missing'
|
|
20
|
+
| 'preview_binary_missing'
|
|
21
|
+
| 'preview_port_busy'
|
|
22
|
+
| 'preview_start_failed'
|
|
23
|
+
| 'preview_sidecar_unavailable'
|
|
24
|
+
| 'turbo_unavailable'
|
|
25
|
+
| null
|
|
26
|
+
|
|
27
|
+
export interface WhisperPreviewCapability {
|
|
28
|
+
requested: WhisperPreviewRequest
|
|
29
|
+
effectiveModel: WhisperPreviewModel
|
|
30
|
+
ready: boolean
|
|
31
|
+
backend: 'whisper-preview-server' | 'whisper-server' | null
|
|
32
|
+
degraded: boolean
|
|
33
|
+
reason: WhisperPreviewReason
|
|
34
|
+
committedModel: 'large-v3-turbo'
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const MODEL_DIR = join(process.env.HOME ?? homedir(), '.local/share/whisper-models')
|
|
38
|
+
export const WHISPER_SMALL_EN_MODEL_PATH = join(MODEL_DIR, 'ggml-small.en.bin')
|
|
39
|
+
const VAD_MODEL_PATH = join(MODEL_DIR, 'ggml-silero-v5.1.2.bin')
|
|
40
|
+
const VAD_ENABLED = process.env.COS_WHISPER_VAD !== '0'
|
|
41
|
+
const WHISPER_SERVER = ['/opt/homebrew/bin/whisper-server', '/usr/local/bin/whisper-server']
|
|
42
|
+
.find(existsSync) ?? '/opt/homebrew/bin/whisper-server'
|
|
43
|
+
const PREVIEW_PORT = 8177
|
|
44
|
+
const PREVIEW_URL = `http://127.0.0.1:${PREVIEW_PORT}`
|
|
45
|
+
|
|
46
|
+
let previewProcess: ChildProcess | null = null
|
|
47
|
+
let previewAvailable = false
|
|
48
|
+
let previewStarting = false
|
|
49
|
+
let previewFailure: WhisperPreviewReason = null
|
|
50
|
+
let warnedInvalidChoice = false
|
|
51
|
+
|
|
52
|
+
export function normalizeWhisperPreviewRequest(raw?: string): WhisperPreviewRequest {
|
|
53
|
+
const value = raw?.trim().toLowerCase()
|
|
54
|
+
// Backward compatibility is deliberate: simply updating the server keeps
|
|
55
|
+
// the old Turbo preview. Guided Setup opts the user into Small.en.
|
|
56
|
+
if (!value) return 'turbo'
|
|
57
|
+
if (value === 'auto' || value === 'adaptive') return 'auto'
|
|
58
|
+
if (value === 'small' || value === 'small.en' || value === 'ggml-small.en.bin') return 'small.en'
|
|
59
|
+
if (value === 'turbo' || value === 'large-v3-turbo' || value === 'ggml-large-v3-turbo.bin') return 'turbo'
|
|
60
|
+
if (value === 'off' || value === 'disabled' || value === '0') return 'off'
|
|
61
|
+
return 'auto'
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function requestedPreviewModel(): WhisperPreviewRequest {
|
|
65
|
+
const raw = process.env.COS_WHISPER_PREVIEW_MODEL
|
|
66
|
+
?? process.env.COS_WHISPER_REALTIME_MODEL // migration alias for early private installs
|
|
67
|
+
const normalized = normalizeWhisperPreviewRequest(raw)
|
|
68
|
+
if (raw && normalized === 'auto' && !['auto', 'adaptive'].includes(raw.trim().toLowerCase()) && !warnedInvalidChoice) {
|
|
69
|
+
warnedInvalidChoice = true
|
|
70
|
+
console.warn(`[whisper-preview] Unknown model "${raw}"; using adaptive selection.`)
|
|
71
|
+
}
|
|
72
|
+
return normalized
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function selectedPreviewModel(requested = requestedPreviewModel()): WhisperPreviewModel {
|
|
76
|
+
if (requested === 'off') return null
|
|
77
|
+
if (requested === 'turbo') return 'large-v3-turbo'
|
|
78
|
+
if (requested === 'small.en') return existsSync(WHISPER_SMALL_EN_MODEL_PATH) ? 'small.en' : 'large-v3-turbo'
|
|
79
|
+
return existsSync(WHISPER_SMALL_EN_MODEL_PATH) ? 'small.en' : 'large-v3-turbo'
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function getWhisperPreviewCapability(): WhisperPreviewCapability {
|
|
83
|
+
const requested = requestedPreviewModel()
|
|
84
|
+
if (requested === 'off') {
|
|
85
|
+
return {
|
|
86
|
+
requested, effectiveModel: null, ready: false, backend: null,
|
|
87
|
+
degraded: false, reason: 'disabled', committedModel: 'large-v3-turbo',
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const smallPresent = existsSync(WHISPER_SMALL_EN_MODEL_PATH)
|
|
92
|
+
const selected = selectedPreviewModel(requested)
|
|
93
|
+
const turboReady = getWhisperHealth().server
|
|
94
|
+
if (selected === 'small.en' && previewAvailable) {
|
|
95
|
+
return {
|
|
96
|
+
requested, effectiveModel: 'small.en', ready: true,
|
|
97
|
+
backend: 'whisper-preview-server', degraded: false, reason: null,
|
|
98
|
+
committedModel: 'large-v3-turbo',
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const smallWasExpected = requested === 'small.en' || (requested === 'auto' && smallPresent)
|
|
103
|
+
const reason: WhisperPreviewReason = requested === 'small.en' && !smallPresent
|
|
104
|
+
? 'small_model_missing'
|
|
105
|
+
: smallWasExpected
|
|
106
|
+
? (previewFailure ?? (previewStarting ? null : 'preview_sidecar_unavailable'))
|
|
107
|
+
: turboReady ? null : 'turbo_unavailable'
|
|
108
|
+
return {
|
|
109
|
+
requested,
|
|
110
|
+
effectiveModel: turboReady ? 'large-v3-turbo' : selected,
|
|
111
|
+
ready: turboReady,
|
|
112
|
+
backend: turboReady ? 'whisper-server' : null,
|
|
113
|
+
degraded: smallWasExpected,
|
|
114
|
+
reason,
|
|
115
|
+
committedModel: 'large-v3-turbo',
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
async function endpointReady(path: '/health' | '/inference', init?: RequestInit, timeoutMs = 1_000): Promise<Response> {
|
|
120
|
+
return fetch(`${PREVIEW_URL}${path}`, { ...init, signal: AbortSignal.timeout(timeoutMs) })
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Start the optional small.en preview worker. Failure is cosmetic: committed
|
|
124
|
+
* Turbo and every recovery/finalization path stay untouched. */
|
|
125
|
+
export async function startWhisperPreviewServer(): Promise<void> {
|
|
126
|
+
const requested = requestedPreviewModel()
|
|
127
|
+
if (selectedPreviewModel(requested) !== 'small.en' || previewProcess || previewAvailable || previewStarting) return
|
|
128
|
+
if (!existsSync(WHISPER_SMALL_EN_MODEL_PATH)) {
|
|
129
|
+
previewFailure = 'small_model_missing'
|
|
130
|
+
return
|
|
131
|
+
}
|
|
132
|
+
if (!existsSync(WHISPER_SERVER)) {
|
|
133
|
+
previewFailure = 'preview_binary_missing'
|
|
134
|
+
return
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
previewStarting = true
|
|
138
|
+
previewFailure = null
|
|
139
|
+
try {
|
|
140
|
+
try {
|
|
141
|
+
const occupied = await endpointReady('/health')
|
|
142
|
+
if (occupied.ok) {
|
|
143
|
+
previewFailure = 'preview_port_busy'
|
|
144
|
+
return
|
|
145
|
+
}
|
|
146
|
+
} catch { /* clear port is expected */ }
|
|
147
|
+
|
|
148
|
+
const args = [
|
|
149
|
+
'-m', WHISPER_SMALL_EN_MODEL_PATH,
|
|
150
|
+
'-t', '16',
|
|
151
|
+
'-l', 'en',
|
|
152
|
+
'-fa',
|
|
153
|
+
'--no-speech-thold', '0.7',
|
|
154
|
+
'--host', '127.0.0.1',
|
|
155
|
+
'--port', String(PREVIEW_PORT),
|
|
156
|
+
]
|
|
157
|
+
if (VAD_ENABLED && existsSync(VAD_MODEL_PATH)) {
|
|
158
|
+
args.push('--vad', '--vad-model', VAD_MODEL_PATH)
|
|
159
|
+
}
|
|
160
|
+
const child = spawn(WHISPER_SERVER, args, { stdio: 'ignore', detached: false })
|
|
161
|
+
previewProcess = child
|
|
162
|
+
child.once('close', code => {
|
|
163
|
+
if (previewProcess !== child) return
|
|
164
|
+
previewProcess = null
|
|
165
|
+
previewAvailable = false
|
|
166
|
+
previewFailure = code === 0 ? 'preview_sidecar_unavailable' : 'preview_start_failed'
|
|
167
|
+
})
|
|
168
|
+
child.once('error', () => {
|
|
169
|
+
previewAvailable = false
|
|
170
|
+
previewFailure = 'preview_start_failed'
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
const deadline = Date.now() + 45_000
|
|
174
|
+
while (Date.now() < deadline) {
|
|
175
|
+
if (child.exitCode !== null || child.signalCode !== null) break
|
|
176
|
+
try {
|
|
177
|
+
const response = await endpointReady('/health', undefined, 1_000)
|
|
178
|
+
if (response.ok) {
|
|
179
|
+
previewAvailable = true
|
|
180
|
+
previewFailure = null
|
|
181
|
+
console.log('[whisper-preview] small.en ready for provisional text; committed text remains Turbo')
|
|
182
|
+
return
|
|
183
|
+
}
|
|
184
|
+
} catch { /* model still loading */ }
|
|
185
|
+
await new Promise(resolve => setTimeout(resolve, 1_000))
|
|
186
|
+
}
|
|
187
|
+
try { child.kill('SIGKILL') } catch { /* already exited */ }
|
|
188
|
+
if (previewProcess === child) previewProcess = null
|
|
189
|
+
previewFailure = 'preview_start_failed'
|
|
190
|
+
} finally {
|
|
191
|
+
previewStarting = false
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
export function stopWhisperPreviewServer(): void {
|
|
196
|
+
const child = previewProcess
|
|
197
|
+
previewProcess = null
|
|
198
|
+
previewAvailable = false
|
|
199
|
+
previewStarting = false
|
|
200
|
+
if (child) {
|
|
201
|
+
try { child.kill('SIGTERM') } catch { /* already exited */ }
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function previewPrompt(): string {
|
|
206
|
+
const vocabulary = getVocabulary()
|
|
207
|
+
return vocabulary.length > 0
|
|
208
|
+
? [getOwnerName(), ...vocabulary].join(', ')
|
|
209
|
+
: `${getOwnerName()}. COS Glasses. Even G2.`
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
async function transcribeViaPreviewServer(audioBuffer: Buffer): Promise<string> {
|
|
213
|
+
const formData = new FormData()
|
|
214
|
+
formData.append('file', new Blob([new Uint8Array(audioBuffer)], { type: 'audio/wav' }), 'recording.wav')
|
|
215
|
+
formData.append('response_format', 'json')
|
|
216
|
+
formData.append('prompt', previewPrompt())
|
|
217
|
+
formData.append('suppress_non_speech', 'true')
|
|
218
|
+
const response = await endpointReady('/inference', { method: 'POST', body: formData }, 5_000)
|
|
219
|
+
if (!response.ok) throw new Error(`preview server ${response.status}`)
|
|
220
|
+
const result = await response.json() as { text?: unknown }
|
|
221
|
+
if (typeof result.text !== 'string') throw new Error('preview server returned invalid text')
|
|
222
|
+
return applyCorrections(result.text.trim())
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Cosmetic preview only. A small-worker failure falls through to the existing
|
|
226
|
+
* non-circuit Turbo decode and can never write committed transcript state. */
|
|
227
|
+
export async function transcribeWhisperPreview(audioBuffer: Buffer): Promise<{
|
|
228
|
+
text: string
|
|
229
|
+
model: 'small.en' | 'large-v3-turbo'
|
|
230
|
+
backend: 'whisper-preview-server' | 'whisper-server'
|
|
231
|
+
}> {
|
|
232
|
+
if (previewAvailable) {
|
|
233
|
+
try {
|
|
234
|
+
return { text: await transcribeViaPreviewServer(audioBuffer), model: 'small.en', backend: 'whisper-preview-server' }
|
|
235
|
+
} catch (error) {
|
|
236
|
+
previewAvailable = false
|
|
237
|
+
previewFailure = 'preview_sidecar_unavailable'
|
|
238
|
+
console.warn(`[whisper-preview] small.en preview failed; falling back to Turbo: ${error instanceof Error ? error.message : error}`)
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
const result = await transcribeLocal(audioBuffer, undefined, undefined, { affectsCircuit: false })
|
|
242
|
+
return { text: result.text, model: 'large-v3-turbo', backend: 'whisper-server' }
|
|
243
|
+
}
|
|
@@ -13,7 +13,7 @@ import { errMsg } from '../lib/utils.js'
|
|
|
13
13
|
import {
|
|
14
14
|
getVocabulary,
|
|
15
15
|
getNegativeRules,
|
|
16
|
-
|
|
16
|
+
getWhisperCorrections,
|
|
17
17
|
updateProfileFields,
|
|
18
18
|
} from '../lib/profile.js'
|
|
19
19
|
import { resetDecoderCaches } from '../lib/whisper-local.js'
|
|
@@ -36,16 +36,7 @@ function looksLikeUrlEmailPath(s: string): boolean {
|
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
function readCorrections(): Record<string, string> {
|
|
39
|
-
|
|
40
|
-
const raw = loadProfileField('whisper_corrections', '')
|
|
41
|
-
if (!raw) return {}
|
|
42
|
-
const parsed = JSON.parse(raw)
|
|
43
|
-
return (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
|
|
44
|
-
? parsed as Record<string, string>
|
|
45
|
-
: {}
|
|
46
|
-
} catch {
|
|
47
|
-
return {}
|
|
48
|
-
}
|
|
39
|
+
return getWhisperCorrections()
|
|
49
40
|
}
|
|
50
41
|
|
|
51
42
|
function currentGlossary() {
|
package/server/routes/health.ts
CHANGED
|
@@ -39,6 +39,8 @@ import { getLocalTtsHealth, refreshLocalTtsHealth } from '../lib/tts-local.js'
|
|
|
39
39
|
import { liveCuesCapability } from '../lib/live-cues-capability.js'
|
|
40
40
|
import { getMeetingSyncSnapshot } from '../lib/meeting-batch-progress.js'
|
|
41
41
|
import { listUnsavedCaptures } from '../lib/unsaved-audio-quarantine.js'
|
|
42
|
+
import { getWhisperPreviewCapability } from '../lib/whisper-preview.js'
|
|
43
|
+
import { getTranscriptionProfileStatus } from '../lib/profile.js'
|
|
42
44
|
|
|
43
45
|
export const healthRouter = Router()
|
|
44
46
|
|
|
@@ -203,6 +205,8 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
203
205
|
const localFirstMeetings = localFirstMeetingsCapability(getServerInstanceId())
|
|
204
206
|
const transcription = getTranscriptionPolicySnapshot()
|
|
205
207
|
const transcriptionHq = getHighQualityTranscriptionCapability()
|
|
208
|
+
const transcriptionLive = getWhisperPreviewCapability()
|
|
209
|
+
const transcriptionProfile = getTranscriptionProfileStatus()
|
|
206
210
|
const recovery = managedRuntimeCapability()
|
|
207
211
|
const maintenance = maintenanceLifecycle.snapshot()
|
|
208
212
|
const tts_local = getLocalTtsHealth()
|
|
@@ -293,7 +297,12 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
293
297
|
meeting_sync,
|
|
294
298
|
unsaved_captures,
|
|
295
299
|
capabilities: {
|
|
296
|
-
transcription: {
|
|
300
|
+
transcription: {
|
|
301
|
+
...transcription,
|
|
302
|
+
live: transcriptionLive,
|
|
303
|
+
hq: transcriptionHq,
|
|
304
|
+
profile: transcriptionProfile,
|
|
305
|
+
},
|
|
297
306
|
recovery,
|
|
298
307
|
maintenance: {
|
|
299
308
|
state: maintenance.state,
|
|
@@ -326,6 +335,8 @@ healthRouter.get('/models', async (req, res) => {
|
|
|
326
335
|
const localFirstMeetings = localFirstMeetingsCapability(getServerInstanceId())
|
|
327
336
|
const transcription = getTranscriptionPolicySnapshot()
|
|
328
337
|
const transcriptionHq = getHighQualityTranscriptionCapability()
|
|
338
|
+
const transcriptionLive = getWhisperPreviewCapability()
|
|
339
|
+
const transcriptionProfile = getTranscriptionProfileStatus()
|
|
329
340
|
const cursorOptions = cursorCatalog.options.filter(option => !!option.id)
|
|
330
341
|
res.json({
|
|
331
342
|
...catalog,
|
|
@@ -341,7 +352,12 @@ healthRouter.get('/models', async (req, res) => {
|
|
|
341
352
|
enabled: durableJobs.enabled,
|
|
342
353
|
protocolVersion: durableJobs.protocolVersion,
|
|
343
354
|
},
|
|
344
|
-
transcription: {
|
|
355
|
+
transcription: {
|
|
356
|
+
...transcription,
|
|
357
|
+
live: transcriptionLive,
|
|
358
|
+
hq: transcriptionHq,
|
|
359
|
+
profile: transcriptionProfile,
|
|
360
|
+
},
|
|
345
361
|
cliDebug: CLI_DEBUG_CAPABILITY,
|
|
346
362
|
recovery: managedRuntimeCapability(),
|
|
347
363
|
// Same helper as /api/health — the companion's 15s liveness poll reads
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
applyNegativeRules,
|
|
35
35
|
} from '../lib/hallucination-filter.js'
|
|
36
36
|
import { applyCorrections } from '../lib/whisper-local.js'
|
|
37
|
+
import { transcribeWhisperPreview } from '../lib/whisper-preview.js'
|
|
37
38
|
import { autoCleanDictation, AUTOCLEAN_MAX_CHARS } from '../lib/dictation-clean.js'
|
|
38
39
|
import { getVocabulary } from '../lib/profile.js'
|
|
39
40
|
import { createBreaker } from '../lib/claude-circuit.js'
|
|
@@ -414,11 +415,7 @@ promptDraftsRouter.post('/prompt-drafts/:draftId/peek', async (req, res) => {
|
|
|
414
415
|
peekTail = peekTail.then(async () => {
|
|
415
416
|
lease.setPhase('active')
|
|
416
417
|
try {
|
|
417
|
-
const result = await
|
|
418
|
-
mode: 'fast',
|
|
419
|
-
policy: 'local-only',
|
|
420
|
-
affectsCircuit: false,
|
|
421
|
-
})
|
|
418
|
+
const result = await transcribeWhisperPreview(audio)
|
|
422
419
|
const text = sanitizeTranscript(draftId, result.text, false)
|
|
423
420
|
if (!text) return
|
|
424
421
|
if (!loadPromptDraftMeta(draftId)) return
|