@gotcos/glasses-server 6.18.8 → 6.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cos-profile.example.json +3 -3
- package/.env.example +18 -0
- package/CHANGELOG.md +78 -0
- package/README.md +23 -0
- package/bin/cli.cjs +156 -4
- package/managed-runtime-contract.json +3 -0
- package/package.json +1 -1
- package/server/index.ts +16 -1
- package/server/lib/hallucination-filter.ts +5 -8
- package/server/lib/maintenance-lifecycle.ts +8 -0
- package/server/lib/meeting-batch-progress.ts +150 -3
- package/server/lib/meeting-batch-transcribe.ts +29 -2
- package/server/lib/profile.ts +81 -2
- package/server/lib/unsaved-audio-quarantine.ts +314 -0
- package/server/lib/whisper-local.ts +7 -14
- package/server/lib/whisper-preview.ts +243 -0
- package/server/routes/glossary.ts +2 -11
- package/server/routes/health.ts +34 -2
- package/server/routes/meeting.ts +281 -2
- package/server/routes/prompt-drafts.ts +2 -5
- package/server/routes/transcribe-stream.ts +65 -13
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
// Quarantine for unsaved meeting audio.
|
|
2
|
+
//
|
|
3
|
+
// Until 6.19.0 the session-audio purge (boot sweep + 60s interval) DELETED any
|
|
4
|
+
// session-audio directory that was not in the in-memory sessions map and did
|
|
5
|
+
// not carry a fresh save-preserved marker. An offline meeting whose deferred
|
|
6
|
+
// save never landed therefore lost its full-fidelity audio within a minute of
|
|
7
|
+
// the server no longer tracking the session — two real meetings were destroyed
|
|
8
|
+
// this way on 2026-08-01. The only surviving fragments were in ext-audio,
|
|
9
|
+
// which is a speaker-enrollment store (unrecognized-speaker chunks, hard
|
|
10
|
+
// capped), not meeting audio.
|
|
11
|
+
//
|
|
12
|
+
// The rule now: audio evidence is moved here, never deleted in place. A
|
|
13
|
+
// quarantined capture is surfaced on /api/health (unsaved_captures) and can be
|
|
14
|
+
// driven to a durable scribe via POST /api/meeting/orphans/:sessionId/recover.
|
|
15
|
+
// Quarantine expires on a retention clock (default 72h) — long enough to
|
|
16
|
+
// survive a weekend away from the Mac, bounded enough not to grow forever.
|
|
17
|
+
|
|
18
|
+
import {
|
|
19
|
+
existsSync,
|
|
20
|
+
mkdirSync,
|
|
21
|
+
readdirSync,
|
|
22
|
+
readFileSync,
|
|
23
|
+
renameSync,
|
|
24
|
+
rmSync,
|
|
25
|
+
statSync,
|
|
26
|
+
writeFileSync,
|
|
27
|
+
} from 'node:fs'
|
|
28
|
+
import { basename, join, resolve } from 'node:path'
|
|
29
|
+
import { dataPath } from './data-dir.js'
|
|
30
|
+
|
|
31
|
+
export const QUARANTINE_MANIFEST = '_quarantine.json'
|
|
32
|
+
export const RECOVERED_RECEIPT = '_recovered.json'
|
|
33
|
+
const CHUNK_WAV_RE = /^chunk_\d{4}\.wav$/
|
|
34
|
+
|
|
35
|
+
const DEFAULT_RETENTION_HOURS = 72
|
|
36
|
+
const MIN_RETENTION_HOURS = 1
|
|
37
|
+
const MAX_RETENTION_HOURS = 720
|
|
38
|
+
|
|
39
|
+
export interface QuarantineManifest {
|
|
40
|
+
schemaVersion: 1
|
|
41
|
+
sessionId: string
|
|
42
|
+
quarantinedAt: string
|
|
43
|
+
reason: string
|
|
44
|
+
chunkFiles: number
|
|
45
|
+
bytes: number
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface UnsavedCapture {
|
|
49
|
+
sessionId: string
|
|
50
|
+
dirName: string
|
|
51
|
+
quarantinedAt: string | null
|
|
52
|
+
ageHours: number | null
|
|
53
|
+
chunkFiles: number
|
|
54
|
+
bytes: number
|
|
55
|
+
reason: string | null
|
|
56
|
+
expiresAt: string | null
|
|
57
|
+
recovered: boolean
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function unsavedAudioRoot(): string {
|
|
61
|
+
return dataPath('unsaved-audio')
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function unsavedAudioRetentionMs(env: NodeJS.ProcessEnv = process.env): number {
|
|
65
|
+
// An EMPTY value means unset, not zero: Number('') === 0, which would clamp
|
|
66
|
+
// to the 1h minimum and silently collapse the 72h safety net — on the one
|
|
67
|
+
// knob whose failure mode is losing the audio this module exists to keep.
|
|
68
|
+
const rawText = (env.COS_UNSAVED_AUDIO_RETENTION_HOURS ?? '').trim()
|
|
69
|
+
if (rawText === '') return DEFAULT_RETENTION_HOURS * 60 * 60 * 1000
|
|
70
|
+
const raw = Number(rawText)
|
|
71
|
+
const hours = Number.isFinite(raw)
|
|
72
|
+
? Math.min(MAX_RETENTION_HOURS, Math.max(MIN_RETENTION_HOURS, raw))
|
|
73
|
+
: DEFAULT_RETENTION_HOURS
|
|
74
|
+
return hours * 60 * 60 * 1000
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function countChunkWavs(dirPath: string): number {
|
|
78
|
+
try {
|
|
79
|
+
return readdirSync(dirPath).filter(name => CHUNK_WAV_RE.test(name)).length
|
|
80
|
+
} catch {
|
|
81
|
+
return 0
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function dirBytes(dirPath: string): number {
|
|
86
|
+
let total = 0
|
|
87
|
+
try {
|
|
88
|
+
for (const name of readdirSync(dirPath)) {
|
|
89
|
+
try { total += statSync(join(dirPath, name)).size } catch { /* skip */ }
|
|
90
|
+
}
|
|
91
|
+
} catch { /* empty */ }
|
|
92
|
+
return total
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Move one session-audio dir into quarantine instead of deleting it.
|
|
96
|
+
* Returns the quarantine path, or null when the move could not be made
|
|
97
|
+
* (in which case the SOURCE IS LEFT IN PLACE — never deleted on failure). */
|
|
98
|
+
export function quarantineSessionAudio(
|
|
99
|
+
sourceDir: string,
|
|
100
|
+
reason: string,
|
|
101
|
+
root: string = unsavedAudioRoot(),
|
|
102
|
+
): string | null {
|
|
103
|
+
const sessionId = basename(sourceDir)
|
|
104
|
+
try {
|
|
105
|
+
if (!existsSync(sourceDir)) return null
|
|
106
|
+
mkdirSync(root, { recursive: true, mode: 0o700 })
|
|
107
|
+
let target = resolve(root, sessionId)
|
|
108
|
+
if (existsSync(target)) target = resolve(root, `${sessionId}.${Date.now()}`)
|
|
109
|
+
renameSync(sourceDir, target)
|
|
110
|
+
const manifest: QuarantineManifest = {
|
|
111
|
+
schemaVersion: 1,
|
|
112
|
+
sessionId,
|
|
113
|
+
quarantinedAt: new Date().toISOString(),
|
|
114
|
+
reason,
|
|
115
|
+
chunkFiles: countChunkWavs(target),
|
|
116
|
+
bytes: dirBytes(target),
|
|
117
|
+
}
|
|
118
|
+
try {
|
|
119
|
+
writeFileSync(resolve(target, QUARANTINE_MANIFEST), `${JSON.stringify(manifest)}\n`, {
|
|
120
|
+
encoding: 'utf8',
|
|
121
|
+
mode: 0o600,
|
|
122
|
+
})
|
|
123
|
+
} catch { /* manifest is observability; the audio move already succeeded */ }
|
|
124
|
+
return target
|
|
125
|
+
} catch {
|
|
126
|
+
// Rename failed (cross-device, permissions, race). Leave the source alone —
|
|
127
|
+
// a skipped purge is recoverable, a deleted capture is not.
|
|
128
|
+
return null
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export type OrphanSweepAction =
|
|
133
|
+
| { dir: string; action: 'kept_live' }
|
|
134
|
+
| { dir: string; action: 'kept_preserved' }
|
|
135
|
+
| { dir: string; action: 'quarantined'; target: string }
|
|
136
|
+
| { dir: string; action: 'quarantine_failed' }
|
|
137
|
+
| { dir: string; action: 'deleted_empty' }
|
|
138
|
+
|
|
139
|
+
/** Shared sweep for the boot path and the 60s interval. A directory that
|
|
140
|
+
* still holds chunk audio is quarantined; only chunk-less directories are
|
|
141
|
+
* deleted. A failed quarantine move keeps the source in place. */
|
|
142
|
+
export function sweepOrphanedSessionAudio(
|
|
143
|
+
sessionAudioRoot: string,
|
|
144
|
+
opts: {
|
|
145
|
+
isLive: (sessionId: string) => boolean
|
|
146
|
+
hasFreshPreservedMarker: (dirPath: string) => boolean
|
|
147
|
+
reason: string
|
|
148
|
+
quarantineRoot?: string
|
|
149
|
+
},
|
|
150
|
+
): OrphanSweepAction[] {
|
|
151
|
+
const actions: OrphanSweepAction[] = []
|
|
152
|
+
let entries: string[] = []
|
|
153
|
+
try {
|
|
154
|
+
entries = readdirSync(sessionAudioRoot)
|
|
155
|
+
} catch {
|
|
156
|
+
return actions
|
|
157
|
+
}
|
|
158
|
+
for (const dir of entries) {
|
|
159
|
+
const dirPath = resolve(sessionAudioRoot, dir)
|
|
160
|
+
try {
|
|
161
|
+
if (!statSync(dirPath).isDirectory()) continue
|
|
162
|
+
} catch { continue }
|
|
163
|
+
if (opts.isLive(dir)) { actions.push({ dir, action: 'kept_live' }); continue }
|
|
164
|
+
if (opts.hasFreshPreservedMarker(dirPath)) { actions.push({ dir, action: 'kept_preserved' }); continue }
|
|
165
|
+
if (countChunkWavs(dirPath) > 0) {
|
|
166
|
+
const target = quarantineSessionAudio(dirPath, opts.reason, opts.quarantineRoot)
|
|
167
|
+
actions.push(target
|
|
168
|
+
? { dir, action: 'quarantined', target }
|
|
169
|
+
: { dir, action: 'quarantine_failed' })
|
|
170
|
+
continue
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
rmSync(dirPath, { recursive: true, force: true })
|
|
174
|
+
actions.push({ dir, action: 'deleted_empty' })
|
|
175
|
+
} catch { /* next sweep retries */ }
|
|
176
|
+
}
|
|
177
|
+
return actions
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function readManifest(dirPath: string): QuarantineManifest | null {
|
|
181
|
+
try {
|
|
182
|
+
const raw = JSON.parse(readFileSync(resolve(dirPath, QUARANTINE_MANIFEST), 'utf8')) as QuarantineManifest
|
|
183
|
+
if (raw?.schemaVersion !== 1 || typeof raw.sessionId !== 'string') return null
|
|
184
|
+
return raw
|
|
185
|
+
} catch {
|
|
186
|
+
return null
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function quarantinedAtMs(dirPath: string, manifest: QuarantineManifest | null): number | null {
|
|
191
|
+
if (manifest) {
|
|
192
|
+
const parsed = Date.parse(manifest.quarantinedAt)
|
|
193
|
+
if (Number.isFinite(parsed)) return parsed
|
|
194
|
+
}
|
|
195
|
+
try {
|
|
196
|
+
return statSync(dirPath).mtimeMs
|
|
197
|
+
} catch {
|
|
198
|
+
return null
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Everything currently in quarantine, newest first. Recovered captures are
|
|
203
|
+
* flagged (they linger until the retention clock clears them) so health can
|
|
204
|
+
* exclude them from the actionable count. */
|
|
205
|
+
export function listUnsavedCaptures(
|
|
206
|
+
root: string = unsavedAudioRoot(),
|
|
207
|
+
retentionMs: number = unsavedAudioRetentionMs(),
|
|
208
|
+
): UnsavedCapture[] {
|
|
209
|
+
const captures: UnsavedCapture[] = []
|
|
210
|
+
let entries: string[] = []
|
|
211
|
+
try {
|
|
212
|
+
entries = readdirSync(root)
|
|
213
|
+
} catch {
|
|
214
|
+
return captures
|
|
215
|
+
}
|
|
216
|
+
for (const dir of entries) {
|
|
217
|
+
const dirPath = resolve(root, dir)
|
|
218
|
+
try {
|
|
219
|
+
if (!statSync(dirPath).isDirectory()) continue
|
|
220
|
+
} catch { continue }
|
|
221
|
+
const manifest = readManifest(dirPath)
|
|
222
|
+
const atMs = quarantinedAtMs(dirPath, manifest)
|
|
223
|
+
captures.push({
|
|
224
|
+
sessionId: manifest?.sessionId ?? dir.replace(/\.\d+$/, ''),
|
|
225
|
+
dirName: dir,
|
|
226
|
+
quarantinedAt: atMs != null ? new Date(atMs).toISOString() : null,
|
|
227
|
+
ageHours: atMs != null ? Math.round(((Date.now() - atMs) / 3_600_000) * 10) / 10 : null,
|
|
228
|
+
chunkFiles: manifest?.chunkFiles ?? countChunkWavs(dirPath),
|
|
229
|
+
bytes: manifest?.bytes ?? dirBytes(dirPath),
|
|
230
|
+
reason: manifest?.reason ?? null,
|
|
231
|
+
expiresAt: atMs != null ? new Date(atMs + retentionMs).toISOString() : null,
|
|
232
|
+
recovered: existsSync(resolve(dirPath, RECOVERED_RECEIPT)),
|
|
233
|
+
})
|
|
234
|
+
}
|
|
235
|
+
captures.sort((a, b) => (b.quarantinedAt ?? '').localeCompare(a.quarantinedAt ?? ''))
|
|
236
|
+
return captures
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Retention: quarantined captures past the clock are removed. This is the
|
|
240
|
+
* ONLY place quarantined audio is ever deleted. */
|
|
241
|
+
export function purgeExpiredQuarantine(
|
|
242
|
+
root: string = unsavedAudioRoot(),
|
|
243
|
+
retentionMs: number = unsavedAudioRetentionMs(),
|
|
244
|
+
): string[] {
|
|
245
|
+
const purged: string[] = []
|
|
246
|
+
for (const capture of listUnsavedCaptures(root, retentionMs)) {
|
|
247
|
+
// Never delete a directory a recovery is actively decoding — a capture
|
|
248
|
+
// sitting exactly at the retention boundary would otherwise lose its
|
|
249
|
+
// audio mid-run, in the module whose thesis is "never delete evidence".
|
|
250
|
+
if (isRecoveryActive(capture.dirName)) continue
|
|
251
|
+
const atMs = capture.quarantinedAt ? Date.parse(capture.quarantinedAt) : NaN
|
|
252
|
+
if (!Number.isFinite(atMs)) continue
|
|
253
|
+
if (Date.now() - atMs > retentionMs) {
|
|
254
|
+
try {
|
|
255
|
+
rmSync(resolve(root, capture.dirName), { recursive: true, force: true })
|
|
256
|
+
purged.push(capture.dirName)
|
|
257
|
+
} catch { /* next sweep retries */ }
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
return purged
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** Resolve the quarantine dir for a sessionId (exact dir or timestamp-suffixed). */
|
|
264
|
+
export function findQuarantineDir(
|
|
265
|
+
sessionId: string,
|
|
266
|
+
root: string = unsavedAudioRoot(),
|
|
267
|
+
): string | null {
|
|
268
|
+
const exact = resolve(root, sessionId)
|
|
269
|
+
if (existsSync(exact)) return exact
|
|
270
|
+
try {
|
|
271
|
+
for (const dir of readdirSync(root)) {
|
|
272
|
+
if (dir === sessionId || dir.startsWith(`${sessionId}.`)) {
|
|
273
|
+
const dirPath = resolve(root, dir)
|
|
274
|
+
if (statSync(dirPath).isDirectory()) return dirPath
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
} catch { /* fall through */ }
|
|
278
|
+
return null
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// ── Active recovery registry ─────────────────────────────────────────────
|
|
282
|
+
// In-memory, single-process (matches the sessions map's own scope). Two
|
|
283
|
+
// consumers: purgeExpiredQuarantine skips dirs being recovered (a capture at
|
|
284
|
+
// the retention boundary must not be deleted mid-decode), and meeting_sync
|
|
285
|
+
// renders an active row so COS Control warns BEFORE committing an Update
|
|
286
|
+
// Server drain into a 20-90 minute recovery it cannot see.
|
|
287
|
+
|
|
288
|
+
const activeRecoveries = new Map<string, { sessionId: string; dirPath: string; startedAt: number }>()
|
|
289
|
+
|
|
290
|
+
export function registerActiveRecovery(sessionId: string, dirPath: string): void {
|
|
291
|
+
activeRecoveries.set(basename(dirPath), { sessionId, dirPath, startedAt: Date.now() })
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
export function clearActiveRecovery(dirPath: string): void {
|
|
295
|
+
activeRecoveries.delete(basename(dirPath))
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export function listActiveRecoveries(): Array<{ sessionId: string; dirPath: string; startedAt: number }> {
|
|
299
|
+
return [...activeRecoveries.values()]
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export function isRecoveryActive(dirName: string): boolean {
|
|
303
|
+
return activeRecoveries.has(dirName)
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
export function markRecovered(dirPath: string, savedFilename: string): void {
|
|
307
|
+
try {
|
|
308
|
+
writeFileSync(
|
|
309
|
+
resolve(dirPath, RECOVERED_RECEIPT),
|
|
310
|
+
`${JSON.stringify({ schemaVersion: 1, recoveredAt: new Date().toISOString(), savedFilename })}\n`,
|
|
311
|
+
{ encoding: 'utf8', mode: 0o600 },
|
|
312
|
+
)
|
|
313
|
+
} catch { /* receipt is best-effort; findBySessionId remains the true guard */ }
|
|
314
|
+
}
|
|
@@ -12,7 +12,7 @@ import { writeFileSync, unlinkSync, existsSync, readFileSync } from 'node:fs'
|
|
|
12
12
|
import { basename, join } from 'node:path'
|
|
13
13
|
import { homedir } from 'node:os'
|
|
14
14
|
import crypto from 'node:crypto'
|
|
15
|
-
import { getVocabulary, getOwnerName } from './profile.js'
|
|
15
|
+
import { getVocabulary, getOwnerName, getWhisperCorrections } from './profile.js'
|
|
16
16
|
import { stripBrandUrls } from './hallucination-filter.js'
|
|
17
17
|
import {
|
|
18
18
|
batchHqMetalEnabled,
|
|
@@ -1022,22 +1022,15 @@ async function transcribeViaCLI(audioBuffer: Buffer, context?: string, isQuiet?:
|
|
|
1022
1022
|
// Post-processing correction dictionary — deterministic fixes for names Whisper garbles.
|
|
1023
1023
|
// Prompt biasing is probabilistic; regex replacement is guaranteed.
|
|
1024
1024
|
// User-specific corrections loaded from .cos-profile.json "whisper_corrections" field.
|
|
1025
|
-
import { loadProfileField } from './profile.js'
|
|
1026
|
-
|
|
1027
1025
|
function buildCorrections(): Array<[RegExp, string]> {
|
|
1028
1026
|
const corrections: Array<[RegExp, string]> = []
|
|
1029
1027
|
|
|
1030
|
-
//
|
|
1031
|
-
//
|
|
1032
|
-
|
|
1033
|
-
const
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
for (const [pattern, replacement] of Object.entries(map)) {
|
|
1037
|
-
corrections.push([new RegExp(`\\b${pattern}\\b`, 'gi'), replacement])
|
|
1038
|
-
}
|
|
1039
|
-
}
|
|
1040
|
-
} catch { /* invalid JSON — skip */ }
|
|
1028
|
+
// Escape correction keys before interpolation. Names such as "A.C.M.E."
|
|
1029
|
+
// are literal vocabulary, never regular-expression programs.
|
|
1030
|
+
for (const [pattern, replacement] of Object.entries(getWhisperCorrections())) {
|
|
1031
|
+
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
1032
|
+
corrections.push([new RegExp(`\\b${escaped}\\b`, 'gi'), replacement])
|
|
1033
|
+
}
|
|
1041
1034
|
|
|
1042
1035
|
return corrections
|
|
1043
1036
|
}
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
// Adaptive provisional transcription. This sidecar is deliberately isolated
|
|
2
|
+
// from the authoritative large-v3-turbo server in whisper-local.ts:
|
|
3
|
+
// small.en -> cosmetic prompt preview only
|
|
4
|
+
// turbo -> committed live transcript (unchanged)
|
|
5
|
+
// large-v3 -> HQ save/polish (unchanged)
|
|
6
|
+
|
|
7
|
+
import { spawn } from 'node:child_process'
|
|
8
|
+
import type { ChildProcess } from 'node:child_process'
|
|
9
|
+
import { existsSync } from 'node:fs'
|
|
10
|
+
import { homedir } from 'node:os'
|
|
11
|
+
import { join } from 'node:path'
|
|
12
|
+
import { applyCorrections, getWhisperHealth, transcribeLocal } from './whisper-local.js'
|
|
13
|
+
import { getOwnerName, getVocabulary } from './profile.js'
|
|
14
|
+
|
|
15
|
+
export type WhisperPreviewRequest = 'auto' | 'small.en' | 'turbo' | 'off'
|
|
16
|
+
export type WhisperPreviewModel = 'small.en' | 'large-v3-turbo' | null
|
|
17
|
+
export type WhisperPreviewReason =
|
|
18
|
+
| 'disabled'
|
|
19
|
+
| 'small_model_missing'
|
|
20
|
+
| 'preview_binary_missing'
|
|
21
|
+
| 'preview_port_busy'
|
|
22
|
+
| 'preview_start_failed'
|
|
23
|
+
| 'preview_sidecar_unavailable'
|
|
24
|
+
| 'turbo_unavailable'
|
|
25
|
+
| null
|
|
26
|
+
|
|
27
|
+
export interface WhisperPreviewCapability {
|
|
28
|
+
requested: WhisperPreviewRequest
|
|
29
|
+
effectiveModel: WhisperPreviewModel
|
|
30
|
+
ready: boolean
|
|
31
|
+
backend: 'whisper-preview-server' | 'whisper-server' | null
|
|
32
|
+
degraded: boolean
|
|
33
|
+
reason: WhisperPreviewReason
|
|
34
|
+
committedModel: 'large-v3-turbo'
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const MODEL_DIR = join(process.env.HOME ?? homedir(), '.local/share/whisper-models')
|
|
38
|
+
export const WHISPER_SMALL_EN_MODEL_PATH = join(MODEL_DIR, 'ggml-small.en.bin')
|
|
39
|
+
const VAD_MODEL_PATH = join(MODEL_DIR, 'ggml-silero-v5.1.2.bin')
|
|
40
|
+
const VAD_ENABLED = process.env.COS_WHISPER_VAD !== '0'
|
|
41
|
+
const WHISPER_SERVER = ['/opt/homebrew/bin/whisper-server', '/usr/local/bin/whisper-server']
|
|
42
|
+
.find(existsSync) ?? '/opt/homebrew/bin/whisper-server'
|
|
43
|
+
const PREVIEW_PORT = 8177
|
|
44
|
+
const PREVIEW_URL = `http://127.0.0.1:${PREVIEW_PORT}`
|
|
45
|
+
|
|
46
|
+
let previewProcess: ChildProcess | null = null
|
|
47
|
+
let previewAvailable = false
|
|
48
|
+
let previewStarting = false
|
|
49
|
+
let previewFailure: WhisperPreviewReason = null
|
|
50
|
+
let warnedInvalidChoice = false
|
|
51
|
+
|
|
52
|
+
export function normalizeWhisperPreviewRequest(raw?: string): WhisperPreviewRequest {
|
|
53
|
+
const value = raw?.trim().toLowerCase()
|
|
54
|
+
// Backward compatibility is deliberate: simply updating the server keeps
|
|
55
|
+
// the old Turbo preview. Guided Setup opts the user into Small.en.
|
|
56
|
+
if (!value) return 'turbo'
|
|
57
|
+
if (value === 'auto' || value === 'adaptive') return 'auto'
|
|
58
|
+
if (value === 'small' || value === 'small.en' || value === 'ggml-small.en.bin') return 'small.en'
|
|
59
|
+
if (value === 'turbo' || value === 'large-v3-turbo' || value === 'ggml-large-v3-turbo.bin') return 'turbo'
|
|
60
|
+
if (value === 'off' || value === 'disabled' || value === '0') return 'off'
|
|
61
|
+
return 'auto'
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function requestedPreviewModel(): WhisperPreviewRequest {
|
|
65
|
+
const raw = process.env.COS_WHISPER_PREVIEW_MODEL
|
|
66
|
+
?? process.env.COS_WHISPER_REALTIME_MODEL // migration alias for early private installs
|
|
67
|
+
const normalized = normalizeWhisperPreviewRequest(raw)
|
|
68
|
+
if (raw && normalized === 'auto' && !['auto', 'adaptive'].includes(raw.trim().toLowerCase()) && !warnedInvalidChoice) {
|
|
69
|
+
warnedInvalidChoice = true
|
|
70
|
+
console.warn(`[whisper-preview] Unknown model "${raw}"; using adaptive selection.`)
|
|
71
|
+
}
|
|
72
|
+
return normalized
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function selectedPreviewModel(requested = requestedPreviewModel()): WhisperPreviewModel {
|
|
76
|
+
if (requested === 'off') return null
|
|
77
|
+
if (requested === 'turbo') return 'large-v3-turbo'
|
|
78
|
+
if (requested === 'small.en') return existsSync(WHISPER_SMALL_EN_MODEL_PATH) ? 'small.en' : 'large-v3-turbo'
|
|
79
|
+
return existsSync(WHISPER_SMALL_EN_MODEL_PATH) ? 'small.en' : 'large-v3-turbo'
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function getWhisperPreviewCapability(): WhisperPreviewCapability {
|
|
83
|
+
const requested = requestedPreviewModel()
|
|
84
|
+
if (requested === 'off') {
|
|
85
|
+
return {
|
|
86
|
+
requested, effectiveModel: null, ready: false, backend: null,
|
|
87
|
+
degraded: false, reason: 'disabled', committedModel: 'large-v3-turbo',
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const smallPresent = existsSync(WHISPER_SMALL_EN_MODEL_PATH)
|
|
92
|
+
const selected = selectedPreviewModel(requested)
|
|
93
|
+
const turboReady = getWhisperHealth().server
|
|
94
|
+
if (selected === 'small.en' && previewAvailable) {
|
|
95
|
+
return {
|
|
96
|
+
requested, effectiveModel: 'small.en', ready: true,
|
|
97
|
+
backend: 'whisper-preview-server', degraded: false, reason: null,
|
|
98
|
+
committedModel: 'large-v3-turbo',
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const smallWasExpected = requested === 'small.en' || (requested === 'auto' && smallPresent)
|
|
103
|
+
const reason: WhisperPreviewReason = requested === 'small.en' && !smallPresent
|
|
104
|
+
? 'small_model_missing'
|
|
105
|
+
: smallWasExpected
|
|
106
|
+
? (previewFailure ?? (previewStarting ? null : 'preview_sidecar_unavailable'))
|
|
107
|
+
: turboReady ? null : 'turbo_unavailable'
|
|
108
|
+
return {
|
|
109
|
+
requested,
|
|
110
|
+
effectiveModel: turboReady ? 'large-v3-turbo' : selected,
|
|
111
|
+
ready: turboReady,
|
|
112
|
+
backend: turboReady ? 'whisper-server' : null,
|
|
113
|
+
degraded: smallWasExpected,
|
|
114
|
+
reason,
|
|
115
|
+
committedModel: 'large-v3-turbo',
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
async function endpointReady(path: '/health' | '/inference', init?: RequestInit, timeoutMs = 1_000): Promise<Response> {
|
|
120
|
+
return fetch(`${PREVIEW_URL}${path}`, { ...init, signal: AbortSignal.timeout(timeoutMs) })
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Start the optional small.en preview worker. Failure is cosmetic: committed
|
|
124
|
+
* Turbo and every recovery/finalization path stay untouched. */
|
|
125
|
+
export async function startWhisperPreviewServer(): Promise<void> {
|
|
126
|
+
const requested = requestedPreviewModel()
|
|
127
|
+
if (selectedPreviewModel(requested) !== 'small.en' || previewProcess || previewAvailable || previewStarting) return
|
|
128
|
+
if (!existsSync(WHISPER_SMALL_EN_MODEL_PATH)) {
|
|
129
|
+
previewFailure = 'small_model_missing'
|
|
130
|
+
return
|
|
131
|
+
}
|
|
132
|
+
if (!existsSync(WHISPER_SERVER)) {
|
|
133
|
+
previewFailure = 'preview_binary_missing'
|
|
134
|
+
return
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
previewStarting = true
|
|
138
|
+
previewFailure = null
|
|
139
|
+
try {
|
|
140
|
+
try {
|
|
141
|
+
const occupied = await endpointReady('/health')
|
|
142
|
+
if (occupied.ok) {
|
|
143
|
+
previewFailure = 'preview_port_busy'
|
|
144
|
+
return
|
|
145
|
+
}
|
|
146
|
+
} catch { /* clear port is expected */ }
|
|
147
|
+
|
|
148
|
+
const args = [
|
|
149
|
+
'-m', WHISPER_SMALL_EN_MODEL_PATH,
|
|
150
|
+
'-t', '16',
|
|
151
|
+
'-l', 'en',
|
|
152
|
+
'-fa',
|
|
153
|
+
'--no-speech-thold', '0.7',
|
|
154
|
+
'--host', '127.0.0.1',
|
|
155
|
+
'--port', String(PREVIEW_PORT),
|
|
156
|
+
]
|
|
157
|
+
if (VAD_ENABLED && existsSync(VAD_MODEL_PATH)) {
|
|
158
|
+
args.push('--vad', '--vad-model', VAD_MODEL_PATH)
|
|
159
|
+
}
|
|
160
|
+
const child = spawn(WHISPER_SERVER, args, { stdio: 'ignore', detached: false })
|
|
161
|
+
previewProcess = child
|
|
162
|
+
child.once('close', code => {
|
|
163
|
+
if (previewProcess !== child) return
|
|
164
|
+
previewProcess = null
|
|
165
|
+
previewAvailable = false
|
|
166
|
+
previewFailure = code === 0 ? 'preview_sidecar_unavailable' : 'preview_start_failed'
|
|
167
|
+
})
|
|
168
|
+
child.once('error', () => {
|
|
169
|
+
previewAvailable = false
|
|
170
|
+
previewFailure = 'preview_start_failed'
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
const deadline = Date.now() + 45_000
|
|
174
|
+
while (Date.now() < deadline) {
|
|
175
|
+
if (child.exitCode !== null || child.signalCode !== null) break
|
|
176
|
+
try {
|
|
177
|
+
const response = await endpointReady('/health', undefined, 1_000)
|
|
178
|
+
if (response.ok) {
|
|
179
|
+
previewAvailable = true
|
|
180
|
+
previewFailure = null
|
|
181
|
+
console.log('[whisper-preview] small.en ready for provisional text; committed text remains Turbo')
|
|
182
|
+
return
|
|
183
|
+
}
|
|
184
|
+
} catch { /* model still loading */ }
|
|
185
|
+
await new Promise(resolve => setTimeout(resolve, 1_000))
|
|
186
|
+
}
|
|
187
|
+
try { child.kill('SIGKILL') } catch { /* already exited */ }
|
|
188
|
+
if (previewProcess === child) previewProcess = null
|
|
189
|
+
previewFailure = 'preview_start_failed'
|
|
190
|
+
} finally {
|
|
191
|
+
previewStarting = false
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
export function stopWhisperPreviewServer(): void {
|
|
196
|
+
const child = previewProcess
|
|
197
|
+
previewProcess = null
|
|
198
|
+
previewAvailable = false
|
|
199
|
+
previewStarting = false
|
|
200
|
+
if (child) {
|
|
201
|
+
try { child.kill('SIGTERM') } catch { /* already exited */ }
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function previewPrompt(): string {
|
|
206
|
+
const vocabulary = getVocabulary()
|
|
207
|
+
return vocabulary.length > 0
|
|
208
|
+
? [getOwnerName(), ...vocabulary].join(', ')
|
|
209
|
+
: `${getOwnerName()}. COS Glasses. Even G2.`
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
async function transcribeViaPreviewServer(audioBuffer: Buffer): Promise<string> {
|
|
213
|
+
const formData = new FormData()
|
|
214
|
+
formData.append('file', new Blob([new Uint8Array(audioBuffer)], { type: 'audio/wav' }), 'recording.wav')
|
|
215
|
+
formData.append('response_format', 'json')
|
|
216
|
+
formData.append('prompt', previewPrompt())
|
|
217
|
+
formData.append('suppress_non_speech', 'true')
|
|
218
|
+
const response = await endpointReady('/inference', { method: 'POST', body: formData }, 5_000)
|
|
219
|
+
if (!response.ok) throw new Error(`preview server ${response.status}`)
|
|
220
|
+
const result = await response.json() as { text?: unknown }
|
|
221
|
+
if (typeof result.text !== 'string') throw new Error('preview server returned invalid text')
|
|
222
|
+
return applyCorrections(result.text.trim())
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Cosmetic preview only. A small-worker failure falls through to the existing
|
|
226
|
+
* non-circuit Turbo decode and can never write committed transcript state. */
|
|
227
|
+
export async function transcribeWhisperPreview(audioBuffer: Buffer): Promise<{
|
|
228
|
+
text: string
|
|
229
|
+
model: 'small.en' | 'large-v3-turbo'
|
|
230
|
+
backend: 'whisper-preview-server' | 'whisper-server'
|
|
231
|
+
}> {
|
|
232
|
+
if (previewAvailable) {
|
|
233
|
+
try {
|
|
234
|
+
return { text: await transcribeViaPreviewServer(audioBuffer), model: 'small.en', backend: 'whisper-preview-server' }
|
|
235
|
+
} catch (error) {
|
|
236
|
+
previewAvailable = false
|
|
237
|
+
previewFailure = 'preview_sidecar_unavailable'
|
|
238
|
+
console.warn(`[whisper-preview] small.en preview failed; falling back to Turbo: ${error instanceof Error ? error.message : error}`)
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
const result = await transcribeLocal(audioBuffer, undefined, undefined, { affectsCircuit: false })
|
|
242
|
+
return { text: result.text, model: 'large-v3-turbo', backend: 'whisper-server' }
|
|
243
|
+
}
|
|
@@ -13,7 +13,7 @@ import { errMsg } from '../lib/utils.js'
|
|
|
13
13
|
import {
|
|
14
14
|
getVocabulary,
|
|
15
15
|
getNegativeRules,
|
|
16
|
-
|
|
16
|
+
getWhisperCorrections,
|
|
17
17
|
updateProfileFields,
|
|
18
18
|
} from '../lib/profile.js'
|
|
19
19
|
import { resetDecoderCaches } from '../lib/whisper-local.js'
|
|
@@ -36,16 +36,7 @@ function looksLikeUrlEmailPath(s: string): boolean {
|
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
function readCorrections(): Record<string, string> {
|
|
39
|
-
|
|
40
|
-
const raw = loadProfileField('whisper_corrections', '')
|
|
41
|
-
if (!raw) return {}
|
|
42
|
-
const parsed = JSON.parse(raw)
|
|
43
|
-
return (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
|
|
44
|
-
? parsed as Record<string, string>
|
|
45
|
-
: {}
|
|
46
|
-
} catch {
|
|
47
|
-
return {}
|
|
48
|
-
}
|
|
39
|
+
return getWhisperCorrections()
|
|
49
40
|
}
|
|
50
41
|
|
|
51
42
|
function currentGlossary() {
|