@gotcos/glasses-server 6.14.1 → 6.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +11 -0
- package/CHANGELOG.md +40 -0
- package/README.md +31 -7
- package/bin/cli.cjs +31 -5
- package/package.json +2 -2
- package/server/index.ts +10 -19
- package/server/lib/api-auth.ts +34 -0
- package/server/lib/tts-cache.ts +32 -5
- package/server/lib/tts-engine.ts +230 -0
- package/server/lib/tts-local.ts +375 -0
- package/server/lib/tts-pronounce.ts +53 -0
- package/server/lib/whisper-local.ts +86 -4
- package/server/routes/health.ts +8 -2
- package/server/routes/tts.ts +579 -199
- package/server/tts-sidecar/bootstrap.sh +57 -0
- package/server/tts-sidecar/requirements.txt +9 -0
- package/server/tts-sidecar/server.py +235 -0
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
// Local Kokoro TTS sidecar lifecycle — Whisper-shaped, port 8179.
|
|
2
|
+
// Local-first engine with an explicit OpenAI fallback path.
|
|
3
|
+
|
|
4
|
+
import { spawn, type ChildProcess } from 'node:child_process'
|
|
5
|
+
import { randomBytes } from 'node:crypto'
|
|
6
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
|
7
|
+
import { arch, homedir, platform } from 'node:os'
|
|
8
|
+
import { dirname, join, resolve } from 'node:path'
|
|
9
|
+
import { fileURLToPath } from 'node:url'
|
|
10
|
+
import { atomicWriteFileSync } from './atomic-fs.js'
|
|
11
|
+
import { dataPath } from './data-dir.js'
|
|
12
|
+
|
|
13
|
+
const __dirname = dirname(fileURLToPath(import.meta.url))
|
|
14
|
+
const SIDECAR_DIR = resolve(__dirname, '..', 'tts-sidecar')
|
|
15
|
+
const SIDECAR_SCRIPT = join(SIDECAR_DIR, 'server.py')
|
|
16
|
+
const BOOTSTRAP = join(SIDECAR_DIR, 'bootstrap.sh')
|
|
17
|
+
|
|
18
|
+
const TTS_PORT = Number(process.env.COS_TTS_LOCAL_PORT || 8179)
|
|
19
|
+
const TTS_HOST = '127.0.0.1'
|
|
20
|
+
const TTS_BASE =
|
|
21
|
+
process.env.COS_TTS_LOCAL_URL?.replace(/\/$/, '') || `http://${TTS_HOST}:${TTS_PORT}`
|
|
22
|
+
const TTS_PROTOCOL = 'cos-tts-v1'
|
|
23
|
+
// Boot-scoped bearer token: an unrelated/orphan process on 8179 cannot be
|
|
24
|
+
// mistaken for COS or receive private text after the owning server restarts.
|
|
25
|
+
const TTS_AUTH_TOKEN = randomBytes(32).toString('hex')
|
|
26
|
+
|
|
27
|
+
const MODEL_DIR = join(process.env.HOME ?? homedir(), '.local/share/cos-tts-models')
|
|
28
|
+
|
|
29
|
+
let serverProcess: ChildProcess | null = null
|
|
30
|
+
let serverAvailable = false
|
|
31
|
+
let serverStarting = false
|
|
32
|
+
let lastError: string | null = null
|
|
33
|
+
let engineVersion: string | null = null
|
|
34
|
+
let localVoice: string | null = null
|
|
35
|
+
let lastFallbackToOpenAI: { at: string; reason: string } | null = null
|
|
36
|
+
let lastHealthProbeAt = 0
|
|
37
|
+
let healthProbeInFlight: Promise<boolean> | null = null
|
|
38
|
+
|
|
39
|
+
const HEALTH_REFRESH_INTERVAL_MS = 2_000
|
|
40
|
+
const HEALTH_REFRESH_TIMEOUT_MS = 400
|
|
41
|
+
|
|
42
|
+
const FALLBACK_STATE_PATH = dataPath('tts-last-fallback.json')
|
|
43
|
+
|
|
44
|
+
function loadPersistedFallback(): void {
|
|
45
|
+
try {
|
|
46
|
+
if (!existsSync(FALLBACK_STATE_PATH)) return
|
|
47
|
+
const raw = JSON.parse(readFileSync(FALLBACK_STATE_PATH, 'utf8')) as {
|
|
48
|
+
at?: string
|
|
49
|
+
reason?: string
|
|
50
|
+
}
|
|
51
|
+
if (typeof raw.at === 'string' && typeof raw.reason === 'string') {
|
|
52
|
+
lastFallbackToOpenAI = { at: raw.at, reason: raw.reason.slice(0, 300) }
|
|
53
|
+
}
|
|
54
|
+
} catch { /* ignore corrupt state */ }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
loadPersistedFallback()
|
|
58
|
+
|
|
59
|
+
/** Record that Voice Mode escaped Kokoro → OpenAI (survives LaunchAgent restart). */
|
|
60
|
+
export function recordLocalTtsFallbackToOpenAI(reason: string): void {
|
|
61
|
+
lastFallbackToOpenAI = { at: new Date().toISOString(), reason: reason.slice(0, 300) }
|
|
62
|
+
console.warn('[tts-local] FALLBACK TO OPENAI:', reason)
|
|
63
|
+
try {
|
|
64
|
+
mkdirSync(dirname(FALLBACK_STATE_PATH), { recursive: true })
|
|
65
|
+
atomicWriteFileSync(FALLBACK_STATE_PATH, `${JSON.stringify(lastFallbackToOpenAI)}\n`)
|
|
66
|
+
} catch {
|
|
67
|
+
try {
|
|
68
|
+
writeFileSync(FALLBACK_STATE_PATH, `${JSON.stringify(lastFallbackToOpenAI)}\n`)
|
|
69
|
+
} catch { /* best-effort persist */ }
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Bound hung sidecar so local_first can fall back before session TTL (~60s). */
|
|
74
|
+
export const LOCAL_TTS_SYNTH_TIMEOUT_MS = Number(
|
|
75
|
+
process.env.COS_TTS_LOCAL_TIMEOUT_MS || 12_000,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
function candidatePythons(): string[] {
|
|
79
|
+
const out: string[] = []
|
|
80
|
+
if (process.env.COS_TTS_PYTHON) out.push(process.env.COS_TTS_PYTHON)
|
|
81
|
+
// Pinned product venv only — bootstrap.sh creates it under MODEL_DIR.
|
|
82
|
+
// Do not hardcode repo/Phase-0 paths (breaks other machines + LaunchAgent hygiene).
|
|
83
|
+
out.push(join(MODEL_DIR, '.venv', 'bin', 'python'))
|
|
84
|
+
return out.filter((p) => p && existsSync(p))
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function resolvePython(): string | null {
|
|
88
|
+
const hits = candidatePythons()
|
|
89
|
+
return hits[0] ?? null
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function ensureBootstrap(): Promise<string | null> {
|
|
93
|
+
let py = resolvePython()
|
|
94
|
+
if (py) return py
|
|
95
|
+
if (!existsSync(BOOTSTRAP)) {
|
|
96
|
+
lastError = `TTS bootstrap missing at ${BOOTSTRAP}`
|
|
97
|
+
return null
|
|
98
|
+
}
|
|
99
|
+
try {
|
|
100
|
+
mkdirSync(MODEL_DIR, { recursive: true })
|
|
101
|
+
console.log('[tts-local] bootstrapping venv via', BOOTSTRAP)
|
|
102
|
+
await new Promise<void>((resolvePromise, rejectPromise) => {
|
|
103
|
+
const child = spawn('/bin/bash', [BOOTSTRAP], {
|
|
104
|
+
stdio: 'inherit',
|
|
105
|
+
detached: false,
|
|
106
|
+
env: process.env,
|
|
107
|
+
})
|
|
108
|
+
child.once('error', rejectPromise)
|
|
109
|
+
child.once('exit', (code, signal) => {
|
|
110
|
+
if (code === 0) {
|
|
111
|
+
resolvePromise()
|
|
112
|
+
return
|
|
113
|
+
}
|
|
114
|
+
rejectPromise(new Error(`bootstrap exited code=${code} signal=${signal}`))
|
|
115
|
+
})
|
|
116
|
+
})
|
|
117
|
+
} catch (err) {
|
|
118
|
+
lastError = `TTS bootstrap failed: ${err instanceof Error ? err.message : String(err)}`
|
|
119
|
+
console.error('[tts-local]', lastError)
|
|
120
|
+
return null
|
|
121
|
+
}
|
|
122
|
+
py = resolvePython()
|
|
123
|
+
if (!py) lastError = 'TTS bootstrap finished but python still missing'
|
|
124
|
+
return py
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async function probeHealth(timeoutMs = 1500): Promise<boolean> {
|
|
128
|
+
try {
|
|
129
|
+
const res = await fetch(`${TTS_BASE}/health`, {
|
|
130
|
+
headers: { Authorization: `Bearer ${TTS_AUTH_TOKEN}` },
|
|
131
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
132
|
+
})
|
|
133
|
+
if (!res.ok) {
|
|
134
|
+
serverAvailable = false
|
|
135
|
+
lastError = `local TTS health ${res.status}`
|
|
136
|
+
return false
|
|
137
|
+
}
|
|
138
|
+
const body = (await res.json()) as {
|
|
139
|
+
ready?: boolean
|
|
140
|
+
protocol?: string
|
|
141
|
+
engine?: string
|
|
142
|
+
voice?: string
|
|
143
|
+
error?: string | null
|
|
144
|
+
}
|
|
145
|
+
if (body.ready && body.protocol === TTS_PROTOCOL) {
|
|
146
|
+
serverAvailable = true
|
|
147
|
+
engineVersion = body.engine ?? 'kokoro'
|
|
148
|
+
localVoice = body.voice ?? null
|
|
149
|
+
lastError = null
|
|
150
|
+
return true
|
|
151
|
+
}
|
|
152
|
+
serverAvailable = false
|
|
153
|
+
lastError = body.protocol !== TTS_PROTOCOL
|
|
154
|
+
? 'local TTS protocol identity mismatch'
|
|
155
|
+
: body.error || 'local TTS reported not ready'
|
|
156
|
+
return false
|
|
157
|
+
} catch (err) {
|
|
158
|
+
serverAvailable = false
|
|
159
|
+
lastError = err instanceof Error ? err.message : String(err)
|
|
160
|
+
return false
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Authenticated liveness refresh used by /api/health; never trusts stale state. */
|
|
165
|
+
export async function refreshLocalTtsHealth(): Promise<void> {
|
|
166
|
+
if (process.env.COS_TTS_LOCAL_DISABLE === '1') {
|
|
167
|
+
serverAvailable = false
|
|
168
|
+
lastError = 'disabled via COS_TTS_LOCAL_DISABLE=1'
|
|
169
|
+
return
|
|
170
|
+
}
|
|
171
|
+
if (platform() !== 'darwin' || arch() !== 'arm64') {
|
|
172
|
+
serverAvailable = false
|
|
173
|
+
lastError = 'Local Kokoro requires an Apple silicon Mac (darwin/arm64)'
|
|
174
|
+
return
|
|
175
|
+
}
|
|
176
|
+
const now = Date.now()
|
|
177
|
+
if (now - lastHealthProbeAt < HEALTH_REFRESH_INTERVAL_MS) return
|
|
178
|
+
if (!healthProbeInFlight) {
|
|
179
|
+
healthProbeInFlight = probeHealth(HEALTH_REFRESH_TIMEOUT_MS).finally(() => {
|
|
180
|
+
lastHealthProbeAt = Date.now()
|
|
181
|
+
healthProbeInFlight = null
|
|
182
|
+
})
|
|
183
|
+
}
|
|
184
|
+
await healthProbeInFlight
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function isLocalTtsReady(): boolean {
|
|
188
|
+
return serverAvailable
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export function getLocalTtsHealth(): {
|
|
192
|
+
ready: boolean
|
|
193
|
+
engine: string | null
|
|
194
|
+
version: string | null
|
|
195
|
+
voice: string | null
|
|
196
|
+
ffmpeg: boolean
|
|
197
|
+
espeak: boolean
|
|
198
|
+
port: number
|
|
199
|
+
url: string
|
|
200
|
+
starting: boolean
|
|
201
|
+
error: string | null
|
|
202
|
+
lastFallbackToOpenAI: { at: string; reason: string } | null
|
|
203
|
+
} {
|
|
204
|
+
return {
|
|
205
|
+
ready: serverAvailable,
|
|
206
|
+
engine: engineVersion,
|
|
207
|
+
version: engineVersion,
|
|
208
|
+
voice: localVoice,
|
|
209
|
+
ffmpeg: existsSync('/opt/homebrew/bin/ffmpeg') || existsSync('/usr/local/bin/ffmpeg'),
|
|
210
|
+
espeak: existsSync('/opt/homebrew/bin/espeak-ng') || existsSync('/usr/bin/espeak-ng'),
|
|
211
|
+
port: TTS_PORT,
|
|
212
|
+
url: TTS_BASE,
|
|
213
|
+
starting: serverStarting,
|
|
214
|
+
error: lastError,
|
|
215
|
+
lastFallbackToOpenAI,
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export async function startLocalTtsServer(): Promise<void> {
|
|
220
|
+
if (process.env.COS_TTS_LOCAL_DISABLE === '1') {
|
|
221
|
+
console.log('[tts-local] disabled via COS_TTS_LOCAL_DISABLE=1')
|
|
222
|
+
return
|
|
223
|
+
}
|
|
224
|
+
if (serverStarting) return
|
|
225
|
+
if (platform() !== 'darwin' || arch() !== 'arm64') {
|
|
226
|
+
lastError = 'Local Kokoro requires an Apple silicon Mac (darwin/arm64)'
|
|
227
|
+
console.warn('[tts-local]', lastError)
|
|
228
|
+
return
|
|
229
|
+
}
|
|
230
|
+
serverStarting = true
|
|
231
|
+
try {
|
|
232
|
+
if (await probeHealth(1500)) {
|
|
233
|
+
// A healthy listener authenticated with this boot token belongs to this
|
|
234
|
+
// server boot; orphan/foreign listeners cannot pass the probe.
|
|
235
|
+
console.log('[tts-local] already healthy on', TTS_PORT)
|
|
236
|
+
return
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
if (!existsSync(SIDECAR_SCRIPT)) {
|
|
240
|
+
lastError = `sidecar missing: ${SIDECAR_SCRIPT}`
|
|
241
|
+
console.error('[tts-local]', lastError)
|
|
242
|
+
return
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
const python = await ensureBootstrap()
|
|
246
|
+
if (!python) return
|
|
247
|
+
|
|
248
|
+
// Clear stale listener if we own a dead handle
|
|
249
|
+
if (serverProcess) {
|
|
250
|
+
try { serverProcess.kill('SIGTERM') } catch { /* ignore */ }
|
|
251
|
+
serverProcess = null
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
console.log(`[tts-local] starting Kokoro sidecar on ${TTS_PORT} via ${python}`)
|
|
255
|
+
const child = spawn(
|
|
256
|
+
python,
|
|
257
|
+
[SIDECAR_SCRIPT, '--host', TTS_HOST, '--port', String(TTS_PORT)],
|
|
258
|
+
{
|
|
259
|
+
stdio: ['ignore', 'inherit', 'inherit'],
|
|
260
|
+
detached: false,
|
|
261
|
+
env: {
|
|
262
|
+
...process.env,
|
|
263
|
+
COS_TTS_AUTH_TOKEN: TTS_AUTH_TOKEN,
|
|
264
|
+
ESPEAK_DATA_PATH: process.env.ESPEAK_DATA_PATH || '/opt/homebrew/share/espeak-ng-data',
|
|
265
|
+
PHONEMIZER_ESPEAK_PATH: process.env.PHONEMIZER_ESPEAK_PATH || '/opt/homebrew/bin/espeak-ng',
|
|
266
|
+
COS_TTS_PORT: String(TTS_PORT),
|
|
267
|
+
},
|
|
268
|
+
},
|
|
269
|
+
)
|
|
270
|
+
serverProcess = child
|
|
271
|
+
child.on('exit', (code, signal) => {
|
|
272
|
+
if (serverProcess === child) {
|
|
273
|
+
serverProcess = null
|
|
274
|
+
serverAvailable = false
|
|
275
|
+
lastError = `sidecar exited code=${code} signal=${signal}`
|
|
276
|
+
console.warn('[tts-local]', lastError)
|
|
277
|
+
}
|
|
278
|
+
})
|
|
279
|
+
|
|
280
|
+
const maxWaitMs = 120_000
|
|
281
|
+
const started = Date.now()
|
|
282
|
+
while (Date.now() - started < maxWaitMs) {
|
|
283
|
+
if (await probeHealth(1500)) {
|
|
284
|
+
console.log(
|
|
285
|
+
`[tts-local] ready on ${TTS_PORT} engine=${engineVersion} voice=${localVoice} ` +
|
|
286
|
+
`(${((Date.now() - started) / 1000).toFixed(1)}s)`,
|
|
287
|
+
)
|
|
288
|
+
return
|
|
289
|
+
}
|
|
290
|
+
await new Promise((r) => setTimeout(r, 1500))
|
|
291
|
+
}
|
|
292
|
+
lastError = `sidecar startup timeout (${maxWaitMs / 1000}s)`
|
|
293
|
+
console.error('[tts-local]', lastError)
|
|
294
|
+
try { child.kill('SIGKILL') } catch { /* ignore */ }
|
|
295
|
+
serverProcess = null
|
|
296
|
+
serverAvailable = false
|
|
297
|
+
} finally {
|
|
298
|
+
serverStarting = false
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export function stopLocalTtsServer(): void {
|
|
303
|
+
if (!serverProcess) return
|
|
304
|
+
try {
|
|
305
|
+
serverProcess.kill('SIGTERM')
|
|
306
|
+
} catch { /* ignore */ }
|
|
307
|
+
serverProcess = null
|
|
308
|
+
serverAvailable = false
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** Synthesize via local OpenAI-shaped speech endpoint. Returns full audio Buffer. */
|
|
312
|
+
export async function synthesizeLocalTts(opts: {
|
|
313
|
+
text: string
|
|
314
|
+
voice: string
|
|
315
|
+
format: string
|
|
316
|
+
signal?: AbortSignal
|
|
317
|
+
}): Promise<Buffer> {
|
|
318
|
+
const timeoutMs = Number.isFinite(LOCAL_TTS_SYNTH_TIMEOUT_MS) && LOCAL_TTS_SYNTH_TIMEOUT_MS > 0
|
|
319
|
+
? LOCAL_TTS_SYNTH_TIMEOUT_MS
|
|
320
|
+
: 12_000
|
|
321
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs)
|
|
322
|
+
const signal =
|
|
323
|
+
opts.signal && typeof AbortSignal.any === 'function'
|
|
324
|
+
? AbortSignal.any([opts.signal, timeoutSignal])
|
|
325
|
+
: opts.signal ?? timeoutSignal
|
|
326
|
+
|
|
327
|
+
let res: Response
|
|
328
|
+
try {
|
|
329
|
+
res = await fetch(`${TTS_BASE}/v1/audio/speech`, {
|
|
330
|
+
method: 'POST',
|
|
331
|
+
headers: {
|
|
332
|
+
Authorization: `Bearer ${TTS_AUTH_TOKEN}`,
|
|
333
|
+
'Content-Type': 'application/json',
|
|
334
|
+
},
|
|
335
|
+
body: JSON.stringify({
|
|
336
|
+
model: 'tts-1',
|
|
337
|
+
input: opts.text,
|
|
338
|
+
voice: opts.voice,
|
|
339
|
+
response_format: opts.format,
|
|
340
|
+
}),
|
|
341
|
+
signal,
|
|
342
|
+
})
|
|
343
|
+
} catch (err) {
|
|
344
|
+
const name = (err as { name?: string })?.name
|
|
345
|
+
// A caller abort means the user/request went away. Preserve that signal so
|
|
346
|
+
// the route returns 499 and never mistakes cancellation for a Kokoro
|
|
347
|
+
// failure eligible for OpenAI fallback.
|
|
348
|
+
if (opts.signal?.aborted) {
|
|
349
|
+
const abortError = new Error('local TTS request aborted')
|
|
350
|
+
abortError.name = 'AbortError'
|
|
351
|
+
throw abortError
|
|
352
|
+
}
|
|
353
|
+
if (timeoutSignal.aborted || name === 'TimeoutError') {
|
|
354
|
+
const timeoutError = new Error(`local TTS timed out after ${timeoutMs}ms`)
|
|
355
|
+
timeoutError.name = 'TimeoutError'
|
|
356
|
+
throw timeoutError
|
|
357
|
+
}
|
|
358
|
+
// Fail closed for an AbortError whose source is unknown. Treating it as a
|
|
359
|
+
// synthesis failure could spend cloud budget after a canceled request.
|
|
360
|
+
if (name === 'AbortError') {
|
|
361
|
+
throw err
|
|
362
|
+
}
|
|
363
|
+
serverAvailable = false
|
|
364
|
+
lastError = err instanceof Error ? err.message : String(err)
|
|
365
|
+
throw err
|
|
366
|
+
}
|
|
367
|
+
if (!res.ok) {
|
|
368
|
+
const errText = await res.text().catch(() => '')
|
|
369
|
+
serverAvailable = false
|
|
370
|
+
lastError = `local TTS ${res.status}: ${errText.slice(0, 160)}`
|
|
371
|
+
throw new Error(`local TTS ${res.status}: ${errText.slice(0, 300)}`)
|
|
372
|
+
}
|
|
373
|
+
const ab = await res.arrayBuffer()
|
|
374
|
+
return Buffer.from(ab)
|
|
375
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// Optional, public-safe pronunciation lexicon for local Kokoro and OpenAI TTS.
|
|
2
|
+
// No personal names ship in the npm package. Operators may opt in with:
|
|
3
|
+
// COS_TTS_PRONUNCIATIONS_JSON='{"Exampleco":{"local":"[Exampleco](/ɪgzˈæmpəlkoʊ/)","openai":"ig-ZAM-pul-co"}}'
|
|
4
|
+
|
|
5
|
+
interface PronunciationEntry {
|
|
6
|
+
local?: string
|
|
7
|
+
openai?: string
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function escapeRegExp(value: string): string {
|
|
11
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function readLexicon(): Array<{ term: string; local?: string; openai?: string }> {
|
|
15
|
+
const raw = (process.env.COS_TTS_PRONUNCIATIONS_JSON || '').trim()
|
|
16
|
+
if (!raw) return []
|
|
17
|
+
try {
|
|
18
|
+
const parsed = JSON.parse(raw) as Record<string, PronunciationEntry>
|
|
19
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return []
|
|
20
|
+
return Object.entries(parsed)
|
|
21
|
+
.slice(0, 64)
|
|
22
|
+
.flatMap(([term, entry]) => {
|
|
23
|
+
const cleanTerm = term.trim()
|
|
24
|
+
if (!cleanTerm || cleanTerm.length > 64 || !entry || typeof entry !== 'object') return []
|
|
25
|
+
const local = typeof entry.local === 'string' && entry.local.length <= 160 ? entry.local : undefined
|
|
26
|
+
const openai = typeof entry.openai === 'string' && entry.openai.length <= 160 ? entry.openai : undefined
|
|
27
|
+
return local || openai ? [{ term: cleanTerm, local, openai }] : []
|
|
28
|
+
})
|
|
29
|
+
} catch {
|
|
30
|
+
console.warn('[tts-pronounce] invalid COS_TTS_PRONUNCIATIONS_JSON; ignoring')
|
|
31
|
+
return []
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function applyLocalPronunciation(text: string): string {
|
|
36
|
+
let out = text
|
|
37
|
+
for (const entry of readLexicon()) {
|
|
38
|
+
if (!entry.local) continue
|
|
39
|
+
const pattern = new RegExp(`(?<!\\[)\\b${escapeRegExp(entry.term)}\\b`, 'gi')
|
|
40
|
+
out = out.replace(pattern, entry.local)
|
|
41
|
+
}
|
|
42
|
+
return out
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function applyOpenAIPronunciation(text: string): string {
|
|
46
|
+
let out = text
|
|
47
|
+
for (const entry of readLexicon()) {
|
|
48
|
+
if (!entry.openai) continue
|
|
49
|
+
const pattern = new RegExp(`\\b${escapeRegExp(entry.term)}\\b`, 'gi')
|
|
50
|
+
out = out.replace(pattern, entry.openai)
|
|
51
|
+
}
|
|
52
|
+
return out
|
|
53
|
+
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import { spawn, execFileSync } from 'node:child_process'
|
|
10
10
|
import type { ChildProcess } from 'node:child_process'
|
|
11
|
-
import { writeFileSync, unlinkSync, existsSync } from 'node:fs'
|
|
11
|
+
import { writeFileSync, unlinkSync, existsSync, readFileSync } from 'node:fs'
|
|
12
12
|
import { join } from 'node:path'
|
|
13
13
|
import { homedir } from 'node:os'
|
|
14
14
|
import crypto from 'node:crypto'
|
|
@@ -522,6 +522,54 @@ async function reconcileWhisperServerHealth(): Promise<boolean> {
|
|
|
522
522
|
return serverHealthProbe
|
|
523
523
|
}
|
|
524
524
|
|
|
525
|
+
/**
|
|
526
|
+
* Parse whisper-cli `-ojf` JSON into plain text + timed words.
|
|
527
|
+
*
|
|
528
|
+
* Batch/save path only. Never uses whisper-server `verbose_json` (the live
|
|
529
|
+
* VAD-empty crash vector). Special tokens like `[_BEG_]` / `<|...|>` are dropped.
|
|
530
|
+
*/
|
|
531
|
+
export function parseWhisperCliFullJson(raw: string): { text: string; words: WhisperWord[] } {
|
|
532
|
+
const data = JSON.parse(raw) as {
|
|
533
|
+
transcription?: Array<{
|
|
534
|
+
text?: unknown
|
|
535
|
+
tokens?: Array<{
|
|
536
|
+
text?: unknown
|
|
537
|
+
p?: unknown
|
|
538
|
+
offsets?: { from?: unknown; to?: unknown }
|
|
539
|
+
}>
|
|
540
|
+
}>
|
|
541
|
+
}
|
|
542
|
+
const segments = Array.isArray(data.transcription) ? data.transcription : []
|
|
543
|
+
const texts: string[] = []
|
|
544
|
+
const words: WhisperWord[] = []
|
|
545
|
+
|
|
546
|
+
for (const segment of segments) {
|
|
547
|
+
if (typeof segment.text === 'string' && segment.text.trim()) {
|
|
548
|
+
texts.push(segment.text.trim())
|
|
549
|
+
}
|
|
550
|
+
for (const token of Array.isArray(segment.tokens) ? segment.tokens : []) {
|
|
551
|
+
const tokenText = typeof token.text === 'string' ? token.text.trim() : ''
|
|
552
|
+
if (!tokenText) continue
|
|
553
|
+
if (tokenText.startsWith('[') && tokenText.endsWith(']')) continue
|
|
554
|
+
if (tokenText.startsWith('<|') && tokenText.endsWith('|>')) continue
|
|
555
|
+
const fromMs = typeof token.offsets?.from === 'number' ? token.offsets.from : Number(token.offsets?.from)
|
|
556
|
+
const toMs = typeof token.offsets?.to === 'number' ? token.offsets.to : Number(token.offsets?.to)
|
|
557
|
+
if (!Number.isFinite(fromMs) || !Number.isFinite(toMs)) continue
|
|
558
|
+
words.push({
|
|
559
|
+
word: tokenText,
|
|
560
|
+
start: fromMs / 1000,
|
|
561
|
+
end: toMs / 1000,
|
|
562
|
+
probability: typeof token.p === 'number' && Number.isFinite(token.p) ? token.p : 0,
|
|
563
|
+
})
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
return {
|
|
568
|
+
text: texts.join(' ').replace(/\s+/g, ' ').trim(),
|
|
569
|
+
words,
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
525
573
|
/**
|
|
526
574
|
* High-quality transcription for batch/post-meeting use.
|
|
527
575
|
* Uses the full Whisper large-v3 (32-layer) decoder + Silero VAD + beam search.
|
|
@@ -542,6 +590,10 @@ export async function transcribeHighQuality(
|
|
|
542
590
|
const start = Date.now()
|
|
543
591
|
const id = crypto.randomUUID().slice(0, 8)
|
|
544
592
|
const tmpWav = join('/tmp', `cos-whisper-hq-${id}.wav`)
|
|
593
|
+
const outBase = join('/tmp', `cos-whisper-hq-${id}`)
|
|
594
|
+
const jsonPath = `${outBase}.json`
|
|
595
|
+
// Word clocks only on post-meeting CPU polish. Live stays compact JSON.
|
|
596
|
+
const captureBatchWords = opts.priority === 'batch'
|
|
545
597
|
|
|
546
598
|
const modelPath = resolveBatchModel()
|
|
547
599
|
const useLargeV3 = modelPath === BATCH_MODEL_LARGE_V3
|
|
@@ -564,6 +616,9 @@ export async function transcribeHighQuality(
|
|
|
564
616
|
'-np',
|
|
565
617
|
'--prompt', buildPrompt(context),
|
|
566
618
|
]
|
|
619
|
+
if (captureBatchWords) {
|
|
620
|
+
args.push('-ojf', '-of', outBase)
|
|
621
|
+
}
|
|
567
622
|
if (useVad) {
|
|
568
623
|
// Same VAD model the streaming path uses. Strips silence windows
|
|
569
624
|
// before the decoder sees them — prevents the silence-hallucination
|
|
@@ -618,13 +673,40 @@ export async function transcribeHighQuality(
|
|
|
618
673
|
})
|
|
619
674
|
})
|
|
620
675
|
|
|
621
|
-
|
|
676
|
+
let finalText = text
|
|
677
|
+
let words: WhisperWord[] | undefined
|
|
678
|
+
if (captureBatchWords) {
|
|
679
|
+
try {
|
|
680
|
+
if (existsSync(jsonPath)) {
|
|
681
|
+
const parsed = parseWhisperCliFullJson(readFileSync(jsonPath, 'utf8'))
|
|
682
|
+
if (parsed.text) finalText = parsed.text
|
|
683
|
+
words = parsed.words.length > 0
|
|
684
|
+
? parsed.words.map(w => ({ ...w, word: applyCorrections(w.word) }))
|
|
685
|
+
: []
|
|
686
|
+
}
|
|
687
|
+
} catch (err) {
|
|
688
|
+
console.warn(
|
|
689
|
+
`[whisper-hq] Batch word JSON parse failed; keeping text-only polish: ` +
|
|
690
|
+
`${err instanceof Error ? err.message : String(err)}`,
|
|
691
|
+
)
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
const corrected = applyCorrections(finalText)
|
|
622
696
|
const elapsed = Date.now() - start
|
|
623
697
|
const modelTag = useLargeV3 ? 'large-v3' : 'turbo'
|
|
624
|
-
console.log(
|
|
625
|
-
|
|
698
|
+
console.log(
|
|
699
|
+
`[whisper-hq] Batch transcribed in ${elapsed}ms ` +
|
|
700
|
+
`(${modelTag}${useVad ? '+vad' : ''}${captureBatchWords ? '+words' : ''}` +
|
|
701
|
+
`${words ? `, ${words.length} words` : ''}): ` +
|
|
702
|
+
`"${corrected.slice(0, 80)}${corrected.length > 80 ? '...' : ''}"`,
|
|
703
|
+
)
|
|
704
|
+
return words ? { text: corrected, words } : { text: corrected }
|
|
626
705
|
} finally {
|
|
627
706
|
try { unlinkSync(tmpWav) } catch { /* cleanup */ }
|
|
707
|
+
if (captureBatchWords) {
|
|
708
|
+
try { unlinkSync(jsonPath) } catch { /* cleanup */ }
|
|
709
|
+
}
|
|
628
710
|
}
|
|
629
711
|
}
|
|
630
712
|
|
package/server/routes/health.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { CLI_DEBUG_CAPABILITY } from '../lib/cli-debug-view.js'
|
|
|
24
24
|
import { managedRuntimeCapability, managedServerVersion } from '../lib/managed-runtime.js'
|
|
25
25
|
import { getServerGenerationId } from '../lib/managed-runtime.js'
|
|
26
26
|
import { maintenanceLifecycle } from '../lib/maintenance-lifecycle.js'
|
|
27
|
+
import { getLocalTtsHealth, refreshLocalTtsHealth } from '../lib/tts-local.js'
|
|
27
28
|
|
|
28
29
|
export const healthRouter = Router()
|
|
29
30
|
|
|
@@ -57,6 +58,7 @@ function durableQueryJobStatus() {
|
|
|
57
58
|
}
|
|
58
59
|
|
|
59
60
|
healthRouter.get('/health', async (_req, res) => {
|
|
61
|
+
await refreshLocalTtsHealth()
|
|
60
62
|
const checks: Record<string, string | number | boolean> = {
|
|
61
63
|
status: 'ok',
|
|
62
64
|
mode: COS_MODE ? 'cos' : 'standalone',
|
|
@@ -150,10 +152,11 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
150
152
|
const transcription = getTranscriptionPolicySnapshot()
|
|
151
153
|
const recovery = managedRuntimeCapability()
|
|
152
154
|
const maintenance = maintenanceLifecycle.snapshot()
|
|
155
|
+
const tts_local = getLocalTtsHealth()
|
|
153
156
|
const features = {
|
|
154
157
|
claude: claudeAvailable,
|
|
155
158
|
codex: codexAvailable,
|
|
156
|
-
voice: keyStatus.hasKey,
|
|
159
|
+
voice: keyStatus.hasKey || tts_local.ready,
|
|
157
160
|
cos_pipeline: COS_MODE,
|
|
158
161
|
whisper: isWhisperLocalAvailable(),
|
|
159
162
|
promptRecovery: true,
|
|
@@ -167,15 +170,17 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
167
170
|
transcriptionPolicy: transcription.mode,
|
|
168
171
|
}
|
|
169
172
|
const voice = {
|
|
173
|
+
available: keyStatus.hasKey || tts_local.ready,
|
|
170
174
|
hasKey: keyStatus.hasKey,
|
|
171
175
|
keySource: keyStatus.source,
|
|
176
|
+
localReady: tts_local.ready,
|
|
177
|
+
engine: tts_local.engine,
|
|
172
178
|
}
|
|
173
179
|
|
|
174
180
|
// Whisper-server health + cloud budget — exposed so glasses + dashboards can
|
|
175
181
|
// see whether we're at risk of falling to cloud and how much budget remains.
|
|
176
182
|
const whisper_health = getWhisperHealth()
|
|
177
183
|
const openai_whisper_budget = getOpenAIWhisperBudgetState()
|
|
178
|
-
|
|
179
184
|
const codex_models = getCodexModelCatalogSnapshot()
|
|
180
185
|
res.json({
|
|
181
186
|
...checks,
|
|
@@ -187,6 +192,7 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
187
192
|
voice,
|
|
188
193
|
whisper_health,
|
|
189
194
|
openai_whisper_budget,
|
|
195
|
+
tts_local,
|
|
190
196
|
codex_models,
|
|
191
197
|
capabilities: {
|
|
192
198
|
transcription,
|