@gotcos/glasses-server 6.14.1 → 6.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +11 -0
- package/CHANGELOG.md +40 -0
- package/README.md +31 -7
- package/bin/cli.cjs +31 -5
- package/package.json +2 -2
- package/server/index.ts +10 -19
- package/server/lib/api-auth.ts +34 -0
- package/server/lib/tts-cache.ts +32 -5
- package/server/lib/tts-engine.ts +230 -0
- package/server/lib/tts-local.ts +375 -0
- package/server/lib/tts-pronounce.ts +53 -0
- package/server/lib/whisper-local.ts +86 -4
- package/server/routes/health.ts +8 -2
- package/server/routes/tts.ts +579 -199
- package/server/tts-sidecar/bootstrap.sh +57 -0
- package/server/tts-sidecar/requirements.txt +9 -0
- package/server/tts-sidecar/server.py +235 -0
package/.env.example
CHANGED
|
@@ -70,6 +70,17 @@ BIND_HOST=0.0.0.0
|
|
|
70
70
|
# COS_OPENAI_WHISPER_FALLBACK=1
|
|
71
71
|
# OPENAI_API_KEY=sk-...
|
|
72
72
|
|
|
73
|
+
# Spoken reply playback defaults to local Kokoro on Apple silicon Macs. The
|
|
74
|
+
# first run creates a private venv and downloads the model. Local mode fails
|
|
75
|
+
# closed: it never sends text to OpenAI. Set openai_primary only when cloud TTS
|
|
76
|
+
# is desired and OPENAI_API_KEY is configured.
|
|
77
|
+
# COS_TTS_ENGINE=local_first # local_first | openai_primary
|
|
78
|
+
# COS_TTS_LOCAL_PORT=8179
|
|
79
|
+
# COS_TTS_LOCAL_TIMEOUT_MS=12000
|
|
80
|
+
# COS_TTS_KOKORO_VOICE=am_echo
|
|
81
|
+
# COS_TTS_LOCAL_DISABLE=1 # disable the local sidecar explicitly
|
|
82
|
+
# COS_TTS_PRONUNCIATIONS_JSON={"Exampleco":{"local":"[Exampleco](/ɪgzˈæmpəlkoʊ/)","openai":"ig-ZAM-pul-co"}}
|
|
83
|
+
|
|
73
84
|
# ── FULL COS PIPELINE (optional — leave unset for standalone) ────────────
|
|
74
85
|
# Power users running the COS Starter Kit can point the glasses at their
|
|
75
86
|
# pipeline to inherit live tasks/calendar/people context. Omit for standalone.
|
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,43 @@
|
|
|
1
|
+
## 6.15.1
|
|
2
|
+
|
|
3
|
+
- Fix prepared TTS playback for native audio clients by allowing only
|
|
4
|
+
`GET`/`HEAD /api/tts/play/<UUID>` through the global API-token boundary. The
|
|
5
|
+
authenticated prepare route mints a random, audio-scoped capability that
|
|
6
|
+
expires after 60 seconds; all other TTS routes remain token-protected.
|
|
7
|
+
- Fix Kokoro first-run provisioning by selecting only Python 3.11 or 3.12,
|
|
8
|
+
the actual compatibility intersection of the pinned `numpy` and `misaki`
|
|
9
|
+
dependencies. Python 3.13 is no longer advertised or selected.
|
|
10
|
+
- Reject an incompatible `COS_TTS_BOOTSTRAP_PYTHON` before installation and
|
|
11
|
+
automatically rebuild stale or partial TTS virtual environments instead of
|
|
12
|
+
repeatedly failing inside pip.
|
|
13
|
+
|
|
14
|
+
## 6.15.0
|
|
15
|
+
|
|
16
|
+
- Add local-first spoken reply playback through a Mac-owned Kokoro sidecar on
|
|
17
|
+
Apple silicon. The existing OpenAI TTS path remains available as an explicit
|
|
18
|
+
engine choice and as the bounded fallback for `local_first` only when a key
|
|
19
|
+
and budget are available.
|
|
20
|
+
- Preserve cancellation across the local synthesis boundary so abandoned
|
|
21
|
+
requests return 499 and can never become accidental cloud fallbacks. An
|
|
22
|
+
explicitly selected local engine fails closed when Kokoro is unavailable.
|
|
23
|
+
- Keep cache entries and resumable sessions engine-aware, expose additive
|
|
24
|
+
`tts_local` and voice-engine health, and announce a real Kokoro-to-OpenAI
|
|
25
|
+
fallback only after cloud audio succeeds.
|
|
26
|
+
- Bootstrap the pinned private Python environment asynchronously on first run,
|
|
27
|
+
avoiding an API event-loop stall while dependencies install. The local
|
|
28
|
+
sidecar is limited to Apple silicon macOS; other hosts retain their existing
|
|
29
|
+
OpenAI behavior.
|
|
30
|
+
- Add opt-in, public-safe pronunciation overrides through
|
|
31
|
+
`COS_TTS_PRONUNCIATIONS_JSON`. The published package contains no personal
|
|
32
|
+
pronunciation dictionary or machine-specific path.
|
|
33
|
+
|
|
34
|
+
## 6.14.2
|
|
35
|
+
|
|
36
|
+
- Restore timed words on post-meeting CPU polish only: whisper-cli `-ojf`
|
|
37
|
+
token offsets feed speaker-word mapping after save. Live ASR stays on
|
|
38
|
+
compact `json` (no `verbose_json`); VAD-empty CLI windows return empty
|
|
39
|
+
transcription safely without reintroducing the live daemon crash.
|
|
40
|
+
|
|
1
41
|
## 6.14.1
|
|
2
42
|
|
|
3
43
|
- Keep real-time `large-v3-turbo` stable on VAD-empty audio by using
|
package/README.md
CHANGED
|
@@ -11,8 +11,8 @@ API key is pasted into the phone for chat.
|
|
|
11
11
|
npx --yes @gotcos/glasses-server@latest
|
|
12
12
|
```
|
|
13
13
|
|
|
14
|
-
For the optional COS Control macOS menu bar app,
|
|
15
|
-
|
|
14
|
+
For the optional COS Control macOS menu bar app, run the same non-mutating
|
|
15
|
+
readiness check it uses before guided installation:
|
|
16
16
|
|
|
17
17
|
```bash
|
|
18
18
|
npx --yes @gotcos/glasses-server@latest --prepare-only
|
|
@@ -21,9 +21,13 @@ npx --yes @gotcos/glasses-server@latest --prepare-only
|
|
|
21
21
|
COS Control then installs the same npm package as a launchd-managed runtime.
|
|
22
22
|
The original foreground command remains supported and unchanged.
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
Normal server start checks Node, finds your CLI, checks voice and image
|
|
25
|
+
processing, writes `~/.cos-glasses/.env`, and starts the server on
|
|
26
|
+
`0.0.0.0:3141`. Optional Whisper and Kokoro models are provisioned when their
|
|
27
|
+
local services start; `--prepare-only` intentionally does not download or
|
|
28
|
+
install optional models, write COS configuration, or start a listener. It may
|
|
29
|
+
invoke an installed agent CLI's read-only version/auth probe, and that CLI may
|
|
30
|
+
maintain its own user cache. On boot the server prints
|
|
27
31
|
an **API token** — paste that into the COS Glasses app. Only one COS Glasses
|
|
28
32
|
server may run on a Mac at a time; a second `npx` or source runner exits before
|
|
29
33
|
opening ports or touching shared conversation/media state. Version 6.6.0 also
|
|
@@ -41,7 +45,10 @@ without silently losing completed replies.
|
|
|
41
45
|
_or_ **Codex CLI** (GPT Frontier/Balanced) — https://developers.openai.com/codex/, then `codex login`
|
|
42
46
|
- **Even G2 glasses** + the **COS Glasses** app from the Even Hub
|
|
43
47
|
- `brew install whisper-cpp` for free local voice (the launcher can download the model)
|
|
44
|
-
- _Optional:_ `brew install ffmpeg` for
|
|
48
|
+
- _Optional:_ `brew install python@3.12 ffmpeg espeak-ng` for local Kokoro
|
|
49
|
+
spoken replies on Apple silicon (Python 3.11-3.12 is supported). `ffmpeg`
|
|
50
|
+
also enables phone/output image attachments; text chat remains available
|
|
51
|
+
without these optional dependencies.
|
|
45
52
|
- _Optional:_ **Tailscale** so your phone reaches your Mac from anywhere
|
|
46
53
|
|
|
47
54
|
> No `ANTHROPIC_API_KEY` is needed — chat runs through your installed CLI, billed
|
|
@@ -99,6 +106,11 @@ range is the exact Tailscale/CGNAT allocation (`100.64.0.0/10`), not all of
|
|
|
99
106
|
Whisper fallback is optional and requires both the exact
|
|
100
107
|
`COS_OPENAI_WHISPER_FALLBACK=1` opt-in and a configured key; a key alone never
|
|
101
108
|
uploads audio.
|
|
109
|
+
- Local-first spoken reply playback through Kokoro on Apple silicon. The first
|
|
110
|
+
use creates a private Python environment and downloads its model without
|
|
111
|
+
blocking the API. Selecting Local fails closed; `local_first` can fall back
|
|
112
|
+
to OpenAI TTS only when a key and budget are available. `/api/health`
|
|
113
|
+
reports the independent `tts_local` state and current engine.
|
|
102
114
|
- Tasks / calendar / people context **if** you run the
|
|
103
115
|
[COS Starter Kit](https://www.gotcos.com) (`COS_SCRIPTS_DIR`); otherwise it is
|
|
104
116
|
glasses + AI only
|
|
@@ -108,7 +120,11 @@ range is the exact Tailscale/CGNAT allocation (`100.64.0.0/10`), not all of
|
|
|
108
120
|
Config lives at `~/.cos-glasses/.env` (created on first run). Every key is
|
|
109
121
|
optional except an installed CLI. Highlights: `BIND_HOST`, `PORT`,
|
|
110
122
|
`COS_API_TOKEN` (auto if unset), `COS_OPENAI_WHISPER_FALLBACK=1` plus
|
|
111
|
-
`OPENAI_API_KEY` (explicit cloud
|
|
123
|
+
`OPENAI_API_KEY` (explicit cloud transcription/TTS fallback),
|
|
124
|
+
`COS_TTS_ENGINE` (`local_first` or `openai_primary`),
|
|
125
|
+
`COS_TTS_KOKORO_VOICE` (local voice id),
|
|
126
|
+
`COS_TTS_LOCAL_DISABLE=1` (disable the sidecar), and
|
|
127
|
+
`COS_TTS_PRONUNCIATIONS_JSON` (optional local/cloud pronunciation overrides),
|
|
112
128
|
`COS_SCRIPTS_DIR` (full pipeline), `COS_DURABLE_QUERY_JOBS=1` (build 204+
|
|
113
129
|
server-owned query recovery), and `COS_MEDIA_ROOT` (optional image-store
|
|
114
130
|
location; default `~/.cos-glasses/data/media`). Your name + transcription vocabulary live in
|
|
@@ -149,6 +165,14 @@ BIND_HOST=0.0.0.0 npm run start:server
|
|
|
149
165
|
confirm `/api/health` reports `features.whisper: true`. A typed retryable 503
|
|
150
166
|
keeps compatible prompt/meeting audio available for retry instead of silently
|
|
151
167
|
sending it to OpenAI.
|
|
168
|
+
- *Local spoken replies unavailable?* — on Apple silicon, install
|
|
169
|
+
`python@3.12 ffmpeg espeak-ng`, restart the server, and wait for the
|
|
170
|
+
first-run Kokoro model download. Confirm `/api/health` reports
|
|
171
|
+
`tts_local.ready: true`.
|
|
172
|
+
If Python lives outside the normal Homebrew paths, set its absolute 3.11 or
|
|
173
|
+
3.12 path as `COS_TTS_BOOTSTRAP_PYTHON` in `~/.cos-glasses/.env`.
|
|
174
|
+
Selecting Local never falls back to cloud; set `COS_TTS_ENGINE=openai_primary`
|
|
175
|
+
only when OpenAI playback is intentionally configured.
|
|
152
176
|
- *Photos unavailable?* — install `ffmpeg`, restart the server, and confirm `/api/health` reports `features.mediaProcessingReady: true`.
|
|
153
177
|
- *Prompt recovery unavailable?* — update with `npx --yes @gotcos/glasses-server@latest`, then confirm `/api/health` reports `features.promptRecovery: true`.
|
|
154
178
|
- *Durable query recovery unavailable?* — build 204+ requires server 6.10.0+ and
|
package/bin/cli.cjs
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// Runs the bundled server for Even G2 smart glasses. The server ships INSIDE this
|
|
5
5
|
// package — there is no clone. Config persists at ~/.cos-glasses/.
|
|
6
6
|
|
|
7
|
-
const { execSync, spawn } = require('child_process')
|
|
7
|
+
const { execFileSync, execSync, spawn } = require('child_process')
|
|
8
8
|
const {
|
|
9
9
|
existsSync,
|
|
10
10
|
mkdirSync,
|
|
@@ -47,6 +47,7 @@ if (process.argv.includes('--help') || process.argv.includes('-h')) {
|
|
|
47
47
|
console.log(' - Node.js 20.11+')
|
|
48
48
|
console.log(' - Claude Code CLI (not Claude Desktop) or Codex CLI')
|
|
49
49
|
console.log(' - Even G2 smart glasses + the COS Glasses app (Even Hub)')
|
|
50
|
+
console.log(' - Optional local Kokoro voice: Apple silicon + Python 3.11 or 3.12')
|
|
50
51
|
console.log('')
|
|
51
52
|
console.log(' No API key is needed for chat — it runs through your installed CLI.')
|
|
52
53
|
console.log(' Config persists at ~/.cos-glasses/.env')
|
|
@@ -78,6 +79,21 @@ function getCliVersion(command, versionArg = '--version') {
|
|
|
78
79
|
return null
|
|
79
80
|
}
|
|
80
81
|
}
|
|
82
|
+
function compatibleKokoroPython() {
|
|
83
|
+
const configured = process.env.COS_TTS_BOOTSTRAP_PYTHON?.trim()
|
|
84
|
+
const candidates = configured ? [configured] : ['python3.12', 'python3.11', 'python3']
|
|
85
|
+
for (const command of candidates) {
|
|
86
|
+
let version = null
|
|
87
|
+
try {
|
|
88
|
+
version = execFileSync(command, ['--version'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 5000 }).trim()
|
|
89
|
+
} catch { /* try the next interpreter */ }
|
|
90
|
+
const match = version?.match(/^Python\s+(\d+)\.(\d+)/i)
|
|
91
|
+
if (match && Number(match[1]) === 3 && [11, 12].includes(Number(match[2]))) {
|
|
92
|
+
return { command, version }
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return null
|
|
96
|
+
}
|
|
81
97
|
function normalizeCodexVersion(raw) {
|
|
82
98
|
if (!raw) return 'available'
|
|
83
99
|
const line = raw.split('\n').map((s) => s.trim()).find((s) => /^codex(?:-cli)?\s+/i.test(s)) || raw.split('\n')[0].trim()
|
|
@@ -175,13 +191,23 @@ try {
|
|
|
175
191
|
process.exit(1)
|
|
176
192
|
}
|
|
177
193
|
|
|
178
|
-
// Controller probes
|
|
179
|
-
// and the packaged runtime without creating config, downloading
|
|
180
|
-
// changing permissions, or starting a
|
|
194
|
+
// Controller probes do not mutate COS state. They verify Node, agent auth,
|
|
195
|
+
// and the packaged runtime without creating COS config, downloading Kokoro
|
|
196
|
+
// models, installing Kokoro packages, changing COS permissions, or starting a
|
|
197
|
+
// listener. An invoked agent CLI may still maintain its own user cache.
|
|
181
198
|
if (process.argv.includes('--prepare-only')) {
|
|
199
|
+
if (process.platform === 'darwin' && process.arch === 'arm64') {
|
|
200
|
+
const python = compatibleKokoroPython()
|
|
201
|
+
if (python) {
|
|
202
|
+
console.log(green(' ✓') + ` Local voice Python ${python.version.replace(/^Python\s+/i, '')}`)
|
|
203
|
+
} else {
|
|
204
|
+
console.log(yellow(' ⚠') + ' Local Kokoro voice needs Python 3.11 or 3.12')
|
|
205
|
+
console.log(' Install: ' + bold('brew install python@3.12 ffmpeg espeak-ng'))
|
|
206
|
+
}
|
|
207
|
+
}
|
|
182
208
|
console.log('')
|
|
183
209
|
console.log(green(' ✓ Non-mutating readiness check complete'))
|
|
184
|
-
console.log('
|
|
210
|
+
console.log(' This checks prerequisites only; normal server start provisions optional models.')
|
|
185
211
|
console.log('')
|
|
186
212
|
process.exit(0)
|
|
187
213
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gotcos/glasses-server",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.15.1",
|
|
4
4
|
"description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by your local Claude Code or Codex CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"scripts": {
|
|
11
11
|
"start": "node bin/cli.cjs",
|
|
12
12
|
"start:server": "node --import tsx/esm server/index.ts",
|
|
13
|
-
"test": "vitest run",
|
|
13
|
+
"test": "vitest run --maxWorkers=1",
|
|
14
14
|
"typecheck": "tsc --noEmit"
|
|
15
15
|
},
|
|
16
16
|
"keywords": [
|
package/server/index.ts
CHANGED
|
@@ -44,6 +44,7 @@ import {
|
|
|
44
44
|
stopCodexModelCatalogRefresh,
|
|
45
45
|
} from './lib/codex-model-catalog.js'
|
|
46
46
|
import { startWhisperServer, stopWhisperServer } from './lib/whisper-local.js'
|
|
47
|
+
import { startLocalTtsServer, stopLocalTtsServer } from './lib/tts-local.js'
|
|
47
48
|
import { initSileroVAD } from './lib/vad-silero.js'
|
|
48
49
|
import { initSessionCache } from './lib/session-cache-writer.js'
|
|
49
50
|
import { initSpeakerEmbeddings } from './lib/speaker-embeddings.js'
|
|
@@ -65,7 +66,7 @@ import {
|
|
|
65
66
|
isAllowedNetworkOrigin,
|
|
66
67
|
isTailscaleIpv4,
|
|
67
68
|
} from './lib/network-policy.js'
|
|
68
|
-
import {
|
|
69
|
+
import { requireApiToken } from './lib/api-auth.js'
|
|
69
70
|
import { isManagedRuntime } from './lib/managed-runtime.js'
|
|
70
71
|
import {
|
|
71
72
|
acquireMaintenanceWork,
|
|
@@ -134,24 +135,12 @@ app.use(cors({
|
|
|
134
135
|
cb(new Error('CORS blocked'))
|
|
135
136
|
},
|
|
136
137
|
}))
|
|
137
|
-
// Auth middleware — always active (token is auto-generated if not set)
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
// boot window we're trying to observe.
|
|
144
|
-
if (
|
|
145
|
-
req.path === '/health' ||
|
|
146
|
-
req.path === '/display-stream' ||
|
|
147
|
-
req.path === '/diag/client' ||
|
|
148
|
-
req.path === '/diag/health'
|
|
149
|
-
) return next()
|
|
150
|
-
if (!timingSafeTokenEqual(req.headers['x-cos-token'], API_TOKEN)) {
|
|
151
|
-
return res.status(401).json({ error: 'unauthorized' })
|
|
152
|
-
}
|
|
153
|
-
next()
|
|
154
|
-
})
|
|
138
|
+
// Auth middleware — always active (token is auto-generated if not set).
|
|
139
|
+
// Mounted before body parsers so rejected uploads cannot consume parse memory.
|
|
140
|
+
// The only capability-URL exception is a canonical /tts/play/<UUID> GET/HEAD;
|
|
141
|
+
// authenticated /tts/prepare mints it for native audio players that cannot set
|
|
142
|
+
// X-Cos-Token headers.
|
|
143
|
+
app.use('/api', requireApiToken(API_TOKEN))
|
|
155
144
|
|
|
156
145
|
// Fail-closed catch-all for mutation routes that do not own a more specific
|
|
157
146
|
// lifecycle lease below. This closes the admission/drain race for secondary
|
|
@@ -266,6 +255,7 @@ async function gracefulShutdown(): Promise<void> {
|
|
|
266
255
|
try { logActiveSessionsOnShutdown() } catch { /* best-effort flush */ }
|
|
267
256
|
stopCodexModelCatalogRefresh()
|
|
268
257
|
stopWhisperServer()
|
|
258
|
+
stopLocalTtsServer()
|
|
269
259
|
clearTimeout(forceExit)
|
|
270
260
|
process.exit(0)
|
|
271
261
|
}
|
|
@@ -385,6 +375,7 @@ listenRequiredServers(listeners).then(() => {
|
|
|
385
375
|
}
|
|
386
376
|
// Start local whisper-server (model stays in RAM for ~50ms transcription)
|
|
387
377
|
startWhisperServer().catch(err => console.error('[startup] Whisper server error:', err))
|
|
378
|
+
startLocalTtsServer().catch(err => console.error('[startup] Local TTS server error:', err))
|
|
388
379
|
// Initialize speaker embeddings (voiceprint-based diarization) — fails soft if model absent
|
|
389
380
|
const embeddingOk = initSpeakerEmbeddings()
|
|
390
381
|
console.log(`[startup] Speaker embeddings: ${embeddingOk ? 'active' : 'disabled (model not found)'}`)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { RequestHandler } from 'express'
|
|
2
|
+
import { timingSafeTokenEqual } from './token-auth.js'
|
|
3
|
+
|
|
4
|
+
// Recovery/setup clients need these availability surfaces before they have a
|
|
5
|
+
// usable token. Keep private provider state and every mutation route out.
|
|
6
|
+
const PUBLIC_API_PATHS = new Set([
|
|
7
|
+
'/health',
|
|
8
|
+
'/display-stream',
|
|
9
|
+
'/diag/client',
|
|
10
|
+
'/diag/health',
|
|
11
|
+
])
|
|
12
|
+
|
|
13
|
+
// Native HTML audio requests cannot attach X-Cos-Token. The UUID minted by
|
|
14
|
+
// authenticated POST /tts/prepare is therefore a short-lived bearer
|
|
15
|
+
// capability. Keep this exception exact: GET/HEAD only, one canonical v4 UUID
|
|
16
|
+
// path segment, and no query-token fallback that could leak into URL logs.
|
|
17
|
+
const TTS_PLAYBACK_CAPABILITY_PATH = /^\/tts\/play\/[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
|
|
18
|
+
|
|
19
|
+
export function isPublicApiRequest(method: string, path: string): boolean {
|
|
20
|
+
if (PUBLIC_API_PATHS.has(path)) return true
|
|
21
|
+
return (method === 'GET' || method === 'HEAD')
|
|
22
|
+
&& TTS_PLAYBACK_CAPABILITY_PATH.test(path)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Global /api authentication boundary. Mount before all body parsers. */
|
|
26
|
+
export function requireApiToken(apiToken: string): RequestHandler {
|
|
27
|
+
return (req, res, next) => {
|
|
28
|
+
if (isPublicApiRequest(req.method, req.path)) return next()
|
|
29
|
+
if (!timingSafeTokenEqual(req.headers['x-cos-token'], apiToken)) {
|
|
30
|
+
return res.status(401).json({ error: 'unauthorized' })
|
|
31
|
+
}
|
|
32
|
+
next()
|
|
33
|
+
}
|
|
34
|
+
}
|
package/server/lib/tts-cache.ts
CHANGED
|
@@ -67,6 +67,10 @@ interface SessionEntry {
|
|
|
67
67
|
text: string
|
|
68
68
|
voice: string
|
|
69
69
|
format: string
|
|
70
|
+
/** One-shot OpenAI opt-in for this session (play cold-miss must honor it). */
|
|
71
|
+
preferOpenAI?: boolean
|
|
72
|
+
/** Settings forced Local/Kokoro — do not auto-escape to OpenAI on play miss. */
|
|
73
|
+
forceLocal?: boolean
|
|
70
74
|
expiresAt: number
|
|
71
75
|
}
|
|
72
76
|
|
|
@@ -114,8 +118,18 @@ let totalDiskBytes = 0 // on-disk bytes (sum of all sidecar sizeBytes)
|
|
|
114
118
|
* a different voice for the same text correctly misses (and gets its own
|
|
115
119
|
* entry). Hash is sha256 truncated to 16 hex chars — collision probability is
|
|
116
120
|
* ~negligible for our scale. */
|
|
117
|
-
export function hashKey(
|
|
118
|
-
|
|
121
|
+
export function hashKey(
|
|
122
|
+
engine: string,
|
|
123
|
+
mappedVoice: string,
|
|
124
|
+
format: string,
|
|
125
|
+
text: string,
|
|
126
|
+
instructions?: string,
|
|
127
|
+
): string {
|
|
128
|
+
let material = `${engine}\0${mappedVoice}\0${format}\0${text}`
|
|
129
|
+
if (engine === 'openai' && instructions && instructions.trim().length > 0) {
|
|
130
|
+
material += `\0${instructions.trim()}`
|
|
131
|
+
}
|
|
132
|
+
return createHash('sha256').update(material).digest('hex').slice(0, 16)
|
|
119
133
|
}
|
|
120
134
|
|
|
121
135
|
function ensureDiskDir(): void {
|
|
@@ -481,9 +495,10 @@ function sweepStaleByAge(): void {
|
|
|
481
495
|
}
|
|
482
496
|
}
|
|
483
497
|
|
|
484
|
-
/** Allocate a new session UUID pointing at
|
|
485
|
-
* bundle. The play route
|
|
486
|
-
*
|
|
498
|
+
/** Allocate a new random v4 session UUID pointing at one
|
|
499
|
+
* (hash, text, voice, format) bundle. The play route may reread it for native
|
|
500
|
+
* Range refills during the 60-second TTL; expired sessions are rejected and
|
|
501
|
+
* reaped by the periodic sweeper below. */
|
|
487
502
|
export function createSession(s: Omit<SessionEntry, 'expiresAt'>): string {
|
|
488
503
|
const uuid = randomUUID()
|
|
489
504
|
sessions.set(uuid, { ...s, expiresAt: Date.now() + SESSION_TTL_MS })
|
|
@@ -512,6 +527,18 @@ export function peekSession(uuid: string): SessionEntry | null {
|
|
|
512
527
|
return s
|
|
513
528
|
}
|
|
514
529
|
|
|
530
|
+
/** Rebind a live session to a new cache hash (openai -> local fallback). */
|
|
531
|
+
export function rebindSessionHash(uuid: string, hash: string): boolean {
|
|
532
|
+
const s = sessions.get(uuid)
|
|
533
|
+
if (!s) return false
|
|
534
|
+
if (s.expiresAt < Date.now()) {
|
|
535
|
+
sessions.delete(uuid)
|
|
536
|
+
return false
|
|
537
|
+
}
|
|
538
|
+
s.hash = hash
|
|
539
|
+
return true
|
|
540
|
+
}
|
|
541
|
+
|
|
515
542
|
/** Strict one-shot lookup — kept for callers that want to invalidate the
|
|
516
543
|
* session immediately on first read. Not used by /play in v5.9.4+. */
|
|
517
544
|
export function consumeSession(uuid: string): SessionEntry | null {
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
// Local TTS engine mode + OpenAI voice → Kokoro preset map.
|
|
2
|
+
// Default product mode: local_first (Kokoro). OpenAI is opt-in escape hatch.
|
|
3
|
+
// Modes: openai | openai_primary | local_first | local
|
|
4
|
+
// Bare "auto" → local_first.
|
|
5
|
+
// Per-request engine: 'local' | 'openai' forces that backend (Settings picker).
|
|
6
|
+
|
|
7
|
+
export type TtsEngineMode = 'openai' | 'openai_primary' | 'local_first' | 'local'
|
|
8
|
+
export type TtsBackend = 'openai' | 'local'
|
|
9
|
+
export type TtsEnginePreference = 'local' | 'openai'
|
|
10
|
+
|
|
11
|
+
/** Cache/engine tag written into hashKey — distinguishes OpenAI audio from Kokoro. */
|
|
12
|
+
export type TtsEngineTag = 'openai' | 'kokoro'
|
|
13
|
+
|
|
14
|
+
export const OPENAI_VOICE_OPTIONS = [
|
|
15
|
+
{ id: 'echo', label: 'Echo (default)' },
|
|
16
|
+
{ id: 'alloy', label: 'Alloy' },
|
|
17
|
+
{ id: 'nova', label: 'Nova' },
|
|
18
|
+
{ id: 'sage', label: 'Sage' },
|
|
19
|
+
{ id: 'onyx', label: 'Onyx' },
|
|
20
|
+
{ id: 'shimmer', label: 'Shimmer' },
|
|
21
|
+
{ id: 'fable', label: 'Fable' },
|
|
22
|
+
{ id: 'ash', label: 'Ash' },
|
|
23
|
+
{ id: 'coral', label: 'Coral' },
|
|
24
|
+
] as const
|
|
25
|
+
|
|
26
|
+
/** American English Kokoro presets verified on disk (mlx-community/Kokoro-82M-bf16). */
|
|
27
|
+
export const KOKORO_VOICE_OPTIONS = [
|
|
28
|
+
{ id: 'am_echo', label: 'Echo (am_echo) · default' },
|
|
29
|
+
{ id: 'am_michael', label: 'Michael (am_michael)' },
|
|
30
|
+
{ id: 'am_fenrir', label: 'Fenrir (am_fenrir)' },
|
|
31
|
+
{ id: 'am_puck', label: 'Puck (am_puck)' },
|
|
32
|
+
{ id: 'am_onyx', label: 'Onyx (am_onyx)' },
|
|
33
|
+
{ id: 'am_eric', label: 'Eric (am_eric)' },
|
|
34
|
+
{ id: 'am_liam', label: 'Liam (am_liam)' },
|
|
35
|
+
{ id: 'am_adam', label: 'Adam (am_adam)' },
|
|
36
|
+
{ id: 'am_santa', label: 'Santa (am_santa)' },
|
|
37
|
+
{ id: 'af_heart', label: 'Heart (af_heart)' },
|
|
38
|
+
{ id: 'af_bella', label: 'Bella (af_bella)' },
|
|
39
|
+
{ id: 'af_nova', label: 'Nova (af_nova)' },
|
|
40
|
+
{ id: 'af_sarah', label: 'Sarah (af_sarah)' },
|
|
41
|
+
{ id: 'af_sky', label: 'Sky (af_sky)' },
|
|
42
|
+
{ id: 'af_nicole', label: 'Nicole (af_nicole)' },
|
|
43
|
+
{ id: 'af_jessica', label: 'Jessica (af_jessica)' },
|
|
44
|
+
{ id: 'af_alloy', label: 'Alloy (af_alloy)' },
|
|
45
|
+
{ id: 'af_aoede', label: 'Aoede (af_aoede)' },
|
|
46
|
+
{ id: 'af_kore', label: 'Kore (af_kore)' },
|
|
47
|
+
{ id: 'af_river', label: 'River (af_river)' },
|
|
48
|
+
] as const
|
|
49
|
+
|
|
50
|
+
export const KOKORO_EN_US_VOICES = KOKORO_VOICE_OPTIONS.map((v) => v.id)
|
|
51
|
+
|
|
52
|
+
// OpenAI id → Kokoro when Settings still sends an OpenAI id on the local path.
|
|
53
|
+
const VOICE_MAP: Record<string, string> = {
|
|
54
|
+
echo: 'am_echo',
|
|
55
|
+
alloy: 'af_heart',
|
|
56
|
+
fable: 'af_bella',
|
|
57
|
+
onyx: 'am_onyx',
|
|
58
|
+
nova: 'af_nova',
|
|
59
|
+
shimmer: 'af_sky',
|
|
60
|
+
ash: 'am_fenrir',
|
|
61
|
+
sage: 'af_sarah',
|
|
62
|
+
coral: 'af_jessica',
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const OPENAI_VOICE_IDS = new Set(OPENAI_VOICE_OPTIONS.map((v) => v.id))
|
|
66
|
+
const KOKORO_VOICE_IDS = new Set(KOKORO_EN_US_VOICES)
|
|
67
|
+
|
|
68
|
+
export function isOpenAIVoiceId(voice: string): boolean {
|
|
69
|
+
return OPENAI_VOICE_IDS.has(voice as typeof OPENAI_VOICE_OPTIONS[number]['id'])
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Kokoro / Misaki voice file ids look like am_echo, af_heart, bm_george. */
|
|
73
|
+
export function isKokoroVoiceId(voice: string): boolean {
|
|
74
|
+
return /^[a-z]{2}_[a-z0-9]+$/i.test(voice)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function getTtsEngineMode(): TtsEngineMode {
|
|
78
|
+
const raw = (process.env.COS_TTS_ENGINE || 'local_first').trim().toLowerCase()
|
|
79
|
+
if (raw === 'auto') return 'local_first'
|
|
80
|
+
if (raw === 'openai' || raw === 'openai_primary' || raw === 'local_first' || raw === 'local') {
|
|
81
|
+
return raw
|
|
82
|
+
}
|
|
83
|
+
console.warn(`[tts-engine] unknown COS_TTS_ENGINE=${raw}; using local_first`)
|
|
84
|
+
return 'local_first'
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Resolve which Kokoro preset to synthesize for a local request. */
|
|
88
|
+
export function resolveLocalVoice(requested: string): string {
|
|
89
|
+
const v = (requested || '').trim()
|
|
90
|
+
if (v && isKokoroVoiceId(v)) {
|
|
91
|
+
const lower = v.toLowerCase()
|
|
92
|
+
// Accept any on-disk-shaped id; known list is the Settings catalog.
|
|
93
|
+
return lower
|
|
94
|
+
}
|
|
95
|
+
// Env pin only when mapping legacy OpenAI ids (not when client sent Kokoro).
|
|
96
|
+
const pinned = (process.env.COS_TTS_KOKORO_VOICE || '').trim()
|
|
97
|
+
if (pinned && isKokoroVoiceId(pinned)) return pinned.toLowerCase()
|
|
98
|
+
return VOICE_MAP[v] ?? VOICE_MAP.echo
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** @deprecated use resolveLocalVoice — kept for tests / call sites. */
|
|
102
|
+
export function mapOpenAIVoiceToLocal(openaiVoice: string): string {
|
|
103
|
+
return resolveLocalVoice(openaiVoice)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function localEngineTag(): TtsEngineTag {
|
|
107
|
+
const pinned = (process.env.COS_TTS_LOCAL_ENGINE || 'kokoro').trim().toLowerCase()
|
|
108
|
+
return pinned === 'kokoro' ? 'kokoro' : 'kokoro'
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export interface TtsRouteDecision {
|
|
112
|
+
backend: TtsBackend
|
|
113
|
+
engineTag: TtsEngineTag
|
|
114
|
+
/** Voice id sent to the chosen backend. */
|
|
115
|
+
backendVoice: string
|
|
116
|
+
/** Voice id from the client (OpenAI or Kokoro). */
|
|
117
|
+
openaiVoice: string
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Pick initial backend for a request.
|
|
122
|
+
*
|
|
123
|
+
* enginePreference:
|
|
124
|
+
* - 'openai' → cloud (Settings “OpenAI”)
|
|
125
|
+
* - 'local' → Kokoro (Settings “Local”)
|
|
126
|
+
* - omitted → daemon COS_TTS_ENGINE (local_first default)
|
|
127
|
+
*/
|
|
128
|
+
export function decideInitialBackend(opts: {
|
|
129
|
+
openaiVoice: string
|
|
130
|
+
openaiKeyPresent: boolean
|
|
131
|
+
openaiBudgetOk: boolean
|
|
132
|
+
localReady: boolean
|
|
133
|
+
preferOpenAI?: boolean
|
|
134
|
+
enginePreference?: TtsEnginePreference | null
|
|
135
|
+
}): TtsRouteDecision {
|
|
136
|
+
const requested = opts.openaiVoice
|
|
137
|
+
const localVoice = resolveLocalVoice(requested)
|
|
138
|
+
const openaiVoice = isOpenAIVoiceId(requested) ? requested : 'echo'
|
|
139
|
+
const local: TtsRouteDecision = {
|
|
140
|
+
backend: 'local',
|
|
141
|
+
engineTag: localEngineTag(),
|
|
142
|
+
backendVoice: localVoice,
|
|
143
|
+
openaiVoice: requested,
|
|
144
|
+
}
|
|
145
|
+
const openai: TtsRouteDecision = {
|
|
146
|
+
backend: 'openai',
|
|
147
|
+
engineTag: 'openai',
|
|
148
|
+
backendVoice: openaiVoice,
|
|
149
|
+
openaiVoice: requested,
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const preferOpenAI =
|
|
153
|
+
opts.preferOpenAI === true || opts.enginePreference === 'openai'
|
|
154
|
+
const forceLocal = opts.enginePreference === 'local'
|
|
155
|
+
|
|
156
|
+
if (forceLocal) {
|
|
157
|
+
if (!opts.localReady) {
|
|
158
|
+
throw new Error('Local TTS selected but Kokoro sidecar is not ready')
|
|
159
|
+
}
|
|
160
|
+
return local
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if (preferOpenAI) {
|
|
164
|
+
if (opts.openaiKeyPresent && opts.openaiBudgetOk) return openai
|
|
165
|
+
if (opts.localReady) {
|
|
166
|
+
console.warn('[tts-engine] OpenAI requested but unavailable; using local')
|
|
167
|
+
return local
|
|
168
|
+
}
|
|
169
|
+
throw new Error('OpenAI TTS requested but key/budget unavailable and local TTS not ready')
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Legacy: bare Kokoro voice id implies local even without enginePreference.
|
|
173
|
+
if (isKokoroVoiceId(requested) && opts.localReady) return local
|
|
174
|
+
|
|
175
|
+
const mode = getTtsEngineMode()
|
|
176
|
+
|
|
177
|
+
if (mode === 'openai') {
|
|
178
|
+
if (!opts.openaiKeyPresent) {
|
|
179
|
+
throw new Error('COS_TTS_ENGINE=openai but OPENAI_API_KEY is missing')
|
|
180
|
+
}
|
|
181
|
+
if (!opts.openaiBudgetOk) {
|
|
182
|
+
throw new Error('COS_TTS_ENGINE=openai but daily OpenAI TTS budget is exhausted')
|
|
183
|
+
}
|
|
184
|
+
return openai
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
if (mode === 'local') {
|
|
188
|
+
if (!opts.localReady) {
|
|
189
|
+
throw new Error('COS_TTS_ENGINE=local but local TTS sidecar is not ready')
|
|
190
|
+
}
|
|
191
|
+
return local
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
if (mode === 'local_first') {
|
|
195
|
+
if (opts.localReady) return local
|
|
196
|
+
if (opts.openaiKeyPresent && opts.openaiBudgetOk) return openai
|
|
197
|
+
throw new Error('local TTS unavailable and OpenAI key/budget not usable')
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// openai_primary
|
|
201
|
+
if (opts.openaiKeyPresent && opts.openaiBudgetOk) return openai
|
|
202
|
+
if (opts.localReady) return local
|
|
203
|
+
if (!opts.openaiKeyPresent) {
|
|
204
|
+
throw new Error('OpenAI TTS key missing and local TTS sidecar is not ready')
|
|
205
|
+
}
|
|
206
|
+
throw new Error('OpenAI TTS budget exhausted and local TTS sidecar is not ready')
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** After an OpenAI failure, can we fall back to local under this mode? */
|
|
210
|
+
export function canFallbackToLocal(
|
|
211
|
+
mode: TtsEngineMode,
|
|
212
|
+
localReady: boolean,
|
|
213
|
+
preferOpenAI = false,
|
|
214
|
+
): boolean {
|
|
215
|
+
if (!localReady) return false
|
|
216
|
+
if (preferOpenAI) return true
|
|
217
|
+
return mode === 'openai_primary' || mode === 'local_first'
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** After a local failure, can we fall back to OpenAI under this mode? */
|
|
221
|
+
export function canFallbackToOpenAI(
|
|
222
|
+
mode: TtsEngineMode,
|
|
223
|
+
openaiKeyPresent: boolean,
|
|
224
|
+
openaiBudgetOk: boolean,
|
|
225
|
+
forceLocal = false,
|
|
226
|
+
): boolean {
|
|
227
|
+
if (forceLocal) return false
|
|
228
|
+
return openaiKeyPresent && openaiBudgetOk && mode === 'local_first'
|
|
229
|
+
}
|
|
230
|
+
|