@gotcos/glasses-server 6.15.3 → 6.15.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/package.json +1 -1
- package/server/index.ts +48 -23
- package/server/lib/api-auth.ts +5 -1
- package/server/lib/audio-enhance.ts +15 -8
- package/server/lib/claude-bridge.ts +48 -18
- package/server/lib/claude-tool-access.ts +73 -7
- package/server/lib/codex-bridge.ts +60 -27
- package/server/lib/maintenance-lifecycle.ts +40 -0
- package/server/lib/provider-process-lifecycle.ts +139 -0
- package/server/lib/provider-proof.ts +28 -10
- package/server/lib/transcribe-audio.ts +10 -2
- package/server/lib/whisper-local.ts +92 -17
- package/server/routes/health.ts +18 -0
- package/server/routes/openai-compat.ts +54 -27
- package/server/routes/prompt-drafts.ts +79 -7
- package/server/routes/provider-proof.ts +40 -2
|
@@ -310,6 +310,7 @@ export class MaintenanceLifecycle {
|
|
|
310
310
|
private blockedGateReason: BlockedGateReason | null = null
|
|
311
311
|
private blockedGateVersion: number | null = null
|
|
312
312
|
private readonly work = new Map<string, WorkEntry>()
|
|
313
|
+
private readonly admissionsOpenListeners = new Set<() => void>()
|
|
313
314
|
|
|
314
315
|
constructor(options: MaintenanceLifecycleOptions = {}) {
|
|
315
316
|
this.path = options.path ?? process.env.COS_MAINTENANCE_GATE_PATH?.trim() ?? DEFAULT_GATE_PATH
|
|
@@ -356,11 +357,44 @@ export class MaintenanceLifecycle {
|
|
|
356
357
|
try {
|
|
357
358
|
removeGate(this.path)
|
|
358
359
|
this.gate = null
|
|
360
|
+
this.notifyAdmissionsOpen()
|
|
359
361
|
} catch {
|
|
360
362
|
this.blockedGateReason = 'invalid_schema'
|
|
361
363
|
}
|
|
362
364
|
}
|
|
363
365
|
|
|
366
|
+
/**
|
|
367
|
+
* Register deferred runtime initialization that is safe only after the
|
|
368
|
+
* durable maintenance gate opens. Delivery is asynchronous so the release
|
|
369
|
+
* response is never held open by model/cache initialization. The callback is
|
|
370
|
+
* also delivered for an already-open lifecycle, closing the boot/release
|
|
371
|
+
* race without forcing index.ts to poll.
|
|
372
|
+
*/
|
|
373
|
+
onAdmissionsOpen(listener: () => void): () => void {
|
|
374
|
+
this.expireSameBootGateIfPermitted()
|
|
375
|
+
this.admissionsOpenListeners.add(listener)
|
|
376
|
+
if (!this.gate && !this.blockedGateReason) this.deliverAdmissionsOpen(listener)
|
|
377
|
+
return () => { this.admissionsOpenListeners.delete(listener) }
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
private deliverAdmissionsOpen(listener: () => void): void {
|
|
381
|
+
queueMicrotask(() => {
|
|
382
|
+
if (!this.admissionsOpenListeners.has(listener)) return
|
|
383
|
+
// State may have closed again between scheduling and microtask delivery.
|
|
384
|
+
// Never start admitted runtime from a stale accepting notification.
|
|
385
|
+
this.expireSameBootGateIfPermitted()
|
|
386
|
+
if (this.gate || this.blockedGateReason) return
|
|
387
|
+
try { listener() } catch (error) {
|
|
388
|
+
console.error('[maintenance] admissions-open listener failed:', error)
|
|
389
|
+
}
|
|
390
|
+
})
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
private notifyAdmissionsOpen(): void {
|
|
394
|
+
if (this.gate || this.blockedGateReason) return
|
|
395
|
+
for (const listener of this.admissionsOpenListeners) this.deliverAdmissionsOpen(listener)
|
|
396
|
+
}
|
|
397
|
+
|
|
364
398
|
private credentialsMatch(credentials: MaintenanceOperationCredentials): {
|
|
365
399
|
leaseMatches: boolean
|
|
366
400
|
operationMatches: boolean
|
|
@@ -615,6 +649,7 @@ export class MaintenanceLifecycle {
|
|
|
615
649
|
)
|
|
616
650
|
}
|
|
617
651
|
this.gate = null
|
|
652
|
+
this.notifyAdmissionsOpen()
|
|
618
653
|
}
|
|
619
654
|
|
|
620
655
|
cancelDrain(identity: MaintenanceOperationIdentity, credentials: MaintenanceOperationCredentials): void {
|
|
@@ -641,6 +676,7 @@ export class MaintenanceLifecycle {
|
|
|
641
676
|
)
|
|
642
677
|
}
|
|
643
678
|
this.gate = null
|
|
679
|
+
this.notifyAdmissionsOpen()
|
|
644
680
|
}
|
|
645
681
|
|
|
646
682
|
snapshot(credentials: MaintenanceOperationCredentials = {}, extraActiveByKind: Record<string, number> = {}) {
|
|
@@ -741,6 +777,10 @@ export function maintenanceAdmissionsOpen(): boolean {
|
|
|
741
777
|
return maintenanceLifecycle.snapshot().admissionsOpen
|
|
742
778
|
}
|
|
743
779
|
|
|
780
|
+
export function onMaintenanceAdmissionsOpen(listener: () => void): () => void {
|
|
781
|
+
return maintenanceLifecycle.onAdmissionsOpen(listener)
|
|
782
|
+
}
|
|
783
|
+
|
|
744
784
|
export function maintenanceErrorPayload(error: MaintenanceLifecycleError) {
|
|
745
785
|
return {
|
|
746
786
|
error: error.code,
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import type { ChildProcess } from 'node:child_process'
|
|
2
|
+
|
|
3
|
+
export interface ProviderTerminationResult {
|
|
4
|
+
closed: boolean
|
|
5
|
+
escalated: boolean
|
|
6
|
+
code: number | null
|
|
7
|
+
signal: NodeJS.Signals | null
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
interface ProviderTerminationOptions {
|
|
11
|
+
termGraceMs?: number
|
|
12
|
+
killWaitMs?: number
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** A detached provider's process group can outlive its CLI leader. ESRCH is
|
|
16
|
+
* the only proof that no member remains; EPERM and unknown probe failures are
|
|
17
|
+
* treated as alive so lifecycle ownership fails closed. */
|
|
18
|
+
function providerGroupAlive(pid: number | undefined): boolean {
|
|
19
|
+
if (!pid || process.platform === 'win32') return false
|
|
20
|
+
try {
|
|
21
|
+
process.kill(-pid, 0)
|
|
22
|
+
return true
|
|
23
|
+
} catch (error: any) {
|
|
24
|
+
return error?.code !== 'ESRCH'
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Signal the detached provider process group so tool subprocesses cannot be
|
|
29
|
+
* orphaned behind the CLI wrapper. Direct-child signaling is a safe fallback
|
|
30
|
+
* for a process that failed before its process group was established. */
|
|
31
|
+
function signalProviderTree(proc: ChildProcess, signal: NodeJS.Signals): void {
|
|
32
|
+
const pid = proc.pid
|
|
33
|
+
if (pid && process.platform !== 'win32') {
|
|
34
|
+
try {
|
|
35
|
+
process.kill(-pid, signal)
|
|
36
|
+
return
|
|
37
|
+
} catch {
|
|
38
|
+
// Fall through to the direct child. close/error remains authoritative.
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
try { proc.kill(signal) } catch { /* process already terminal */ }
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Terminate provider work and resolve only after Node observes leader close
|
|
46
|
+
* and the detached process group no longer exists.
|
|
47
|
+
* A caller must retain its maintenance lease when `closed` is false: releasing
|
|
48
|
+
* without a close event could let Control restart while a tool is still alive.
|
|
49
|
+
*/
|
|
50
|
+
export function terminateProviderProcess(
|
|
51
|
+
proc: ChildProcess,
|
|
52
|
+
options: ProviderTerminationOptions = {},
|
|
53
|
+
): Promise<ProviderTerminationResult> {
|
|
54
|
+
const termGraceMs = Math.max(10, options.termGraceMs ?? 2_000)
|
|
55
|
+
const killWaitMs = Math.max(10, options.killWaitMs ?? 2_000)
|
|
56
|
+
const groupPid = proc.pid
|
|
57
|
+
const initiallyClosed = proc.exitCode !== null || proc.signalCode !== null
|
|
58
|
+
|
|
59
|
+
if (initiallyClosed && !providerGroupAlive(groupPid)) {
|
|
60
|
+
return Promise.resolve({
|
|
61
|
+
closed: true,
|
|
62
|
+
escalated: false,
|
|
63
|
+
code: proc.exitCode,
|
|
64
|
+
signal: proc.signalCode,
|
|
65
|
+
})
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return new Promise(resolve => {
|
|
69
|
+
let settled = false
|
|
70
|
+
let escalated = false
|
|
71
|
+
let leaderClosed = initiallyClosed
|
|
72
|
+
let leaderCode = proc.exitCode
|
|
73
|
+
let leaderSignal = proc.signalCode
|
|
74
|
+
let escalationTimer: ReturnType<typeof setTimeout> | undefined
|
|
75
|
+
let terminalTimer: ReturnType<typeof setTimeout> | undefined
|
|
76
|
+
|
|
77
|
+
const finish = (result: ProviderTerminationResult) => {
|
|
78
|
+
if (settled) return
|
|
79
|
+
settled = true
|
|
80
|
+
if (escalationTimer) clearTimeout(escalationTimer)
|
|
81
|
+
if (terminalTimer) clearTimeout(terminalTimer)
|
|
82
|
+
proc.removeListener('close', onClose)
|
|
83
|
+
proc.removeListener('error', onError)
|
|
84
|
+
resolve(result)
|
|
85
|
+
}
|
|
86
|
+
const finishIfTreeClosed = (): boolean => {
|
|
87
|
+
if (!leaderClosed || providerGroupAlive(groupPid)) return false
|
|
88
|
+
finish({
|
|
89
|
+
closed: true,
|
|
90
|
+
escalated,
|
|
91
|
+
code: leaderCode,
|
|
92
|
+
signal: leaderSignal,
|
|
93
|
+
})
|
|
94
|
+
return true
|
|
95
|
+
}
|
|
96
|
+
const onClose = (code: number | null, signal: NodeJS.Signals | null) => {
|
|
97
|
+
leaderClosed = true
|
|
98
|
+
leaderCode = code
|
|
99
|
+
leaderSignal = signal
|
|
100
|
+
finishIfTreeClosed()
|
|
101
|
+
}
|
|
102
|
+
const onError = () => {
|
|
103
|
+
// A spawn failure has no live process to retain. Running children still
|
|
104
|
+
// produce close after an error, so wait for that authoritative event.
|
|
105
|
+
if (!proc.pid) {
|
|
106
|
+
leaderClosed = true
|
|
107
|
+
leaderCode = null
|
|
108
|
+
leaderSignal = null
|
|
109
|
+
finishIfTreeClosed()
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
proc.once('close', onClose)
|
|
114
|
+
proc.once('error', onError)
|
|
115
|
+
try { proc.stdin?.destroy() } catch { /* best effort */ }
|
|
116
|
+
signalProviderTree(proc, 'SIGTERM')
|
|
117
|
+
|
|
118
|
+
escalationTimer = setTimeout(() => {
|
|
119
|
+
if (finishIfTreeClosed()) return
|
|
120
|
+
escalated = true
|
|
121
|
+
signalProviderTree(proc, 'SIGKILL')
|
|
122
|
+
const deadline = Date.now() + killWaitMs
|
|
123
|
+
const pollForTreeExit = () => {
|
|
124
|
+
if (finishIfTreeClosed()) return
|
|
125
|
+
if (Date.now() < deadline) {
|
|
126
|
+
terminalTimer = setTimeout(pollForTreeExit, 25)
|
|
127
|
+
return
|
|
128
|
+
}
|
|
129
|
+
finish({
|
|
130
|
+
closed: false,
|
|
131
|
+
escalated: true,
|
|
132
|
+
code: leaderCode,
|
|
133
|
+
signal: leaderSignal,
|
|
134
|
+
})
|
|
135
|
+
}
|
|
136
|
+
terminalTimer = setTimeout(pollForTreeExit, 25)
|
|
137
|
+
}, termGraceMs)
|
|
138
|
+
})
|
|
139
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process'
|
|
2
2
|
import { cosBrainDir } from './launch-dir.js'
|
|
3
|
+
import { terminateProviderProcess } from './provider-process-lifecycle.js'
|
|
3
4
|
|
|
4
5
|
export type ProofProvider = 'claude' | 'codex'
|
|
5
6
|
|
|
@@ -21,6 +22,7 @@ interface ProcessResult {
|
|
|
21
22
|
stdout: string
|
|
22
23
|
stderr: string
|
|
23
24
|
timedOut: boolean
|
|
25
|
+
aborted: boolean
|
|
24
26
|
}
|
|
25
27
|
|
|
26
28
|
function runBounded(
|
|
@@ -28,15 +30,20 @@ function runBounded(
|
|
|
28
30
|
args: string[],
|
|
29
31
|
input: string,
|
|
30
32
|
timeoutMs = 120_000,
|
|
33
|
+
signal?: AbortSignal,
|
|
31
34
|
): Promise<ProcessResult> {
|
|
32
35
|
return new Promise((resolvePromise) => {
|
|
36
|
+
if (signal?.aborted) {
|
|
37
|
+
resolvePromise({ code: null, stdout: '', stderr: '', timedOut: false, aborted: true })
|
|
38
|
+
return
|
|
39
|
+
}
|
|
33
40
|
const env = { ...process.env }
|
|
34
41
|
delete env.CLAUDECODE
|
|
35
42
|
const child = spawn(command, args, {
|
|
36
43
|
cwd: cosBrainDir() ?? process.cwd(),
|
|
37
44
|
env,
|
|
38
45
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
39
|
-
detached:
|
|
46
|
+
detached: true,
|
|
40
47
|
})
|
|
41
48
|
let stdout = ''
|
|
42
49
|
let stderr = ''
|
|
@@ -48,15 +55,25 @@ function runBounded(
|
|
|
48
55
|
if (settled) return
|
|
49
56
|
settled = true
|
|
50
57
|
clearTimeout(timer)
|
|
58
|
+
signal?.removeEventListener('abort', abort)
|
|
51
59
|
resolvePromise(result)
|
|
52
60
|
}
|
|
53
61
|
const timer = setTimeout(() => {
|
|
54
|
-
|
|
55
|
-
|
|
62
|
+
void terminateProviderProcess(child, { termGraceMs: 50 }).then(result => {
|
|
63
|
+
if (result.closed) finish({ code: result.code, stdout, stderr, timedOut: true, aborted: false })
|
|
64
|
+
else console.error('[provider-proof] timed-out provider did not close after SIGKILL; retaining request ownership')
|
|
65
|
+
})
|
|
56
66
|
}, timeoutMs)
|
|
57
67
|
timer.unref?.()
|
|
58
|
-
|
|
59
|
-
|
|
68
|
+
const abort = () => {
|
|
69
|
+
void terminateProviderProcess(child).then(result => {
|
|
70
|
+
if (result.closed) finish({ code: result.code, stdout, stderr, timedOut: false, aborted: true })
|
|
71
|
+
else console.error('[provider-proof] canceled provider did not close after SIGKILL; retaining request ownership')
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
child.once('error', err => finish({ code: null, stdout, stderr: err.message, timedOut: false, aborted: false }))
|
|
75
|
+
child.once('close', code => finish({ code, stdout, stderr, timedOut: false, aborted: false }))
|
|
76
|
+
signal?.addEventListener('abort', abort, { once: true })
|
|
60
77
|
child.stdin.on('error', () => { /* close/error is authoritative */ })
|
|
61
78
|
child.stdin.end(input)
|
|
62
79
|
})
|
|
@@ -94,12 +111,13 @@ export function codexProofText(stdout: string): string {
|
|
|
94
111
|
}
|
|
95
112
|
|
|
96
113
|
function safeProofError(result: ProcessResult): string {
|
|
114
|
+
if (result.aborted) return 'provider proof canceled'
|
|
97
115
|
if (result.timedOut) return 'provider proof timed out'
|
|
98
116
|
if (result.code !== 0) return `provider process exited ${result.code ?? 'before launch'}`
|
|
99
117
|
return 'provider returned no valid proof response'
|
|
100
118
|
}
|
|
101
119
|
|
|
102
|
-
async function executeProof(provider: ProofProvider): Promise<ProviderProofResult> {
|
|
120
|
+
async function executeProof(provider: ProofProvider, signal?: AbortSignal): Promise<ProviderProofResult> {
|
|
103
121
|
const started = Date.now()
|
|
104
122
|
const result = provider === 'claude'
|
|
105
123
|
? await runBounded('claude', [
|
|
@@ -110,7 +128,7 @@ async function executeProof(provider: ProofProvider): Promise<ProviderProofResul
|
|
|
110
128
|
'--allowedTools', '',
|
|
111
129
|
'--system-prompt', PROOF_PROMPT,
|
|
112
130
|
PROOF_PROMPT,
|
|
113
|
-
], '')
|
|
131
|
+
], '', 120_000, signal)
|
|
114
132
|
: await runBounded('codex', [
|
|
115
133
|
'exec',
|
|
116
134
|
'--sandbox', 'read-only',
|
|
@@ -119,7 +137,7 @@ async function executeProof(provider: ProofProvider): Promise<ProviderProofResul
|
|
|
119
137
|
'--cd', cosBrainDir() ?? process.cwd(),
|
|
120
138
|
'--ephemeral',
|
|
121
139
|
'-',
|
|
122
|
-
], PROOF_PROMPT)
|
|
140
|
+
], PROOF_PROMPT, 120_000, signal)
|
|
123
141
|
const text = provider === 'claude'
|
|
124
142
|
? claudeProofText(result.stdout)
|
|
125
143
|
: codexProofText(result.stdout)
|
|
@@ -134,12 +152,12 @@ async function executeProof(provider: ProofProvider): Promise<ProviderProofResul
|
|
|
134
152
|
}
|
|
135
153
|
|
|
136
154
|
/** Actual no-tool model turn, cached only after success for this server boot. */
|
|
137
|
-
export async function runProviderProof(provider: ProofProvider): Promise<ProviderProofResult> {
|
|
155
|
+
export async function runProviderProof(provider: ProofProvider, signal?: AbortSignal): Promise<ProviderProofResult> {
|
|
138
156
|
const cached = successCache.get(provider)
|
|
139
157
|
if (cached) return { ...cached, cached: true }
|
|
140
158
|
const existing = inFlight.get(provider)
|
|
141
159
|
if (existing) return existing
|
|
142
|
-
const operation = executeProof(provider).then(result => {
|
|
160
|
+
const operation = executeProof(provider, signal).then(result => {
|
|
143
161
|
if (result.ok) successCache.set(provider, result)
|
|
144
162
|
return result
|
|
145
163
|
}).finally(() => {
|
|
@@ -62,6 +62,12 @@ export class NoSpeechDetectedError extends Error {
|
|
|
62
62
|
// ceiling (anything longer is a dictation, not a query — use meetings instead).
|
|
63
63
|
const HQ_MAX_SECONDS = 60
|
|
64
64
|
|
|
65
|
+
/** Short interactive clips use light enhance (highpass only). Override via env. */
|
|
66
|
+
function hqEnhanceLightMaxSeconds(): number {
|
|
67
|
+
const value = Number.parseInt(process.env.COS_HQ_ENHANCE_LIGHT_MAX_SEC || '15', 10)
|
|
68
|
+
return Number.isFinite(value) && value >= 0 ? value : 15
|
|
69
|
+
}
|
|
70
|
+
|
|
65
71
|
function unavailableAfterLocalFailure(): TranscriptionUnavailableError | null {
|
|
66
72
|
const fallback = getTranscriptionPolicySnapshot()
|
|
67
73
|
if (fallback.openaiFallbackReady) return null
|
|
@@ -158,10 +164,12 @@ export async function transcribeAudioBuffer(
|
|
|
158
164
|
|
|
159
165
|
if (effectiveMode === 'hq') {
|
|
160
166
|
try {
|
|
161
|
-
const
|
|
167
|
+
const lightMax = hqEnhanceLightMaxSeconds()
|
|
168
|
+
const enhanceProfile = audioSeconds < lightMax ? 'light' as const : 'full' as const
|
|
169
|
+
const enhanced = await enhanceAudio(audioBuffer, { profile: enhanceProfile })
|
|
162
170
|
const result = await transcribeHighQuality(enhanced)
|
|
163
171
|
text = result.text
|
|
164
|
-
backend = 'hq-large-v3'
|
|
172
|
+
backend = enhanceProfile === 'light' ? 'hq-large-v3-light' : 'hq-large-v3'
|
|
165
173
|
actualQuality = 'hq'
|
|
166
174
|
} catch (hqErr: any) {
|
|
167
175
|
console.warn(`[transcribe] HQ path failed, falling back to fast: ${hqErr.message}`)
|
|
@@ -6,10 +6,10 @@
|
|
|
6
6
|
// 2. whisper-cli (spawned per request, model loaded from disk) → ~500-700ms
|
|
7
7
|
// 3. OpenAI API (cloud, handled by transcribe.ts) → ~1000-3000ms
|
|
8
8
|
|
|
9
|
-
import { spawn,
|
|
9
|
+
import { spawn, execFile } from 'node:child_process'
|
|
10
10
|
import type { ChildProcess } from 'node:child_process'
|
|
11
11
|
import { writeFileSync, unlinkSync, existsSync, readFileSync } from 'node:fs'
|
|
12
|
-
import { join } from 'node:path'
|
|
12
|
+
import { basename, join } from 'node:path'
|
|
13
13
|
import { homedir } from 'node:os'
|
|
14
14
|
import crypto from 'node:crypto'
|
|
15
15
|
import { getVocabulary, getOwnerName } from './profile.js'
|
|
@@ -137,6 +137,10 @@ function getWhisperPrompt(): string {
|
|
|
137
137
|
// whisper-server runs on this port (started at server boot or by LaunchAgent)
|
|
138
138
|
const WHISPER_SERVER_PORT = 8178
|
|
139
139
|
const WHISPER_SERVER_URL = `http://127.0.0.1:${WHISPER_SERVER_PORT}`
|
|
140
|
+
const PROCESS_PROBE_TIMEOUT_MS = 2_000
|
|
141
|
+
const PROCESS_PROBE_MAX_BUFFER = 1024 * 1024
|
|
142
|
+
const PS_BIN = '/bin/ps'
|
|
143
|
+
const LSOF_BIN = existsSync('/usr/sbin/lsof') ? '/usr/sbin/lsof' : 'lsof'
|
|
140
144
|
|
|
141
145
|
// Track which backends are available
|
|
142
146
|
let cliAvailable = false
|
|
@@ -153,6 +157,10 @@ let serverStarting = false // Initial model load is not a circuit fai
|
|
|
153
157
|
let serverStartPromise: Promise<void> | null = null
|
|
154
158
|
let serverRestartPromise: Promise<WhisperRestartResult> | null = null
|
|
155
159
|
let serverHealthProbe: Promise<boolean> | null = null
|
|
160
|
+
type WhisperStartupState = 'not_started' | 'preflight' | 'loading' | 'ready' | 'unavailable' | 'failed' | 'stopped'
|
|
161
|
+
let serverStartupState: WhisperStartupState = 'not_started'
|
|
162
|
+
let serverLastAttemptAt: string | null = null
|
|
163
|
+
let serverLastError: string | null = null
|
|
156
164
|
|
|
157
165
|
interface ProcessEntry {
|
|
158
166
|
pid: number
|
|
@@ -167,10 +175,34 @@ export interface WhisperRestartResult {
|
|
|
167
175
|
|
|
168
176
|
const sleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms))
|
|
169
177
|
|
|
170
|
-
function
|
|
178
|
+
function boundedError(value: unknown): string {
|
|
179
|
+
const message = value instanceof Error ? value.message : String(value)
|
|
180
|
+
return message.replace(/[\r\n\t]+/g, ' ').slice(0, 240)
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function runProcessProbe(file: string, args: string[]): Promise<string> {
|
|
184
|
+
return new Promise((resolve, reject) => {
|
|
185
|
+
execFile(file, args, {
|
|
186
|
+
encoding: 'utf8',
|
|
187
|
+
timeout: PROCESS_PROBE_TIMEOUT_MS,
|
|
188
|
+
maxBuffer: PROCESS_PROBE_MAX_BUFFER,
|
|
189
|
+
killSignal: 'SIGKILL',
|
|
190
|
+
}, (error, stdout) => {
|
|
191
|
+
if (error) reject(error)
|
|
192
|
+
else resolve(String(stdout))
|
|
193
|
+
})
|
|
194
|
+
})
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
async function listProcesses(): Promise<ProcessEntry[]> {
|
|
171
198
|
// `command=` includes arguments, which lets us distinguish this COS-owned
|
|
172
199
|
// port/model signature from unrelated whisper-server instances.
|
|
173
|
-
|
|
200
|
+
let output: string
|
|
201
|
+
try {
|
|
202
|
+
output = await runProcessProbe(PS_BIN, ['-axww', '-o', 'pid=,ppid=,command='])
|
|
203
|
+
} catch (error) {
|
|
204
|
+
throw new Error(`unable to inspect process table: ${boundedError(error)}`)
|
|
205
|
+
}
|
|
174
206
|
return output.split('\n').flatMap(line => {
|
|
175
207
|
const match = line.match(/^\s*(\d+)\s+(\d+)\s+(.+?)\s*$/)
|
|
176
208
|
if (!match) return []
|
|
@@ -178,24 +210,25 @@ function listProcesses(): ProcessEntry[] {
|
|
|
178
210
|
})
|
|
179
211
|
}
|
|
180
212
|
|
|
181
|
-
function listeningPids(): number[] {
|
|
213
|
+
async function listeningPids(): Promise<number[]> {
|
|
182
214
|
try {
|
|
183
|
-
const output =
|
|
184
|
-
|
|
215
|
+
const output = await runProcessProbe(
|
|
216
|
+
LSOF_BIN,
|
|
185
217
|
['-nP', `-iTCP:${WHISPER_SERVER_PORT}`, '-sTCP:LISTEN', '-t'],
|
|
186
|
-
{ encoding: 'utf8' },
|
|
187
218
|
)
|
|
188
219
|
return output.split(/\s+/).map(Number).filter(pid => Number.isInteger(pid) && pid > 0)
|
|
189
220
|
} catch (err: any) {
|
|
190
221
|
// lsof uses exit 1 for "no matches". Anything else means we could not
|
|
191
222
|
// prove the port state, so startup must fail closed.
|
|
192
|
-
if (err?.status === 1) return []
|
|
193
|
-
throw new Error(`unable to inspect whisper-server port ${WHISPER_SERVER_PORT}: ${err
|
|
223
|
+
if (err?.code === 1 || err?.status === 1) return []
|
|
224
|
+
throw new Error(`unable to inspect whisper-server port ${WHISPER_SERVER_PORT}: ${boundedError(err)}`)
|
|
194
225
|
}
|
|
195
226
|
}
|
|
196
227
|
|
|
197
228
|
function isCosWhisperServerCommand(command: string): boolean {
|
|
198
|
-
const
|
|
229
|
+
const firstToken = command.trim().match(/^(?:"([^"]+)"|'([^']+)'|(\S+))/)
|
|
230
|
+
const executablePath = firstToken?.[1] ?? firstToken?.[2] ?? firstToken?.[3] ?? ''
|
|
231
|
+
const executable = basename(executablePath) === 'whisper-server'
|
|
199
232
|
const configuredPort = new RegExp(`(?:^|\\s)--port(?:=|\\s+)${WHISPER_SERVER_PORT}(?:\\s|$)`).test(command)
|
|
200
233
|
return executable && configuredPort && command.includes(MODEL_PATH)
|
|
201
234
|
}
|
|
@@ -249,7 +282,7 @@ async function killAndReapWhisperProcesses(): Promise<void> {
|
|
|
249
282
|
serverAvailable = false
|
|
250
283
|
|
|
251
284
|
for (let round = 0; round < 3; round++) {
|
|
252
|
-
const processes = listProcesses()
|
|
285
|
+
const processes = await listProcesses()
|
|
253
286
|
const ownedPids = [...ownedServerChildren]
|
|
254
287
|
.map(child => child.pid)
|
|
255
288
|
.filter((pid): pid is number => typeof pid === 'number')
|
|
@@ -288,7 +321,7 @@ async function killAndReapWhisperProcesses(): Promise<void> {
|
|
|
288
321
|
await sleep(50)
|
|
289
322
|
}
|
|
290
323
|
|
|
291
|
-
const remaining = listProcesses().filter(entry => isCosWhisperServerCommand(entry.command))
|
|
324
|
+
const remaining = (await listProcesses()).filter(entry => isCosWhisperServerCommand(entry.command))
|
|
292
325
|
if (remaining.length > 0) {
|
|
293
326
|
throw new Error(`stale whisper-server process(es) remain: ${remaining.map(entry => entry.pid).join(', ')}`)
|
|
294
327
|
}
|
|
@@ -298,11 +331,11 @@ async function killAndReapWhisperProcesses(): Promise<void> {
|
|
|
298
331
|
|
|
299
332
|
async function proveWhisperPortClear(): Promise<void> {
|
|
300
333
|
for (let attempt = 0; attempt < 20; attempt++) {
|
|
301
|
-
const pids = listeningPids()
|
|
334
|
+
const pids = await listeningPids()
|
|
302
335
|
if (pids.length === 0) return
|
|
303
336
|
if (attempt < 19) await sleep(250)
|
|
304
337
|
}
|
|
305
|
-
const pids = listeningPids()
|
|
338
|
+
const pids = await listeningPids()
|
|
306
339
|
throw new Error(
|
|
307
340
|
`whisper-server port ${WHISPER_SERVER_PORT} remains occupied${pids.length ? ` by PID(s) ${pids.join(', ')}` : ''}`,
|
|
308
341
|
)
|
|
@@ -334,10 +367,17 @@ export async function startWhisperServer(): Promise<void> {
|
|
|
334
367
|
if (serverAvailable && serverProcess) return
|
|
335
368
|
|
|
336
369
|
serverStarting = true
|
|
370
|
+
serverStartupState = 'preflight'
|
|
371
|
+
serverLastAttemptAt = new Date().toISOString()
|
|
372
|
+
serverLastError = null
|
|
337
373
|
const operation = startWhisperServerAttempt()
|
|
338
374
|
serverStartPromise = operation
|
|
339
375
|
try {
|
|
340
376
|
await operation
|
|
377
|
+
} catch (error) {
|
|
378
|
+
serverStartupState = 'failed'
|
|
379
|
+
serverLastError = boundedError(error)
|
|
380
|
+
throw error
|
|
341
381
|
} finally {
|
|
342
382
|
if (serverStartPromise === operation) serverStartPromise = null
|
|
343
383
|
serverStarting = false
|
|
@@ -346,6 +386,8 @@ export async function startWhisperServer(): Promise<void> {
|
|
|
346
386
|
|
|
347
387
|
async function startWhisperServerAttempt(preflightCompleted = false): Promise<void> {
|
|
348
388
|
if (!existsSync(WHISPER_SERVER) || !existsSync(MODEL_PATH)) {
|
|
389
|
+
serverStartupState = 'unavailable'
|
|
390
|
+
serverLastError = 'whisper-server or model not found'
|
|
349
391
|
console.log('[whisper-local] whisper-server or model not found — using CLI fallback')
|
|
350
392
|
return
|
|
351
393
|
}
|
|
@@ -353,9 +395,12 @@ async function startWhisperServerAttempt(preflightCompleted = false): Promise<vo
|
|
|
353
395
|
// The API process is the sole Whisper owner. Never adopt an untracked daemon:
|
|
354
396
|
// reap stale trees, then prove the fixed local port is free before spawning.
|
|
355
397
|
if (!preflightCompleted) {
|
|
398
|
+
console.log('[whisper-local] preflight: inspecting owned processes and port 8178')
|
|
399
|
+
serverStartupState = 'preflight'
|
|
356
400
|
await killAndReapWhisperProcesses()
|
|
357
401
|
await proveWhisperPortClear()
|
|
358
402
|
}
|
|
403
|
+
serverStartupState = 'loading'
|
|
359
404
|
|
|
360
405
|
// Assemble startup args. VAD only attaches if the ggml model is actually on
|
|
361
406
|
// disk — missing-file is logged, not fatal (server still boots without VAD).
|
|
@@ -402,6 +447,8 @@ async function startWhisperServerAttempt(preflightCompleted = false): Promise<vo
|
|
|
402
447
|
if (serverProcess !== child) return
|
|
403
448
|
serverAvailable = false
|
|
404
449
|
serverProcess = null
|
|
450
|
+
serverStartupState = 'failed'
|
|
451
|
+
serverLastError = `whisper-server exited${code == null ? '' : ` with code ${code}`}`
|
|
405
452
|
if (code !== null && code !== 0) {
|
|
406
453
|
console.warn(`[whisper-local] whisper-server exited with code ${code}`)
|
|
407
454
|
}
|
|
@@ -422,6 +469,8 @@ async function startWhisperServerAttempt(preflightCompleted = false): Promise<vo
|
|
|
422
469
|
const res = await fetch(`${WHISPER_SERVER_URL}/health`, { signal: AbortSignal.timeout(1000) })
|
|
423
470
|
if (res.ok) {
|
|
424
471
|
serverAvailable = true
|
|
472
|
+
serverStartupState = 'ready'
|
|
473
|
+
serverLastError = null
|
|
425
474
|
const loadTime = ((Date.now() - startTime) / 1000).toFixed(1)
|
|
426
475
|
console.log(`[whisper-local] whisper-server ready on port ${WHISPER_SERVER_PORT} (loaded in ${loadTime}s)`)
|
|
427
476
|
return
|
|
@@ -444,6 +493,8 @@ async function startWhisperServerAttempt(preflightCompleted = false): Promise<vo
|
|
|
444
493
|
console.error(`[whisper-local] ${failure} — reaping child and keeping local backend unavailable`)
|
|
445
494
|
await killAndReapWhisperProcesses()
|
|
446
495
|
await proveWhisperPortClear()
|
|
496
|
+
serverStartupState = 'failed'
|
|
497
|
+
serverLastError = failure
|
|
447
498
|
throw new Error(failure)
|
|
448
499
|
}
|
|
449
500
|
|
|
@@ -461,6 +512,7 @@ export function stopWhisperServer(): void {
|
|
|
461
512
|
try { proc.kill('SIGKILL') } catch { /* already exited */ }
|
|
462
513
|
}
|
|
463
514
|
ownedHqChildren.clear()
|
|
515
|
+
serverStartupState = 'stopped'
|
|
464
516
|
}
|
|
465
517
|
|
|
466
518
|
export function isWhisperLocalAvailable(): boolean {
|
|
@@ -480,17 +532,25 @@ export function getWhisperBackend(): 'server' | 'cli' | 'none' {
|
|
|
480
532
|
/** Detailed health status for diagnostics (/api/health) */
|
|
481
533
|
export function getWhisperHealth(): {
|
|
482
534
|
server: boolean
|
|
535
|
+
serverConfigured: boolean
|
|
483
536
|
cli: boolean
|
|
484
537
|
consecutiveFailures: number
|
|
485
538
|
restarting: boolean
|
|
486
539
|
circuitOpen: boolean
|
|
540
|
+
startupState: WhisperStartupState
|
|
541
|
+
lastAttemptAt: string | null
|
|
542
|
+
lastError: string | null
|
|
487
543
|
} {
|
|
488
544
|
return {
|
|
489
545
|
server: serverAvailable,
|
|
546
|
+
serverConfigured: existsSync(WHISPER_SERVER) && existsSync(MODEL_PATH),
|
|
490
547
|
cli: cliAvailable,
|
|
491
548
|
consecutiveFailures: serverConsecutiveFailures,
|
|
492
549
|
restarting: serverRestarting || serverStarting,
|
|
493
550
|
circuitOpen: serverConsecutiveFailures >= SERVER_FAILURE_THRESHOLD,
|
|
551
|
+
startupState: serverStartupState,
|
|
552
|
+
lastAttemptAt: serverLastAttemptAt,
|
|
553
|
+
lastError: serverLastError,
|
|
494
554
|
}
|
|
495
555
|
}
|
|
496
556
|
|
|
@@ -604,14 +664,22 @@ export async function transcribeHighQuality(
|
|
|
604
664
|
|
|
605
665
|
const text = await new Promise<string>((resolve, reject) => {
|
|
606
666
|
const isolateBatchFromLiveMetal = opts.priority === 'batch'
|
|
667
|
+
// Interactive HQ: narrower beam (default 2) for latency. Meeting batch keeps 5.
|
|
668
|
+
// Override: COS_HQ_BEAM_INTERACTIVE=N
|
|
669
|
+
const interactiveBeamRaw = Number.parseInt(process.env.COS_HQ_BEAM_INTERACTIVE || '2', 10)
|
|
670
|
+
const interactiveBeam = Number.isFinite(interactiveBeamRaw) && interactiveBeamRaw >= 1
|
|
671
|
+
? Math.min(interactiveBeamRaw, 5)
|
|
672
|
+
: 2
|
|
673
|
+
const beam = isolateBatchFromLiveMetal ? 5 : interactiveBeam
|
|
674
|
+
const bestOf = beam
|
|
607
675
|
const args = [
|
|
608
676
|
'-m', modelPath,
|
|
609
677
|
'-f', tmpWav,
|
|
610
678
|
'-t', isolateBatchFromLiveMetal ? '8' : '16',
|
|
611
679
|
'-l', 'en',
|
|
612
680
|
...(isolateBatchFromLiveMetal ? ['-ng'] : ['-fa']),
|
|
613
|
-
'-bs',
|
|
614
|
-
'-bo',
|
|
681
|
+
'-bs', String(beam),
|
|
682
|
+
'-bo', String(bestOf),
|
|
615
683
|
'--no-timestamps',
|
|
616
684
|
'-np',
|
|
617
685
|
'--prompt', buildPrompt(context),
|
|
@@ -966,6 +1034,9 @@ export async function restartWhisperServer(): Promise<WhisperRestartResult> {
|
|
|
966
1034
|
if (serverRestartPromise) return serverRestartPromise
|
|
967
1035
|
|
|
968
1036
|
serverRestarting = true
|
|
1037
|
+
serverStartupState = 'preflight'
|
|
1038
|
+
serverLastAttemptAt = new Date().toISOString()
|
|
1039
|
+
serverLastError = null
|
|
969
1040
|
const priorStart = serverStartPromise
|
|
970
1041
|
const operation = (async (): Promise<WhisperRestartResult> => {
|
|
971
1042
|
try {
|
|
@@ -984,6 +1055,8 @@ export async function restartWhisperServer(): Promise<WhisperRestartResult> {
|
|
|
984
1055
|
throw new Error('whisper-server did not become healthy')
|
|
985
1056
|
}
|
|
986
1057
|
serverConsecutiveFailures = 0
|
|
1058
|
+
serverStartupState = 'ready'
|
|
1059
|
+
serverLastError = null
|
|
987
1060
|
console.log('[whisper-local] Server restarted successfully — circuit breaker CLOSED')
|
|
988
1061
|
return { status: 'recovered' }
|
|
989
1062
|
} catch (err: any) {
|
|
@@ -992,6 +1065,8 @@ export async function restartWhisperServer(): Promise<WhisperRestartResult> {
|
|
|
992
1065
|
// breaker cycle, and the next three failed calls may request one new cycle.
|
|
993
1066
|
serverConsecutiveFailures = 0
|
|
994
1067
|
const message = err?.message ?? String(err)
|
|
1068
|
+
serverStartupState = 'failed'
|
|
1069
|
+
serverLastError = boundedError(message)
|
|
995
1070
|
console.error(`[whisper-local] Server restart error: ${message} — will retry after next 3 failures`)
|
|
996
1071
|
return { status: 'failed', error: message }
|
|
997
1072
|
}
|
package/server/routes/health.ts
CHANGED
|
@@ -180,6 +180,23 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
180
180
|
// Whisper-server health + cloud budget — exposed so glasses + dashboards can
|
|
181
181
|
// see whether we're at risk of falling to cloud and how much budget remains.
|
|
182
182
|
const whisper_health = getWhisperHealth()
|
|
183
|
+
const whisperReadiness = !whisper_health.serverConfigured
|
|
184
|
+
? 'not_configured'
|
|
185
|
+
: whisper_health.server
|
|
186
|
+
? 'ready'
|
|
187
|
+
: whisper_health.startupState === 'preflight' || whisper_health.startupState === 'loading'
|
|
188
|
+
? 'starting'
|
|
189
|
+
: 'degraded'
|
|
190
|
+
const readiness = {
|
|
191
|
+
// /api/health remains a liveness endpoint and intentionally returns HTTP
|
|
192
|
+
// 200 while the server can answer. This separate field prevents an HTTP-
|
|
193
|
+
// green response from hiding a configured local subsystem failure.
|
|
194
|
+
status: whisperReadiness === 'degraded' ? 'degraded' : 'ready',
|
|
195
|
+
admissions: maintenance.admissionsOpen ? 'open' : 'maintenance',
|
|
196
|
+
whisper: whisperReadiness,
|
|
197
|
+
whisperError: whisper_health.lastError,
|
|
198
|
+
localTts: tts_local.ready ? 'ready' : 'unavailable',
|
|
199
|
+
}
|
|
183
200
|
const openai_whisper_budget = getOpenAIWhisperBudgetState()
|
|
184
201
|
const codex_models = getCodexModelCatalogSnapshot()
|
|
185
202
|
res.json({
|
|
@@ -190,6 +207,7 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
190
207
|
generation_id: getServerGenerationId(),
|
|
191
208
|
features,
|
|
192
209
|
voice,
|
|
210
|
+
readiness,
|
|
193
211
|
whisper_health,
|
|
194
212
|
openai_whisper_budget,
|
|
195
213
|
tts_local,
|