@gotcos/glasses-server 6.17.0 → 6.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +39 -0
- package/CHANGELOG.md +71 -0
- package/managed-runtime-contract.json +8 -1
- package/package.json +2 -2
- package/server/index.ts +12 -0
- package/server/lib/g2-enrichment-runner.ts +179 -0
- package/server/lib/g2-ops-handoff.ts +136 -0
- package/server/lib/live-cues-capability.ts +83 -0
- package/server/lib/live-cues-cursor.ts +167 -0
- package/server/lib/live-cues-engine.ts +461 -0
- package/server/lib/live-cues-memory.ts +202 -0
- package/server/lib/live-cues-prompt.ts +98 -0
- package/server/lib/maintenance-lifecycle.ts +6 -0
- package/server/lib/meeting-batch-transcribe.ts +21 -1
- package/server/lib/meeting-store.ts +10 -1
- package/server/lib/whisper-local.ts +49 -6
- package/server/lib/whisper-metal-gate.ts +207 -0
- package/server/routes/health.ts +10 -0
- package/server/routes/live-cues.ts +58 -0
- package/server/routes/meeting.ts +43 -12
- package/server/routes/transcribe-stream.ts +54 -7
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
// GPU admission control for post-meeting batch HQ transcription.
|
|
2
|
+
//
|
|
3
|
+
// THE PROBLEM: Miles goes meeting -> meeting. Meeting A's post-save HQ polish
|
|
4
|
+
// (whisper-cli large-v3) often overlaps Meeting B's live ASR. Both want Metal.
|
|
5
|
+
// 6.14.1 fixed the clash bluntly by pinning batch to CPU forever, which taxes
|
|
6
|
+
// every idle polish to protect the overlap case.
|
|
7
|
+
//
|
|
8
|
+
// THE INVARIANT: never Metal+Metal. Live always wins admission. Batch still
|
|
9
|
+
// progresses on busy days, just on CPU. Metal is used only when the operator
|
|
10
|
+
// has opted in AND nothing live is contending.
|
|
11
|
+
//
|
|
12
|
+
// NO CIRCULAR IMPORT: transcribe-stream REGISTERS its live-activity probe here;
|
|
13
|
+
// this module never imports the stream, and whisper-local imports only this.
|
|
14
|
+
|
|
15
|
+
import type { ChildProcess } from 'node:child_process'
|
|
16
|
+
import { maintenanceLifecycle } from './maintenance-lifecycle.js'
|
|
17
|
+
|
|
18
|
+
/** Recent-activity window. A session counts as live only if it has been active
|
|
19
|
+
* inside this window. Anything colder is treated as an ORPHAN and must NOT pin
|
|
20
|
+
* batch to CPU — proven necessary: an active-sessions file sat untouched for
|
|
21
|
+
* 3+ hours on 2026-07-27, and an "any session in the map" rule would have
|
|
22
|
+
* pinned batch to CPU until a server restart, silently, forever. */
|
|
23
|
+
export const LIVE_ACTIVITY_WINDOW_MS = 180_000
|
|
24
|
+
|
|
25
|
+
/** Maintenance kinds that own the same Metal family as batch HQ. */
|
|
26
|
+
const METAL_FAMILY_WORK_KINDS = [
|
|
27
|
+
'recording_chunk',
|
|
28
|
+
'one_shot_transcription',
|
|
29
|
+
'prompt_draft_warm',
|
|
30
|
+
'prompt_draft_finalize',
|
|
31
|
+
] as const
|
|
32
|
+
|
|
33
|
+
export type BatchDevice = 'metal' | 'cpu'
|
|
34
|
+
|
|
35
|
+
export type ContentionReason =
|
|
36
|
+
| 'session_recent'
|
|
37
|
+
| 'recording_chunk'
|
|
38
|
+
| 'one_shot_transcription'
|
|
39
|
+
| 'prompt_draft_warm'
|
|
40
|
+
| 'prompt_draft_finalize'
|
|
41
|
+
| 'metal_batch_in_flight'
|
|
42
|
+
|
|
43
|
+
export type DeviceReason =
|
|
44
|
+
| 'force_cpu'
|
|
45
|
+
| 'metal_opt_out'
|
|
46
|
+
| ContentionReason
|
|
47
|
+
| 'idle'
|
|
48
|
+
|
|
49
|
+
export interface BatchDeviceDecision {
|
|
50
|
+
device: BatchDevice
|
|
51
|
+
reason: DeviceReason
|
|
52
|
+
metalEnabled: boolean
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// ── Env ──────────────────────────────────────────────────────────────────────
|
|
56
|
+
|
|
57
|
+
/** Blunt rollback. Wins over everything, and keeps working after the default
|
|
58
|
+
* eventually flips to Metal-on. */
|
|
59
|
+
export function batchHqForceCpu(): boolean {
|
|
60
|
+
return process.env.COS_BATCH_HQ_FORCE_CPU === '1'
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Metal batch is OPT-IN (3B). Default stays always-CPU — today's proven
|
|
64
|
+
* behavior — until a meeting-to-meeting smoke passes. Read live, not at module
|
|
65
|
+
* load, so tests and Control-updated env are visible. */
|
|
66
|
+
export function batchHqMetalEnabled(): boolean {
|
|
67
|
+
return !batchHqForceCpu() && process.env.COS_BATCH_HQ_METAL === '1'
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// ── Injected live-activity probe ─────────────────────────────────────────────
|
|
71
|
+
|
|
72
|
+
type LiveActivityProbe = () => number | null
|
|
73
|
+
|
|
74
|
+
let liveActivityProbe: LiveActivityProbe | null = null
|
|
75
|
+
|
|
76
|
+
/** transcribe-stream calls this at module load. Returns the most recent
|
|
77
|
+
* lastActivityAt across in-memory sessions, or null when there are none. */
|
|
78
|
+
export function registerLiveActivityProbe(probe: LiveActivityProbe): void {
|
|
79
|
+
liveActivityProbe = probe
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Test seam — clears the probe and any tracked children. */
|
|
83
|
+
export function resetMetalGateForTests(): void {
|
|
84
|
+
liveActivityProbe = null
|
|
85
|
+
metalChildren.clear()
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function recentSessionActivity(now: number): boolean {
|
|
89
|
+
if (!liveActivityProbe) return false
|
|
90
|
+
let last: number | null = null
|
|
91
|
+
try {
|
|
92
|
+
last = liveActivityProbe()
|
|
93
|
+
} catch {
|
|
94
|
+
// A probe failure must fail SAFE (assume contended): a wrong "idle" starts
|
|
95
|
+
// a Metal batch against a live meeting, which is the one thing this whole
|
|
96
|
+
// module exists to prevent. A wrong "busy" only costs batch speed.
|
|
97
|
+
return true
|
|
98
|
+
}
|
|
99
|
+
if (last == null || !Number.isFinite(last)) return false
|
|
100
|
+
return now - last < LIVE_ACTIVITY_WINDOW_MS
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function activeMetalFamilyWork(): ContentionReason | null {
|
|
104
|
+
let byKind: Record<string, number>
|
|
105
|
+
try {
|
|
106
|
+
byKind = maintenanceLifecycle.snapshot().activeByKind as Record<string, number>
|
|
107
|
+
} catch {
|
|
108
|
+
return 'recording_chunk' // fail safe, same reasoning as above
|
|
109
|
+
}
|
|
110
|
+
for (const kind of METAL_FAMILY_WORK_KINDS) {
|
|
111
|
+
if ((byKind[kind] ?? 0) > 0) return kind
|
|
112
|
+
}
|
|
113
|
+
return null
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Is something live currently entitled to Metal? */
|
|
117
|
+
export function isLiveMetalContended(now: number = Date.now()): { contended: boolean; reason: ContentionReason | null } {
|
|
118
|
+
const work = activeMetalFamilyWork()
|
|
119
|
+
if (work) return { contended: true, reason: work }
|
|
120
|
+
if (recentSessionActivity(now)) return { contended: true, reason: 'session_recent' }
|
|
121
|
+
if (metalChildren.size > 0) return { contended: true, reason: 'metal_batch_in_flight' }
|
|
122
|
+
return { contended: false, reason: null }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Device for the NEXT batch segment. Re-evaluated per segment so a meeting
|
|
126
|
+
* that starts mid-batch moves subsequent segments to CPU without a preempt. */
|
|
127
|
+
export function chooseBatchDevice(now: number = Date.now()): BatchDeviceDecision {
|
|
128
|
+
if (batchHqForceCpu()) return { device: 'cpu', reason: 'force_cpu', metalEnabled: false }
|
|
129
|
+
const metalEnabled = batchHqMetalEnabled()
|
|
130
|
+
if (!metalEnabled) return { device: 'cpu', reason: 'metal_opt_out', metalEnabled: false }
|
|
131
|
+
const { contended, reason } = isLiveMetalContended(now)
|
|
132
|
+
if (contended) return { device: 'cpu', reason: reason ?? 'session_recent', metalEnabled: true }
|
|
133
|
+
return { device: 'metal', reason: 'idle', metalEnabled: true }
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// ── Metal child registry + preempt ───────────────────────────────────────────
|
|
137
|
+
|
|
138
|
+
interface MetalChildEntry {
|
|
139
|
+
proc: ChildProcess
|
|
140
|
+
/** Set by the owner so the close handler can DISCARD partial output rather
|
|
141
|
+
* than resolving a truncated transcript into a saved meeting. */
|
|
142
|
+
markPreempted: (reason: string) => void
|
|
143
|
+
escalate?: ReturnType<typeof setTimeout>
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const metalChildren = new Map<ChildProcess, MetalChildEntry>()
|
|
147
|
+
|
|
148
|
+
/** whisper-local registers ONLY Metal-device HQ children. CPU children are
|
|
149
|
+
* deliberately absent: they do not contend for the GPU, so preempting them
|
|
150
|
+
* would slow batch for no benefit. */
|
|
151
|
+
export function registerMetalBatchChild(proc: ChildProcess, markPreempted: (reason: string) => void): void {
|
|
152
|
+
metalChildren.set(proc, { proc, markPreempted })
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function unregisterMetalBatchChild(proc: ChildProcess): void {
|
|
156
|
+
const entry = metalChildren.get(proc)
|
|
157
|
+
if (entry?.escalate) clearTimeout(entry.escalate)
|
|
158
|
+
metalChildren.delete(proc)
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export function metalBatchInFlight(): boolean {
|
|
162
|
+
return metalChildren.size > 0
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Live work needs the GPU: kill every in-flight Metal batch child.
|
|
167
|
+
*
|
|
168
|
+
* Idempotent — a child already preempted is skipped, so the create-path hook
|
|
169
|
+
* and the recording_chunk backstop can both fire without double-killing.
|
|
170
|
+
* Returns how many children were preempted (0 is the overwhelmingly common
|
|
171
|
+
* case: no Metal batch running, or Metal not opted in at all).
|
|
172
|
+
*/
|
|
173
|
+
export function preemptMetalBatchForLive(reason: string): number {
|
|
174
|
+
if (metalChildren.size === 0) return 0
|
|
175
|
+
let preempted = 0
|
|
176
|
+
for (const entry of [...metalChildren.values()]) {
|
|
177
|
+
if (entry.escalate) continue // already preempted, escalation pending
|
|
178
|
+
preempted++
|
|
179
|
+
// Mark FIRST: the close handler must see the preempt flag before the signal
|
|
180
|
+
// lands, or it could treat a truncated run as a clean exit.
|
|
181
|
+
try { entry.markPreempted(reason) } catch { /* never block the kill */ }
|
|
182
|
+
try { entry.proc.kill('SIGTERM') } catch { /* already exited */ }
|
|
183
|
+
entry.escalate = setTimeout(() => {
|
|
184
|
+
try { entry.proc.kill('SIGKILL') } catch { /* already exited */ }
|
|
185
|
+
}, 2_000)
|
|
186
|
+
entry.escalate.unref?.()
|
|
187
|
+
}
|
|
188
|
+
if (preempted > 0) {
|
|
189
|
+
console.log(`[metal-gate] Preempted ${preempted} Metal batch child(ren) for live work (${reason})`)
|
|
190
|
+
}
|
|
191
|
+
return preempted
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Distinguishable error so the batch pipeline can retry the SAME segment on
|
|
195
|
+
* CPU instead of treating it as a generic transcription failure. */
|
|
196
|
+
export class MetalBatchPreemptedError extends Error {
|
|
197
|
+
readonly preempted = true
|
|
198
|
+
constructor(readonly preemptReason: string) {
|
|
199
|
+
super(`Metal batch preempted by live work (${preemptReason})`)
|
|
200
|
+
this.name = 'MetalBatchPreemptedError'
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export function isMetalBatchPreempted(error: unknown): error is MetalBatchPreemptedError {
|
|
205
|
+
return error instanceof MetalBatchPreemptedError
|
|
206
|
+
|| (typeof error === 'object' && error !== null && (error as { preempted?: boolean }).preempted === true)
|
|
207
|
+
}
|
package/server/routes/health.ts
CHANGED
|
@@ -36,6 +36,7 @@ import { managedRuntimeCapability, managedServerVersion } from '../lib/managed-r
|
|
|
36
36
|
import { getServerGenerationId } from '../lib/managed-runtime.js'
|
|
37
37
|
import { maintenanceLifecycle } from '../lib/maintenance-lifecycle.js'
|
|
38
38
|
import { getLocalTtsHealth, refreshLocalTtsHealth } from '../lib/tts-local.js'
|
|
39
|
+
import { liveCuesCapability } from '../lib/live-cues-capability.js'
|
|
39
40
|
|
|
40
41
|
export const healthRouter = Router()
|
|
41
42
|
|
|
@@ -203,6 +204,9 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
203
204
|
const recovery = managedRuntimeCapability()
|
|
204
205
|
const maintenance = maintenanceLifecycle.snapshot()
|
|
205
206
|
const tts_local = getLocalTtsHealth()
|
|
207
|
+
// Computed once per request; the same value feeds features.liveCues and
|
|
208
|
+
// capabilities.liveCues so the two surfaces can never disagree.
|
|
209
|
+
const liveCues = liveCuesCapability()
|
|
206
210
|
const features = {
|
|
207
211
|
claude: claudeAvailable,
|
|
208
212
|
codex: codexAvailable,
|
|
@@ -219,6 +223,7 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
219
223
|
durableQueryJobsProtocol: durableJobs.protocolVersion,
|
|
220
224
|
localFirstMeetings: localFirstMeetings !== null,
|
|
221
225
|
transcriptionPolicy: transcription.mode,
|
|
226
|
+
liveCues: liveCues.available,
|
|
222
227
|
}
|
|
223
228
|
const voice = {
|
|
224
229
|
available: keyStatus.hasKey || tts_local.ready,
|
|
@@ -277,6 +282,7 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
277
282
|
carriedAcrossBoot: maintenance.operation?.carriedAcrossBoot ?? false,
|
|
278
283
|
},
|
|
279
284
|
cliDebug: CLI_DEBUG_CAPABILITY,
|
|
285
|
+
liveCues,
|
|
280
286
|
...(localFirstMeetings ? { localFirstMeetings } : {}),
|
|
281
287
|
},
|
|
282
288
|
// /api/health is intentionally unauthenticated for setup diagnostics.
|
|
@@ -319,6 +325,10 @@ healthRouter.get('/models', async (req, res) => {
|
|
|
319
325
|
transcription: { ...transcription, hq: transcriptionHq },
|
|
320
326
|
cliDebug: CLI_DEBUG_CAPABILITY,
|
|
321
327
|
recovery: managedRuntimeCapability(),
|
|
328
|
+
// Same helper as /api/health — the companion's 15s liveness poll reads
|
|
329
|
+
// THIS surface, so a value present only on /api/health leaves the
|
|
330
|
+
// live-cues indicator blind.
|
|
331
|
+
liveCues: liveCuesCapability(),
|
|
322
332
|
...(localFirstMeetings ? { localFirstMeetings } : {}),
|
|
323
333
|
},
|
|
324
334
|
})
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// Live Cues routes. Authenticated by the global /api X-Cos-Token middleware
|
|
2
|
+
// (index.ts:148) like every other /api route — no route-local auth needed.
|
|
3
|
+
//
|
|
4
|
+
// POST /live-cues/start { sessionId } -> { armed, sessionId, pipelinesUsed }
|
|
5
|
+
// POST /live-cues/stop { sessionId } -> { nudgesGenerated, nudges }
|
|
6
|
+
// GET /live-cues/status -> engine snapshot + capability
|
|
7
|
+
|
|
8
|
+
import { Router } from 'express'
|
|
9
|
+
import { errMsg } from '../lib/utils.js'
|
|
10
|
+
import {
|
|
11
|
+
armLiveCues,
|
|
12
|
+
disarmLiveCues,
|
|
13
|
+
getLiveCuesStatus,
|
|
14
|
+
LiveCuesArmError,
|
|
15
|
+
} from '../lib/live-cues-engine.js'
|
|
16
|
+
import { liveCuesCapability } from '../lib/live-cues-capability.js'
|
|
17
|
+
|
|
18
|
+
export const liveCuesRouter = Router()
|
|
19
|
+
|
|
20
|
+
liveCuesRouter.post('/live-cues/start', async (req, res) => {
|
|
21
|
+
const sessionId = typeof (req.body as { sessionId?: unknown })?.sessionId === 'string'
|
|
22
|
+
? String((req.body as { sessionId: string }).sessionId).trim()
|
|
23
|
+
: ''
|
|
24
|
+
// A start with no sessionId is refused, never silently armed on a global
|
|
25
|
+
// counter — the per-meeting cap is only real when it is session-scoped.
|
|
26
|
+
if (!sessionId || !/^[A-Za-z0-9:_-]{3,96}$/.test(sessionId)) {
|
|
27
|
+
return res.status(400).json({ error: 'sessionId required', reason: 'missing_session_id' })
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
const result = await armLiveCues(sessionId)
|
|
31
|
+
return res.json(result)
|
|
32
|
+
} catch (error) {
|
|
33
|
+
if (error instanceof LiveCuesArmError) {
|
|
34
|
+
return res.status(error.status).json({ error: error.message, reason: error.code })
|
|
35
|
+
}
|
|
36
|
+
return res.status(500).json({ error: errMsg(error) })
|
|
37
|
+
}
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
liveCuesRouter.post('/live-cues/stop', (req, res) => {
|
|
41
|
+
const sessionId = typeof (req.body as { sessionId?: unknown })?.sessionId === 'string'
|
|
42
|
+
? String((req.body as { sessionId: string }).sessionId).trim()
|
|
43
|
+
: ''
|
|
44
|
+
if (!sessionId) {
|
|
45
|
+
return res.status(400).json({ error: 'sessionId required', reason: 'missing_session_id' })
|
|
46
|
+
}
|
|
47
|
+
// Shape matches the existing client stop handler: it reads nudgesGenerated
|
|
48
|
+
// for its log line and hydrates meetingNudgeHistory from nudges when SSE
|
|
49
|
+
// missed events (the 200-event replay buffer is flooded by transcript_chunk).
|
|
50
|
+
return res.json(disarmLiveCues(sessionId))
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
liveCuesRouter.get('/live-cues/status', (_req, res) => {
|
|
54
|
+
res.json({
|
|
55
|
+
capability: liveCuesCapability(),
|
|
56
|
+
sessions: getLiveCuesStatus(),
|
|
57
|
+
})
|
|
58
|
+
})
|
package/server/routes/meeting.ts
CHANGED
|
@@ -41,6 +41,13 @@ import {
|
|
|
41
41
|
} from './transcribe-stream.js'
|
|
42
42
|
import { getServerInstanceId } from '../lib/server-instance-id.js'
|
|
43
43
|
import { acquireMaintenanceWork, type MaintenanceWorkLease } from '../lib/maintenance-lifecycle.js'
|
|
44
|
+
import { handoffMeetingToOperations } from '../lib/g2-ops-handoff.js'
|
|
45
|
+
|
|
46
|
+
function cosOpsPipelineConfigured(): boolean {
|
|
47
|
+
// Read env live (not the module-load COS_SCRIPTS_DIR const) so unit tests that
|
|
48
|
+
// clear ops env stay standalone, and Control-updated env is visible.
|
|
49
|
+
return Boolean(process.env.COS_SCRIPTS_DIR?.trim())
|
|
50
|
+
}
|
|
44
51
|
|
|
45
52
|
interface MeetingSessionSource {
|
|
46
53
|
getTranscript(sessionId: string): string | null
|
|
@@ -313,24 +320,48 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
|
|
|
313
320
|
allowDuringDrain: true,
|
|
314
321
|
phase: 'queued',
|
|
315
322
|
})
|
|
316
|
-
const task = Promise.resolve().then(() => {
|
|
323
|
+
const task = Promise.resolve().then(async () => {
|
|
317
324
|
batchLease.setPhase('active')
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
325
|
+
try {
|
|
326
|
+
await finalizeBatch({
|
|
327
|
+
audioDir: pendingAudioDir,
|
|
328
|
+
entries: chunkEntries.map(entry => ({ ...entry, chunk: { ...entry.chunk } })),
|
|
329
|
+
streamingWordCount: countWords(transcript),
|
|
330
|
+
meetingPath: saved.filepath,
|
|
331
|
+
sidecarPath: saved.sidecarPath,
|
|
332
|
+
runBatch,
|
|
333
|
+
})
|
|
334
|
+
} catch (error) {
|
|
335
|
+
// Raw audio deliberately remains for bounded cleanup / retry.
|
|
336
|
+
console.error(
|
|
337
|
+
`[meeting/save] Batch finalization failed for ${sessionId}: `
|
|
338
|
+
+ `${error instanceof Error ? error.message : String(error)}`,
|
|
339
|
+
)
|
|
340
|
+
}
|
|
341
|
+
// Always hand off the durable local scribe (streaming or batch) into
|
|
342
|
+
// operations/ when COS pipeline is configured — this is what was
|
|
343
|
+
// missing on managed public server and left today's G2 files unsynced.
|
|
344
|
+
if (cosOpsPipelineConfigured()) {
|
|
345
|
+
await handoffMeetingToOperations(saved.filepath)
|
|
346
|
+
}
|
|
326
347
|
}).catch(error => {
|
|
327
|
-
// Raw audio deliberately remains for the existing two-hour cleanup.
|
|
328
348
|
console.error(
|
|
329
|
-
`[meeting/save]
|
|
349
|
+
`[meeting/save] G2 ops handoff failed for ${sessionId}: `
|
|
330
350
|
+ `${error instanceof Error ? error.message : String(error)}`,
|
|
331
351
|
)
|
|
332
352
|
}).finally(() => batchLease.release())
|
|
333
353
|
scheduleBackground(task)
|
|
354
|
+
} else if (cosOpsPipelineConfigured()) {
|
|
355
|
+
// No HQ batch (no audio / incomplete writes) — still hand off the
|
|
356
|
+
// streaming scribe into operations when COS pipeline is configured.
|
|
357
|
+
scheduleBackground(
|
|
358
|
+
handoffMeetingToOperations(saved.filepath).catch(error => {
|
|
359
|
+
console.error(
|
|
360
|
+
`[meeting/save] G2 ops handoff failed for ${sessionId}: `
|
|
361
|
+
+ `${error instanceof Error ? error.message : String(error)}`,
|
|
362
|
+
)
|
|
363
|
+
}),
|
|
364
|
+
)
|
|
334
365
|
}
|
|
335
366
|
} catch (error) {
|
|
336
367
|
if (error instanceof MeetingStoreError) {
|
|
@@ -399,7 +430,7 @@ async function finalizeBatch(options: {
|
|
|
399
430
|
if (canDeletePendingBatchAudio(transcriptApplied, metadataPersisted)) {
|
|
400
431
|
rmSync(options.audioDir, { recursive: true, force: true })
|
|
401
432
|
} else {
|
|
402
|
-
console.warn('[meeting/save] Pending raw audio retained for bounded
|
|
433
|
+
console.warn('[meeting/save] Pending raw audio retained for bounded cleanup')
|
|
403
434
|
}
|
|
404
435
|
}
|
|
405
436
|
|
|
@@ -46,8 +46,38 @@ import { getServerInstanceId } from '../lib/server-instance-id.js'
|
|
|
46
46
|
import {
|
|
47
47
|
acquireMaintenanceWork,
|
|
48
48
|
maintenanceAdmissionsOpen,
|
|
49
|
+
maintenanceLifecycle,
|
|
49
50
|
type MaintenanceWorkLease,
|
|
50
51
|
} from '../lib/maintenance-lifecycle.js'
|
|
52
|
+
import { feedLiveCueTranscript } from '../lib/live-cues-engine.js'
|
|
53
|
+
import { preemptMetalBatchForLive, registerLiveActivityProbe } from '../lib/whisper-metal-gate.js'
|
|
54
|
+
|
|
55
|
+
/** Preempt hook 2 of 2 (1C): the recording_chunk backstop. Live audio is
|
|
56
|
+
* arriving, so any Metal batch must yield the GPU now. This covers recovery,
|
|
57
|
+
* reconnect, and restart adoption — paths that reach chunk upload WITHOUT
|
|
58
|
+
* going through session creation. Idempotent with the create hook.
|
|
59
|
+
*
|
|
60
|
+
* Every recording_chunk lease goes through here rather than calling
|
|
61
|
+
* acquireMaintenanceWork directly, so a future call site cannot silently skip
|
|
62
|
+
* the preempt. Preempt runs BEFORE acquire so the GPU frees even if the drain
|
|
63
|
+
* gate rejects the lease. */
|
|
64
|
+
function acquireRecordingChunkLease(options: { allowDuringDrain?: boolean }): MaintenanceWorkLease {
|
|
65
|
+
preemptMetalBatchForLive('recording_chunk')
|
|
66
|
+
return acquireMaintenanceWork('recording_chunk', options)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Live-activity probe for the Metal gate (2B). Reports the most recent
|
|
70
|
+
* lastActivityAt across in-memory sessions; the gate applies its own
|
|
71
|
+
* 180s window so a cold orphan cannot pin batch to CPU. Registered here to
|
|
72
|
+
* keep the dependency one-way (stream -> gate), avoiding a circular import. */
|
|
73
|
+
registerLiveActivityProbe(() => {
|
|
74
|
+
let newest: number | null = null
|
|
75
|
+
for (const session of sessions.values()) {
|
|
76
|
+
const at = session.lastActivityAt
|
|
77
|
+
if (typeof at === 'number' && Number.isFinite(at) && (newest === null || at > newest)) newest = at
|
|
78
|
+
}
|
|
79
|
+
return newest
|
|
80
|
+
})
|
|
51
81
|
|
|
52
82
|
function ensurePrivateDirectory(path: string): void {
|
|
53
83
|
if (!existsSync(path)) mkdirSync(path, { recursive: true, mode: 0o700 })
|
|
@@ -636,11 +666,15 @@ setInterval(() => {
|
|
|
636
666
|
}
|
|
637
667
|
}
|
|
638
668
|
} catch {}
|
|
639
|
-
// Purge stale pending-batch dirs
|
|
669
|
+
// Purge stale pending-batch dirs. HQ large-v3 on long meetings + Control
|
|
670
|
+
// restart can exceed 2h (2026-07-27: two sessions purged before batch).
|
|
671
|
+
// Use 12h, and never purge while a meeting_batch_finalization lease is held.
|
|
640
672
|
// Age measured by _batch_pending.marker mtime (set by moveSessionAudioToPending), NOT the first
|
|
641
673
|
// chunk's mtime — chunk files keep their original write-time across the atomic rename, so for
|
|
642
674
|
// meetings > 1 hour, chunk mtimes would always look stale. Fallback: dir mtime for older marker-less dirs.
|
|
643
675
|
try {
|
|
676
|
+
const batchBusy = ((maintenanceLifecycle.snapshot().activeByKind as Record<string, number>)
|
|
677
|
+
.meeting_batch_finalization ?? 0) > 0
|
|
644
678
|
for (const dir of readdirSync(PENDING_BATCH_DIR)) {
|
|
645
679
|
const dirPath = resolve(PENDING_BATCH_DIR, dir)
|
|
646
680
|
try {
|
|
@@ -655,7 +689,11 @@ setInterval(() => {
|
|
|
655
689
|
// Fallback for pre-v5.4.3 dirs: use directory ctime (changes on rename)
|
|
656
690
|
ageSource = statSync(dirPath).ctimeMs
|
|
657
691
|
}
|
|
658
|
-
if (Date.now() - ageSource >
|
|
692
|
+
if (Date.now() - ageSource > 12 * 60 * 60 * 1000) {
|
|
693
|
+
if (batchBusy) {
|
|
694
|
+
console.warn(`[cleanup] Retaining stale pending-batch while batch lease held: ${dir}`)
|
|
695
|
+
continue
|
|
696
|
+
}
|
|
659
697
|
rmSync(dirPath, { recursive: true, force: true })
|
|
660
698
|
console.log(`[cleanup] Purged stale pending-batch: ${dir}`)
|
|
661
699
|
}
|
|
@@ -700,6 +738,10 @@ export function getSession(sessionId: string): TranscriptSession {
|
|
|
700
738
|
emptyCompletions: {},
|
|
701
739
|
}
|
|
702
740
|
sessions.set(sessionId, session)
|
|
741
|
+
// Preempt hook 1 of 2 (1C). CREATE ONLY — deliberately inside the `!session`
|
|
742
|
+
// branch, never on an ordinary getSession() touch, or a status read of a
|
|
743
|
+
// stale session would falsely evict a healthy Metal batch.
|
|
744
|
+
preemptMetalBatchForLive('session_create')
|
|
703
745
|
}
|
|
704
746
|
if (!session.providerCandidates) session.providerCandidates = {}
|
|
705
747
|
if (!session.receivedIndices) session.receivedIndices = []
|
|
@@ -1538,6 +1580,11 @@ async function processStreamChunk(opts: {
|
|
|
1538
1580
|
console.log(`[perf] persistSession: ${(performance.now() - tPersist).toFixed(1)}ms (${session.chunks.filter(c => c).length} chunks)`)
|
|
1539
1581
|
|
|
1540
1582
|
emitDisplay({ type: 'transcript_chunk', data: { text: trimmedText, speaker, chunkIndex, elapsed, sessionId } })
|
|
1583
|
+
// Live Cues feed — fire-and-forget, NEVER awaited: transcription must not
|
|
1584
|
+
// block on a cue, and no LLM runs on this path. .catch() is required — a
|
|
1585
|
+
// bare `void` on a rejecting promise is an unhandled rejection, which Node
|
|
1586
|
+
// throws on by default. All gates live inside the feed.
|
|
1587
|
+
void feedLiveCueTranscript(sessionId, trimmedText, Boolean(fallbackReason)).catch(() => {})
|
|
1541
1588
|
|
|
1542
1589
|
console.log(`[perf] TOTAL request: ${(performance.now() - tReq).toFixed(1)}ms | chunk #${chunkIndex} | ${audioBuffer.length}b | rms=${Math.round(rms)} q=${isQuiet ? 1 : 0} | ${asrProvider} | "${trimmedText.slice(0, 50)}"`)
|
|
1543
1590
|
return canonicalChunkResponse(chunk, sessionId, chunkIndex)
|
|
@@ -1571,7 +1618,7 @@ transcribeStreamRouter.post('/transcribe-stream', async (req, res) => {
|
|
|
1571
1618
|
// Reject a wrong Mac before consuming or persisting any upload bytes.
|
|
1572
1619
|
assertPinnedServerIdentity(req.get('X-COS-Server-Instance'), req.query.serverInstanceId)
|
|
1573
1620
|
const sessionId = (req.query.sessionId as string) || `g2_${Date.now()}`
|
|
1574
|
-
maintenanceLease =
|
|
1621
|
+
maintenanceLease = acquireRecordingChunkLease({
|
|
1575
1622
|
allowDuringDrain: sessions.has(sessionId),
|
|
1576
1623
|
})
|
|
1577
1624
|
const chunkIndex = parseInt((req.query.chunkIndex as string) || '0', 10)
|
|
@@ -1607,7 +1654,7 @@ transcribeStreamRouter.post('/transcribe-stream/offline-sessions/start', async (
|
|
|
1607
1654
|
const body = req.body ?? {}
|
|
1608
1655
|
const sessionId = String(body.sessionId ?? '')
|
|
1609
1656
|
validateSessionId(sessionId)
|
|
1610
|
-
maintenanceLease =
|
|
1657
|
+
maintenanceLease = acquireRecordingChunkLease({
|
|
1611
1658
|
allowDuringDrain: sessions.has(sessionId),
|
|
1612
1659
|
})
|
|
1613
1660
|
if (isSessionDeleted(sessionId)) throw makeHttpError(410, 'session is closed', 'session_closed')
|
|
@@ -1633,7 +1680,7 @@ transcribeStreamRouter.post('/transcribe-stream/offline-sessions/:sessionId/chun
|
|
|
1633
1680
|
if (!isIosAsrCandidateEnabled()) throw makeHttpError(403, 'iPhone ASR candidates disabled', 'iphone_asr_disabled')
|
|
1634
1681
|
const sessionId = String(req.params.sessionId ?? '')
|
|
1635
1682
|
validateSessionId(sessionId)
|
|
1636
|
-
maintenanceLease =
|
|
1683
|
+
maintenanceLease = acquireRecordingChunkLease({
|
|
1637
1684
|
allowDuringDrain: sessions.has(sessionId),
|
|
1638
1685
|
})
|
|
1639
1686
|
if (isSessionDeleted(sessionId)) throw makeHttpError(410, 'session is closed', 'session_closed')
|
|
@@ -1679,7 +1726,7 @@ transcribeStreamRouter.post('/transcribe-stream/offline-sessions/:sessionId/fina
|
|
|
1679
1726
|
if (!isIosAsrCandidateEnabled()) throw makeHttpError(403, 'iPhone ASR candidates disabled', 'iphone_asr_disabled')
|
|
1680
1727
|
const sessionId = String(req.params.sessionId ?? '')
|
|
1681
1728
|
validateSessionId(sessionId)
|
|
1682
|
-
maintenanceLease =
|
|
1729
|
+
maintenanceLease = acquireRecordingChunkLease({
|
|
1683
1730
|
allowDuringDrain: sessions.has(sessionId),
|
|
1684
1731
|
})
|
|
1685
1732
|
if (isSessionDeleted(sessionId)) throw makeHttpError(410, 'session is closed', 'session_closed')
|
|
@@ -1713,7 +1760,7 @@ transcribeStreamRouter.post('/transcribe-stream/candidates', async (req, res) =>
|
|
|
1713
1760
|
const sessionId = String(body.sessionId ?? '')
|
|
1714
1761
|
const chunkIndex = Number(body.chunkIndex)
|
|
1715
1762
|
validateSessionId(sessionId)
|
|
1716
|
-
maintenanceLease =
|
|
1763
|
+
maintenanceLease = acquireRecordingChunkLease({
|
|
1717
1764
|
allowDuringDrain: sessions.has(sessionId),
|
|
1718
1765
|
})
|
|
1719
1766
|
validateChunkIndex(chunkIndex)
|