@gotcos/glasses-server 6.12.7 → 6.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +10 -0
- package/bin/cli.cjs +12 -0
- package/bin/managed-server.cjs +28 -0
- package/managed-runtime-contract.json +23 -0
- package/package.json +6 -3
- package/server/index.ts +106 -35
- package/server/lib/bookmarks.ts +96 -0
- package/server/lib/handoff-store.ts +404 -0
- package/server/lib/launch-dir.ts +13 -7
- package/server/lib/maintenance-lifecycle.ts +735 -0
- package/server/lib/managed-runtime.ts +44 -0
- package/server/lib/openai-tts-budget.ts +142 -0
- package/server/lib/prompt-edit.ts +224 -0
- package/server/lib/query-job-coordinator.ts +36 -4
- package/server/lib/query-job-runtime.ts +38 -26
- package/server/lib/recovery-activity.ts +168 -0
- package/server/lib/speaker-trainer.ts +532 -0
- package/server/lib/tts-cache.ts +596 -0
- package/server/routes/bookmarks.ts +59 -0
- package/server/routes/glossary.ts +153 -0
- package/server/routes/handoffs.ts +101 -0
- package/server/routes/health.ts +15 -12
- package/server/routes/maintenance.ts +160 -0
- package/server/routes/meeting.ts +25 -8
- package/server/routes/openai-compat.ts +82 -36
- package/server/routes/prompt-drafts.ts +51 -8
- package/server/routes/prompt-edit.ts +33 -0
- package/server/routes/query.ts +52 -24
- package/server/routes/recovery.ts +76 -0
- package/server/routes/transcribe-stream.ts +49 -3
- package/server/routes/transcribe.ts +14 -0
- package/server/routes/tts.ts +709 -0
- package/server/routes/voice.ts +317 -0
- package/shared/handoff-intent.ts +90 -0
|
@@ -14,6 +14,11 @@ import {
|
|
|
14
14
|
import { tryInstantResponse } from '../lib/response-cache.js'
|
|
15
15
|
import crypto from 'node:crypto'
|
|
16
16
|
import { timingSafeTokenEqual } from '../lib/token-auth.js'
|
|
17
|
+
import {
|
|
18
|
+
acquireMaintenanceWork,
|
|
19
|
+
MaintenanceLifecycleError,
|
|
20
|
+
type MaintenanceWorkLease,
|
|
21
|
+
} from '../lib/maintenance-lifecycle.js'
|
|
17
22
|
|
|
18
23
|
export const openaiCompatRouter = Router()
|
|
19
24
|
|
|
@@ -131,6 +136,27 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
131
136
|
})
|
|
132
137
|
}
|
|
133
138
|
|
|
139
|
+
// Admission closes before cache lookup, dedup bookkeeping, latency logging,
|
|
140
|
+
// or provider work. Maintenance therefore has one linearization point for
|
|
141
|
+
// every OpenAI-compatible request, including instant and duplicate replies.
|
|
142
|
+
let maintenanceLease: MaintenanceWorkLease
|
|
143
|
+
try {
|
|
144
|
+
maintenanceLease = acquireMaintenanceWork('openai_query')
|
|
145
|
+
} catch (error) {
|
|
146
|
+
if (error instanceof MaintenanceLifecycleError) {
|
|
147
|
+
if (error.retryAfterSeconds != null) res.setHeader('Retry-After', String(error.retryAfterSeconds))
|
|
148
|
+
return res.status(error.status).json({
|
|
149
|
+
error: {
|
|
150
|
+
message: error.message,
|
|
151
|
+
type: 'server_error',
|
|
152
|
+
code: error.code,
|
|
153
|
+
retryable: error.retryable,
|
|
154
|
+
},
|
|
155
|
+
})
|
|
156
|
+
}
|
|
157
|
+
throw error
|
|
158
|
+
}
|
|
159
|
+
|
|
134
160
|
|
|
135
161
|
// Log request entry — tells us if Even sends stream: true or false
|
|
136
162
|
console.log(`[g2] Request: stream=${!!stream}, query="${query.slice(0, 50)}"`)
|
|
@@ -184,11 +210,12 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
184
210
|
res.write(`data: ${JSON.stringify(finalChunk)}\n\n`)
|
|
185
211
|
res.write('data: [DONE]\n\n')
|
|
186
212
|
res.end()
|
|
213
|
+
maintenanceLease.release()
|
|
187
214
|
return
|
|
188
215
|
}
|
|
189
216
|
|
|
190
217
|
// Non-streaming cache response
|
|
191
|
-
|
|
218
|
+
const response = res.json({
|
|
192
219
|
id: completionId,
|
|
193
220
|
object: 'chat.completion',
|
|
194
221
|
created: timestamp,
|
|
@@ -200,6 +227,8 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
200
227
|
}],
|
|
201
228
|
usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
|
|
202
229
|
})
|
|
230
|
+
maintenanceLease.release()
|
|
231
|
+
return response
|
|
203
232
|
}
|
|
204
233
|
|
|
205
234
|
// ── Dedup guard — if same query is already in-flight within 2s, reuse result ──
|
|
@@ -241,9 +270,10 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
241
270
|
res.write(`data: ${JSON.stringify({ id: completionId, object: 'chat.completion.chunk', created: timestamp, model: responseModel, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] })}\n\n`)
|
|
242
271
|
res.write('data: [DONE]\n\n')
|
|
243
272
|
res.end()
|
|
273
|
+
maintenanceLease.release()
|
|
244
274
|
return
|
|
245
275
|
}
|
|
246
|
-
|
|
276
|
+
const response = res.json({
|
|
247
277
|
id: completionId,
|
|
248
278
|
object: 'chat.completion',
|
|
249
279
|
created: timestamp,
|
|
@@ -251,6 +281,8 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
251
281
|
choices: [{ index: 0, message: { role: 'assistant', content: dedupResult }, finish_reason: 'stop' }],
|
|
252
282
|
usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
|
|
253
283
|
})
|
|
284
|
+
maintenanceLease.release()
|
|
285
|
+
return response
|
|
254
286
|
} catch {
|
|
255
287
|
// Original request failed — fall through to make a fresh request
|
|
256
288
|
}
|
|
@@ -311,11 +343,12 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
311
343
|
}
|
|
312
344
|
},
|
|
313
345
|
onDone: (fullText) => {
|
|
314
|
-
|
|
315
|
-
done
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
346
|
+
try {
|
|
347
|
+
if (!done) {
|
|
348
|
+
done = true
|
|
349
|
+
resolveInflight!(fullText || '')
|
|
350
|
+
inflightQueries.delete(dedupKey)
|
|
351
|
+
logLatency({
|
|
319
352
|
timestamp: new Date().toISOString(),
|
|
320
353
|
query: query.slice(0, 50),
|
|
321
354
|
ttfb_ms: actualTtfbMs,
|
|
@@ -325,35 +358,42 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
325
358
|
contextInjected: /\b(schedule|calendar|meeting|task|tasks|today|tomorrow|next meeting|who do i meet|what's next)\b/i.test(query),
|
|
326
359
|
cacheHit: false,
|
|
327
360
|
stream_requested: true,
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
361
|
+
})
|
|
362
|
+
// Final chunk with finish_reason
|
|
363
|
+
const finalChunk = {
|
|
331
364
|
id: completionId,
|
|
332
365
|
object: 'chat.completion.chunk',
|
|
333
366
|
created: timestamp,
|
|
334
367
|
model: responseModel,
|
|
335
368
|
choices: [{ index: 0, delta: {}, finish_reason: 'stop' }],
|
|
369
|
+
}
|
|
370
|
+
res.write(`data: ${JSON.stringify(finalChunk)}\n\n`)
|
|
371
|
+
res.write('data: [DONE]\n\n')
|
|
372
|
+
res.end()
|
|
336
373
|
}
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
res.end()
|
|
374
|
+
} finally {
|
|
375
|
+
maintenanceLease.release()
|
|
340
376
|
}
|
|
341
377
|
},
|
|
342
378
|
onError: (error) => {
|
|
343
|
-
|
|
344
|
-
done
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
379
|
+
try {
|
|
380
|
+
if (!done) {
|
|
381
|
+
done = true
|
|
382
|
+
rejectInflight!(new Error(error))
|
|
383
|
+
inflightQueries.delete(dedupKey)
|
|
384
|
+
const errChunk = {
|
|
348
385
|
id: completionId,
|
|
349
386
|
object: 'chat.completion.chunk',
|
|
350
387
|
created: timestamp,
|
|
351
388
|
model: responseModel,
|
|
352
389
|
choices: [{ index: 0, delta: { content: `Error: ${error}` }, finish_reason: 'stop' }],
|
|
390
|
+
}
|
|
391
|
+
res.write(`data: ${JSON.stringify(errChunk)}\n\n`)
|
|
392
|
+
res.write('data: [DONE]\n\n')
|
|
393
|
+
res.end()
|
|
353
394
|
}
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
res.end()
|
|
395
|
+
} finally {
|
|
396
|
+
maintenanceLease.release()
|
|
357
397
|
}
|
|
358
398
|
},
|
|
359
399
|
onToolStatus: (status) => {
|
|
@@ -367,6 +407,7 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
367
407
|
// Persist session ID for multi-turn context on subsequent G2 queries
|
|
368
408
|
g2SessionId = returnedSid
|
|
369
409
|
} catch (err: any) {
|
|
410
|
+
maintenanceLease.release()
|
|
370
411
|
if (!done) {
|
|
371
412
|
done = true
|
|
372
413
|
rejectInflight!(err)
|
|
@@ -399,6 +440,7 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
399
440
|
const fail = (error: unknown) => {
|
|
400
441
|
if (settled) return
|
|
401
442
|
settled = true
|
|
443
|
+
maintenanceLease.release()
|
|
402
444
|
const err = error instanceof Error ? error : new Error(String(error))
|
|
403
445
|
rejectInflight!(err)
|
|
404
446
|
inflightQueries.delete(dedupKey)
|
|
@@ -412,21 +454,25 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
|
|
|
412
454
|
onDone: (fullText) => {
|
|
413
455
|
if (settled) return
|
|
414
456
|
settled = true
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
457
|
+
try {
|
|
458
|
+
const text = fullText || result
|
|
459
|
+
resolveInflight!(text)
|
|
460
|
+
inflightQueries.delete(dedupKey)
|
|
461
|
+
logLatency({
|
|
462
|
+
timestamp: new Date().toISOString(),
|
|
463
|
+
query: query.slice(0, 50),
|
|
464
|
+
ttfb_ms: nsFirstChunkMs,
|
|
465
|
+
total_ms: Date.now() - requestReceivedAt,
|
|
466
|
+
model: resolvedModel,
|
|
467
|
+
resumed: !!currentSessionIdNS,
|
|
468
|
+
contextInjected: /\b(schedule|calendar|meeting|task|tasks|today|tomorrow)\b/i.test(query),
|
|
469
|
+
cacheHit: false,
|
|
470
|
+
stream_requested: false,
|
|
471
|
+
})
|
|
472
|
+
resolve(text)
|
|
473
|
+
} finally {
|
|
474
|
+
maintenanceLease.release()
|
|
475
|
+
}
|
|
430
476
|
},
|
|
431
477
|
onError: (error) => {
|
|
432
478
|
fail(new Error(error))
|
|
@@ -39,6 +39,13 @@ import { logTokenAudit } from '../lib/token-audit.js'
|
|
|
39
39
|
import { atomicWriteFileSync } from '../lib/atomic-fs.js'
|
|
40
40
|
import { dataPath } from '../lib/data-dir.js'
|
|
41
41
|
import { emitDisplay } from '../lib/display-bus.js'
|
|
42
|
+
import {
|
|
43
|
+
acquireMaintenanceWork,
|
|
44
|
+
maintenanceAdmissionsOpen,
|
|
45
|
+
MaintenanceLifecycleError,
|
|
46
|
+
maintenanceErrorPayload,
|
|
47
|
+
type MaintenanceWorkLease,
|
|
48
|
+
} from '../lib/maintenance-lifecycle.js'
|
|
42
49
|
|
|
43
50
|
export const promptDraftsRouter = Router()
|
|
44
51
|
|
|
@@ -138,6 +145,12 @@ function sanitizeTranscript(draftId: string, text: string, learnInline = true):
|
|
|
138
145
|
}
|
|
139
146
|
|
|
140
147
|
async function sendDraftError(res: Response, draftId: string, err: any): Promise<void> {
|
|
148
|
+
// Maintenance rejection is an admission result, not a draft failure. Keep
|
|
149
|
+
// the recoverable draft untouched so the phone can retry after maintenance.
|
|
150
|
+
if (err instanceof MaintenanceLifecycleError) {
|
|
151
|
+
if (err.retryAfterSeconds != null) res.setHeader('Retry-After', String(err.retryAfterSeconds))
|
|
152
|
+
return void res.status(err.status).json({ ...maintenanceErrorPayload(err), draftPreserved: true })
|
|
153
|
+
}
|
|
141
154
|
if (err instanceof NoSpeechDetectedError) return void res.status(204).send()
|
|
142
155
|
if (err instanceof OpenAIWhisperBudgetExhaustedError) {
|
|
143
156
|
await markPromptDraftError(draftId, err.message)
|
|
@@ -211,31 +224,53 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
|
|
|
211
224
|
return { draftId, text: finalText, recovered: true, chunkCount: finalized.receivedChunkIndexes.length, missingChunks: getMissingChunkIndexes(finalized), expiresAt: finalized.expiresAt }
|
|
212
225
|
}
|
|
213
226
|
|
|
214
|
-
const prunedAtBoot = prunePromptDrafts()
|
|
227
|
+
const prunedAtBoot = maintenanceAdmissionsOpen() ? prunePromptDrafts() : 0
|
|
215
228
|
if (prunedAtBoot) console.log(`[prompt-draft] pruned ${prunedAtBoot} expired draft(s)`)
|
|
216
|
-
const pruneTimer = setInterval(() =>
|
|
229
|
+
const pruneTimer = setInterval(() => {
|
|
230
|
+
if (maintenanceAdmissionsOpen()) prunePromptDrafts()
|
|
231
|
+
}, 60 * 60 * 1000)
|
|
217
232
|
pruneTimer.unref?.()
|
|
218
233
|
|
|
219
234
|
promptDraftsRouter.post('/prompt-drafts/start', (req, res) => {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
235
|
+
let maintenanceLease: MaintenanceWorkLease | undefined
|
|
236
|
+
try {
|
|
237
|
+
maintenanceLease = acquireMaintenanceWork('prompt_draft_write')
|
|
238
|
+
const requestedId = typeof req.body?.recoveryId === 'string' ? req.body.recoveryId : undefined
|
|
239
|
+
const meta = createPromptDraft(requestedId)
|
|
240
|
+
res.json({ draftId: meta.draftId, recoveryId: requestedId ?? meta.draftId, remapped: Boolean(requestedId && requestedId !== meta.draftId), expiresAt: meta.expiresAt, status: meta.status })
|
|
241
|
+
} catch (err: any) {
|
|
242
|
+
if (err instanceof MaintenanceLifecycleError) {
|
|
243
|
+
if (err.retryAfterSeconds != null) res.setHeader('Retry-After', String(err.retryAfterSeconds))
|
|
244
|
+
res.status(err.status).json(maintenanceErrorPayload(err))
|
|
245
|
+
return
|
|
246
|
+
}
|
|
247
|
+
res.status(err.status ?? 500).json({ error: err.message })
|
|
248
|
+
} finally {
|
|
249
|
+
maintenanceLease?.release()
|
|
250
|
+
}
|
|
223
251
|
})
|
|
224
252
|
|
|
225
253
|
promptDraftsRouter.post('/prompt-drafts/:draftId/chunks', async (req, res) => {
|
|
254
|
+
let maintenanceLease: MaintenanceWorkLease | undefined
|
|
226
255
|
try {
|
|
227
256
|
const raw = Array.isArray(req.query.chunkIndex) ? req.query.chunkIndex[0] : req.query.chunkIndex
|
|
228
257
|
const chunkIndex = Number(raw)
|
|
229
258
|
if (!Number.isInteger(chunkIndex) || chunkIndex < 0 || chunkIndex >= MAX_CHUNKS) return res.status(400).json({ error: 'invalid chunkIndex' })
|
|
230
|
-
const audio = await readRawBody(req)
|
|
231
|
-
if (audio.length < 44) return res.status(400).json({ error: 'audio too short' })
|
|
232
259
|
const before = loadPromptDraftMeta(req.params.draftId)
|
|
233
260
|
if (!before) return res.status(404).json({ error: 'draft not found' })
|
|
261
|
+
maintenanceLease = acquireMaintenanceWork('prompt_draft_write')
|
|
262
|
+
const audio = await readRawBody(req)
|
|
263
|
+
if (audio.length < 44) return res.status(400).json({ error: 'audio too short' })
|
|
234
264
|
const existingBytes = before.chunkBytes[String(chunkIndex)] ?? 0
|
|
235
265
|
const nextTotal = Object.values(before.chunkBytes).reduce((sum, bytes) => sum + bytes, 0) - existingBytes + audio.length
|
|
236
266
|
if (nextTotal > MAX_DRAFT_BYTES) return res.status(413).json({ error: 'prompt draft too large' })
|
|
237
267
|
const meta = await savePromptDraftChunk(req.params.draftId, chunkIndex, audio)
|
|
268
|
+
const warmLease = acquireMaintenanceWork('prompt_draft_warm', {
|
|
269
|
+
allowDuringDrain: true,
|
|
270
|
+
phase: 'queued',
|
|
271
|
+
})
|
|
238
272
|
warmTail = warmTail.then(async () => {
|
|
273
|
+
warmLease.setPhase('active')
|
|
239
274
|
const text = await transcribeChunk(req.params.draftId, chunkIndex, audio, 'fast', 'warm')
|
|
240
275
|
// The durability ACK above remains immediate. Publish the optional warm
|
|
241
276
|
// transcript only after rechecking that this exact audio still owns the
|
|
@@ -247,10 +282,16 @@ promptDraftsRouter.post('/prompt-drafts/:draftId/chunks', async (req, res) => {
|
|
|
247
282
|
})
|
|
248
283
|
}).catch(err => {
|
|
249
284
|
console.warn(`[prompt-draft] warm transcription failed ${req.params.draftId}/${chunkIndex}: ${err.message}`)
|
|
250
|
-
})
|
|
285
|
+
}).finally(() => warmLease.release())
|
|
251
286
|
res.json({ draftId: meta.draftId, chunkIndex, acked: true, receivedChunkIndexes: meta.receivedChunkIndexes, chunkBytes: meta.chunkBytes[String(chunkIndex)] ?? audio.length, transcriptPending: true, expiresAt: meta.expiresAt })
|
|
252
287
|
} catch (err: any) {
|
|
288
|
+
if (err instanceof MaintenanceLifecycleError) {
|
|
289
|
+
if (err.retryAfterSeconds != null) res.setHeader('Retry-After', String(err.retryAfterSeconds))
|
|
290
|
+
return void res.status(err.status).json(maintenanceErrorPayload(err))
|
|
291
|
+
}
|
|
253
292
|
res.status(err.status ?? (err.message === 'draft not found' ? 404 : 500)).json({ error: err.message })
|
|
293
|
+
} finally {
|
|
294
|
+
maintenanceLease?.release()
|
|
254
295
|
}
|
|
255
296
|
})
|
|
256
297
|
|
|
@@ -262,7 +303,9 @@ async function finalizeRequest(req: any, res: Response): Promise<void> {
|
|
|
262
303
|
const key = `${req.params.draftId}:${mode}`
|
|
263
304
|
let job = finalizeJobs.get(key)
|
|
264
305
|
if (!job) {
|
|
306
|
+
const finalizeLease = acquireMaintenanceWork('prompt_draft_finalize')
|
|
265
307
|
job = finalizeDraft(req.params.draftId, mode, routeAutoClean(req), abort.signal)
|
|
308
|
+
.finally(() => finalizeLease.release())
|
|
266
309
|
finalizeJobs.set(key, job)
|
|
267
310
|
job.finally(() => finalizeJobs.delete(key)).catch(() => {})
|
|
268
311
|
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { Router } from 'express'
|
|
2
|
+
import {
|
|
3
|
+
PromptEditValidationError,
|
|
4
|
+
applyPromptEdit,
|
|
5
|
+
normalizePromptEditInput,
|
|
6
|
+
} from '../lib/prompt-edit.js'
|
|
7
|
+
import { errMsg } from '../lib/utils.js'
|
|
8
|
+
|
|
9
|
+
export const promptEditRouter = Router()
|
|
10
|
+
|
|
11
|
+
promptEditRouter.post('/prompt-edit', async (req, res) => {
|
|
12
|
+
const abort = new AbortController()
|
|
13
|
+
// Abort the model spawn ONLY if the client disconnects before we respond.
|
|
14
|
+
// Must listen on `res`, not `req`: express.json() fully drains the request
|
|
15
|
+
// body stream before this handler runs, so `req` emits 'close' on the next
|
|
16
|
+
// tick and would abort EVERY edit mid-flight (instant 500 "Prompt edit
|
|
17
|
+
// aborted"). `res` 'close' + !writableEnded fires only on a genuine premature
|
|
18
|
+
// disconnect. (House pattern: see routes/query.ts.)
|
|
19
|
+
res.on('close', () => {
|
|
20
|
+
if (!res.writableEnded) abort.abort()
|
|
21
|
+
})
|
|
22
|
+
try {
|
|
23
|
+
const input = normalizePromptEditInput(req.body)
|
|
24
|
+
const revisedText = await applyPromptEdit(input, abort.signal)
|
|
25
|
+
res.json({ revisedText })
|
|
26
|
+
} catch (err) {
|
|
27
|
+
if (err instanceof PromptEditValidationError) {
|
|
28
|
+
res.status(err.status).json({ error: err.message })
|
|
29
|
+
return
|
|
30
|
+
}
|
|
31
|
+
res.status(500).json({ error: errMsg(err) })
|
|
32
|
+
}
|
|
33
|
+
})
|
package/server/routes/query.ts
CHANGED
|
@@ -9,6 +9,11 @@ import { normalizeEffortPreference, normalizeModelPreference } from '../../share
|
|
|
9
9
|
import { QueryAttachmentError, resolveQueryAttachments } from '../lib/query-attachments.js'
|
|
10
10
|
import { getMediaStore } from '../lib/media-store.js'
|
|
11
11
|
import { mergeMediaAttachmentRefs } from '../../shared/media-attachment.js'
|
|
12
|
+
import {
|
|
13
|
+
acquireMaintenanceWork,
|
|
14
|
+
MaintenanceLifecycleError,
|
|
15
|
+
maintenanceErrorPayload,
|
|
16
|
+
} from '../lib/maintenance-lifecycle.js'
|
|
12
17
|
|
|
13
18
|
const TOOL_STATUS_MESSAGES: Record<string, string> = {
|
|
14
19
|
WebSearch: 'Searching web...',
|
|
@@ -19,6 +24,16 @@ const TOOL_STATUS_MESSAGES: Record<string, string> = {
|
|
|
19
24
|
export const queryRouter = Router()
|
|
20
25
|
|
|
21
26
|
queryRouter.post('/query', async (req, res) => {
|
|
27
|
+
let maintenanceLease
|
|
28
|
+
try {
|
|
29
|
+
maintenanceLease = acquireMaintenanceWork('legacy_query')
|
|
30
|
+
} catch (error) {
|
|
31
|
+
if (error instanceof MaintenanceLifecycleError) {
|
|
32
|
+
if (error.retryAfterSeconds != null) res.setHeader('Retry-After', String(error.retryAfterSeconds))
|
|
33
|
+
return res.status(error.status).json(maintenanceErrorPayload(error))
|
|
34
|
+
}
|
|
35
|
+
throw error
|
|
36
|
+
}
|
|
22
37
|
const { query, sessionId, model, effort, reference, globalMsgNum } = req.body
|
|
23
38
|
const activityToolMode = req.body.activityToolMode === 'off' || req.body.activityToolMode === 'preview'
|
|
24
39
|
? req.body.activityToolMode
|
|
@@ -31,8 +46,10 @@ queryRouter.post('/query', async (req, res) => {
|
|
|
31
46
|
resolvedAttachments = await resolveQueryAttachments(req.body)
|
|
32
47
|
} catch (err) {
|
|
33
48
|
if (err instanceof QueryAttachmentError) {
|
|
49
|
+
maintenanceLease.release()
|
|
34
50
|
return res.status(err.status).json({ error: err.code, detail: err.message })
|
|
35
51
|
}
|
|
52
|
+
maintenanceLease.release()
|
|
36
53
|
return res.status(500).json({ error: errMsg(err) })
|
|
37
54
|
}
|
|
38
55
|
const imageInputs = resolvedAttachments.inputs.length > 0 ? resolvedAttachments.inputs : undefined
|
|
@@ -42,6 +59,7 @@ queryRouter.post('/query', async (req, res) => {
|
|
|
42
59
|
|
|
43
60
|
// Vision queries can have an empty query (default to "describe what you see")
|
|
44
61
|
if ((!resolvedQuery || typeof resolvedQuery !== 'string') && !imageInputs) {
|
|
62
|
+
maintenanceLease.release()
|
|
45
63
|
return res.status(400).json({ error: 'query string or image required' })
|
|
46
64
|
}
|
|
47
65
|
|
|
@@ -107,34 +125,43 @@ queryRouter.post('/query', async (req, res) => {
|
|
|
107
125
|
}
|
|
108
126
|
},
|
|
109
127
|
} : {}),
|
|
110
|
-
onDone: (fullText, model, cliSessionId, metadata) => {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
done = true
|
|
121
|
-
const attachments = mergeMediaAttachmentRefs(attachmentRefs, metadata?.outputAttachments)
|
|
122
|
-
const { outputAttachments: _outputAttachments, ...runMetadata } = metadata ?? {}
|
|
123
|
-
const payload = {
|
|
124
|
-
text: fullText, sessionId: sid, model, cliSessionId, ...runMetadata,
|
|
125
|
-
...(attachments.length > 0 ? { attachments } : {}),
|
|
128
|
+
onDone: async (fullText, model, cliSessionId, metadata) => {
|
|
129
|
+
try {
|
|
130
|
+
// Durable association is independent of the SSE socket.
|
|
131
|
+
// Backgrounding the phone cannot leave request media reserved until
|
|
132
|
+
// expiry or make maintenance proof outrun the pending write.
|
|
133
|
+
if (resolvedAttachments.ids.length > 0) {
|
|
134
|
+
await getMediaStore().associate(resolvedAttachments.ids, {
|
|
135
|
+
sessionId: sid,
|
|
136
|
+
...(validGlobalMsgNum ? { globalMsgNum: validGlobalMsgNum } : {}),
|
|
137
|
+
}).catch((err) => console.error('[query] attachment association failed:', err))
|
|
126
138
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
139
|
+
if (!done) {
|
|
140
|
+
done = true
|
|
141
|
+
const attachments = mergeMediaAttachmentRefs(attachmentRefs, metadata?.outputAttachments)
|
|
142
|
+
const { outputAttachments: _outputAttachments, ...runMetadata } = metadata ?? {}
|
|
143
|
+
const payload = {
|
|
144
|
+
text: fullText, sessionId: sid, model, cliSessionId, ...runMetadata,
|
|
145
|
+
...(attachments.length > 0 ? { attachments } : {}),
|
|
146
|
+
}
|
|
147
|
+
res.write(`event: done\ndata: ${JSON.stringify(payload)}\n\n`)
|
|
148
|
+
emitDisplay({ type: 'done', data: payload })
|
|
149
|
+
res.end()
|
|
150
|
+
}
|
|
151
|
+
} finally {
|
|
152
|
+
maintenanceLease.release()
|
|
130
153
|
}
|
|
131
154
|
},
|
|
132
155
|
onError: (error) => {
|
|
133
|
-
|
|
134
|
-
done
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
156
|
+
try {
|
|
157
|
+
if (!done) {
|
|
158
|
+
done = true
|
|
159
|
+
res.write(`event: error\ndata: ${JSON.stringify({ error })}\n\n`)
|
|
160
|
+
emitDisplay({ type: 'error', data: { error } })
|
|
161
|
+
res.end()
|
|
162
|
+
}
|
|
163
|
+
} finally {
|
|
164
|
+
maintenanceLease.release()
|
|
138
165
|
}
|
|
139
166
|
},
|
|
140
167
|
}, validModel, imageInputs,
|
|
@@ -146,6 +173,7 @@ queryRouter.post('/query', async (req, res) => {
|
|
|
146
173
|
{ abortSignal: abortController.signal, effort: validEffort },
|
|
147
174
|
)
|
|
148
175
|
} catch (err: unknown) {
|
|
176
|
+
maintenanceLease.release()
|
|
149
177
|
if (!done) {
|
|
150
178
|
done = true
|
|
151
179
|
res.write(`event: error\ndata: ${JSON.stringify({ error: errMsg(err) })}\n\n`)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { Router } from 'express'
|
|
2
|
+
import { readFileSync } from 'node:fs'
|
|
3
|
+
import { resolve } from 'node:path'
|
|
4
|
+
import { atomicWriteFileSync } from '../lib/atomic-fs.js'
|
|
5
|
+
import { acquireMaintenance, getRecoveryActivityStatus } from '../lib/recovery-activity.js'
|
|
6
|
+
import { getWhisperHealth, restartWhisperServer } from '../lib/whisper-local.js'
|
|
7
|
+
import { serverMetrics } from '../lib/server-metrics.js'
|
|
8
|
+
|
|
9
|
+
export const recoveryRouter = Router()
|
|
10
|
+
const COOLDOWN_MS = 60_000
|
|
11
|
+
const cooldownPath = resolve(import.meta.dirname, '../data/.recovery_restart.json')
|
|
12
|
+
|
|
13
|
+
function lastRestartAt(): number {
|
|
14
|
+
try { return Number(JSON.parse(readFileSync(cooldownPath, 'utf8'))?.at) || 0 } catch { return 0 }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
recoveryRouter.get('/live', (_req, res) => {
|
|
18
|
+
res.setHeader('Cache-Control', 'no-store')
|
|
19
|
+
res.json({
|
|
20
|
+
status: 'ok', bootId: serverMetrics.bootId, pid: process.pid,
|
|
21
|
+
uptimeSeconds: Math.round((Date.now() - serverMetrics.startedAt) / 1000),
|
|
22
|
+
managed: process.env.COS_HARNESS === 'daemon',
|
|
23
|
+
})
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
recoveryRouter.get('/recovery/status', (_req, res) => {
|
|
27
|
+
res.json({
|
|
28
|
+
bootId: serverMetrics.bootId,
|
|
29
|
+
managed: process.env.COS_HARNESS === 'daemon',
|
|
30
|
+
whisper: getWhisperHealth(),
|
|
31
|
+
asr: { hqActive: false, hqQueued: 0, fastRestarting: false }, // public build: no HQ/fast ASR scheduler in this server
|
|
32
|
+
activity: getRecoveryActivityStatus(),
|
|
33
|
+
})
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
recoveryRouter.post('/recovery/whisper/restart', async (_req, res) => {
|
|
37
|
+
const gate = acquireMaintenance()
|
|
38
|
+
if (!gate.ok) {
|
|
39
|
+
gate.release()
|
|
40
|
+
return res.status(409).json({ error: 'Recovery blocked by active work', reason: 'recovery_busy', busy: gate.busy })
|
|
41
|
+
}
|
|
42
|
+
try {
|
|
43
|
+
const result = await restartWhisperServer()
|
|
44
|
+
res.status(result.status === 'failed' ? 503 : 200).json(result)
|
|
45
|
+
} finally { gate.release() }
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
recoveryRouter.post('/recovery/server/restart', (_req, res) => {
|
|
49
|
+
if (process.env.COS_HARNESS !== 'daemon') {
|
|
50
|
+
return res.status(409).json({ error: 'Server is not managed by the COS LaunchAgent', reason: 'restart_unmanaged' })
|
|
51
|
+
}
|
|
52
|
+
const elapsed = Date.now() - lastRestartAt()
|
|
53
|
+
if (elapsed < COOLDOWN_MS) {
|
|
54
|
+
return res.status(429).json({ error: 'Restart cooldown active', reason: 'restart_cooldown', retryAfterMs: COOLDOWN_MS - elapsed })
|
|
55
|
+
}
|
|
56
|
+
const gate = acquireMaintenance()
|
|
57
|
+
if (!gate.ok) {
|
|
58
|
+
gate.release()
|
|
59
|
+
return res.status(409).json({ error: 'Restart blocked by active work', reason: 'recovery_busy', busy: gate.busy })
|
|
60
|
+
}
|
|
61
|
+
atomicWriteFileSync(cooldownPath, JSON.stringify({ at: Date.now(), bootId: serverMetrics.bootId }))
|
|
62
|
+
res.status(202).json({ accepted: true, oldBootId: serverMetrics.bootId })
|
|
63
|
+
// Schedule independently of the response socket. The phone may change
|
|
64
|
+
// network/close the sheet immediately after receiving 202; that must not
|
|
65
|
+
// cancel an accepted restart or strand maintenance forever.
|
|
66
|
+
const timer = setTimeout(() => {
|
|
67
|
+
if (process.env.COS_DISABLE_SELF_RESTART === '1') { gate.release(); return }
|
|
68
|
+
try {
|
|
69
|
+
process.kill(process.pid, 'SIGTERM')
|
|
70
|
+
} catch (error) {
|
|
71
|
+
gate.release()
|
|
72
|
+
console.error('[recovery] Failed to signal managed server restart:', error)
|
|
73
|
+
}
|
|
74
|
+
}, 350)
|
|
75
|
+
timer.unref?.()
|
|
76
|
+
})
|