@gotcos/glasses-server 6.12.7 → 6.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,6 +14,11 @@ import {
14
14
  import { tryInstantResponse } from '../lib/response-cache.js'
15
15
  import crypto from 'node:crypto'
16
16
  import { timingSafeTokenEqual } from '../lib/token-auth.js'
17
+ import {
18
+ acquireMaintenanceWork,
19
+ MaintenanceLifecycleError,
20
+ type MaintenanceWorkLease,
21
+ } from '../lib/maintenance-lifecycle.js'
17
22
 
18
23
  export const openaiCompatRouter = Router()
19
24
 
@@ -131,6 +136,27 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
131
136
  })
132
137
  }
133
138
 
139
+ // Admission closes before cache lookup, dedup bookkeeping, latency logging,
140
+ // or provider work. Maintenance therefore has one linearization point for
141
+ // every OpenAI-compatible request, including instant and duplicate replies.
142
+ let maintenanceLease: MaintenanceWorkLease
143
+ try {
144
+ maintenanceLease = acquireMaintenanceWork('openai_query')
145
+ } catch (error) {
146
+ if (error instanceof MaintenanceLifecycleError) {
147
+ if (error.retryAfterSeconds != null) res.setHeader('Retry-After', String(error.retryAfterSeconds))
148
+ return res.status(error.status).json({
149
+ error: {
150
+ message: error.message,
151
+ type: 'server_error',
152
+ code: error.code,
153
+ retryable: error.retryable,
154
+ },
155
+ })
156
+ }
157
+ throw error
158
+ }
159
+
134
160
 
135
161
  // Log request entry — tells us if Even sends stream: true or false
136
162
  console.log(`[g2] Request: stream=${!!stream}, query="${query.slice(0, 50)}"`)
@@ -184,11 +210,12 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
184
210
  res.write(`data: ${JSON.stringify(finalChunk)}\n\n`)
185
211
  res.write('data: [DONE]\n\n')
186
212
  res.end()
213
+ maintenanceLease.release()
187
214
  return
188
215
  }
189
216
 
190
217
  // Non-streaming cache response
191
- return res.json({
218
+ const response = res.json({
192
219
  id: completionId,
193
220
  object: 'chat.completion',
194
221
  created: timestamp,
@@ -200,6 +227,8 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
200
227
  }],
201
228
  usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
202
229
  })
230
+ maintenanceLease.release()
231
+ return response
203
232
  }
204
233
 
205
234
  // ── Dedup guard — if same query is already in-flight within 2s, reuse result ──
@@ -241,9 +270,10 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
241
270
  res.write(`data: ${JSON.stringify({ id: completionId, object: 'chat.completion.chunk', created: timestamp, model: responseModel, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] })}\n\n`)
242
271
  res.write('data: [DONE]\n\n')
243
272
  res.end()
273
+ maintenanceLease.release()
244
274
  return
245
275
  }
246
- return res.json({
276
+ const response = res.json({
247
277
  id: completionId,
248
278
  object: 'chat.completion',
249
279
  created: timestamp,
@@ -251,6 +281,8 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
251
281
  choices: [{ index: 0, message: { role: 'assistant', content: dedupResult }, finish_reason: 'stop' }],
252
282
  usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
253
283
  })
284
+ maintenanceLease.release()
285
+ return response
254
286
  } catch {
255
287
  // Original request failed — fall through to make a fresh request
256
288
  }
@@ -311,11 +343,12 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
311
343
  }
312
344
  },
313
345
  onDone: (fullText) => {
314
- if (!done) {
315
- done = true
316
- resolveInflight!(fullText || '')
317
- inflightQueries.delete(dedupKey)
318
- logLatency({
346
+ try {
347
+ if (!done) {
348
+ done = true
349
+ resolveInflight!(fullText || '')
350
+ inflightQueries.delete(dedupKey)
351
+ logLatency({
319
352
  timestamp: new Date().toISOString(),
320
353
  query: query.slice(0, 50),
321
354
  ttfb_ms: actualTtfbMs,
@@ -325,35 +358,42 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
325
358
  contextInjected: /\b(schedule|calendar|meeting|task|tasks|today|tomorrow|next meeting|who do i meet|what's next)\b/i.test(query),
326
359
  cacheHit: false,
327
360
  stream_requested: true,
328
- })
329
- // Final chunk with finish_reason
330
- const finalChunk = {
361
+ })
362
+ // Final chunk with finish_reason
363
+ const finalChunk = {
331
364
  id: completionId,
332
365
  object: 'chat.completion.chunk',
333
366
  created: timestamp,
334
367
  model: responseModel,
335
368
  choices: [{ index: 0, delta: {}, finish_reason: 'stop' }],
369
+ }
370
+ res.write(`data: ${JSON.stringify(finalChunk)}\n\n`)
371
+ res.write('data: [DONE]\n\n')
372
+ res.end()
336
373
  }
337
- res.write(`data: ${JSON.stringify(finalChunk)}\n\n`)
338
- res.write('data: [DONE]\n\n')
339
- res.end()
374
+ } finally {
375
+ maintenanceLease.release()
340
376
  }
341
377
  },
342
378
  onError: (error) => {
343
- if (!done) {
344
- done = true
345
- rejectInflight!(new Error(error))
346
- inflightQueries.delete(dedupKey)
347
- const errChunk = {
379
+ try {
380
+ if (!done) {
381
+ done = true
382
+ rejectInflight!(new Error(error))
383
+ inflightQueries.delete(dedupKey)
384
+ const errChunk = {
348
385
  id: completionId,
349
386
  object: 'chat.completion.chunk',
350
387
  created: timestamp,
351
388
  model: responseModel,
352
389
  choices: [{ index: 0, delta: { content: `Error: ${error}` }, finish_reason: 'stop' }],
390
+ }
391
+ res.write(`data: ${JSON.stringify(errChunk)}\n\n`)
392
+ res.write('data: [DONE]\n\n')
393
+ res.end()
353
394
  }
354
- res.write(`data: ${JSON.stringify(errChunk)}\n\n`)
355
- res.write('data: [DONE]\n\n')
356
- res.end()
395
+ } finally {
396
+ maintenanceLease.release()
357
397
  }
358
398
  },
359
399
  onToolStatus: (status) => {
@@ -367,6 +407,7 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
367
407
  // Persist session ID for multi-turn context on subsequent G2 queries
368
408
  g2SessionId = returnedSid
369
409
  } catch (err: any) {
410
+ maintenanceLease.release()
370
411
  if (!done) {
371
412
  done = true
372
413
  rejectInflight!(err)
@@ -399,6 +440,7 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
399
440
  const fail = (error: unknown) => {
400
441
  if (settled) return
401
442
  settled = true
443
+ maintenanceLease.release()
402
444
  const err = error instanceof Error ? error : new Error(String(error))
403
445
  rejectInflight!(err)
404
446
  inflightQueries.delete(dedupKey)
@@ -412,21 +454,25 @@ openaiCompatRouter.post('/v1/chat/completions', async (req, res) => {
412
454
  onDone: (fullText) => {
413
455
  if (settled) return
414
456
  settled = true
415
- const text = fullText || result
416
- resolveInflight!(text)
417
- inflightQueries.delete(dedupKey)
418
- logLatency({
419
- timestamp: new Date().toISOString(),
420
- query: query.slice(0, 50),
421
- ttfb_ms: nsFirstChunkMs,
422
- total_ms: Date.now() - requestReceivedAt,
423
- model: resolvedModel,
424
- resumed: !!currentSessionIdNS,
425
- contextInjected: /\b(schedule|calendar|meeting|task|tasks|today|tomorrow)\b/i.test(query),
426
- cacheHit: false,
427
- stream_requested: false,
428
- })
429
- resolve(text)
457
+ try {
458
+ const text = fullText || result
459
+ resolveInflight!(text)
460
+ inflightQueries.delete(dedupKey)
461
+ logLatency({
462
+ timestamp: new Date().toISOString(),
463
+ query: query.slice(0, 50),
464
+ ttfb_ms: nsFirstChunkMs,
465
+ total_ms: Date.now() - requestReceivedAt,
466
+ model: resolvedModel,
467
+ resumed: !!currentSessionIdNS,
468
+ contextInjected: /\b(schedule|calendar|meeting|task|tasks|today|tomorrow)\b/i.test(query),
469
+ cacheHit: false,
470
+ stream_requested: false,
471
+ })
472
+ resolve(text)
473
+ } finally {
474
+ maintenanceLease.release()
475
+ }
430
476
  },
431
477
  onError: (error) => {
432
478
  fail(new Error(error))
@@ -39,6 +39,13 @@ import { logTokenAudit } from '../lib/token-audit.js'
39
39
  import { atomicWriteFileSync } from '../lib/atomic-fs.js'
40
40
  import { dataPath } from '../lib/data-dir.js'
41
41
  import { emitDisplay } from '../lib/display-bus.js'
42
+ import {
43
+ acquireMaintenanceWork,
44
+ maintenanceAdmissionsOpen,
45
+ MaintenanceLifecycleError,
46
+ maintenanceErrorPayload,
47
+ type MaintenanceWorkLease,
48
+ } from '../lib/maintenance-lifecycle.js'
42
49
 
43
50
  export const promptDraftsRouter = Router()
44
51
 
@@ -138,6 +145,12 @@ function sanitizeTranscript(draftId: string, text: string, learnInline = true):
138
145
  }
139
146
 
140
147
  async function sendDraftError(res: Response, draftId: string, err: any): Promise<void> {
148
+ // Maintenance rejection is an admission result, not a draft failure. Keep
149
+ // the recoverable draft untouched so the phone can retry after maintenance.
150
+ if (err instanceof MaintenanceLifecycleError) {
151
+ if (err.retryAfterSeconds != null) res.setHeader('Retry-After', String(err.retryAfterSeconds))
152
+ return void res.status(err.status).json({ ...maintenanceErrorPayload(err), draftPreserved: true })
153
+ }
141
154
  if (err instanceof NoSpeechDetectedError) return void res.status(204).send()
142
155
  if (err instanceof OpenAIWhisperBudgetExhaustedError) {
143
156
  await markPromptDraftError(draftId, err.message)
@@ -211,31 +224,53 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
211
224
  return { draftId, text: finalText, recovered: true, chunkCount: finalized.receivedChunkIndexes.length, missingChunks: getMissingChunkIndexes(finalized), expiresAt: finalized.expiresAt }
212
225
  }
213
226
 
214
- const prunedAtBoot = prunePromptDrafts()
227
+ const prunedAtBoot = maintenanceAdmissionsOpen() ? prunePromptDrafts() : 0
215
228
  if (prunedAtBoot) console.log(`[prompt-draft] pruned ${prunedAtBoot} expired draft(s)`)
216
- const pruneTimer = setInterval(() => prunePromptDrafts(), 60 * 60 * 1000)
229
+ const pruneTimer = setInterval(() => {
230
+ if (maintenanceAdmissionsOpen()) prunePromptDrafts()
231
+ }, 60 * 60 * 1000)
217
232
  pruneTimer.unref?.()
218
233
 
219
234
  promptDraftsRouter.post('/prompt-drafts/start', (req, res) => {
220
- const requestedId = typeof req.body?.recoveryId === 'string' ? req.body.recoveryId : undefined
221
- const meta = createPromptDraft(requestedId)
222
- res.json({ draftId: meta.draftId, recoveryId: requestedId ?? meta.draftId, remapped: Boolean(requestedId && requestedId !== meta.draftId), expiresAt: meta.expiresAt, status: meta.status })
235
+ let maintenanceLease: MaintenanceWorkLease | undefined
236
+ try {
237
+ maintenanceLease = acquireMaintenanceWork('prompt_draft_write')
238
+ const requestedId = typeof req.body?.recoveryId === 'string' ? req.body.recoveryId : undefined
239
+ const meta = createPromptDraft(requestedId)
240
+ res.json({ draftId: meta.draftId, recoveryId: requestedId ?? meta.draftId, remapped: Boolean(requestedId && requestedId !== meta.draftId), expiresAt: meta.expiresAt, status: meta.status })
241
+ } catch (err: any) {
242
+ if (err instanceof MaintenanceLifecycleError) {
243
+ if (err.retryAfterSeconds != null) res.setHeader('Retry-After', String(err.retryAfterSeconds))
244
+ res.status(err.status).json(maintenanceErrorPayload(err))
245
+ return
246
+ }
247
+ res.status(err.status ?? 500).json({ error: err.message })
248
+ } finally {
249
+ maintenanceLease?.release()
250
+ }
223
251
  })
224
252
 
225
253
  promptDraftsRouter.post('/prompt-drafts/:draftId/chunks', async (req, res) => {
254
+ let maintenanceLease: MaintenanceWorkLease | undefined
226
255
  try {
227
256
  const raw = Array.isArray(req.query.chunkIndex) ? req.query.chunkIndex[0] : req.query.chunkIndex
228
257
  const chunkIndex = Number(raw)
229
258
  if (!Number.isInteger(chunkIndex) || chunkIndex < 0 || chunkIndex >= MAX_CHUNKS) return res.status(400).json({ error: 'invalid chunkIndex' })
230
- const audio = await readRawBody(req)
231
- if (audio.length < 44) return res.status(400).json({ error: 'audio too short' })
232
259
  const before = loadPromptDraftMeta(req.params.draftId)
233
260
  if (!before) return res.status(404).json({ error: 'draft not found' })
261
+ maintenanceLease = acquireMaintenanceWork('prompt_draft_write')
262
+ const audio = await readRawBody(req)
263
+ if (audio.length < 44) return res.status(400).json({ error: 'audio too short' })
234
264
  const existingBytes = before.chunkBytes[String(chunkIndex)] ?? 0
235
265
  const nextTotal = Object.values(before.chunkBytes).reduce((sum, bytes) => sum + bytes, 0) - existingBytes + audio.length
236
266
  if (nextTotal > MAX_DRAFT_BYTES) return res.status(413).json({ error: 'prompt draft too large' })
237
267
  const meta = await savePromptDraftChunk(req.params.draftId, chunkIndex, audio)
268
+ const warmLease = acquireMaintenanceWork('prompt_draft_warm', {
269
+ allowDuringDrain: true,
270
+ phase: 'queued',
271
+ })
238
272
  warmTail = warmTail.then(async () => {
273
+ warmLease.setPhase('active')
239
274
  const text = await transcribeChunk(req.params.draftId, chunkIndex, audio, 'fast', 'warm')
240
275
  // The durability ACK above remains immediate. Publish the optional warm
241
276
  // transcript only after rechecking that this exact audio still owns the
@@ -247,10 +282,16 @@ promptDraftsRouter.post('/prompt-drafts/:draftId/chunks', async (req, res) => {
247
282
  })
248
283
  }).catch(err => {
249
284
  console.warn(`[prompt-draft] warm transcription failed ${req.params.draftId}/${chunkIndex}: ${err.message}`)
250
- })
285
+ }).finally(() => warmLease.release())
251
286
  res.json({ draftId: meta.draftId, chunkIndex, acked: true, receivedChunkIndexes: meta.receivedChunkIndexes, chunkBytes: meta.chunkBytes[String(chunkIndex)] ?? audio.length, transcriptPending: true, expiresAt: meta.expiresAt })
252
287
  } catch (err: any) {
288
+ if (err instanceof MaintenanceLifecycleError) {
289
+ if (err.retryAfterSeconds != null) res.setHeader('Retry-After', String(err.retryAfterSeconds))
290
+ return void res.status(err.status).json(maintenanceErrorPayload(err))
291
+ }
253
292
  res.status(err.status ?? (err.message === 'draft not found' ? 404 : 500)).json({ error: err.message })
293
+ } finally {
294
+ maintenanceLease?.release()
254
295
  }
255
296
  })
256
297
 
@@ -262,7 +303,9 @@ async function finalizeRequest(req: any, res: Response): Promise<void> {
262
303
  const key = `${req.params.draftId}:${mode}`
263
304
  let job = finalizeJobs.get(key)
264
305
  if (!job) {
306
+ const finalizeLease = acquireMaintenanceWork('prompt_draft_finalize')
265
307
  job = finalizeDraft(req.params.draftId, mode, routeAutoClean(req), abort.signal)
308
+ .finally(() => finalizeLease.release())
266
309
  finalizeJobs.set(key, job)
267
310
  job.finally(() => finalizeJobs.delete(key)).catch(() => {})
268
311
  }
@@ -0,0 +1,33 @@
1
+ import { Router } from 'express'
2
+ import {
3
+ PromptEditValidationError,
4
+ applyPromptEdit,
5
+ normalizePromptEditInput,
6
+ } from '../lib/prompt-edit.js'
7
+ import { errMsg } from '../lib/utils.js'
8
+
9
+ export const promptEditRouter = Router()
10
+
11
+ promptEditRouter.post('/prompt-edit', async (req, res) => {
12
+ const abort = new AbortController()
13
+ // Abort the model spawn ONLY if the client disconnects before we respond.
14
+ // Must listen on `res`, not `req`: express.json() fully drains the request
15
+ // body stream before this handler runs, so `req` emits 'close' on the next
16
+ // tick and would abort EVERY edit mid-flight (instant 500 "Prompt edit
17
+ // aborted"). `res` 'close' + !writableEnded fires only on a genuine premature
18
+ // disconnect. (House pattern: see routes/query.ts.)
19
+ res.on('close', () => {
20
+ if (!res.writableEnded) abort.abort()
21
+ })
22
+ try {
23
+ const input = normalizePromptEditInput(req.body)
24
+ const revisedText = await applyPromptEdit(input, abort.signal)
25
+ res.json({ revisedText })
26
+ } catch (err) {
27
+ if (err instanceof PromptEditValidationError) {
28
+ res.status(err.status).json({ error: err.message })
29
+ return
30
+ }
31
+ res.status(500).json({ error: errMsg(err) })
32
+ }
33
+ })
@@ -9,6 +9,11 @@ import { normalizeEffortPreference, normalizeModelPreference } from '../../share
9
9
  import { QueryAttachmentError, resolveQueryAttachments } from '../lib/query-attachments.js'
10
10
  import { getMediaStore } from '../lib/media-store.js'
11
11
  import { mergeMediaAttachmentRefs } from '../../shared/media-attachment.js'
12
+ import {
13
+ acquireMaintenanceWork,
14
+ MaintenanceLifecycleError,
15
+ maintenanceErrorPayload,
16
+ } from '../lib/maintenance-lifecycle.js'
12
17
 
13
18
  const TOOL_STATUS_MESSAGES: Record<string, string> = {
14
19
  WebSearch: 'Searching web...',
@@ -19,6 +24,16 @@ const TOOL_STATUS_MESSAGES: Record<string, string> = {
19
24
  export const queryRouter = Router()
20
25
 
21
26
  queryRouter.post('/query', async (req, res) => {
27
+ let maintenanceLease
28
+ try {
29
+ maintenanceLease = acquireMaintenanceWork('legacy_query')
30
+ } catch (error) {
31
+ if (error instanceof MaintenanceLifecycleError) {
32
+ if (error.retryAfterSeconds != null) res.setHeader('Retry-After', String(error.retryAfterSeconds))
33
+ return res.status(error.status).json(maintenanceErrorPayload(error))
34
+ }
35
+ throw error
36
+ }
22
37
  const { query, sessionId, model, effort, reference, globalMsgNum } = req.body
23
38
  const activityToolMode = req.body.activityToolMode === 'off' || req.body.activityToolMode === 'preview'
24
39
  ? req.body.activityToolMode
@@ -31,8 +46,10 @@ queryRouter.post('/query', async (req, res) => {
31
46
  resolvedAttachments = await resolveQueryAttachments(req.body)
32
47
  } catch (err) {
33
48
  if (err instanceof QueryAttachmentError) {
49
+ maintenanceLease.release()
34
50
  return res.status(err.status).json({ error: err.code, detail: err.message })
35
51
  }
52
+ maintenanceLease.release()
36
53
  return res.status(500).json({ error: errMsg(err) })
37
54
  }
38
55
  const imageInputs = resolvedAttachments.inputs.length > 0 ? resolvedAttachments.inputs : undefined
@@ -42,6 +59,7 @@ queryRouter.post('/query', async (req, res) => {
42
59
 
43
60
  // Vision queries can have an empty query (default to "describe what you see")
44
61
  if ((!resolvedQuery || typeof resolvedQuery !== 'string') && !imageInputs) {
62
+ maintenanceLease.release()
45
63
  return res.status(400).json({ error: 'query string or image required' })
46
64
  }
47
65
 
@@ -107,34 +125,43 @@ queryRouter.post('/query', async (req, res) => {
107
125
  }
108
126
  },
109
127
  } : {}),
110
- onDone: (fullText, model, cliSessionId, metadata) => {
111
- // Durable association is independent of the SSE socket. Backgrounding
112
- // the phone cannot leave request media reserved until expiry.
113
- if (resolvedAttachments.ids.length > 0) {
114
- getMediaStore().associate(resolvedAttachments.ids, {
115
- sessionId: sid,
116
- ...(validGlobalMsgNum ? { globalMsgNum: validGlobalMsgNum } : {}),
117
- }).catch((err) => console.error('[query] attachment association failed:', err))
118
- }
119
- if (!done) {
120
- done = true
121
- const attachments = mergeMediaAttachmentRefs(attachmentRefs, metadata?.outputAttachments)
122
- const { outputAttachments: _outputAttachments, ...runMetadata } = metadata ?? {}
123
- const payload = {
124
- text: fullText, sessionId: sid, model, cliSessionId, ...runMetadata,
125
- ...(attachments.length > 0 ? { attachments } : {}),
128
+ onDone: async (fullText, model, cliSessionId, metadata) => {
129
+ try {
130
+ // Durable association is independent of the SSE socket.
131
+ // Backgrounding the phone cannot leave request media reserved until
132
+ // expiry or make maintenance proof outrun the pending write.
133
+ if (resolvedAttachments.ids.length > 0) {
134
+ await getMediaStore().associate(resolvedAttachments.ids, {
135
+ sessionId: sid,
136
+ ...(validGlobalMsgNum ? { globalMsgNum: validGlobalMsgNum } : {}),
137
+ }).catch((err) => console.error('[query] attachment association failed:', err))
126
138
  }
127
- res.write(`event: done\ndata: ${JSON.stringify(payload)}\n\n`)
128
- emitDisplay({ type: 'done', data: payload })
129
- res.end()
139
+ if (!done) {
140
+ done = true
141
+ const attachments = mergeMediaAttachmentRefs(attachmentRefs, metadata?.outputAttachments)
142
+ const { outputAttachments: _outputAttachments, ...runMetadata } = metadata ?? {}
143
+ const payload = {
144
+ text: fullText, sessionId: sid, model, cliSessionId, ...runMetadata,
145
+ ...(attachments.length > 0 ? { attachments } : {}),
146
+ }
147
+ res.write(`event: done\ndata: ${JSON.stringify(payload)}\n\n`)
148
+ emitDisplay({ type: 'done', data: payload })
149
+ res.end()
150
+ }
151
+ } finally {
152
+ maintenanceLease.release()
130
153
  }
131
154
  },
132
155
  onError: (error) => {
133
- if (!done) {
134
- done = true
135
- res.write(`event: error\ndata: ${JSON.stringify({ error })}\n\n`)
136
- emitDisplay({ type: 'error', data: { error } })
137
- res.end()
156
+ try {
157
+ if (!done) {
158
+ done = true
159
+ res.write(`event: error\ndata: ${JSON.stringify({ error })}\n\n`)
160
+ emitDisplay({ type: 'error', data: { error } })
161
+ res.end()
162
+ }
163
+ } finally {
164
+ maintenanceLease.release()
138
165
  }
139
166
  },
140
167
  }, validModel, imageInputs,
@@ -146,6 +173,7 @@ queryRouter.post('/query', async (req, res) => {
146
173
  { abortSignal: abortController.signal, effort: validEffort },
147
174
  )
148
175
  } catch (err: unknown) {
176
+ maintenanceLease.release()
149
177
  if (!done) {
150
178
  done = true
151
179
  res.write(`event: error\ndata: ${JSON.stringify({ error: errMsg(err) })}\n\n`)
@@ -0,0 +1,76 @@
1
+ import { Router } from 'express'
2
+ import { readFileSync } from 'node:fs'
3
+ import { resolve } from 'node:path'
4
+ import { atomicWriteFileSync } from '../lib/atomic-fs.js'
5
+ import { acquireMaintenance, getRecoveryActivityStatus } from '../lib/recovery-activity.js'
6
+ import { getWhisperHealth, restartWhisperServer } from '../lib/whisper-local.js'
7
+ import { serverMetrics } from '../lib/server-metrics.js'
8
+
9
+ export const recoveryRouter = Router()
10
+ const COOLDOWN_MS = 60_000
11
+ const cooldownPath = resolve(import.meta.dirname, '../data/.recovery_restart.json')
12
+
13
+ function lastRestartAt(): number {
14
+ try { return Number(JSON.parse(readFileSync(cooldownPath, 'utf8'))?.at) || 0 } catch { return 0 }
15
+ }
16
+
17
+ recoveryRouter.get('/live', (_req, res) => {
18
+ res.setHeader('Cache-Control', 'no-store')
19
+ res.json({
20
+ status: 'ok', bootId: serverMetrics.bootId, pid: process.pid,
21
+ uptimeSeconds: Math.round((Date.now() - serverMetrics.startedAt) / 1000),
22
+ managed: process.env.COS_HARNESS === 'daemon',
23
+ })
24
+ })
25
+
26
+ recoveryRouter.get('/recovery/status', (_req, res) => {
27
+ res.json({
28
+ bootId: serverMetrics.bootId,
29
+ managed: process.env.COS_HARNESS === 'daemon',
30
+ whisper: getWhisperHealth(),
31
+ asr: { hqActive: false, hqQueued: 0, fastRestarting: false }, // public build: no HQ/fast ASR scheduler in this server
32
+ activity: getRecoveryActivityStatus(),
33
+ })
34
+ })
35
+
36
+ recoveryRouter.post('/recovery/whisper/restart', async (_req, res) => {
37
+ const gate = acquireMaintenance()
38
+ if (!gate.ok) {
39
+ gate.release()
40
+ return res.status(409).json({ error: 'Recovery blocked by active work', reason: 'recovery_busy', busy: gate.busy })
41
+ }
42
+ try {
43
+ const result = await restartWhisperServer()
44
+ res.status(result.status === 'failed' ? 503 : 200).json(result)
45
+ } finally { gate.release() }
46
+ })
47
+
48
+ recoveryRouter.post('/recovery/server/restart', (_req, res) => {
49
+ if (process.env.COS_HARNESS !== 'daemon') {
50
+ return res.status(409).json({ error: 'Server is not managed by the COS LaunchAgent', reason: 'restart_unmanaged' })
51
+ }
52
+ const elapsed = Date.now() - lastRestartAt()
53
+ if (elapsed < COOLDOWN_MS) {
54
+ return res.status(429).json({ error: 'Restart cooldown active', reason: 'restart_cooldown', retryAfterMs: COOLDOWN_MS - elapsed })
55
+ }
56
+ const gate = acquireMaintenance()
57
+ if (!gate.ok) {
58
+ gate.release()
59
+ return res.status(409).json({ error: 'Restart blocked by active work', reason: 'recovery_busy', busy: gate.busy })
60
+ }
61
+ atomicWriteFileSync(cooldownPath, JSON.stringify({ at: Date.now(), bootId: serverMetrics.bootId }))
62
+ res.status(202).json({ accepted: true, oldBootId: serverMetrics.bootId })
63
+ // Schedule independently of the response socket. The phone may change
64
+ // network/close the sheet immediately after receiving 202; that must not
65
+ // cancel an accepted restart or strand maintenance forever.
66
+ const timer = setTimeout(() => {
67
+ if (process.env.COS_DISABLE_SELF_RESTART === '1') { gate.release(); return }
68
+ try {
69
+ process.kill(process.pid, 'SIGTERM')
70
+ } catch (error) {
71
+ gate.release()
72
+ console.error('[recovery] Failed to signal managed server restart:', error)
73
+ }
74
+ }, 350)
75
+ timer.unref?.()
76
+ })