@gotcos/glasses-server 6.21.2 → 6.21.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +12 -6
- package/bin/cli.cjs +1 -1
- package/package.json +1 -1
- package/server/lib/claude-bridge.ts +3 -1
- package/server/lib/codex-bridge.ts +3 -1
- package/server/lib/cursor-bridge.ts +1 -0
- package/server/lib/media-store.ts +53 -1
- package/server/lib/query-job-runtime.ts +1 -0
- package/server/lib/run-output-images.ts +10 -1
- package/server/lib/whisper-local.ts +25 -2
- package/server/lib/whisper-metal-gate.ts +53 -0
- package/server/lib/whisper-preview.ts +94 -42
- package/server/routes/media.ts +4 -0
- package/server/routes/query.ts +2 -0
- package/server/routes/sessions.ts +58 -9
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,27 @@
|
|
|
1
|
+
## 6.21.4
|
|
2
|
+
|
|
3
|
+
- Preserve validated phone-photo references on Claude and Codex conversation
|
|
4
|
+
exchanges so Recent Glasses and message-history clients can recover the
|
|
5
|
+
original visual context instead of receiving a text-only marker.
|
|
6
|
+
- Recover validated refs at read time from the durable media association index,
|
|
7
|
+
keyed by exact session ID, global message number, and message era. Pre-6.21.4
|
|
8
|
+
unversioned refs are recovered only for the active era when both creation and
|
|
9
|
+
association occurred after its boundary; ambiguous historical refs fail
|
|
10
|
+
closed rather than risking the wrong photo.
|
|
11
|
+
|
|
12
|
+
## 6.21.3
|
|
13
|
+
|
|
14
|
+
- Route Max-tier provisional dictation preview through the resident Turbo
|
|
15
|
+
preview sidecar while keeping authoritative live commit and saved-work
|
|
16
|
+
polish on Large-v3. This corrects the 6.21.0 behavior that made cosmetic
|
|
17
|
+
preview pay Large-v3 latency.
|
|
18
|
+
- Give canonical transcription strict GPU priority. A cosmetic preview is
|
|
19
|
+
dropped or aborted when canonical/HQ Metal work begins, and preview failures
|
|
20
|
+
remain outside the Whisper circuit breaker and all persistence paths.
|
|
21
|
+
- Report the effective Max lanes truthfully as Turbo preview, Large-v3 commit,
|
|
22
|
+
and Large-v3 polish. Balanced remains Small.en preview, Turbo commit, and
|
|
23
|
+
Large-v3 polish.
|
|
24
|
+
|
|
1
25
|
## 6.21.2
|
|
2
26
|
|
|
3
27
|
- Cache Python, Claude, Codex, and Cursor process probes for 30 seconds so the
|
package/README.md
CHANGED
|
@@ -112,6 +112,10 @@ range is the exact Tailscale/CGNAT allocation (`100.64.0.0/10`), not all of
|
|
|
112
112
|
authenticated query that requested it
|
|
113
113
|
- Message History + cross-day "reference message N" — your chats are archived by day
|
|
114
114
|
and every message keeps a permanent number you can recall (`/api/archive`, `/api/message/:num`)
|
|
115
|
+
- Recent/history responses preserve validated photo references. Recovery uses
|
|
116
|
+
exact session + global-message + message-era identity without exposing storage
|
|
117
|
+
paths. Ambiguous pre-version historical refs fail closed; unversioned refs are
|
|
118
|
+
recovered only inside the active era when created and associated after its boundary.
|
|
115
119
|
- Send phone photos with queued prompts, and review assistant-selected generated,
|
|
116
120
|
research, or explicitly used email images in Messages and on the G2 lens
|
|
117
121
|
- Recover long voice prompts after phone, network, or server interruptions. Audio
|
|
@@ -257,12 +261,14 @@ update the server remain on Turbo until Guided Setup opts them into Small.en.
|
|
|
257
261
|
npx --yes @gotcos/glasses-server@latest --setup-transcription --transcription-tier max
|
|
258
262
|
```
|
|
259
263
|
|
|
260
|
-
Max
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
is
|
|
265
|
-
|
|
264
|
+
Max keeps Turbo resident in the isolated preview sidecar for low-latency
|
|
265
|
+
provisional words, while Large-v3 remains authoritative for live commit and
|
|
266
|
+
saved-work polish. Canonical transcription has strict GPU priority: a cosmetic
|
|
267
|
+
preview is dropped or aborted instead of competing with a committed decode.
|
|
268
|
+
If Large-v3 is missing, health reports the downgrade and the server falls back
|
|
269
|
+
to Turbo rather than making transcription unavailable. COS Control is the
|
|
270
|
+
supported owner of the machine-wide tier; the per-lane environment variables
|
|
271
|
+
remain advanced overrides.
|
|
266
272
|
|
|
267
273
|
The first server start downloads the real-time turbo model. True HQ additionally
|
|
268
274
|
requires the full `ggml-large-v3.bin` model (about 3.1 GB):
|
package/bin/cli.cjs
CHANGED
|
@@ -379,7 +379,7 @@ if (SETUP_TRANSCRIPTION) {
|
|
|
379
379
|
process.env.COS_WHISPER_PREVIEW_MODEL = previewModel
|
|
380
380
|
process.env.COS_WHISPER_COMMIT_MODEL = commitModel
|
|
381
381
|
const laneSummary = TRANSCRIPTION_TIER === 'max'
|
|
382
|
-
? 'Large-v3
|
|
382
|
+
? 'Turbo preview · Large-v3 commit · Large-v3 HQ'
|
|
383
383
|
: 'Small.en preview · Turbo commit · Large-v3 HQ'
|
|
384
384
|
console.log(green(' ✓') + ` ${TRANSCRIPTION_TIER === 'max' ? 'Max' : 'Balanced'} transcription selected ` + dim(`— ${laneSummary}`))
|
|
385
385
|
}
|
package/package.json
CHANGED
|
@@ -428,6 +428,7 @@ export async function callClaudeStreaming(
|
|
|
428
428
|
outputImagePublisher = createRunOutputImagePublisher({
|
|
429
429
|
sessionId: sid,
|
|
430
430
|
globalMsgNum,
|
|
431
|
+
messageEra: options?.messageEra,
|
|
431
432
|
maxImages: outputImageBudget,
|
|
432
433
|
})
|
|
433
434
|
} catch (err) {
|
|
@@ -473,6 +474,7 @@ export async function callClaudeStreaming(
|
|
|
473
474
|
// Record user message (with [Photo]/[N Photos] prefix for vision queries)
|
|
474
475
|
const photoPrefix = imagePaths.length === 1 ? '[Photo]' : imagePaths.length > 1 ? `[${imagePaths.length} Photos]` : ''
|
|
475
476
|
const historyQuery = photoPrefix ? `${photoPrefix} ${query || 'What do you see?'}` : query
|
|
477
|
+
const inboundAttachments = imageInputs.length > 0 ? imageInputs.map(input => input.attachment) : undefined
|
|
476
478
|
const exchangeProvenance = {
|
|
477
479
|
clientJobId: options?.clientJobId,
|
|
478
480
|
generation: options?.generation,
|
|
@@ -482,7 +484,7 @@ export async function callClaudeStreaming(
|
|
|
482
484
|
'user',
|
|
483
485
|
historyQuery,
|
|
484
486
|
globalMsgNum,
|
|
485
|
-
|
|
487
|
+
inboundAttachments,
|
|
486
488
|
exchangeProvenance,
|
|
487
489
|
resolvedModel,
|
|
488
490
|
)
|
|
@@ -321,6 +321,7 @@ export async function callCodexStreaming(
|
|
|
321
321
|
sessionId: sid,
|
|
322
322
|
globalMsgNum,
|
|
323
323
|
runId: run.runId,
|
|
324
|
+
messageEra: options?.messageEra,
|
|
324
325
|
maxImages: outputImageBudget,
|
|
325
326
|
})
|
|
326
327
|
} catch (err) {
|
|
@@ -363,6 +364,7 @@ export async function callCodexStreaming(
|
|
|
363
364
|
const isFirstQuery = isNewSession(sid)
|
|
364
365
|
const photoPrefix = imagePaths.length === 1 ? '[Photo]' : imagePaths.length > 1 ? `[${imagePaths.length} Photos]` : ''
|
|
365
366
|
const historyQuery = photoPrefix ? `${photoPrefix} ${query || 'What do you see?'}` : query
|
|
367
|
+
const inboundAttachments = imageInputs.length > 0 ? imageInputs.map(input => input.attachment) : undefined
|
|
366
368
|
const exchangeProvenance = {
|
|
367
369
|
clientJobId: options?.clientJobId,
|
|
368
370
|
generation: options?.generation,
|
|
@@ -372,7 +374,7 @@ export async function callCodexStreaming(
|
|
|
372
374
|
'user',
|
|
373
375
|
historyQuery,
|
|
374
376
|
globalMsgNum,
|
|
375
|
-
|
|
377
|
+
inboundAttachments,
|
|
376
378
|
exchangeProvenance,
|
|
377
379
|
model,
|
|
378
380
|
)
|
|
@@ -33,6 +33,7 @@ import { atomicWriteFileSync, loadJsonOrQuarantine } from './atomic-fs.js'
|
|
|
33
33
|
import { dataPath } from './data-dir.js'
|
|
34
34
|
import {
|
|
35
35
|
isValidMediaId,
|
|
36
|
+
mergeMediaAttachmentRefs,
|
|
36
37
|
parseMediaAttachmentRef,
|
|
37
38
|
type MediaAttachmentRef,
|
|
38
39
|
type MediaKind,
|
|
@@ -131,6 +132,7 @@ export interface MediaRecord {
|
|
|
131
132
|
clientQueueItemId?: string
|
|
132
133
|
runId?: string
|
|
133
134
|
globalMsgNum?: number
|
|
135
|
+
messageEra?: string
|
|
134
136
|
createdAtMs: number
|
|
135
137
|
updatedAtMs: number
|
|
136
138
|
reservedAtMs?: number
|
|
@@ -197,6 +199,9 @@ function sanitizeRecord(raw: unknown): MediaRecord | null {
|
|
|
197
199
|
...(typeof r.clientQueueItemId === 'string' ? { clientQueueItemId: r.clientQueueItemId } : {}),
|
|
198
200
|
...(typeof r.runId === 'string' ? { runId: r.runId } : {}),
|
|
199
201
|
...(typeof r.globalMsgNum === 'number' ? { globalMsgNum: r.globalMsgNum } : {}),
|
|
202
|
+
...(typeof r.messageEra === 'string' && /^[a-z0-9][a-z0-9._-]{0,79}$/i.test(r.messageEra)
|
|
203
|
+
? { messageEra: r.messageEra }
|
|
204
|
+
: {}),
|
|
200
205
|
createdAtMs: typeof r.createdAtMs === 'number' ? r.createdAtMs : Date.now(),
|
|
201
206
|
updatedAtMs: typeof r.updatedAtMs === 'number' ? r.updatedAtMs : Date.now(),
|
|
202
207
|
...(typeof r.reservedAtMs === 'number' ? { reservedAtMs: r.reservedAtMs } : {}),
|
|
@@ -464,6 +469,51 @@ export class MediaStore {
|
|
|
464
469
|
return this.getRecord(id)?.ref ?? null
|
|
465
470
|
}
|
|
466
471
|
|
|
472
|
+
/** Recover the public refs associated with one exact conversation turn.
|
|
473
|
+
*
|
|
474
|
+
* The media index is already the durable lifecycle authority for uploaded
|
|
475
|
+
* and generated images. Older/legacy query paths associated the media here
|
|
476
|
+
* but failed to copy the refs onto the conversation exchange. Readers may
|
|
477
|
+
* use this exact session + message key to repair that projection without
|
|
478
|
+
* exposing storage paths or inventing a second attachment database.
|
|
479
|
+
*/
|
|
480
|
+
getAssociatedRefs(target: {
|
|
481
|
+
sessionId: string
|
|
482
|
+
globalMsgNum: number
|
|
483
|
+
messageEra?: string
|
|
484
|
+
activeMessageEra: string
|
|
485
|
+
activeEraStartedAt: number
|
|
486
|
+
}): MediaAttachmentRef[] {
|
|
487
|
+
if (!target.sessionId || !Number.isSafeInteger(target.globalMsgNum) || target.globalMsgNum < 1) return []
|
|
488
|
+
const validEra = (value: unknown): value is string =>
|
|
489
|
+
typeof value === 'string' && /^[a-z0-9][a-z0-9._-]{0,79}$/i.test(value)
|
|
490
|
+
if (!validEra(target.activeMessageEra) || !Number.isFinite(target.activeEraStartedAt)) return []
|
|
491
|
+
const targetEra = validEra(target.messageEra) ? target.messageEra : undefined
|
|
492
|
+
const eraMatches = (rec: MediaRecord): boolean => {
|
|
493
|
+
if (rec.messageEra) return targetEra != null && rec.messageEra === targetEra
|
|
494
|
+
|
|
495
|
+
// Pre-6.21.4 associations did not persist an era. Recover them only for
|
|
496
|
+
// the active era when both creation and association are on/after its
|
|
497
|
+
// boundary. Requiring both closes the race where a pre-reset upload was
|
|
498
|
+
// associated only after the reset. Anything older may belong to legacy
|
|
499
|
+
// or any prior named era and fails closed.
|
|
500
|
+
const associatedAt = rec.associatedAtMs ?? rec.createdAtMs
|
|
501
|
+
if (target.activeMessageEra === 'legacy') return targetEra == null || targetEra === 'legacy'
|
|
502
|
+
if (targetEra === target.activeMessageEra) {
|
|
503
|
+
return rec.createdAtMs >= target.activeEraStartedAt && associatedAt >= target.activeEraStartedAt
|
|
504
|
+
}
|
|
505
|
+
return false
|
|
506
|
+
}
|
|
507
|
+
const matching = [...this.records.values()]
|
|
508
|
+
.filter(rec => rec.lifecycle === 'associated'
|
|
509
|
+
&& rec.sessionId === target.sessionId
|
|
510
|
+
&& rec.globalMsgNum === target.globalMsgNum
|
|
511
|
+
&& eraMatches(rec))
|
|
512
|
+
.sort((a, b) => a.createdAtMs - b.createdAtMs)
|
|
513
|
+
.map(rec => rec.ref)
|
|
514
|
+
return mergeMediaAttachmentRefs(matching)
|
|
515
|
+
}
|
|
516
|
+
|
|
467
517
|
/** Resolve content for serving/model input, honoring lifecycle + TTLs. */
|
|
468
518
|
getContent(id: string, variant: 'phone' | 'thumb' | 'g2' = 'phone'): MediaContentResult {
|
|
469
519
|
const rec = this.getRecord(id)
|
|
@@ -667,7 +717,7 @@ export class MediaStore {
|
|
|
667
717
|
|
|
668
718
|
/** Bind media to its final run/message. Safe to replay; wins over a
|
|
669
719
|
* delayed release. */
|
|
670
|
-
associate(ids: string[], target: { sessionId?: string; runId?: string; globalMsgNum?: number }): Promise<void> {
|
|
720
|
+
associate(ids: string[], target: { sessionId?: string; runId?: string; globalMsgNum?: number; messageEra?: string }): Promise<void> {
|
|
671
721
|
return this.withLock(() => {
|
|
672
722
|
const now = Date.now()
|
|
673
723
|
let dirty = false
|
|
@@ -684,6 +734,8 @@ export class MediaStore {
|
|
|
684
734
|
if (target.sessionId && rec.sessionId !== target.sessionId) { rec.sessionId = target.sessionId; recDirty = true }
|
|
685
735
|
if (target.runId && rec.runId !== target.runId) { rec.runId = target.runId; recDirty = true }
|
|
686
736
|
if (target.globalMsgNum != null && rec.globalMsgNum !== target.globalMsgNum) { rec.globalMsgNum = target.globalMsgNum; recDirty = true }
|
|
737
|
+
if (target.messageEra && /^[a-z0-9][a-z0-9._-]{0,79}$/i.test(target.messageEra)
|
|
738
|
+
&& rec.messageEra !== target.messageEra) { rec.messageEra = target.messageEra; recDirty = true }
|
|
687
739
|
if (recDirty) {
|
|
688
740
|
rec.updatedAtMs = now
|
|
689
741
|
dirty = true
|
|
@@ -227,6 +227,7 @@ const runner: QueryJobRunner = async ({ jobId, turnId, request, signal, callback
|
|
|
227
227
|
await getMediaStore().associate(resolvedAttachments.ids, {
|
|
228
228
|
sessionId: request.sessionId,
|
|
229
229
|
...(request.globalMsgNum ? { globalMsgNum: request.globalMsgNum } : {}),
|
|
230
|
+
messageEra: request.messageEra,
|
|
230
231
|
}).catch(error => console.error('[query-jobs] attachment association failed:', error))
|
|
231
232
|
}
|
|
232
233
|
const { outputAttachments: _outputAttachments, ...runMetadata } = metadata ?? {}
|
|
@@ -41,6 +41,7 @@ export interface RunOutputImageTarget {
|
|
|
41
41
|
sessionId: string
|
|
42
42
|
globalMsgNum?: number
|
|
43
43
|
runId?: string
|
|
44
|
+
messageEra?: string
|
|
44
45
|
}
|
|
45
46
|
|
|
46
47
|
export interface CreateRunOutputImagePublisherOptions extends RunOutputImageTarget {
|
|
@@ -223,7 +224,15 @@ function safeTarget(options: CreateRunOutputImagePublisherOptions): RunOutputIma
|
|
|
223
224
|
const runId = typeof options.runId === 'string' && options.runId.trim()
|
|
224
225
|
? options.runId.trim().slice(0, 120)
|
|
225
226
|
: undefined
|
|
226
|
-
|
|
227
|
+
const messageEra = typeof options.messageEra === 'string' && /^[a-z0-9][a-z0-9._-]{0,79}$/i.test(options.messageEra)
|
|
228
|
+
? options.messageEra
|
|
229
|
+
: undefined
|
|
230
|
+
return {
|
|
231
|
+
sessionId,
|
|
232
|
+
...(globalMsgNum ? { globalMsgNum } : {}),
|
|
233
|
+
...(runId ? { runId } : {}),
|
|
234
|
+
...(messageEra ? { messageEra } : {}),
|
|
235
|
+
}
|
|
227
236
|
}
|
|
228
237
|
|
|
229
238
|
function assertTempRoot(rawRoot: string): string {
|
|
@@ -16,9 +16,12 @@ import { getVocabulary, getOwnerName, getWhisperCorrections } from './profile.js
|
|
|
16
16
|
import { stripBrandUrls } from './hallucination-filter.js'
|
|
17
17
|
import {
|
|
18
18
|
batchHqMetalEnabled,
|
|
19
|
+
beginCanonicalMetal,
|
|
19
20
|
chooseBatchDevice,
|
|
20
21
|
MetalBatchPreemptedError,
|
|
22
|
+
MetalPreviewContendedError,
|
|
21
23
|
registerMetalBatchChild,
|
|
24
|
+
tryAcquireMetalPreview,
|
|
22
25
|
unregisterMetalBatchChild,
|
|
23
26
|
} from './whisper-metal-gate.js'
|
|
24
27
|
|
|
@@ -1026,6 +1029,7 @@ async function transcribeViaServer(
|
|
|
1026
1029
|
context?: string,
|
|
1027
1030
|
isQuiet?: boolean,
|
|
1028
1031
|
promptPolicy: 'full-vocabulary' | 'none' = 'full-vocabulary',
|
|
1032
|
+
signal?: AbortSignal,
|
|
1029
1033
|
): Promise<{ text: string; words?: WhisperWord[] }> {
|
|
1030
1034
|
const formData = new FormData()
|
|
1031
1035
|
// Convert Buffer to Uint8Array to satisfy Blob's BlobPart type constraint
|
|
@@ -1043,10 +1047,11 @@ async function transcribeViaServer(
|
|
|
1043
1047
|
// legitimate speech from quiet sources (laptop speakers through G2 mic).
|
|
1044
1048
|
formData.append('suppress_non_speech', 'true') // Suppress special/non-speech tokens (benign)
|
|
1045
1049
|
|
|
1050
|
+
const timeoutSignal = AbortSignal.timeout(10_000)
|
|
1046
1051
|
const response = await fetch(`${WHISPER_SERVER_URL}/inference`, {
|
|
1047
1052
|
method: 'POST',
|
|
1048
1053
|
body: formData,
|
|
1049
|
-
signal: AbortSignal.
|
|
1054
|
+
signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal,
|
|
1050
1055
|
})
|
|
1051
1056
|
|
|
1052
1057
|
if (!response.ok) {
|
|
@@ -1183,6 +1188,7 @@ export async function transcribeLocal(
|
|
|
1183
1188
|
opts?: {
|
|
1184
1189
|
affectsCircuit?: boolean
|
|
1185
1190
|
promptPolicy?: 'full-vocabulary' | 'none'
|
|
1191
|
+
metalPriority?: 'canonical' | 'preview'
|
|
1186
1192
|
},
|
|
1187
1193
|
): Promise<{ text: string; backend: 'server' | 'cli'; words?: WhisperWord[] }> {
|
|
1188
1194
|
const start = Date.now()
|
|
@@ -1198,8 +1204,21 @@ export async function transcribeLocal(
|
|
|
1198
1204
|
|
|
1199
1205
|
// Try whisper-server first (fastest: ~50-100ms, includes DTW word timestamps)
|
|
1200
1206
|
if (serverAvailable) {
|
|
1207
|
+
const previewLease = opts?.metalPriority === 'preview' ? tryAcquireMetalPreview() : null
|
|
1208
|
+
if (opts?.metalPriority === 'preview' && !previewLease) {
|
|
1209
|
+
throw new MetalPreviewContendedError()
|
|
1210
|
+
}
|
|
1211
|
+
const releaseCanonical = opts?.metalPriority === 'preview'
|
|
1212
|
+
? null
|
|
1213
|
+
: beginCanonicalMetal('whisper_server')
|
|
1201
1214
|
try {
|
|
1202
|
-
const result = await transcribeViaServer(
|
|
1215
|
+
const result = await transcribeViaServer(
|
|
1216
|
+
audioBuffer,
|
|
1217
|
+
context,
|
|
1218
|
+
isQuiet,
|
|
1219
|
+
opts?.promptPolicy,
|
|
1220
|
+
previewLease?.signal,
|
|
1221
|
+
)
|
|
1203
1222
|
const text = applyCorrections(result.text)
|
|
1204
1223
|
const words = result.words?.map(w => ({ ...w, word: applyCorrections(w.word) }))
|
|
1205
1224
|
const elapsed = Date.now() - start
|
|
@@ -1211,6 +1230,7 @@ export async function transcribeLocal(
|
|
|
1211
1230
|
console.log(`[whisper-local] Server transcribed in ${elapsed}ms (${words?.length ?? 0} words): "${text.slice(0, 80)}${text.length > 80 ? '...' : ''}"`)
|
|
1212
1231
|
return { text, backend: 'server', words }
|
|
1213
1232
|
} catch (err: any) {
|
|
1233
|
+
if (previewLease?.signal.aborted) throw new MetalPreviewContendedError()
|
|
1214
1234
|
if (affectsCircuit) {
|
|
1215
1235
|
serverConsecutiveFailures++
|
|
1216
1236
|
const isTimeout = err.message.includes('timeout') || err.message.includes('aborted')
|
|
@@ -1239,6 +1259,9 @@ export async function transcribeLocal(
|
|
|
1239
1259
|
|
|
1240
1260
|
// Throw to let caller fall to OpenAI cloud (1-3s) — much faster than CLI cold-start (11s)
|
|
1241
1261
|
throw new Error(`whisper-server unavailable: ${err.message}`)
|
|
1262
|
+
} finally {
|
|
1263
|
+
previewLease?.release()
|
|
1264
|
+
releaseCanonical?.()
|
|
1242
1265
|
}
|
|
1243
1266
|
}
|
|
1244
1267
|
|
|
@@ -39,6 +39,7 @@ export type ContentionReason =
|
|
|
39
39
|
| 'prompt_draft_warm'
|
|
40
40
|
| 'prompt_draft_finalize'
|
|
41
41
|
| 'metal_batch_in_flight'
|
|
42
|
+
| 'canonical_in_flight'
|
|
42
43
|
|
|
43
44
|
export type DeviceReason =
|
|
44
45
|
| 'force_cpu'
|
|
@@ -72,6 +73,8 @@ export function batchHqMetalEnabled(): boolean {
|
|
|
72
73
|
type LiveActivityProbe = () => number | null
|
|
73
74
|
|
|
74
75
|
let liveActivityProbe: LiveActivityProbe | null = null
|
|
76
|
+
let canonicalMetalRequests = 0
|
|
77
|
+
const previewMetalRequests = new Set<AbortController>()
|
|
75
78
|
|
|
76
79
|
/** transcribe-stream calls this at module load. Returns the most recent
|
|
77
80
|
* lastActivityAt across in-memory sessions, or null when there are none. */
|
|
@@ -83,6 +86,9 @@ export function registerLiveActivityProbe(probe: LiveActivityProbe): void {
|
|
|
83
86
|
export function resetMetalGateForTests(): void {
|
|
84
87
|
liveActivityProbe = null
|
|
85
88
|
metalChildren.clear()
|
|
89
|
+
canonicalMetalRequests = 0
|
|
90
|
+
for (const controller of previewMetalRequests) controller.abort('test_reset')
|
|
91
|
+
previewMetalRequests.clear()
|
|
86
92
|
}
|
|
87
93
|
|
|
88
94
|
function recentSessionActivity(now: number): boolean {
|
|
@@ -115,6 +121,7 @@ function activeMetalFamilyWork(): ContentionReason | null {
|
|
|
115
121
|
|
|
116
122
|
/** Is something live currently entitled to Metal? */
|
|
117
123
|
export function isLiveMetalContended(now: number = Date.now()): { contended: boolean; reason: ContentionReason | null } {
|
|
124
|
+
if (canonicalMetalRequests > 0) return { contended: true, reason: 'canonical_in_flight' }
|
|
118
125
|
const work = activeMetalFamilyWork()
|
|
119
126
|
if (work) return { contended: true, reason: work }
|
|
120
127
|
if (recentSessionActivity(now)) return { contended: true, reason: 'session_recent' }
|
|
@@ -122,6 +129,52 @@ export function isLiveMetalContended(now: number = Date.now()): { contended: boo
|
|
|
122
129
|
return { contended: false, reason: null }
|
|
123
130
|
}
|
|
124
131
|
|
|
132
|
+
export interface MetalPreviewLease {
|
|
133
|
+
signal: AbortSignal
|
|
134
|
+
release: () => void
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Cosmetic preview receives Metal only while canonical work is idle. The
|
|
138
|
+
* registration is synchronous, so a canonical request that begins afterward
|
|
139
|
+
* can abort this request before starting its own inference. */
|
|
140
|
+
export function tryAcquireMetalPreview(): MetalPreviewLease | null {
|
|
141
|
+
// Recent session activity is not itself GPU work. Allow preview between
|
|
142
|
+
// canonical chunks, but never alongside an active canonical or HQ Metal job.
|
|
143
|
+
if (canonicalMetalRequests > 0 || metalChildren.size > 0) return null
|
|
144
|
+
const controller = new AbortController()
|
|
145
|
+
previewMetalRequests.add(controller)
|
|
146
|
+
let released = false
|
|
147
|
+
return {
|
|
148
|
+
signal: controller.signal,
|
|
149
|
+
release: () => {
|
|
150
|
+
if (released) return
|
|
151
|
+
released = true
|
|
152
|
+
previewMetalRequests.delete(controller)
|
|
153
|
+
},
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Canonical live transcription always wins. Abort any cosmetic preview first,
|
|
158
|
+
* then hold a synchronous counter so no new preview can enter until release. */
|
|
159
|
+
export function beginCanonicalMetal(reason: string): () => void {
|
|
160
|
+
canonicalMetalRequests++
|
|
161
|
+
for (const controller of previewMetalRequests) controller.abort(reason)
|
|
162
|
+
let released = false
|
|
163
|
+
return () => {
|
|
164
|
+
if (released) return
|
|
165
|
+
released = true
|
|
166
|
+
canonicalMetalRequests = Math.max(0, canonicalMetalRequests - 1)
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export class MetalPreviewContendedError extends Error {
|
|
171
|
+
readonly contended = true
|
|
172
|
+
constructor() {
|
|
173
|
+
super('Whisper preview yielded to canonical transcription')
|
|
174
|
+
this.name = 'MetalPreviewContendedError'
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
125
178
|
/** Device for the NEXT batch segment. Re-evaluated per segment so a meeting
|
|
126
179
|
* that starts mid-batch moves subsequent segments to CPU without a preempt. */
|
|
127
180
|
export function chooseBatchDevice(now: number = Date.now()): BatchDeviceDecision {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Adaptive provisional transcription:
|
|
2
2
|
// Balanced -> isolated Small.en cosmetic preview + Turbo live commit
|
|
3
|
-
// Max ->
|
|
3
|
+
// Max -> isolated Turbo cosmetic preview + Large-v3 live commit
|
|
4
4
|
// polish -> Large-v3 save pass (unchanged)
|
|
5
5
|
|
|
6
6
|
import { execFile, spawn } from 'node:child_process'
|
|
@@ -16,9 +16,11 @@ import {
|
|
|
16
16
|
type WhisperCommitModel,
|
|
17
17
|
type WhisperTranscriptionTier,
|
|
18
18
|
} from './whisper-local.js'
|
|
19
|
+
import { MetalPreviewContendedError, tryAcquireMetalPreview } from './whisper-metal-gate.js'
|
|
19
20
|
|
|
20
21
|
export type WhisperPreviewRequest = 'auto' | 'small.en' | 'turbo' | 'off'
|
|
21
22
|
export type WhisperPreviewModel = 'small.en' | WhisperCommitModel | null
|
|
23
|
+
type WhisperPreviewSidecarModel = 'small.en' | 'large-v3-turbo'
|
|
22
24
|
export type WhisperPreviewReason =
|
|
23
25
|
| 'disabled'
|
|
24
26
|
| 'small_model_missing'
|
|
@@ -50,6 +52,7 @@ export interface WhisperPreviewCapability {
|
|
|
50
52
|
|
|
51
53
|
const MODEL_DIR = join(process.env.HOME ?? homedir(), '.local/share/whisper-models')
|
|
52
54
|
export const WHISPER_SMALL_EN_MODEL_PATH = join(MODEL_DIR, 'ggml-small.en.bin')
|
|
55
|
+
const WHISPER_TURBO_MODEL_PATH = join(MODEL_DIR, 'ggml-large-v3-turbo.bin')
|
|
53
56
|
const VAD_MODEL_PATH = join(MODEL_DIR, 'ggml-silero-v5.1.2.bin')
|
|
54
57
|
const VAD_ENABLED = process.env.COS_WHISPER_VAD !== '0'
|
|
55
58
|
const WHISPER_SERVER = ['/opt/homebrew/bin/whisper-server', '/usr/local/bin/whisper-server']
|
|
@@ -63,6 +66,7 @@ let previewProcess: ChildProcess | null = null
|
|
|
63
66
|
let previewAvailable = false
|
|
64
67
|
let previewStarting = false
|
|
65
68
|
let previewFailure: WhisperPreviewReason = null
|
|
69
|
+
let previewWorkerModel: WhisperPreviewSidecarModel | null = null
|
|
66
70
|
let warnedInvalidChoice = false
|
|
67
71
|
|
|
68
72
|
interface ProcessEntry {
|
|
@@ -103,10 +107,11 @@ function isCosPreviewCommand(command: string): boolean {
|
|
|
103
107
|
const executablePath = firstToken?.[1] ?? firstToken?.[2] ?? firstToken?.[3] ?? ''
|
|
104
108
|
return basename(executablePath) === 'whisper-server'
|
|
105
109
|
&& new RegExp(`(?:^|\\s)--port(?:=|\\s+)${PREVIEW_PORT}(?:\\s|$)`).test(command)
|
|
106
|
-
&&
|
|
110
|
+
&& [WHISPER_SMALL_EN_MODEL_PATH, WHISPER_TURBO_MODEL_PATH]
|
|
111
|
+
.some(modelPath => command.includes(modelPath))
|
|
107
112
|
}
|
|
108
113
|
|
|
109
|
-
/** Reap only a listener proven to be our exact
|
|
114
|
+
/** Reap only a listener proven to be our exact COS preview/8177 command. An
|
|
110
115
|
* unrelated local service is never contacted with audio or terminated. */
|
|
111
116
|
async function reclaimPreviewPort(): Promise<'clear' | 'reaped' | 'foreign'> {
|
|
112
117
|
const listeners = await previewListeningPids()
|
|
@@ -135,7 +140,7 @@ async function reclaimPreviewPort(): Promise<'clear' | 'reaped' | 'foreign'> {
|
|
|
135
140
|
if ((await previewListeningPids()).length === 0) return 'reaped'
|
|
136
141
|
await new Promise(resolve => setTimeout(resolve, 100))
|
|
137
142
|
}
|
|
138
|
-
throw new Error(`verified
|
|
143
|
+
throw new Error(`verified COS preview worker still owns port ${PREVIEW_PORT}`)
|
|
139
144
|
}
|
|
140
145
|
|
|
141
146
|
export function normalizeWhisperPreviewRequest(raw?: string): WhisperPreviewRequest {
|
|
@@ -169,11 +174,21 @@ function requestedPreviewModel(): WhisperPreviewRequest {
|
|
|
169
174
|
function selectedPreviewModel(requested = requestedPreviewModel()): WhisperPreviewModel {
|
|
170
175
|
if (requested === 'off') return null
|
|
171
176
|
const primary = getWhisperCommitCapability().effectiveModel
|
|
172
|
-
if (requested === 'turbo')
|
|
177
|
+
if (requested === 'turbo') {
|
|
178
|
+
return existsSync(WHISPER_TURBO_MODEL_PATH) ? 'large-v3-turbo' : primary
|
|
179
|
+
}
|
|
173
180
|
if (requested === 'small.en') return existsSync(WHISPER_SMALL_EN_MODEL_PATH) ? 'small.en' : primary
|
|
174
181
|
return existsSync(WHISPER_SMALL_EN_MODEL_PATH) ? 'small.en' : primary
|
|
175
182
|
}
|
|
176
183
|
|
|
184
|
+
function sidecarModelPath(model: WhisperPreviewSidecarModel): string {
|
|
185
|
+
return model === 'small.en' ? WHISPER_SMALL_EN_MODEL_PATH : WHISPER_TURBO_MODEL_PATH
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function needsPreviewSidecar(model: WhisperPreviewModel): model is WhisperPreviewSidecarModel {
|
|
189
|
+
return model !== null && model !== getWhisperCommitCapability().effectiveModel
|
|
190
|
+
}
|
|
191
|
+
|
|
177
192
|
export function getWhisperPreviewCapability(): WhisperPreviewCapability {
|
|
178
193
|
const commit = getWhisperCommitCapability()
|
|
179
194
|
const requested = requestedPreviewModel()
|
|
@@ -189,12 +204,12 @@ export function getWhisperPreviewCapability(): WhisperPreviewCapability {
|
|
|
189
204
|
}
|
|
190
205
|
}
|
|
191
206
|
|
|
192
|
-
const smallPresent = existsSync(WHISPER_SMALL_EN_MODEL_PATH)
|
|
193
207
|
const selected = selectedPreviewModel(requested)
|
|
194
|
-
const
|
|
195
|
-
|
|
208
|
+
const primaryReady = getWhisperHealth().server
|
|
209
|
+
const sidecarExpected = needsPreviewSidecar(selected)
|
|
210
|
+
if (sidecarExpected && previewAvailable && previewWorkerModel === selected) {
|
|
196
211
|
return {
|
|
197
|
-
requested, effectiveModel:
|
|
212
|
+
requested, effectiveModel: selected, ready: true,
|
|
198
213
|
backend: 'whisper-preview-server', degraded: commit.degraded, reason: commit.reason,
|
|
199
214
|
previewDegraded: false, commitDegraded: commit.degraded, commitReason: commit.reason,
|
|
200
215
|
committedModel: commit.effectiveModel,
|
|
@@ -204,20 +219,24 @@ export function getWhisperPreviewCapability(): WhisperPreviewCapability {
|
|
|
204
219
|
}
|
|
205
220
|
}
|
|
206
221
|
|
|
207
|
-
const
|
|
208
|
-
|
|
222
|
+
const selectedModelMissing = requested === 'small.en' && !existsSync(WHISPER_SMALL_EN_MODEL_PATH)
|
|
223
|
+
? 'small_model_missing'
|
|
224
|
+
: requested === 'turbo' && !existsSync(WHISPER_TURBO_MODEL_PATH)
|
|
225
|
+
? 'turbo_model_missing'
|
|
226
|
+
: null
|
|
227
|
+
const previewDegraded = sidecarExpected || selectedModelMissing !== null
|
|
209
228
|
const reason: WhisperPreviewReason = commit.reason
|
|
210
229
|
? commit.reason
|
|
211
|
-
:
|
|
212
|
-
|
|
213
|
-
|
|
230
|
+
: selectedModelMissing
|
|
231
|
+
? selectedModelMissing
|
|
232
|
+
: sidecarExpected
|
|
214
233
|
? (previewFailure ?? (previewStarting ? null : 'preview_sidecar_unavailable'))
|
|
215
|
-
:
|
|
234
|
+
: primaryReady ? null : 'turbo_unavailable'
|
|
216
235
|
return {
|
|
217
236
|
requested,
|
|
218
|
-
effectiveModel:
|
|
219
|
-
ready:
|
|
220
|
-
backend:
|
|
237
|
+
effectiveModel: primaryReady ? commit.effectiveModel : selected,
|
|
238
|
+
ready: primaryReady,
|
|
239
|
+
backend: primaryReady ? 'whisper-server' : null,
|
|
221
240
|
degraded: previewDegraded || commit.degraded,
|
|
222
241
|
reason,
|
|
223
242
|
previewDegraded,
|
|
@@ -232,16 +251,21 @@ export function getWhisperPreviewCapability(): WhisperPreviewCapability {
|
|
|
232
251
|
}
|
|
233
252
|
|
|
234
253
|
async function endpointReady(path: '/health' | '/inference', init?: RequestInit, timeoutMs = 1_000): Promise<Response> {
|
|
235
|
-
|
|
254
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs)
|
|
255
|
+
const signal = init?.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal
|
|
256
|
+
return fetch(`${PREVIEW_URL}${path}`, { ...init, signal })
|
|
236
257
|
}
|
|
237
258
|
|
|
238
|
-
/** Start the optional
|
|
239
|
-
* Turbo
|
|
259
|
+
/** Start the optional isolated preview worker. Balanced uses Small.en; Max
|
|
260
|
+
* uses Turbo while its canonical live worker remains Large-v3. Failure is
|
|
261
|
+
* cosmetic: every recovery/finalization path stays untouched. */
|
|
240
262
|
export async function startWhisperPreviewServer(): Promise<void> {
|
|
241
263
|
const requested = requestedPreviewModel()
|
|
242
|
-
|
|
243
|
-
if (!
|
|
244
|
-
|
|
264
|
+
const selected = selectedPreviewModel(requested)
|
|
265
|
+
if (!needsPreviewSidecar(selected) || previewProcess || previewAvailable || previewStarting) return
|
|
266
|
+
const modelPath = sidecarModelPath(selected)
|
|
267
|
+
if (!existsSync(modelPath)) {
|
|
268
|
+
previewFailure = selected === 'small.en' ? 'small_model_missing' : 'turbo_model_missing'
|
|
245
269
|
return
|
|
246
270
|
}
|
|
247
271
|
if (!existsSync(WHISPER_SERVER)) {
|
|
@@ -259,7 +283,7 @@ export async function startWhisperPreviewServer(): Promise<void> {
|
|
|
259
283
|
return
|
|
260
284
|
}
|
|
261
285
|
if (portState === 'reaped') {
|
|
262
|
-
console.log('[whisper-preview] reaped a stale
|
|
286
|
+
console.log('[whisper-preview] reaped a stale COS preview worker before restart')
|
|
263
287
|
}
|
|
264
288
|
} catch (error) {
|
|
265
289
|
previewFailure = 'preview_start_failed'
|
|
@@ -268,7 +292,7 @@ export async function startWhisperPreviewServer(): Promise<void> {
|
|
|
268
292
|
}
|
|
269
293
|
|
|
270
294
|
const args = [
|
|
271
|
-
'-m',
|
|
295
|
+
'-m', modelPath,
|
|
272
296
|
'-t', '16',
|
|
273
297
|
'-l', 'en',
|
|
274
298
|
'-fa',
|
|
@@ -281,10 +305,12 @@ export async function startWhisperPreviewServer(): Promise<void> {
|
|
|
281
305
|
}
|
|
282
306
|
const child = spawn(WHISPER_SERVER, args, { stdio: 'ignore', detached: false })
|
|
283
307
|
previewProcess = child
|
|
308
|
+
previewWorkerModel = selected
|
|
284
309
|
child.once('close', code => {
|
|
285
310
|
if (previewProcess !== child) return
|
|
286
311
|
previewProcess = null
|
|
287
312
|
previewAvailable = false
|
|
313
|
+
previewWorkerModel = null
|
|
288
314
|
previewFailure = code === 0 ? 'preview_sidecar_unavailable' : 'preview_start_failed'
|
|
289
315
|
})
|
|
290
316
|
child.once('error', () => {
|
|
@@ -300,7 +326,8 @@ export async function startWhisperPreviewServer(): Promise<void> {
|
|
|
300
326
|
if (response.ok) {
|
|
301
327
|
previewAvailable = true
|
|
302
328
|
previewFailure = null
|
|
303
|
-
|
|
329
|
+
const committed = getWhisperCommitCapability().effectiveModel
|
|
330
|
+
console.log(`[whisper-preview] ${selected} ready for provisional text; committed text remains ${committed}`)
|
|
304
331
|
return
|
|
305
332
|
}
|
|
306
333
|
} catch { /* model still loading */ }
|
|
@@ -308,6 +335,7 @@ export async function startWhisperPreviewServer(): Promise<void> {
|
|
|
308
335
|
}
|
|
309
336
|
try { child.kill('SIGKILL') } catch { /* already exited */ }
|
|
310
337
|
if (previewProcess === child) previewProcess = null
|
|
338
|
+
previewWorkerModel = null
|
|
311
339
|
previewFailure = 'preview_start_failed'
|
|
312
340
|
} finally {
|
|
313
341
|
previewStarting = false
|
|
@@ -336,6 +364,7 @@ export async function stopWhisperPreviewServer(): Promise<void> {
|
|
|
336
364
|
previewProcess = null
|
|
337
365
|
previewAvailable = false
|
|
338
366
|
previewStarting = false
|
|
367
|
+
previewWorkerModel = null
|
|
339
368
|
if (child) {
|
|
340
369
|
try { child.kill('SIGTERM') } catch { /* already exited */ }
|
|
341
370
|
if (!await waitForPreviewClose(child, 2_000)) {
|
|
@@ -345,40 +374,63 @@ export async function stopWhisperPreviewServer(): Promise<void> {
|
|
|
345
374
|
}
|
|
346
375
|
}
|
|
347
376
|
|
|
348
|
-
async function transcribeViaPreviewServer(audioBuffer: Buffer): Promise<string> {
|
|
377
|
+
async function transcribeViaPreviewServer(audioBuffer: Buffer, signal: AbortSignal): Promise<string> {
|
|
349
378
|
const formData = new FormData()
|
|
350
379
|
formData.append('file', new Blob([new Uint8Array(audioBuffer)], { type: 'audio/wav' }), 'recording.wav')
|
|
351
380
|
formData.append('response_format', 'json')
|
|
352
381
|
// Preview text is cosmetic and the same audio is authoritatively decoded by
|
|
353
|
-
// the commit lane.
|
|
354
|
-
// windows, so never bias this provisional decode with profile vocabulary.
|
|
382
|
+
// the commit lane. Never bias this provisional decode with profile vocabulary.
|
|
355
383
|
formData.append('suppress_non_speech', 'true')
|
|
356
|
-
const response = await endpointReady('/inference', { method: 'POST', body: formData }, 5_000)
|
|
384
|
+
const response = await endpointReady('/inference', { method: 'POST', body: formData, signal }, 5_000)
|
|
357
385
|
if (!response.ok) throw new Error(`preview server ${response.status}`)
|
|
358
386
|
const result = await response.json() as { text?: unknown }
|
|
359
387
|
if (typeof result.text !== 'string') throw new Error('preview server returned invalid text')
|
|
360
388
|
return applyCorrections(result.text.trim())
|
|
361
389
|
}
|
|
362
390
|
|
|
363
|
-
/** Cosmetic preview only. A
|
|
364
|
-
* non-circuit
|
|
391
|
+
/** Cosmetic preview only. A sidecar failure falls through to the existing
|
|
392
|
+
* non-circuit canonical decode and can never write committed transcript state. */
|
|
365
393
|
export async function transcribeWhisperPreview(audioBuffer: Buffer): Promise<{
|
|
366
394
|
text: string
|
|
367
395
|
model: 'small.en' | WhisperCommitModel
|
|
368
396
|
backend: 'whisper-preview-server' | 'whisper-server'
|
|
369
|
-
}> {
|
|
370
|
-
if (previewAvailable) {
|
|
397
|
+
}> {
|
|
398
|
+
if (previewAvailable && previewWorkerModel) {
|
|
399
|
+
const workerModel = previewWorkerModel
|
|
400
|
+
const previewLease = tryAcquireMetalPreview()
|
|
401
|
+
if (!previewLease) {
|
|
402
|
+
return { text: '', model: workerModel, backend: 'whisper-preview-server' }
|
|
403
|
+
}
|
|
371
404
|
try {
|
|
372
|
-
return {
|
|
405
|
+
return {
|
|
406
|
+
text: await transcribeViaPreviewServer(audioBuffer, previewLease.signal),
|
|
407
|
+
model: workerModel,
|
|
408
|
+
backend: 'whisper-preview-server',
|
|
409
|
+
}
|
|
373
410
|
} catch (error) {
|
|
411
|
+
if (previewLease.signal.aborted) {
|
|
412
|
+
return { text: '', model: workerModel, backend: 'whisper-preview-server' }
|
|
413
|
+
}
|
|
414
|
+
const failedModel = previewWorkerModel
|
|
374
415
|
previewAvailable = false
|
|
416
|
+
previewWorkerModel = null
|
|
375
417
|
previewFailure = 'preview_sidecar_unavailable'
|
|
376
|
-
console.warn(`[whisper-preview]
|
|
418
|
+
console.warn(`[whisper-preview] ${failedModel} preview failed; falling back to canonical worker: ${error instanceof Error ? error.message : error}`)
|
|
419
|
+
} finally {
|
|
420
|
+
previewLease.release()
|
|
377
421
|
}
|
|
378
422
|
}
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
423
|
+
try {
|
|
424
|
+
const result = await transcribeLocal(audioBuffer, undefined, undefined, {
|
|
425
|
+
affectsCircuit: false,
|
|
426
|
+
promptPolicy: 'none',
|
|
427
|
+
metalPriority: 'preview',
|
|
428
|
+
})
|
|
429
|
+
return { text: result.text, model: getWhisperCommitCapability().effectiveModel, backend: 'whisper-server' }
|
|
430
|
+
} catch (error) {
|
|
431
|
+
if (error instanceof MetalPreviewContendedError) {
|
|
432
|
+
return { text: '', model: getWhisperCommitCapability().effectiveModel, backend: 'whisper-server' }
|
|
433
|
+
}
|
|
434
|
+
throw error
|
|
435
|
+
}
|
|
384
436
|
}
|
package/server/routes/media.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
getMediaStore,
|
|
29
29
|
MediaStoreError,
|
|
30
30
|
} from '../lib/media-store.js'
|
|
31
|
+
import { currentMessageEra } from '../lib/message-era.js'
|
|
31
32
|
import {
|
|
32
33
|
ImageSafetyError,
|
|
33
34
|
MAX_BATCH_BYTES,
|
|
@@ -175,6 +176,9 @@ mediaRouter.post('/media/associate', async (req: Request, res: Response) => {
|
|
|
175
176
|
sessionId: safeString(req.body?.sessionId, 64),
|
|
176
177
|
runId: safeString(req.body?.runId, 120),
|
|
177
178
|
globalMsgNum,
|
|
179
|
+
// Association always belongs to the server's active message era. Never
|
|
180
|
+
// trust a client-supplied era to make media visible in historical turns.
|
|
181
|
+
messageEra: currentMessageEra(),
|
|
178
182
|
})
|
|
179
183
|
res.json({ ok: true })
|
|
180
184
|
} catch (err) {
|
package/server/routes/query.ts
CHANGED
|
@@ -160,6 +160,7 @@ queryRouter.post('/query', async (req, res) => {
|
|
|
160
160
|
await getMediaStore().associate(resolvedAttachments.ids, {
|
|
161
161
|
sessionId: sid,
|
|
162
162
|
...(validGlobalMsgNum ? { globalMsgNum: validGlobalMsgNum } : {}),
|
|
163
|
+
messageEra: activeMessageEra,
|
|
163
164
|
}).catch((err) => console.error('[query] attachment association failed:', err))
|
|
164
165
|
}
|
|
165
166
|
if (!done) {
|
|
@@ -199,6 +200,7 @@ queryRouter.post('/query', async (req, res) => {
|
|
|
199
200
|
{
|
|
200
201
|
abortSignal: abortController.signal,
|
|
201
202
|
effort: validEffort,
|
|
203
|
+
messageEra: activeMessageEra,
|
|
202
204
|
...(validModel && isCursorModel(validModel) ? { cursorExecutionMode } : {}),
|
|
203
205
|
},
|
|
204
206
|
)
|
|
@@ -8,9 +8,46 @@ import { getArchiveDayMessages } from '../lib/archive.js'
|
|
|
8
8
|
import { localDay } from '../lib/local-day.js'
|
|
9
9
|
import { clearCodexEngineSession } from '../lib/codex-engine-sessions.js'
|
|
10
10
|
import { mergeMediaAttachmentRefs, type MediaAttachmentRef } from '../../shared/media-attachment.js'
|
|
11
|
-
import {
|
|
11
|
+
import { currentMessageEraState, exchangeBelongsToEra, type MessageEraState } from '../lib/message-era.js'
|
|
12
|
+
import { getMediaStore } from '../lib/media-store.js'
|
|
12
13
|
|
|
13
14
|
export const sessionsRouter = Router()
|
|
15
|
+
let mediaAssociationLookupWarned = false
|
|
16
|
+
|
|
17
|
+
/** Conversation exchanges are the normal attachment projection. The media
|
|
18
|
+
* store fallback repairs turns created by the legacy query path, which did
|
|
19
|
+
* durably associate media to session + message but did not stamp the refs on
|
|
20
|
+
* the exchange. Both sources contain public refs only and remain capped by the
|
|
21
|
+
* shared merge validator. */
|
|
22
|
+
function turnAttachments(
|
|
23
|
+
sessionId: string,
|
|
24
|
+
globalMsgNum: number | undefined,
|
|
25
|
+
messageEra: string | undefined,
|
|
26
|
+
activeEra: MessageEraState,
|
|
27
|
+
...sources: unknown[]
|
|
28
|
+
): MediaAttachmentRef[] {
|
|
29
|
+
let associated: MediaAttachmentRef[] = []
|
|
30
|
+
if (globalMsgNum != null) {
|
|
31
|
+
try {
|
|
32
|
+
associated = getMediaStore().getAssociatedRefs({
|
|
33
|
+
sessionId,
|
|
34
|
+
globalMsgNum,
|
|
35
|
+
messageEra,
|
|
36
|
+
activeMessageEra: activeEra.era,
|
|
37
|
+
activeEraStartedAt: activeEra.startedAt,
|
|
38
|
+
})
|
|
39
|
+
} catch (error) {
|
|
40
|
+
// Conversation history remains useful when optional media recovery is
|
|
41
|
+
// unavailable. Warn once per process instead of turning a text endpoint
|
|
42
|
+
// into a 500 or flooding logs once per turn.
|
|
43
|
+
if (!mediaAssociationLookupWarned) {
|
|
44
|
+
mediaAssociationLookupWarned = true
|
|
45
|
+
console.warn(`[sessions] media association lookup unavailable; serving text-only history: ${String(error)}`)
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return mergeMediaAttachmentRefs(...sources, associated)
|
|
50
|
+
}
|
|
14
51
|
|
|
15
52
|
sessionsRouter.get('/sessions/recent', (_req, res) => {
|
|
16
53
|
const sessions = getRecentSessions(24 * 60 * 60_000)
|
|
@@ -62,19 +99,22 @@ sessionsRouter.get('/sessions/:id/messages', (req, res) => {
|
|
|
62
99
|
attachments?: MediaAttachmentRef[]
|
|
63
100
|
}> = []
|
|
64
101
|
const session = getSessionRaw(req.params.id)
|
|
102
|
+
const activeEra = currentMessageEraState()
|
|
65
103
|
for (let i = 0; i < exchanges.length; i++) {
|
|
66
104
|
const ex = exchanges[i]
|
|
67
105
|
if (ex.role === 'user') {
|
|
68
106
|
const next = exchanges[i + 1]
|
|
69
107
|
if (next && next.role === 'assistant') {
|
|
70
|
-
const
|
|
108
|
+
const globalMsgNum = ex.globalMsgNum ?? next.globalMsgNum
|
|
109
|
+
const messageEra = ex.messageEra ?? next.messageEra
|
|
110
|
+
const attachments = turnAttachments(req.params.id, globalMsgNum, messageEra, activeEra, ex.attachments, next.attachments)
|
|
71
111
|
const modelPreference = resolveExchangePairModel(ex, next, session?.modelPreference)
|
|
72
112
|
messages.push({
|
|
73
113
|
query: ex.content,
|
|
74
114
|
text: next.content,
|
|
75
115
|
timestamp: next.timestamp,
|
|
76
116
|
sessionId: req.params.id,
|
|
77
|
-
...(
|
|
117
|
+
...(globalMsgNum != null ? { no: globalMsgNum } : {}),
|
|
78
118
|
...(modelPreference ? { modelPreference } : {}),
|
|
79
119
|
...(attachments.length > 0 ? { attachments } : {}),
|
|
80
120
|
})
|
|
@@ -246,14 +286,23 @@ sessionsRouter.get('/sessions/today/live-chats', (_req, res) => {
|
|
|
246
286
|
// back to the archived chat's sessionId via getArchiveDayMessages.
|
|
247
287
|
sessionsRouter.get('/sessions/today/all-messages', (_req, res) => {
|
|
248
288
|
const todayDate = localDay()
|
|
249
|
-
const
|
|
289
|
+
const activeEra = currentMessageEraState()
|
|
290
|
+
const era = activeEra.era
|
|
250
291
|
|
|
251
292
|
const archivedMessages = getArchiveDayMessages(todayDate)
|
|
252
293
|
.filter(m => exchangeBelongsToEra(m, era))
|
|
253
|
-
.map(m =>
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
294
|
+
.map(m => {
|
|
295
|
+
const globalMsgNum = m.globalMsgNum ?? m.no
|
|
296
|
+
const attachments = turnAttachments(m.sessionId, globalMsgNum, m.messageEra, activeEra, m.attachments)
|
|
297
|
+
// Never pass an unvalidated archive attachment array through the object
|
|
298
|
+
// spread when the merge rejects it.
|
|
299
|
+
const { attachments: _rawAttachments, ...message } = m
|
|
300
|
+
return {
|
|
301
|
+
...message,
|
|
302
|
+
source: 'archive' as const,
|
|
303
|
+
...(attachments.length > 0 ? { attachments } : {}),
|
|
304
|
+
}
|
|
305
|
+
})
|
|
257
306
|
|
|
258
307
|
const liveMessages: Array<{
|
|
259
308
|
query: string
|
|
@@ -278,9 +327,9 @@ sessionsRouter.get('/sessions/today/all-messages', (_req, res) => {
|
|
|
278
327
|
if (!exchangeBelongsToEra(ex, era)) continue
|
|
279
328
|
const next = session.exchanges[i + 1]
|
|
280
329
|
if (next && next.role === 'assistant') {
|
|
281
|
-
const attachments = mergeMediaAttachmentRefs(ex.attachments, next.attachments)
|
|
282
330
|
const globalMsgNum = ex.globalMsgNum ?? next.globalMsgNum
|
|
283
331
|
const messageEra = ex.messageEra ?? next.messageEra
|
|
332
|
+
const attachments = turnAttachments(session.id, globalMsgNum, messageEra, activeEra, ex.attachments, next.attachments)
|
|
284
333
|
const modelPreference = resolveExchangePairModel(ex, next, session.modelPreference)
|
|
285
334
|
liveMessages.push({
|
|
286
335
|
query: ex.content,
|