@gotcos/glasses-server 6.6.0 → 6.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/README.md +3 -0
- package/package.json +1 -1
- package/server/index.ts +6 -0
- package/server/lib/atomic-fs.ts +79 -3
- package/server/lib/batch-transcript-quality.ts +243 -0
- package/server/lib/dictation-clean.ts +76 -0
- package/server/lib/meeting-batch-persistence.ts +53 -0
- package/server/lib/meeting-batch-transcribe.ts +249 -0
- package/server/lib/meeting-store.ts +594 -0
- package/server/lib/prompt-draft-store.ts +279 -0
- package/server/lib/transcribe-audio.ts +53 -5
- package/server/lib/whisper-local.ts +49 -1
- package/server/routes/health.ts +2 -0
- package/server/routes/meeting.ts +329 -0
- package/server/routes/meetings.ts +66 -0
- package/server/routes/prompt-drafts.ts +276 -0
- package/server/routes/transcribe-stream.ts +116 -23
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
// POST /api/meeting/save — finalize an existing transcribe-stream session into
|
|
2
|
+
// the standalone public meeting store. The live transcript and chunk metadata
|
|
3
|
+
// are durable before the session is closed; batch improvement runs afterward.
|
|
4
|
+
|
|
5
|
+
import { rmSync } from 'node:fs'
|
|
6
|
+
import { Router } from 'express'
|
|
7
|
+
import { emitDisplay } from '../lib/display-bus.js'
|
|
8
|
+
import { cleanTranscriptLines } from '../lib/hallucination-filter.js'
|
|
9
|
+
import {
|
|
10
|
+
getMeetingStore,
|
|
11
|
+
MeetingStore,
|
|
12
|
+
MeetingStoreError,
|
|
13
|
+
type SavedMeeting,
|
|
14
|
+
} from '../lib/meeting-store.js'
|
|
15
|
+
import {
|
|
16
|
+
canDeletePendingBatchAudio,
|
|
17
|
+
persistBatchDecisionSidecar,
|
|
18
|
+
replaceMeetingTranscriptAtomic,
|
|
19
|
+
} from '../lib/meeting-batch-persistence.js'
|
|
20
|
+
import { runMeetingBatchPipeline } from '../lib/meeting-batch-transcribe.js'
|
|
21
|
+
import {
|
|
22
|
+
selectBatchTranscriptForPersistence,
|
|
23
|
+
type BatchTranscription,
|
|
24
|
+
} from '../lib/batch-transcript-quality.js'
|
|
25
|
+
import {
|
|
26
|
+
analyzeTranscriptGaps,
|
|
27
|
+
deleteSession,
|
|
28
|
+
drainSessionAudioWrites,
|
|
29
|
+
getSessionChunkEntries,
|
|
30
|
+
getSessionChunks,
|
|
31
|
+
getSessionProviderCandidates,
|
|
32
|
+
getSessionStartTime,
|
|
33
|
+
getSessionTranscript,
|
|
34
|
+
hasSessionAudio,
|
|
35
|
+
moveSessionAudioToPending,
|
|
36
|
+
type IndexedTranscriptChunk,
|
|
37
|
+
type ProviderCandidateRecord,
|
|
38
|
+
type TranscriptChunk,
|
|
39
|
+
type TranscriptGapReport,
|
|
40
|
+
} from './transcribe-stream.js'
|
|
41
|
+
|
|
42
|
+
interface MeetingSessionSource {
|
|
43
|
+
getTranscript(sessionId: string): string | null
|
|
44
|
+
getStartTime(sessionId: string): number | null
|
|
45
|
+
getChunks(sessionId: string): TranscriptChunk[] | null
|
|
46
|
+
getChunkEntries(sessionId: string): IndexedTranscriptChunk[] | null
|
|
47
|
+
getProviderCandidates(sessionId: string): Record<string, ProviderCandidateRecord>
|
|
48
|
+
getIntegrity(sessionId: string): TranscriptGapReport | null
|
|
49
|
+
drainAudioWrites(sessionId: string): Promise<void>
|
|
50
|
+
hasAudio(sessionId: string): boolean
|
|
51
|
+
moveAudioToPending(sessionId: string): string | null
|
|
52
|
+
delete(sessionId: string, options?: { preserveAudio?: boolean }): void
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface MeetingRouteDependencies {
|
|
56
|
+
store?: MeetingStore
|
|
57
|
+
sessions?: MeetingSessionSource
|
|
58
|
+
runBatch?: (
|
|
59
|
+
audioDir: string,
|
|
60
|
+
entries: IndexedTranscriptChunk[],
|
|
61
|
+
streamingWordCount: number,
|
|
62
|
+
) => Promise<BatchTranscription>
|
|
63
|
+
scheduleBackground?: (task: Promise<void>) => void
|
|
64
|
+
emit?: typeof emitDisplay
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const defaultSessionSource: MeetingSessionSource = {
|
|
68
|
+
getTranscript: sessionId => getSessionTranscript(sessionId, { withGaps: true }),
|
|
69
|
+
getStartTime: getSessionStartTime,
|
|
70
|
+
getChunks: getSessionChunks,
|
|
71
|
+
getChunkEntries: getSessionChunkEntries,
|
|
72
|
+
getProviderCandidates: getSessionProviderCandidates,
|
|
73
|
+
getIntegrity: analyzeTranscriptGaps,
|
|
74
|
+
drainAudioWrites: drainSessionAudioWrites,
|
|
75
|
+
hasAudio: hasSessionAudio,
|
|
76
|
+
moveAudioToPending: moveSessionAudioToPending,
|
|
77
|
+
delete: deleteSession,
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function countWords(text: string): number {
|
|
81
|
+
return text.trim().split(/\s+/).filter(Boolean).length
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function cleanFinalTranscript(transcript: string): string {
|
|
85
|
+
try {
|
|
86
|
+
return process.env.COS_WHISPER_STRIP_BRAND_URLS === '0'
|
|
87
|
+
? transcript
|
|
88
|
+
: cleanTranscriptLines(transcript)
|
|
89
|
+
} catch {
|
|
90
|
+
return transcript
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function publicSaveResponse(saved: SavedMeeting, replayed = false): Record<string, unknown> {
|
|
95
|
+
const integrity = saved.transferIntegrity ?? null
|
|
96
|
+
const missingCount = integrity?.missingIndices.length ?? 0
|
|
97
|
+
const completenessPct = integrity
|
|
98
|
+
? Math.floor(integrity.completeness * 1_000) / 10
|
|
99
|
+
: 100
|
|
100
|
+
return {
|
|
101
|
+
saved: true,
|
|
102
|
+
// Keep the build199 string field without leaking an absolute host path.
|
|
103
|
+
filepath: `recordings/${saved.month}/${saved.filename}`,
|
|
104
|
+
filename: saved.filename,
|
|
105
|
+
durationMin: saved.durationMin,
|
|
106
|
+
domain: saved.domain,
|
|
107
|
+
transcriptionQuality: 'streaming',
|
|
108
|
+
...(replayed ? { replayed: true } : {}),
|
|
109
|
+
transferIntegrity: integrity ? {
|
|
110
|
+
completeness: completenessPct,
|
|
111
|
+
received: integrity.received,
|
|
112
|
+
expected: integrity.expected,
|
|
113
|
+
missingChunks: missingCount,
|
|
114
|
+
missingIndices: integrity.missingIndices.slice(0, 50),
|
|
115
|
+
} : null,
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router {
|
|
120
|
+
const store = deps.store ?? getMeetingStore()
|
|
121
|
+
const sessions = deps.sessions ?? defaultSessionSource
|
|
122
|
+
const runBatch = deps.runBatch ?? runMeetingBatchPipeline
|
|
123
|
+
const scheduleBackground = deps.scheduleBackground ?? (task => { void task })
|
|
124
|
+
const emit = deps.emit ?? emitDisplay
|
|
125
|
+
const router = Router()
|
|
126
|
+
const savingSessions = new Set<string>()
|
|
127
|
+
|
|
128
|
+
router.post('/meeting/save', async (req, res) => {
|
|
129
|
+
let lockedSessionId: string | null = null
|
|
130
|
+
try {
|
|
131
|
+
const body = req.body as Record<string, unknown> | undefined
|
|
132
|
+
const sessionId = typeof body?.sessionId === 'string' ? body.sessionId : ''
|
|
133
|
+
if (!sessionId) {
|
|
134
|
+
res.status(400).json({ error: 'sessionId required', reason: 'missing_session_id' })
|
|
135
|
+
return
|
|
136
|
+
}
|
|
137
|
+
if (!/^[A-Za-z0-9:_-]{3,96}$/.test(sessionId)) {
|
|
138
|
+
res.status(400).json({ error: 'Invalid sessionId', reason: 'invalid_session_id' })
|
|
139
|
+
return
|
|
140
|
+
}
|
|
141
|
+
if (body?.title !== undefined && typeof body.title !== 'string') {
|
|
142
|
+
res.status(400).json({ error: 'Invalid title', reason: 'invalid_title' })
|
|
143
|
+
return
|
|
144
|
+
}
|
|
145
|
+
if (body?.domain !== undefined && typeof body.domain !== 'string') {
|
|
146
|
+
res.status(400).json({ error: 'Invalid domain', reason: 'invalid_domain' })
|
|
147
|
+
return
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// A response can be lost after both durable files were committed. Find
|
|
151
|
+
// the sidecar by session ID so client retry/restart is idempotent.
|
|
152
|
+
const alreadySaved = store.findBySessionId(sessionId)
|
|
153
|
+
if (alreadySaved) {
|
|
154
|
+
res.set('Cache-Control', 'private, no-store')
|
|
155
|
+
res.json(publicSaveResponse(alreadySaved, true))
|
|
156
|
+
return
|
|
157
|
+
}
|
|
158
|
+
if (savingSessions.has(sessionId)) {
|
|
159
|
+
res.status(409).json({ error: 'Meeting save already in progress', reason: 'save_in_progress' })
|
|
160
|
+
return
|
|
161
|
+
}
|
|
162
|
+
savingSessions.add(sessionId)
|
|
163
|
+
lockedSessionId = sessionId
|
|
164
|
+
|
|
165
|
+
const transcript = sessions.getTranscript(sessionId)
|
|
166
|
+
if (!transcript?.trim()) {
|
|
167
|
+
res.status(404).json({
|
|
168
|
+
error: `No transcript found for session ${sessionId}`,
|
|
169
|
+
reason: 'session_not_found',
|
|
170
|
+
})
|
|
171
|
+
return
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const chunks = sessions.getChunks(sessionId) ?? []
|
|
175
|
+
const chunkEntries = sessions.getChunkEntries(sessionId)
|
|
176
|
+
?? chunks.map((chunk, chunkIndex) => ({ chunkIndex, chunk }))
|
|
177
|
+
const startTime = sessions.getStartTime(sessionId) ?? Date.now()
|
|
178
|
+
const durationFromTimeline = chunks.reduce(
|
|
179
|
+
(maximum, chunk) => Math.max(maximum, chunk?.elapsed ?? 0),
|
|
180
|
+
0,
|
|
181
|
+
)
|
|
182
|
+
const durationMs = durationFromTimeline > 0
|
|
183
|
+
? durationFromTimeline
|
|
184
|
+
: Math.max(0, Date.now() - startTime)
|
|
185
|
+
const integrity = sessions.getIntegrity(sessionId)
|
|
186
|
+
|
|
187
|
+
// Initial canonical text + structured metadata are published before any
|
|
188
|
+
// live state is removed or background work is scheduled.
|
|
189
|
+
const saved = store.save({
|
|
190
|
+
sessionId,
|
|
191
|
+
title: body?.title as string | undefined,
|
|
192
|
+
domain: body?.domain as string | undefined,
|
|
193
|
+
transcript: cleanFinalTranscript(transcript),
|
|
194
|
+
startTime,
|
|
195
|
+
durationMs,
|
|
196
|
+
chunks,
|
|
197
|
+
chunkEntries,
|
|
198
|
+
providerCandidates: sessions.getProviderCandidates(sessionId),
|
|
199
|
+
transferIntegrity: integrity,
|
|
200
|
+
})
|
|
201
|
+
|
|
202
|
+
// Wait for every raw-WAV write before rename. If any write failed, retain
|
|
203
|
+
// surviving audio for recovery but do not run an incomplete batch.
|
|
204
|
+
let audioWritesReady = true
|
|
205
|
+
try {
|
|
206
|
+
await sessions.drainAudioWrites(sessionId)
|
|
207
|
+
} catch (error) {
|
|
208
|
+
audioWritesReady = false
|
|
209
|
+
console.warn(
|
|
210
|
+
`[meeting/save] One or more raw audio writes failed for ${sessionId}: `
|
|
211
|
+
+ `${error instanceof Error ? error.message : String(error)}`,
|
|
212
|
+
)
|
|
213
|
+
}
|
|
214
|
+
const hadSessionAudio = sessions.hasAudio(sessionId)
|
|
215
|
+
// drainAudioWrites uses allSettled, so even its error path has no live
|
|
216
|
+
// writes. Move surviving evidence to the normal two-hour pending store,
|
|
217
|
+
// but do not batch an incomplete capture.
|
|
218
|
+
const pendingAudioDir = hadSessionAudio ? sessions.moveAudioToPending(sessionId) : null
|
|
219
|
+
const preserveSourceAudio = hadSessionAudio && !pendingAudioDir
|
|
220
|
+
sessions.delete(sessionId, { preserveAudio: preserveSourceAudio })
|
|
221
|
+
if (preserveSourceAudio) {
|
|
222
|
+
console.warn(`[meeting/save] Source audio for ${sessionId} retained after failed pending handoff`)
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
try {
|
|
226
|
+
emit({
|
|
227
|
+
type: 'recording_stop',
|
|
228
|
+
data: {
|
|
229
|
+
sessionId,
|
|
230
|
+
filename: saved.filename,
|
|
231
|
+
durationMin: saved.durationMin,
|
|
232
|
+
domain: saved.domain,
|
|
233
|
+
},
|
|
234
|
+
})
|
|
235
|
+
} catch (error) {
|
|
236
|
+
console.warn('[meeting/save] Display notification failed after durable save:', error)
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
res.set('Cache-Control', 'private, no-store')
|
|
240
|
+
res.json(publicSaveResponse(saved))
|
|
241
|
+
|
|
242
|
+
if (audioWritesReady && pendingAudioDir && chunkEntries.length > 0) {
|
|
243
|
+
const task = finalizeBatch({
|
|
244
|
+
audioDir: pendingAudioDir,
|
|
245
|
+
entries: chunkEntries.map(entry => ({ ...entry, chunk: { ...entry.chunk } })),
|
|
246
|
+
streamingWordCount: countWords(transcript),
|
|
247
|
+
meetingPath: saved.filepath,
|
|
248
|
+
sidecarPath: saved.sidecarPath,
|
|
249
|
+
runBatch,
|
|
250
|
+
}).catch(error => {
|
|
251
|
+
// Raw audio deliberately remains for the existing two-hour cleanup.
|
|
252
|
+
console.error(
|
|
253
|
+
`[meeting/save] Batch finalization failed for ${sessionId}: `
|
|
254
|
+
+ `${error instanceof Error ? error.message : String(error)}`,
|
|
255
|
+
)
|
|
256
|
+
})
|
|
257
|
+
scheduleBackground(task)
|
|
258
|
+
}
|
|
259
|
+
} catch (error) {
|
|
260
|
+
if (error instanceof MeetingStoreError) {
|
|
261
|
+
res.status(error.status).json({ error: error.message, reason: error.code })
|
|
262
|
+
return
|
|
263
|
+
}
|
|
264
|
+
console.error('[meeting/save] Finalization failed:', error)
|
|
265
|
+
res.status(500).json({ error: 'Meeting save failed', reason: 'meeting_save_error' })
|
|
266
|
+
} finally {
|
|
267
|
+
if (lockedSessionId) savingSessions.delete(lockedSessionId)
|
|
268
|
+
}
|
|
269
|
+
})
|
|
270
|
+
|
|
271
|
+
return router
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
async function finalizeBatch(options: {
|
|
275
|
+
audioDir: string
|
|
276
|
+
entries: IndexedTranscriptChunk[]
|
|
277
|
+
streamingWordCount: number
|
|
278
|
+
meetingPath: string
|
|
279
|
+
sidecarPath: string
|
|
280
|
+
runBatch: NonNullable<MeetingRouteDependencies['runBatch']>
|
|
281
|
+
}): Promise<void> {
|
|
282
|
+
const result = await options.runBatch(
|
|
283
|
+
options.audioDir,
|
|
284
|
+
options.entries,
|
|
285
|
+
options.streamingWordCount,
|
|
286
|
+
)
|
|
287
|
+
let transcriptApplied = false
|
|
288
|
+
let metadataPersisted = false
|
|
289
|
+
let persistedResult = result
|
|
290
|
+
|
|
291
|
+
if (result.transcriptionQuality === 'batch' && result.batchTranscript) {
|
|
292
|
+
const selected = selectBatchTranscriptForPersistence(result.batchTranscript, result.batchSegments)
|
|
293
|
+
const canonicalText = cleanFinalTranscript(selected.text)
|
|
294
|
+
if (canonicalText.trim()) {
|
|
295
|
+
transcriptApplied = replaceMeetingTranscriptAtomic(options.meetingPath, canonicalText)
|
|
296
|
+
} else {
|
|
297
|
+
console.error('[meeting/save] Accepted batch candidate cleaned to empty; canonical text retained')
|
|
298
|
+
}
|
|
299
|
+
// Metadata records the exact selected text that became canonical, while
|
|
300
|
+
// batchSegments retain the full diagnostic evidence.
|
|
301
|
+
persistedResult = { ...result, batchTranscript: canonicalText }
|
|
302
|
+
} else if (result.qualityReport) {
|
|
303
|
+
console.warn(
|
|
304
|
+
`[meeting/save] Batch candidate rejected (${result.qualityReport.reason}); `
|
|
305
|
+
+ 'canonical streaming transcript retained',
|
|
306
|
+
)
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
try {
|
|
310
|
+
metadataPersisted = persistBatchDecisionSidecar(
|
|
311
|
+
options.sidecarPath,
|
|
312
|
+
persistedResult,
|
|
313
|
+
transcriptApplied,
|
|
314
|
+
)
|
|
315
|
+
} catch (error) {
|
|
316
|
+
console.error(
|
|
317
|
+
'[meeting/save] Batch decision metadata was not durable:',
|
|
318
|
+
error instanceof Error ? error.message : String(error),
|
|
319
|
+
)
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
if (canDeletePendingBatchAudio(transcriptApplied, metadataPersisted)) {
|
|
323
|
+
rmSync(options.audioDir, { recursive: true, force: true })
|
|
324
|
+
} else {
|
|
325
|
+
console.warn('[meeting/save] Pending raw audio retained for bounded two-hour cleanup')
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
export const meetingRouter = createMeetingRouter()
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// Standalone meeting archive backed only by the public server's private data
|
|
2
|
+
// directory. No COS operations paths, classifiers, or user-specific stores.
|
|
3
|
+
|
|
4
|
+
import { Router } from 'express'
|
|
5
|
+
import { getMeetingStore, MeetingStore, MeetingStoreError } from '../lib/meeting-store.js'
|
|
6
|
+
|
|
7
|
+
export function createMeetingsRouter(store: MeetingStore = getMeetingStore()): Router {
|
|
8
|
+
const router = Router()
|
|
9
|
+
|
|
10
|
+
// GET /api/meetings?limit=20&domain=all
|
|
11
|
+
router.get('/meetings', (req, res) => {
|
|
12
|
+
try {
|
|
13
|
+
const rawLimit = typeof req.query.limit === 'string' ? Number.parseInt(req.query.limit, 10) : 20
|
|
14
|
+
const limit = Number.isFinite(rawLimit) ? rawLimit : 20
|
|
15
|
+
const domain = typeof req.query.domain === 'string' ? req.query.domain : 'all'
|
|
16
|
+
res.set('Cache-Control', 'private, no-store')
|
|
17
|
+
res.json({ meetings: store.list({ limit, domain }) })
|
|
18
|
+
} catch (error) {
|
|
19
|
+
sendMeetingStoreError(res, error)
|
|
20
|
+
}
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
// Query-form detail is convenient for generic API consumers. Register the
|
|
24
|
+
// literal route before the build199 dynamic compatibility route.
|
|
25
|
+
router.get('/meetings/detail', (req, res) => {
|
|
26
|
+
try {
|
|
27
|
+
const domain = typeof req.query.domain === 'string' ? req.query.domain : ''
|
|
28
|
+
const month = typeof req.query.month === 'string' ? req.query.month : ''
|
|
29
|
+
const filename = typeof req.query.filename === 'string' ? req.query.filename : ''
|
|
30
|
+
if (!domain || !month || !filename) {
|
|
31
|
+
res.status(400).json({ error: 'domain, month, and filename are required', reason: 'invalid_meeting_ref' })
|
|
32
|
+
return
|
|
33
|
+
}
|
|
34
|
+
res.set('Cache-Control', 'private, no-store')
|
|
35
|
+
res.json(store.detail(domain, month, filename))
|
|
36
|
+
} catch (error) {
|
|
37
|
+
sendMeetingStoreError(res, error)
|
|
38
|
+
}
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
// Build199 compatibility: detail requests carry the list row's domain even
|
|
42
|
+
// though standalone files all live in one fixed recordings/YYYY-MM store.
|
|
43
|
+
router.get('/meetings/:domain/:month/:filename', (req, res) => {
|
|
44
|
+
try {
|
|
45
|
+
res.set('Cache-Control', 'private, no-store')
|
|
46
|
+
res.json(store.detail(req.params.domain, req.params.month, req.params.filename))
|
|
47
|
+
} catch (error) {
|
|
48
|
+
sendMeetingStoreError(res, error)
|
|
49
|
+
}
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
return router
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function sendMeetingStoreError(
|
|
56
|
+
res: { status: (status: number) => { json: (body: unknown) => unknown } },
|
|
57
|
+
error: unknown,
|
|
58
|
+
): unknown {
|
|
59
|
+
if (error instanceof MeetingStoreError) {
|
|
60
|
+
return res.status(error.status).json({ error: error.message, reason: error.code })
|
|
61
|
+
}
|
|
62
|
+
console.error('[meetings] Store read failed:', error)
|
|
63
|
+
return res.status(500).json({ error: 'Meeting store unavailable', reason: 'meeting_store_error' })
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export const meetingsRouter = createMeetingsRouter()
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import { Router } from 'express'
|
|
2
|
+
import { createHash } from 'node:crypto'
|
|
3
|
+
import type { Response } from 'express'
|
|
4
|
+
import { existsSync, mkdirSync, readFileSync } from 'node:fs'
|
|
5
|
+
import { resolve, dirname } from 'node:path'
|
|
6
|
+
import {
|
|
7
|
+
createPromptDraft,
|
|
8
|
+
loadPromptDraftMeta,
|
|
9
|
+
savePromptDraftChunk,
|
|
10
|
+
readPromptDraftChunks,
|
|
11
|
+
markPromptDraftFinalized,
|
|
12
|
+
markPromptDraftChunkTranscript,
|
|
13
|
+
markPromptDraftError,
|
|
14
|
+
getMissingChunkIndexes,
|
|
15
|
+
prunePromptDrafts,
|
|
16
|
+
type PromptDraftTranscriptRecord,
|
|
17
|
+
} from '../lib/prompt-draft-store.js'
|
|
18
|
+
import {
|
|
19
|
+
transcribeAudioBuffer,
|
|
20
|
+
resolveTranscribeMode,
|
|
21
|
+
NoSpeechDetectedError,
|
|
22
|
+
OpenAIWhisperBudgetExhaustedError,
|
|
23
|
+
TranscriptionUnavailableError,
|
|
24
|
+
} from '../lib/transcribe-audio.js'
|
|
25
|
+
import {
|
|
26
|
+
stripInlineHallucinationsOneShot,
|
|
27
|
+
stripInlineHallucinations,
|
|
28
|
+
stripPromptDictationArtifacts,
|
|
29
|
+
isFullHallucination,
|
|
30
|
+
isBrandUrlOnly,
|
|
31
|
+
clearSessionHallucinationState,
|
|
32
|
+
applyNegativeRules,
|
|
33
|
+
} from '../lib/hallucination-filter.js'
|
|
34
|
+
import { applyCorrections } from '../lib/whisper-local.js'
|
|
35
|
+
import { autoCleanDictation, AUTOCLEAN_MAX_CHARS } from '../lib/dictation-clean.js'
|
|
36
|
+
import { getVocabulary } from '../lib/profile.js'
|
|
37
|
+
import { createBreaker } from '../lib/claude-circuit.js'
|
|
38
|
+
import { logTokenAudit } from '../lib/token-audit.js'
|
|
39
|
+
import { atomicWriteFileSync } from '../lib/atomic-fs.js'
|
|
40
|
+
import { dataPath } from '../lib/data-dir.js'
|
|
41
|
+
|
|
42
|
+
export const promptDraftsRouter = Router()
|
|
43
|
+
|
|
44
|
+
const MAX_CHUNK_BYTES = 25 * 1024 * 1024
|
|
45
|
+
const MAX_DRAFT_BYTES = 256 * 1024 * 1024
|
|
46
|
+
const MAX_CHUNKS = 600
|
|
47
|
+
const chunkTranscriptJobs = new Map<string, Promise<string>>()
|
|
48
|
+
const finalizeJobs = new Map<string, Promise<any>>()
|
|
49
|
+
let warmTail: Promise<void> = Promise.resolve()
|
|
50
|
+
|
|
51
|
+
const autoCleanBreaker = createBreaker({ label: 'dictation-autoclean' })
|
|
52
|
+
const autoCleanCountFile = () => process.env.COS_DICTATION_AUTOCLEAN_COUNT_FILE || dataPath('.dictation_autoclean_count.json')
|
|
53
|
+
const autoCleanDefaultEnabled = () => ['1', 'true', 'on'].includes((process.env.COS_DICTATION_AUTOCLEAN ?? '').toLowerCase())
|
|
54
|
+
const autoCleanDailyCap = () => {
|
|
55
|
+
const value = Number.parseInt(process.env.COS_DICTATION_AUTOCLEAN_MAX_PER_DAY || '200', 10)
|
|
56
|
+
return Number.isFinite(value) && value > 0 ? value : 200
|
|
57
|
+
}
|
|
58
|
+
function autoCleanCountToday(): number {
|
|
59
|
+
try {
|
|
60
|
+
const raw = JSON.parse(readFileSync(autoCleanCountFile(), 'utf-8'))
|
|
61
|
+
if (raw?.date === new Date().toISOString().slice(0, 10) && Number.isFinite(raw?.count)) return raw.count
|
|
62
|
+
} catch {}
|
|
63
|
+
return 0
|
|
64
|
+
}
|
|
65
|
+
function recordAutoCleanCall(): void {
|
|
66
|
+
try {
|
|
67
|
+
const file = autoCleanCountFile()
|
|
68
|
+
if (!existsSync(dirname(file))) mkdirSync(dirname(file), { recursive: true })
|
|
69
|
+
atomicWriteFileSync(file, JSON.stringify({ date: new Date().toISOString().slice(0, 10), count: autoCleanCountToday() + 1 }), { mode: 0o600 })
|
|
70
|
+
} catch {}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
interface AutoCleanRequest { enabled?: boolean; model?: 'haiku' | 'sonnet' }
|
|
74
|
+
function routeAutoClean(req: { body?: any; query?: any }): AutoCleanRequest {
|
|
75
|
+
const rawEnabled = req.body?.autoclean ?? req.query?.autoclean
|
|
76
|
+
const enabled = rawEnabled === undefined ? undefined : ['1', 'true', 'on'].includes(String(rawEnabled).toLowerCase())
|
|
77
|
+
const rawModel = String(req.body?.autocleanModel ?? req.query?.autocleanModel ?? '').toLowerCase()
|
|
78
|
+
return { enabled, model: rawModel === 'sonnet' ? 'sonnet' : rawModel === 'haiku' ? 'haiku' : undefined }
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function cleanOutboundDictation(text: string, opts: AutoCleanRequest & { signal?: AbortSignal }): Promise<string> {
|
|
82
|
+
let cleaned = applyNegativeRules(applyCorrections(text)).replace(/\s+/g, ' ').trim() || text
|
|
83
|
+
if (!(opts.enabled ?? autoCleanDefaultEnabled())) return cleaned
|
|
84
|
+
if (cleaned.length > AUTOCLEAN_MAX_CHARS || autoCleanBreaker.isOpen() || autoCleanCountToday() >= autoCleanDailyCap()) return cleaned
|
|
85
|
+
const startedAt = Date.now()
|
|
86
|
+
const model = opts.model === 'sonnet' ? 'sonnet' : 'haiku'
|
|
87
|
+
recordAutoCleanCall()
|
|
88
|
+
try {
|
|
89
|
+
const polished = (await autoCleanDictation(cleaned, getVocabulary(), { model, signal: opts.signal })).trim()
|
|
90
|
+
autoCleanBreaker.recordSuccess()
|
|
91
|
+
logTokenAudit({
|
|
92
|
+
source: 'g2-dictation-autoclean', model, inputChars: cleaned.length, outputChars: polished.length,
|
|
93
|
+
durationMs: Date.now() - startedAt, caller: 'dictation_autoclean',
|
|
94
|
+
})
|
|
95
|
+
return polished || cleaned
|
|
96
|
+
} catch (err: any) {
|
|
97
|
+
autoCleanBreaker.recordFailure()
|
|
98
|
+
console.warn(`[prompt-draft] auto-clean failed (glossary-only): ${err?.message ?? err}`)
|
|
99
|
+
return cleaned
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function readRawBody(req: AsyncIterable<Buffer | Uint8Array | string>): Promise<Buffer> {
|
|
104
|
+
const chunks: Buffer[] = []
|
|
105
|
+
let total = 0
|
|
106
|
+
for await (const chunk of req) {
|
|
107
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
|
108
|
+
total += buffer.length
|
|
109
|
+
if (total > MAX_CHUNK_BYTES) throw Object.assign(new Error('audio chunk too large'), { status: 413 })
|
|
110
|
+
chunks.push(buffer)
|
|
111
|
+
}
|
|
112
|
+
return Buffer.concat(chunks)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function routeMode(req: { body?: { mode?: string }; query?: { mode?: string | string[] } }) {
|
|
116
|
+
return resolveTranscribeMode(
|
|
117
|
+
(typeof req.body?.mode === 'string' ? req.body.mode : undefined) ??
|
|
118
|
+
(typeof req.query?.mode === 'string' ? req.query.mode : undefined),
|
|
119
|
+
)
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const sessionId = (draftId: string) => `prompt-draft:${draftId}`
|
|
123
|
+
const audioHash = (audio: Buffer) => createHash('sha256').update(audio).digest('hex')
|
|
124
|
+
function isCurrentChunk(draftId: string, chunkIndex: number, audio: Buffer): boolean {
|
|
125
|
+
try {
|
|
126
|
+
return Boolean(readPromptDraftChunks(draftId).find(chunk => chunk.chunkIndex === chunkIndex)?.audioBuffer.equals(audio))
|
|
127
|
+
} catch {
|
|
128
|
+
return false
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
function sanitizeTranscript(draftId: string, text: string, learnInline = true): string {
|
|
132
|
+
const artifactCleaned = stripPromptDictationArtifacts(text).trim()
|
|
133
|
+
if (isBrandUrlOnly(artifactCleaned)) return ''
|
|
134
|
+
const oneShot = stripInlineHallucinationsOneShot(artifactCleaned).trim()
|
|
135
|
+
const cleaned = learnInline ? stripInlineHallucinations(oneShot, sessionId(draftId)).trim() : oneShot
|
|
136
|
+
return !cleaned || isFullHallucination(cleaned) ? '' : cleaned
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function sendDraftError(res: Response, draftId: string, err: any): Promise<void> {
|
|
140
|
+
if (err instanceof NoSpeechDetectedError) return void res.status(204).send()
|
|
141
|
+
if (err instanceof OpenAIWhisperBudgetExhaustedError) {
|
|
142
|
+
await markPromptDraftError(draftId, err.message)
|
|
143
|
+
return void res.status(503).json({ error: err.message, reason: 'openai_whisper_budget_exhausted', spent_today_usd: err.spentTodayUsd, cap_usd: err.capUsd })
|
|
144
|
+
}
|
|
145
|
+
if (err instanceof TranscriptionUnavailableError) {
|
|
146
|
+
await markPromptDraftError(draftId, err.message)
|
|
147
|
+
return void res.status(err.status).json({ error: err.message, reason: err.reason, retryable: true, draftPreserved: true })
|
|
148
|
+
}
|
|
149
|
+
await markPromptDraftError(draftId, err.message).catch(() => null)
|
|
150
|
+
res.status(err.status ?? 500).json({ error: err.message })
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffer, mode: 'hq' | 'fast', purpose: 'warm' | 'final'): Promise<string> {
|
|
154
|
+
const hash = audioHash(audio)
|
|
155
|
+
const key = `${draftId}:${chunkIndex}:${purpose}:${mode}:${hash}`
|
|
156
|
+
const existing = chunkTranscriptJobs.get(key)
|
|
157
|
+
if (existing) return existing
|
|
158
|
+
const job = (async () => {
|
|
159
|
+
try {
|
|
160
|
+
const result = await transcribeAudioBuffer(audio, { mode, policy: purpose === 'warm' ? 'local-only' : 'automatic' })
|
|
161
|
+
if (!isCurrentChunk(draftId, chunkIndex, audio)) return ''
|
|
162
|
+
const text = sanitizeTranscript(draftId, result.text)
|
|
163
|
+
const record: PromptDraftTranscriptRecord = {
|
|
164
|
+
text, hash, requestedMode: result.requestedMode, actualQuality: result.actualQuality,
|
|
165
|
+
backend: result.backend, degraded: result.degraded,
|
|
166
|
+
}
|
|
167
|
+
await markPromptDraftChunkTranscript(draftId, chunkIndex, record, purpose)
|
|
168
|
+
console.log(`[prompt-draft] chunk ${draftId}/${chunkIndex}: ${result.elapsedMs.toFixed(1)}ms | ${result.backend} | ${text.length} chars`)
|
|
169
|
+
return text
|
|
170
|
+
} catch (err) {
|
|
171
|
+
if (err instanceof NoSpeechDetectedError) {
|
|
172
|
+
await markPromptDraftChunkTranscript(draftId, chunkIndex, {
|
|
173
|
+
text: '', hash, requestedMode: mode, actualQuality: mode, backend: 'no-speech', degraded: false,
|
|
174
|
+
}, purpose)
|
|
175
|
+
return ''
|
|
176
|
+
}
|
|
177
|
+
throw err
|
|
178
|
+
} finally {
|
|
179
|
+
chunkTranscriptJobs.delete(key)
|
|
180
|
+
}
|
|
181
|
+
})()
|
|
182
|
+
chunkTranscriptJobs.set(key, job)
|
|
183
|
+
return job
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: AutoCleanRequest, signal?: AbortSignal) {
|
|
187
|
+
const meta = loadPromptDraftMeta(draftId)
|
|
188
|
+
if (!meta) throw Object.assign(new Error('draft not found'), { status: 404 })
|
|
189
|
+
const texts: string[] = []
|
|
190
|
+
for (const chunk of readPromptDraftChunks(draftId)) {
|
|
191
|
+
try {
|
|
192
|
+
const current = loadPromptDraftMeta(draftId)
|
|
193
|
+
const cached = current?.finalTranscripts?.[String(chunk.chunkIndex)] ?? current?.warmTranscripts?.[String(chunk.chunkIndex)]
|
|
194
|
+
const reusable = Boolean(cached && cached.hash === audioHash(chunk.audioBuffer) && (mode === 'fast' ? cached.actualQuality === 'fast' : cached.actualQuality === 'hq'))
|
|
195
|
+
const raw = reusable ? cached!.text : await transcribeChunk(draftId, chunk.chunkIndex, chunk.audioBuffer, mode, 'final')
|
|
196
|
+
const text = sanitizeTranscript(draftId, raw, !reusable)
|
|
197
|
+
if (text.trim()) texts.push(text.trim())
|
|
198
|
+
} catch (err) {
|
|
199
|
+
if (err instanceof NoSpeechDetectedError) continue
|
|
200
|
+
throw err
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
const text = texts.join(' ').replace(/\s+/g, ' ').trim()
|
|
204
|
+
if (!text) {
|
|
205
|
+
await markPromptDraftError(draftId, 'No speech detected')
|
|
206
|
+
throw new NoSpeechDetectedError()
|
|
207
|
+
}
|
|
208
|
+
const finalText = await cleanOutboundDictation(text, { ...autoClean, signal })
|
|
209
|
+
const finalized = await markPromptDraftFinalized(draftId, finalText)
|
|
210
|
+
return { draftId, text: finalText, recovered: true, chunkCount: finalized.receivedChunkIndexes.length, missingChunks: getMissingChunkIndexes(finalized), expiresAt: finalized.expiresAt }
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const prunedAtBoot = prunePromptDrafts()
|
|
214
|
+
if (prunedAtBoot) console.log(`[prompt-draft] pruned ${prunedAtBoot} expired draft(s)`)
|
|
215
|
+
const pruneTimer = setInterval(() => prunePromptDrafts(), 60 * 60 * 1000)
|
|
216
|
+
pruneTimer.unref?.()
|
|
217
|
+
|
|
218
|
+
promptDraftsRouter.post('/prompt-drafts/start', (req, res) => {
|
|
219
|
+
const requestedId = typeof req.body?.recoveryId === 'string' ? req.body.recoveryId : undefined
|
|
220
|
+
const meta = createPromptDraft(requestedId)
|
|
221
|
+
res.json({ draftId: meta.draftId, recoveryId: requestedId ?? meta.draftId, remapped: Boolean(requestedId && requestedId !== meta.draftId), expiresAt: meta.expiresAt, status: meta.status })
|
|
222
|
+
})
|
|
223
|
+
|
|
224
|
+
promptDraftsRouter.post('/prompt-drafts/:draftId/chunks', async (req, res) => {
|
|
225
|
+
try {
|
|
226
|
+
const raw = Array.isArray(req.query.chunkIndex) ? req.query.chunkIndex[0] : req.query.chunkIndex
|
|
227
|
+
const chunkIndex = Number(raw)
|
|
228
|
+
if (!Number.isInteger(chunkIndex) || chunkIndex < 0 || chunkIndex >= MAX_CHUNKS) return res.status(400).json({ error: 'invalid chunkIndex' })
|
|
229
|
+
const audio = await readRawBody(req)
|
|
230
|
+
if (audio.length < 44) return res.status(400).json({ error: 'audio too short' })
|
|
231
|
+
const before = loadPromptDraftMeta(req.params.draftId)
|
|
232
|
+
if (!before) return res.status(404).json({ error: 'draft not found' })
|
|
233
|
+
const existingBytes = before.chunkBytes[String(chunkIndex)] ?? 0
|
|
234
|
+
const nextTotal = Object.values(before.chunkBytes).reduce((sum, bytes) => sum + bytes, 0) - existingBytes + audio.length
|
|
235
|
+
if (nextTotal > MAX_DRAFT_BYTES) return res.status(413).json({ error: 'prompt draft too large' })
|
|
236
|
+
const meta = await savePromptDraftChunk(req.params.draftId, chunkIndex, audio)
|
|
237
|
+
warmTail = warmTail.then(() => transcribeChunk(req.params.draftId, chunkIndex, audio, 'fast', 'warm').then(() => undefined)).catch(err => {
|
|
238
|
+
console.warn(`[prompt-draft] warm transcription failed ${req.params.draftId}/${chunkIndex}: ${err.message}`)
|
|
239
|
+
})
|
|
240
|
+
res.json({ draftId: meta.draftId, chunkIndex, acked: true, receivedChunkIndexes: meta.receivedChunkIndexes, chunkBytes: meta.chunkBytes[String(chunkIndex)] ?? audio.length, transcriptPending: true, expiresAt: meta.expiresAt })
|
|
241
|
+
} catch (err: any) {
|
|
242
|
+
res.status(err.status ?? (err.message === 'draft not found' ? 404 : 500)).json({ error: err.message })
|
|
243
|
+
}
|
|
244
|
+
})
|
|
245
|
+
|
|
246
|
+
async function finalizeRequest(req: any, res: Response): Promise<void> {
|
|
247
|
+
const abort = new AbortController()
|
|
248
|
+
res.on('close', () => { if (!res.writableEnded) abort.abort() })
|
|
249
|
+
try {
|
|
250
|
+
const mode = routeMode(req)
|
|
251
|
+
const key = `${req.params.draftId}:${mode}`
|
|
252
|
+
let job = finalizeJobs.get(key)
|
|
253
|
+
if (!job) {
|
|
254
|
+
job = finalizeDraft(req.params.draftId, mode, routeAutoClean(req), abort.signal)
|
|
255
|
+
finalizeJobs.set(key, job)
|
|
256
|
+
job.finally(() => finalizeJobs.delete(key)).catch(() => {})
|
|
257
|
+
}
|
|
258
|
+
res.json(await job)
|
|
259
|
+
} catch (err: any) {
|
|
260
|
+
await sendDraftError(res, req.params.draftId, err)
|
|
261
|
+
} finally {
|
|
262
|
+
clearSessionHallucinationState(sessionId(req.params.draftId))
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
promptDraftsRouter.post('/prompt-drafts/:draftId/finalize', finalizeRequest)
|
|
267
|
+
promptDraftsRouter.post('/prompt-drafts/:draftId/retry', finalizeRequest)
|
|
268
|
+
promptDraftsRouter.get('/prompt-drafts/:draftId', (req, res) => {
|
|
269
|
+
try {
|
|
270
|
+
const meta = loadPromptDraftMeta(req.params.draftId)
|
|
271
|
+
if (!meta) return res.status(404).json({ error: 'draft not found' })
|
|
272
|
+
res.json({ ...meta, missingChunks: getMissingChunkIndexes(meta) })
|
|
273
|
+
} catch (err: any) {
|
|
274
|
+
res.status(500).json({ error: err.message })
|
|
275
|
+
}
|
|
276
|
+
})
|