@gotcos/glasses-server 6.71.13 → 6.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +16 -6
- package/CHANGELOG.md +29 -0
- package/README.md +160 -1
- package/docs/stt-672-evaluation.md +113 -0
- package/package.json +4 -1
- package/server/index.ts +18 -1
- package/server/lib/batch-transcript-quality.ts +46 -0
- package/server/lib/meeting-batch-persistence.ts +23 -1
- package/server/lib/meeting-batch-transcribe.ts +408 -45
- package/server/lib/parakeet-manifest.ts +41 -0
- package/server/lib/parakeet.ts +665 -0
- package/server/lib/stale-active-session-files.ts +178 -0
- package/server/lib/stt-live.ts +60 -0
- package/server/lib/stt-resolver.ts +327 -0
- package/server/lib/stt-runtime.ts +225 -0
- package/server/lib/stt-settings.ts +342 -0
- package/server/lib/unsaved-audio-quarantine.ts +114 -0
- package/server/lib/vad-silero.ts +52 -0
- package/server/lib/whisper-batch-timing.ts +13 -3
- package/server/lib/whisper-local.ts +58 -17
- package/server/routes/meeting.ts +84 -4
- package/server/routes/stt-settings.ts +85 -0
- package/server/routes/transcribe-stream.ts +179 -24
|
@@ -347,3 +347,117 @@ export function markRecovered(dirPath: string, savedFilename: string): void {
|
|
|
347
347
|
)
|
|
348
348
|
} catch { /* receipt is best-effort; findBySessionId remains the true guard */ }
|
|
349
349
|
}
|
|
350
|
+
|
|
351
|
+
// ── Terminal disposition of a capture the server no longer holds live ───────
|
|
352
|
+
//
|
|
353
|
+
// 6.71.14. The quarantine is the only record of what happened to a capture whose
|
|
354
|
+
// live session is gone: its audio was quarantined, then recovered into a meeting or
|
|
355
|
+
// receipted as no_speech. Two readers need that answer and used to get "missing":
|
|
356
|
+
// the session status route (COS Control rendered "Recording ended without saving"
|
|
357
|
+
// for a capture that had been recovered and receipted) and the chunk route, which
|
|
358
|
+
// recreated a live session for a late chunk of a capture that was already done.
|
|
359
|
+
|
|
360
|
+
export type CaptureDispositionKind = 'quarantined' | 'recovered' | 'no_speech'
|
|
361
|
+
|
|
362
|
+
export interface CaptureDisposition {
|
|
363
|
+
kind: CaptureDispositionKind
|
|
364
|
+
sessionId: string
|
|
365
|
+
dirName: string
|
|
366
|
+
quarantinedAt: string | null
|
|
367
|
+
/** When the recovery receipt was written, for `recovered` and `no_speech`. */
|
|
368
|
+
receiptedAt: string | null
|
|
369
|
+
chunkFiles: number
|
|
370
|
+
/** The quarantine manifest's reason (why the audio was moved), when known. */
|
|
371
|
+
quarantineReason: string | null
|
|
372
|
+
/** Set for `recovered`: the meeting file the recovery saved. */
|
|
373
|
+
savedFilename: string | null
|
|
374
|
+
/** A recovery is decoding this capture right now. */
|
|
375
|
+
recovering: boolean
|
|
376
|
+
/** When quarantine retention removes the audio. */
|
|
377
|
+
retainedUntil: string | null
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
interface RecoveredReceipt {
|
|
381
|
+
recoveredAt?: unknown
|
|
382
|
+
outcome?: unknown
|
|
383
|
+
savedFilename?: unknown
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
function readRecoveredReceipt(dirPath: string): RecoveredReceipt | null {
|
|
387
|
+
const path = resolve(dirPath, RECOVERED_RECEIPT)
|
|
388
|
+
if (!existsSync(path)) return null
|
|
389
|
+
try {
|
|
390
|
+
const raw = JSON.parse(readFileSync(path, 'utf8')) as unknown
|
|
391
|
+
return raw && typeof raw === 'object' ? raw as RecoveredReceipt : {}
|
|
392
|
+
} catch {
|
|
393
|
+
// Present but unreadable still means a recovery finished: the receipt is only
|
|
394
|
+
// ever written at the end of one (listUnsavedCaptures counts it the same way).
|
|
395
|
+
return {}
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/** Every quarantine dir for a session id: the exact name and any `<id>.<ms>` copies. */
|
|
400
|
+
function quarantineDirsFor(sessionId: string, root: string): string[] {
|
|
401
|
+
const dirs: string[] = []
|
|
402
|
+
let entries: string[] = []
|
|
403
|
+
try { entries = readdirSync(root) } catch { return dirs }
|
|
404
|
+
for (const dir of entries) {
|
|
405
|
+
// Same match as findQuarantineDir: session ids carry no dots, so `<id>.` is this id.
|
|
406
|
+
if (dir !== sessionId && !dir.startsWith(`${sessionId}.`)) continue
|
|
407
|
+
const dirPath = resolve(root, dir)
|
|
408
|
+
try { if (statSync(dirPath).isDirectory()) dirs.push(dirPath) } catch { /* vanished: skip */ }
|
|
409
|
+
}
|
|
410
|
+
return dirs.sort()
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function dispositionOf(sessionId: string, dirPath: string, retentionMs: number): CaptureDisposition {
|
|
414
|
+
const manifest = readManifest(dirPath)
|
|
415
|
+
const atMs = quarantinedAtMs(dirPath, manifest)
|
|
416
|
+
const receipt = readRecoveredReceipt(dirPath)
|
|
417
|
+
const receiptedMs = receipt && typeof receipt.recoveredAt === 'string' ? Date.parse(receipt.recoveredAt) : NaN
|
|
418
|
+
const savedFilename = receipt && typeof receipt.savedFilename === 'string' && receipt.savedFilename
|
|
419
|
+
? receipt.savedFilename
|
|
420
|
+
: null
|
|
421
|
+
const kind: CaptureDispositionKind = !receipt
|
|
422
|
+
? 'quarantined'
|
|
423
|
+
: receipt.outcome === 'no_speech'
|
|
424
|
+
? 'no_speech'
|
|
425
|
+
: 'recovered'
|
|
426
|
+
return {
|
|
427
|
+
kind,
|
|
428
|
+
sessionId,
|
|
429
|
+
dirName: basename(dirPath),
|
|
430
|
+
quarantinedAt: atMs != null ? new Date(atMs).toISOString() : null,
|
|
431
|
+
receiptedAt: Number.isFinite(receiptedMs) ? new Date(receiptedMs).toISOString() : null,
|
|
432
|
+
chunkFiles: manifest?.chunkFiles ?? countChunkWavs(dirPath),
|
|
433
|
+
quarantineReason: manifest?.reason ?? null,
|
|
434
|
+
savedFilename,
|
|
435
|
+
recovering: isRecoveryActive(basename(dirPath)),
|
|
436
|
+
retainedUntil: atMs != null ? new Date(atMs + retentionMs).toISOString() : null,
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
const DISPOSITION_RANK: Record<CaptureDispositionKind, number> = { quarantined: 0, recovered: 1, no_speech: 2 }
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* What became of a capture whose audio reached quarantine, or null when the
|
|
444
|
+
* quarantine holds nothing for this session id.
|
|
445
|
+
*
|
|
446
|
+
* `no_speech` and `recovered` are terminal: the capture was decoded once and its
|
|
447
|
+
* outcome receipted. `quarantined` is held audio that a recovery has not finished
|
|
448
|
+
* yet (the sweep picks one per minute, or a person presses Recover).
|
|
449
|
+
*
|
|
450
|
+
* One id can own SEVERAL dirs (`<id>` and `<id>.<ms>`, when the same id was
|
|
451
|
+
* quarantined twice). The answer covers all of them: ANY unrecovered dir means the
|
|
452
|
+
* capture is still `quarantined`; otherwise a saved recovery outranks no_speech.
|
|
453
|
+
*/
|
|
454
|
+
export function readCaptureDisposition(
|
|
455
|
+
sessionId: string,
|
|
456
|
+
root: string = unsavedAudioRoot(),
|
|
457
|
+
retentionMs: number = unsavedAudioRetentionMs(),
|
|
458
|
+
): CaptureDisposition | null {
|
|
459
|
+
if (!/^[A-Za-z0-9:_-]{3,96}$/.test(sessionId)) return null
|
|
460
|
+
const all = quarantineDirsFor(sessionId, root).map(dir => dispositionOf(sessionId, dir, retentionMs))
|
|
461
|
+
if (all.length === 0) return null
|
|
462
|
+
return all.reduce((best, next) => DISPOSITION_RANK[next.kind] < DISPOSITION_RANK[best.kind] ? next : best)
|
|
463
|
+
}
|
package/server/lib/vad-silero.ts
CHANGED
|
@@ -27,6 +27,12 @@ const MIN_SAMPLES = 512
|
|
|
27
27
|
let sherpaOnnx: any = null
|
|
28
28
|
let available = false
|
|
29
29
|
let vadInstance: any = null
|
|
30
|
+
// 6.72.0 quiet gate: its own detector, stricter than the trimming one. Chosen on
|
|
31
|
+
// bake-off canary 5 (docs/stt-672-evaluation.md): probability 0.8 with a 0.25 s
|
|
32
|
+
// minimum, and quiet below 0.5 s of speech, fired on 80.8% of Silero-negative
|
|
33
|
+
// chunks and 0.91% of speech chunks.
|
|
34
|
+
export const QUIET_GATE_VAD = { threshold: 0.8, minSilenceDuration: 0.3, minSpeechDuration: 0.25 } as const
|
|
35
|
+
let gateVad: any = null
|
|
30
36
|
|
|
31
37
|
export interface TrimResult {
|
|
32
38
|
trimmedWav: Buffer
|
|
@@ -60,6 +66,11 @@ export function initSileroVAD(): boolean {
|
|
|
60
66
|
10, // bufferSizeInSeconds — chunks are max 6s
|
|
61
67
|
)
|
|
62
68
|
vadInstance.reset() // verify it works
|
|
69
|
+
gateVad = new sherpaOnnx.Vad(
|
|
70
|
+
{ sileroVad: { model: MODEL_PATH, ...QUIET_GATE_VAD, windowSize: 512 }, sampleRate: SAMPLE_RATE, numThreads: 1, provider: 'cpu' },
|
|
71
|
+
10,
|
|
72
|
+
)
|
|
73
|
+
gateVad.reset()
|
|
63
74
|
available = true
|
|
64
75
|
console.log('[silero-vad] Initialized: Silero VAD active (reused instance, bufferSize=10s)')
|
|
65
76
|
return true
|
|
@@ -182,3 +193,44 @@ export function trimSilence(wavBuffer: Buffer): TrimResult {
|
|
|
182
193
|
return fallback
|
|
183
194
|
}
|
|
184
195
|
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Seconds of speech the quiet-gate Silero (QUIET_GATE_VAD) hears in a WAV (6.72.0).
|
|
199
|
+
*
|
|
200
|
+
* Feeds the detector one 512-sample window at a time and drains finished
|
|
201
|
+
* segments after each window. Handing sherpa-onnx the whole chunk in one
|
|
202
|
+
* acceptWaveform call (as trimSilence does) keeps only the last segment:
|
|
203
|
+
* measured 2026-10-13, a 6 s chunk of continuous speech read 0.32 s that way
|
|
204
|
+
* and 5.03 s windowed. Null when the detector is unavailable or the audio is
|
|
205
|
+
* too short to measure: a caller must never read null as silence.
|
|
206
|
+
*/
|
|
207
|
+
export function measureSpeechSeconds(wavBuffer: Buffer): number | null {
|
|
208
|
+
if (!available || !gateVad) return null
|
|
209
|
+
try {
|
|
210
|
+
if (wavBuffer.length <= WAV_HEADER_SIZE + MIN_SAMPLES * BYTES_PER_SAMPLE) return null
|
|
211
|
+
const pcmBytes = wavBuffer.buffer.slice(wavBuffer.byteOffset + WAV_HEADER_SIZE, wavBuffer.byteOffset + wavBuffer.length - ((wavBuffer.length - WAV_HEADER_SIZE) % 2))
|
|
212
|
+
const int16 = new Int16Array(pcmBytes)
|
|
213
|
+
if (int16.length < MIN_SAMPLES) return null
|
|
214
|
+
const float32 = new Float32Array(int16.length)
|
|
215
|
+
for (let i = 0; i < int16.length; i++) float32[i] = int16[i] / 32768
|
|
216
|
+
gateVad.reset()
|
|
217
|
+
let speechSamples = 0
|
|
218
|
+
const drain = () => {
|
|
219
|
+
while (!gateVad.isEmpty()) {
|
|
220
|
+
speechSamples += gateVad.front().samples.length
|
|
221
|
+
gateVad.pop()
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
for (let offset = 0; offset + MIN_SAMPLES <= float32.length; offset += MIN_SAMPLES) {
|
|
225
|
+
gateVad.acceptWaveform(float32.subarray(offset, offset + MIN_SAMPLES))
|
|
226
|
+
drain()
|
|
227
|
+
}
|
|
228
|
+
gateVad.flush()
|
|
229
|
+
drain()
|
|
230
|
+
gateVad.reset()
|
|
231
|
+
return speechSamples / SAMPLE_RATE
|
|
232
|
+
} catch (err: any) {
|
|
233
|
+
console.error('[silero-vad] measureSpeechSeconds error:', err.message)
|
|
234
|
+
return null
|
|
235
|
+
}
|
|
236
|
+
}
|
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
import type { WhisperWord } from './whisper-local.js'
|
|
2
|
+
import { readSttSettings } from './stt-settings.js'
|
|
2
3
|
|
|
3
|
-
/**
|
|
4
|
+
/** Meeting batch HQ timestamps (6.72.0 default: on). The HQ setting decides:
|
|
5
|
+
* "large-v3" decodes with timestamps and remaps them through Whisper's own VAD
|
|
6
|
+
* table; "large-v3-legacy" keeps the pre-6.72 `--no-timestamps` decoder.
|
|
7
|
+
* COS_BATCH_HQ_TIMESTAMPS still overrides: "1" forces timestamps on, "0" off.
|
|
4
8
|
* Live HTTP keeps compact JSON; interactive HQ and DTW are unchanged. */
|
|
5
|
-
export function
|
|
6
|
-
|
|
9
|
+
export function batchTimestampsEnabled(hq?: string): boolean {
|
|
10
|
+
const flag = process.env.COS_BATCH_HQ_TIMESTAMPS
|
|
11
|
+
if (flag === '1') return true
|
|
12
|
+
if (flag === '0') return false
|
|
13
|
+
return (hq ?? readSttSettings().hq) !== 'large-v3-legacy'
|
|
7
14
|
}
|
|
8
15
|
|
|
16
|
+
/** Pre-6.72 name, kept for callers and tests. */
|
|
17
|
+
export const batchTimestampCanaryEnabled = (): boolean => batchTimestampsEnabled()
|
|
18
|
+
|
|
9
19
|
export interface VadTimeSegment {
|
|
10
20
|
originalStart: number
|
|
11
21
|
originalEnd: number
|
|
@@ -13,7 +13,8 @@ import { writeFileSync, unlinkSync, existsSync, readFileSync, statSync } from 'n
|
|
|
13
13
|
import { basename, join } from 'node:path'
|
|
14
14
|
import { homedir } from 'node:os'
|
|
15
15
|
import crypto from 'node:crypto'
|
|
16
|
-
import {
|
|
16
|
+
import { batchTimestampsEnabled, parseNativeVadTimeline, remapBatchWords } from './whisper-batch-timing.js'
|
|
17
|
+
import { readSttSettings, type FrontEndId } from './stt-settings.js'
|
|
17
18
|
import { getVocabulary, getOwnerName, getWhisperCorrections } from './profile.js'
|
|
18
19
|
import { stripBrandUrls } from './hallucination-filter.js'
|
|
19
20
|
import {
|
|
@@ -70,21 +71,34 @@ export interface HighQualityTranscriptionCapability {
|
|
|
70
71
|
export interface HighQualityTranscriptionResult {
|
|
71
72
|
text: string
|
|
72
73
|
words?: WhisperWord[]
|
|
74
|
+
/** Sub-word tokens merged into spoken words (6.72.0), NOT name-corrected.
|
|
75
|
+
* Clocks are only meaningful when the decode ran with timestamps. */
|
|
76
|
+
spokenWords?: WhisperWord[]
|
|
77
|
+
/** True when this decode ran with timestamps and the VAD remap. */
|
|
78
|
+
timestamps?: boolean
|
|
79
|
+
/** Where this decode's own VAD heard speech (seconds, input clock). Timestamp mode only. */
|
|
80
|
+
speechSpans?: Array<[number, number]>
|
|
73
81
|
model: 'large-v3' | 'turbo'
|
|
74
82
|
backend: 'whisper-cli' | 'whisper-server'
|
|
75
83
|
actualQuality: 'hq' | 'fast'
|
|
76
84
|
degradationReason?: HighQualityUnavailableReason
|
|
77
85
|
}
|
|
78
86
|
|
|
79
|
-
|
|
80
|
-
let highQualityCheckpointTimingMode: boolean | null = null
|
|
87
|
+
const highQualityCheckpointFingerprints = new Map<string, string>()
|
|
81
88
|
|
|
82
89
|
/** Stable, path-free cache identity for progressive meeting HQ checkpoints.
|
|
83
90
|
* Model/configuration is immutable for the lifetime of one server process, so
|
|
84
|
-
* cache this instead of stat'ing multi-GB model files on every health poll.
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
91
|
+
* cache this instead of stat'ing multi-GB model files on every health poll.
|
|
92
|
+
* 6.72.0: the timing mode and the ASR front end are part of the identity, so a
|
|
93
|
+
* checkpoint decoded under one setting is never reused under another. With
|
|
94
|
+
* timestamps off and the full front end the identity is the pre-6.72 one, byte
|
|
95
|
+
* for byte. */
|
|
96
|
+
export function getHighQualityCheckpointFingerprint(opts: { timestamps?: boolean; frontEnd?: FrontEndId } = {}): string {
|
|
97
|
+
const timingMode = opts.timestamps ?? batchTimestampsEnabled()
|
|
98
|
+
const frontEnd: FrontEndId = opts.frontEnd ?? readSttSettings().frontEnd
|
|
99
|
+
const cacheKey = `${timingMode ? 1 : 0}|${frontEnd}`
|
|
100
|
+
const cached = highQualityCheckpointFingerprints.get(cacheKey)
|
|
101
|
+
if (cached) return cached
|
|
88
102
|
const identity = (path: string): Record<string, number | boolean> => {
|
|
89
103
|
try {
|
|
90
104
|
const stat = statSync(path)
|
|
@@ -93,9 +107,9 @@ export function getHighQualityCheckpointFingerprint(): string {
|
|
|
93
107
|
return { present: false, size: 0, mtimeMs: 0 }
|
|
94
108
|
}
|
|
95
109
|
}
|
|
96
|
-
|
|
110
|
+
const fingerprint = crypto.createHash('sha256').update(JSON.stringify({
|
|
97
111
|
schema: 2,
|
|
98
|
-
// Keep the existing legacy identity byte-for-byte when
|
|
112
|
+
// Keep the existing legacy identity byte-for-byte when timestamps are off.
|
|
99
113
|
...(timingMode ? { batchTiming: 'native-vad-timestamps-v1' } : {}),
|
|
100
114
|
serverVersion: process.env.COS_SERVER_VERSION?.trim() || 'development',
|
|
101
115
|
whisperCli: identity(WHISPER_CLI),
|
|
@@ -104,12 +118,12 @@ export function getHighQualityCheckpointFingerprint(): string {
|
|
|
104
118
|
beam: 5,
|
|
105
119
|
bestOf: 5,
|
|
106
120
|
vadEnabled: hqCliVadEnabled('batch'),
|
|
107
|
-
enhancement: 'audio-enhance-v1',
|
|
121
|
+
enhancement: frontEnd === 'full' ? 'audio-enhance-v1' : 'audio-enhance-highpass-v1',
|
|
108
122
|
prompt: buildWhisperPrompt(),
|
|
109
123
|
corrections: getWhisperCorrections(),
|
|
110
124
|
})).digest('hex')
|
|
111
|
-
|
|
112
|
-
return
|
|
125
|
+
highQualityCheckpointFingerprints.set(cacheKey, fingerprint)
|
|
126
|
+
return fingerprint
|
|
113
127
|
}
|
|
114
128
|
|
|
115
129
|
export function classifyHighQualityTranscriptionCapability(input: {
|
|
@@ -882,7 +896,7 @@ async function reconcileWhisperServerHealth(): Promise<boolean> {
|
|
|
882
896
|
* Batch/save path only. Never uses whisper-server `verbose_json` (the live
|
|
883
897
|
* VAD-empty crash vector). Special tokens like `[_BEG_]` / `<|...|>` are dropped.
|
|
884
898
|
*/
|
|
885
|
-
export function parseWhisperCliFullJson(raw: string): { text: string; words: WhisperWord[] } {
|
|
899
|
+
export function parseWhisperCliFullJson(raw: string): { text: string; words: WhisperWord[]; spokenWords: WhisperWord[] } {
|
|
886
900
|
const data = JSON.parse(raw) as {
|
|
887
901
|
transcription?: Array<{
|
|
888
902
|
text?: unknown
|
|
@@ -896,6 +910,7 @@ export function parseWhisperCliFullJson(raw: string): { text: string; words: Whi
|
|
|
896
910
|
const segments = Array.isArray(data.transcription) ? data.transcription : []
|
|
897
911
|
const texts: string[] = []
|
|
898
912
|
const words: WhisperWord[] = []
|
|
913
|
+
const spokenWords: WhisperWord[] = []
|
|
899
914
|
|
|
900
915
|
for (const segment of segments) {
|
|
901
916
|
if (typeof segment.text === 'string' && segment.text.trim()) {
|
|
@@ -909,18 +924,30 @@ export function parseWhisperCliFullJson(raw: string): { text: string; words: Whi
|
|
|
909
924
|
const fromMs = typeof token.offsets?.from === 'number' ? token.offsets.from : Number(token.offsets?.from)
|
|
910
925
|
const toMs = typeof token.offsets?.to === 'number' ? token.offsets.to : Number(token.offsets?.to)
|
|
911
926
|
if (!Number.isFinite(fromMs) || !Number.isFinite(toMs)) continue
|
|
912
|
-
|
|
927
|
+
const word: WhisperWord = {
|
|
913
928
|
word: tokenText,
|
|
914
929
|
start: fromMs / 1000,
|
|
915
930
|
end: toMs / 1000,
|
|
916
931
|
probability: typeof token.p === 'number' && Number.isFinite(token.p) ? token.p : 0,
|
|
917
|
-
}
|
|
932
|
+
}
|
|
933
|
+
words.push(word)
|
|
934
|
+
// A token without a leading space continues the previous word ("I" + "'m").
|
|
935
|
+
const rawText = typeof token.text === 'string' ? token.text : ''
|
|
936
|
+
const last = spokenWords.at(-1)
|
|
937
|
+
if (last && !/^\s/.test(rawText) && !/^[\s]*$/.test(rawText)) {
|
|
938
|
+
last.word += tokenText
|
|
939
|
+
last.end = Math.max(last.end, word.end)
|
|
940
|
+
last.probability = Math.min(last.probability, word.probability)
|
|
941
|
+
} else {
|
|
942
|
+
spokenWords.push({ ...word })
|
|
943
|
+
}
|
|
918
944
|
}
|
|
919
945
|
}
|
|
920
946
|
|
|
921
947
|
return {
|
|
922
948
|
text: texts.join(' ').replace(/\s+/g, ' ').trim(),
|
|
923
949
|
words,
|
|
950
|
+
spokenWords,
|
|
924
951
|
}
|
|
925
952
|
}
|
|
926
953
|
|
|
@@ -949,6 +976,8 @@ export async function transcribeHighQuality(
|
|
|
949
976
|
threads?: number
|
|
950
977
|
backgroundCpu?: boolean
|
|
951
978
|
signal?: AbortSignal
|
|
979
|
+
/** Meeting batch only: timestamps on/off for this decode (the job's snapshot). */
|
|
980
|
+
timestamps?: boolean
|
|
952
981
|
} = {},
|
|
953
982
|
): Promise<HighQualityTranscriptionResult> {
|
|
954
983
|
if (!cliAvailable) {
|
|
@@ -972,7 +1001,9 @@ export async function transcribeHighQuality(
|
|
|
972
1001
|
// Word clocks on post-meeting CPU polish, and on any decode that asks for
|
|
973
1002
|
// them (the prompt tail guard). Other live decodes stay compact JSON.
|
|
974
1003
|
const captureWords = opts.priority === 'batch' || opts.words === true
|
|
975
|
-
|
|
1004
|
+
// 6.72.0: on by default for meeting batch (HQ setting "large-v3"); the job
|
|
1005
|
+
// passes its own snapshot so a settings change never lands mid-meeting.
|
|
1006
|
+
const timestampCanary = opts.priority === 'batch' && (opts.timestamps ?? batchTimestampsEnabled())
|
|
976
1007
|
let nativeDiagnostics = ''
|
|
977
1008
|
|
|
978
1009
|
const modelPath = resolveBatchModel()
|
|
@@ -1144,6 +1175,8 @@ export async function transcribeHighQuality(
|
|
|
1144
1175
|
|
|
1145
1176
|
let finalText = text
|
|
1146
1177
|
let words: WhisperWord[] | undefined
|
|
1178
|
+
let spokenWords: WhisperWord[] | undefined
|
|
1179
|
+
let speechSpans: Array<[number, number]> | undefined
|
|
1147
1180
|
if (captureWords) {
|
|
1148
1181
|
try {
|
|
1149
1182
|
if (existsSync(jsonPath)) {
|
|
@@ -1157,6 +1190,11 @@ export async function transcribeHighQuality(
|
|
|
1157
1190
|
words = timed.length > 0
|
|
1158
1191
|
? timed.map(w => ({ ...w, word: applyCorrections(w.word) }))
|
|
1159
1192
|
: []
|
|
1193
|
+
// Spoken words stay uncorrected: the resolver joins them and corrects the text once.
|
|
1194
|
+
spokenWords = timestampCanary ? remapBatchWords(parsed.spokenWords, nativeDiagnostics, useVad) : parsed.spokenWords
|
|
1195
|
+
if (timestampCanary && useVad && parsed.words.length) {
|
|
1196
|
+
speechSpans = parseNativeVadTimeline(nativeDiagnostics).map(s => [s.originalStart, s.originalEnd] as [number, number])
|
|
1197
|
+
}
|
|
1160
1198
|
} else if (timestampCanary) {
|
|
1161
1199
|
throw new Error('HQ timestamp candidate did not produce CLI JSON')
|
|
1162
1200
|
}
|
|
@@ -1193,7 +1231,10 @@ export async function transcribeHighQuality(
|
|
|
1193
1231
|
? 'large_v3_model_missing' as const
|
|
1194
1232
|
: 'hq_disabled' as const,
|
|
1195
1233
|
}
|
|
1196
|
-
|
|
1234
|
+
const timing = isBatch ? { timestamps: timestampCanary } : {}
|
|
1235
|
+
return words
|
|
1236
|
+
? { text: corrected, words, ...(spokenWords ? { spokenWords } : {}), ...(speechSpans ? { speechSpans } : {}), ...timing, ...metadata }
|
|
1237
|
+
: { text: corrected, ...timing, ...metadata }
|
|
1197
1238
|
} finally {
|
|
1198
1239
|
try { unlinkSync(tmpWav) } catch { /* cleanup */ }
|
|
1199
1240
|
if (captureWords) {
|
package/server/routes/meeting.ts
CHANGED
|
@@ -66,6 +66,7 @@ import {
|
|
|
66
66
|
import {
|
|
67
67
|
canDeletePendingBatchAudio,
|
|
68
68
|
persistBatchDecisionSidecar,
|
|
69
|
+
persistSttSidecar,
|
|
69
70
|
replaceMeetingTranscriptAtomic,
|
|
70
71
|
} from '../lib/meeting-batch-persistence.js'
|
|
71
72
|
import {
|
|
@@ -77,6 +78,8 @@ import {
|
|
|
77
78
|
import {
|
|
78
79
|
enqueueSerializedHqWork,
|
|
79
80
|
runMeetingBatchPipeline,
|
|
81
|
+
recoveryHqJob,
|
|
82
|
+
summarizeStt,
|
|
80
83
|
segmentTranscriptChunks,
|
|
81
84
|
stopProgressiveHqSession,
|
|
82
85
|
transcribeSegments,
|
|
@@ -87,7 +90,9 @@ import {
|
|
|
87
90
|
listUnsavedCaptures,
|
|
88
91
|
markRecovered,
|
|
89
92
|
markRecoveredNoSpeech,
|
|
93
|
+
readCaptureDisposition,
|
|
90
94
|
registerActiveRecovery,
|
|
95
|
+
type CaptureDisposition,
|
|
91
96
|
} from '../lib/unsaved-audio-quarantine.js'
|
|
92
97
|
import {
|
|
93
98
|
clearSessionHallucinationState,
|
|
@@ -101,6 +106,7 @@ import {
|
|
|
101
106
|
analyzeTranscriptGaps,
|
|
102
107
|
deleteSession,
|
|
103
108
|
drainSessionAudioWrites,
|
|
109
|
+
durableSessionRecoveryHasRun,
|
|
104
110
|
getSessionChunkEntries,
|
|
105
111
|
getSessionChunks,
|
|
106
112
|
getSessionProviderCandidates,
|
|
@@ -475,6 +481,48 @@ function publicSaveResponse(saved: SavedMeeting, replayed = false): Record<strin
|
|
|
475
481
|
}
|
|
476
482
|
}
|
|
477
483
|
|
|
484
|
+
/**
|
|
485
|
+
* 6.71.14: what became of a capture whose live session is gone, as an ADDITIVE
|
|
486
|
+
* field beside `state`.
|
|
487
|
+
*
|
|
488
|
+
* `state` keeps its 6.71.13 values and precedence (saved > active > closed >
|
|
489
|
+
* missing) on purpose. Phones branch on it (`missing` is recovery-required or a
|
|
490
|
+
* collision, `closed` is terminal-unsaved), and a new value would fall through to
|
|
491
|
+
* their resume path. So the quarantine's answer rides alongside:
|
|
492
|
+
* receipted recovery finished and found no speech
|
|
493
|
+
* recovered recovery saved a meeting (the store may no longer find it by id)
|
|
494
|
+
* quarantined the audio is held and no recovery has finished yet
|
|
495
|
+
* Null when `state` is saved (the receipt is the answer) or active (a live session
|
|
496
|
+
* reports itself), and when the quarantine holds nothing for the id.
|
|
497
|
+
*/
|
|
498
|
+
export type MeetingCaptureOutcome = 'receipted' | 'recovered' | 'quarantined'
|
|
499
|
+
|
|
500
|
+
export function captureOutcome(
|
|
501
|
+
state: 'saved' | 'active' | 'closed' | 'missing',
|
|
502
|
+
disposition: Pick<CaptureDisposition, 'kind'> | null,
|
|
503
|
+
): MeetingCaptureOutcome | null {
|
|
504
|
+
if (state === 'saved' || state === 'active' || !disposition) return null
|
|
505
|
+
if (disposition.kind === 'no_speech') return 'receipted'
|
|
506
|
+
if (disposition.kind === 'recovered') return 'recovered'
|
|
507
|
+
return 'quarantined'
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
function publicDisposition(disposition: CaptureDisposition | null): Record<string, unknown> | null {
|
|
511
|
+
if (!disposition) return null
|
|
512
|
+
return {
|
|
513
|
+
kind: disposition.kind,
|
|
514
|
+
// Why the capture is terminal: `no_speech` or `recovered`; null while it is only held.
|
|
515
|
+
reason: disposition.kind === 'quarantined' ? null : disposition.kind,
|
|
516
|
+
quarantinedAt: disposition.quarantinedAt,
|
|
517
|
+
quarantineReason: disposition.quarantineReason,
|
|
518
|
+
receiptedAt: disposition.receiptedAt,
|
|
519
|
+
chunkFiles: disposition.chunkFiles,
|
|
520
|
+
savedFilename: disposition.savedFilename,
|
|
521
|
+
recovering: disposition.recovering,
|
|
522
|
+
audioRetainedUntil: disposition.retainedUntil,
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
|
|
478
526
|
function suppliedServerPin(
|
|
479
527
|
req: { get: (name: string) => string | undefined },
|
|
480
528
|
fallback: unknown,
|
|
@@ -497,6 +545,7 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
|
|
|
497
545
|
const savingSessions = new Set<string>()
|
|
498
546
|
setNamingSessionLiveness(id => sessions.getStartTime(id) !== null || savingSessions.has(id) || activeFinalizationJobs.has(id))
|
|
499
547
|
|
|
548
|
+
let recoveryPendingLogged = false
|
|
500
549
|
router.get('/meeting/sessions/:sessionId/status', (req, res) => {
|
|
501
550
|
const sessionId = String(req.params.sessionId ?? '')
|
|
502
551
|
if (!/^[A-Za-z0-9:_-]{3,96}$/.test(sessionId)) {
|
|
@@ -514,11 +563,35 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
|
|
|
514
563
|
return
|
|
515
564
|
}
|
|
516
565
|
const saved = store.findBySessionId(sessionId)
|
|
566
|
+
// 6.71.14: before durable recovery has run (a boot behind a closed or stuck
|
|
567
|
+
// maintenance gate), a persisted meeting is not in memory YET, and answering
|
|
568
|
+
// `missing` would tell the phone the Mac lost it. Every phone status caller
|
|
569
|
+
// treats a thrown non-2xx as defer-and-retry (the uploader's finalize defers;
|
|
570
|
+
// origin-loss recovery backs off), so say "not yet" instead. A saved meeting
|
|
571
|
+
// is already authoritative and is answered as usual.
|
|
572
|
+
if (!saved && !durableSessionRecoveryHasRun()) {
|
|
573
|
+
if (!recoveryPendingLogged) {
|
|
574
|
+
recoveryPendingLogged = true
|
|
575
|
+
console.warn('[meeting/status] Answering 503 recovery_pending: durable session recovery has not run (maintenance gate closed)')
|
|
576
|
+
}
|
|
577
|
+
res.set('Cache-Control', 'private, no-store')
|
|
578
|
+
res.set('Retry-After', '5')
|
|
579
|
+
res.status(503).json({
|
|
580
|
+
error: 'Meeting session recovery has not run yet; retry shortly',
|
|
581
|
+
reason: 'recovery_pending',
|
|
582
|
+
retryable: true,
|
|
583
|
+
})
|
|
584
|
+
return
|
|
585
|
+
}
|
|
517
586
|
const live = getMeetingSessionStatus(sessionId)
|
|
587
|
+
const state = saved ? 'saved' : live.state
|
|
588
|
+
// A saved meeting or a live session is the truth for itself; the quarantine is
|
|
589
|
+
// read only when neither is.
|
|
590
|
+
const disposition = state === 'saved' || state === 'active' ? null : readCaptureDisposition(sessionId)
|
|
518
591
|
res.set('Cache-Control', 'private, no-store')
|
|
519
592
|
res.json({
|
|
520
593
|
sessionId,
|
|
521
|
-
state
|
|
594
|
+
state,
|
|
522
595
|
serverInstanceId,
|
|
523
596
|
receivedRanges: live.receivedRanges,
|
|
524
597
|
receivedCount: live.receivedCount,
|
|
@@ -530,6 +603,10 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
|
|
|
530
603
|
lastActivityAt: live.lastActivityAt,
|
|
531
604
|
retainedUntil: saved ? null : live.retainedUntil,
|
|
532
605
|
saveReceipt: saved ? publicSaveResponse(saved) : null,
|
|
606
|
+
// 6.71.14, additive: what became of a capture whose audio reached quarantine
|
|
607
|
+
// (null otherwise). Older clients ignore both.
|
|
608
|
+
outcome: captureOutcome(state, disposition),
|
|
609
|
+
disposition: publicDisposition(disposition),
|
|
533
610
|
})
|
|
534
611
|
})
|
|
535
612
|
|
|
@@ -2104,9 +2181,9 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
|
|
|
2104
2181
|
// decoder serialization — that lives in the queue tail. Chain onto it,
|
|
2105
2182
|
// or two back-to-back recoveries plus a live post-meeting batch would
|
|
2106
2183
|
// run parallel 16-thread large-v3 decoders on the user's Mac.
|
|
2107
|
-
const
|
|
2108
|
-
|
|
2109
|
-
)
|
|
2184
|
+
const hqJob = recoveryHqJob(quarantineDir, segments, entries)
|
|
2185
|
+
const plan = hqJob.plan
|
|
2186
|
+
const results = await enqueueSerializedHqWork(hqJob.run)
|
|
2110
2187
|
// Batch whisper over a dead session has no streaming baseline to
|
|
2111
2188
|
// compare against (evaluateBatchQuality needs one), but the inline
|
|
2112
2189
|
// hallucination filter needs none — run the same two-pass the boot
|
|
@@ -2166,6 +2243,9 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
|
|
|
2166
2243
|
})),
|
|
2167
2244
|
transferIntegrity: null,
|
|
2168
2245
|
})
|
|
2246
|
+
try { persistSttSidecar(saved.sidecarPath, results, summarizeStt(plan, results, segments.length)) } catch (error) {
|
|
2247
|
+
console.warn(`[meeting/orphans] ${sessionId}: STT hypotheses not saved: ${error instanceof Error ? error.message : error}`)
|
|
2248
|
+
}
|
|
2169
2249
|
markRecovered(quarantineDir, saved.filename)
|
|
2170
2250
|
console.log(
|
|
2171
2251
|
`[meeting/orphans] Recovered ${sessionId} → ${saved.filename} `
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// Speech-to-text settings (6.72.0). Token-protected by the /api middleware.
|
|
2
|
+
//
|
|
3
|
+
// GET /api/stt-settings the settings, every option with availability, Parakeet status
|
|
4
|
+
// PUT /api/stt-settings partial body: {"hq":"parakeet-ultra"}, {"resolver":false}, ...
|
|
5
|
+
// POST /api/stt-settings/parakeet/install install the pinned Parakeet CLI and models (202; poll GET)
|
|
6
|
+
// body {} | {"localSource":"/abs/path"} | {"localSource":"...","allowDownload":true}
|
|
7
|
+
// DELETE /api/stt-settings/parakeet/install cancel a running install (nothing partial is kept)
|
|
8
|
+
//
|
|
9
|
+
// Clients find the route with GET: a 404 means an older server, so they hide the switcher.
|
|
10
|
+
// The contract is shared with COS Control and the phone (see README "Speech-to-text settings").
|
|
11
|
+
|
|
12
|
+
import { Router } from 'express'
|
|
13
|
+
import { errMsg } from '../lib/utils.js'
|
|
14
|
+
import { cancelParakeetInstall, installParakeet, parakeetInstalled, parakeetInstallRunning, type InstallOptions } from '../lib/parakeet.js'
|
|
15
|
+
import { currentSttCapabilities, ensurePreviewWarm, parakeetStatus } from '../lib/stt-runtime.js'
|
|
16
|
+
import {
|
|
17
|
+
applySettingsPatch, readLastSttJob, readSttSettings, settingsView, SttSettingsError, writeSttSettings,
|
|
18
|
+
type SttCapabilities,
|
|
19
|
+
} from '../lib/stt-settings.js'
|
|
20
|
+
|
|
21
|
+
export interface SttSettingsRouterDeps {
|
|
22
|
+
recordingLive: () => boolean
|
|
23
|
+
capabilities?: () => SttCapabilities
|
|
24
|
+
install?: (options: InstallOptions) => Promise<void>
|
|
25
|
+
onChanged?: () => void
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function createSttSettingsRouter(deps: SttSettingsRouterDeps): Router {
|
|
29
|
+
const router = Router()
|
|
30
|
+
const caps = deps.capabilities ?? currentSttCapabilities
|
|
31
|
+
const install = deps.install ?? installParakeet
|
|
32
|
+
const view = () => settingsView(readSttSettings(), caps(), parakeetStatus(deps.recordingLive()), readLastSttJob())
|
|
33
|
+
|
|
34
|
+
router.get('/stt-settings', (_req, res) => {
|
|
35
|
+
try { res.json(view()) } catch (err) { res.status(500).json({ error: errMsg(err) }) }
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
router.put('/stt-settings', (req, res) => {
|
|
39
|
+
try {
|
|
40
|
+
const next = applySettingsPatch(readSttSettings(), req.body, caps())
|
|
41
|
+
writeSttSettings(next)
|
|
42
|
+
ensurePreviewWarm()
|
|
43
|
+
deps.onChanged?.()
|
|
44
|
+
res.json(view())
|
|
45
|
+
} catch (err) {
|
|
46
|
+
if (err instanceof SttSettingsError) return void res.status(400).json({ error: err.message })
|
|
47
|
+
res.status(500).json({ error: errMsg(err) })
|
|
48
|
+
}
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
router.post('/stt-settings/parakeet/install', (req, res) => {
|
|
52
|
+
try {
|
|
53
|
+
if (deps.recordingLive()) return void res.status(409).json({ error: 'A recording is live. Install Parakeet after it is saved.' })
|
|
54
|
+
if (parakeetInstalled().installed && !parakeetInstallRunning()) return void res.status(200).json({ started: false, ...view() })
|
|
55
|
+
const body = (req.body ?? {}) as { localSource?: unknown; allowDownload?: unknown }
|
|
56
|
+
const raw = body.localSource
|
|
57
|
+
if (raw !== undefined && (typeof raw !== 'string' || !raw.startsWith('/') || raw.length > 1024)) {
|
|
58
|
+
return void res.status(400).json({ error: 'localSource must be an absolute path.' })
|
|
59
|
+
}
|
|
60
|
+
if (body.allowDownload !== undefined && typeof body.allowDownload !== 'boolean') return void res.status(400).json({ error: 'allowDownload must be true or false.' })
|
|
61
|
+
// Only bytes whose SHA-256 matches the pinned manifest are ever copied from it.
|
|
62
|
+
// A named localSource that lacks a file fails plainly unless allowDownload is true.
|
|
63
|
+
const options: InstallOptions = raw ? { localSources: [raw], requireLocal: body.allowDownload !== true } : {}
|
|
64
|
+
if (!parakeetInstallRunning()) {
|
|
65
|
+
void install(options).then(() => { ensurePreviewWarm(); deps.onChanged?.() }).catch(error => {
|
|
66
|
+
console.warn(`[stt] Parakeet install failed: ${errMsg(error)}`)
|
|
67
|
+
})
|
|
68
|
+
}
|
|
69
|
+
res.status(202).json({ started: true, ...view() })
|
|
70
|
+
} catch (err) {
|
|
71
|
+
res.status(500).json({ error: errMsg(err) })
|
|
72
|
+
}
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
router.delete('/stt-settings/parakeet/install', (_req, res) => {
|
|
76
|
+
try {
|
|
77
|
+
const cancelled = cancelParakeetInstall()
|
|
78
|
+
res.status(cancelled ? 202 : 200).json({ cancelled, ...view() })
|
|
79
|
+
} catch (err) {
|
|
80
|
+
res.status(500).json({ error: errMsg(err) })
|
|
81
|
+
}
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
return router
|
|
85
|
+
}
|