@gotcos/glasses-server 6.71.13 → 6.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +16 -6
- package/CHANGELOG.md +29 -0
- package/README.md +160 -1
- package/docs/stt-672-evaluation.md +113 -0
- package/package.json +4 -1
- package/server/index.ts +18 -1
- package/server/lib/batch-transcript-quality.ts +46 -0
- package/server/lib/meeting-batch-persistence.ts +23 -1
- package/server/lib/meeting-batch-transcribe.ts +408 -45
- package/server/lib/parakeet-manifest.ts +41 -0
- package/server/lib/parakeet.ts +665 -0
- package/server/lib/stale-active-session-files.ts +178 -0
- package/server/lib/stt-live.ts +60 -0
- package/server/lib/stt-resolver.ts +327 -0
- package/server/lib/stt-runtime.ts +225 -0
- package/server/lib/stt-settings.ts +342 -0
- package/server/lib/unsaved-audio-quarantine.ts +114 -0
- package/server/lib/vad-silero.ts +52 -0
- package/server/lib/whisper-batch-timing.ts +13 -3
- package/server/lib/whisper-local.ts +58 -17
- package/server/routes/meeting.ts +84 -4
- package/server/routes/stt-settings.ts +85 -0
- package/server/routes/transcribe-stream.ts +179 -24
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// Active-session files the server no longer owns.
|
|
2
|
+
//
|
|
3
|
+
// THE INCIDENT (2026-10-12, server 6.71.11). `active-sessions/<id>.json` is written by
|
|
4
|
+
// the chunk route while a capture is live, and removed when the session closes. A
|
|
5
|
+
// capture of two silent chunks was persisted, then the server restarted behind a
|
|
6
|
+
// closed startup maintenance gate. Boot recovery was gated on admissions being open
|
|
7
|
+
// and never ran once the controller released the gate, so the file was neither
|
|
8
|
+
// adopted into the in-memory session map nor tombstoned. The audio was quarantined,
|
|
9
|
+
// recovered and receipted no_speech, and the file stayed: COS Control lists the
|
|
10
|
+
// directory directly and showed "Live now ... No audio for 1054m" for 17 hours, and
|
|
11
|
+
// the status route answered `missing`.
|
|
12
|
+
//
|
|
13
|
+
// The deferred recovery fix in routes/transcribe-stream.ts closes that door. This
|
|
14
|
+
// sweep is the backstop for any other door: a file whose session is not live and
|
|
15
|
+
// that has been idle past the retention horizon is moved aside, never deleted in
|
|
16
|
+
// place, with a reason written next to it. The horizon is the same one the session
|
|
17
|
+
// itself would be promoted or tombstoned at, so this can never act earlier than the
|
|
18
|
+
// server's own definition of "done".
|
|
19
|
+
|
|
20
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync } from 'node:fs'
|
|
21
|
+
import { resolve } from 'node:path'
|
|
22
|
+
import { atomicWriteFileSync } from './atomic-fs.js'
|
|
23
|
+
|
|
24
|
+
export const STALE_ACTIVE_SESSION_DIR_NAME = 'stale-active-session-files'
|
|
25
|
+
|
|
26
|
+
/** Moved files carry transcript text, so they are not kept forever either. */
|
|
27
|
+
export const STALE_ACTIVE_SESSION_FILE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000
|
|
28
|
+
|
|
29
|
+
export type StaleActiveSessionReason =
|
|
30
|
+
/** Not in the live session map and idle past the retention horizon. */
|
|
31
|
+
| 'not_live_idle_expired'
|
|
32
|
+
/** Boot recovery found the capture already recovered or receipted no_speech. */
|
|
33
|
+
| 'terminal_capture'
|
|
34
|
+
|
|
35
|
+
export interface StaleActiveSessionMove {
|
|
36
|
+
sessionId: string
|
|
37
|
+
reason: StaleActiveSessionReason
|
|
38
|
+
/** Where the file went, or null when the move failed and the file stayed put. */
|
|
39
|
+
target: string | null
|
|
40
|
+
lastActivityAt: number | null
|
|
41
|
+
idleMs: number | null
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const SESSION_FILE_RE = /^([A-Za-z0-9:_-]{3,96})\.json$/
|
|
45
|
+
/** `<id>.<move ms>[-n].json` and its `.reason.json`. The move time is in the NAME. */
|
|
46
|
+
const MOVED_FILE_RE = /^[A-Za-z0-9:_-]{3,96}\.(\d{13})(?:-\d+)?(?:\.reason)?\.json$/
|
|
47
|
+
const REASON_SUFFIX = '.reason.json'
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* When the session in this file was last active, for staleness: the EARLIER of its
|
|
51
|
+
* own `lastActivityAt` and the file's mtime. The file is rewritten on every accepted
|
|
52
|
+
* chunk, so its mtime is a true upper bound; a future-dated or corrupt
|
|
53
|
+
* `lastActivityAt` therefore cannot pin a dead file forever. Null when neither can
|
|
54
|
+
* be read.
|
|
55
|
+
*/
|
|
56
|
+
export function activeSessionFileActivity(filePath: string): number | null {
|
|
57
|
+
let mtimeMs: number | null = null
|
|
58
|
+
try {
|
|
59
|
+
const stat = statSync(filePath)
|
|
60
|
+
if (Number.isFinite(stat.mtimeMs)) mtimeMs = stat.mtimeMs
|
|
61
|
+
} catch { /* fall through: a missing file has no age */ }
|
|
62
|
+
let recorded: number | null = null
|
|
63
|
+
try {
|
|
64
|
+
const data = JSON.parse(readFileSync(filePath, 'utf8')) as { lastActivityAt?: unknown }
|
|
65
|
+
if (typeof data?.lastActivityAt === 'number' && Number.isFinite(data.lastActivityAt)) recorded = data.lastActivityAt
|
|
66
|
+
} catch { /* torn or legacy file: the mtime still gives it an age */ }
|
|
67
|
+
if (recorded == null) return mtimeMs
|
|
68
|
+
if (mtimeMs == null) return recorded
|
|
69
|
+
return Math.min(recorded, mtimeMs)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Move one active-session file into the stale directory. The reason is written
|
|
74
|
+
* FIRST and atomically, so a moved file never exists without its reason; if the
|
|
75
|
+
* reason cannot be written nothing is moved. Returns the new path, or null when the
|
|
76
|
+
* move did not happen (the source then stays exactly where it was).
|
|
77
|
+
*/
|
|
78
|
+
export function moveActiveSessionFileAside(
|
|
79
|
+
sourcePath: string,
|
|
80
|
+
sessionId: string,
|
|
81
|
+
targetDir: string,
|
|
82
|
+
reason: StaleActiveSessionReason,
|
|
83
|
+
now: number,
|
|
84
|
+
detail: { lastActivityAt: number | null; note?: string } = { lastActivityAt: null },
|
|
85
|
+
): string | null {
|
|
86
|
+
try {
|
|
87
|
+
if (!existsSync(sourcePath)) return null
|
|
88
|
+
mkdirSync(targetDir, { recursive: true, mode: 0o700 })
|
|
89
|
+
// Two moves in the same millisecond must not overwrite each other.
|
|
90
|
+
let base = `${sessionId}.${now}`
|
|
91
|
+
for (let n = 1; existsSync(resolve(targetDir, `${base}.json`)) || existsSync(resolve(targetDir, `${base}${REASON_SUFFIX}`)); n++) {
|
|
92
|
+
base = `${sessionId}.${now}-${n}`
|
|
93
|
+
}
|
|
94
|
+
const target = resolve(targetDir, `${base}.json`)
|
|
95
|
+
const reasonPath = resolve(targetDir, `${base}${REASON_SUFFIX}`)
|
|
96
|
+
atomicWriteFileSync(reasonPath, `${JSON.stringify({
|
|
97
|
+
schemaVersion: 1,
|
|
98
|
+
sessionId,
|
|
99
|
+
reason,
|
|
100
|
+
movedAt: new Date(now).toISOString(),
|
|
101
|
+
lastActivityAt: detail.lastActivityAt != null ? new Date(detail.lastActivityAt).toISOString() : null,
|
|
102
|
+
...(detail.note ? { note: detail.note } : {}),
|
|
103
|
+
})}\n`, { mode: 0o600 })
|
|
104
|
+
try {
|
|
105
|
+
renameSync(sourcePath, target)
|
|
106
|
+
} catch (error) {
|
|
107
|
+
// No reason may outlive a move that did not happen.
|
|
108
|
+
try { rmSync(reasonPath, { force: true }) } catch { /* the next sweep's own reason supersedes it */ }
|
|
109
|
+
throw error
|
|
110
|
+
}
|
|
111
|
+
return target
|
|
112
|
+
} catch {
|
|
113
|
+
return null
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* One pass over the active-sessions directory. A file is moved aside only when its
|
|
119
|
+
* session is NOT live and it has been idle for at least `idleRetentionMs`. A live
|
|
120
|
+
* session's file is never touched, however old it looks. A failed move is returned
|
|
121
|
+
* with `target: null` so the caller can say so.
|
|
122
|
+
*/
|
|
123
|
+
export function sweepStaleActiveSessionFiles(opts: {
|
|
124
|
+
dir: string
|
|
125
|
+
targetDir: string
|
|
126
|
+
isLive: (sessionId: string) => boolean
|
|
127
|
+
now: number
|
|
128
|
+
idleRetentionMs: number
|
|
129
|
+
}): StaleActiveSessionMove[] {
|
|
130
|
+
const moves: StaleActiveSessionMove[] = []
|
|
131
|
+
let entries: string[]
|
|
132
|
+
try { entries = readdirSync(opts.dir) } catch { return moves }
|
|
133
|
+
for (const name of entries) {
|
|
134
|
+
const match = SESSION_FILE_RE.exec(name)
|
|
135
|
+
if (!match) continue
|
|
136
|
+
const sessionId = match[1]
|
|
137
|
+
if (opts.isLive(sessionId)) continue
|
|
138
|
+
const filePath = resolve(opts.dir, name)
|
|
139
|
+
// The mtime bound inside activeSessionFileActivity is what defeats a
|
|
140
|
+
// future-dated value. A further min(..., now) would change no decision: a
|
|
141
|
+
// future activity is negative idle, and zero idle is just as far from the horizon.
|
|
142
|
+
const lastActivityAt = activeSessionFileActivity(filePath)
|
|
143
|
+
if (lastActivityAt == null) continue
|
|
144
|
+
const idleMs = opts.now - lastActivityAt
|
|
145
|
+
if (idleMs < opts.idleRetentionMs) continue
|
|
146
|
+
const target = moveActiveSessionFileAside(filePath, sessionId, opts.targetDir, 'not_live_idle_expired', opts.now, {
|
|
147
|
+
lastActivityAt,
|
|
148
|
+
})
|
|
149
|
+
moves.push({ sessionId, reason: 'not_live_idle_expired', target, lastActivityAt, idleMs })
|
|
150
|
+
}
|
|
151
|
+
return moves
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Remove moved files (and their reasons) whose move time, read from the file NAME,
|
|
156
|
+
* is older than the retention. Ages never come from mtime: a rename keeps the old
|
|
157
|
+
* mtime, which would turn "move" into "delete a minute later". A name that does not
|
|
158
|
+
* carry a move time is never removed. Returns the count removed.
|
|
159
|
+
*/
|
|
160
|
+
export function purgeExpiredStaleActiveSessionFiles(
|
|
161
|
+
targetDir: string,
|
|
162
|
+
now: number,
|
|
163
|
+
retentionMs: number = STALE_ACTIVE_SESSION_FILE_RETENTION_MS,
|
|
164
|
+
): number {
|
|
165
|
+
let removed = 0
|
|
166
|
+
let entries: string[]
|
|
167
|
+
try { entries = readdirSync(targetDir) } catch { return 0 }
|
|
168
|
+
for (const name of entries) {
|
|
169
|
+
const match = MOVED_FILE_RE.exec(name)
|
|
170
|
+
if (!match) continue
|
|
171
|
+
if (now - Number(match[1]) <= retentionMs) continue
|
|
172
|
+
try {
|
|
173
|
+
rmSync(resolve(targetDir, name), { force: true })
|
|
174
|
+
removed++
|
|
175
|
+
} catch { /* next sweep retries */ }
|
|
176
|
+
}
|
|
177
|
+
return removed
|
|
178
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// The live chunk's quiet gate and ASR front end (6.72.0). Kept apart from the
|
|
2
|
+
// decoders so the live route imports only audio code.
|
|
3
|
+
|
|
4
|
+
import { enhanceAudio } from './audio-enhance.js'
|
|
5
|
+
import { isSileroAvailable, measureSpeechSeconds } from './vad-silero.js'
|
|
6
|
+
import type { FrontEndId, QuietGateId } from './stt-settings.js'
|
|
7
|
+
|
|
8
|
+
/** Under this many seconds of Silero speech a chunk is quiet (bake-off canary 5). */
|
|
9
|
+
export const QUIET_SPEECH_SEC = 0.5
|
|
10
|
+
export const QUIET_RMS = 150
|
|
11
|
+
|
|
12
|
+
export interface QuietDecision {
|
|
13
|
+
isQuiet: boolean
|
|
14
|
+
rms: number
|
|
15
|
+
/** Silero speech seconds; null when the gate fell back to RMS. */
|
|
16
|
+
speechSec: number | null
|
|
17
|
+
gate: 'silero' | 'rms' | 'rms_fallback'
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function chunkRms(wav: Buffer): number {
|
|
21
|
+
const pcm = wav.subarray(44)
|
|
22
|
+
const n = Math.floor(pcm.length / 2)
|
|
23
|
+
let sumSq = 0
|
|
24
|
+
for (let i = 0; i < n; i++) { const s = pcm.readInt16LE(i * 2); sumSq += s * s }
|
|
25
|
+
return Math.sqrt(sumSq / Math.max(1, n))
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Quiet means "do not prompt, do not enhance, drop silence echoes". RMS 150
|
|
30
|
+
* fired on 1% of G2 chunks while Silero found no speech in 9.3%, so 6.72.0
|
|
31
|
+
* gates on Silero speech seconds (the in-process sherpa Silero, about 17 ms a
|
|
32
|
+
* 6 s chunk). Silero unavailable or the clip too short: the old RMS gate.
|
|
33
|
+
*/
|
|
34
|
+
export function decideQuiet(wav: Buffer, gate: QuietGateId, vad: { available: () => boolean; speechSeconds: (wav: Buffer) => number | null } = { available: () => isSileroAvailable(), speechSeconds: w => measureSpeechSeconds(w) }): QuietDecision {
|
|
35
|
+
const rms = chunkRms(wav)
|
|
36
|
+
if (gate === 'silero' && vad.available()) {
|
|
37
|
+
const speechSec = vad.speechSeconds(wav)
|
|
38
|
+
if (speechSec !== null && Number.isFinite(speechSec)) {
|
|
39
|
+
return { isQuiet: speechSec < QUIET_SPEECH_SEC, rms, speechSec, gate: 'silero' }
|
|
40
|
+
}
|
|
41
|
+
return { isQuiet: rms < QUIET_RMS, rms, speechSec: null, gate: 'rms_fallback' }
|
|
42
|
+
}
|
|
43
|
+
return { isQuiet: rms < QUIET_RMS, rms, speechSec: null, gate: 'rms' }
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Whether the silence-hallucination TEXT drops (URL-only, repeated thank-you,
|
|
48
|
+
* quiet vocabulary echo) may fire. They stay on the pre-6.72 RMS test: the
|
|
49
|
+
* Silero gate also calls a chunk holding one short real name quiet, and those
|
|
50
|
+
* drops would delete it.
|
|
51
|
+
*/
|
|
52
|
+
export function silenceDropsAllowed(decision: QuietDecision): boolean {
|
|
53
|
+
return decision.rms < QUIET_RMS
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** ASR input for a live chunk or an HQ window. Playback audio is never touched here. */
|
|
57
|
+
export function asrFrontEnd(audio: Buffer, frontEnd: FrontEndId): Promise<Buffer> {
|
|
58
|
+
return frontEnd === 'full' ? enhanceAudio(audio) : enhanceAudio(audio, { profile: 'light' })
|
|
59
|
+
}
|
|
60
|
+
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
// Second-opinion resolver for meeting HQ (6.72.0). Algorithmic only: no model
|
|
2
|
+
// call, no network, no LLM. Transcription must never call `claude -p`.
|
|
3
|
+
//
|
|
4
|
+
// Two decoders hear the same window. Their words are aligned by edit distance
|
|
5
|
+
// over normalized words (an optional time-skew limit exists but is off: it
|
|
6
|
+
// measured worse), and every place they disagree is recorded. The resolver
|
|
7
|
+
// then applies two measured rules, and nothing else:
|
|
8
|
+
//
|
|
9
|
+
// 1. Gap fill. Large-v3 is deletion-prone on G2 audio (bake-off silver
|
|
10
|
+
// deletions 6.1% for the fixed Large-v3, 6.6% on the shipped path, against
|
|
11
|
+
// 2.1% for Parakeet Ultra). Where the primary is a Whisper engine and the
|
|
12
|
+
// second opinion heard words the primary has nothing for, those words are
|
|
13
|
+
// taken, but only inside a span the primary's own VAD marked as speech and
|
|
14
|
+
// only when the second opinion's mean word confidence is at least 0.5.
|
|
15
|
+
// 2. Vocabulary rescue. Where the second opinion's side of a short
|
|
16
|
+
// disagreement (at most 4 words, and the primary's side at most 2 longer)
|
|
17
|
+
// holds a profile vocabulary term the primary's side does not, the second
|
|
18
|
+
// opinion's words are taken.
|
|
19
|
+
//
|
|
20
|
+
// Everything else stays as the primary heard it. No region whose two sides
|
|
21
|
+
// differ in a negation, a number or a unit is ever changed, whatever its length
|
|
22
|
+
// (the 2026-10-13 diagnostic saw "born lacking" become "born having" and
|
|
23
|
+
// "18 months" become "18 minutes"); it is listed as a protected disagreement.
|
|
24
|
+
|
|
25
|
+
export interface HypWord {
|
|
26
|
+
word: string
|
|
27
|
+
start: number
|
|
28
|
+
end: number
|
|
29
|
+
probability?: number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface Hypothesis {
|
|
33
|
+
engine: string
|
|
34
|
+
text: string
|
|
35
|
+
/** Spoken words (not sub-word tokens), seconds from the start of the window. */
|
|
36
|
+
words?: HypWord[]
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type DisagreementKind = 'substitution' | 'primary_only' | 'second_only'
|
|
40
|
+
export type ResolutionReason = 'gap_fill' | 'vocabulary' | 'protected' | 'not_speech' | 'low_confidence' | 'too_wide' | null
|
|
41
|
+
|
|
42
|
+
export interface Disagreement {
|
|
43
|
+
kind: DisagreementKind
|
|
44
|
+
primary: string
|
|
45
|
+
second: string
|
|
46
|
+
start: number | null
|
|
47
|
+
end: number | null
|
|
48
|
+
taken: 'primary' | 'second'
|
|
49
|
+
reason: ResolutionReason
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface ResolveOptions {
|
|
53
|
+
vocabulary: readonly string[]
|
|
54
|
+
/** Gap fill only for a deletion-prone primary (the Whisper engines). */
|
|
55
|
+
primaryDeletionProne: boolean
|
|
56
|
+
/** Where the primary's own VAD heard speech, seconds on the window clock.
|
|
57
|
+
* Gap fill needs it: a run outside every span is background the primary's VAD
|
|
58
|
+
* rejected, and is not taken. Absent: no gap fill at all. */
|
|
59
|
+
speechSpans?: ReadonlyArray<readonly [number, number]> | null
|
|
60
|
+
/** Mean word confidence a gap-fill run needs (second opinion's own scores). */
|
|
61
|
+
minGapConfidence?: number
|
|
62
|
+
/** false: report disagreements, change nothing. */
|
|
63
|
+
apply: boolean
|
|
64
|
+
/** When set, two words more than this far apart in time never align. Off by
|
|
65
|
+
* default: on the production-path silver decode every limit tried (2, 3, 5,
|
|
66
|
+
* 10 s) made the resolver worse than plain text alignment, because Large-v3's
|
|
67
|
+
* remapped token clocks have a heavy tail (median skew 0.23 s, p90 1.0 s). */
|
|
68
|
+
maxSkewSec?: number | null
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface ResolveResult {
|
|
72
|
+
text: string
|
|
73
|
+
words: HypWord[]
|
|
74
|
+
/** True when the resolver's words carry clocks for every word. */
|
|
75
|
+
timed: boolean
|
|
76
|
+
disagreements: Disagreement[]
|
|
77
|
+
wordsFromSecond: number
|
|
78
|
+
applied: boolean
|
|
79
|
+
skipped?: 'too_long'
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export const RESOLVER_VERSION = 'stt-resolver-v1'
|
|
83
|
+
/** Alignment is quadratic; a 30 s window is about 100 words a side. */
|
|
84
|
+
export const MAX_ALIGNMENT_CELLS = 2_000_000
|
|
85
|
+
export const DEFAULT_MAX_SKEW_SEC: number | null = null
|
|
86
|
+
/** Agreed words each side of a region that a vocabulary match may span. */
|
|
87
|
+
export const CONTEXT_WORDS = 3
|
|
88
|
+
/** A vocabulary rescue replaces at most this many second-opinion words... */
|
|
89
|
+
export const MAX_VOCAB_REGION_WORDS = 4
|
|
90
|
+
/** ...and at most this many more primary words than it inserts. */
|
|
91
|
+
export const MAX_VOCAB_REGION_SLACK = 2
|
|
92
|
+
export const DEFAULT_MIN_GAP_CONFIDENCE = 0.5
|
|
93
|
+
/** Share of a gap-fill run's words whose midpoint must lie in a speech span. */
|
|
94
|
+
export const MIN_SPEECH_SHARE = 0.5
|
|
95
|
+
|
|
96
|
+
const NEGATIONS = new Set([
|
|
97
|
+
'not', 'no', 'never', 'none', 'nor', 'nothing', 'nobody', 'neither', 'without', 'cannot',
|
|
98
|
+
'dont', 'doesnt', 'didnt', 'isnt', 'arent', 'wasnt', 'werent', 'wont', 'wouldnt', 'cant',
|
|
99
|
+
'couldnt', 'shouldnt', 'havent', 'hasnt', 'hadnt', 'aint', 'lacking', 'lack', 'lacks',
|
|
100
|
+
])
|
|
101
|
+
// Unambiguous units and number words only. "am", "one", "second" and "first"
|
|
102
|
+
// are everyday words far more often than quantities, and protecting them would
|
|
103
|
+
// block most real gap fills.
|
|
104
|
+
const UNITS = new Set([
|
|
105
|
+
'seconds', 'minute', 'minutes', 'hour', 'hours', 'day', 'days', 'week', 'weeks',
|
|
106
|
+
'month', 'months', 'year', 'years', 'percent', 'dollar', 'dollars', 'cents',
|
|
107
|
+
'thousand', 'million', 'billion',
|
|
108
|
+
])
|
|
109
|
+
const NUMBER_WORDS = new Set([
|
|
110
|
+
'zero', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',
|
|
111
|
+
'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen',
|
|
112
|
+
'nineteen', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety',
|
|
113
|
+
'hundred', 'half', 'quarter', 'twice',
|
|
114
|
+
])
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
/** Lowercase letters and digits only. "Don't," and "dont" are the same word. */
|
|
118
|
+
export function wordKey(word: string): string {
|
|
119
|
+
return word.normalize('NFKD').replace(/\p{M}+/gu, '').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function isProtectedKey(key: string): boolean {
|
|
123
|
+
return NEGATIONS.has(key) || UNITS.has(key) || NUMBER_WORDS.has(key) || /\d/.test(key)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
interface Token { display: string; key: string; start: number | null; end: number | null; probability?: number }
|
|
127
|
+
|
|
128
|
+
function tokens(h: Hypothesis): Token[] {
|
|
129
|
+
const timedWords = Array.isArray(h.words) && h.words.length > 0 ? h.words : null
|
|
130
|
+
const source = timedWords
|
|
131
|
+
? timedWords.map(w => ({
|
|
132
|
+
display: String(w.word ?? '').trim(),
|
|
133
|
+
start: Number.isFinite(w.start) ? w.start : null,
|
|
134
|
+
end: Number.isFinite(w.end) ? w.end : null,
|
|
135
|
+
probability: w.probability,
|
|
136
|
+
}))
|
|
137
|
+
: String(h.text ?? '').split(/\s+/).map(display => ({ display, start: null, end: null, probability: undefined }))
|
|
138
|
+
return source
|
|
139
|
+
.filter(t => t.display.length > 0)
|
|
140
|
+
.map(t => ({ ...t, key: wordKey(t.display) }))
|
|
141
|
+
.filter(t => t.key.length > 0)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function vocabSequences(vocabulary: readonly string[]): string[][] {
|
|
145
|
+
const out: string[][] = []
|
|
146
|
+
const seen = new Set<string>()
|
|
147
|
+
for (const term of vocabulary) {
|
|
148
|
+
const seq = String(term).split(/\s+/).map(wordKey).filter(Boolean)
|
|
149
|
+
if (!seq.length) continue
|
|
150
|
+
const id = seq.join(' ')
|
|
151
|
+
if (seen.has(id)) continue
|
|
152
|
+
seen.add(id)
|
|
153
|
+
out.push(seq)
|
|
154
|
+
// "ThriftCart" is also heard as "thrift cart": both spellings count.
|
|
155
|
+
const joined = seq.join('')
|
|
156
|
+
if (seq.length > 1 && !seen.has(joined)) { seen.add(joined); out.push([joined]) }
|
|
157
|
+
}
|
|
158
|
+
return out
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function termsIn(keys: string[], sequences: string[][]): Set<string> {
|
|
162
|
+
const found = new Set<string>()
|
|
163
|
+
for (const seq of sequences) {
|
|
164
|
+
for (let i = 0; i + seq.length <= keys.length; i++) {
|
|
165
|
+
let hit = true
|
|
166
|
+
for (let k = 0; k < seq.length; k++) if (keys[i + k] !== seq[k]) { hit = false; break }
|
|
167
|
+
if (hit) { found.add(seq.join('')); break }
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return found
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function mid(t: Token): number | null {
|
|
174
|
+
if (t.start === null) return null
|
|
175
|
+
return t.end === null ? t.start : (t.start + t.end) / 2
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
type Op = { kind: '=' | 's' | 'p' | 'q'; i: number | null; j: number | null }
|
|
179
|
+
|
|
180
|
+
/** Edit-distance alignment. Two words far apart in time cannot pair. */
|
|
181
|
+
function align(a: Token[], b: Token[], maxSkew: number | null): Op[] {
|
|
182
|
+
const n = a.length, m = b.length
|
|
183
|
+
const INF = 1e9
|
|
184
|
+
const D = new Float64Array((n + 1) * (m + 1))
|
|
185
|
+
const W = m + 1
|
|
186
|
+
for (let i = 0; i <= n; i++) D[i * W] = i
|
|
187
|
+
for (let j = 0; j <= m; j++) D[j] = j
|
|
188
|
+
const pairCost = (i: number, j: number): number => {
|
|
189
|
+
const ta = a[i], tb = b[j]
|
|
190
|
+
const ma = mid(ta), mb = mid(tb)
|
|
191
|
+
if (maxSkew !== null && ma !== null && mb !== null && Math.abs(ma - mb) > maxSkew) return INF
|
|
192
|
+
return ta.key === tb.key ? 0 : 1
|
|
193
|
+
}
|
|
194
|
+
for (let i = 1; i <= n; i++) {
|
|
195
|
+
for (let j = 1; j <= m; j++) {
|
|
196
|
+
const diag = D[(i - 1) * W + j - 1] + pairCost(i - 1, j - 1)
|
|
197
|
+
const up = D[(i - 1) * W + j] + 1
|
|
198
|
+
const left = D[i * W + j - 1] + 1
|
|
199
|
+
D[i * W + j] = Math.min(diag, up, left)
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
const ops: Op[] = []
|
|
203
|
+
let i = n, j = m
|
|
204
|
+
while (i > 0 || j > 0) {
|
|
205
|
+
if (i > 0 && j > 0) {
|
|
206
|
+
const c = pairCost(i - 1, j - 1)
|
|
207
|
+
if (c < INF && D[i * W + j] === D[(i - 1) * W + j - 1] + c) {
|
|
208
|
+
ops.push({ kind: c === 0 ? '=' : 's', i: i - 1, j: j - 1 }); i--; j--; continue
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
if (i > 0 && D[i * W + j] === D[(i - 1) * W + j] + 1) { ops.push({ kind: 'p', i: i - 1, j: null }); i--; continue }
|
|
212
|
+
ops.push({ kind: 'q', i: null, j: j - 1 }); j--
|
|
213
|
+
}
|
|
214
|
+
return ops.reverse()
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function span(ts: Token[]): { start: number | null; end: number | null } {
|
|
218
|
+
const starts = ts.map(t => t.start).filter((x): x is number => x !== null)
|
|
219
|
+
const ends = ts.map(t => t.end ?? t.start).filter((x): x is number => x !== null)
|
|
220
|
+
return { start: starts.length ? Math.min(...starts) : null, end: ends.length ? Math.max(...ends) : null }
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function sameProtected(p: Token[], q: Token[]): boolean {
|
|
224
|
+
const pk = p.map(t => t.key).filter(isProtectedKey).join(' ')
|
|
225
|
+
const qk = q.map(t => t.key).filter(isProtectedKey).join(' ')
|
|
226
|
+
return pk === qk
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** A second-opinion-only run is taken only where the primary's VAD heard
|
|
230
|
+
* speech and the second opinion was confident. */
|
|
231
|
+
function gapVerdict(q: Token[], options: ResolveOptions): ResolutionReason {
|
|
232
|
+
const spans = options.speechSpans
|
|
233
|
+
if (!spans || !spans.length) return 'not_speech'
|
|
234
|
+
const timed = q.filter(t => t.start !== null)
|
|
235
|
+
if (!timed.length) return 'not_speech'
|
|
236
|
+
const inside = timed.filter(t => { const m = mid(t)!; return spans.some(([a, b]) => m >= a && m <= b) }).length
|
|
237
|
+
if (inside / timed.length < MIN_SPEECH_SHARE) return 'not_speech'
|
|
238
|
+
const scored = q.filter(t => typeof t.probability === 'number' && Number.isFinite(t.probability))
|
|
239
|
+
const floor = options.minGapConfidence ?? DEFAULT_MIN_GAP_CONFIDENCE
|
|
240
|
+
if (scored.length && scored.reduce((n, t) => n + (t.probability as number), 0) / scored.length < floor) return 'low_confidence'
|
|
241
|
+
return 'gap_fill'
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export function resolveHypotheses(primary: Hypothesis, second: Hypothesis, options: ResolveOptions): ResolveResult {
|
|
245
|
+
const P = tokens(primary), Q = tokens(second)
|
|
246
|
+
const toWords = (ts: Token[]): HypWord[] => ts.map(t => ({
|
|
247
|
+
word: t.display,
|
|
248
|
+
start: t.start ?? 0,
|
|
249
|
+
end: Math.max(t.start ?? 0, t.end ?? t.start ?? 0),
|
|
250
|
+
...(t.probability !== undefined ? { probability: t.probability } : {}),
|
|
251
|
+
}))
|
|
252
|
+
const primaryOnly = (): ResolveResult => ({
|
|
253
|
+
text: P.map(t => t.display).join(' '),
|
|
254
|
+
words: toWords(P),
|
|
255
|
+
timed: P.length > 0 && P.every(t => t.start !== null),
|
|
256
|
+
disagreements: [],
|
|
257
|
+
wordsFromSecond: 0,
|
|
258
|
+
applied: false,
|
|
259
|
+
})
|
|
260
|
+
if ((P.length + 1) * (Q.length + 1) > MAX_ALIGNMENT_CELLS) return { ...primaryOnly(), skipped: 'too_long' }
|
|
261
|
+
|
|
262
|
+
const sequences = vocabSequences(options.vocabulary)
|
|
263
|
+
const ops = align(P, Q, options.maxSkewSec ?? DEFAULT_MAX_SKEW_SEC)
|
|
264
|
+
const out: Token[] = []
|
|
265
|
+
const disagreements: Disagreement[] = []
|
|
266
|
+
let wordsFromSecond = 0
|
|
267
|
+
let k = 0
|
|
268
|
+
while (k < ops.length) {
|
|
269
|
+
if (ops[k].kind === '=') { out.push(P[ops[k].i!]); k++; continue }
|
|
270
|
+
const p: Token[] = [], q: Token[] = []
|
|
271
|
+
const regionStart = k
|
|
272
|
+
while (k < ops.length && ops[k].kind !== '=') {
|
|
273
|
+
if (ops[k].i !== null) p.push(P[ops[k].i!])
|
|
274
|
+
if (ops[k].j !== null) q.push(Q[ops[k].j!])
|
|
275
|
+
k++
|
|
276
|
+
}
|
|
277
|
+
// Agreed words on both sides, so a term split across the edge of the
|
|
278
|
+
// region ("drift cart" vs "thrift cart") is still seen whole.
|
|
279
|
+
const before: string[] = [], after: string[] = []
|
|
280
|
+
for (let b = regionStart - 1; b >= 0 && before.length < CONTEXT_WORDS && ops[b].kind === '='; b--) before.unshift(P[ops[b].i!].key)
|
|
281
|
+
for (let a = k; a < ops.length && after.length < CONTEXT_WORDS && ops[a].kind === '='; a++) after.push(P[ops[a].i!].key)
|
|
282
|
+
const kind: DisagreementKind = !p.length ? 'second_only' : !q.length ? 'primary_only' : 'substitution'
|
|
283
|
+
let take: 'primary' | 'second' = 'primary'
|
|
284
|
+
let reason: ResolutionReason = null
|
|
285
|
+
if (q.length) {
|
|
286
|
+
const withContext = (side: Token[]) => [...before, ...side.map(t => t.key), ...after]
|
|
287
|
+
const primaryTerms = termsIn(withContext(p), sequences)
|
|
288
|
+
const newTerms = [...termsIn(withContext(q), sequences)].filter(t => !primaryTerms.has(t))
|
|
289
|
+
const wantGap = kind === 'second_only' && options.primaryDeletionProne
|
|
290
|
+
const wantVocab = newTerms.length > 0
|
|
291
|
+
if (wantGap || wantVocab) {
|
|
292
|
+
if (!sameProtected(p, q)) reason = 'protected'
|
|
293
|
+
else if (wantVocab && (q.length > MAX_VOCAB_REGION_WORDS || p.length > q.length + MAX_VOCAB_REGION_SLACK)) {
|
|
294
|
+
// A term inside a long disagreement is not a reason to swap the whole region.
|
|
295
|
+
reason = wantGap ? gapVerdict(q, options) : 'too_wide'
|
|
296
|
+
if (reason === 'gap_fill') take = 'second'
|
|
297
|
+
} else if (wantVocab) { take = 'second'; reason = 'vocabulary' }
|
|
298
|
+
else { reason = gapVerdict(q, options); if (reason === 'gap_fill') take = 'second' }
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
const where = span([...p, ...q])
|
|
302
|
+
disagreements.push({
|
|
303
|
+
kind,
|
|
304
|
+
primary: p.map(t => t.display).join(' '),
|
|
305
|
+
second: q.map(t => t.display).join(' '),
|
|
306
|
+
start: where.start,
|
|
307
|
+
end: where.end,
|
|
308
|
+
taken: options.apply ? take : 'primary',
|
|
309
|
+
reason,
|
|
310
|
+
})
|
|
311
|
+
if (options.apply && take === 'second') { out.push(...q); wordsFromSecond += q.length }
|
|
312
|
+
else out.push(...p)
|
|
313
|
+
}
|
|
314
|
+
return {
|
|
315
|
+
text: out.map(t => t.display).join(' '),
|
|
316
|
+
words: toWords(out),
|
|
317
|
+
timed: out.length > 0 && out.every(t => t.start !== null),
|
|
318
|
+
disagreements,
|
|
319
|
+
wordsFromSecond,
|
|
320
|
+
applied: options.apply,
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** Whisper's deletion-prone profile is the reason gap fill exists. */
|
|
325
|
+
export function isDeletionProneEngine(engine: string): boolean {
|
|
326
|
+
return engine === 'large-v3' || engine === 'large-v3-legacy'
|
|
327
|
+
}
|