@gotcos/glasses-server 6.60.0 → 6.61.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +8 -0
- package/CHANGELOG.md +51 -0
- package/README.md +64 -0
- package/managed-runtime-contract.json +4 -1
- package/package.json +3 -1
- package/server/index.ts +11 -0
- package/server/lib/diarizer-backend.ts +163 -0
- package/server/lib/meeting-relabel-enrolment.ts +44 -7
- package/server/lib/meeting-store.ts +3 -1
- package/server/lib/nemotron-cli.ts +256 -0
- package/server/lib/nemotron-final.ts +1005 -0
- package/server/lib/nemotron-live.ts +736 -0
- package/server/lib/nemotron-segments.ts +358 -0
- package/server/lib/speaker-embeddings.ts +9 -5
- package/server/lib/voiceprint-floor.ts +12 -0
- package/server/routes/health.ts +10 -0
- package/server/routes/meeting.ts +55 -2
- package/server/routes/transcribe-stream.ts +75 -6
|
@@ -0,0 +1,1005 @@
|
|
|
1
|
+
// The final transcript: one Nemotron pass over the whole meeting at save time.
|
|
2
|
+
//
|
|
3
|
+
// Live chunks are diarized one at a time, so a chunk's Nemotron channels mean
|
|
4
|
+
// nothing in the next chunk. Run once over the meeting, the channels are
|
|
5
|
+
// consistent for the whole call, and the canary's method maps each to a name:
|
|
6
|
+
// 1. place every chunk WAV at its real `elapsed` on one timeline (join
|
|
7
|
+
// chunkEntries[].chunkIndex to chunk_NNNN.wav, check audioSha256, trim only
|
|
8
|
+
// the tail that would run into the next chunk);
|
|
9
|
+
// 2. score the raw 10 ms probabilities at 0.5, 8 channels;
|
|
10
|
+
// 3. a channel takes a name only from chunks the voiceprint is sure of
|
|
11
|
+
// (similarity >= 0.65, a real name, one-sided by Even Hub, or by Nemotron
|
|
12
|
+
// itself when the chunk has no Even Hub histogram), with at least
|
|
13
|
+
// FINAL_MIN_SUPPORT such chunks and FINAL_PURITY of them agreeing. A channel
|
|
14
|
+
// two people share stays unmapped.
|
|
15
|
+
// Every word then takes its channel's name, or keeps its per-chunk name (the
|
|
16
|
+
// live segment it sat in, else its existing label) where the channel is unmapped.
|
|
17
|
+
//
|
|
18
|
+
// SAFETY. Only labels change. The saved transcript's words are relabelled in
|
|
19
|
+
// place (never regenerated and re-cleaned), and nothing is written unless the
|
|
20
|
+
// result holds exactly the same words, in the same order. The original
|
|
21
|
+
// labels stay recoverable: every changed batch word carries `originalSpeaker`,
|
|
22
|
+
// every changed chunk carries `originalSpeaker`, and the sidecar keeps the whole
|
|
23
|
+
// pre-relabel transcript under `diarization.originalTranscript`. Any failure
|
|
24
|
+
// leaves the meeting exactly as it was saved. A human correction always wins:
|
|
25
|
+
// a sidecar with correctionRevision > 0 is never relabelled.
|
|
26
|
+
//
|
|
27
|
+
// No LLM anywhere. Transcription must never call `claude -p`.
|
|
28
|
+
|
|
29
|
+
import { createHash } from 'node:crypto'
|
|
30
|
+
import { chmodSync, closeSync, constants as fsConstants, existsSync, lstatSync, mkdtempSync, openSync, readFileSync, readSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
31
|
+
import { homedir, tmpdir } from 'node:os'
|
|
32
|
+
import { readFile, writeFile } from 'node:fs/promises'
|
|
33
|
+
import { join } from 'node:path'
|
|
34
|
+
import { durableAtomicWriteFileSync } from './atomic-fs.js'
|
|
35
|
+
import { selectBatchTranscriptForPersistence } from './batch-transcript-quality.js'
|
|
36
|
+
import {
|
|
37
|
+
NEMOTRON_BUNDLE,
|
|
38
|
+
NEMOTRON_VARIANT,
|
|
39
|
+
activeDiarizer,
|
|
40
|
+
nemotronCliPath,
|
|
41
|
+
prepareNemotronModelsDir,
|
|
42
|
+
} from './diarizer-backend.js'
|
|
43
|
+
import { CONFIDENT_SIMILARITY, isUnattributed } from './meeting-speaker-review.js'
|
|
44
|
+
import { getOwnerSpeakerLabel } from './profile.js'
|
|
45
|
+
import { OWNER_VERIFY_SIMILARITY } from './voiceprint-floor.js'
|
|
46
|
+
import {
|
|
47
|
+
NEMOTRON_TMP_PREFIX,
|
|
48
|
+
NemotronRunError,
|
|
49
|
+
runNemotronCli,
|
|
50
|
+
trackNemotronDir,
|
|
51
|
+
untrackNemotronDir,
|
|
52
|
+
type NemotronPreds,
|
|
53
|
+
type RunNemotronOptions,
|
|
54
|
+
} from './nemotron-cli.js'
|
|
55
|
+
import type { ChunkSpeakerSegment } from './nemotron-live.js'
|
|
56
|
+
import {
|
|
57
|
+
channelForSpan,
|
|
58
|
+
estimateTokenTimes,
|
|
59
|
+
frameRange,
|
|
60
|
+
isActive,
|
|
61
|
+
isWordToken,
|
|
62
|
+
readPcm16Mono,
|
|
63
|
+
splitTokens,
|
|
64
|
+
tokenTimesFromWords,
|
|
65
|
+
tracksIn,
|
|
66
|
+
type Activity,
|
|
67
|
+
type TokenTime,
|
|
68
|
+
wavHeader,
|
|
69
|
+
} from './nemotron-segments.js'
|
|
70
|
+
import { NEMOTRON_CHANNELS } from './nemotron-cli.js'
|
|
71
|
+
import type { WhisperWord } from './whisper-local.js'
|
|
72
|
+
|
|
73
|
+
/** A name is evidence only from a chunk the voiceprint was sure of: the review panel's own floor. */
|
|
74
|
+
export const FINAL_MIN_SIMILARITY = CONFIDENT_SIMILARITY
|
|
75
|
+
export const FINAL_ONE_SIDED = 0.8
|
|
76
|
+
export const FINAL_MIN_SUPPORT = 3
|
|
77
|
+
export const FINAL_PURITY = 0.8
|
|
78
|
+
/** The canary's eligibility for a meeting-wide map: at most this many identities... */
|
|
79
|
+
export const FINAL_MAX_IDENTITIES = 4
|
|
80
|
+
/** ...and fewer channels in use than Nemotron has, or two people already share one. */
|
|
81
|
+
export const FINAL_MAX_CHANNELS = NEMOTRON_CHANNELS
|
|
82
|
+
/** A timeline longer than this many times the audio it holds (plus slack) is a bad clock, not a meeting. */
|
|
83
|
+
export const FINAL_MAX_SPAN_RATIO = 1.5
|
|
84
|
+
export const FINAL_SPAN_SLACK_SEC = 600
|
|
85
|
+
const SR = 16_000
|
|
86
|
+
const WAV_HEADER = 44
|
|
87
|
+
|
|
88
|
+
interface EvenRole { frames?: number; self?: number; other?: number }
|
|
89
|
+
|
|
90
|
+
export interface SidecarChunk {
|
|
91
|
+
text: string
|
|
92
|
+
speaker: string
|
|
93
|
+
elapsed: number
|
|
94
|
+
similarity: number
|
|
95
|
+
words?: WhisperWord[]
|
|
96
|
+
segments?: ChunkSpeakerSegment[]
|
|
97
|
+
evenHubSpeakerRole?: EvenRole
|
|
98
|
+
audioSha256?: string
|
|
99
|
+
originalSpeaker?: string
|
|
100
|
+
finalSegments?: ChunkSpeakerSegment[]
|
|
101
|
+
[key: string]: unknown
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export interface SidecarEntry { chunkIndex: number; chunk: SidecarChunk }
|
|
105
|
+
|
|
106
|
+
export interface SpeakerWord {
|
|
107
|
+
word: string
|
|
108
|
+
start: number
|
|
109
|
+
end: number
|
|
110
|
+
speaker: string
|
|
111
|
+
similarity: number
|
|
112
|
+
originalSpeaker?: string
|
|
113
|
+
[key: string]: unknown
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export interface SidecarBatchSegment {
|
|
117
|
+
startChunkIdx: number
|
|
118
|
+
endChunkIdx: number
|
|
119
|
+
startElapsed: number
|
|
120
|
+
endElapsed: number
|
|
121
|
+
text: string
|
|
122
|
+
words?: WhisperWord[]
|
|
123
|
+
speakerWords: SpeakerWord[]
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export interface TimelineWindow {
|
|
127
|
+
chunkIndex: number
|
|
128
|
+
/** Seconds on the meeting timeline. */
|
|
129
|
+
start: number
|
|
130
|
+
end: number
|
|
131
|
+
chunk: SidecarChunk
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export interface MeetingTimeline {
|
|
135
|
+
/** A complete 16 kHz mono WAV. */
|
|
136
|
+
wav: Buffer
|
|
137
|
+
durationSec: number
|
|
138
|
+
windows: TimelineWindow[]
|
|
139
|
+
byIndex: Map<number, TimelineWindow>
|
|
140
|
+
/** PCM samples each chunk file holds as concatenateWavChunks reads it (bytes after a 44-byte header). */
|
|
141
|
+
fileSamples: Map<number, number>
|
|
142
|
+
excluded: { missing: number; hashMismatch: number; unreadable: number }
|
|
143
|
+
/** Seconds of chunk audio placed on the timeline. */
|
|
144
|
+
audioSec: number
|
|
145
|
+
/** True when the span is far beyond the audio: no WAV was built. */
|
|
146
|
+
tooLong: boolean
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const chunkFile = (audioDir: string, index: number): string => join(audioDir, `chunk_${String(index).padStart(4, '0')}.wav`)
|
|
150
|
+
|
|
151
|
+
/** lstat, so a symlink or a FIFO in the audio directory is never read. */
|
|
152
|
+
function isRegularFile(path: string): boolean {
|
|
153
|
+
try { return lstatSync(path).isFile() } catch { return false }
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Samples concatenateWavChunks reads from a chunk file: everything after a 44-byte header, if it is RIFF. */
|
|
157
|
+
function concatSamples(path: string): number | null {
|
|
158
|
+
try {
|
|
159
|
+
const stat = statSync(path)
|
|
160
|
+
if (!stat.isFile() || stat.size <= WAV_HEADER) return null
|
|
161
|
+
const size = stat.size
|
|
162
|
+
// Non-blocking and no symlinks: a FIFO planted in its place must never hang the server.
|
|
163
|
+
const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NONBLOCK | fsConstants.O_NOFOLLOW)
|
|
164
|
+
try {
|
|
165
|
+
const head = Buffer.alloc(4)
|
|
166
|
+
readSync(fd, head, 0, 4, 0)
|
|
167
|
+
return head.toString('ascii') === 'RIFF' ? Math.floor((size - WAV_HEADER) / 2) : null
|
|
168
|
+
} finally { closeSync(fd) }
|
|
169
|
+
} catch {
|
|
170
|
+
return null
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* The canary's timeline: each WAV at its real elapsed, only the overlapping tail
|
|
176
|
+
* trimmed. Two passes so that only the timeline itself is held in memory (a
|
|
177
|
+
* 90-minute meeting is about 170 MB of 16-bit audio): the first checks every
|
|
178
|
+
* chunk and keeps its place, the second copies its audio in.
|
|
179
|
+
*/
|
|
180
|
+
/** Give the event loop back every this many chunks while the timeline is built. */
|
|
181
|
+
const TIMELINE_YIELD_EVERY = 25
|
|
182
|
+
const yieldToLoop = (): Promise<void> => new Promise(resolve => setImmediate(resolve))
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Async and chunked (QA 6.61.0, N7): reads are non-blocking and the loop is given
|
|
186
|
+
* back every TIMELINE_YIELD_EVERY chunks, so a long meeting's timeline does not
|
|
187
|
+
* freeze live chunk uploads while it is built.
|
|
188
|
+
*
|
|
189
|
+
* Returns `tooLong` instead of a timeline when the span the clocks claim is far
|
|
190
|
+
* beyond the audio the chunks hold (QA W5: one chunk stamped three hours late
|
|
191
|
+
* made a 346 MB timeline out of 12 s of audio). Nothing is allocated then.
|
|
192
|
+
*/
|
|
193
|
+
export async function buildMeetingTimeline(entries: SidecarEntry[], audioDir: string): Promise<MeetingTimeline> {
|
|
194
|
+
const excluded = { missing: 0, hashMismatch: 0, unreadable: 0 }
|
|
195
|
+
const fileSamples = new Map<number, number>()
|
|
196
|
+
const indices = entries.map(entry => entry.chunkIndex)
|
|
197
|
+
const maxIndex = indices.length ? Math.max(...indices) : -1
|
|
198
|
+
for (let index = 0; index <= maxIndex; index++) {
|
|
199
|
+
const samples = concatSamples(chunkFile(audioDir, index))
|
|
200
|
+
if (samples !== null) fileSamples.set(index, samples)
|
|
201
|
+
}
|
|
202
|
+
const placed: Array<{ entry: SidecarEntry; at: number; samples: number; dataOffset: number }> = []
|
|
203
|
+
let seen = 0
|
|
204
|
+
for (const entry of entries) {
|
|
205
|
+
if (++seen % TIMELINE_YIELD_EVERY === 0) await yieldToLoop()
|
|
206
|
+
const path = chunkFile(audioDir, entry.chunkIndex)
|
|
207
|
+
if (!existsSync(path)) { excluded.missing++; continue }
|
|
208
|
+
if (!isRegularFile(path)) { excluded.unreadable++; continue }
|
|
209
|
+
let wav: Buffer
|
|
210
|
+
try { wav = await readFile(path) } catch { excluded.unreadable++; continue }
|
|
211
|
+
const expected = entry.chunk.audioSha256
|
|
212
|
+
if (expected && createHash('sha256').update(wav).digest('hex') !== expected) { excluded.hashMismatch++; continue }
|
|
213
|
+
const pcm = readPcm16Mono(wav)
|
|
214
|
+
if (!pcm || pcm.sampleRate !== SR || !Number.isFinite(entry.chunk.elapsed)) { excluded.unreadable++; continue }
|
|
215
|
+
placed.push({ entry, at: Math.max(0, Math.round((entry.chunk.elapsed / 1000) * SR)), samples: pcm.pcm.length / 2, dataOffset: pcm.pcm.byteOffset - wav.byteOffset })
|
|
216
|
+
}
|
|
217
|
+
placed.sort((x, y) => x.at - y.at || x.entry.chunkIndex - y.entry.chunkIndex)
|
|
218
|
+
let total = 0
|
|
219
|
+
let audio = 0
|
|
220
|
+
for (const item of placed) { total = Math.max(total, item.at + item.samples); audio += item.samples }
|
|
221
|
+
total += SR
|
|
222
|
+
const empty = { wav: Buffer.alloc(0), windows: [], byIndex: new Map<number, TimelineWindow>(), fileSamples, excluded }
|
|
223
|
+
if (total / SR > (audio / SR) * FINAL_MAX_SPAN_RATIO + FINAL_SPAN_SLACK_SEC) {
|
|
224
|
+
return { ...empty, durationSec: total / SR, audioSec: audio / SR, tooLong: true }
|
|
225
|
+
}
|
|
226
|
+
const out = Buffer.alloc(WAV_HEADER + total * 2)
|
|
227
|
+
wavHeader(total * 2, SR).copy(out, 0)
|
|
228
|
+
const windows: TimelineWindow[] = []
|
|
229
|
+
for (let i = 0; i < placed.length; i++) {
|
|
230
|
+
if ((i + 1) % TIMELINE_YIELD_EVERY === 0) await yieldToLoop()
|
|
231
|
+
const item = placed[i]
|
|
232
|
+
const next = i + 1 < placed.length ? placed[i + 1].at : item.at + item.samples
|
|
233
|
+
const room = Math.max(0, next - item.at)
|
|
234
|
+
const use = Math.min(item.samples, room)
|
|
235
|
+
let copied = 0
|
|
236
|
+
try {
|
|
237
|
+
if (!isRegularFile(chunkFile(audioDir, item.entry.chunkIndex))) throw new Error('not a regular file')
|
|
238
|
+
const wav = await readFile(chunkFile(audioDir, item.entry.chunkIndex))
|
|
239
|
+
copied = wav.copy(out, WAV_HEADER + item.at * 2, item.dataOffset, Math.min(wav.length, item.dataOffset + use * 2)) / 2
|
|
240
|
+
} catch { /* vanished between passes: its window stays silent */ }
|
|
241
|
+
windows.push({ chunkIndex: item.entry.chunkIndex, start: item.at / SR, end: (item.at + Math.floor(copied)) / SR, chunk: item.entry.chunk })
|
|
242
|
+
}
|
|
243
|
+
return {
|
|
244
|
+
wav: out,
|
|
245
|
+
durationSec: total / SR,
|
|
246
|
+
audioSec: audio / SR,
|
|
247
|
+
tooLong: false,
|
|
248
|
+
windows,
|
|
249
|
+
byIndex: new Map(windows.map(window => [window.chunkIndex, window])),
|
|
250
|
+
fileSamples,
|
|
251
|
+
excluded,
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** Too full for one meeting-wide map: more than four identities, or every channel in use. */
|
|
256
|
+
export function isCrowded(identities: number, channelsUsed: number): boolean {
|
|
257
|
+
return identities > FINAL_MAX_IDENTITIES || channelsUsed >= FINAL_MAX_CHANNELS
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** Names with at least three chunks the voiceprint is sure of: the canary's identity count. */
|
|
261
|
+
export function crowdIdentities(windows: TimelineWindow[]): number {
|
|
262
|
+
const counts = new Map<string, number>()
|
|
263
|
+
for (const window of windows) {
|
|
264
|
+
const chunk = window.chunk
|
|
265
|
+
if (isName(chunk.speaker) && Number(chunk.similarity) >= FINAL_MIN_SIMILARITY) counts.set(chunk.speaker, (counts.get(chunk.speaker) ?? 0) + 1)
|
|
266
|
+
}
|
|
267
|
+
return [...counts.values()].filter(count => count >= FINAL_MIN_SUPPORT).length
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function ownerLabelOrNull(): string | null {
|
|
271
|
+
try { return getOwnerSpeakerLabel() } catch { return null }
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function isName(label: unknown): label is string {
|
|
275
|
+
return typeof label === 'string' && !isUnattributed(label.trim())
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
export interface ChannelMapping {
|
|
279
|
+
map: Map<number, string>
|
|
280
|
+
evidence: Array<{ channel: number; name: string | null; support: number; purity: number; top: string | null }>
|
|
281
|
+
eligible: number
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** The canary's dominant-channel map, made many-to-one with a purity floor. */
|
|
285
|
+
export function mapChannelsToNames(windows: TimelineWindow[], a: Activity, options: {
|
|
286
|
+
minSimilarity?: number
|
|
287
|
+
oneSided?: number
|
|
288
|
+
minSupport?: number
|
|
289
|
+
purity?: number
|
|
290
|
+
} = {}): ChannelMapping {
|
|
291
|
+
const minSimilarity = options.minSimilarity ?? FINAL_MIN_SIMILARITY
|
|
292
|
+
const oneSided = options.oneSided ?? FINAL_ONE_SIDED
|
|
293
|
+
const minSupport = options.minSupport ?? FINAL_MIN_SUPPORT
|
|
294
|
+
const purity = options.purity ?? FINAL_PURITY
|
|
295
|
+
const tallies = new Map<number, Map<string, number>>()
|
|
296
|
+
let eligible = 0
|
|
297
|
+
for (const window of windows) {
|
|
298
|
+
const chunk = window.chunk
|
|
299
|
+
if (!isName(chunk.speaker) || !(Number(chunk.similarity) >= minSimilarity)) continue
|
|
300
|
+
const [f0, f1] = frameRange(a, window.start, window.end)
|
|
301
|
+
const counts = new Array<number>(NEMOTRON_CHANNELS).fill(0)
|
|
302
|
+
let speech = 0
|
|
303
|
+
for (let f = f0; f < f1; f++) {
|
|
304
|
+
let any = false
|
|
305
|
+
for (let c = 0; c < NEMOTRON_CHANNELS; c++) if (isActive(a, f, c)) { counts[c]++; any = true }
|
|
306
|
+
if (any) speech++
|
|
307
|
+
}
|
|
308
|
+
if (speech === 0) continue
|
|
309
|
+
let dom = 0
|
|
310
|
+
for (let c = 1; c < NEMOTRON_CHANNELS; c++) if (counts[c] > counts[dom]) dom = c
|
|
311
|
+
const role = chunk.evenHubSpeakerRole
|
|
312
|
+
const frames = Number(role?.frames ?? 0)
|
|
313
|
+
const sided = frames > 0
|
|
314
|
+
? Math.max(Number(role?.self ?? 0), Number(role?.other ?? 0)) / frames >= oneSided
|
|
315
|
+
: counts[dom] / speech >= oneSided
|
|
316
|
+
if (!sided) continue
|
|
317
|
+
eligible++
|
|
318
|
+
const tally = tallies.get(dom) ?? new Map<string, number>()
|
|
319
|
+
tally.set(chunk.speaker, (tally.get(chunk.speaker) ?? 0) + 1)
|
|
320
|
+
tallies.set(dom, tally)
|
|
321
|
+
}
|
|
322
|
+
const map = new Map<number, string>()
|
|
323
|
+
const evidence: ChannelMapping['evidence'] = []
|
|
324
|
+
for (const [channel, tally] of [...tallies.entries()].sort((x, y) => x[0] - y[0])) {
|
|
325
|
+
let top: string | null = null
|
|
326
|
+
let topCount = 0
|
|
327
|
+
let support = 0
|
|
328
|
+
for (const [name, count] of tally) {
|
|
329
|
+
support += count
|
|
330
|
+
if (count > topCount || (count === topCount && top !== null && name < top)) { top = name; topCount = count }
|
|
331
|
+
}
|
|
332
|
+
const share = support ? topCount / support : 0
|
|
333
|
+
const accepted = top !== null && support >= minSupport && share >= purity
|
|
334
|
+
if (accepted) map.set(channel, top!)
|
|
335
|
+
evidence.push({ channel, name: accepted ? top : null, support, purity: Math.round(share * 1000) / 1000, top })
|
|
336
|
+
}
|
|
337
|
+
return { map, evidence, eligible }
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/** The live segment that held chunk-relative time t, else null. */
|
|
341
|
+
/**
|
|
342
|
+
* 6.61.1 owner guard for per-chunk names. 6.61.0 named split tracks as the wearer
|
|
343
|
+
* from the glasses' default label (similarity 0) or a weak search match
|
|
344
|
+
* (0.55 to 0.65). In a chunk with more than one live turn, the wearer's name
|
|
345
|
+
* counts only at owner-verify confidence; otherwise it is Ext. A one-turn chunk
|
|
346
|
+
* keeps its label exactly as 6.60.0 gave it.
|
|
347
|
+
*/
|
|
348
|
+
export function guardOwnerName(name: string, similarity: unknown, chunk: SidecarChunk, owner: string | null | undefined): string {
|
|
349
|
+
if (!owner || name !== owner) return name
|
|
350
|
+
const split = Array.isArray(chunk.segments) && chunk.segments.length > 1
|
|
351
|
+
if (!split) return name
|
|
352
|
+
return Number(similarity) >= OWNER_VERIFY_SIMILARITY ? name : 'Ext'
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function liveSpeakerAt(chunk: SidecarChunk, t: number, owner?: string | null): string | null {
|
|
356
|
+
const segments = chunk.segments
|
|
357
|
+
if (!Array.isArray(segments) || segments.length === 0) return null
|
|
358
|
+
let best: ChunkSpeakerSegment | null = null
|
|
359
|
+
let distance = Number.POSITIVE_INFINITY
|
|
360
|
+
for (const segment of segments) {
|
|
361
|
+
const s = Number(segment.startSec)
|
|
362
|
+
const e = Number(segment.endSec)
|
|
363
|
+
if (!Number.isFinite(s) || !Number.isFinite(e)) continue
|
|
364
|
+
const d = t < s ? s - t : t > e ? t - e : 0
|
|
365
|
+
if (d < distance) { distance = d; best = segment }
|
|
366
|
+
}
|
|
367
|
+
return best && typeof best.speaker === 'string' ? guardOwnerName(best.speaker, best.similarity, chunk, owner) : null
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
export interface RelabelStats {
|
|
371
|
+
total: number
|
|
372
|
+
/** Words whose label changed. */
|
|
373
|
+
relabeled: number
|
|
374
|
+
/** Words labelled through a mapped channel. */
|
|
375
|
+
viaChannel: number
|
|
376
|
+
/** Words whose channel was unmapped or silent, kept on their per-chunk name. */
|
|
377
|
+
kept: number
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** Word times in a batch segment are relative to its concatenated chunk files. Put them on the timeline. */
|
|
381
|
+
export function batchWordToTimeline(segment: SidecarBatchSegment, t: number, timeline: MeetingTimeline): { window: TimelineWindow; inChunk: number; at: number } | null {
|
|
382
|
+
let offset = 0
|
|
383
|
+
let lastWindow: { window: TimelineWindow; inChunk: number; at: number } | null = null
|
|
384
|
+
for (let index = segment.startChunkIdx; index <= segment.endChunkIdx; index++) {
|
|
385
|
+
const samples = timeline.fileSamples.get(index)
|
|
386
|
+
if (samples === undefined) continue
|
|
387
|
+
const length = samples / SR
|
|
388
|
+
const window = timeline.byIndex.get(index)
|
|
389
|
+
if (t < offset + length || index === segment.endChunkIdx) {
|
|
390
|
+
if (!window) return null
|
|
391
|
+
const inChunk = Math.max(0, t - offset)
|
|
392
|
+
const span = Math.max(0, window.end - window.start - 0.005)
|
|
393
|
+
return { window, inChunk, at: window.start + Math.min(inChunk, span) }
|
|
394
|
+
}
|
|
395
|
+
if (window) lastWindow = { window, inChunk: length, at: window.end - 0.005 }
|
|
396
|
+
offset += length
|
|
397
|
+
}
|
|
398
|
+
return lastWindow
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
export function relabelBatchSegments(
|
|
402
|
+
segments: SidecarBatchSegment[],
|
|
403
|
+
timeline: MeetingTimeline,
|
|
404
|
+
a: Activity,
|
|
405
|
+
map: Map<number, string>,
|
|
406
|
+
owner?: string | null,
|
|
407
|
+
): { segments: SidecarBatchSegment[]; stats: RelabelStats } {
|
|
408
|
+
const stats: RelabelStats = { total: 0, relabeled: 0, viaChannel: 0, kept: 0 }
|
|
409
|
+
const out = segments.map(segment => ({
|
|
410
|
+
...segment,
|
|
411
|
+
speakerWords: (segment.speakerWords ?? []).map(word => {
|
|
412
|
+
stats.total++
|
|
413
|
+
const original = typeof word.originalSpeaker === 'string' ? word.originalSpeaker : word.speaker
|
|
414
|
+
const start = batchWordToTimeline(segment, Number(word.start), timeline)
|
|
415
|
+
const end = batchWordToTimeline(segment, Number(word.end), timeline)
|
|
416
|
+
let name: string | null = null
|
|
417
|
+
if (start) {
|
|
418
|
+
const channel = channelForSpan(a, start.at, Math.max(start.at, end?.window === start.window ? end.at : start.at + 0.01))
|
|
419
|
+
if (channel !== null && map.has(channel)) { name = map.get(channel)!; stats.viaChannel++ }
|
|
420
|
+
}
|
|
421
|
+
if (name === null) {
|
|
422
|
+
stats.kept++
|
|
423
|
+
name = (start && liveSpeakerAt(start.window.chunk, start.inChunk, owner))
|
|
424
|
+
?? (start ? guardOwnerName(original, start.window.chunk.similarity, start.window.chunk, owner) : original)
|
|
425
|
+
}
|
|
426
|
+
if (name !== word.speaker) stats.relabeled++
|
|
427
|
+
const next: SpeakerWord = { ...word, speaker: name }
|
|
428
|
+
if (name !== original) next.originalSpeaker = original
|
|
429
|
+
else delete next.originalSpeaker
|
|
430
|
+
return next
|
|
431
|
+
}),
|
|
432
|
+
}))
|
|
433
|
+
return { segments: out, stats }
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/** Token times for a streaming chunk, chunk-relative seconds. */
|
|
437
|
+
function streamingTokenTimes(chunk: SidecarChunk, tokens: string[], window: TimelineWindow, a: Activity): TokenTime[] | null {
|
|
438
|
+
if (Array.isArray(chunk.words) && chunk.words.length) {
|
|
439
|
+
const fromWords = tokenTimesFromWords(tokens, chunk.words)
|
|
440
|
+
if (fromWords) return fromWords
|
|
441
|
+
}
|
|
442
|
+
const shift = (times: TokenTime[] | null): TokenTime[] | null => times && times.map(time => ({ start: time.start - window.start, end: time.end - window.start }))
|
|
443
|
+
const segments = Array.isArray(chunk.segments) ? chunk.segments : []
|
|
444
|
+
const counts = segments.map(segment => splitTokens(String(segment.text ?? '')).length)
|
|
445
|
+
if (segments.length && counts.reduce((sum, n) => sum + n, 0) === tokens.length && segments.map(s => s.text).join(' ') === chunk.text) {
|
|
446
|
+
const out: TokenTime[] = []
|
|
447
|
+
let from = 0
|
|
448
|
+
for (let i = 0; i < segments.length; i++) {
|
|
449
|
+
const piece = tokens.slice(from, from + counts[i])
|
|
450
|
+
const s = window.start + Number(segments[i].startSec ?? 0)
|
|
451
|
+
const e = window.start + Number(segments[i].endSec ?? (window.end - window.start))
|
|
452
|
+
const times = shift(estimateTokenTimes(piece, a, s, Math.max(s + 0.01, e)))
|
|
453
|
+
?? piece.map(() => ({ start: s - window.start, end: e - window.start }))
|
|
454
|
+
out.push(...times)
|
|
455
|
+
from += counts[i]
|
|
456
|
+
}
|
|
457
|
+
return out
|
|
458
|
+
}
|
|
459
|
+
return shift(estimateTokenTimes(tokens, a, window.start, window.end))
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
export function relabelStreamingChunks(
|
|
463
|
+
timeline: MeetingTimeline,
|
|
464
|
+
a: Activity,
|
|
465
|
+
map: Map<number, string>,
|
|
466
|
+
owner?: string | null,
|
|
467
|
+
): { finals: Map<number, { speaker: string; segments: ChunkSpeakerSegment[] }>; stats: RelabelStats } {
|
|
468
|
+
const stats: RelabelStats = { total: 0, relabeled: 0, viaChannel: 0, kept: 0 }
|
|
469
|
+
const finals = new Map<number, { speaker: string; segments: ChunkSpeakerSegment[] }>()
|
|
470
|
+
for (const window of timeline.windows) {
|
|
471
|
+
const chunk = window.chunk
|
|
472
|
+
if (typeof chunk.text !== 'string' || !chunk.text.trim()) continue
|
|
473
|
+
const tokens = splitTokens(chunk.text)
|
|
474
|
+
const times = streamingTokenTimes(chunk, tokens, window, a)
|
|
475
|
+
const base = guardOwnerName(typeof chunk.originalSpeaker === 'string' ? chunk.originalSpeaker : chunk.speaker, chunk.similarity, chunk, owner)
|
|
476
|
+
const labels: string[] = []
|
|
477
|
+
let carry: string | null = null
|
|
478
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
479
|
+
if (!isWordToken(tokens[i]) || !times) { labels.push(carry ?? liveSpeakerAt(chunk, 0, owner) ?? base); continue }
|
|
480
|
+
stats.total++
|
|
481
|
+
const span = times[i]
|
|
482
|
+
const channel = channelForSpan(a, window.start + span.start, window.start + Math.max(span.end, span.start + 0.01))
|
|
483
|
+
let name: string
|
|
484
|
+
if (channel !== null && map.has(channel)) { name = map.get(channel)!; stats.viaChannel++ }
|
|
485
|
+
else { name = liveSpeakerAt(chunk, span.start, owner) ?? base; stats.kept++ }
|
|
486
|
+
labels.push(name)
|
|
487
|
+
carry = name
|
|
488
|
+
}
|
|
489
|
+
const segments: ChunkSpeakerSegment[] = []
|
|
490
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
491
|
+
const last = segments.at(-1)
|
|
492
|
+
if (last && last.speaker === labels[i]) { last.text += ` ${tokens[i]}`; if (times) last.endSec = Math.round(times[i].end * 100) / 100; continue }
|
|
493
|
+
segments.push({
|
|
494
|
+
speaker: labels[i],
|
|
495
|
+
text: tokens[i],
|
|
496
|
+
...(times ? { startSec: Math.round(times[i].start * 100) / 100, endSec: Math.round(times[i].end * 100) / 100 } : {}),
|
|
497
|
+
})
|
|
498
|
+
}
|
|
499
|
+
const wordsBy = new Map<string, number>()
|
|
500
|
+
for (const segment of segments) wordsBy.set(segment.speaker, (wordsBy.get(segment.speaker) ?? 0) + splitTokens(segment.text).filter(isWordToken).length)
|
|
501
|
+
let speaker = chunk.speaker
|
|
502
|
+
let most = -1
|
|
503
|
+
for (const [name, count] of wordsBy) if (count > most) { most = count; speaker = name }
|
|
504
|
+
const liveLabels = Array.isArray(chunk.segments) && chunk.segments.length
|
|
505
|
+
? chunk.segments.flatMap(segment => splitTokens(String(segment.text ?? '')).map(() => segment.speaker))
|
|
506
|
+
: tokens.map(() => chunk.speaker)
|
|
507
|
+
for (let i = 0; i < tokens.length; i++) if (isWordToken(tokens[i]) && labels[i] !== (liveLabels[i] ?? chunk.speaker)) stats.relabeled++
|
|
508
|
+
finals.set(window.chunkIndex, { speaker, segments })
|
|
509
|
+
}
|
|
510
|
+
return { finals, stats }
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const LABEL_PREFIX = /^\[[^\]\n]+\]:\s*/
|
|
514
|
+
|
|
515
|
+
/** The words of a transcript with speaker labels removed. Gap markers stay words. */
|
|
516
|
+
export function transcriptWords(transcript: string): string[] {
|
|
517
|
+
return transcript.split('\n').flatMap(line => line.replace(LABEL_PREFIX, '').split(/\s+/).filter(Boolean))
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
export function sameWords(before: string, after: string): boolean {
|
|
521
|
+
const x = transcriptWords(before)
|
|
522
|
+
const y = transcriptWords(after)
|
|
523
|
+
return x.length === y.length && x.every((word, i) => word === y[i])
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/** Re-render a streaming transcript: each `[X]: text` line that is exactly one chunk's text becomes that chunk's final turns. */
|
|
527
|
+
export function relabelStreamingTranscript(
|
|
528
|
+
transcript: string,
|
|
529
|
+
chunksInOrder: Array<{ chunkIndex: number; text: string }>,
|
|
530
|
+
finals: Map<number, { speaker: string; segments: ChunkSpeakerSegment[] }>,
|
|
531
|
+
): string {
|
|
532
|
+
const out: string[] = []
|
|
533
|
+
let pointer = 0
|
|
534
|
+
for (const line of transcript.split('\n')) {
|
|
535
|
+
const match = /^\[([^\]\n]+)\]: ([\s\S]*)$/.exec(line)
|
|
536
|
+
if (!match) { out.push(line); continue }
|
|
537
|
+
let found = -1
|
|
538
|
+
for (let k = pointer; k < Math.min(chunksInOrder.length, pointer + 8); k++) {
|
|
539
|
+
if (chunksInOrder[k].text === match[2]) { found = k; break }
|
|
540
|
+
}
|
|
541
|
+
if (found < 0) { out.push(line); continue }
|
|
542
|
+
pointer = found + 1
|
|
543
|
+
const final = finals.get(chunksInOrder[found].chunkIndex)
|
|
544
|
+
if (!final) { out.push(line); continue }
|
|
545
|
+
for (const segment of final.segments) out.push(`[${segment.speaker}]: ${segment.text}`)
|
|
546
|
+
}
|
|
547
|
+
return out.join('\n')
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* Relabel a transcript that is already on disk, word by word, without touching a
|
|
552
|
+
* word. The saved batch transcript is the speaker-words grouping AFTER cleaning
|
|
553
|
+
* (URL-only lines dropped, negative rules applied), so regenerating it from the
|
|
554
|
+
* relabelled words and cleaning again can change which words survive when the
|
|
555
|
+
* groups change. Instead each transcript word is matched, in order, to the next
|
|
556
|
+
* relabelled word with the same text (a small window, then a three-word anchor
|
|
557
|
+
* to resynchronise past anything cleaning removed). A word with no match keeps
|
|
558
|
+
* the turn it is in. Gap markers and other unlabelled lines are kept as they are.
|
|
559
|
+
*/
|
|
560
|
+
export function relabelTranscriptWords(transcript: string, sequence: Array<{ word: string; speaker: string }>): string {
|
|
561
|
+
const WINDOW = 12
|
|
562
|
+
const ANCHOR = 3
|
|
563
|
+
const RESYNC = 600
|
|
564
|
+
const lines = transcript.split('\n')
|
|
565
|
+
const parsed = lines.map(line => {
|
|
566
|
+
const match = /^\[([^\]\n]+)\]:\s*([\s\S]*)$/.exec(line)
|
|
567
|
+
return match ? { speaker: match[1], words: match[2].split(/\s+/).filter(Boolean) } : null
|
|
568
|
+
})
|
|
569
|
+
const flat: string[] = parsed.flatMap(line => line?.words ?? [])
|
|
570
|
+
const out: string[] = []
|
|
571
|
+
let pointer = 0
|
|
572
|
+
let flatIndex = 0
|
|
573
|
+
for (let li = 0; li < lines.length; li++) {
|
|
574
|
+
const line = parsed[li]
|
|
575
|
+
if (!line) { out.push(lines[li]); continue }
|
|
576
|
+
// Line breaks are kept (QA 6.61.0 W4): a line splits where its words change
|
|
577
|
+
// speaker and is never merged with its neighbour, so a pass that changes no
|
|
578
|
+
// label reproduces the transcript byte for byte.
|
|
579
|
+
const turns: Array<{ speaker: string; words: string[] }> = []
|
|
580
|
+
for (const word of line.words) {
|
|
581
|
+
let speaker: string | null = null
|
|
582
|
+
for (let k = pointer; k < Math.min(sequence.length, pointer + WINDOW); k++) {
|
|
583
|
+
if (sequence[k].word === word) { speaker = sequence[k].speaker; pointer = k + 1; break }
|
|
584
|
+
}
|
|
585
|
+
if (speaker === null) {
|
|
586
|
+
const anchor = flat.slice(flatIndex, flatIndex + ANCHOR)
|
|
587
|
+
if (anchor.length === ANCHOR) {
|
|
588
|
+
for (let k = pointer; k <= Math.min(sequence.length - ANCHOR, pointer + RESYNC); k++) {
|
|
589
|
+
if (anchor.every((w, a) => sequence[k + a].word === w)) { speaker = sequence[k].speaker; pointer = k + 1; break }
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
if (speaker === null) speaker = turns.at(-1)?.speaker ?? line.speaker
|
|
594
|
+
const turn = turns.at(-1)
|
|
595
|
+
if (turn && turn.speaker === speaker) turn.words.push(word)
|
|
596
|
+
else turns.push({ speaker, words: [word] })
|
|
597
|
+
flatIndex++
|
|
598
|
+
}
|
|
599
|
+
if (turns.length === 1 && turns[0].speaker === line.speaker) { out.push(lines[li]); continue }
|
|
600
|
+
for (const turn of turns) out.push(`[${turn.speaker}]: ${turn.words.join(' ')}`)
|
|
601
|
+
}
|
|
602
|
+
return out.join('\n')
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
const TRANSCRIPT_SECTION = /## Transcript\n\n([\s\S]*)$/
|
|
606
|
+
|
|
607
|
+
export function readTranscriptSection(markdown: string): string | null {
|
|
608
|
+
const match = TRANSCRIPT_SECTION.exec(markdown)
|
|
609
|
+
return match ? match[1].replace(/\n+$/, '') : null
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/** Replace the transcript section and nothing else (the quality row stays as saved). */
|
|
613
|
+
export function replaceTranscriptSection(markdown: string, transcript: string): string | null {
|
|
614
|
+
if (!TRANSCRIPT_SECTION.test(markdown)) return null
|
|
615
|
+
return markdown.replace(TRANSCRIPT_SECTION, () => `## Transcript\n\n${transcript}\n`)
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
const sha = (text: string): string => createHash('sha256').update(text).digest('hex')
|
|
619
|
+
|
|
620
|
+
export type FinalPassStatus = 'applied' | 'skipped' | 'failed'
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* Every reason a final pass can give, a fixed set. `/api/health` is public (no
|
|
624
|
+
* token) and the server listens on the LAN, so health may carry only these codes
|
|
625
|
+
* and counts: never a speaker name, a path, a meeting title or a raw error
|
|
626
|
+
* message (QA 6.61.0 blocker). Anything else is reported as `error`, and the raw
|
|
627
|
+
* text goes to server.log only.
|
|
628
|
+
*/
|
|
629
|
+
export const FINAL_REASON_CODES = [
|
|
630
|
+
'embedding_requested', 'cli_missing', 'models_missing', 'meeting_missing', 'no_audio', 'human_corrected',
|
|
631
|
+
'no_transcript', 'already_applied', 'no_chunks', 'nemotron_timeout', 'nemotron_cli_failed',
|
|
632
|
+
'nemotron_bad_output', 'nemotron_spawn_failed', 'nemotron_error', 'no_speaker_words', 'chunk_map_ambiguous',
|
|
633
|
+
'word_mismatch', 'no_change', 'changed_during_pass', 'markdown_write', 'sidecar_restore_failed', 'too_long',
|
|
634
|
+
'metadata_not_persisted', 'error',
|
|
635
|
+
] as const
|
|
636
|
+
export type FinalReasonCode = typeof FINAL_REASON_CODES[number]
|
|
637
|
+
const REASON_SET: ReadonlySet<string> = new Set(FINAL_REASON_CODES)
|
|
638
|
+
export const finalReasonCode = (reason: string | undefined): FinalReasonCode | null =>
|
|
639
|
+
reason === undefined ? null : REASON_SET.has(reason) ? reason as FinalReasonCode : 'error'
|
|
640
|
+
|
|
641
|
+
export interface FinalPassOutcome {
|
|
642
|
+
status: FinalPassStatus
|
|
643
|
+
reason?: string
|
|
644
|
+
/** Raw failure text for server.log. Never published. */
|
|
645
|
+
detail?: string
|
|
646
|
+
mode?: 'batch' | 'streaming'
|
|
647
|
+
/** True when the room was too full for a meeting-wide map (only per-chunk turns applied). */
|
|
648
|
+
crowded?: boolean
|
|
649
|
+
identities?: number
|
|
650
|
+
ms?: number
|
|
651
|
+
audioSec?: number
|
|
652
|
+
channels?: number
|
|
653
|
+
mapped?: Record<string, string>
|
|
654
|
+
words?: RelabelStats
|
|
655
|
+
/** Chunks left off the timeline: no file, a hash that disagrees, unreadable. */
|
|
656
|
+
excluded?: { missing: number; hashMismatch: number; unreadable: number }
|
|
657
|
+
/** Distinct speakers in the transcript before and after. */
|
|
658
|
+
speakersBefore?: number
|
|
659
|
+
speakersAfter?: number
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
export interface FinalPassOptions {
|
|
663
|
+
meetingPath: string
|
|
664
|
+
sidecarPath: string
|
|
665
|
+
audioDir: string
|
|
666
|
+
env?: NodeJS.ProcessEnv
|
|
667
|
+
home?: string
|
|
668
|
+
run?: (options: RunNemotronOptions) => Promise<NemotronPreds & { loadSec?: number | null }>
|
|
669
|
+
tmpRoot?: string
|
|
670
|
+
log?: (line: string) => void
|
|
671
|
+
now?: () => number
|
|
672
|
+
/** The wearer's label, for the 6.61.1 owner guard (default: the profile's). */
|
|
673
|
+
owner?: string | null
|
|
674
|
+
/** Seam: how files are written (durable atomic by default). */
|
|
675
|
+
write?: (path: string, data: string) => void
|
|
676
|
+
/** Seam: the batch relabeller (word-aligned in place by default). */
|
|
677
|
+
relabelWords?: (transcript: string, sequence: Array<{ word: string; speaker: string }>) => string
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
/** 20x realtime is the floor (the canary ran 155x), plus two minutes for a cold Neural Engine compile. */
|
|
681
|
+
export function finalPassTimeoutMs(durationSec: number): number {
|
|
682
|
+
return 120_000 + Math.round((durationSec * 1000) / 20)
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/** What `/api/health` may say about the last final pass: codes and counts only. */
|
|
686
|
+
export interface PublicFinalOutcome {
|
|
687
|
+
status: FinalPassStatus
|
|
688
|
+
reasonCode: FinalReasonCode | null
|
|
689
|
+
mode: 'batch' | 'streaming' | null
|
|
690
|
+
crowded: boolean | null
|
|
691
|
+
channels: number | null
|
|
692
|
+
mappedChannels: number | null
|
|
693
|
+
wordsRelabeled: number | null
|
|
694
|
+
wordsTotal: number | null
|
|
695
|
+
excludedChunks: number | null
|
|
696
|
+
ms: number | null
|
|
697
|
+
at: string
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
let finalLane: Promise<unknown> = Promise.resolve()
|
|
701
|
+
let lastFinal: PublicFinalOutcome | null = null
|
|
702
|
+
const finalCounts: Record<string, number> = {}
|
|
703
|
+
|
|
704
|
+
export function publicFinalOutcome(outcome: FinalPassOutcome, at: string): PublicFinalOutcome {
|
|
705
|
+
return {
|
|
706
|
+
status: outcome.status,
|
|
707
|
+
reasonCode: finalReasonCode(outcome.reason),
|
|
708
|
+
mode: outcome.mode ?? null,
|
|
709
|
+
crowded: typeof outcome.crowded === 'boolean' ? outcome.crowded : null,
|
|
710
|
+
channels: typeof outcome.channels === 'number' ? outcome.channels : null,
|
|
711
|
+
mappedChannels: outcome.mapped ? Object.keys(outcome.mapped).length : null,
|
|
712
|
+
wordsRelabeled: outcome.words?.relabeled ?? null,
|
|
713
|
+
wordsTotal: outcome.words?.total ?? null,
|
|
714
|
+
excludedChunks: outcome.excluded ? outcome.excluded.missing + outcome.excluded.hashMismatch + outcome.excluded.unreadable : null,
|
|
715
|
+
ms: typeof outcome.ms === 'number' ? outcome.ms : null,
|
|
716
|
+
at,
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
export function finalPassSnapshot(): { last: PublicFinalOutcome | null; counts: Record<string, number> } {
|
|
721
|
+
return { last: lastFinal ? { ...lastFinal } : null, counts: { ...finalCounts } }
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
/** One meeting's final pass, serialised with every other final pass. Never throws. */
|
|
725
|
+
export function runNemotronFinalPass(options: FinalPassOptions): Promise<FinalPassOutcome> {
|
|
726
|
+
const job = finalLane.then(() => finalPassNow(options)).catch((error): FinalPassOutcome => ({
|
|
727
|
+
status: 'failed',
|
|
728
|
+
reason: 'error',
|
|
729
|
+
detail: error instanceof Error ? error.message.slice(0, 300) : String(error).slice(0, 300),
|
|
730
|
+
}))
|
|
731
|
+
finalLane = job.then(() => undefined, () => undefined)
|
|
732
|
+
return job.then(outcome => {
|
|
733
|
+
const published = publicFinalOutcome(outcome, new Date((options.now ?? Date.now)()).toISOString())
|
|
734
|
+
const key = outcome.status === 'applied' ? 'applied' : `${outcome.status}:${published.reasonCode ?? 'error'}`
|
|
735
|
+
finalCounts[key] = (finalCounts[key] ?? 0) + 1
|
|
736
|
+
lastFinal = published
|
|
737
|
+
const log = options.log ?? console.log
|
|
738
|
+
if (outcome.status === 'applied') {
|
|
739
|
+
log(`[diarizer] final pass applied (${outcome.mode}${outcome.crowded ? `, crowded: ${outcome.identities} identities, per-chunk turns only` : ''}) in ${outcome.ms} ms: ${outcome.channels} channels, mapped ${JSON.stringify(outcome.mapped)}, ${outcome.words?.relabeled}/${outcome.words?.total} words relabelled, speakers ${outcome.speakersBefore} -> ${outcome.speakersAfter}${excludedNote(outcome)}`)
|
|
740
|
+
} else if (outcome.reason === 'sidecar_restore_failed') {
|
|
741
|
+
log(`[diarizer] final pass FAILED and could not restore the sidecar: ${outcome.detail}`)
|
|
742
|
+
} else {
|
|
743
|
+
log(`[diarizer] final pass ${outcome.status}: ${outcome.reason}${outcome.detail ? ` (${outcome.detail})` : ''}${excludedNote(outcome)}; the saved transcript is unchanged`)
|
|
744
|
+
}
|
|
745
|
+
return outcome
|
|
746
|
+
})
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
function excludedNote(outcome: FinalPassOutcome): string {
|
|
750
|
+
const e = outcome.excluded
|
|
751
|
+
if (!e || e.missing + e.hashMismatch + e.unreadable === 0) return ''
|
|
752
|
+
return `; chunks left off the timeline: ${e.missing} missing, ${e.hashMismatch} hash mismatch, ${e.unreadable} unreadable`
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
/** A pass the caller decided not to run (the save path's metadata guard): counted and logged like any other. */
|
|
756
|
+
export function recordFinalPassSkip(reason: FinalReasonCode, detail: string, options: { log?: (line: string) => void; now?: () => number } = {}): void {
|
|
757
|
+
const outcome: FinalPassOutcome = { status: 'skipped', reason, detail }
|
|
758
|
+
const published = publicFinalOutcome(outcome, new Date((options.now ?? Date.now)()).toISOString())
|
|
759
|
+
const key = `skipped:${published.reasonCode ?? 'error'}`
|
|
760
|
+
finalCounts[key] = (finalCounts[key] ?? 0) + 1
|
|
761
|
+
lastFinal = published
|
|
762
|
+
;(options.log ?? console.log)(`[diarizer] final pass skipped: ${reason} (${detail}); the saved transcript is unchanged`)
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
function distinctSpeakers(transcript: string): number {
|
|
766
|
+
const names = new Set<string>()
|
|
767
|
+
for (const line of transcript.split('\n')) {
|
|
768
|
+
const match = /^\[([^\]\n]+)\]:/.exec(line)
|
|
769
|
+
if (match) names.add(match[1])
|
|
770
|
+
}
|
|
771
|
+
return names.size
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
async function finalPassNow(options: FinalPassOptions): Promise<FinalPassOutcome> {
|
|
775
|
+
const started = (options.now ?? Date.now)()
|
|
776
|
+
const env = options.env ?? process.env
|
|
777
|
+
const home = options.home ?? homedir()
|
|
778
|
+
const choice = activeDiarizer(env, home)
|
|
779
|
+
if (choice.requested === 'embedding') return { status: 'skipped', reason: 'embedding_requested' }
|
|
780
|
+
if (choice.fallback) return { status: 'skipped', reason: choice.fallback }
|
|
781
|
+
if (!existsSync(options.sidecarPath) || !existsSync(options.meetingPath)) return { status: 'skipped', reason: 'meeting_missing' }
|
|
782
|
+
if (!existsSync(options.audioDir)) return { status: 'skipped', reason: 'no_audio' }
|
|
783
|
+
|
|
784
|
+
const sidecarText = readFileSync(options.sidecarPath, 'utf8')
|
|
785
|
+
const sidecar = JSON.parse(sidecarText) as Record<string, any>
|
|
786
|
+
const owner = options.owner !== undefined ? options.owner : ownerLabelOrNull()
|
|
787
|
+
if (Number(sidecar.correctionRevision ?? 0) > 0) return { status: 'skipped', reason: 'human_corrected' }
|
|
788
|
+
const markdown = readFileSync(options.meetingPath, 'utf8')
|
|
789
|
+
const transcript = readTranscriptSection(markdown)
|
|
790
|
+
if (transcript === null) return { status: 'skipped', reason: 'no_transcript' }
|
|
791
|
+
if (sidecar.diarization?.status === 'applied' && sidecar.diarization?.outputSha256 === sha(transcript)) {
|
|
792
|
+
return { status: 'skipped', reason: 'already_applied' }
|
|
793
|
+
}
|
|
794
|
+
const entries = (Array.isArray(sidecar.chunkEntries) ? sidecar.chunkEntries : []) as SidecarEntry[]
|
|
795
|
+
if (!entries.length) return { status: 'skipped', reason: 'no_chunks' }
|
|
796
|
+
|
|
797
|
+
const timeline = await buildMeetingTimeline(entries, options.audioDir)
|
|
798
|
+
const excluded = timeline.excluded
|
|
799
|
+
if (timeline.tooLong) {
|
|
800
|
+
return { status: 'skipped', reason: 'too_long', excluded, detail: `timeline ${Math.round(timeline.durationSec)} s for ${Math.round(timeline.audioSec)} s of audio` }
|
|
801
|
+
}
|
|
802
|
+
if (!timeline.windows.length) return { status: 'skipped', reason: 'no_audio', excluded }
|
|
803
|
+
|
|
804
|
+
const cli = nemotronCliPath(env, home)
|
|
805
|
+
const modelsDir = prepareNemotronModelsDir(env, home)
|
|
806
|
+
if (!cli || !modelsDir) return { status: 'skipped', reason: !cli ? 'cli_missing' : 'models_missing' }
|
|
807
|
+
|
|
808
|
+
// Tracked, so a server exit mid-run kills the CLI and deletes this directory
|
|
809
|
+
// (it holds the whole meeting's audio); a sweep at the next start catches a crash.
|
|
810
|
+
const work = mkdtempSync(join(options.tmpRoot ?? tmpdir(), `${NEMOTRON_TMP_PREFIX}final-`))
|
|
811
|
+
trackNemotronDir(work)
|
|
812
|
+
let run: NemotronPreds
|
|
813
|
+
try {
|
|
814
|
+
try { chmodSync(work, 0o700) } catch { /* mkdtemp is already 0700 */ }
|
|
815
|
+
const wavPath = join(work, 'meeting.wav')
|
|
816
|
+
await writeFile(wavPath, timeline.wav, { mode: 0o600 })
|
|
817
|
+
try {
|
|
818
|
+
run = await (options.run ?? runNemotronCli)({
|
|
819
|
+
cli,
|
|
820
|
+
modelsDir,
|
|
821
|
+
wavPath,
|
|
822
|
+
timeoutMs: finalPassTimeoutMs(timeline.durationSec),
|
|
823
|
+
computeUnits: 'ane',
|
|
824
|
+
niceness: 10,
|
|
825
|
+
tmpRoot: options.tmpRoot,
|
|
826
|
+
})
|
|
827
|
+
} catch (error) {
|
|
828
|
+
return {
|
|
829
|
+
status: 'failed',
|
|
830
|
+
reason: error instanceof NemotronRunError ? `nemotron_${error.reason}` : 'nemotron_error',
|
|
831
|
+
detail: (error instanceof Error ? error.message : String(error)).slice(0, 300),
|
|
832
|
+
excluded,
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
} finally {
|
|
836
|
+
untrackNemotronDir(work)
|
|
837
|
+
try { rmSync(work, { recursive: true, force: true }) } catch { /* best effort */ }
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
const activity: Activity = run
|
|
841
|
+
// A channel is in use once it holds a track's worth of speech (QA 6.61.0 N2:
|
|
842
|
+
// one 10 ms blip on a third channel made a two-person call "crowded").
|
|
843
|
+
const channelsUsed = new Set(tracksIn(activity, 0, activity.frames).map(track => track.channel))
|
|
844
|
+
// The canary's rule for a meeting-wide map (OUTCOME_2026-10-01, retro plan 4):
|
|
845
|
+
// four or fewer identities and fewer than eight channels. A crowded room
|
|
846
|
+
// (Wed 07:31 filled all eight with five names) gets no channel map; its words
|
|
847
|
+
// keep their per-chunk names, which are the live Nemotron turns where those exist.
|
|
848
|
+
const identities = crowdIdentities(timeline.windows)
|
|
849
|
+
const crowded = isCrowded(identities, channelsUsed.size)
|
|
850
|
+
const mapping = crowded
|
|
851
|
+
? { ...mapChannelsToNames(timeline.windows, activity), map: new Map<number, string>() }
|
|
852
|
+
: mapChannelsToNames(timeline.windows, activity)
|
|
853
|
+
|
|
854
|
+
let mode: 'batch' | 'streaming'
|
|
855
|
+
let next: string
|
|
856
|
+
let stats: RelabelStats
|
|
857
|
+
const nextSidecar: Record<string, any> = JSON.parse(sidecarText)
|
|
858
|
+
if (sidecar.batchApplied === true) {
|
|
859
|
+
mode = 'batch'
|
|
860
|
+
const segments = (Array.isArray(sidecar.batchSegments) ? sidecar.batchSegments : []) as SidecarBatchSegment[]
|
|
861
|
+
if (!segments.length) return { status: 'skipped', reason: 'no_speaker_words' }
|
|
862
|
+
const relabeled = relabelBatchSegments(segments, timeline, activity, mapping.map, owner)
|
|
863
|
+
stats = relabeled.stats
|
|
864
|
+
// finalizeBatch wrote labelled turns only when every segment had speaker words.
|
|
865
|
+
const selected = selectBatchTranscriptForPersistence(
|
|
866
|
+
String(sidecar.batchTranscript ?? ''),
|
|
867
|
+
segments.map(segment => ({ text: segment.text, speakerWords: segment.speakerWords })),
|
|
868
|
+
)
|
|
869
|
+
if (selected.source !== 'speaker-words') return { status: 'skipped', reason: 'no_speaker_words' }
|
|
870
|
+
const sequence = relabeled.segments.flatMap(segment => segment.speakerWords
|
|
871
|
+
.map(word => ({ word: String(word.word ?? '').trim(), speaker: word.speaker || 'Ext' }))
|
|
872
|
+
.filter(word => word.word))
|
|
873
|
+
next = (options.relabelWords ?? relabelTranscriptWords)(transcript, sequence)
|
|
874
|
+
nextSidecar.batchSegments = relabeled.segments
|
|
875
|
+
// finalizeBatch records the exact canonical text here; keep that true.
|
|
876
|
+
if (typeof sidecar.batchTranscript === 'string') nextSidecar.batchTranscript = next
|
|
877
|
+
} else {
|
|
878
|
+
mode = 'streaming'
|
|
879
|
+
const relabeled = relabelStreamingChunks(timeline, activity, mapping.map, owner)
|
|
880
|
+
stats = relabeled.stats
|
|
881
|
+
const textEntries = entries.filter(entry => typeof entry.chunk?.text === 'string' && entry.chunk.text.trim())
|
|
882
|
+
next = relabelStreamingTranscript(transcript, textEntries.map(entry => ({ chunkIndex: entry.chunkIndex, text: entry.chunk.text })), relabeled.finals)
|
|
883
|
+
const chunks = Array.isArray(nextSidecar.chunks) ? nextSidecar.chunks as SidecarChunk[] : []
|
|
884
|
+
const nextEntries = (Array.isArray(nextSidecar.chunkEntries) ? nextSidecar.chunkEntries : []) as SidecarEntry[]
|
|
885
|
+
const textNext = nextEntries.filter(entry => typeof entry.chunk?.text === 'string' && entry.chunk.text.trim())
|
|
886
|
+
const aligned = textNext.length === chunks.length && textNext.every((entry, i) => entry.chunk.text === chunks[i]?.text)
|
|
887
|
+
if (!aligned) return { status: 'skipped', reason: 'chunk_map_ambiguous' }
|
|
888
|
+
textNext.forEach((entry, i) => {
|
|
889
|
+
const final = relabeled.finals.get(entry.chunkIndex)
|
|
890
|
+
if (!final) return
|
|
891
|
+
for (const chunk of [entry.chunk, chunks[i]]) {
|
|
892
|
+
const original = typeof chunk.originalSpeaker === 'string' ? chunk.originalSpeaker : chunk.speaker
|
|
893
|
+
chunk.finalSegments = final.segments
|
|
894
|
+
chunk.speaker = final.speaker
|
|
895
|
+
if (final.speaker !== original) chunk.originalSpeaker = original
|
|
896
|
+
else delete chunk.originalSpeaker
|
|
897
|
+
}
|
|
898
|
+
})
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
// Both relabelers keep every word by construction (the batch one matches the
|
|
902
|
+
// saved words, the streaming one splits a line only into its own chunk text).
|
|
903
|
+
// This check is the last line of defence should either ever change.
|
|
904
|
+
if (!sameWords(transcript, next)) return { status: 'skipped', reason: 'word_mismatch', mode, excluded }
|
|
905
|
+
// Both relabellers keep line breaks, so an unchanged transcript means no label
|
|
906
|
+
// changed: nothing is written and no revision moves.
|
|
907
|
+
if (next === transcript) return { status: 'skipped', reason: 'no_change', mode, words: stats, excluded, crowded, identities, channels: channelsUsed.size }
|
|
908
|
+
|
|
909
|
+
const speakers = new Set<string>(Array.isArray(sidecar.speakers) ? sidecar.speakers : [])
|
|
910
|
+
for (const line of next.split('\n')) {
|
|
911
|
+
const match = /^\[([^\]\n]+)\]:/.exec(line)
|
|
912
|
+
if (match && isName(match[1])) speakers.add(match[1])
|
|
913
|
+
}
|
|
914
|
+
const ms = (options.now ?? Date.now)() - started
|
|
915
|
+
const mapped = Object.fromEntries([...mapping.map.entries()].map(([channel, name]) => [String(channel), name]))
|
|
916
|
+
nextSidecar.speakers = [...speakers]
|
|
917
|
+
nextSidecar.lifecycleRevision = Number(sidecar.lifecycleRevision ?? 0) + 1
|
|
918
|
+
nextSidecar.diarization = {
|
|
919
|
+
engine: 'nemotron3',
|
|
920
|
+
variant: NEMOTRON_VARIANT,
|
|
921
|
+
bundle: NEMOTRON_BUNDLE,
|
|
922
|
+
computeUnits: 'ane',
|
|
923
|
+
status: 'applied',
|
|
924
|
+
mode,
|
|
925
|
+
appliedAt: new Date((options.now ?? Date.now)()).toISOString(),
|
|
926
|
+
ms,
|
|
927
|
+
audioSec: Math.round(timeline.durationSec * 10) / 10,
|
|
928
|
+
frames: run.frames,
|
|
929
|
+
channelsUsed: channelsUsed.size,
|
|
930
|
+
channelMap: mapped,
|
|
931
|
+
mapping: {
|
|
932
|
+
crowded,
|
|
933
|
+
identities,
|
|
934
|
+
eligibleChunks: mapping.eligible,
|
|
935
|
+
minSimilarity: FINAL_MIN_SIMILARITY,
|
|
936
|
+
oneSided: FINAL_ONE_SIDED,
|
|
937
|
+
minSupport: FINAL_MIN_SUPPORT,
|
|
938
|
+
purity: FINAL_PURITY,
|
|
939
|
+
evidence: mapping.evidence,
|
|
940
|
+
},
|
|
941
|
+
words: stats,
|
|
942
|
+
excludedChunks: timeline.excluded,
|
|
943
|
+
originalTranscript: transcript,
|
|
944
|
+
outputSha256: sha(next),
|
|
945
|
+
}
|
|
946
|
+
const nextMarkdown = replaceTranscriptSection(markdown, next)
|
|
947
|
+
if (nextMarkdown === null) return { status: 'skipped', reason: 'no_transcript' }
|
|
948
|
+
|
|
949
|
+
// The CLI ran for tens of seconds. A human relabel or deattribution, or a
|
|
950
|
+
// summary, may have landed meanwhile; writing from the copy read before the
|
|
951
|
+
// run would erase it. Re-read both files: any change and this pass is stale.
|
|
952
|
+
// Everything from here to the writes is synchronous, so no request handled by
|
|
953
|
+
// this process can land in between.
|
|
954
|
+
let sidecarNow: string
|
|
955
|
+
let markdownNow: string
|
|
956
|
+
try {
|
|
957
|
+
sidecarNow = readFileSync(options.sidecarPath, 'utf8')
|
|
958
|
+
markdownNow = readFileSync(options.meetingPath, 'utf8')
|
|
959
|
+
} catch (error) {
|
|
960
|
+
return { status: 'skipped', reason: 'changed_during_pass', detail: (error instanceof Error ? error.message : String(error)).slice(0, 300) }
|
|
961
|
+
}
|
|
962
|
+
if (sidecarNow !== sidecarText || markdownNow !== markdown) {
|
|
963
|
+
return { status: 'skipped', reason: 'changed_during_pass', detail: sidecarNow !== sidecarText ? 'sidecar changed' : 'markdown changed' }
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
// Sidecar first, markdown second, as the store does: the markdown is the
|
|
967
|
+
// visible commit. If the markdown write fails, put the sidecar back, and say
|
|
968
|
+
// so loudly if even that fails (the two files then disagree).
|
|
969
|
+
const write = options.write ?? ((path: string, data: string) => durableAtomicWriteFileSync(path, data, { mode: 0o600 }))
|
|
970
|
+
try {
|
|
971
|
+
write(options.sidecarPath, JSON.stringify(nextSidecar, null, 2))
|
|
972
|
+
} catch (error) {
|
|
973
|
+
return { status: 'failed', reason: 'markdown_write', detail: `sidecar write: ${(error instanceof Error ? error.message : String(error)).slice(0, 280)}`, excluded }
|
|
974
|
+
}
|
|
975
|
+
try {
|
|
976
|
+
write(options.meetingPath, nextMarkdown)
|
|
977
|
+
} catch (error) {
|
|
978
|
+
const why = (error instanceof Error ? error.message : String(error)).slice(0, 200)
|
|
979
|
+
try {
|
|
980
|
+
write(options.sidecarPath, sidecarText)
|
|
981
|
+
} catch (restoreError) {
|
|
982
|
+
return {
|
|
983
|
+
status: 'failed',
|
|
984
|
+
reason: 'sidecar_restore_failed',
|
|
985
|
+
detail: `markdown write: ${why}; sidecar restore: ${(restoreError instanceof Error ? restoreError.message : String(restoreError)).slice(0, 200)}; the sidecar holds the relabel and the markdown does not`,
|
|
986
|
+
excluded,
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
return { status: 'failed', reason: 'markdown_write', detail: `${why}; sidecar restored`, excluded }
|
|
990
|
+
}
|
|
991
|
+
return {
|
|
992
|
+
status: 'applied',
|
|
993
|
+
mode,
|
|
994
|
+
crowded,
|
|
995
|
+
identities,
|
|
996
|
+
ms,
|
|
997
|
+
audioSec: Math.round(timeline.durationSec),
|
|
998
|
+
channels: channelsUsed.size,
|
|
999
|
+
mapped,
|
|
1000
|
+
words: stats,
|
|
1001
|
+
excluded,
|
|
1002
|
+
speakersBefore: distinctSpeakers(transcript),
|
|
1003
|
+
speakersAfter: distinctSpeakers(next),
|
|
1004
|
+
}
|
|
1005
|
+
}
|