@gotcos/glasses-server 6.88.0 → 6.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/CHANGELOG.md +67 -0
- package/SECURITY.md +5 -10
- package/docs/tailscale-self-heal.md +159 -0
- package/managed-runtime-contract.json +2 -0
- package/package.json +4 -3
- package/server/index.ts +21 -12
- package/server/lib/api-auth.ts +0 -7
- package/server/lib/mac-voice-labels.ts +50 -0
- package/server/lib/meeting-batch-transcribe.ts +9 -3
- package/server/lib/meeting-relabel-enrolment.ts +12 -1
- package/server/lib/meeting-relabel.ts +8 -1
- package/server/lib/meeting-speaker-review.ts +4 -1
- package/server/lib/meeting-store.ts +3 -1
- package/server/lib/nemotron-final.ts +314 -7
- package/server/lib/nemotron-live.ts +3 -1
- package/server/lib/speaker-embeddings.ts +39 -1
- package/server/lib/tailscale-nudge.ts +540 -0
- package/server/routes/health.ts +3 -15
- package/server/routes/transcribe-stream.ts +13 -2
- package/docs/lan-first.md +0 -158
- package/server/lib/lan-first.ts +0 -236
- package/server/routes/route-proof.ts +0 -83
|
@@ -68,6 +68,7 @@ import {
|
|
|
68
68
|
wavHeader,
|
|
69
69
|
} from './nemotron-segments.js'
|
|
70
70
|
import { captureSpeakerMode, type SpeakerMode } from './meeting-speaker-policy.js'
|
|
71
|
+
import { isMacCaptureSidecar, isVoiceLabel, macVoiceLabelsEnabled, voiceLabel } from './mac-voice-labels.js'
|
|
71
72
|
import { NEMOTRON_CHANNELS } from './nemotron-cli.js'
|
|
72
73
|
import type { WhisperWord } from './whisper-local.js'
|
|
73
74
|
|
|
@@ -633,7 +634,7 @@ export const FINAL_REASON_CODES = [
|
|
|
633
634
|
'no_transcript', 'already_applied', 'no_chunks', 'nemotron_timeout', 'nemotron_cli_failed',
|
|
634
635
|
'nemotron_bad_output', 'nemotron_spawn_failed', 'nemotron_error', 'no_speaker_words', 'chunk_map_ambiguous',
|
|
635
636
|
'word_mismatch', 'no_change', 'changed_during_pass', 'markdown_write', 'sidecar_restore_failed', 'too_long',
|
|
636
|
-
'metadata_not_persisted', 'speaker_policy', 'error',
|
|
637
|
+
'metadata_not_persisted', 'speaker_policy', 'voices_crowded', 'voices_unconfident', 'error',
|
|
637
638
|
] as const
|
|
638
639
|
export type FinalReasonCode = typeof FINAL_REASON_CODES[number]
|
|
639
640
|
const REASON_SET: ReadonlySet<string> = new Set(FINAL_REASON_CODES)
|
|
@@ -659,6 +660,10 @@ export interface FinalPassOutcome {
|
|
|
659
660
|
/** Distinct speakers in the transcript before and after. */
|
|
660
661
|
speakersBefore?: number
|
|
661
662
|
speakersAfter?: number
|
|
663
|
+
/** 6.90.0: a Mac recording labelled with unnamed `Voice N` turns (no profile was read). */
|
|
664
|
+
unnamed?: boolean
|
|
665
|
+
/** How many `Voice N` labels the meeting carries after an unnamed pass. */
|
|
666
|
+
voices?: number
|
|
662
667
|
}
|
|
663
668
|
|
|
664
669
|
export interface FinalPassOptions {
|
|
@@ -738,7 +743,7 @@ export function runNemotronFinalPass(options: FinalPassOptions): Promise<FinalPa
|
|
|
738
743
|
lastFinal = published
|
|
739
744
|
const log = options.log ?? console.log
|
|
740
745
|
if (outcome.status === 'applied') {
|
|
741
|
-
log(`[diarizer] final pass applied (${outcome.mode}${outcome.crowded ? `, crowded: ${outcome.identities} identities, per-chunk turns only` : ''}) in ${outcome.ms} ms: ${outcome.channels} channels, mapped ${JSON.stringify(outcome.mapped)}, ${outcome.words?.relabeled}/${outcome.words?.total} words relabelled, speakers ${outcome.speakersBefore} -> ${outcome.speakersAfter}${excludedNote(outcome)}`)
|
|
746
|
+
log(`[diarizer] final pass applied (${outcome.mode}${outcome.unnamed ? `, unnamed: ${outcome.voices} voices, no profile read` : ''}${outcome.crowded ? `, crowded: ${outcome.identities} identities, per-chunk turns only` : ''}) in ${outcome.ms} ms: ${outcome.channels} channels, mapped ${JSON.stringify(outcome.mapped)}, ${outcome.words?.relabeled}/${outcome.words?.total} words relabelled, speakers ${outcome.speakersBefore} -> ${outcome.speakersAfter}${excludedNote(outcome)}`)
|
|
742
747
|
} else if (outcome.reason === 'sidecar_restore_failed') {
|
|
743
748
|
log(`[diarizer] final pass FAILED and could not restore the sidecar: ${outcome.detail}`)
|
|
744
749
|
} else {
|
|
@@ -798,9 +803,14 @@ async function finalPassNow(options: FinalPassOptions): Promise<FinalPassOutcome
|
|
|
798
803
|
// Final diarization builds meeting-wide channel/name maps. Earlier Standard
|
|
799
804
|
// identities cannot safely authorize names for later strict-policy audio.
|
|
800
805
|
// Preserve the canonical transcript (including earlier names) byte-for-byte.
|
|
806
|
+
const sidecarChunks = (Array.isArray(sidecar.chunks) ? sidecar.chunks : []) as SidecarChunk[]
|
|
807
|
+
let unnamed = false
|
|
801
808
|
if (entries.some(entry => captureSpeakerMode(entry.chunk.speakerPolicy) !== 'standard')
|
|
802
|
-
||
|
|
803
|
-
|
|
809
|
+
|| sidecarChunks.some(chunk => captureSpeakerMode(chunk.speakerPolicy) !== 'standard')) {
|
|
810
|
+
// 6.90.0: a Mac recording may still take UNNAMED voices (see unnamedVoicePass).
|
|
811
|
+
// Every other strict capture keeps the 6.73.6 skip, byte for byte.
|
|
812
|
+
if (!unnamedVoicesEligible(sidecar, entries, sidecarChunks, env)) return { status: 'skipped', reason: 'speaker_policy' }
|
|
813
|
+
unnamed = true
|
|
804
814
|
}
|
|
805
815
|
|
|
806
816
|
const timeline = await buildMeetingTimeline(entries, options.audioDir)
|
|
@@ -850,6 +860,9 @@ async function finalPassNow(options: FinalPassOptions): Promise<FinalPassOutcome
|
|
|
850
860
|
// A channel is in use once it holds a track's worth of speech (QA 6.61.0 N2:
|
|
851
861
|
// one 10 ms blip on a third channel made a two-person call "crowded").
|
|
852
862
|
const channelsUsed = new Set(tracksIn(activity, 0, activity.frames).map(track => track.channel))
|
|
863
|
+
if (unnamed) {
|
|
864
|
+
return unnamedVoicePass({ options, sidecar, sidecarText, markdown, transcript, entries, timeline, activity, run, channelsUsed: channelsUsed.size, started })
|
|
865
|
+
}
|
|
853
866
|
// The canary's rule for a meeting-wide map (OUTCOME_2026-10-01, retro plan 4):
|
|
854
867
|
// four or fewer identities and fewer than eight channels. A crowded room
|
|
855
868
|
// (Wed 07:31 filled all eight with five names) gets no channel map; its words
|
|
@@ -955,6 +968,37 @@ async function finalPassNow(options: FinalPassOptions): Promise<FinalPassOutcome
|
|
|
955
968
|
const nextMarkdown = replaceTranscriptSection(markdown, next)
|
|
956
969
|
if (nextMarkdown === null) return { status: 'skipped', reason: 'no_transcript' }
|
|
957
970
|
|
|
971
|
+
const committed = commitFinalPass(options, sidecarText, markdown, nextSidecar, nextMarkdown, excluded)
|
|
972
|
+
if (committed) return committed
|
|
973
|
+
return {
|
|
974
|
+
status: 'applied',
|
|
975
|
+
mode,
|
|
976
|
+
crowded,
|
|
977
|
+
identities,
|
|
978
|
+
ms,
|
|
979
|
+
audioSec: Math.round(timeline.durationSec),
|
|
980
|
+
channels: channelsUsed.size,
|
|
981
|
+
mapped,
|
|
982
|
+
words: stats,
|
|
983
|
+
excluded,
|
|
984
|
+
speakersBefore: distinctSpeakers(transcript),
|
|
985
|
+
speakersAfter: distinctSpeakers(next),
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
/**
|
|
990
|
+
* Write a computed pass: re-read both files first (a human edit during the CLI
|
|
991
|
+
* run makes the pass stale), then sidecar, then markdown, restoring the sidecar
|
|
992
|
+
* if the markdown write fails. Null means both files were written.
|
|
993
|
+
*/
|
|
994
|
+
function commitFinalPass(
|
|
995
|
+
options: FinalPassOptions,
|
|
996
|
+
sidecarText: string,
|
|
997
|
+
markdown: string,
|
|
998
|
+
nextSidecar: Record<string, any>,
|
|
999
|
+
nextMarkdown: string,
|
|
1000
|
+
excluded: MeetingTimeline['excluded'],
|
|
1001
|
+
): FinalPassOutcome | null {
|
|
958
1002
|
// The CLI ran for tens of seconds. A human relabel or deattribution, or a
|
|
959
1003
|
// summary, may have landed meanwhile; writing from the copy read before the
|
|
960
1004
|
// run would erase it. Re-read both files: any change and this pass is stale.
|
|
@@ -997,14 +1041,277 @@ async function finalPassNow(options: FinalPassOptions): Promise<FinalPassOutcome
|
|
|
997
1041
|
}
|
|
998
1042
|
return { status: 'failed', reason: 'markdown_write', detail: `${why}; sidecar restored`, excluded }
|
|
999
1043
|
}
|
|
1044
|
+
return null
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
// ── 6.90.0: unnamed, meeting-stable voices for Mac recordings ───────────────────
|
|
1048
|
+
//
|
|
1049
|
+
// The named pass above maps a channel to a name from chunks the voiceprint was
|
|
1050
|
+
// sure of. A Mac recording has no such chunks (every one is `Ext` at the 0.88
|
|
1051
|
+
// floor) and must never borrow a name, so it skips that pass (`speaker_policy`).
|
|
1052
|
+
// This pass keeps the channels and drops the names: each channel that holds the
|
|
1053
|
+
// floor in enough chunks becomes `Voice 1`, `Voice 2`, ... in order of first
|
|
1054
|
+
// speech. No profile is read or written anywhere below; `owner` is never passed,
|
|
1055
|
+
// so the wearer guard cannot fire and no name can enter. A crowded room (more
|
|
1056
|
+
// than FINAL_MAX_IDENTITIES voices, or every channel in use) and a room with no
|
|
1057
|
+
// confident channel stay `Ext` exactly as saved.
|
|
1058
|
+
|
|
1059
|
+
/** A channel becomes a voice only if it holds the floor, one-sided, in at least this many chunks. */
|
|
1060
|
+
export const VOICE_MIN_SUPPORT = FINAL_MIN_SUPPORT
|
|
1061
|
+
|
|
1062
|
+
/**
|
|
1063
|
+
* Unnamed mode applies to a Mac recording, with the switch on, when every chunk
|
|
1064
|
+
* that carries words is the Mac strict policy and none already carries a real
|
|
1065
|
+
* name (a 0.88 match is rare, and a pass must never overwrite one with a
|
|
1066
|
+
* placeholder).
|
|
1067
|
+
*/
|
|
1068
|
+
export function unnamedVoicesEligible(
|
|
1069
|
+
sidecar: Record<string, unknown>,
|
|
1070
|
+
entries: SidecarEntry[],
|
|
1071
|
+
chunks: SidecarChunk[],
|
|
1072
|
+
env: NodeJS.ProcessEnv,
|
|
1073
|
+
): boolean {
|
|
1074
|
+
if (!macVoiceLabelsEnabled(env) || !isMacCaptureSidecar(sidecar)) return false
|
|
1075
|
+
const hasText = (chunk: SidecarChunk) => typeof chunk?.text === 'string' && chunk.text.trim().length > 0
|
|
1076
|
+
const spoken = [...entries.map(entry => entry.chunk), ...chunks].filter(hasText)
|
|
1077
|
+
if (!spoken.length) return false
|
|
1078
|
+
// Measured on the six 10/16-10/22 Mac recordings: every text-less entry (an
|
|
1079
|
+
// empty completion) is stored with no policy marker, so only the chunks that
|
|
1080
|
+
// carry words must prove the Mac pin. A text-less entry may be unmarked or
|
|
1081
|
+
// strict; anything else (anonymous, a malformed marker) still refuses.
|
|
1082
|
+
const silentOk = entries.filter(entry => !hasText(entry.chunk))
|
|
1083
|
+
.every(entry => entry.chunk?.speakerPolicy === undefined || entry.chunk.speakerPolicy === 'high-confidence')
|
|
1084
|
+
return silentOk && spoken.every(chunk => chunk.speakerPolicy === 'high-confidence' && !isName(chunk.speaker))
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
export interface VoiceChannels {
|
|
1088
|
+
/** Accepted channels, in order of first supporting chunk. */
|
|
1089
|
+
channels: number[]
|
|
1090
|
+
evidence: Array<{ channel: number; support: number; firstSec: number; accepted: boolean }>
|
|
1091
|
+
eligible: number
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
/** Channels that hold the floor, one-sided, in VOICE_MIN_SUPPORT or more chunks. Names are never read. */
|
|
1095
|
+
export function mapChannelsToVoices(windows: TimelineWindow[], a: Activity, options: { oneSided?: number; minSupport?: number } = {}): VoiceChannels {
|
|
1096
|
+
const oneSided = options.oneSided ?? FINAL_ONE_SIDED
|
|
1097
|
+
const minSupport = options.minSupport ?? VOICE_MIN_SUPPORT
|
|
1098
|
+
const support = new Map<number, { support: number; firstSec: number }>()
|
|
1099
|
+
let eligible = 0
|
|
1100
|
+
for (const window of windows) {
|
|
1101
|
+
const [f0, f1] = frameRange(a, window.start, window.end)
|
|
1102
|
+
const counts = new Array<number>(NEMOTRON_CHANNELS).fill(0)
|
|
1103
|
+
let speech = 0
|
|
1104
|
+
for (let f = f0; f < f1; f++) {
|
|
1105
|
+
let any = false
|
|
1106
|
+
for (let c = 0; c < NEMOTRON_CHANNELS; c++) if (isActive(a, f, c)) { counts[c]++; any = true }
|
|
1107
|
+
if (any) speech++
|
|
1108
|
+
}
|
|
1109
|
+
if (speech === 0) continue
|
|
1110
|
+
let dom = 0
|
|
1111
|
+
for (let c = 1; c < NEMOTRON_CHANNELS; c++) if (counts[c] > counts[dom]) dom = c
|
|
1112
|
+
if (counts[dom] / speech < oneSided) continue
|
|
1113
|
+
eligible++
|
|
1114
|
+
const row = support.get(dom) ?? { support: 0, firstSec: window.start }
|
|
1115
|
+
row.support++
|
|
1116
|
+
row.firstSec = Math.min(row.firstSec, window.start)
|
|
1117
|
+
support.set(dom, row)
|
|
1118
|
+
}
|
|
1119
|
+
const ordered = [...support.entries()].sort((x, y) => x[1].firstSec - y[1].firstSec || x[0] - y[0])
|
|
1120
|
+
const evidence = ordered.map(([channel, row]) => ({ channel, support: row.support, firstSec: Math.round(row.firstSec * 100) / 100, accepted: row.support >= minSupport }))
|
|
1121
|
+
return { channels: evidence.filter(row => row.accepted).map(row => row.channel), evidence, eligible }
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
/** `Voice 1` for the first channel, `Voice 2` for the next, and so on. */
|
|
1125
|
+
export function numberVoices(channels: number[]): Map<number, string> {
|
|
1126
|
+
return new Map(channels.map((channel, i) => [channel, voiceLabel(i + 1)]))
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
function voicesIn(transcript: string): Set<string> {
|
|
1130
|
+
const out = new Set<string>()
|
|
1131
|
+
for (const line of transcript.split('\n')) {
|
|
1132
|
+
const match = /^\[([^\]\n]+)\]:/.exec(line)
|
|
1133
|
+
if (match && isVoiceLabel(match[1])) out.add(match[1])
|
|
1134
|
+
}
|
|
1135
|
+
return out
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
/** The label with the most words, ties to the earlier label. */
|
|
1139
|
+
function mostWords(counts: Map<string, number>): string | null {
|
|
1140
|
+
let best: string | null = null
|
|
1141
|
+
let n = -1
|
|
1142
|
+
for (const [label, count] of counts) if (count > n) { best = label; n = count }
|
|
1143
|
+
return best
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
interface UnnamedInput {
|
|
1147
|
+
options: FinalPassOptions
|
|
1148
|
+
sidecar: Record<string, any>
|
|
1149
|
+
sidecarText: string
|
|
1150
|
+
markdown: string
|
|
1151
|
+
transcript: string
|
|
1152
|
+
entries: SidecarEntry[]
|
|
1153
|
+
timeline: MeetingTimeline
|
|
1154
|
+
activity: Activity
|
|
1155
|
+
run: NemotronPreds
|
|
1156
|
+
channelsUsed: number
|
|
1157
|
+
started: number
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
interface UnnamedLabelling {
|
|
1161
|
+
next: string
|
|
1162
|
+
stats: RelabelStats
|
|
1163
|
+
/** chunkIndex -> the chunk's label (the one with the most words). */
|
|
1164
|
+
chunkSpeakers: Map<number, string>
|
|
1165
|
+
batchSegments?: SidecarBatchSegment[]
|
|
1166
|
+
finals?: Map<number, { speaker: string; segments: ChunkSpeakerSegment[] }>
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
function labelUnnamed(input: UnnamedInput, mode: 'batch' | 'streaming', map: Map<number, string>): UnnamedLabelling | FinalPassOutcome {
|
|
1170
|
+
const { sidecar, timeline, activity, transcript, entries } = input
|
|
1171
|
+
if (mode === 'batch') {
|
|
1172
|
+
const segments = (Array.isArray(sidecar.batchSegments) ? sidecar.batchSegments : []) as SidecarBatchSegment[]
|
|
1173
|
+
if (!segments.length) return { status: 'skipped', reason: 'no_speaker_words' }
|
|
1174
|
+
const selected = selectBatchTranscriptForPersistence(
|
|
1175
|
+
String(sidecar.batchTranscript ?? ''),
|
|
1176
|
+
segments.map(segment => ({ text: segment.text, speakerWords: segment.speakerWords })),
|
|
1177
|
+
)
|
|
1178
|
+
if (selected.source !== 'speaker-words') return { status: 'skipped', reason: 'no_speaker_words' }
|
|
1179
|
+
const relabeled = relabelBatchSegments(segments, timeline, activity, map, null)
|
|
1180
|
+
const sequence = relabeled.segments.flatMap(segment => segment.speakerWords
|
|
1181
|
+
.map(word => ({ word: String(word.word ?? '').trim(), speaker: word.speaker || 'Ext' }))
|
|
1182
|
+
.filter(word => word.word))
|
|
1183
|
+
const next = (input.options.relabelWords ?? relabelTranscriptWords)(transcript, sequence)
|
|
1184
|
+
const perChunk = new Map<number, Map<string, number>>()
|
|
1185
|
+
for (const segment of relabeled.segments) {
|
|
1186
|
+
for (const word of segment.speakerWords) {
|
|
1187
|
+
const at = batchWordToTimeline(segment, Number(word.start), timeline)
|
|
1188
|
+
if (!at) continue
|
|
1189
|
+
const counts = perChunk.get(at.window.chunkIndex) ?? new Map<string, number>()
|
|
1190
|
+
counts.set(word.speaker || 'Ext', (counts.get(word.speaker || 'Ext') ?? 0) + 1)
|
|
1191
|
+
perChunk.set(at.window.chunkIndex, counts)
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
const chunkSpeakers = new Map<number, string>()
|
|
1195
|
+
for (const [index, counts] of perChunk) { const top = mostWords(counts); if (top) chunkSpeakers.set(index, top) }
|
|
1196
|
+
return { next, stats: relabeled.stats, chunkSpeakers, batchSegments: relabeled.segments }
|
|
1197
|
+
}
|
|
1198
|
+
const relabeled = relabelStreamingChunks(timeline, activity, map, null)
|
|
1199
|
+
const textEntries = entries.filter(entry => typeof entry.chunk?.text === 'string' && entry.chunk.text.trim())
|
|
1200
|
+
const next = relabelStreamingTranscript(transcript, textEntries.map(entry => ({ chunkIndex: entry.chunkIndex, text: entry.chunk.text })), relabeled.finals)
|
|
1201
|
+
const chunkSpeakers = new Map<number, string>([...relabeled.finals.entries()].map(([index, final]) => [index, final.speaker]))
|
|
1202
|
+
return { next, stats: relabeled.stats, chunkSpeakers, finals: relabeled.finals }
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
/**
|
|
1206
|
+
* The unnamed pass. Labels are kept consistent between the transcript and the
|
|
1207
|
+
* chunks the relabel route edits: a voice that would appear in turns but lead no
|
|
1208
|
+
* chunk (or lead a chunk but have no turn, or neither) could never be renamed in
|
|
1209
|
+
* one action, so its channel is dropped back to `Ext` and the rest are
|
|
1210
|
+
* renumbered. A pass with no voice left is `voices_unconfident`.
|
|
1211
|
+
*/
|
|
1212
|
+
async function unnamedVoicePass(input: UnnamedInput): Promise<FinalPassOutcome> {
|
|
1213
|
+
const { options, sidecar, sidecarText, markdown, transcript, entries, timeline, activity, run, channelsUsed, started } = input
|
|
1214
|
+
const excluded = timeline.excluded
|
|
1215
|
+
const mode: 'batch' | 'streaming' = sidecar.batchApplied === true ? 'batch' : 'streaming'
|
|
1216
|
+
const found = mapChannelsToVoices(timeline.windows, activity)
|
|
1217
|
+
const crowded = isCrowded(found.channels.length, channelsUsed)
|
|
1218
|
+
if (crowded) {
|
|
1219
|
+
return { status: 'skipped', reason: 'voices_crowded', unnamed: true, mode, crowded: true, identities: found.channels.length, channels: channelsUsed, excluded }
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
const nextSidecar: Record<string, any> = JSON.parse(sidecarText)
|
|
1223
|
+
const chunks = Array.isArray(nextSidecar.chunks) ? nextSidecar.chunks as SidecarChunk[] : []
|
|
1224
|
+
const nextEntries = (Array.isArray(nextSidecar.chunkEntries) ? nextSidecar.chunkEntries : []) as SidecarEntry[]
|
|
1225
|
+
const textNext = nextEntries.filter(entry => typeof entry.chunk?.text === 'string' && entry.chunk.text.trim())
|
|
1226
|
+
const aligned = textNext.length === chunks.length && textNext.every((entry, i) => entry.chunk.text === chunks[i]?.text)
|
|
1227
|
+
if (!aligned) return { status: 'skipped', reason: 'chunk_map_ambiguous', unnamed: true, mode, excluded }
|
|
1228
|
+
|
|
1229
|
+
let channels = [...found.channels]
|
|
1230
|
+
let map = numberVoices(channels)
|
|
1231
|
+
let labelled: UnnamedLabelling | null = null
|
|
1232
|
+
for (let round = 0; round <= NEMOTRON_CHANNELS && channels.length; round++) {
|
|
1233
|
+
map = numberVoices(channels)
|
|
1234
|
+
const attempt = labelUnnamed(input, mode, map)
|
|
1235
|
+
if ('status' in attempt) return { ...attempt, unnamed: true, mode, excluded }
|
|
1236
|
+
const inTurns = voicesIn(attempt.next)
|
|
1237
|
+
const leading = new Set(textNext.map(entry => attempt.chunkSpeakers.get(entry.chunkIndex)).filter((label): label is string => isVoiceLabel(label)))
|
|
1238
|
+
// A voice must both speak a turn and lead a chunk. One that does neither (a
|
|
1239
|
+
// channel with no words) is dropped too, so numbering never skips a voice.
|
|
1240
|
+
const kept = channels.filter(channel => inTurns.has(map.get(channel)!) && leading.has(map.get(channel)!))
|
|
1241
|
+
if (kept.length === channels.length) { labelled = attempt; break }
|
|
1242
|
+
channels = kept
|
|
1243
|
+
}
|
|
1244
|
+
// No confident channel at all, or none left once orphans are dropped: the room stays Ext.
|
|
1245
|
+
if (!labelled || !channels.length) return { status: 'skipped', reason: 'voices_unconfident', unnamed: true, mode, channels: channelsUsed, excluded }
|
|
1246
|
+
const { next, stats } = labelled
|
|
1247
|
+
|
|
1248
|
+
if (!sameWords(transcript, next)) return { status: 'skipped', reason: 'word_mismatch', unnamed: true, mode, excluded }
|
|
1249
|
+
if (next === transcript) return { status: 'skipped', reason: 'no_change', unnamed: true, mode, words: stats, excluded, channels: channelsUsed }
|
|
1250
|
+
|
|
1251
|
+
textNext.forEach((entry, i) => {
|
|
1252
|
+
const label = labelled!.chunkSpeakers.get(entry.chunkIndex)
|
|
1253
|
+
if (!label) return
|
|
1254
|
+
const final = labelled!.finals?.get(entry.chunkIndex)
|
|
1255
|
+
for (const chunk of [entry.chunk, chunks[i]]) {
|
|
1256
|
+
const original = typeof chunk.originalSpeaker === 'string' ? chunk.originalSpeaker : chunk.speaker
|
|
1257
|
+
if (final) chunk.finalSegments = final.segments
|
|
1258
|
+
chunk.speaker = label
|
|
1259
|
+
if (label !== original) chunk.originalSpeaker = original
|
|
1260
|
+
else delete chunk.originalSpeaker
|
|
1261
|
+
}
|
|
1262
|
+
})
|
|
1263
|
+
if (labelled.batchSegments) {
|
|
1264
|
+
nextSidecar.batchSegments = labelled.batchSegments
|
|
1265
|
+
if (typeof sidecar.batchTranscript === 'string') nextSidecar.batchTranscript = next
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
const voices = voicesIn(next).size
|
|
1269
|
+
const ms = (options.now ?? Date.now)() - started
|
|
1270
|
+
const mapped = Object.fromEntries([...map.entries()].map(([channel, label]) => [String(channel), label]))
|
|
1271
|
+
// `speakers` is left exactly as saved: a `Voice N` placeholder is never a speaker name.
|
|
1272
|
+
nextSidecar.lifecycleRevision = Number(sidecar.lifecycleRevision ?? 0) + 1
|
|
1273
|
+
nextSidecar.diarization = {
|
|
1274
|
+
engine: 'nemotron3',
|
|
1275
|
+
variant: NEMOTRON_VARIANT,
|
|
1276
|
+
bundle: NEMOTRON_BUNDLE,
|
|
1277
|
+
computeUnits: 'ane',
|
|
1278
|
+
status: 'applied',
|
|
1279
|
+
mode,
|
|
1280
|
+
labels: 'unnamed',
|
|
1281
|
+
appliedAt: new Date((options.now ?? Date.now)()).toISOString(),
|
|
1282
|
+
ms,
|
|
1283
|
+
audioSec: Math.round(timeline.durationSec * 10) / 10,
|
|
1284
|
+
frames: run.frames,
|
|
1285
|
+
channelsUsed,
|
|
1286
|
+
channelMap: mapped,
|
|
1287
|
+
mapping: {
|
|
1288
|
+
unnamed: true,
|
|
1289
|
+
crowded: false,
|
|
1290
|
+
voices,
|
|
1291
|
+
eligibleChunks: found.eligible,
|
|
1292
|
+
oneSided: FINAL_ONE_SIDED,
|
|
1293
|
+
minSupport: VOICE_MIN_SUPPORT,
|
|
1294
|
+
evidence: found.evidence,
|
|
1295
|
+
},
|
|
1296
|
+
words: stats,
|
|
1297
|
+
excludedChunks: timeline.excluded,
|
|
1298
|
+
originalTranscript: transcript,
|
|
1299
|
+
outputSha256: sha(next),
|
|
1300
|
+
}
|
|
1301
|
+
const nextMarkdown = replaceTranscriptSection(markdown, next)
|
|
1302
|
+
if (nextMarkdown === null) return { status: 'skipped', reason: 'no_transcript' }
|
|
1303
|
+
const committed = commitFinalPass(options, sidecarText, markdown, nextSidecar, nextMarkdown, excluded)
|
|
1304
|
+
if (committed) return { ...committed, unnamed: true }
|
|
1000
1305
|
return {
|
|
1001
1306
|
status: 'applied',
|
|
1307
|
+
unnamed: true,
|
|
1308
|
+
voices,
|
|
1002
1309
|
mode,
|
|
1003
|
-
crowded,
|
|
1004
|
-
identities,
|
|
1310
|
+
crowded: false,
|
|
1311
|
+
identities: 0,
|
|
1005
1312
|
ms,
|
|
1006
1313
|
audioSec: Math.round(timeline.durationSec),
|
|
1007
|
-
channels: channelsUsed
|
|
1314
|
+
channels: channelsUsed,
|
|
1008
1315
|
mapped,
|
|
1009
1316
|
words: stats,
|
|
1010
1317
|
excluded,
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
//
|
|
16
16
|
// No LLM anywhere. Transcription must never call `claude -p`.
|
|
17
17
|
|
|
18
|
+
import { isVoiceLabel } from './mac-voice-labels.js'
|
|
18
19
|
import { createHash } from 'node:crypto'
|
|
19
20
|
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
20
21
|
import { homedir, tmpdir } from 'node:os'
|
|
@@ -104,7 +105,8 @@ const round = (value: number, places = 2): number => Math.round(value * 10 ** pl
|
|
|
104
105
|
|
|
105
106
|
function isRealLabel(label: string): boolean {
|
|
106
107
|
const value = label.trim()
|
|
107
|
-
|
|
108
|
+
// 6.90.0: a Mac `Voice N` placeholder is not a name either.
|
|
109
|
+
return value.length > 0 && value !== 'Unknown' && value !== 'Ext' && !isVoiceLabel(value)
|
|
108
110
|
}
|
|
109
111
|
|
|
110
112
|
/**
|
|
@@ -376,7 +376,10 @@ export function identifySpeaker(
|
|
|
376
376
|
expectedSpeakers?: string[],
|
|
377
377
|
/** 6.61.0: `event` tags the calibration rows, e.g. 'nemotron-track' for one
|
|
378
378
|
* Nemotron track's audio, so they are never read as whole-chunk checks. */
|
|
379
|
-
|
|
379
|
+
/** 6.90.0 `shadowEvent`: when the strict floor finds nobody, also log the best
|
|
380
|
+
* match at the 0.55 search floor under this event. Calibration only: the
|
|
381
|
+
* returned label is unchanged (`Ext`), and nothing is enrolled. */
|
|
382
|
+
options: { event?: string; minSimilarity?: number; prioritizeOwner?: boolean; shadowEvent?: string } = {},
|
|
380
383
|
): { speaker: string; similarity: number; embedding?: Float32Array } | null {
|
|
381
384
|
if (!extractor || !manager) return null
|
|
382
385
|
const minimum = Number.isFinite(options.minSimilarity) ? Math.max(0, Math.min(1, options.minSimilarity!)) : 0
|
|
@@ -422,6 +425,10 @@ export function identifySpeaker(
|
|
|
422
425
|
// unidentified voice's vector is the most valuable thing to retain, because
|
|
423
426
|
// naming it later is exactly the correction that has no other evidence.
|
|
424
427
|
logCalibration('Ext', 0, false, options.event)
|
|
428
|
+
if (options.shadowEvent) {
|
|
429
|
+
const shadow = shadowBestMatch(manager, embedding)
|
|
430
|
+
logCalibration(shadow.speaker, shadow.similarity, false, options.shadowEvent)
|
|
431
|
+
}
|
|
425
432
|
return { speaker: 'Ext', similarity: 0, embedding }
|
|
426
433
|
} catch (err: unknown) {
|
|
427
434
|
console.error('[speaker] Identification error:', errMsg(err))
|
|
@@ -829,6 +836,37 @@ function computeCosineSimilarity(_embedding: Float32Array, _name: string): numbe
|
|
|
829
836
|
return (lo + hi) / 2
|
|
830
837
|
}
|
|
831
838
|
|
|
839
|
+
/** The subset of the sherpa manager the shadow score needs. */
|
|
840
|
+
export interface ShadowSearchManager {
|
|
841
|
+
search(query: { v: Float32Array; threshold: number }): string
|
|
842
|
+
verify(query: { name: string; v: Float32Array; threshold: number }): boolean
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
/** The floor shadow scoring reports against: the ordinary search floor. */
|
|
846
|
+
export const SHADOW_FLOOR = SEARCH_THRESHOLD
|
|
847
|
+
|
|
848
|
+
/**
|
|
849
|
+
* 6.90.0 shadow score for Mac chunks: who the voice would be at the ordinary
|
|
850
|
+
* 0.55 search floor, and how close. Calibration data only. The caller never
|
|
851
|
+
* labels, enrols or trains from it. `Ext` with 0 when nobody clears the floor.
|
|
852
|
+
*/
|
|
853
|
+
export function shadowBestMatch(m: ShadowSearchManager, embedding: Float32Array, floor = SHADOW_FLOOR): { speaker: string; similarity: number } {
|
|
854
|
+
try {
|
|
855
|
+
const found = m.search({ v: embedding, threshold: floor })
|
|
856
|
+
if (!found) return { speaker: 'Ext', similarity: 0 }
|
|
857
|
+
let lo = 0
|
|
858
|
+
let hi = 1
|
|
859
|
+
for (let i = 0; i < 10; i++) {
|
|
860
|
+
const mid = (lo + hi) / 2
|
|
861
|
+
if (m.verify({ name: found, v: embedding, threshold: mid })) lo = mid
|
|
862
|
+
else hi = mid
|
|
863
|
+
}
|
|
864
|
+
return { speaker: found, similarity: (lo + hi) / 2 }
|
|
865
|
+
} catch {
|
|
866
|
+
return { speaker: 'Ext', similarity: 0 }
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
|
|
832
870
|
/** Log calibration data for threshold tuning */
|
|
833
871
|
function logCalibration(speaker: string, similarity: number, matched: boolean, event?: string): void {
|
|
834
872
|
try {
|