@gotcos/glasses-server 6.21.9 → 6.21.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +77 -0
- package/package.json +1 -1
- package/server/lib/audio-retention.ts +50 -0
- package/server/lib/meeting-speaker-review.ts +268 -0
- package/server/lib/meeting-store.ts +57 -2
- package/server/lib/speaker-calibration-log.ts +146 -0
- package/server/lib/speaker-embeddings.ts +266 -36
- package/server/lib/voice-profile-store.ts +507 -0
- package/server/routes/health.ts +11 -3
- package/server/routes/meeting.ts +56 -1
- package/server/routes/transcribe-stream.ts +46 -0
- package/server/routes/voice.ts +347 -22
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,80 @@
|
|
|
1
|
+
## 6.21.13
|
|
2
|
+
|
|
3
|
+
- Carry each meeting's `sessionId` on the meetings list, so a Control row can
|
|
4
|
+
open the per-meeting speaker review. The review is keyed on the session
|
|
5
|
+
because that is what lets the store's own hardened lookup find the chunk
|
|
6
|
+
sidecar again; without this field the two surfaces could not be joined.
|
|
7
|
+
- Read only the head of a sidecar to lift that one field. Sidecars run to
|
|
8
|
+
megabytes, and reading them whole would make listing cost scale with total
|
|
9
|
+
transcript size — and silently drop the id on any sidecar above the
|
|
10
|
+
whole-file size cap.
|
|
11
|
+
- Omit the field rather than invent one when a sidecar is absent, corrupt,
|
|
12
|
+
symlinked outside its month directory, or carries an implausible id. A
|
|
13
|
+
meeting with no readable sidecar is still listed.
|
|
14
|
+
|
|
15
|
+
## 6.21.12
|
|
16
|
+
|
|
17
|
+
- Add `GET /api/meeting/:sessionId/speakers`, the read surface behind COS
|
|
18
|
+
Control's speaker-naming panel. Read-only: it reports what a saved meeting's
|
|
19
|
+
chunk sidecar already holds and never writes. Naming, merging, and rebuilding
|
|
20
|
+
stay on the `/api/voice/*` routes, each with its own confirmation.
|
|
21
|
+
- Return two to three representative verbatim lines per voice, spread across the
|
|
22
|
+
meeting and timestamped. These are the primary output: a similarity score
|
|
23
|
+
cannot tell you who someone is, and a remembered sentence can.
|
|
24
|
+
- Report per-voice reliability from run length rather than similarity alone. Two
|
|
25
|
+
labels that swap every few segments are the identifier oscillating mid-turn,
|
|
26
|
+
which means those profiles cannot be told apart and any name applied to either
|
|
27
|
+
would be a guess. A high similarity score does not override that verdict.
|
|
28
|
+
- Restrict the run-length comparison to each pair of speakers, so a third person
|
|
29
|
+
interjecting cannot make two others look like they are swapping.
|
|
30
|
+
- Report a recovered meeting as unattributed rather than as a meeting with no
|
|
31
|
+
speakers, and still return phrases for it — on those meetings the phrases are
|
|
32
|
+
the only way in.
|
|
33
|
+
|
|
34
|
+
## 6.21.11
|
|
35
|
+
|
|
36
|
+
- Add `POST /api/voice/merge-profiles` for the case where one person holds two
|
|
37
|
+
profiles. The sherpa manager registers one centroid per name, so a split
|
|
38
|
+
identity has both halves competing on every search and each capped at twenty
|
|
39
|
+
samples independently — a weaker representation than either half deserves.
|
|
40
|
+
- Refuse a merge whose centroid similarity falls below the search-accept
|
|
41
|
+
threshold, since two profiles further apart than the value at which
|
|
42
|
+
identification would match them are not one voice. `force` overrides and is
|
|
43
|
+
recorded in the response.
|
|
44
|
+
- Preserve provenance through a merge and select the surviving samples for
|
|
45
|
+
acoustic diversity, so capping the union keeps both profiles represented
|
|
46
|
+
rather than silently discarding the absorbed one.
|
|
47
|
+
- Relabel the absorbed name's calibration history instead of deleting it: after
|
|
48
|
+
a merge it is one person's history, and it is the only evidence for whether
|
|
49
|
+
the merge improved identification.
|
|
50
|
+
- Refuse to absorb the owner label, which the live identification path checks
|
|
51
|
+
first on every chunk.
|
|
52
|
+
|
|
53
|
+
## 6.21.10
|
|
54
|
+
|
|
55
|
+
- Make the voice profile store durable. `voice-profiles.json` is written
|
|
56
|
+
atomically with rotating hourly backups, a corrupt file is quarantined and
|
|
57
|
+
recovered from the newest usable backup, and a save can no longer replace a
|
|
58
|
+
populated store with an empty one.
|
|
59
|
+
- Keep embedding provenance aligned. `sources[]` is now added, evicted, and
|
|
60
|
+
repaired in lockstep with `embeddings[]`, and centroids use the modal
|
|
61
|
+
dimension so a single wrong-length row cannot reduce a speaker's registered
|
|
62
|
+
vector to NaN.
|
|
63
|
+
- Read saved speaker audio from the runtime data directory, matching where the
|
|
64
|
+
transcription pipeline writes it. `train-g2`, `saved-audio`, `ext-audio`, and
|
|
65
|
+
`enroll-ext` previously resolved a path inside the installed package and
|
|
66
|
+
reported an empty system on every managed install.
|
|
67
|
+
- Require confirmation before an unscoped `train-g2` or `enroll-ext` rewrites
|
|
68
|
+
profiles and deletes source audio, cap G2 training at ten diverse samples per
|
|
69
|
+
speaker so a large backlog cannot evict an existing profile, and retain source
|
|
70
|
+
audio whenever nothing was enrolled.
|
|
71
|
+
- Add `readiness.speakerId` to health. A voiceprint model that is installed but
|
|
72
|
+
rejected by the runtime now reports degraded instead of passing as working
|
|
73
|
+
diarization; an install with no model configured is unaffected.
|
|
74
|
+
- Expire saved training audio after 14 days per file, add a confirm-gated
|
|
75
|
+
`POST /api/voice/delete-person` that reports per-store removal counts, and add
|
|
76
|
+
`GET /api/voice/profiles` for review surfaces.
|
|
77
|
+
|
|
1
78
|
## 6.21.9
|
|
2
79
|
|
|
3
80
|
- Prevent an unclosed or abandoned recording from monopolizing progressive HQ.
|
package/package.json
CHANGED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// Age-based retention for saved speaker audio.
|
|
2
|
+
//
|
|
3
|
+
// The count cap on training-audio (30 WAVs per speaker) is a storage bound, not
|
|
4
|
+
// a retention policy: a speaker who is never trained keeps 30 chunks of their
|
|
5
|
+
// recorded voice forever, and the only path that ever deleted them was a manual
|
|
6
|
+
// /voice/train-g2 call. An unenforced retention policy is worse than none —
|
|
7
|
+
// it is a promise the code does not keep.
|
|
8
|
+
//
|
|
9
|
+
// Split out as a pure function so the expiry rule can be tested by execution
|
|
10
|
+
// without a clock, a filesystem, or a running server.
|
|
11
|
+
|
|
12
|
+
export interface RetentionCandidate {
|
|
13
|
+
name: string
|
|
14
|
+
mtimeMs: number
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface RetentionSplit<T extends RetentionCandidate> {
|
|
18
|
+
expired: T[]
|
|
19
|
+
retained: T[]
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Partition files by age. Per-file rather than per-directory: chunks for one
|
|
24
|
+
* speaker accumulate across weeks, so an all-or-nothing directory check either
|
|
25
|
+
* keeps month-old audio alive because one chunk is fresh, or deletes today's
|
|
26
|
+
* capture because the directory is old.
|
|
27
|
+
*
|
|
28
|
+
* A file with an unreadable/zero mtime is RETAINED. Treating "I could not read
|
|
29
|
+
* the timestamp" as "this is ancient" would delete data on the strength of a
|
|
30
|
+
* failed stat.
|
|
31
|
+
*/
|
|
32
|
+
export function partitionExpiredAudio<T extends RetentionCandidate>(
|
|
33
|
+
files: T[],
|
|
34
|
+
nowMs: number,
|
|
35
|
+
ttlMs: number,
|
|
36
|
+
): RetentionSplit<T> {
|
|
37
|
+
const expired: T[] = []
|
|
38
|
+
const retained: T[] = []
|
|
39
|
+
for (const file of files) {
|
|
40
|
+
const age = nowMs - file.mtimeMs
|
|
41
|
+
if (file.mtimeMs > 0 && Number.isFinite(file.mtimeMs) && age > ttlMs) expired.push(file)
|
|
42
|
+
else retained.push(file)
|
|
43
|
+
}
|
|
44
|
+
return { expired, retained }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Human-readable age, for the log line that reports a purge. */
|
|
48
|
+
export function ageHours(mtimeMs: number, nowMs: number): number {
|
|
49
|
+
return Math.round(((nowMs - mtimeMs) / (60 * 60 * 1000)) * 10) / 10
|
|
50
|
+
}
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
// Per-meeting speaker review — the data behind COS Control's naming panel.
|
|
2
|
+
//
|
|
3
|
+
// The panel's job is to let a human name voices from what they REMEMBER, so the
|
|
4
|
+
// primary output here is verbatim phrases, not similarity scores. A score cannot
|
|
5
|
+
// tell you who someone is; "I'll follow up with both Andrews on music targeting"
|
|
6
|
+
// can.
|
|
7
|
+
//
|
|
8
|
+
// It also carries the diagnostic that actually catches a broken profile.
|
|
9
|
+
// Similarity between profile centroids does NOT distinguish "two people who
|
|
10
|
+
// sound alike" from "one voice split across two labels" — measured on the live
|
|
11
|
+
// store, Chris Krubeck and Luke Henry sat at 0.85 while genuinely being
|
|
12
|
+
// separate rows. What discriminates is RUN LENGTH within a single meeting:
|
|
13
|
+
//
|
|
14
|
+
// MU + Chris Krubeck (known two people) : flip rate 0.069, mean run 30.0
|
|
15
|
+
// Luke H + Luke Henry : flip rate 0.361, mean run 2.79
|
|
16
|
+
// Chris Krubeck + Luke Henry : flip rate 0.381, mean run 3.47
|
|
17
|
+
// Scott Taylor + Dylan Jackson : flip rate 0.344, mean run 2.80
|
|
18
|
+
//
|
|
19
|
+
// Real conversation holds the floor for ~30 consecutive segments. A pair that
|
|
20
|
+
// swaps every ~3 is the identifier oscillating mid-turn, which means those two
|
|
21
|
+
// profiles cannot be told apart and any name applied to either is a guess.
|
|
22
|
+
//
|
|
23
|
+
// Everything here is pure over a chunk array so it can be tested by execution.
|
|
24
|
+
|
|
25
|
+
/** One transcript chunk as stored in the `.g2-chunks.json` sidecar. */
|
|
26
|
+
export interface ReviewChunk {
|
|
27
|
+
text?: string
|
|
28
|
+
speaker?: string
|
|
29
|
+
/** MILLISECONDS from meeting start. Confirmed against the writer:
|
|
30
|
+
* meeting.ts accumulates `elapsed += row.durationMs`. Treating this as
|
|
31
|
+
* seconds reports a 32-minute meeting as 30,936 minutes. */
|
|
32
|
+
elapsed?: number
|
|
33
|
+
similarity?: number
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Labels that mean "nobody was identified", not a person. */
|
|
37
|
+
export const UNATTRIBUTED = new Set(['Unknown', 'Ext', '', 'Speaker 1', 'Speaker 2', 'Speaker 3'])
|
|
38
|
+
|
|
39
|
+
/** Calibrated from the control pair above. A pair must be BOTH flip-happy and
|
|
40
|
+
* short-run to be called unreliable — either alone has honest explanations
|
|
41
|
+
* (a rapid-fire exchange is flip-happy; a brief interjection is short-run). */
|
|
42
|
+
export const THRASH_FLIP_RATE = 0.20
|
|
43
|
+
export const THRASH_MEAN_RUN = 8
|
|
44
|
+
/** Below the search-accept threshold a name was never asserted with confidence. */
|
|
45
|
+
export const CONFIDENT_SIMILARITY = 0.65
|
|
46
|
+
|
|
47
|
+
export type Reliability = 'confident' | 'weak' | 'unreliable' | 'unattributed'
|
|
48
|
+
|
|
49
|
+
export interface ThrashPair {
|
|
50
|
+
speaker: string
|
|
51
|
+
flipRate: number
|
|
52
|
+
meanRun: number
|
|
53
|
+
sharedSegments: number
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface Phrase {
|
|
57
|
+
text: string
|
|
58
|
+
/** Milliseconds from meeting start, matching the sidecar. */
|
|
59
|
+
atMs: number
|
|
60
|
+
similarity: number | null
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface VoiceReview {
|
|
64
|
+
label: string
|
|
65
|
+
segments: number
|
|
66
|
+
meanSimilarity: number | null
|
|
67
|
+
/** Mean consecutive-run length across the WHOLE meeting. Not comparable to
|
|
68
|
+
* the pair-scoped run in `thrashesWith`: in a 13-voice meeting every
|
|
69
|
+
* speaker's whole-sequence run is short (measured: 3.36 for the owner), so
|
|
70
|
+
* this is a talkativeness texture, never a reliability signal. */
|
|
71
|
+
meanRun: number
|
|
72
|
+
longestRun: number
|
|
73
|
+
isOwner: boolean
|
|
74
|
+
reliability: Reliability
|
|
75
|
+
thrashesWith: ThrashPair[]
|
|
76
|
+
phrases: Phrase[]
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface MeetingSpeakerReview {
|
|
80
|
+
segments: number
|
|
81
|
+
/** False when no chunk carries a real speaker — a recovered capture. */
|
|
82
|
+
attributed: boolean
|
|
83
|
+
durationMs: number
|
|
84
|
+
voices: VoiceReview[]
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function mean(xs: number[]): number {
|
|
88
|
+
return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Consecutive-run lengths for one speaker across the whole meeting. */
|
|
92
|
+
export function speakerRuns(sequence: string[], speaker: string): number[] {
|
|
93
|
+
const runs: number[] = []
|
|
94
|
+
let cur = 0
|
|
95
|
+
for (const s of sequence) {
|
|
96
|
+
if (s === speaker) cur++
|
|
97
|
+
else if (cur > 0) { runs.push(cur); cur = 0 }
|
|
98
|
+
}
|
|
99
|
+
if (cur > 0) runs.push(cur)
|
|
100
|
+
return runs
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* How often the label flips between exactly two speakers, ignoring everyone
|
|
105
|
+
* else. Restricting to the pair is the point: a third person interjecting
|
|
106
|
+
* should not make two other speakers look like they are thrashing.
|
|
107
|
+
*/
|
|
108
|
+
export function pairFlipRate(
|
|
109
|
+
sequence: string[],
|
|
110
|
+
a: string,
|
|
111
|
+
b: string,
|
|
112
|
+
): { flipRate: number; meanRun: number; sharedSegments: number } | null {
|
|
113
|
+
const pair = sequence.filter(s => s === a || s === b)
|
|
114
|
+
if (pair.length < 6) return null // too little to characterise
|
|
115
|
+
if (!pair.includes(a) || !pair.includes(b)) return null
|
|
116
|
+
|
|
117
|
+
let flips = 0
|
|
118
|
+
const runs: number[] = []
|
|
119
|
+
let cur = 1
|
|
120
|
+
for (let i = 1; i < pair.length; i++) {
|
|
121
|
+
if (pair[i] === pair[i - 1]) cur++
|
|
122
|
+
else { flips++; runs.push(cur); cur = 1 }
|
|
123
|
+
}
|
|
124
|
+
runs.push(cur)
|
|
125
|
+
return {
|
|
126
|
+
flipRate: flips / (pair.length - 1),
|
|
127
|
+
meanRun: mean(runs),
|
|
128
|
+
sharedSegments: pair.length,
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Words that carry no identifying information. */
|
|
133
|
+
const FILLER = /^(?:yeah|yep|okay|ok|right|sure|mm+|uh+|um+|hmm+|got it|exactly|thanks?|no|yes)[.!?]?$/i
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Score a line by how much it would help a human recognise the speaker.
|
|
137
|
+
*
|
|
138
|
+
* Length matters (a longer line carries more voice), but so does specificity:
|
|
139
|
+
* a proper noun, a number, or a domain term is what makes someone say "that's
|
|
140
|
+
* Graham." A long line of pleasantries scores lower than a short line naming a
|
|
141
|
+
* brand and a figure.
|
|
142
|
+
*/
|
|
143
|
+
export function phraseScore(text: string): number {
|
|
144
|
+
const trimmed = text.trim()
|
|
145
|
+
if (!trimmed || FILLER.test(trimmed)) return 0
|
|
146
|
+
const words = trimmed.split(/\s+/)
|
|
147
|
+
if (words.length < 6) return 0
|
|
148
|
+
|
|
149
|
+
let score = Math.min(words.length, 28)
|
|
150
|
+
// Mid-sentence capitals — names, brands, products.
|
|
151
|
+
const propers = words.slice(1).filter(w => /^[A-Z][a-zA-Z]{2,}/.test(w)).length
|
|
152
|
+
score += Math.min(propers, 4) * 6
|
|
153
|
+
// Figures, percentages, money — highly memorable.
|
|
154
|
+
if (/\d/.test(trimmed)) score += 5
|
|
155
|
+
// Penalise a line that is mostly filler even if long.
|
|
156
|
+
const fillerWords = words.filter(w => FILLER.test(w)).length
|
|
157
|
+
score -= fillerWords * 3
|
|
158
|
+
return Math.max(score, 0)
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Pick up to `limit` representative lines, spread across the meeting.
|
|
163
|
+
*
|
|
164
|
+
* Spread is enforced deliberately: the three best-scoring lines often sit
|
|
165
|
+
* seconds apart in one monologue, which tells you far less about a speaker than
|
|
166
|
+
* three lines from the beginning, middle, and end. Every phrase carries its
|
|
167
|
+
* timestamp so a reviewer can place it against their memory of the meeting.
|
|
168
|
+
*/
|
|
169
|
+
export function selectPhrases(
|
|
170
|
+
chunks: ReviewChunk[],
|
|
171
|
+
speaker: string,
|
|
172
|
+
limit = 3,
|
|
173
|
+
durationMs = 0,
|
|
174
|
+
): Phrase[] {
|
|
175
|
+
const owned = chunks
|
|
176
|
+
.map((c, i) => ({ c, i }))
|
|
177
|
+
.filter(({ c }) => (c.speaker ?? '') === speaker)
|
|
178
|
+
.map(({ c }) => ({
|
|
179
|
+
text: (c.text ?? '').trim(),
|
|
180
|
+
atMs: typeof c.elapsed === 'number' ? c.elapsed : 0,
|
|
181
|
+
similarity: typeof c.similarity === 'number' ? c.similarity : null,
|
|
182
|
+
score: phraseScore(c.text ?? ''),
|
|
183
|
+
}))
|
|
184
|
+
.filter(p => p.score > 0)
|
|
185
|
+
|
|
186
|
+
if (owned.length === 0) return []
|
|
187
|
+
|
|
188
|
+
const span = durationMs > 0 ? durationMs : Math.max(...owned.map(p => p.atMs), 1)
|
|
189
|
+
const minGap = span / (limit * 2) // no two picks from the same stretch
|
|
190
|
+
|
|
191
|
+
const picked: typeof owned = []
|
|
192
|
+
for (const cand of [...owned].sort((a, b) => b.score - a.score)) {
|
|
193
|
+
if (picked.length >= limit) break
|
|
194
|
+
if (picked.some(p => Math.abs(p.atMs - cand.atMs) < minGap)) continue
|
|
195
|
+
picked.push(cand)
|
|
196
|
+
}
|
|
197
|
+
// Backfill on a short meeting where the gap rule excluded everything.
|
|
198
|
+
if (picked.length < limit) {
|
|
199
|
+
for (const cand of [...owned].sort((a, b) => b.score - a.score)) {
|
|
200
|
+
if (picked.length >= limit) break
|
|
201
|
+
if (!picked.includes(cand)) picked.push(cand)
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return picked
|
|
205
|
+
.sort((a, b) => a.atMs - b.atMs)
|
|
206
|
+
.map(({ text, atMs, similarity }) => ({ text, atMs, similarity }))
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** Build the whole review for one meeting's chunks. */
|
|
210
|
+
export function reviewMeetingSpeakers(
|
|
211
|
+
chunks: ReviewChunk[],
|
|
212
|
+
options: { owner?: string; phrasesPerVoice?: number } = {},
|
|
213
|
+
): MeetingSpeakerReview {
|
|
214
|
+
const owner = options.owner ?? 'Me'
|
|
215
|
+
const limit = options.phrasesPerVoice ?? 3
|
|
216
|
+
const sequence = chunks.map(c => c.speaker ?? '')
|
|
217
|
+
const durationMs = chunks.reduce((max, c) => Math.max(max, typeof c.elapsed === 'number' ? c.elapsed : 0), 0)
|
|
218
|
+
|
|
219
|
+
const labels = [...new Set(sequence)].filter(s => s.length > 0)
|
|
220
|
+
const named = labels.filter(l => !UNATTRIBUTED.has(l))
|
|
221
|
+
|
|
222
|
+
const voices: VoiceReview[] = labels.map(label => {
|
|
223
|
+
const own = chunks.filter(c => (c.speaker ?? '') === label)
|
|
224
|
+
const sims = own.map(c => c.similarity).filter((s): s is number => typeof s === 'number' && s > 0)
|
|
225
|
+
const runs = speakerRuns(sequence, label)
|
|
226
|
+
const unattributed = UNATTRIBUTED.has(label)
|
|
227
|
+
|
|
228
|
+
const thrashesWith: ThrashPair[] = []
|
|
229
|
+
if (!unattributed) {
|
|
230
|
+
for (const other of named) {
|
|
231
|
+
if (other === label) continue
|
|
232
|
+
const p = pairFlipRate(sequence, label, other)
|
|
233
|
+
if (!p) continue
|
|
234
|
+
if (p.flipRate > THRASH_FLIP_RATE && p.meanRun < THRASH_MEAN_RUN) {
|
|
235
|
+
thrashesWith.push({
|
|
236
|
+
speaker: other,
|
|
237
|
+
flipRate: Math.round(p.flipRate * 1000) / 1000,
|
|
238
|
+
meanRun: Math.round(p.meanRun * 100) / 100,
|
|
239
|
+
sharedSegments: p.sharedSegments,
|
|
240
|
+
})
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
thrashesWith.sort((a, b) => b.flipRate - a.flipRate)
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const meanSim = sims.length ? Math.round(mean(sims) * 1000) / 1000 : null
|
|
247
|
+
const reliability: Reliability = unattributed
|
|
248
|
+
? 'unattributed'
|
|
249
|
+
: thrashesWith.length > 0
|
|
250
|
+
? 'unreliable'
|
|
251
|
+
: (meanSim ?? 0) >= CONFIDENT_SIMILARITY ? 'confident' : 'weak'
|
|
252
|
+
|
|
253
|
+
return {
|
|
254
|
+
label,
|
|
255
|
+
segments: own.length,
|
|
256
|
+
meanSimilarity: meanSim,
|
|
257
|
+
meanRun: Math.round(mean(runs) * 100) / 100,
|
|
258
|
+
longestRun: runs.length ? Math.max(...runs) : 0,
|
|
259
|
+
isOwner: label === owner,
|
|
260
|
+
reliability,
|
|
261
|
+
thrashesWith,
|
|
262
|
+
phrases: selectPhrases(chunks, label, limit, durationMs),
|
|
263
|
+
}
|
|
264
|
+
})
|
|
265
|
+
|
|
266
|
+
voices.sort((a, b) => b.segments - a.segments)
|
|
267
|
+
return { segments: chunks.length, attributed: named.length > 0, durationMs, voices }
|
|
268
|
+
}
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
mkdirSync,
|
|
9
9
|
openSync,
|
|
10
10
|
readFileSync,
|
|
11
|
+
readSync,
|
|
11
12
|
readdirSync,
|
|
12
13
|
realpathSync,
|
|
13
14
|
unlinkSync,
|
|
@@ -43,6 +44,11 @@ export class MeetingStoreError extends Error {
|
|
|
43
44
|
|
|
44
45
|
export interface MeetingMeta {
|
|
45
46
|
filename: string
|
|
47
|
+
/** From the chunk sidecar, absent on meetings saved without one. Carried so a
|
|
48
|
+
* list row can open the per-meeting speaker review, which is keyed on the
|
|
49
|
+
* session rather than the filename — the session is what lets the store's
|
|
50
|
+
* own hardened lookup find the sidecar again. */
|
|
51
|
+
sessionId?: string
|
|
46
52
|
title: string
|
|
47
53
|
date: string
|
|
48
54
|
domain: string
|
|
@@ -314,7 +320,7 @@ export function boundedMeetingSource(content: string): { sourceContent: string;
|
|
|
314
320
|
return { sourceContent: bytes.subarray(0, end).toString('utf8'), sourceTruncated: true }
|
|
315
321
|
}
|
|
316
322
|
|
|
317
|
-
function toMeta(detail: MeetingDetail): MeetingMeta {
|
|
323
|
+
function toMeta(detail: MeetingDetail, sessionId?: string): MeetingMeta {
|
|
318
324
|
const detailCharEstimate = [
|
|
319
325
|
detail.title,
|
|
320
326
|
detail.date,
|
|
@@ -327,6 +333,7 @@ function toMeta(detail: MeetingDetail): MeetingMeta {
|
|
|
327
333
|
].join('\n\n').trim().length
|
|
328
334
|
return {
|
|
329
335
|
filename: detail.filename,
|
|
336
|
+
...(sessionId ? { sessionId } : {}),
|
|
330
337
|
title: detail.title,
|
|
331
338
|
date: detail.date,
|
|
332
339
|
domain: detail.domain,
|
|
@@ -344,6 +351,9 @@ function toMeta(detail: MeetingDetail): MeetingMeta {
|
|
|
344
351
|
}
|
|
345
352
|
}
|
|
346
353
|
|
|
354
|
+
/** Enough to clear the sidecar's leading metadata keys whatever their order. */
|
|
355
|
+
const SIDECAR_HEAD_BYTES = 4096
|
|
356
|
+
|
|
347
357
|
function isContained(parent: string, child: string): boolean {
|
|
348
358
|
return child === parent || child.startsWith(`${parent}${sep}`)
|
|
349
359
|
}
|
|
@@ -542,7 +552,7 @@ export class MeetingStore {
|
|
|
542
552
|
if (content === null) continue
|
|
543
553
|
const detail = parseMeeting(content, filename, month)
|
|
544
554
|
if (domain !== 'all' && detail.domain !== domain) continue
|
|
545
|
-
meetings.push(toMeta(detail))
|
|
555
|
+
meetings.push(toMeta(detail, this.sidecarSessionId(monthDir, monthReal, filename)))
|
|
546
556
|
} catch {
|
|
547
557
|
// One unreadable/corrupt entry must not hide the rest of the store.
|
|
548
558
|
}
|
|
@@ -601,6 +611,51 @@ export class MeetingStore {
|
|
|
601
611
|
return this.safeReadFile(monthDir, monthReal, filename)
|
|
602
612
|
}
|
|
603
613
|
|
|
614
|
+
/** Read only the first `bytes` of a file, with the same symlink,
|
|
615
|
+
* containment, and O_NOFOLLOW guards as safeReadFile.
|
|
616
|
+
*
|
|
617
|
+
* Exists so `list()` can lift one field out of a chunk sidecar without
|
|
618
|
+
* reading it whole: sidecars run to megabytes (1.3 MB for a 32-minute
|
|
619
|
+
* meeting) and would also trip safeReadFile's MAX_MEETING_BYTES cap, which
|
|
620
|
+
* is sized for markdown. */
|
|
621
|
+
private safeReadFileHead(monthDir: string, monthReal: string, filename: string, bytes: number): string | null {
|
|
622
|
+
const filepath = join(monthDir, filename)
|
|
623
|
+
let fd: number | null = null
|
|
624
|
+
try {
|
|
625
|
+
const linkStat = lstatSync(filepath)
|
|
626
|
+
if (linkStat.isSymbolicLink() || !linkStat.isFile()) return null
|
|
627
|
+
const real = realpathSync(filepath)
|
|
628
|
+
if (!isContained(monthReal, real) || dirname(real) !== monthReal) return null
|
|
629
|
+
fd = openSync(filepath, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0))
|
|
630
|
+
const stat = fstatSync(fd)
|
|
631
|
+
if (!stat.isFile()) return null
|
|
632
|
+
const buffer = Buffer.alloc(Math.min(bytes, stat.size))
|
|
633
|
+
if (buffer.length === 0) return ''
|
|
634
|
+
const read = readSync(fd, buffer, 0, buffer.length, 0)
|
|
635
|
+
return buffer.subarray(0, read).toString('utf8')
|
|
636
|
+
} catch {
|
|
637
|
+
return null
|
|
638
|
+
} finally {
|
|
639
|
+
if (fd !== null) {
|
|
640
|
+
try { closeSync(fd) } catch { /* already closed */ }
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
/** The sessionId recorded in a meeting's chunk sidecar, if it has one.
|
|
646
|
+
*
|
|
647
|
+
* Matched with a regex over the file's head rather than JSON.parse: the goal
|
|
648
|
+
* is one small field, and parsing a megabyte of chunks per list row to reach
|
|
649
|
+
* it would make listing cost scale with total transcript size. */
|
|
650
|
+
private sidecarSessionId(monthDir: string, monthReal: string, meetingFilename: string): string | undefined {
|
|
651
|
+
const sidecarName = meetingFilename.replace(/\.md$/, '.g2-chunks.json')
|
|
652
|
+
if (sidecarName === meetingFilename) return undefined
|
|
653
|
+
const head = this.safeReadFileHead(monthDir, monthReal, sidecarName, SIDECAR_HEAD_BYTES)
|
|
654
|
+
if (!head) return undefined
|
|
655
|
+
const match = head.match(/"sessionId"\s*:\s*"([A-Za-z0-9:_-]{3,96})"/)
|
|
656
|
+
return match ? match[1] : undefined
|
|
657
|
+
}
|
|
658
|
+
|
|
604
659
|
private safeReadFile(monthDir: string, monthReal: string, filename: string): string | null {
|
|
605
660
|
const filepath = join(monthDir, filename)
|
|
606
661
|
let fd: number | null = null
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
// speaker-calibration.jsonl — threshold-tuning telemetry, one row per
|
|
2
|
+
// identification decision. Every row carries a speaker NAME, so a person's
|
|
3
|
+
// trace survives deleting their voice profile unless this file is swept too.
|
|
4
|
+
//
|
|
5
|
+
// The log is append-only and written from the live identification path with a
|
|
6
|
+
// fire-and-forget `appendFileSync`. A rewrite therefore has a genuine (if
|
|
7
|
+
// millisecond-wide) race with concurrent appends. That is acceptable HERE and
|
|
8
|
+
// nowhere else in this feature: the file is explicitly non-critical tuning data,
|
|
9
|
+
// already best-effort, and losing a row written during the swap costs nothing —
|
|
10
|
+
// whereas leaving a deleted person's name in 21k rows is the privacy gap the
|
|
11
|
+
// delete exists to close. Embedding data is never touched by this module.
|
|
12
|
+
|
|
13
|
+
import { existsSync, readFileSync } from 'node:fs'
|
|
14
|
+
import { durableAtomicWriteFileSync } from './atomic-fs.js'
|
|
15
|
+
|
|
16
|
+
export interface CalibrationPurgeResult {
|
|
17
|
+
/** Rows whose `speaker` matched and were dropped. */
|
|
18
|
+
removed: number
|
|
19
|
+
/** Rows retained, including rows that could not be parsed. */
|
|
20
|
+
retained: number
|
|
21
|
+
/** Rows that were not valid JSON. Kept — a malformed row is not evidence that
|
|
22
|
+
* it belongs to the person being deleted, and discarding it would be a
|
|
23
|
+
* silent data loss dressed up as a privacy fix. */
|
|
24
|
+
unparsable: number
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Filter JSONL text, dropping rows whose `speaker` field matches.
|
|
29
|
+
*
|
|
30
|
+
* Pure so the matching rule can be tested without touching the filesystem.
|
|
31
|
+
* Matching is exact on the `speaker` field only: a substring match would delete
|
|
32
|
+
* every "Miles Mallard" row when removing "Miles", and the name also appears in
|
|
33
|
+
* no other field.
|
|
34
|
+
*/
|
|
35
|
+
export function filterCalibrationRows(
|
|
36
|
+
raw: string,
|
|
37
|
+
speakerName: string,
|
|
38
|
+
): { text: string; result: CalibrationPurgeResult } {
|
|
39
|
+
const lines = raw.split('\n')
|
|
40
|
+
const kept: string[] = []
|
|
41
|
+
const result: CalibrationPurgeResult = { removed: 0, retained: 0, unparsable: 0 }
|
|
42
|
+
|
|
43
|
+
for (const line of lines) {
|
|
44
|
+
if (line.trim() === '') continue
|
|
45
|
+
let speaker: unknown
|
|
46
|
+
try {
|
|
47
|
+
speaker = (JSON.parse(line) as { speaker?: unknown }).speaker
|
|
48
|
+
} catch {
|
|
49
|
+
result.unparsable++
|
|
50
|
+
result.retained++
|
|
51
|
+
kept.push(line)
|
|
52
|
+
continue
|
|
53
|
+
}
|
|
54
|
+
if (speaker === speakerName) {
|
|
55
|
+
result.removed++
|
|
56
|
+
continue
|
|
57
|
+
}
|
|
58
|
+
result.retained++
|
|
59
|
+
kept.push(line)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return { text: kept.length > 0 ? kept.join('\n') + '\n' : '', result }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Rename a speaker across the log, for a profile merge.
|
|
67
|
+
*
|
|
68
|
+
* Relabel rather than delete: a merge means the two names were always one
|
|
69
|
+
* person, so their calibration history is one history. Dropping the absorbed
|
|
70
|
+
* name's rows would silently discard exactly the evidence needed to tell
|
|
71
|
+
* whether the merge improved identification.
|
|
72
|
+
*/
|
|
73
|
+
export function relabelCalibrationRows(
|
|
74
|
+
raw: string,
|
|
75
|
+
fromName: string,
|
|
76
|
+
toName: string,
|
|
77
|
+
): { text: string; relabeled: number; retained: number; unparsable: number } {
|
|
78
|
+
const out: string[] = []
|
|
79
|
+
let relabeled = 0, retained = 0, unparsable = 0
|
|
80
|
+
for (const line of raw.split('\n')) {
|
|
81
|
+
if (line.trim() === '') continue
|
|
82
|
+
let row: Record<string, unknown>
|
|
83
|
+
try {
|
|
84
|
+
row = JSON.parse(line) as Record<string, unknown>
|
|
85
|
+
} catch {
|
|
86
|
+
unparsable++; retained++; out.push(line); continue
|
|
87
|
+
}
|
|
88
|
+
// `retained` counts every row that survives, relabeled or not, so
|
|
89
|
+
// `retained === total input rows` is the invariant a caller can assert: a
|
|
90
|
+
// merge must never lose history. (Contrast filterCalibrationRows, where
|
|
91
|
+
// removed + retained = total.)
|
|
92
|
+
retained++
|
|
93
|
+
if (row.speaker === fromName) {
|
|
94
|
+
row.speaker = toName
|
|
95
|
+
relabeled++
|
|
96
|
+
out.push(JSON.stringify(row))
|
|
97
|
+
} else {
|
|
98
|
+
out.push(line)
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return { text: out.length > 0 ? out.join('\n') + '\n' : '', relabeled, retained, unparsable }
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Apply a merge relabel to the log on disk. */
|
|
105
|
+
export function relabelSpeakerCalibrationRows(
|
|
106
|
+
logPath: string,
|
|
107
|
+
fromName: string,
|
|
108
|
+
toName: string,
|
|
109
|
+
options: { dryRun?: boolean } = {},
|
|
110
|
+
): { relabeled: number; retained: number; unparsable: number } {
|
|
111
|
+
if (!existsSync(logPath)) return { relabeled: 0, retained: 0, unparsable: 0 }
|
|
112
|
+
let raw: string
|
|
113
|
+
try {
|
|
114
|
+
raw = readFileSync(logPath, 'utf-8')
|
|
115
|
+
} catch {
|
|
116
|
+
return { relabeled: 0, retained: 0, unparsable: 0 }
|
|
117
|
+
}
|
|
118
|
+
const { text, relabeled, retained, unparsable } = relabelCalibrationRows(raw, fromName, toName)
|
|
119
|
+
if (relabeled > 0 && !options.dryRun) {
|
|
120
|
+
durableAtomicWriteFileSync(logPath, text, { mode: 0o600 })
|
|
121
|
+
}
|
|
122
|
+
return { relabeled, retained, unparsable }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Rewrite the log without a given speaker's rows. */
|
|
126
|
+
export function purgeSpeakerCalibrationRows(
|
|
127
|
+
logPath: string,
|
|
128
|
+
speakerName: string,
|
|
129
|
+
options: { dryRun?: boolean } = {},
|
|
130
|
+
): CalibrationPurgeResult {
|
|
131
|
+
if (!existsSync(logPath)) return { removed: 0, retained: 0, unparsable: 0 }
|
|
132
|
+
let raw: string
|
|
133
|
+
try {
|
|
134
|
+
raw = readFileSync(logPath, 'utf-8')
|
|
135
|
+
} catch {
|
|
136
|
+
return { removed: 0, retained: 0, unparsable: 0 }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const { text, result } = filterCalibrationRows(raw, speakerName)
|
|
140
|
+
if (result.removed === 0 || options.dryRun) return result
|
|
141
|
+
|
|
142
|
+
// Atomic: a torn rewrite of a 2 MB log would leave a half-row that every
|
|
143
|
+
// later parse trips over.
|
|
144
|
+
durableAtomicWriteFileSync(logPath, text, { mode: 0o600 })
|
|
145
|
+
return result
|
|
146
|
+
}
|