@gotcos/glasses-server 6.27.0 → 6.27.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/package.json +1 -1
- package/server/lib/media-store.ts +10 -1
- package/server/lib/query-attachments.ts +17 -1
- package/server/lib/voice-directory.ts +418 -0
- package/server/routes/prompt-drafts.ts +28 -0
- package/server/routes/voice.ts +33 -0
- package/shared/media-attachment.ts +30 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,62 @@
|
|
|
1
|
+
## 6.27.2
|
|
2
|
+
|
|
3
|
+
### One finalizer for phone and Mac dictation
|
|
4
|
+
|
|
5
|
+
- Added authenticated `POST /api/dictation/finalize` for already-transcribed
|
|
6
|
+
Moonshine text. It reuses the exact glossary, negative rules, Haiku/Sonnet
|
|
7
|
+
polish, circuit breaker, token audit, and daily cap already used by recovered
|
|
8
|
+
server prompt drafts.
|
|
9
|
+
- The route accepts bounded text only. Phone-local audio and rolling preview
|
|
10
|
+
audio do not leave the iPhone, and a finalizer failure falls back to the
|
|
11
|
+
deterministic glossary result instead of losing the transcription.
|
|
12
|
+
- Both Message commits and sealed Meeting preview phrases can use the same
|
|
13
|
+
final quality pass while canonical Large-v3 meeting transcription remains
|
|
14
|
+
unchanged.
|
|
15
|
+
|
|
16
|
+
## 6.27.1
|
|
17
|
+
|
|
18
|
+
### The 16-frame video from 6.27.0 now actually works end to end
|
|
19
|
+
|
|
20
|
+
6.27.0 raised video stills from 1-3 to 8-16 entirely inside the extractor. Four
|
|
21
|
+
consumers were never checked, and each one silently rejected what the extractor
|
|
22
|
+
had started writing. Every failure was invisible: three dropped an optional field
|
|
23
|
+
with no error, and the fourth threw only when the user asked a question.
|
|
24
|
+
|
|
25
|
+
- **A video of 75 seconds or longer 400'd when you asked about it.**
|
|
26
|
+
`query-attachments.ts` capped model image inputs at 12 and threw a hard
|
|
27
|
+
`too_many_attachment_frames` rather than trimming, while `round(75/6) = 13`
|
|
28
|
+
frames. The video uploaded, stored its frames, and failed at ask time. The
|
|
29
|
+
ceiling is now `VIDEO_SUMMARY_FRAMES_MAX`, expressed as the symbol so the two
|
|
30
|
+
cannot drift again, and a test walks every duration to 30 minutes. Nothing in
|
|
31
|
+
the suite had ever asserted `too_many_attachment_frames`.
|
|
32
|
+
- **A 16-frame video became an 8-frame video after any restart.**
|
|
33
|
+
`sanitizeRecord` ran `derivativePaths.slice(0, 8)` on index load, so the record
|
|
34
|
+
kept 16 frames in memory and 8 on reload - and Update Server restarts the
|
|
35
|
+
server. The other 8 files stayed on disk orphaned: unreferenced, never served,
|
|
36
|
+
never swept. PDFs are unaffected; their producer caps itself at 8.
|
|
37
|
+
- **`frameCount` above 8 was dropped by the parser.** `parseMediaAttachmentRef`
|
|
38
|
+
is a whitelist and the field is optional, so the count vanished with no error
|
|
39
|
+
for every video past ~90 seconds.
|
|
40
|
+
- **`bytes` above 64 MiB was dropped by the parser** - so the 100 MiB and chunked
|
|
41
|
+
2 GiB videos shipped in 6.26.0 lost their byte count too. Raised to the chunked
|
|
42
|
+
ceiling and pinned to it.
|
|
43
|
+
- `durationMs` above 1 hour was likewise dropped. Unreachable today behind the
|
|
44
|
+
20-minute ingest cap, fixed now because it is the same one-line class and would
|
|
45
|
+
have been the next invisible ceiling.
|
|
46
|
+
|
|
47
|
+
The parser bounds are now named constants documented as sanity bounds rather than
|
|
48
|
+
policy, mirrored from the server ceilings and pinned by test - `shared/` cannot
|
|
49
|
+
import `server/`, and an unpinned copy is what allowed all of this to drift.
|
|
50
|
+
|
|
51
|
+
Known limitation, unchanged: attaching three or more videos to one prompt still
|
|
52
|
+
returns 400, because 3 x the 8-frame floor exceeds the 16-input ceiling. Frames
|
|
53
|
+
are a video's only visual representation, so refusing is honest where silently
|
|
54
|
+
dropping half of one would not be.
|
|
55
|
+
|
|
56
|
+
New tests: 8 parser round-trip, 4 restart round-trip, 5 frame-budget. Every one
|
|
57
|
+
round-trips through the real reader - asserting on the writer is what missed all
|
|
58
|
+
four defects, since the writer was correct in every case. 8 mutations, all caught.
|
|
59
|
+
|
|
1
60
|
## 6.27.0
|
|
2
61
|
|
|
3
62
|
### Video review frames: 8-16 stills instead of 1-3
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gotcos/glasses-server",
|
|
3
|
-
"version": "6.27.
|
|
3
|
+
"version": "6.27.2",
|
|
4
4
|
"description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -54,6 +54,7 @@ import {
|
|
|
54
54
|
validateSourceImage,
|
|
55
55
|
} from './image-safety.js'
|
|
56
56
|
import {
|
|
57
|
+
VIDEO_SUMMARY_FRAMES_MAX,
|
|
57
58
|
prepareRichMediaFromFile,
|
|
58
59
|
type MediaTransferMode,
|
|
59
60
|
type PreparedRichMediaFile,
|
|
@@ -278,8 +279,16 @@ function sanitizeRecord(raw: unknown): MediaRecord | null {
|
|
|
278
279
|
&& resolve(sep, value).startsWith(`${expectedAbsolute}${sep}`)
|
|
279
280
|
if (!isOwnedPath(r.storagePath) || !isOwnedPath(r.thumbPath)) return null
|
|
280
281
|
const textPath = isOwnedPath(r.textPath) ? r.textPath : undefined
|
|
282
|
+
// Was slice(0, 8). sanitizeRecord runs on INDEX LOAD, so a 16-frame video kept all
|
|
283
|
+
// 16 frames in memory and dropped to 8 after any restart — and COS Control's Update
|
|
284
|
+
// Server restarts the server. The other 8 files stayed on disk orphaned:
|
|
285
|
+
// unreferenced, never served, never swept. The model therefore saw the whole video
|
|
286
|
+
// before a restart and the first half after one, which is invisible in every status
|
|
287
|
+
// surface. This is a sanity bound on untrusted index data, so it tracks the largest
|
|
288
|
+
// count any producer can emit; PDFs are unaffected, their producer caps itself at
|
|
289
|
+
// MAX_DERIVATIVE_IMAGES.
|
|
281
290
|
const derivativePaths = Array.isArray(r.derivativePaths)
|
|
282
|
-
? r.derivativePaths.filter(isOwnedPath).slice(0,
|
|
291
|
+
? r.derivativePaths.filter(isOwnedPath).slice(0, VIDEO_SUMMARY_FRAMES_MAX)
|
|
283
292
|
: undefined
|
|
284
293
|
const videoCompression = sanitizeVideoCompression(r.videoCompression)
|
|
285
294
|
return {
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
type MediaAttachmentRef,
|
|
21
21
|
} from '../../shared/media-attachment.js'
|
|
22
22
|
import { getMediaStore, MediaStoreError } from './media-store.js'
|
|
23
|
+
import { VIDEO_SUMMARY_FRAMES_MAX } from './rich-media-safety.js'
|
|
23
24
|
import { strictBase64Decode, ImageSafetyError } from './image-safety.js'
|
|
24
25
|
import type { ModelImageInput } from './model-image-input.js'
|
|
25
26
|
import { formatAttachmentSourceData, type AttachmentSourceData } from './prompt-reference-boundary.js'
|
|
@@ -34,7 +35,22 @@ export interface ResolvedQueryAttachments {
|
|
|
34
35
|
promptBlock?: string
|
|
35
36
|
}
|
|
36
37
|
|
|
37
|
-
|
|
38
|
+
/**
|
|
39
|
+
* Image inputs one prompt may carry.
|
|
40
|
+
*
|
|
41
|
+
* This MUST be at least VIDEO_SUMMARY_FRAMES_MAX, because a video contributes one
|
|
42
|
+
* input per extracted frame with no clamp (see the video branch below) and exceeding
|
|
43
|
+
* this throws a hard 400. It sat at 12 while frames went to 16 in 6.27.0, so every
|
|
44
|
+
* video of 75 seconds or longer uploaded successfully, stored its frames, and then
|
|
45
|
+
* failed the instant the user asked a question about it:
|
|
46
|
+
*
|
|
47
|
+
* Math.round(75 / 6) = 13 frames > 12 -> 400 too_many_attachment_frames
|
|
48
|
+
*
|
|
49
|
+
* Frames are a video's ONLY visual representation, unlike PDF page images which are
|
|
50
|
+
* an optional aid over canonical text — so this ceiling cannot be treated as a
|
|
51
|
+
* budget to trim silently. It has to fit a whole video.
|
|
52
|
+
*/
|
|
53
|
+
const MAX_MODEL_IMAGE_INPUTS = VIDEO_SUMMARY_FRAMES_MAX
|
|
38
54
|
const MAX_ATTACHMENT_PROMPT_CHARS = 60_000
|
|
39
55
|
|
|
40
56
|
export class QueryAttachmentError extends Error {
|
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-meeting voice directory.
|
|
3
|
+
*
|
|
4
|
+
* Profiles answer WHO is enrolled. Meeting sidecars answer WHERE a profile was
|
|
5
|
+
* observed and how strong those occurrence-level matches were. Keeping those
|
|
6
|
+
* jobs separate matters: an embedding count is training coverage, never a
|
|
7
|
+
* confidence score, and Ext/Unidentified clusters are meeting-local—not people.
|
|
8
|
+
*
|
|
9
|
+
* The scan is asynchronous, bounded, single-flight, and cached. It never runs a
|
|
10
|
+
* client-side N+1 fan-out and it never exposes filesystem paths.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { lstat, readFile, readdir, realpath } from 'node:fs/promises'
|
|
14
|
+
import { basename, dirname, join } from 'node:path'
|
|
15
|
+
import { setImmediate as yieldToEventLoop } from 'node:timers/promises'
|
|
16
|
+
import { dataPath } from './data-dir.js'
|
|
17
|
+
import { confirmedLabels } from './meeting-corrections.js'
|
|
18
|
+
import { resolveCosOperationsDir, resolveMeetingLibrary } from './cos-operations-meetings.js'
|
|
19
|
+
import {
|
|
20
|
+
isUnattributed,
|
|
21
|
+
reviewMeetingSpeakers,
|
|
22
|
+
type Reliability,
|
|
23
|
+
type ReviewChunk,
|
|
24
|
+
type SpeakerWordSegment,
|
|
25
|
+
} from './meeting-speaker-review.js'
|
|
26
|
+
import { getOwnerSpeakerLabel } from './profile.js'
|
|
27
|
+
import { readVoiceProfiles, type VoiceProfile } from './speaker-embeddings.js'
|
|
28
|
+
|
|
29
|
+
const MONTH = /^\d{4}-(0[1-9]|1[0-2])$/
|
|
30
|
+
const SIDECAR = /\.g2-chunks\.json$/
|
|
31
|
+
const SESSION = /^[A-Za-z0-9:_-]{3,96}$/
|
|
32
|
+
const MAX_EVIDENCE_FILES = 1_200
|
|
33
|
+
const MAX_SIDECAR_BYTES = 12 * 1024 * 1024
|
|
34
|
+
const MAX_TOTAL_BYTES = 512 * 1024 * 1024
|
|
35
|
+
const MAX_APPEARANCES_PER_VOICE = 24
|
|
36
|
+
const CACHE_MS = 5 * 60_000
|
|
37
|
+
|
|
38
|
+
type EvidenceSource = 'cos_operations' | 'direct_library' | 'standalone_recordings'
|
|
39
|
+
|
|
40
|
+
interface EvidenceCandidate {
|
|
41
|
+
sidecarPath: string
|
|
42
|
+
markdownPath: string
|
|
43
|
+
source: EvidenceSource
|
|
44
|
+
mutable: boolean
|
|
45
|
+
date: string
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface VoiceAppearance {
|
|
49
|
+
sessionId: string
|
|
50
|
+
title: string
|
|
51
|
+
date: string
|
|
52
|
+
source: EvidenceSource
|
|
53
|
+
mutable: boolean
|
|
54
|
+
segments: number
|
|
55
|
+
speakingMs: number
|
|
56
|
+
speakingTimeSource: 'words' | 'chunks'
|
|
57
|
+
observedMatch: number | null
|
|
58
|
+
reliability: Reliability
|
|
59
|
+
confirmedByHuman: boolean
|
|
60
|
+
needsReview: boolean
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface VoiceDirectoryProfile {
|
|
64
|
+
name: string
|
|
65
|
+
isOwner: boolean
|
|
66
|
+
embeddings: number
|
|
67
|
+
sources: Record<string, number>
|
|
68
|
+
sourcesAligned: boolean
|
|
69
|
+
assertedSegments: number
|
|
70
|
+
candidateSegments: number
|
|
71
|
+
assertedSpeakingMs: number
|
|
72
|
+
candidateSpeakingMs: number
|
|
73
|
+
speakingTimeSources: Record<'words' | 'chunks', number>
|
|
74
|
+
meetingCount: number
|
|
75
|
+
reviewMeetingCount: number
|
|
76
|
+
observedMatch: number | null
|
|
77
|
+
observedMatchSegments: number
|
|
78
|
+
reliabilityCounts: Record<Reliability, number>
|
|
79
|
+
firstSeen: string | null
|
|
80
|
+
lastSeen: string | null
|
|
81
|
+
appearances: VoiceAppearance[]
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export interface VoiceDirectorySnapshot {
|
|
85
|
+
schemaVersion: 1
|
|
86
|
+
generatedAt: string
|
|
87
|
+
owner: string
|
|
88
|
+
profileCount: number
|
|
89
|
+
totalEmbeddings: number
|
|
90
|
+
meetingsScanned: number
|
|
91
|
+
sidecarsSkipped: number
|
|
92
|
+
truncated: boolean
|
|
93
|
+
unresolvedMeetings: number
|
|
94
|
+
unresolvedSegments: number
|
|
95
|
+
profiles: VoiceDirectoryProfile[]
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
interface AppearanceAccumulator {
|
|
99
|
+
assertedSegments: number
|
|
100
|
+
candidateSegments: number
|
|
101
|
+
assertedSpeakingMs: number
|
|
102
|
+
candidateSpeakingMs: number
|
|
103
|
+
speakingTimeSources: Record<'words' | 'chunks', number>
|
|
104
|
+
observedWeighted: number
|
|
105
|
+
observedSegments: number
|
|
106
|
+
reliabilityCounts: Record<Reliability, number>
|
|
107
|
+
assertedMeetings: Set<string>
|
|
108
|
+
reviewMeetings: Set<string>
|
|
109
|
+
firstSeen: string | null
|
|
110
|
+
lastSeen: string | null
|
|
111
|
+
appearances: VoiceAppearance[]
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
let cached: { at: number; snapshot: VoiceDirectorySnapshot } | null = null
|
|
115
|
+
let building: Promise<VoiceDirectorySnapshot> | null = null
|
|
116
|
+
|
|
117
|
+
function sourceCounts(profile: VoiceProfile): Record<string, number> {
|
|
118
|
+
const counts: Record<string, number> = {}
|
|
119
|
+
for (const source of profile.sources ?? []) {
|
|
120
|
+
const key = source.startsWith('auto:') ? 'auto' : source
|
|
121
|
+
counts[key] = (counts[key] ?? 0) + 1
|
|
122
|
+
}
|
|
123
|
+
return counts
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function cleanTitle(markdown: string, fallback: string): string {
|
|
127
|
+
const heading = markdown.match(/^#\s+(.+)$/m)?.[1]?.trim()
|
|
128
|
+
if (heading) return heading.slice(0, 180)
|
|
129
|
+
return fallback
|
|
130
|
+
.replace(/\.g2-chunks\.json$/, '')
|
|
131
|
+
.replace(/^\d{4}-\d{2}-\d{2}_/, '')
|
|
132
|
+
.replace(/_/g, ' ')
|
|
133
|
+
.slice(0, 180) || 'Untitled meeting'
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async function safeDirectory(path: string): Promise<string | null> {
|
|
137
|
+
try {
|
|
138
|
+
const info = await lstat(path)
|
|
139
|
+
if (!info.isDirectory() || info.isSymbolicLink()) return null
|
|
140
|
+
return await realpath(path)
|
|
141
|
+
} catch { return null }
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async function childDirectory(parentReal: string, name: string): Promise<string | null> {
|
|
145
|
+
const child = await safeDirectory(join(parentReal, name))
|
|
146
|
+
return child && dirname(child) === parentReal ? child : null
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function collectMonths(
|
|
150
|
+
base: string,
|
|
151
|
+
source: EvidenceSource,
|
|
152
|
+
mutable: boolean,
|
|
153
|
+
): Promise<EvidenceCandidate[]> {
|
|
154
|
+
const root = await safeDirectory(base)
|
|
155
|
+
if (!root) return []
|
|
156
|
+
const candidates: EvidenceCandidate[] = []
|
|
157
|
+
const months = (await readdir(root)).filter(name => MONTH.test(name)).sort().reverse()
|
|
158
|
+
for (const month of months) {
|
|
159
|
+
const monthDir = await childDirectory(root, month)
|
|
160
|
+
if (!monthDir) continue
|
|
161
|
+
const files = (await readdir(monthDir)).filter(name => SIDECAR.test(name)).sort().reverse()
|
|
162
|
+
for (const file of files) {
|
|
163
|
+
candidates.push({
|
|
164
|
+
sidecarPath: join(monthDir, file),
|
|
165
|
+
markdownPath: join(monthDir, file.replace(SIDECAR, '.md')),
|
|
166
|
+
source,
|
|
167
|
+
mutable,
|
|
168
|
+
date: file.match(/^\d{4}-\d{2}-\d{2}/)?.[0] ?? month,
|
|
169
|
+
})
|
|
170
|
+
if (candidates.length >= MAX_EVIDENCE_FILES) return candidates
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return candidates
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async function boundedMarkdownTitle(path: string, fallback: string): Promise<string> {
|
|
177
|
+
try {
|
|
178
|
+
const link = await lstat(path)
|
|
179
|
+
if (!link.isFile() || link.isSymbolicLink() || link.size > 10 * 1024 * 1024) return fallback
|
|
180
|
+
const real = await realpath(path)
|
|
181
|
+
if (dirname(real) !== dirname(path)) return fallback
|
|
182
|
+
return cleanTitle((await readFile(real, 'utf8')).slice(0, 8_192), fallback)
|
|
183
|
+
} catch {
|
|
184
|
+
return fallback
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
async function collectCandidates(): Promise<{ rows: EvidenceCandidate[]; truncated: boolean }> {
|
|
189
|
+
const groups: EvidenceCandidate[][] = []
|
|
190
|
+
const canonicalRoots = new Set<string>()
|
|
191
|
+
const operations = resolveCosOperationsDir()
|
|
192
|
+
if (operations) {
|
|
193
|
+
const operationsReal = await safeDirectory(operations)
|
|
194
|
+
if (operationsReal) {
|
|
195
|
+
canonicalRoots.add(operationsReal)
|
|
196
|
+
const domains = (await readdir(operationsReal)).sort()
|
|
197
|
+
const rows: EvidenceCandidate[] = []
|
|
198
|
+
for (const domain of domains) {
|
|
199
|
+
const domainDir = await childDirectory(operationsReal, domain)
|
|
200
|
+
if (!domainDir) continue
|
|
201
|
+
rows.push(...await collectMonths(join(domainDir, 'meetings'), 'cos_operations', true))
|
|
202
|
+
if (rows.length >= MAX_EVIDENCE_FILES) break
|
|
203
|
+
}
|
|
204
|
+
groups.push(rows)
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const library = resolveMeetingLibrary()
|
|
209
|
+
if (library.layout === 'direct' && library.root) {
|
|
210
|
+
const directReal = await safeDirectory(library.root)
|
|
211
|
+
if (directReal && !canonicalRoots.has(directReal)) {
|
|
212
|
+
canonicalRoots.add(directReal)
|
|
213
|
+
groups.push(await collectMonths(directReal, 'direct_library', false))
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const standalone = await safeDirectory(dataPath('recordings'))
|
|
218
|
+
if (standalone && !canonicalRoots.has(standalone)) {
|
|
219
|
+
groups.push(await collectMonths(standalone, 'standalone_recordings', true))
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const total = groups.reduce((sum, rows) => sum + rows.length, 0)
|
|
223
|
+
// Source order is precedence order. Duplicates are removed after parsing by
|
|
224
|
+
// session id, so the titled operations copy wins over direct/raw copies.
|
|
225
|
+
return { rows: groups.flat().slice(0, MAX_EVIDENCE_FILES), truncated: total > MAX_EVIDENCE_FILES }
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function emptyAccumulator(): AppearanceAccumulator {
|
|
229
|
+
return {
|
|
230
|
+
assertedSegments: 0,
|
|
231
|
+
candidateSegments: 0,
|
|
232
|
+
assertedSpeakingMs: 0,
|
|
233
|
+
candidateSpeakingMs: 0,
|
|
234
|
+
speakingTimeSources: { words: 0, chunks: 0 },
|
|
235
|
+
observedWeighted: 0,
|
|
236
|
+
observedSegments: 0,
|
|
237
|
+
reliabilityCounts: { confident: 0, weak: 0, unreliable: 0, unattributed: 0 },
|
|
238
|
+
assertedMeetings: new Set(),
|
|
239
|
+
reviewMeetings: new Set(),
|
|
240
|
+
firstSeen: null,
|
|
241
|
+
lastSeen: null,
|
|
242
|
+
appearances: [],
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export async function buildVoiceDirectorySnapshot(): Promise<VoiceDirectorySnapshot> {
|
|
247
|
+
const owner = getOwnerSpeakerLabel()
|
|
248
|
+
const { profiles } = readVoiceProfiles()
|
|
249
|
+
const byName = new Map(profiles.map(profile => [profile.name, emptyAccumulator()]))
|
|
250
|
+
const known = new Set(profiles.map(profile => profile.name))
|
|
251
|
+
const candidates = await collectCandidates()
|
|
252
|
+
const seenSessions = new Set<string>()
|
|
253
|
+
let totalBytes = 0
|
|
254
|
+
let meetingsScanned = 0
|
|
255
|
+
let sidecarsSkipped = 0
|
|
256
|
+
let truncated = candidates.truncated
|
|
257
|
+
let unresolvedSegments = 0
|
|
258
|
+
const unresolvedMeetings = new Set<string>()
|
|
259
|
+
|
|
260
|
+
for (let i = 0; i < candidates.rows.length; i++) {
|
|
261
|
+
if (i > 0 && i % 8 === 0) await yieldToEventLoop()
|
|
262
|
+
const candidate = candidates.rows[i]
|
|
263
|
+
try {
|
|
264
|
+
const link = await lstat(candidate.sidecarPath)
|
|
265
|
+
if (!link.isFile() || link.isSymbolicLink() || link.size <= 0 || link.size > MAX_SIDECAR_BYTES) {
|
|
266
|
+
sidecarsSkipped++
|
|
267
|
+
continue
|
|
268
|
+
}
|
|
269
|
+
if (totalBytes + link.size > MAX_TOTAL_BYTES) {
|
|
270
|
+
truncated = true
|
|
271
|
+
break
|
|
272
|
+
}
|
|
273
|
+
const real = await realpath(candidate.sidecarPath)
|
|
274
|
+
if (dirname(real) !== dirname(candidate.sidecarPath)) {
|
|
275
|
+
sidecarsSkipped++
|
|
276
|
+
continue
|
|
277
|
+
}
|
|
278
|
+
totalBytes += link.size
|
|
279
|
+
const raw = JSON.parse(await readFile(real, 'utf8')) as Record<string, unknown> | ReviewChunk[]
|
|
280
|
+
const chunks = Array.isArray(raw) ? raw : raw.chunks
|
|
281
|
+
const sessionId = Array.isArray(raw) ? '' : String(raw.sessionId ?? '')
|
|
282
|
+
if (!Array.isArray(chunks) || !SESSION.test(sessionId) || seenSessions.has(sessionId)) {
|
|
283
|
+
sidecarsSkipped++
|
|
284
|
+
continue
|
|
285
|
+
}
|
|
286
|
+
seenSessions.add(sessionId)
|
|
287
|
+
|
|
288
|
+
const record = Array.isArray(raw) ? {} : raw
|
|
289
|
+
const storedTitle = typeof record.title === 'string' && record.title.trim()
|
|
290
|
+
? record.title.trim().slice(0, 180)
|
|
291
|
+
: cleanTitle('', basename(candidate.sidecarPath))
|
|
292
|
+
const title = await boundedMarkdownTitle(
|
|
293
|
+
candidate.markdownPath,
|
|
294
|
+
storedTitle,
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
const review = reviewMeetingSpeakers(chunks as ReviewChunk[], {
|
|
298
|
+
owner,
|
|
299
|
+
phrasesPerVoice: 1,
|
|
300
|
+
confirmed: confirmedLabels(sessionId),
|
|
301
|
+
durationMs: typeof record.durationMs === 'number' ? record.durationMs : undefined,
|
|
302
|
+
batchSegments: Array.isArray(record.batchSegments)
|
|
303
|
+
? record.batchSegments as SpeakerWordSegment[]
|
|
304
|
+
: undefined,
|
|
305
|
+
})
|
|
306
|
+
meetingsScanned++
|
|
307
|
+
|
|
308
|
+
for (const voice of review.voices) {
|
|
309
|
+
if (isUnattributed(voice.label)) {
|
|
310
|
+
unresolvedSegments += voice.segments
|
|
311
|
+
unresolvedMeetings.add(sessionId)
|
|
312
|
+
continue
|
|
313
|
+
}
|
|
314
|
+
if (!known.has(voice.label)) continue
|
|
315
|
+
const acc = byName.get(voice.label)!
|
|
316
|
+
const needsReview = !voice.nameAsserted
|
|
317
|
+
if (needsReview) {
|
|
318
|
+
acc.candidateSegments += voice.segments
|
|
319
|
+
acc.candidateSpeakingMs += voice.speakingMs
|
|
320
|
+
acc.reviewMeetings.add(sessionId)
|
|
321
|
+
} else {
|
|
322
|
+
acc.assertedSegments += voice.segments
|
|
323
|
+
acc.assertedSpeakingMs += voice.speakingMs
|
|
324
|
+
acc.assertedMeetings.add(sessionId)
|
|
325
|
+
}
|
|
326
|
+
acc.speakingTimeSources[review.speakingTimeSource] += voice.speakingMs
|
|
327
|
+
acc.reliabilityCounts[voice.reliability] += voice.segments
|
|
328
|
+
if (!acc.firstSeen || candidate.date < acc.firstSeen) acc.firstSeen = candidate.date
|
|
329
|
+
if (!acc.lastSeen || candidate.date > acc.lastSeen) acc.lastSeen = candidate.date
|
|
330
|
+
if (voice.meanSimilarity != null) {
|
|
331
|
+
acc.observedWeighted += voice.meanSimilarity * voice.segments
|
|
332
|
+
acc.observedSegments += voice.segments
|
|
333
|
+
}
|
|
334
|
+
acc.appearances.push({
|
|
335
|
+
sessionId,
|
|
336
|
+
title,
|
|
337
|
+
date: candidate.date,
|
|
338
|
+
source: candidate.source,
|
|
339
|
+
mutable: candidate.mutable,
|
|
340
|
+
segments: voice.segments,
|
|
341
|
+
speakingMs: voice.speakingMs,
|
|
342
|
+
speakingTimeSource: review.speakingTimeSource,
|
|
343
|
+
observedMatch: voice.meanSimilarity,
|
|
344
|
+
reliability: voice.reliability,
|
|
345
|
+
confirmedByHuman: voice.confirmedByHuman,
|
|
346
|
+
needsReview,
|
|
347
|
+
})
|
|
348
|
+
}
|
|
349
|
+
} catch {
|
|
350
|
+
sidecarsSkipped++
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
const directory = profiles.map(profile => {
|
|
355
|
+
const acc = byName.get(profile.name) ?? emptyAccumulator()
|
|
356
|
+
const appearances = acc.appearances
|
|
357
|
+
.sort((a, b) => b.date.localeCompare(a.date) || b.segments - a.segments)
|
|
358
|
+
.slice(0, MAX_APPEARANCES_PER_VOICE)
|
|
359
|
+
return {
|
|
360
|
+
name: profile.name,
|
|
361
|
+
isOwner: profile.name === owner,
|
|
362
|
+
embeddings: profile.embeddings.length,
|
|
363
|
+
sources: sourceCounts(profile),
|
|
364
|
+
sourcesAligned: (profile.sources?.length ?? 0) === profile.embeddings.length,
|
|
365
|
+
assertedSegments: acc.assertedSegments,
|
|
366
|
+
candidateSegments: acc.candidateSegments,
|
|
367
|
+
assertedSpeakingMs: acc.assertedSpeakingMs,
|
|
368
|
+
candidateSpeakingMs: acc.candidateSpeakingMs,
|
|
369
|
+
speakingTimeSources: acc.speakingTimeSources,
|
|
370
|
+
meetingCount: acc.assertedMeetings.size,
|
|
371
|
+
reviewMeetingCount: acc.reviewMeetings.size,
|
|
372
|
+
observedMatch: acc.observedSegments > 0
|
|
373
|
+
? Math.round((acc.observedWeighted / acc.observedSegments) * 1_000) / 1_000
|
|
374
|
+
: null,
|
|
375
|
+
observedMatchSegments: acc.observedSegments,
|
|
376
|
+
reliabilityCounts: acc.reliabilityCounts,
|
|
377
|
+
firstSeen: acc.firstSeen,
|
|
378
|
+
lastSeen: acc.lastSeen,
|
|
379
|
+
appearances,
|
|
380
|
+
}
|
|
381
|
+
}).sort((a, b) => {
|
|
382
|
+
const attentionA = a.reviewMeetingCount > 0 || !a.sourcesAligned ? 1 : 0
|
|
383
|
+
const attentionB = b.reviewMeetingCount > 0 || !b.sourcesAligned ? 1 : 0
|
|
384
|
+
return attentionB - attentionA || b.meetingCount - a.meetingCount || a.name.localeCompare(b.name)
|
|
385
|
+
})
|
|
386
|
+
|
|
387
|
+
return {
|
|
388
|
+
schemaVersion: 1,
|
|
389
|
+
generatedAt: new Date().toISOString(),
|
|
390
|
+
owner,
|
|
391
|
+
profileCount: profiles.length,
|
|
392
|
+
totalEmbeddings: profiles.reduce((sum, profile) => sum + profile.embeddings.length, 0),
|
|
393
|
+
meetingsScanned,
|
|
394
|
+
sidecarsSkipped,
|
|
395
|
+
truncated,
|
|
396
|
+
unresolvedMeetings: unresolvedMeetings.size,
|
|
397
|
+
unresolvedSegments,
|
|
398
|
+
profiles: directory,
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
export async function getVoiceDirectorySnapshot(force = false): Promise<VoiceDirectorySnapshot> {
|
|
403
|
+
if (!force && cached && Date.now() - cached.at < CACHE_MS) return cached.snapshot
|
|
404
|
+
if (!building) {
|
|
405
|
+
building = buildVoiceDirectorySnapshot()
|
|
406
|
+
.then(snapshot => {
|
|
407
|
+
cached = { at: Date.now(), snapshot }
|
|
408
|
+
return snapshot
|
|
409
|
+
})
|
|
410
|
+
.finally(() => { building = null })
|
|
411
|
+
}
|
|
412
|
+
return building
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/** Test and post-mutation hook. */
|
|
416
|
+
export function invalidateVoiceDirectory(): void {
|
|
417
|
+
cached = null
|
|
418
|
+
}
|
|
@@ -124,6 +124,34 @@ async function cleanOutboundDictation(text: string, opts: AutoCleanRequest & { s
|
|
|
124
124
|
}
|
|
125
125
|
}
|
|
126
126
|
|
|
127
|
+
/** Finalize already-transcribed phone text through the same glossary + optional
|
|
128
|
+
* AI cleanup used by server-owned prompt drafts. This route accepts text only:
|
|
129
|
+
* Moonshine audio and rolling preview audio never leave the phone. */
|
|
130
|
+
promptDraftsRouter.post('/dictation/finalize', async (req, res) => {
|
|
131
|
+
const text = typeof req.body?.text === 'string' ? req.body.text.trim() : ''
|
|
132
|
+
const surface = req.body?.surface
|
|
133
|
+
const model = req.body?.autocleanModel
|
|
134
|
+
if (!text) return res.status(400).json({ error: 'text_required' })
|
|
135
|
+
if (text.length > AUTOCLEAN_MAX_CHARS) {
|
|
136
|
+
return res.status(413).json({ error: 'text_too_large', maxChars: AUTOCLEAN_MAX_CHARS })
|
|
137
|
+
}
|
|
138
|
+
if (surface !== 'message' && surface !== 'meeting') {
|
|
139
|
+
return res.status(400).json({ error: 'invalid_surface' })
|
|
140
|
+
}
|
|
141
|
+
if (model !== undefined && model !== 'haiku' && model !== 'sonnet') {
|
|
142
|
+
return res.status(400).json({ error: 'invalid_autoclean_model' })
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const abort = new AbortController()
|
|
146
|
+
res.on('close', () => { if (!res.writableEnded) abort.abort() })
|
|
147
|
+
const finalText = await cleanOutboundDictation(text, {
|
|
148
|
+
enabled: true,
|
|
149
|
+
model,
|
|
150
|
+
signal: abort.signal,
|
|
151
|
+
})
|
|
152
|
+
res.json({ text: finalText, surface, polished: finalText !== text })
|
|
153
|
+
})
|
|
154
|
+
|
|
127
155
|
async function readRawBody(req: AsyncIterable<Buffer | Uint8Array | string>): Promise<Buffer> {
|
|
128
156
|
const chunks: Buffer[] = []
|
|
129
157
|
let total = 0
|
package/server/routes/voice.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { dataPath } from '../lib/data-dir.js'
|
|
|
12
12
|
import { purgeSpeakerCalibrationRows, relabelSpeakerCalibrationRows } from '../lib/speaker-calibration-log.js'
|
|
13
13
|
import { trainingSourceFor } from '../lib/training-audio-provenance.js'
|
|
14
14
|
import { sendAudioFile } from '../lib/send-audio.js'
|
|
15
|
+
import { getVoiceDirectorySnapshot, invalidateVoiceDirectory } from '../lib/voice-directory.js'
|
|
15
16
|
|
|
16
17
|
// These MUST match the writer in transcribe-stream.ts, which saves under
|
|
17
18
|
// dataPath(). They previously resolved relative to __dirname — i.e. inside the
|
|
@@ -56,6 +57,7 @@ voiceRouter.post('/voice/enroll', async (req, res) => {
|
|
|
56
57
|
}
|
|
57
58
|
|
|
58
59
|
const result = enrollSpeaker(name, audioBuffer)
|
|
60
|
+
invalidateVoiceDirectory()
|
|
59
61
|
res.json(result)
|
|
60
62
|
} catch (err: unknown) {
|
|
61
63
|
res.status(500).json({ success: false, error: errMsg(err) })
|
|
@@ -532,6 +534,35 @@ voiceRouter.get('/voice/profiles', (_req, res) => {
|
|
|
532
534
|
}
|
|
533
535
|
})
|
|
534
536
|
|
|
537
|
+
// GET /api/voice/directory — enrolled identities plus bounded cross-meeting
|
|
538
|
+
// evidence. Confidence belongs to an observed match, not to a person, so the
|
|
539
|
+
// contract deliberately calls the aggregate `observedMatch` and includes its
|
|
540
|
+
// segment basis. Ext/Unidentified clusters stay in corpus totals rather than
|
|
541
|
+
// being promoted into fictional global people.
|
|
542
|
+
voiceRouter.get('/voice/directory', async (req, res) => {
|
|
543
|
+
res.set('Cache-Control', 'private, no-store')
|
|
544
|
+
try {
|
|
545
|
+
const requestedLimit = Number.parseInt(String(req.query.limit ?? '100'), 10)
|
|
546
|
+
const requestedOffset = Number.parseInt(String(req.query.offset ?? '0'), 10)
|
|
547
|
+
const limit = Math.min(Math.max(Number.isFinite(requestedLimit) ? requestedLimit : 100, 1), 100)
|
|
548
|
+
const offset = Math.max(Number.isFinite(requestedOffset) ? requestedOffset : 0, 0)
|
|
549
|
+
const snapshot = await getVoiceDirectorySnapshot(req.query.refresh === '1')
|
|
550
|
+
res.json({
|
|
551
|
+
...snapshot,
|
|
552
|
+
profiles: snapshot.profiles.slice(offset, offset + limit),
|
|
553
|
+
offset,
|
|
554
|
+
limit,
|
|
555
|
+
hasMore: offset + limit < snapshot.profiles.length,
|
|
556
|
+
})
|
|
557
|
+
} catch (err: unknown) {
|
|
558
|
+
res.status(503).json({
|
|
559
|
+
error: 'Voice directory is temporarily unavailable',
|
|
560
|
+
reason: 'voice_directory_unavailable',
|
|
561
|
+
detail: errMsg(err).slice(0, 240),
|
|
562
|
+
})
|
|
563
|
+
}
|
|
564
|
+
})
|
|
565
|
+
|
|
535
566
|
// POST /api/voice/merge-profiles — fold two names for one person together.
|
|
536
567
|
// Body: { into, from: string[]|string, confirm: true, dryRun?, force? }
|
|
537
568
|
//
|
|
@@ -605,6 +636,7 @@ voiceRouter.post('/voice/merge-profiles', (req, res) => {
|
|
|
605
636
|
}
|
|
606
637
|
}
|
|
607
638
|
|
|
639
|
+
if (!dryRun) invalidateVoiceDirectory()
|
|
608
640
|
res.json({ ...report, dryRun, forced: force, calibrationRowsRelabeled: calibration })
|
|
609
641
|
} catch (err: unknown) {
|
|
610
642
|
res.status(500).json({ error: errMsg(err) })
|
|
@@ -683,6 +715,7 @@ voiceRouter.post('/voice/delete-person', (req, res) => {
|
|
|
683
715
|
|
|
684
716
|
// 3. Calibration rows (the name appears in every row).
|
|
685
717
|
const calibration = purgeSpeakerCalibrationRows(CALIBRATION_LOG, target)
|
|
718
|
+
invalidateVoiceDirectory()
|
|
686
719
|
|
|
687
720
|
res.json({
|
|
688
721
|
name: target,
|
|
@@ -44,6 +44,33 @@ export interface MediaAttachmentRef {
|
|
|
44
44
|
* query resolution, and the phone composer. */
|
|
45
45
|
export const MAX_ATTACHMENTS_PER_PROMPT = 5
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Sanity bounds for the PARSER, not policy.
|
|
49
|
+
*
|
|
50
|
+
* `parseMediaAttachmentRef` is a whitelist that drops any field failing its
|
|
51
|
+
* bound, and every field here is optional — so a value the writer sets correctly
|
|
52
|
+
* and the parser rejects vanishes with no error anywhere. That is not theoretical:
|
|
53
|
+
* both of these were live defects found 2026-08-11.
|
|
54
|
+
*
|
|
55
|
+
* - `bytes` was hardcoded `64 * 1024 * 1024` while the server had already moved
|
|
56
|
+
* the video ceiling to 100 MiB and chunked uploads to 2 GiB, so every video
|
|
57
|
+
* over 64 MiB silently lost its byte count.
|
|
58
|
+
* - `frameCount` was hardcoded `8` while VIDEO_SUMMARY_FRAMES_MAX went to 16, so
|
|
59
|
+
* every video over ~90 seconds silently lost its frame count.
|
|
60
|
+
*
|
|
61
|
+
* These are deliberately GENEROUS: their job is to reject garbage from outside,
|
|
62
|
+
* while the real limits are enforced at ingest (rich-media-safety.ts) and
|
|
63
|
+
* advertised on /api/health. A parser bound that doubles as policy is how the
|
|
64
|
+
* policy ends up in two places and drifts.
|
|
65
|
+
*
|
|
66
|
+
* `shared/` must not import from `server/`, so these mirror the server ceilings
|
|
67
|
+
* and are pinned to them by media-attachment-bounds.test.ts — the same pattern as
|
|
68
|
+
* ADVERTISED_VIDEO_COMPRESSION_LABEL. Change one, the test fails.
|
|
69
|
+
*/
|
|
70
|
+
export const MAX_PLAUSIBLE_MEDIA_BYTES = 2 * 1024 * 1024 * 1024 // mirrors MAX_CHUNKED_MEDIA_BYTES
|
|
71
|
+
export const MAX_PARSED_VIDEO_FRAMES = 16 // mirrors VIDEO_SUMMARY_FRAMES_MAX
|
|
72
|
+
export const MAX_PARSED_DURATION_MS = 24 * 60 * 60_000 // a day; ingest enforces the real cap
|
|
73
|
+
|
|
47
74
|
// ── Media IDs ────────────────────────────────────────────────────────────────
|
|
48
75
|
// One strict generated format, one strict validator. The id builds filesystem
|
|
49
76
|
// paths on the server, so the validator rejects anything that isn't exactly
|
|
@@ -108,13 +135,13 @@ export function parseMediaAttachmentRef(raw: unknown): MediaAttachmentRef | null
|
|
|
108
135
|
// Do not add category to legacy image refs that never carried it. Their
|
|
109
136
|
// canonical JSON is part of persisted durable-query fingerprints.
|
|
110
137
|
if (r.category === inferredCategory) ref.category = inferredCategory
|
|
111
|
-
if (typeof r.bytes === 'number' && Number.isSafeInteger(r.bytes) && r.bytes >= 0 && r.bytes <=
|
|
138
|
+
if (typeof r.bytes === 'number' && Number.isSafeInteger(r.bytes) && r.bytes >= 0 && r.bytes <= MAX_PLAUSIBLE_MEDIA_BYTES) {
|
|
112
139
|
ref.bytes = r.bytes
|
|
113
140
|
}
|
|
114
|
-
if (typeof r.durationMs === 'number' && Number.isSafeInteger(r.durationMs) && r.durationMs >= 0 && r.durationMs <=
|
|
141
|
+
if (typeof r.durationMs === 'number' && Number.isSafeInteger(r.durationMs) && r.durationMs >= 0 && r.durationMs <= MAX_PARSED_DURATION_MS) {
|
|
115
142
|
ref.durationMs = r.durationMs
|
|
116
143
|
}
|
|
117
|
-
if (typeof r.frameCount === 'number' && Number.isSafeInteger(r.frameCount) && r.frameCount >= 0 && r.frameCount <=
|
|
144
|
+
if (typeof r.frameCount === 'number' && Number.isSafeInteger(r.frameCount) && r.frameCount >= 0 && r.frameCount <= MAX_PARSED_VIDEO_FRAMES) {
|
|
118
145
|
ref.frameCount = r.frameCount
|
|
119
146
|
}
|
|
120
147
|
if (typeof r.textChars === 'number' && Number.isSafeInteger(r.textChars) && r.textChars >= 0 && r.textChars <= 100_000) {
|