@gotcos/glasses-server 6.45.2 → 6.45.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/package.json +1 -1
- package/server/index.ts +2 -0
- package/server/lib/archive.ts +2 -1
- package/server/lib/conversation.ts +48 -4
- package/server/lib/embedding-eviction.ts +19 -0
- package/server/lib/held-voice-groups.ts +752 -0
- package/server/lib/prompt-tail-guard.ts +257 -0
- package/server/lib/recent-messages.ts +56 -0
- package/server/lib/speaker-embeddings.ts +8 -1
- package/server/lib/training-audio-provenance.ts +1 -1
- package/server/lib/transcribe-audio.ts +9 -1
- package/server/lib/vad-silero.ts +8 -3
- package/server/lib/voice-enrolment-selection.ts +31 -5
- package/server/lib/whisper-local.ts +12 -6
- package/server/lib/workspace-skills.ts +209 -0
- package/server/routes/archive.ts +3 -0
- package/server/routes/prompt-drafts.ts +13 -2
- package/server/routes/sessions.ts +25 -22
- package/server/routes/skills.ts +13 -0
- package/server/routes/transcribe-stream.ts +5 -3
- package/server/routes/voice.ts +139 -1
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
// Catalog of skills the glasses agent can actually run: the selected COS
|
|
2
|
+
// workspace plus the user's global Claude skills. Labels only — never paths.
|
|
3
|
+
//
|
|
4
|
+
// Walk matches skill_sync.py: `.agents/skills` is canonical (nested folders
|
|
5
|
+
// flatten with `:`), `.claude/skills` is a one-level mirror, `source-command-*`
|
|
6
|
+
// is excluded as a generated duplicate. User skills live in `~/.claude/skills`.
|
|
7
|
+
|
|
8
|
+
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
|
|
9
|
+
import { homedir } from 'node:os'
|
|
10
|
+
import { basename, join, relative } from 'node:path'
|
|
11
|
+
import { COS_SCRIPTS_DIR } from './python-bridge.js'
|
|
12
|
+
import { resolveProviderWorkDir } from './launch-dir.js'
|
|
13
|
+
|
|
14
|
+
export const SKILLS_CATALOG_SCHEMA = 1
|
|
15
|
+
export const SKILLS_CATALOG_MAX = 200
|
|
16
|
+
const SKILL_FILE_MAX_BYTES = 8_192
|
|
17
|
+
const SKILL_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/
|
|
18
|
+
const EXCLUDE_PREFIXES = ['source-command-']
|
|
19
|
+
|
|
20
|
+
export interface WorkspaceSkill {
|
|
21
|
+
name: string
|
|
22
|
+
slash: string
|
|
23
|
+
description: string
|
|
24
|
+
group: string
|
|
25
|
+
where: 'workspace' | 'user'
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface WorkspaceSkillsCatalog {
|
|
29
|
+
schemaVersion: number
|
|
30
|
+
skills: WorkspaceSkill[]
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function parseSkillFrontmatter(text: string): Record<string, string> {
|
|
34
|
+
const src = text.replace(/^\uFEFF/, '')
|
|
35
|
+
if (!src.startsWith('---')) return {}
|
|
36
|
+
const end = src.indexOf('\n---', 3)
|
|
37
|
+
if (end < 0) return {}
|
|
38
|
+
const block = src.slice(3, end).replace(/^\r?\n/, '')
|
|
39
|
+
const out: Record<string, string> = {}
|
|
40
|
+
const lines = block.split(/\r?\n/)
|
|
41
|
+
let i = 0
|
|
42
|
+
while (i < lines.length) {
|
|
43
|
+
const line = lines[i]
|
|
44
|
+
const match = /^([A-Za-z_][\w-]*):\s*(.*)$/.exec(line)
|
|
45
|
+
if (!match) { i += 1; continue }
|
|
46
|
+
const key = match[1]
|
|
47
|
+
let raw = match[2].trim()
|
|
48
|
+
if (raw === '>' || raw === '>-' || raw === '|' || raw === '|-') {
|
|
49
|
+
const parts: string[] = []
|
|
50
|
+
i += 1
|
|
51
|
+
while (i < lines.length && /^\s+\S/.test(lines[i]) && !/^[A-Za-z_][\w-]*:/.test(lines[i])) {
|
|
52
|
+
parts.push(lines[i].trim())
|
|
53
|
+
i += 1
|
|
54
|
+
}
|
|
55
|
+
out[key] = unquote(parts.join(' '))
|
|
56
|
+
continue
|
|
57
|
+
}
|
|
58
|
+
out[key] = unquote(raw)
|
|
59
|
+
i += 1
|
|
60
|
+
}
|
|
61
|
+
return out
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function unquote(value: string): string {
|
|
65
|
+
const trimmed = value.trim()
|
|
66
|
+
if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
|
|
67
|
+
return trimmed.slice(1, -1).replace(/\\"/g, '"').replace(/\\'/g, "'").trim()
|
|
68
|
+
}
|
|
69
|
+
return trimmed
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function excludedName(name: string): boolean {
|
|
73
|
+
return EXCLUDE_PREFIXES.some(prefix => name === prefix.slice(0, -1) || name.startsWith(prefix))
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function skillNameFromRel(rel: string): string | null {
|
|
77
|
+
const parts = rel.split(/[/\\]/).filter(part => part && part !== '.')
|
|
78
|
+
if (parts.length === 0 || parts.some(part => part.startsWith('.') || excludedName(part))) return null
|
|
79
|
+
const name = parts.join(':')
|
|
80
|
+
return SKILL_NAME_RE.test(name) ? name : null
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function readSkillFile(path: string): { name?: string; description: string } | null {
|
|
84
|
+
let stat
|
|
85
|
+
try { stat = statSync(path) } catch { return null }
|
|
86
|
+
if (!stat.isFile() || stat.size <= 0) return null
|
|
87
|
+
const bytes = Math.min(stat.size, SKILL_FILE_MAX_BYTES)
|
|
88
|
+
let text: string
|
|
89
|
+
try {
|
|
90
|
+
text = readFileSync(path, { encoding: 'utf8' }).slice(0, bytes)
|
|
91
|
+
} catch {
|
|
92
|
+
return null
|
|
93
|
+
}
|
|
94
|
+
const meta = parseSkillFrontmatter(text)
|
|
95
|
+
const description = (meta.description || '').replace(/\s+/g, ' ').trim().slice(0, 160)
|
|
96
|
+
const name = meta.name && SKILL_NAME_RE.test(meta.name) && !excludedName(meta.name) ? meta.name : undefined
|
|
97
|
+
return { name, description }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function walkImmediateSkills(root: string): Array<{ rel: string; path: string }> {
|
|
101
|
+
let entries
|
|
102
|
+
try { entries = readdirSync(root, { withFileTypes: true }) } catch { return [] }
|
|
103
|
+
const found: Array<{ rel: string; path: string }> = []
|
|
104
|
+
for (const entry of entries) {
|
|
105
|
+
if (!entry.isDirectory() || entry.name.startsWith('.')) continue
|
|
106
|
+
const path = join(root, entry.name, 'SKILL.md')
|
|
107
|
+
if (existsSync(path)) found.push({ rel: entry.name, path })
|
|
108
|
+
}
|
|
109
|
+
return found
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function walkNestedSkills(root: string): Array<{ rel: string; path: string }> {
|
|
113
|
+
const found: Array<{ rel: string; path: string }> = []
|
|
114
|
+
const stack = [root]
|
|
115
|
+
while (stack.length) {
|
|
116
|
+
const dir = stack.pop()!
|
|
117
|
+
let entries
|
|
118
|
+
try { entries = readdirSync(dir, { withFileTypes: true }) } catch { continue }
|
|
119
|
+
for (const entry of entries) {
|
|
120
|
+
if (entry.name.startsWith('.') || entry.name === 'node_modules') continue
|
|
121
|
+
const full = join(dir, entry.name)
|
|
122
|
+
if (entry.isDirectory()) {
|
|
123
|
+
stack.push(full)
|
|
124
|
+
continue
|
|
125
|
+
}
|
|
126
|
+
if (entry.isFile() && entry.name === 'SKILL.md') {
|
|
127
|
+
const rel = relative(root, dir)
|
|
128
|
+
if (rel && rel !== '.') found.push({ rel, path: full })
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return found
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function groupFor(name: string, where: WorkspaceSkill['where']): string {
|
|
136
|
+
const colon = name.indexOf(':')
|
|
137
|
+
if (colon > 0) return name.slice(0, colon).replace(/[-_]/g, ' ').toUpperCase()
|
|
138
|
+
return where === 'user' ? 'USER' : 'WORKSPACE'
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function addSkill(
|
|
142
|
+
seen: Map<string, WorkspaceSkill>,
|
|
143
|
+
rel: string,
|
|
144
|
+
path: string,
|
|
145
|
+
where: WorkspaceSkill['where'],
|
|
146
|
+
): void {
|
|
147
|
+
if (seen.size >= SKILLS_CATALOG_MAX) return
|
|
148
|
+
const fromDir = skillNameFromRel(rel)
|
|
149
|
+
if (!fromDir) return
|
|
150
|
+
const parsed = readSkillFile(path)
|
|
151
|
+
if (!parsed) return
|
|
152
|
+
// Directory name is the loader identity. Frontmatter `name` is a label; if it
|
|
153
|
+
// disagrees with the folder (common on generated mirrors), keep the folder.
|
|
154
|
+
const name = parsed.name && basename(rel) === parsed.name ? parsed.name : fromDir
|
|
155
|
+
if (seen.has(name) || excludedName(name)) return
|
|
156
|
+
seen.set(name, {
|
|
157
|
+
name,
|
|
158
|
+
slash: `/${name}`,
|
|
159
|
+
description: parsed.description,
|
|
160
|
+
group: groupFor(name, where),
|
|
161
|
+
where,
|
|
162
|
+
})
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function listWorkspaceSkills(options: {
|
|
166
|
+
workDir?: string
|
|
167
|
+
home?: string
|
|
168
|
+
} = {}): WorkspaceSkill[] {
|
|
169
|
+
const workDir = options.workDir ?? resolveProviderWorkDir({ scriptsDir: COS_SCRIPTS_DIR })
|
|
170
|
+
const home = options.home ?? homedir()
|
|
171
|
+
const seen = new Map<string, WorkspaceSkill>()
|
|
172
|
+
|
|
173
|
+
if (workDir) {
|
|
174
|
+
const agents = join(workDir, '.agents', 'skills')
|
|
175
|
+
if (existsSync(agents)) {
|
|
176
|
+
for (const skill of walkNestedSkills(agents)) addSkill(seen, skill.rel, skill.path, 'workspace')
|
|
177
|
+
}
|
|
178
|
+
const claude = join(workDir, '.claude', 'skills')
|
|
179
|
+
if (existsSync(claude)) {
|
|
180
|
+
for (const skill of walkImmediateSkills(claude)) addSkill(seen, skill.rel, skill.path, 'workspace')
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const userClaude = join(home, '.claude', 'skills')
|
|
185
|
+
if (existsSync(userClaude)) {
|
|
186
|
+
for (const skill of walkImmediateSkills(userClaude)) addSkill(seen, skill.rel, skill.path, 'user')
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
return [...seen.values()].sort((a, b) => {
|
|
190
|
+
if (a.group !== b.group) {
|
|
191
|
+
if (a.group === 'WORKSPACE') return -1
|
|
192
|
+
if (b.group === 'WORKSPACE') return 1
|
|
193
|
+
if (a.group === 'USER') return 1
|
|
194
|
+
if (b.group === 'USER') return -1
|
|
195
|
+
return a.group.localeCompare(b.group)
|
|
196
|
+
}
|
|
197
|
+
return a.name.localeCompare(b.name)
|
|
198
|
+
})
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
export function workspaceSkillsCatalog(options?: {
|
|
202
|
+
workDir?: string
|
|
203
|
+
home?: string
|
|
204
|
+
}): WorkspaceSkillsCatalog {
|
|
205
|
+
return {
|
|
206
|
+
schemaVersion: SKILLS_CATALOG_SCHEMA,
|
|
207
|
+
skills: listWorkspaceSkills(options),
|
|
208
|
+
}
|
|
209
|
+
}
|
package/server/routes/archive.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { Router } from 'express'
|
|
3
3
|
import { listArchiveDateStrings, archiveDir, archiveIndexPath, loadArchive, getArchiveChats, getArchiveDayMessages, appendToArchive } from '../lib/archive.js'
|
|
4
4
|
import { getArchiveChatMessagesNumbered } from './message-ref.js'
|
|
5
|
+
import { ensureArchiveMirrorForDay } from '../lib/conversation.js'
|
|
5
6
|
import { searchArchive, MAX_LIMIT, DEFAULT_LIMIT } from '../lib/archive-search.js'
|
|
6
7
|
import { refreshArchiveIndex } from '../lib/archive-index.js'
|
|
7
8
|
import { getActiveSessions } from '../lib/conversation.js'
|
|
@@ -24,6 +25,8 @@ archiveRouter.param('date', (req, res, next, date) => {
|
|
|
24
25
|
|
|
25
26
|
// GET /api/archive — list all archive dates with summaries
|
|
26
27
|
archiveRouter.get('/archive', async (_req, res) => {
|
|
28
|
+
// 6.45.3 — the listing files yesterday's finished sessions before it answers.
|
|
29
|
+
await ensureArchiveMirrorForDay().catch(() => {})
|
|
27
30
|
// Index-backed. The previous implementation parsed every day file to reach four
|
|
28
31
|
// summary fields; see archive-index.ts for the measurements that killed it.
|
|
29
32
|
const { entries, rebuilt, fromCache } = await refreshArchiveIndex(archiveDir(), archiveIndexPath())
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
transcriptQualityRank,
|
|
17
17
|
type PromptDraftTranscriptRecord,
|
|
18
18
|
} from '../lib/prompt-draft-store.js'
|
|
19
|
+
import { SPEECH_UNKNOWN, guardPromptTail, speechWindowsFromWav } from '../lib/prompt-tail-guard.js'
|
|
19
20
|
import {
|
|
20
21
|
transcribeAudioBuffer,
|
|
21
22
|
resolveTranscribeMode,
|
|
@@ -291,7 +292,15 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
|
|
|
291
292
|
const policy = purpose === 'warm' ? 'local-only' as const : 'automatic' as const
|
|
292
293
|
const result = await transcribeAudioBuffer(audio, { mode, policy })
|
|
293
294
|
if (!isCurrentChunk(draftId, chunkIndex, audio)) return ''
|
|
294
|
-
|
|
295
|
+
// 6.45.4 — the sentence Whisper invents after the speaker stops. Never trims
|
|
296
|
+
// audio; drops a trailing sentence only on the junk lexicon or on silence
|
|
297
|
+
// plus the decoder's own low confidence (prompt-tail-guard.ts).
|
|
298
|
+
const guarded = guardPromptTail({ text: result.text, words: result.words, speech: speechWindowsFromWav(audio) })
|
|
299
|
+
for (const drop of guarded.dropped) {
|
|
300
|
+
console.warn(`[prompt-draft] tail_drop ${draftId}/${chunkIndex} ${purpose}/${mode} (${drop.reason}, start=${drop.startSec ?? '?'}s, lastSpeech=${guarded.lastSpeechEndSec ?? '?'}s, p=${drop.meanProbability?.toFixed(2) ?? '?'}): "${drop.text.slice(0, 100)}"`)
|
|
301
|
+
}
|
|
302
|
+
if (guarded.dropped.length > 0 && !guarded.text.trim()) throw new NoSpeechDetectedError(result.text)
|
|
303
|
+
const text = sanitizeTranscript(draftId, guarded.text)
|
|
295
304
|
const record: PromptDraftTranscriptRecord = {
|
|
296
305
|
text, hash, requestedMode: result.requestedMode, actualQuality: result.actualQuality,
|
|
297
306
|
backend: result.backend, degraded: result.degraded,
|
|
@@ -497,7 +506,9 @@ promptDraftsRouter.post('/prompt-drafts/:draftId/peek', async (req, res) => {
|
|
|
497
506
|
lease.setPhase('active')
|
|
498
507
|
try {
|
|
499
508
|
const result = await transcribeWhisperPreview(audio)
|
|
500
|
-
|
|
509
|
+
// 6.45.4: the same whole-sentence filler rule as the commit path, so an
|
|
510
|
+
// invented closing line never paints on the lens only to vanish later.
|
|
511
|
+
const text = sanitizeTranscript(draftId, guardPromptTail({ text: result.text, speech: SPEECH_UNKNOWN }).text, false)
|
|
501
512
|
if (!text) return
|
|
502
513
|
if (!loadPromptDraftMeta(draftId)) return
|
|
503
514
|
emitDisplay({
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
import { Router } from 'express'
|
|
3
3
|
import { readFileSync } from 'fs'
|
|
4
4
|
import { join } from 'path'
|
|
5
|
-
import { getRecentSessions, getHistory, sessionExists, addContextBreak, endSession, getSessionRaw, getActiveSessions, resolveExchangePairModel } from '../lib/conversation.js'
|
|
5
|
+
import { getRecentSessions, getHistory, sessionExists, addContextBreak, endSession, getSessionRaw, getActiveSessions, resolveExchangePairModel, ensureArchiveMirrorForDay } from '../lib/conversation.js'
|
|
6
6
|
import { buildSessionLogEntry, writeSessionLog } from '../lib/session-log.js'
|
|
7
|
-
import { getArchiveDayMessages } from '../lib/archive.js'
|
|
7
|
+
import { getArchiveDayMessages, listArchiveDateStrings } from '../lib/archive.js'
|
|
8
|
+
import { recentMessagesLimit, selectRecentMessages } from '../lib/recent-messages.js'
|
|
8
9
|
import { localDay } from '../lib/local-day.js'
|
|
9
10
|
import { clearCodexEngineSession } from '../lib/codex-engine-sessions.js'
|
|
10
11
|
import { mergeMediaAttachmentRefs, type MediaAttachmentRef } from '../../shared/media-attachment.js'
|
|
@@ -282,16 +283,23 @@ sessionsRouter.get('/sessions/today/live-chats', (_req, res) => {
|
|
|
282
283
|
res.json({ chats })
|
|
283
284
|
})
|
|
284
285
|
|
|
285
|
-
// GET /api/sessions/today/all-messages —
|
|
286
|
-
//
|
|
287
|
-
//
|
|
288
|
-
//
|
|
289
|
-
|
|
286
|
+
// GET /api/sessions/today/all-messages — the newest messages, live + archived, one window.
|
|
287
|
+
// 6.45.3 — the path keeps its name for the two clients that call it (COS Control's
|
|
288
|
+
// Recent view, the phone's history recovery) but the view is a ROLLING WINDOW of the
|
|
289
|
+
// newest `limit` (default 30, `?limit=` up to 100) across every live session and as many
|
|
290
|
+
// archived days as it takes, not a calendar day. Dedup key is `sessionId|timestamp`
|
|
291
|
+
// (was bare timestamp, which collided on NTP skew or same-ms adds); `sessionId` is
|
|
292
|
+
// always known for live exchanges; archive messages fall back to the archived chat's
|
|
293
|
+
// sessionId via getArchiveDayMessages. `date` stays in the response for compatibility.
|
|
294
|
+
sessionsRouter.get('/sessions/today/all-messages', async (req, res) => {
|
|
295
|
+
// 6.45.3 — first read after midnight files yesterday before answering.
|
|
296
|
+
await ensureArchiveMirrorForDay().catch(() => {})
|
|
290
297
|
const todayDate = localDay()
|
|
298
|
+
const limit = recentMessagesLimit(req.query.limit)
|
|
291
299
|
const activeEra = currentMessageEraState()
|
|
292
300
|
const era = activeEra.era
|
|
293
301
|
|
|
294
|
-
const
|
|
302
|
+
const archivedDay = (date: string) => getArchiveDayMessages(date)
|
|
295
303
|
.filter(m => exchangeBelongsToEra(m, era))
|
|
296
304
|
.map(m => {
|
|
297
305
|
const globalMsgNum = m.globalMsgNum ?? m.no
|
|
@@ -323,8 +331,7 @@ sessionsRouter.get('/sessions/today/all-messages', (_req, res) => {
|
|
|
323
331
|
}> = []
|
|
324
332
|
const liveSessions = getActiveSessions()
|
|
325
333
|
for (const session of liveSessions) {
|
|
326
|
-
|
|
327
|
-
if (sessionDay !== todayDate) continue
|
|
334
|
+
// Every live session, whatever day it last spoke: the window decides, not the calendar.
|
|
328
335
|
for (let i = 0; i < session.exchanges.length; i++) {
|
|
329
336
|
const ex = session.exchanges[i]
|
|
330
337
|
if (ex.role === 'user') {
|
|
@@ -358,17 +365,13 @@ sessionsRouter.get('/sessions/today/all-messages', (_req, res) => {
|
|
|
358
365
|
}
|
|
359
366
|
}
|
|
360
367
|
|
|
361
|
-
//
|
|
362
|
-
|
|
363
|
-
const
|
|
364
|
-
|
|
365
|
-
.
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
seen.add(k)
|
|
369
|
-
return true
|
|
370
|
-
})
|
|
371
|
-
.sort((a, b) => a.timestamp - b.timestamp)
|
|
368
|
+
// Newest `limit` across live sessions and archived days (newest day first, read
|
|
369
|
+
// only while the window is short), dedup by (sessionId, timestamp), chronological.
|
|
370
|
+
const merged = selectRecentMessages(
|
|
371
|
+
liveMessages as Array<(typeof liveMessages)[number] | ReturnType<typeof archivedDay>[number]>,
|
|
372
|
+
listArchiveDateStrings().map(date => () => archivedDay(date)),
|
|
373
|
+
limit,
|
|
374
|
+
)
|
|
372
375
|
|
|
373
|
-
res.json({ messages: merged, date: todayDate })
|
|
376
|
+
res.json({ messages: merged, date: todayDate, window: { kind: 'recent', limit } })
|
|
374
377
|
})
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// GET /api/skills — slash commands the glasses agent can run from this Mac.
|
|
2
|
+
// Workspace `.agents/skills` (canonical) + `.claude/skills` + `~/.claude/skills`.
|
|
3
|
+
// Authenticated by the /api token middleware. Paths never leave the box.
|
|
4
|
+
|
|
5
|
+
import { Router } from 'express'
|
|
6
|
+
import { workspaceSkillsCatalog } from '../lib/workspace-skills.js'
|
|
7
|
+
|
|
8
|
+
export const skillsRouter = Router()
|
|
9
|
+
|
|
10
|
+
skillsRouter.get('/skills', (_req, res) => {
|
|
11
|
+
res.set('Cache-Control', 'private, no-store')
|
|
12
|
+
res.json(workspaceSkillsCatalog())
|
|
13
|
+
})
|
|
@@ -39,7 +39,7 @@ import { errMsg } from '../lib/utils.js'
|
|
|
39
39
|
import { transcribeLocal, applyCorrections, type WhisperWord } from '../lib/whisper-local.js'
|
|
40
40
|
import { enhanceAudio } from '../lib/audio-enhance.js'
|
|
41
41
|
import { trimSilence, isSileroAvailable } from '../lib/vad-silero.js'
|
|
42
|
-
import { identifySpeaker, isEmbeddingAvailable, autoEnroll, getEmbeddingCount } from '../lib/speaker-embeddings.js'
|
|
42
|
+
import { identifySpeaker, isEmbeddingAvailable, autoEnroll, getEmbeddingCount, AUTO_ENROLL_CANDIDATE_SIMILARITY } from '../lib/speaker-embeddings.js'
|
|
43
43
|
import {
|
|
44
44
|
assertOpenAIWhisperBudget,
|
|
45
45
|
recordOpenAIWhisperUsage,
|
|
@@ -1157,7 +1157,9 @@ setInterval(() => {
|
|
|
1157
1157
|
const dirPath = resolve(EXT_AUDIO_DIR, dir)
|
|
1158
1158
|
try {
|
|
1159
1159
|
const files = readdirSync(dirPath)
|
|
1160
|
-
|
|
1160
|
+
// 6.45.4: an emptied session (every held sample named or discarded)
|
|
1161
|
+
// frees its per-session cap too, so the next stranger can be held.
|
|
1162
|
+
if (files.length === 0) { rmSync(dirPath, { recursive: true, force: true }); extAudioCounts.delete(dir); continue }
|
|
1161
1163
|
const { mtimeMs } = statSync(resolve(dirPath, files[0]))
|
|
1162
1164
|
if (Date.now() - mtimeMs > EXT_AUDIO_TTL_MS) {
|
|
1163
1165
|
rmSync(dirPath, { recursive: true, force: true })
|
|
@@ -1897,7 +1899,7 @@ function identifyChunkSpeaker(audioBuffer: Buffer, sessionId: string, chunkIndex
|
|
|
1897
1899
|
console.log(`[speaker] Embedding: ${speaker} vs Amplitude: ${clientSpeaker} (sim: ${embeddingResult.similarity.toFixed(2)})`)
|
|
1898
1900
|
}
|
|
1899
1901
|
|
|
1900
|
-
if (embeddingResult.similarity >=
|
|
1902
|
+
if (embeddingResult.similarity >= AUTO_ENROLL_CANDIDATE_SIMILARITY && speaker !== 'Ext') {
|
|
1901
1903
|
const enrollResult = autoEnroll(speaker, audioBuffer, embeddingResult.similarity, sessionId)
|
|
1902
1904
|
if (enrollResult.enrolled) {
|
|
1903
1905
|
console.log(`[speaker] Auto-enrolled ${speaker} from G2 mic (sim: ${embeddingResult.similarity.toFixed(3)})`)
|
package/server/routes/voice.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Voice enrollment, status, and multi-speaker training endpoints
|
|
2
2
|
|
|
3
|
-
import { Router } from 'express'
|
|
3
|
+
import express, { Router } from 'express'
|
|
4
4
|
import { errMsg } from '../lib/utils.js'
|
|
5
5
|
import { readdirSync, readFileSync, unlinkSync, existsSync, rmdirSync, rmSync } from 'node:fs'
|
|
6
6
|
import { resolve } from 'node:path'
|
|
@@ -13,9 +13,11 @@ import { dataPath } from '../lib/data-dir.js'
|
|
|
13
13
|
import { purgeSpeakerCalibrationRows, relabelSpeakerCalibrationRows } from '../lib/speaker-calibration-log.js'
|
|
14
14
|
import { trainingSourceFor } from '../lib/training-audio-provenance.js'
|
|
15
15
|
import { sendAudioFile } from '../lib/send-audio.js'
|
|
16
|
+
import { extAudioChunkPath, listExtAudioChunks } from '../lib/meeting-audio-archive.js'
|
|
16
17
|
import { getVoiceDirectorySnapshot, invalidateVoiceDirectory } from '../lib/voice-directory.js'
|
|
17
18
|
import { greedyDiversitySelect } from '../lib/voice-enrolment-selection.js'
|
|
18
19
|
import { fanOutSpeakerRename, type SpeakerRenameFanOut } from '../lib/speaker-rename-fanout.js'
|
|
20
|
+
import { HeldGroupError, discardHeldSamples, enrollHeldGroup, heldVoiceGroups, parseHeldMembers, previewDiscard } from '../lib/held-voice-groups.js'
|
|
19
21
|
import { resolveCosOperationsDir } from '../lib/cos-operations-meetings.js'
|
|
20
22
|
|
|
21
23
|
// These MUST match the writer in transcribe-stream.ts, which saves under
|
|
@@ -336,6 +338,10 @@ voiceRouter.get('/voice/ext-audio', (_req, res) => {
|
|
|
336
338
|
chunks: wavFiles.length,
|
|
337
339
|
ageHours: parseFloat(ageHours),
|
|
338
340
|
expiresIn: `${Math.max(0, 72 - parseFloat(ageHours)).toFixed(1)}h`,
|
|
341
|
+
// 6.45.3 — the chunk indices a reviewer can ask to hear (`?chunk=` on
|
|
342
|
+
// the sample route). Queen, 2026-09-12: the Add-a-voice panel let her
|
|
343
|
+
// name a session but not listen to it; naming is a guess without this.
|
|
344
|
+
chunkIndices: listExtAudioChunks(d.name),
|
|
339
345
|
}
|
|
340
346
|
}).filter(s => s.chunks > 0)
|
|
341
347
|
|
|
@@ -496,6 +502,8 @@ voiceRouter.get('/voice/profiles/:name/sample', (req, res) => {
|
|
|
496
502
|
})
|
|
497
503
|
|
|
498
504
|
// GET /api/voice/ext-audio/:sessionId/sample — hear an unidentified voice.
|
|
505
|
+
// 6.45.3 — `?chunk=<index>` picks one held chunk (indices come from the listing's
|
|
506
|
+
// `chunkIndices`); without it the newest chunk is served, as before.
|
|
499
507
|
voiceRouter.get('/voice/ext-audio/:sessionId/sample', (req, res) => {
|
|
500
508
|
res.set('Cache-Control', 'private, no-store')
|
|
501
509
|
const sessionId = String(req.params.sessionId ?? '')
|
|
@@ -504,6 +512,21 @@ voiceRouter.get('/voice/ext-audio/:sessionId/sample', (req, res) => {
|
|
|
504
512
|
res.status(400).json({ error: 'Invalid sessionId', reason: 'invalid_session_id' })
|
|
505
513
|
return
|
|
506
514
|
}
|
|
515
|
+
const chunkRaw = req.query.chunk
|
|
516
|
+
if (chunkRaw !== undefined) {
|
|
517
|
+
const chunkIndex = Number.parseInt(String(chunkRaw), 10)
|
|
518
|
+
if (!Number.isInteger(chunkIndex) || chunkIndex < 0) {
|
|
519
|
+
res.status(400).json({ error: 'Invalid chunk', reason: 'invalid_chunk' })
|
|
520
|
+
return
|
|
521
|
+
}
|
|
522
|
+
const chunkWav = extAudioChunkPath(sessionId, chunkIndex)
|
|
523
|
+
if (!chunkWav) {
|
|
524
|
+
res.status(404).json({ error: 'No ext-audio retained for that chunk', reason: 'no_ext_audio_chunk' })
|
|
525
|
+
return
|
|
526
|
+
}
|
|
527
|
+
sendAudioFile(res, chunkWav)
|
|
528
|
+
return
|
|
529
|
+
}
|
|
507
530
|
const wav = existsSync(dirPath) ? newestWav(dirPath) : null
|
|
508
531
|
if (!wav) {
|
|
509
532
|
res.status(404).json({
|
|
@@ -515,6 +538,121 @@ voiceRouter.get('/voice/ext-audio/:sessionId/sample', (req, res) => {
|
|
|
515
538
|
sendAudioFile(res, wav)
|
|
516
539
|
})
|
|
517
540
|
|
|
541
|
+
// ── Held voices, grouped (6.45.4) ─────────────────────────────────────────
|
|
542
|
+
//
|
|
543
|
+
// The Add-a-voice panel listed held audio by SESSION. A session is not a voice:
|
|
544
|
+
// one meeting holds several strangers, and one stranger recurs across meetings.
|
|
545
|
+
// These three routes work at the grain a reviewer actually names — a GROUP of
|
|
546
|
+
// samples that sound like one person, across the whole retention window — and
|
|
547
|
+
// let the random artifacts be thrown out instead of named. See
|
|
548
|
+
// lib/held-voice-groups.ts for the rule (mutually coherent at the identifier's
|
|
549
|
+
// own floor) and for why the vectors cost no decode in the ordinary case.
|
|
550
|
+
//
|
|
551
|
+
// Both mutations fail closed like every sibling here: without `confirm: true`
|
|
552
|
+
// they answer 400 `confirmation required` with a preview of exactly what would
|
|
553
|
+
// change, and `dryRun: true` returns that preview as a 200. COS Control sends
|
|
554
|
+
// `confirm` after its own two-click gate.
|
|
555
|
+
|
|
556
|
+
// GET /api/voice/held-groups
|
|
557
|
+
voiceRouter.get('/voice/held-groups', (_req, res) => {
|
|
558
|
+
try {
|
|
559
|
+
res.set('Cache-Control', 'private, no-store')
|
|
560
|
+
res.json(heldVoiceGroups())
|
|
561
|
+
} catch (err: unknown) {
|
|
562
|
+
res.status(500).json({ error: errMsg(err) })
|
|
563
|
+
}
|
|
564
|
+
})
|
|
565
|
+
|
|
566
|
+
function heldGroupFailure(res: express.Response, err: unknown): void {
|
|
567
|
+
if (err instanceof HeldGroupError) {
|
|
568
|
+
res.status(err.status).json({ success: false, error: err.message, reason: err.reason, ...err.details })
|
|
569
|
+
return
|
|
570
|
+
}
|
|
571
|
+
res.status(500).json({ success: false, error: errMsg(err) })
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
// POST /api/voice/held-groups/enroll — name a group (or any set of held
|
|
575
|
+
// samples) as one person. Body: { name, members: [{ sessionId, chunkIndex }],
|
|
576
|
+
// confirm?: true, dryRun?: true }. A name that already has a profile is
|
|
577
|
+
// APPENDED to: "add more fidelity in samples to a given voice". Only the
|
|
578
|
+
// coherent core is written and only its wavs are removed; samples that were
|
|
579
|
+
// not this voice stay held.
|
|
580
|
+
voiceRouter.post('/voice/held-groups/enroll', (req, res) => {
|
|
581
|
+
try {
|
|
582
|
+
const nameCheck = checkSpeakerName(req.body?.name, { ownerLabel: getOwnerSpeakerLabel() })
|
|
583
|
+
if (!nameCheck.ok) {
|
|
584
|
+
return res.status(400).json({ success: false, error: nameCheck.message, reason: nameCheck.reason })
|
|
585
|
+
}
|
|
586
|
+
const name = String(req.body.name).trim()
|
|
587
|
+
const members = parseHeldMembers(req.body?.members)
|
|
588
|
+
const dryRun = req.body?.dryRun === true
|
|
589
|
+
const confirm = req.body?.confirm === true
|
|
590
|
+
if (dryRun || !confirm) {
|
|
591
|
+
const plan = enrollHeldGroup(name, members, { dryRun: true })
|
|
592
|
+
const verb = plan.created ? 'create' : 'add to'
|
|
593
|
+
const summary = `Would ${verb} ${name} from ${plan.coherent} of ${plan.resolved} held sample${plan.resolved === 1 ? '' : 's'}`
|
|
594
|
+
+ (plan.leftBehind.length > 0 ? `, leaving ${plan.leftBehind.length} that do not sound like the same person` : '')
|
|
595
|
+
+ (plan.notReady.length > 0 ? `, with ${plan.notReady.length} still being read` : '')
|
|
596
|
+
+ `, and delete the audio it used.`
|
|
597
|
+
if (dryRun) return res.json({ success: true, ...plan, message: summary })
|
|
598
|
+
return res.status(400).json({
|
|
599
|
+
success: false,
|
|
600
|
+
error: 'confirmation required',
|
|
601
|
+
reason: 'confirmation_required',
|
|
602
|
+
message: `${summary} Pass { confirm: true } to proceed.`,
|
|
603
|
+
preview: plan,
|
|
604
|
+
})
|
|
605
|
+
}
|
|
606
|
+
const result = enrollHeldGroup(name, members)
|
|
607
|
+
invalidateVoiceDirectory()
|
|
608
|
+
const verb = result.created ? 'Created' : 'Added to'
|
|
609
|
+
res.json({
|
|
610
|
+
success: true,
|
|
611
|
+
...result,
|
|
612
|
+
message: result.leftBehind.length > 0
|
|
613
|
+
? `${verb} ${name} from ${result.coherent} of ${result.resolved} samples; ${result.leftBehind.length} did not sound like the same person and stay held.`
|
|
614
|
+
: `${verb} ${name} from ${result.coherent} sample${result.coherent === 1 ? '' : 's'}.`
|
|
615
|
+
+ (result.notReady.length > 0 ? ` ${result.notReady.length} still being read; they stay held.` : ''),
|
|
616
|
+
})
|
|
617
|
+
} catch (err: unknown) {
|
|
618
|
+
heldGroupFailure(res, err)
|
|
619
|
+
}
|
|
620
|
+
})
|
|
621
|
+
|
|
622
|
+
// POST /api/voice/held-groups/discard — throw held samples out without naming
|
|
623
|
+
// them. Body: { members: [{ sessionId, chunkIndex }], confirm?: true, dryRun?: true }.
|
|
624
|
+
// This is how the loose artifacts leave the panel.
|
|
625
|
+
voiceRouter.post('/voice/held-groups/discard', (req, res) => {
|
|
626
|
+
try {
|
|
627
|
+
const members = parseHeldMembers(req.body?.members)
|
|
628
|
+
const dryRun = req.body?.dryRun === true
|
|
629
|
+
const confirm = req.body?.confirm === true
|
|
630
|
+
if (dryRun || !confirm) {
|
|
631
|
+
const preview = previewDiscard(members)
|
|
632
|
+
const summary = `Would discard ${preview.present.length} held sample${preview.present.length === 1 ? '' : 's'}`
|
|
633
|
+
+ (preview.missing.length > 0 ? ` (${preview.missing.length} already gone)` : '') + '.'
|
|
634
|
+
if (dryRun) return res.json({ success: true, dryRun: true, wouldRemove: preview.present.length, missing: preview.missing, message: summary })
|
|
635
|
+
return res.status(400).json({
|
|
636
|
+
success: false,
|
|
637
|
+
error: 'confirmation required',
|
|
638
|
+
reason: 'confirmation_required',
|
|
639
|
+
message: `${summary} Pass { confirm: true } to proceed.`,
|
|
640
|
+
wouldRemove: preview.present.length,
|
|
641
|
+
missing: preview.missing,
|
|
642
|
+
})
|
|
643
|
+
}
|
|
644
|
+
const result = discardHeldSamples(members)
|
|
645
|
+
res.json({
|
|
646
|
+
success: true,
|
|
647
|
+
removed: result.removed.length,
|
|
648
|
+
missing: result.missing,
|
|
649
|
+
message: `Discarded ${result.removed.length} held sample${result.removed.length === 1 ? '' : 's'}.`,
|
|
650
|
+
})
|
|
651
|
+
} catch (err: unknown) {
|
|
652
|
+
heldGroupFailure(res, err)
|
|
653
|
+
}
|
|
654
|
+
})
|
|
655
|
+
|
|
518
656
|
// GET /api/voice/profiles — enrolled people with sample counts and provenance.
|
|
519
657
|
// The review surfaces need to see the store; until now the only window into it
|
|
520
658
|
// was a per-name count, so a misattributed profile was invisible.
|