@gotcos/glasses-server 6.71.0 → 6.71.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +8 -0
- package/CHANGELOG.md +40 -0
- package/README.md +6 -0
- package/docs/asr-canaries.md +103 -0
- package/docs/test-plan-6.71.2.md +111 -0
- package/docs/test-plan-6.71.3.md +30 -0
- package/managed-runtime-contract.json +2 -0
- package/package.json +8 -2
- package/server/lib/context-library-search.ts +3 -2
- package/server/lib/live-cos-ask.ts +73 -16
- package/server/lib/meeting-ask-confidence.ts +16 -0
- package/server/lib/meeting-ask-hq-transcript.ts +40 -0
- package/server/lib/meeting-ask-orientation.ts +47 -0
- package/server/lib/meeting-ask-refinement.ts +26 -0
- package/server/lib/meeting-ask-runtime.ts +132 -16
- package/server/lib/meeting-ask-store.ts +29 -1
- package/server/lib/meeting-ask-transcript-sources.ts +13 -0
- package/server/lib/meeting-ask-transcript.ts +132 -0
- package/server/lib/meeting-batch-transcribe.ts +36 -0
- package/server/lib/meeting-library-search.ts +3 -1
- package/server/lib/whisper-batch-timing.ts +86 -0
- package/server/lib/whisper-local.ts +43 -8
- package/server/routes/health.ts +3 -2
- package/server/routes/meeting-ask.ts +26 -8
- package/shared/meeting-ask.ts +46 -1
|
@@ -356,6 +356,42 @@ export const __progressiveHqTesting = {
|
|
|
356
356
|
},
|
|
357
357
|
}
|
|
358
358
|
|
|
359
|
+
/** Read-only, default-off candidate for in-meeting Ask evidence. Does not
|
|
360
|
+
* rewrite canonical chunks, run a decoder, or include the open tail. A complete
|
|
361
|
+
* audio/config/context chain is required; any invalid checkpoint leaves the
|
|
362
|
+
* remaining live transcript untouched. */
|
|
363
|
+
export function readProgressiveAskRefinements(
|
|
364
|
+
audioDir: string,
|
|
365
|
+
sessionId: string,
|
|
366
|
+
entries: IndexedTranscriptChunk[],
|
|
367
|
+
asrCompletedIndices: number[],
|
|
368
|
+
): BatchResult[] {
|
|
369
|
+
if (process.env.COS_MEETING_ASK_HQ_EVIDENCE !== '1'
|
|
370
|
+
|| process.env.COS_BATCH_HQ_TIMESTAMPS !== '1'
|
|
371
|
+
|| !progressiveMeetingHqEnabled()) return []
|
|
372
|
+
const results: BatchResult[] = []
|
|
373
|
+
let previousText: string | undefined
|
|
374
|
+
for (const segment of segmentTranscriptChunks(entries, 30_000, false)) {
|
|
375
|
+
if (!segmentRangeIsComplete(segment, asrCompletedIndices)) break
|
|
376
|
+
const cached = readProgressiveCheckpoint(audioDir, segment, previousText, sessionId)
|
|
377
|
+
if (!cached) break
|
|
378
|
+
previousText = cached.text
|
|
379
|
+
const originals = entries.filter(e => e.chunkIndex >= segment.startChunkIdx && e.chunkIndex <= segment.endChunkIdx)
|
|
380
|
+
const baseline = originals.map(e => e.chunk.text).join(' ')
|
|
381
|
+
const count = baseline.trim().split(/\s+/).filter(Boolean).length
|
|
382
|
+
const quality = evaluateBatchQuality([cached], count)
|
|
383
|
+
// Runtime checks are conservative corruption guards, not proof of ASR
|
|
384
|
+
// accuracy. Human-corrected facts and latency remain qualification gates.
|
|
385
|
+
const numbers = baseline.match(/\b\d+(?:[.,]\d+)*\b/g) ?? []
|
|
386
|
+
const refinedNumbers = new Set(cached.text.match(/\b\d+(?:[.,]\d+)*\b/g) ?? [])
|
|
387
|
+
if (count >= 8 && quality.accepted && quality.coverageRatio >= 0.85
|
|
388
|
+
&& quality.coverageRatio <= 1.75
|
|
389
|
+
&& cached.words.filter(word => word.end > word.start).length >= quality.batchWordCount * 0.75
|
|
390
|
+
&& numbers.every(number => refinedNumbers.has(number))) results.push(cached)
|
|
391
|
+
}
|
|
392
|
+
return results
|
|
393
|
+
}
|
|
394
|
+
|
|
359
395
|
export async function transcribeSegments(
|
|
360
396
|
audioDir: string,
|
|
361
397
|
segments: BatchSegment[],
|
|
@@ -577,12 +577,14 @@ export async function searchMeetingLibrary(options: {
|
|
|
577
577
|
domain?: string
|
|
578
578
|
limit?: number
|
|
579
579
|
signal?: AbortSignal
|
|
580
|
+
/** false (6.71.1, Meeting Ask "keyword only") skips the embedding search. */
|
|
581
|
+
semantic?: boolean
|
|
580
582
|
}, store: MeetingStore = getMeetingStore()): Promise<MeetingSearchResult> {
|
|
581
583
|
const query = options.query.trim()
|
|
582
584
|
const domain = options.domain || 'all'
|
|
583
585
|
const limit = Math.max(1, Math.min(options.limit ?? 20, 50))
|
|
584
586
|
const keywordHits = keywordSearchMeetings(query, domain, store)
|
|
585
|
-
const semantic = await semanticSearchMeetings(query, domain, limit, options.signal)
|
|
587
|
+
const semantic = options.semantic === false ? { hits: [], reason: 'keyword_only' } : await semanticSearchMeetings(query, domain, limit, options.signal)
|
|
586
588
|
const hits = mergeMeetingSearchHits(keywordHits, semantic.hits, limit)
|
|
587
589
|
return {
|
|
588
590
|
hits,
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { WhisperWord } from './whisper-local.js'
|
|
2
|
+
|
|
3
|
+
/** Canary-only until human-corrected meeting/fact and native contention gates pass.
|
|
4
|
+
* Live HTTP keeps compact JSON; interactive HQ and DTW are unchanged. */
|
|
5
|
+
export function batchTimestampCanaryEnabled(): boolean {
|
|
6
|
+
return process.env.COS_BATCH_HQ_TIMESTAMPS === '1'
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface VadTimeSegment {
|
|
10
|
+
originalStart: number
|
|
11
|
+
originalEnd: number
|
|
12
|
+
processedStart: number
|
|
13
|
+
processedEnd: number
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Read the mapping emitted by the SAME native decode, rather than run a second
|
|
17
|
+
* VAD with potentially different weights, padding or thresholds. whisper.cpp
|
|
18
|
+
* 1.9.1/1.9.5 emit this centisecond table, even though CLI -ojf token offsets
|
|
19
|
+
* still use the compressed clock. Unknown/truncated diagnostics fail closed. */
|
|
20
|
+
export function parseNativeVadTimeline(stderr: string): VadTimeSegment[] {
|
|
21
|
+
const count = [...stderr.matchAll(/^whisper_vad: detected (\d+) speech segments\s*$/gm)]
|
|
22
|
+
const reduced = [...stderr.matchAll(/^whisper_vad: Reduced audio from (\d+) to (\d+) samples .+$/gm)]
|
|
23
|
+
const rows = [...stderr.matchAll(/^whisper_vad: vad_segment_info: orig_start: ([\d.]+), orig_end: ([\d.]+), vad_start: ([\d.]+), vad_end: ([\d.]+)\s*$/gm)]
|
|
24
|
+
if (count.length !== 1 || reduced.length !== 1 || rows.length !== Number(count[0][1]) || !rows.length) {
|
|
25
|
+
throw new Error('HQ VAD timing evidence missing or incomplete')
|
|
26
|
+
}
|
|
27
|
+
const duration = Number(reduced[0][1]) / 16000
|
|
28
|
+
const processedDuration = Number(reduced[0][2]) / 16000
|
|
29
|
+
const table = rows.map(row => ({
|
|
30
|
+
originalStart: Number(row[1]), originalEnd: Number(row[2]),
|
|
31
|
+
processedStart: Number(row[3]), processedEnd: Number(row[4]),
|
|
32
|
+
}))
|
|
33
|
+
for (let i = 0; i < table.length; i++) {
|
|
34
|
+
const s = table[i], prior = table[i - 1]
|
|
35
|
+
if (!Object.values(s).every(Number.isFinite)
|
|
36
|
+
|| s.originalStart < 0 || s.processedStart < 0
|
|
37
|
+
|| s.originalEnd <= s.originalStart || s.processedEnd <= s.processedStart
|
|
38
|
+
// Silero's final speech padding can extend the reported original end
|
|
39
|
+
// beyond the last sample by up to 30 ms (+ centisecond rounding).
|
|
40
|
+
|| s.originalEnd > duration + 0.041 || s.processedEnd > processedDuration + 0.011
|
|
41
|
+
|| (i === 0 && s.processedStart !== 0)
|
|
42
|
+
|| (prior && (s.originalStart < prior.originalEnd || s.processedStart <= prior.processedEnd))) {
|
|
43
|
+
throw new Error('HQ VAD timing evidence inconsistent')
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (Math.abs(table.at(-1)!.processedEnd - processedDuration) > 0.011) {
|
|
47
|
+
throw new Error('HQ VAD timing evidence does not cover the decoded audio')
|
|
48
|
+
}
|
|
49
|
+
table.at(-1)!.originalEnd = Math.min(table.at(-1)!.originalEnd, duration)
|
|
50
|
+
return table
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Replica of whisper_map_token_time_segment_aware. A token in an inserted gap
|
|
54
|
+
* snaps to the nearest speech boundary; a linear map through the gap would put
|
|
55
|
+
* words into silence. The native table already includes overlap and padding. */
|
|
56
|
+
export function mapVadTime(time: number, table: readonly VadTimeSegment[]): number {
|
|
57
|
+
if (!Number.isFinite(time) || time < 0 || !table.length) throw new Error('Invalid HQ word clock')
|
|
58
|
+
if (time <= table[0].processedStart) return table[0].originalStart
|
|
59
|
+
if (time >= table.at(-1)!.processedEnd) return table.at(-1)!.originalEnd
|
|
60
|
+
for (let i = 0; i < table.length; i++) {
|
|
61
|
+
const s = table[i], next = table[i + 1]
|
|
62
|
+
if (time >= s.processedStart && time <= s.processedEnd) {
|
|
63
|
+
return s.originalStart + (time - s.processedStart) *
|
|
64
|
+
(s.originalEnd - s.originalStart) / (s.processedEnd - s.processedStart)
|
|
65
|
+
}
|
|
66
|
+
if (next && time > s.processedEnd && time < next.processedStart) {
|
|
67
|
+
return time <= (s.processedEnd + next.processedStart) / 2 ? s.originalEnd : next.originalStart
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
throw new Error('Unmapped HQ word clock')
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function remapBatchWords(words: readonly WhisperWord[], stderr: string, useVad: boolean): WhisperWord[] {
|
|
74
|
+
// VAD-empty CLI output is a normal empty array, not a request for a new decode.
|
|
75
|
+
if (!words.length) return []
|
|
76
|
+
const table = useVad ? parseNativeVadTimeline(stderr) : []
|
|
77
|
+
return words.map(word => {
|
|
78
|
+
if (!Number.isFinite(word.start) || !Number.isFinite(word.end)
|
|
79
|
+
|| word.start < 0 || word.end < 0) throw new Error('Invalid HQ word clock')
|
|
80
|
+
const start = useVad ? mapVadTime(word.start, table) : word.start
|
|
81
|
+
// Match whisper.cpp's mapped t1 getter: token heuristics can emit t1 < t0.
|
|
82
|
+
// A zero-duration token is not evidence of a precisely timed spoken word.
|
|
83
|
+
const end = Math.max(start, useVad ? mapVadTime(word.end, table) : word.end)
|
|
84
|
+
return { ...word, start, end }
|
|
85
|
+
})
|
|
86
|
+
}
|
|
@@ -13,6 +13,7 @@ import { writeFileSync, unlinkSync, existsSync, readFileSync, statSync } from 'n
|
|
|
13
13
|
import { basename, join } from 'node:path'
|
|
14
14
|
import { homedir } from 'node:os'
|
|
15
15
|
import crypto from 'node:crypto'
|
|
16
|
+
import { batchTimestampCanaryEnabled, remapBatchWords } from './whisper-batch-timing.js'
|
|
16
17
|
import { getVocabulary, getOwnerName, getWhisperCorrections } from './profile.js'
|
|
17
18
|
import { stripBrandUrls } from './hallucination-filter.js'
|
|
18
19
|
import {
|
|
@@ -38,7 +39,7 @@ import {
|
|
|
38
39
|
const PROMPT_V2 = process.env.COS_PROMPT_V2 === '1'
|
|
39
40
|
const STRIP_BRAND_URLS = process.env.COS_WHISPER_STRIP_BRAND_URLS !== '0'
|
|
40
41
|
|
|
41
|
-
// Word
|
|
42
|
+
// Word/token timing from the batch CLI; live HTTP does not request word clocks.
|
|
42
43
|
export interface WhisperWord {
|
|
43
44
|
word: string
|
|
44
45
|
start: number // seconds
|
|
@@ -76,12 +77,14 @@ export interface HighQualityTranscriptionResult {
|
|
|
76
77
|
}
|
|
77
78
|
|
|
78
79
|
let highQualityCheckpointFingerprint: string | null = null
|
|
80
|
+
let highQualityCheckpointTimingMode: boolean | null = null
|
|
79
81
|
|
|
80
82
|
/** Stable, path-free cache identity for progressive meeting HQ checkpoints.
|
|
81
83
|
* Model/configuration is immutable for the lifetime of one server process, so
|
|
82
84
|
* cache this instead of stat'ing multi-GB model files on every health poll. */
|
|
83
85
|
export function getHighQualityCheckpointFingerprint(): string {
|
|
84
|
-
|
|
86
|
+
const timingMode = batchTimestampCanaryEnabled()
|
|
87
|
+
if (highQualityCheckpointFingerprint && highQualityCheckpointTimingMode === timingMode) return highQualityCheckpointFingerprint
|
|
85
88
|
const identity = (path: string): Record<string, number | boolean> => {
|
|
86
89
|
try {
|
|
87
90
|
const stat = statSync(path)
|
|
@@ -92,6 +95,8 @@ export function getHighQualityCheckpointFingerprint(): string {
|
|
|
92
95
|
}
|
|
93
96
|
highQualityCheckpointFingerprint = crypto.createHash('sha256').update(JSON.stringify({
|
|
94
97
|
schema: 2,
|
|
98
|
+
// Keep the existing legacy identity byte-for-byte when the canary is off.
|
|
99
|
+
...(timingMode ? { batchTiming: 'native-vad-timestamps-v1' } : {}),
|
|
95
100
|
serverVersion: process.env.COS_SERVER_VERSION?.trim() || 'development',
|
|
96
101
|
whisperCli: identity(WHISPER_CLI),
|
|
97
102
|
model: identity(resolveBatchModel()),
|
|
@@ -103,6 +108,7 @@ export function getHighQualityCheckpointFingerprint(): string {
|
|
|
103
108
|
prompt: buildWhisperPrompt(),
|
|
104
109
|
corrections: getWhisperCorrections(),
|
|
105
110
|
})).digest('hex')
|
|
111
|
+
highQualityCheckpointTimingMode = timingMode
|
|
106
112
|
return highQualityCheckpointFingerprint
|
|
107
113
|
}
|
|
108
114
|
|
|
@@ -966,10 +972,27 @@ export async function transcribeHighQuality(
|
|
|
966
972
|
// Word clocks on post-meeting CPU polish, and on any decode that asks for
|
|
967
973
|
// them (the prompt tail guard). Other live decodes stay compact JSON.
|
|
968
974
|
const captureWords = opts.priority === 'batch' || opts.words === true
|
|
975
|
+
const timestampCanary = opts.priority === 'batch' && batchTimestampCanaryEnabled()
|
|
976
|
+
let nativeDiagnostics = ''
|
|
969
977
|
|
|
970
978
|
const modelPath = resolveBatchModel()
|
|
971
979
|
const useLargeV3 = modelPath === BATCH_MODEL_LARGE_V3
|
|
972
980
|
|
|
981
|
+
// A valid empty PCM WAV has nothing to decode. Some native CLI versions exit
|
|
982
|
+
// zero without writing JSON for it; distinguish it from malformed audio.
|
|
983
|
+
if (timestampCanary && audioBuffer.length === 44
|
|
984
|
+
&& audioBuffer.toString('ascii', 0, 4) === 'RIFF'
|
|
985
|
+
&& audioBuffer.readUInt32LE(4) === 36
|
|
986
|
+
&& audioBuffer.toString('ascii', 8, 12) === 'WAVE'
|
|
987
|
+
&& audioBuffer.toString('ascii', 12, 16) === 'fmt '
|
|
988
|
+
&& audioBuffer.readUInt32LE(16) === 16
|
|
989
|
+
&& audioBuffer.readUInt16LE(20) === 1 && audioBuffer.readUInt16LE(22) === 1
|
|
990
|
+
&& audioBuffer.readUInt32LE(24) === 16000 && audioBuffer.readUInt32LE(28) === 32000
|
|
991
|
+
&& audioBuffer.readUInt16LE(32) === 2 && audioBuffer.readUInt16LE(34) === 16
|
|
992
|
+
&& audioBuffer.toString('ascii', 36, 40) === 'data' && audioBuffer.readUInt32LE(40) === 0) {
|
|
993
|
+
return { text: '', words: [], model: useLargeV3 ? 'large-v3' : 'turbo', backend: 'whisper-cli', actualQuality: useLargeV3 ? 'hq' : 'fast' }
|
|
994
|
+
}
|
|
995
|
+
|
|
973
996
|
try {
|
|
974
997
|
writeFileSync(tmpWav, audioBuffer)
|
|
975
998
|
|
|
@@ -1008,8 +1031,9 @@ export async function transcribeHighQuality(
|
|
|
1008
1031
|
...(useMetal ? ['-fa'] : ['-ng']),
|
|
1009
1032
|
'-bs', String(beam),
|
|
1010
1033
|
'-bo', String(bestOf),
|
|
1011
|
-
|
|
1012
|
-
|
|
1034
|
+
// The candidate needs native VAD diagnostics for the exact mapping.
|
|
1035
|
+
// Ordinary timestamp decoding never enables DTW or live verbose JSON.
|
|
1036
|
+
...(timestampCanary ? [] : ['--no-timestamps', '-np']),
|
|
1013
1037
|
'--prompt', buildPrompt(context),
|
|
1014
1038
|
]
|
|
1015
1039
|
if (captureWords) {
|
|
@@ -1103,6 +1127,7 @@ export async function transcribeHighQuality(
|
|
|
1103
1127
|
reject(new Error(`whisper-cli HQ exit ${code}: ${stderr.trim().slice(0, 200)}`))
|
|
1104
1128
|
return
|
|
1105
1129
|
}
|
|
1130
|
+
nativeDiagnostics = stderr
|
|
1106
1131
|
resolve(stdout.trim())
|
|
1107
1132
|
})
|
|
1108
1133
|
|
|
@@ -1122,13 +1147,23 @@ export async function transcribeHighQuality(
|
|
|
1122
1147
|
if (captureWords) {
|
|
1123
1148
|
try {
|
|
1124
1149
|
if (existsSync(jsonPath)) {
|
|
1125
|
-
const
|
|
1126
|
-
if (
|
|
1127
|
-
|
|
1128
|
-
|
|
1150
|
+
const raw = readFileSync(jsonPath, 'utf8')
|
|
1151
|
+
if (timestampCanary && !Array.isArray(JSON.parse(raw)?.transcription)) {
|
|
1152
|
+
throw new Error('HQ timestamp candidate returned invalid CLI JSON')
|
|
1153
|
+
}
|
|
1154
|
+
const parsed = parseWhisperCliFullJson(raw)
|
|
1155
|
+
if (timestampCanary || parsed.text) finalText = parsed.text
|
|
1156
|
+
const timed = timestampCanary ? remapBatchWords(parsed.words, nativeDiagnostics, useVad) : parsed.words
|
|
1157
|
+
words = timed.length > 0
|
|
1158
|
+
? timed.map(w => ({ ...w, word: applyCorrections(w.word) }))
|
|
1129
1159
|
: []
|
|
1160
|
+
} else if (timestampCanary) {
|
|
1161
|
+
throw new Error('HQ timestamp candidate did not produce CLI JSON')
|
|
1130
1162
|
}
|
|
1131
1163
|
} catch (err) {
|
|
1164
|
+
// Timestamp-mode stdout contains display clocks. Missing mapping/JSON
|
|
1165
|
+
// must preserve the prior transcript, never persist that stdout as text.
|
|
1166
|
+
if (timestampCanary) throw err
|
|
1132
1167
|
console.warn(
|
|
1133
1168
|
`[whisper-hq] Batch word JSON parse failed; keeping text-only polish: ` +
|
|
1134
1169
|
`${err instanceof Error ? err.message : String(err)}`,
|
package/server/routes/health.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
import { getOpenAIWhisperBudgetState } from '../lib/openai-whisper-budget.js'
|
|
29
29
|
import { getMeetingSummaryBudgetState } from '../lib/meeting-summary-budget.js'
|
|
30
30
|
import { meetingWebCapability } from '../lib/meeting-ask-web.js'
|
|
31
|
+
import { MEETING_ASK_CAPABILITIES } from '../lib/meeting-ask-runtime.js'
|
|
31
32
|
import { lensGistPublicHealth } from '../lib/lens-gist.js'
|
|
32
33
|
import { meetingSummaryLLMEnabled } from '../lib/meeting-summary.js'
|
|
33
34
|
import { getKeyStatus } from '../lib/openai-key.js'
|
|
@@ -466,7 +467,7 @@ healthRouter.get('/health', async (_req, res) => {
|
|
|
466
467
|
review_audio: reviewAudio,
|
|
467
468
|
...(voiceProvenance ? { voice_provenance: voiceProvenance } : {}),
|
|
468
469
|
capabilities: {
|
|
469
|
-
meetingAsk: {
|
|
470
|
+
meetingAsk: { ...MEETING_ASK_CAPABILITIES, webRetrieval: meetingWebCapability().available },
|
|
470
471
|
dictationCleanup: { models: DICTATION_CLEAN_MODELS },
|
|
471
472
|
// 6.67.0: glasses pairing (docs/pairing-contract.md). Public on purpose: the phone
|
|
472
473
|
// reads it before it holds a token, and an older server omits it, which is how
|
|
@@ -632,7 +633,7 @@ healthRouter.get('/models', async (req, res) => {
|
|
|
632
633
|
// Changing that is a breaking change that needs a client version gate first.
|
|
633
634
|
...(apiToken ? { displayStreamTicket: mintDisplayTicket(apiToken) } : {}),
|
|
634
635
|
capabilities: {
|
|
635
|
-
meetingAsk: {
|
|
636
|
+
meetingAsk: { ...MEETING_ASK_CAPABILITIES, webRetrieval: meetingWebCapability().available },
|
|
636
637
|
dictationCleanup: { models: DICTATION_CLEAN_MODELS },
|
|
637
638
|
durableQueryJobs: {
|
|
638
639
|
enabled: durableJobs.enabled,
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { transcriptSources } from '../lib/meeting-ask-transcript-sources.js';
|
|
1
2
|
import { Router, type Response } from 'express';
|
|
2
3
|
import { AskError } from '../lib/live-cos-ask.js';
|
|
3
4
|
import { meetingAskRuntime, type MeetingAskRuntime } from '../lib/meeting-ask-runtime.js';
|
|
4
|
-
import {
|
|
5
|
+
import { refineActiveMeetingAsk } from '../lib/meeting-ask-refinement.js';
|
|
6
|
+
import { resolveAskTranscript, type AskTranscriptSources } from '../lib/meeting-ask-transcript.js';
|
|
5
7
|
import { ASK_SESSION } from '../lib/meeting-ask-store.js';
|
|
6
|
-
import { redactSecrets } from '../lib/lens-gist-engines.js';
|
|
7
8
|
import type { MeetingAskExchange, MeetingAskFile } from '../../shared/meeting-ask.js';
|
|
8
9
|
function failure(res: Response, error: unknown) { res.status(error instanceof AskError ? error.status : 503).json({ error: error instanceof AskError ? error.reason : 'meeting_assistant_unavailable' }); }
|
|
9
10
|
/** Safe Markdown export. Citations are namespaced to the request because T1 is answer-local. */
|
|
@@ -41,7 +42,7 @@ export function exportMeetingAsk(exchanges: MeetingAskExchange[], files: Meeting
|
|
|
41
42
|
throw new AskError('export_too_large', 413);
|
|
42
43
|
return text;
|
|
43
44
|
}
|
|
44
|
-
export function createMeetingAskRouter(runtime: MeetingAskRuntime = meetingAskRuntime) {
|
|
45
|
+
export function createMeetingAskRouter(runtime: MeetingAskRuntime = meetingAskRuntime, transcripts: AskTranscriptSources = transcriptSources) {
|
|
45
46
|
const router = Router();
|
|
46
47
|
router.use((_q, res, next) => { res.set('Cache-Control', 'no-store'); next(); });
|
|
47
48
|
router.get('/meeting-ask/config', async (_q, res) => { try {
|
|
@@ -56,6 +57,14 @@ export function createMeetingAskRouter(runtime: MeetingAskRuntime = meetingAskRu
|
|
|
56
57
|
catch (e) {
|
|
57
58
|
failure(res, e);
|
|
58
59
|
} });
|
|
60
|
+
router.get('/meeting-ask/context', async (req, res) => { try {
|
|
61
|
+
const sessionId = String(req.query.sessionId ?? '');
|
|
62
|
+
res.json(await runtime.context(sessionId, req.query.serverInstanceId, transcripts.live(sessionId)?.startTime ?? null));
|
|
63
|
+
} catch (e) { failure(res, e); } });
|
|
64
|
+
router.put('/meeting-ask/context', async (req, res) => { try {
|
|
65
|
+
const sessionId = String(req.body?.sessionId ?? '');
|
|
66
|
+
res.json({ orientation: await runtime.setContext(sessionId, req.body?.serverInstanceId, transcripts.live(sessionId)?.startTime ?? null, req.body) });
|
|
67
|
+
} catch (e) { failure(res, e); } });
|
|
59
68
|
router.post('/meeting-ask/requests', async (req, res) => { try {
|
|
60
69
|
res.status(202).json({ exchange: await runtime.submit(req.body) });
|
|
61
70
|
}
|
|
@@ -90,6 +99,12 @@ export function createMeetingAskRouter(runtime: MeetingAskRuntime = meetingAskRu
|
|
|
90
99
|
catch (e) {
|
|
91
100
|
failure(res, e);
|
|
92
101
|
} });
|
|
102
|
+
router.post('/meeting-ask/files/media', (req, res) => { try {
|
|
103
|
+
res.json({ files: runtime.attachMedia(String(req.body?.sessionId ?? ''), req.body?.serverInstanceId, req.body?.media) });
|
|
104
|
+
}
|
|
105
|
+
catch (e) {
|
|
106
|
+
failure(res, e);
|
|
107
|
+
} });
|
|
93
108
|
router.get('/meeting-ask/export', (req, res) => {
|
|
94
109
|
try {
|
|
95
110
|
const exchanges: MeetingAskExchange[] = [];
|
|
@@ -113,11 +128,14 @@ export function createMeetingAskRouter(runtime: MeetingAskRuntime = meetingAskRu
|
|
|
113
128
|
const sessionId = String(req.query.sessionId ?? '');
|
|
114
129
|
if (!ASK_SESSION.test(sessionId))
|
|
115
130
|
throw new AskError('invalid_session_id');
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
131
|
+
const snapshot = resolveAskTranscript(sessionId, transcripts);
|
|
132
|
+
// Ended/saved transcripts keep the released resolver and source metadata.
|
|
133
|
+
// Only a live snapshot may use the default-off HQ cache overlay.
|
|
134
|
+
if (snapshot.source === 'live' && process.env.COS_MEETING_ASK_HQ_EVIDENCE === '1') {
|
|
135
|
+
const refined = refineActiveMeetingAsk({ sessionId, action: 'catch_up', question: '',
|
|
136
|
+
elapsedMs: snapshot.elapsedMs, turns: snapshot.turns, history: [] });
|
|
137
|
+
res.json({ ...snapshot, turns: refined.input.turns, notices: [...snapshot.notices, ...refined.notices] });
|
|
138
|
+
} else res.json(snapshot);
|
|
121
139
|
}
|
|
122
140
|
catch (e) {
|
|
123
141
|
failure(res, e);
|
package/shared/meeting-ask.ts
CHANGED
|
@@ -11,6 +11,20 @@ export interface MeetingAskTurn {
|
|
|
11
11
|
speaker: string;
|
|
12
12
|
text: string;
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* GET /api/meeting-ask/transcript. A live meeting answers from memory. A meeting that has ended (6.71.1)
|
|
16
|
+
* answers from its saved record with ended: true, so it stays askable. 404 meeting_transcript_unavailable
|
|
17
|
+
* only when no source holds its words.
|
|
18
|
+
*/
|
|
19
|
+
export interface MeetingAskTranscript {
|
|
20
|
+
turns: MeetingAskTurn[];
|
|
21
|
+
elapsedMs: number;
|
|
22
|
+
notices: string[];
|
|
23
|
+
ended?: true;
|
|
24
|
+
/** When the recording stopped, if known. */
|
|
25
|
+
endedAt?: string;
|
|
26
|
+
source?: 'live' | 'active_file' | 'saved';
|
|
27
|
+
}
|
|
14
28
|
export interface MeetingAskFile {
|
|
15
29
|
id: string;
|
|
16
30
|
revision: string;
|
|
@@ -18,8 +32,12 @@ export interface MeetingAskFile {
|
|
|
18
32
|
status: 'ready' | 'pending' | 'unsupported' | 'failed' | 'removed' | 'secret-held';
|
|
19
33
|
text: string;
|
|
20
34
|
notice?: string;
|
|
35
|
+
/** "phone" for a file the phone attached through POST files/media (6.71.1). Control's rows omit it. */
|
|
36
|
+
origin?: 'phone';
|
|
21
37
|
}
|
|
22
38
|
export interface MeetingAskSource {
|
|
39
|
+
/** Server-owned accepted post-stop quotation; no live turn target or precise word clock. */
|
|
40
|
+
finalizedTranscript?: boolean
|
|
23
41
|
id: string;
|
|
24
42
|
kind: 'transcript' | 'memory' | 'meeting' | 'file' | 'web';
|
|
25
43
|
title: string;
|
|
@@ -155,11 +173,24 @@ export interface MeetingAskCapabilities {
|
|
|
155
173
|
durableHistory: true;
|
|
156
174
|
attachmentContext: true;
|
|
157
175
|
webRetrieval: boolean;
|
|
176
|
+
/** 6.71.1: POST /api/meeting-ask/files/media turns uploaded media into meeting files. */
|
|
177
|
+
phoneFiles?: true;
|
|
178
|
+
/** 6.71.1: config carries cosSearch and PUT config {cosSearch} changes it. */
|
|
179
|
+
cosSearchMode?: true;
|
|
180
|
+
meetingContext?: true;
|
|
158
181
|
}
|
|
182
|
+
/**
|
|
183
|
+
* How Meeting Ask searches COS memory and earlier meetings (6.71.1). 'meaning' (default) adds the embedding
|
|
184
|
+
* search, which sends the search phrase to the COS install's embedding provider; 'keyword' matches words
|
|
185
|
+
* locally only, with no embedding request and no tokens.
|
|
186
|
+
*/
|
|
187
|
+
export type MeetingAskCosSearch = 'meaning' | 'keyword';
|
|
159
188
|
export interface MeetingAskConfig {
|
|
160
189
|
protocolVersion: 2;
|
|
161
190
|
serverInstanceId: string;
|
|
162
191
|
selected: MeetingAskModel;
|
|
192
|
+
/** 6.71.1. Missing on older servers, which always used 'meaning'. */
|
|
193
|
+
cosSearch?: MeetingAskCosSearch;
|
|
163
194
|
models: Array<MeetingAskModel & {
|
|
164
195
|
available: boolean;
|
|
165
196
|
reason?: string;
|
|
@@ -171,12 +202,26 @@ export interface MeetingAskConfig {
|
|
|
171
202
|
};
|
|
172
203
|
capabilities: MeetingAskCapabilities;
|
|
173
204
|
}
|
|
174
|
-
/** All paths under /api/meeting-ask. GET config; PUT config {engine,model}.
|
|
205
|
+
/** All paths under /api/meeting-ask. GET config; PUT config {engine,model} and/or {cosSearch} (6.71.1); either part may come alone.
|
|
175
206
|
* POST requests MeetingAskSubmit -> 202 {exchange}; GET requests/:requestId -> {exchange}.
|
|
176
207
|
* POST requests/:requestId/cancel {serverInstanceId} -> {exchange}.
|
|
177
208
|
* GET history?sessionId=...&serverInstanceId=...&offset=0&limit=50&order=asc|desc -> {exchanges,nextOffset,files}.
|
|
178
209
|
* PUT files {sessionId,serverInstanceId,files:MeetingAskFile[]} -> {files}. Upsert only; removed is explicit.
|
|
210
|
+
* POST files/media {sessionId,serverInstanceId,media:[{id,name?}]} -> {files} (6.71.1, capabilities.phoneFiles):
|
|
211
|
+
* media already uploaded through /api/media joins the meeting's file list as id "phone_<mediaId>", origin "phone".
|
|
179
212
|
* GET export?sessionId=...&serverInstanceId=... -> {text}; composes persisted Ask/file context,
|
|
180
213
|
* client prepends its current canonical transcript. All operations use X-Cos-Token.
|
|
181
214
|
* GET config is the capability probe (404 means old server); health.capabilities.meetingAsk mirrors capabilities.
|
|
182
215
|
*/
|
|
216
|
+
|
|
217
|
+
/** User-confirmed navigation context for one meeting, never canonical transcript evidence. */
|
|
218
|
+
export type MeetingAskDomain = string;
|
|
219
|
+
export interface MeetingAskCalendarMatch {
|
|
220
|
+
id: string; title: string; start: string; end: string; suggestedDomain?: MeetingAskDomain;
|
|
221
|
+
}
|
|
222
|
+
export interface MeetingAskOrientation {
|
|
223
|
+
revision: number; domain: MeetingAskDomain | null; calendar: MeetingAskCalendarMatch | null;
|
|
224
|
+
}
|
|
225
|
+
export interface MeetingAskContext {
|
|
226
|
+
orientation: MeetingAskOrientation; domains: Array<{id: string; label: string}>; candidates: MeetingAskCalendarMatch[]; notices: string[];
|
|
227
|
+
}
|