@gotcos/glasses-server 6.84.0 → 6.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +272 -6
- package/README.md +23 -0
- package/docs/active-query-jobs.md +123 -0
- package/managed-runtime-contract.json +1 -0
- package/package.json +6 -1
- package/server/index.ts +3 -0
- package/server/lib/dictation-clean.ts +53 -8
- package/server/lib/dictation-polish-instructions.ts +118 -0
- package/server/lib/dictation-prewarm.ts +2 -2
- package/server/lib/fuzzy-correct.ts +3 -5
- package/server/lib/live-cue-verdicts.ts +3 -2
- package/server/lib/live-cues-v2-prompts-kinds.ts +38 -0
- package/server/lib/live-cues-v2.ts +95 -21
- package/server/lib/live-provisional.ts +86 -0
- package/server/lib/meeting-ask-owner.ts +1 -1
- package/server/lib/meeting-fact-check.ts +1 -1
- package/server/lib/query-job-active.ts +140 -0
- package/server/lib/query-job-coordinator.ts +8 -0
- package/server/lib/query-job-store.ts +25 -0
- package/server/lib/token-audit.ts +7 -0
- package/server/lib/transcript-cleanup-context.ts +2 -2
- package/server/lib/transcript-cleanup-model.ts +1 -1
- package/server/lib/transcript-cleanup-spans.ts +1 -1
- package/server/lib/transcript-cleanup.ts +1 -1
- package/server/lib/whisper-local.ts +2 -1
- package/server/routes/dictation-polish-settings.ts +91 -0
- package/server/routes/health.ts +21 -4
- package/server/routes/live-cues.ts +8 -2
- package/server/routes/prompt-drafts.ts +33 -10
- package/server/routes/query-jobs.ts +22 -0
- package/server/routes/transcribe-stream.ts +53 -11
- package/server/routes/voice.ts +1 -1
- package/server/routes/welcome-context.ts +29 -6
- package/shared/live-cues-v2.ts +19 -3
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
import { getVocabulary, getWhisperCorrections } from '../lib/profile.js'
|
|
53
53
|
import { correctionOnlyForms } from '../lib/glossary-learn.js'
|
|
54
54
|
import { polishLinkProblem, renderSpokenPaths, spokenPathsEnabled } from '../lib/dictation-format.js'
|
|
55
|
+
import { customInstructionsHash, readCustomInstructions } from '../lib/dictation-polish-instructions.js'
|
|
55
56
|
import { createBreaker } from '../lib/claude-circuit.js'
|
|
56
57
|
import { logTokenAudit } from '../lib/token-audit.js'
|
|
57
58
|
import { emitDisplay } from '../lib/display-bus.js'
|
|
@@ -109,7 +110,7 @@ function routeAutoClean(req: { body?: any; query?: any }): AutoCleanRequest {
|
|
|
109
110
|
async function cleanOutboundDictation(
|
|
110
111
|
text: string,
|
|
111
112
|
opts: AutoCleanRequest & { bucket: PolishBucket; signal?: AbortSignal; prewarmToken?: string },
|
|
112
|
-
): Promise<
|
|
113
|
+
): Promise<PolishResult> {
|
|
113
114
|
const result = await polishOutboundDictation(text, opts)
|
|
114
115
|
// A warm process the model call did not take must not sit until its TTL.
|
|
115
116
|
if (opts.prewarmToken && !result.prewarm) {
|
|
@@ -118,10 +119,23 @@ async function cleanOutboundDictation(
|
|
|
118
119
|
return result
|
|
119
120
|
}
|
|
120
121
|
|
|
122
|
+
/** 6.85.0, additive and present only when the model was given the user's polish instructions. */
|
|
123
|
+
interface PolishInstructionsUse { customInstructions?: true; instructionsHash?: string }
|
|
124
|
+
type PolishResult = { text: string; polishSkipped?: PolishSkipped; prewarm?: PrewarmOutcome } & PolishInstructionsUse
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* 6.85.0: the user's polish instructions apply to dictation (Mac dictation and G2 messages, which
|
|
128
|
+
* are surface "message", and prompt drafts). Meeting lines are a faithful record, so they stay on
|
|
129
|
+
* the built-in rules alone, the same way they skip the filler pass and spoken paths.
|
|
130
|
+
*/
|
|
131
|
+
export function polishInstructionsFor(bucket: PolishBucket): string {
|
|
132
|
+
return bucket === 'meeting' ? '' : readCustomInstructions().customInstructions
|
|
133
|
+
}
|
|
134
|
+
|
|
121
135
|
async function polishOutboundDictation(
|
|
122
136
|
text: string,
|
|
123
137
|
opts: AutoCleanRequest & { bucket: PolishBucket; signal?: AbortSignal; prewarmToken?: string },
|
|
124
|
-
): Promise<
|
|
138
|
+
): Promise<PolishResult> {
|
|
125
139
|
let cleaned = applyNegativeRules(applyCorrections(text)).replace(/\s+/g, ' ').trim() || text
|
|
126
140
|
// applyCorrections above is an EXACT string map, so it only fixes misspellings someone
|
|
127
141
|
// already hand-authored. A novel miss ("Rileyy" for Riley, "Exampel" for Example) sails
|
|
@@ -159,6 +173,10 @@ async function polishOutboundDictation(
|
|
|
159
173
|
if (polishBudgetExhausted(opts.bucket)) return { text: cleaned, polishSkipped: 'daily_cap' }
|
|
160
174
|
const startedAt = Date.now()
|
|
161
175
|
recordPolishCall(opts.bucket)
|
|
176
|
+
// Read once per polish: the warm and cold paths see the same instructions.
|
|
177
|
+
const customInstructions = polishInstructionsFor(opts.bucket)
|
|
178
|
+
const instructionsHash = customInstructionsHash(customInstructions)
|
|
179
|
+
const used: PolishInstructionsUse = customInstructions ? { customInstructions: true, instructionsHash } : {}
|
|
162
180
|
// 6.79.2: a warm process started at key press answers without the CLI's start-up cost.
|
|
163
181
|
const warmEntry = opts.prewarmToken ? takePolishPrewarm(opts.prewarmToken, model) : null
|
|
164
182
|
let prewarm: PrewarmOutcome | undefined = opts.prewarmToken ? 'missed' : undefined
|
|
@@ -166,18 +184,19 @@ async function polishOutboundDictation(
|
|
|
166
184
|
let polished: string | null = null
|
|
167
185
|
if (warmEntry) {
|
|
168
186
|
try {
|
|
169
|
-
polished = (await runWarmPolish(warmEntry, cleaned, getVocabulary(), { signal: opts.signal })).trim()
|
|
187
|
+
polished = (await runWarmPolish(warmEntry, cleaned, getVocabulary(), { signal: opts.signal, customInstructions })).trim()
|
|
170
188
|
prewarm = 'used'
|
|
171
189
|
} catch (err) {
|
|
172
190
|
// Nothing was sent: the cold path below is the same call, not a second one.
|
|
173
191
|
if (!(err instanceof PrewarmUnavailableError)) throw err
|
|
174
192
|
}
|
|
175
193
|
}
|
|
176
|
-
if (polished === null) polished = (await autoCleanDictation(cleaned, getVocabulary(), { model, signal: opts.signal })).trim()
|
|
194
|
+
if (polished === null) polished = (await autoCleanDictation(cleaned, getVocabulary(), { model, signal: opts.signal, customInstructions })).trim()
|
|
177
195
|
autoCleanBreaker.recordSuccess()
|
|
178
196
|
logTokenAudit({
|
|
179
197
|
source: 'g2-dictation-autoclean', model, inputChars: cleaned.length, outputChars: polished.length,
|
|
180
198
|
durationMs: Date.now() - startedAt, caller: `dictation_autoclean_${opts.bucket}`,
|
|
199
|
+
polishInstructions: { on: Boolean(customInstructions), hash: instructionsHash },
|
|
181
200
|
})
|
|
182
201
|
// 6.83.1: the model never invents a link and never drops one the dictionary wrote. Its
|
|
183
202
|
// answer is refused (the pre-polish text goes out, polishSkipped "link_guard") rather than
|
|
@@ -185,13 +204,13 @@ async function polishOutboundDictation(
|
|
|
185
204
|
const linkProblem = polished ? polishLinkProblem(cleaned, polished, correctionOnlyForms(getWhisperCorrections()), getVocabulary()) : null
|
|
186
205
|
if (linkProblem) {
|
|
187
206
|
console.warn(`[prompt-draft] auto-clean answer refused: ${linkProblem}`)
|
|
188
|
-
return { text: cleaned, ...(prewarm ? { prewarm } : {}), polishSkipped: 'link_guard' }
|
|
207
|
+
return { text: cleaned, ...(prewarm ? { prewarm } : {}), polishSkipped: 'link_guard', ...used }
|
|
189
208
|
}
|
|
190
|
-
return { text: polished || cleaned, ...(prewarm ? { prewarm } : {}) }
|
|
209
|
+
return { text: polished || cleaned, ...(prewarm ? { prewarm } : {}), ...used }
|
|
191
210
|
} catch (err: any) {
|
|
192
211
|
autoCleanBreaker.recordFailure()
|
|
193
212
|
console.warn(`[prompt-draft] auto-clean failed (glossary-only): ${err?.message ?? err}`)
|
|
194
|
-
return { text: cleaned, polishSkipped: 'failed', ...(prewarm ? { prewarm } : {}) }
|
|
213
|
+
return { text: cleaned, polishSkipped: 'failed', ...(prewarm ? { prewarm } : {}), ...used }
|
|
195
214
|
}
|
|
196
215
|
}
|
|
197
216
|
|
|
@@ -219,7 +238,7 @@ promptDraftsRouter.post('/dictation/finalize', async (req, res) => {
|
|
|
219
238
|
|
|
220
239
|
const abort = new AbortController()
|
|
221
240
|
res.on('close', () => { if (!res.writableEnded) abort.abort() })
|
|
222
|
-
const { text: finalText, polishSkipped, prewarm } = await cleanOutboundDictation(text, {
|
|
241
|
+
const { text: finalText, polishSkipped, prewarm, customInstructions, instructionsHash } = await cleanOutboundDictation(text, {
|
|
223
242
|
enabled: true,
|
|
224
243
|
model,
|
|
225
244
|
bucket: surface,
|
|
@@ -231,6 +250,8 @@ promptDraftsRouter.post('/dictation/finalize', async (req, res) => {
|
|
|
231
250
|
res.json({
|
|
232
251
|
text: finalText, surface, polished: finalText !== text,
|
|
233
252
|
...(polishSkipped ? { polishSkipped } : {}), ...(prewarm ? { prewarm } : {}),
|
|
253
|
+
// 6.85.0: only when the model was given the user's polish instructions (a hash, never the text).
|
|
254
|
+
...(customInstructions ? { customInstructions, instructionsHash } : {}),
|
|
234
255
|
})
|
|
235
256
|
})
|
|
236
257
|
|
|
@@ -393,7 +414,7 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
|
|
|
393
414
|
// plus the decoder's own low confidence (prompt-tail-guard.ts).
|
|
394
415
|
const guarded = guardPromptTail({ text: result.text, words: result.words, speech: speechWindowsFromWav(audio) })
|
|
395
416
|
for (const drop of guarded.dropped) {
|
|
396
|
-
console.warn(`[prompt-draft] tail_drop ${draftId}/${chunkIndex} ${purpose}/${mode} (${drop.reason}, start=${drop.startSec ?? '?'}s, lastSpeech=${guarded.lastSpeechEndSec ?? '?'}s, p=${drop.meanProbability?.toFixed(2) ?? '?'}):
|
|
417
|
+
console.warn(`[prompt-draft] tail_drop ${draftId}/${chunkIndex} ${purpose}/${mode} (${drop.reason}, start=${drop.startSec ?? '?'}s, lastSpeech=${guarded.lastSpeechEndSec ?? '?'}s, p=${drop.meanProbability?.toFixed(2) ?? '?'}): chars=${drop.text.length}`)
|
|
397
418
|
}
|
|
398
419
|
if (guarded.dropped.length > 0 && !guarded.text.trim()) throw new NoSpeechDetectedError(result.text)
|
|
399
420
|
const text = sanitizeTranscript(draftId, guarded.text)
|
|
@@ -518,7 +539,7 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
|
|
|
518
539
|
await markPromptDraftError(draftId, 'No speech detected')
|
|
519
540
|
throw new NoSpeechDetectedError()
|
|
520
541
|
}
|
|
521
|
-
const { text: finalText, polishSkipped } = await cleanOutboundDictation(text, { ...autoClean, bucket: 'prompt_draft', signal })
|
|
542
|
+
const { text: finalText, polishSkipped, customInstructions, instructionsHash } = await cleanOutboundDictation(text, { ...autoClean, bucket: 'prompt_draft', signal })
|
|
522
543
|
const finalized = await markPromptDraftFinalized(draftId, finalText)
|
|
523
544
|
const qualities = transcriptRecords.map(record => record.actualQuality)
|
|
524
545
|
const actualQuality: 'hq' | 'fast' | 'cloud' = qualities.includes('cloud')
|
|
@@ -548,6 +569,8 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
|
|
|
548
569
|
...(engineReason ? { engineReason } : {}),
|
|
549
570
|
// 6.79.1, additive.
|
|
550
571
|
...(polishSkipped ? { polishSkipped } : {}),
|
|
572
|
+
// 6.85.0, additive.
|
|
573
|
+
...(customInstructions ? { customInstructions, instructionsHash } : {}),
|
|
551
574
|
}
|
|
552
575
|
}
|
|
553
576
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Router, type Request, type Response } from 'express'
|
|
2
2
|
import { durableQueryJobsEnabled, messagesSteerEnabled } from '../lib/query-job-feature.js'
|
|
3
|
+
import { ACTIVE_QUERY_JOBS_LIST_MAX, ACTIVE_QUERY_JOBS_PROTOCOL } from '../lib/query-job-active.js'
|
|
3
4
|
import {
|
|
4
5
|
QueryJobCoordinator,
|
|
5
6
|
QueryJobCoordinatorError,
|
|
@@ -197,6 +198,27 @@ export function createQueryJobsRouter(
|
|
|
197
198
|
}
|
|
198
199
|
})
|
|
199
200
|
|
|
201
|
+
// 6.84.1: the in-flight jobs for COS Control's Running panel (docs/active-query-jobs.md).
|
|
202
|
+
// Read-only and in memory; registered ahead of `/query-jobs/:jobId`, which would answer
|
|
203
|
+
// 400 invalid_job_id for "active". A title is the prompt's first line, cut to 80
|
|
204
|
+
// characters; no prompt body, answer, activity or trail is ever in this answer.
|
|
205
|
+
router.get('/query-jobs/active', (_req, res) => {
|
|
206
|
+
try {
|
|
207
|
+
const now = Date.now()
|
|
208
|
+
const { items, count } = coordinator.activeJobs(now)
|
|
209
|
+
return res.json({
|
|
210
|
+
protocolVersion: ACTIVE_QUERY_JOBS_PROTOCOL,
|
|
211
|
+
enabled: enabled(),
|
|
212
|
+
now: new Date(now).toISOString(),
|
|
213
|
+
count,
|
|
214
|
+
items: items.slice(0, ACTIVE_QUERY_JOBS_LIST_MAX),
|
|
215
|
+
})
|
|
216
|
+
} catch (error) {
|
|
217
|
+
const wire = wireError(error)
|
|
218
|
+
return res.status(wire.status).json(wire.body)
|
|
219
|
+
}
|
|
220
|
+
})
|
|
221
|
+
|
|
200
222
|
// Lost-202 recovery is keyed by the immutable client identity. Keep this
|
|
201
223
|
// route explicit and ahead of the dynamic job lookup for future router
|
|
202
224
|
// changes that may broaden the latter's matcher.
|
|
@@ -115,6 +115,8 @@ import {
|
|
|
115
115
|
type MaintenanceWorkLease,
|
|
116
116
|
} from '../lib/maintenance-lifecycle.js'
|
|
117
117
|
import { feedLiveCueTranscript } from '../lib/live-cues-engine.js'
|
|
118
|
+
import { armMacLiveCues, type MacLiveCuesAnswer } from '../lib/live-cues-v2.js'
|
|
119
|
+
import { liveProvisional } from '../lib/live-provisional.js'
|
|
118
120
|
import { preemptMetalBatchForLive, registerLiveActivityProbe } from '../lib/whisper-metal-gate.js'
|
|
119
121
|
import {
|
|
120
122
|
scheduleProgressiveHqCheckpoint,
|
|
@@ -1409,10 +1411,23 @@ function recordReceivedChunk(session: TranscriptSession, chunkIndex: number): vo
|
|
|
1409
1411
|
}
|
|
1410
1412
|
}
|
|
1411
1413
|
|
|
1412
|
-
function recordAsrCompleted(session: TranscriptSession, chunkIndex: number): void {
|
|
1414
|
+
function recordAsrCompleted(session: TranscriptSession, chunkIndex: number, sessionId: string): void {
|
|
1413
1415
|
if (!Number.isInteger(chunkIndex) || chunkIndex < 0) return
|
|
1414
1416
|
if (!session.asrCompletedIndices) session.asrCompletedIndices = []
|
|
1415
1417
|
insertSortedUnique(session.asrCompletedIndices, chunkIndex)
|
|
1418
|
+
// 6.86.0: the canonical answer replaces the grey provisional line for this chunk (live-provisional.ts).
|
|
1419
|
+
liveProvisional().settle(sessionId, chunkIndex)
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
/** 6.86.0: keep the newest preview of a LIVE meeting for COS Control's grey line. Only a session this server holds (a
|
|
1423
|
+
* close removes it from `sessions`), and only while that chunk is still open: a preview that lands after its chunk (or a
|
|
1424
|
+
* later one) settled is stale. Memory only; never logged. */
|
|
1425
|
+
function noteLiveProvisional(sessionId: string, chunkIndex: number, previewGen: number, text: string, model: string): void {
|
|
1426
|
+
const session = sessions.get(sessionId)
|
|
1427
|
+
if (!session) return
|
|
1428
|
+
const completed = session.asrCompletedIndices ?? []
|
|
1429
|
+
if (completed.length && chunkIndex <= completed[completed.length - 1]) return
|
|
1430
|
+
liveProvisional().record(sessionId, { chunkIndex, previewGen, text, model })
|
|
1416
1431
|
}
|
|
1417
1432
|
|
|
1418
1433
|
function canonicalIndices(session: TranscriptSession): number[] {
|
|
@@ -1773,6 +1788,7 @@ function finishClosingTranscriptSession(
|
|
|
1773
1788
|
sessions.delete(sessionId)
|
|
1774
1789
|
sessionAudioBytes.delete(sessionId)
|
|
1775
1790
|
sessionAudioWrites.delete(sessionId)
|
|
1791
|
+
liveProvisional().clear(sessionId)
|
|
1776
1792
|
// Clean up inline hallucination tracking (was leaking until 4-hour interval fired)
|
|
1777
1793
|
clearSessionHallucinationState(sessionId)
|
|
1778
1794
|
// Track as deleted so orphan heartbeats get 410 Gone (prevents zombie client spam)
|
|
@@ -2088,7 +2104,7 @@ function sanitizeStreamTranscript(sessionId: string, session: TranscriptSession,
|
|
|
2088
2104
|
const dropReason = streamSilenceDropReason(trimmedText, isQuiet)
|
|
2089
2105
|
const flagAllows = dropReason === 'thankyou_silence' ? THANKYOU_FILTER : STRIP_BRAND_URLS
|
|
2090
2106
|
if (dropReason && flagAllows) {
|
|
2091
|
-
console.log(`[hallucination] Dropped (${dropReason}, q=${isQuiet ? 1 : 0}):
|
|
2107
|
+
console.log(`[hallucination] Dropped (${dropReason}, q=${isQuiet ? 1 : 0}): chars=${trimmedText.length}`)
|
|
2092
2108
|
return { text: '', fallbackReason: dropReason }
|
|
2093
2109
|
}
|
|
2094
2110
|
}
|
|
@@ -2106,7 +2122,7 @@ function sanitizeStreamTranscript(sessionId: string, session: TranscriptSession,
|
|
|
2106
2122
|
const streak = (session.vocabEchoStreak ?? 0) + 1
|
|
2107
2123
|
session.vocabEchoStreak = streak
|
|
2108
2124
|
if (isQuiet || streak >= 2 || isCrossChunkRepeat(session, trimmedText)) {
|
|
2109
|
-
console.log(`[hallucination] Dropped (vocab_echo, q=${isQuiet ? 1 : 0}, streak=${streak}):
|
|
2125
|
+
console.log(`[hallucination] Dropped (vocab_echo, q=${isQuiet ? 1 : 0}, streak=${streak}): chars=${trimmedText.length}`)
|
|
2110
2126
|
return { text: '', fallbackReason: 'vocab_echo' }
|
|
2111
2127
|
}
|
|
2112
2128
|
} else if (trimmedText) {
|
|
@@ -2377,7 +2393,7 @@ async function processStreamChunk(opts: {
|
|
|
2377
2393
|
if (alreadyCanonical?.text) {
|
|
2378
2394
|
session.lastActivityAt = Date.now()
|
|
2379
2395
|
recordReceivedChunk(session, chunkIndex)
|
|
2380
|
-
recordAsrCompleted(session, chunkIndex)
|
|
2396
|
+
recordAsrCompleted(session, chunkIndex, sessionId)
|
|
2381
2397
|
if (candidate && candidateRecordKey) {
|
|
2382
2398
|
session.providerCandidates![candidateRecordKey].accepted =
|
|
2383
2399
|
alreadyCanonical.asrProvider === 'iphone-whisperkit-beta' && alreadyCanonical.audioSha256 === audioSha256
|
|
@@ -2390,7 +2406,7 @@ async function processStreamChunk(opts: {
|
|
|
2390
2406
|
if (alreadyEmpty) {
|
|
2391
2407
|
session.lastActivityAt = Date.now()
|
|
2392
2408
|
recordReceivedChunk(session, chunkIndex)
|
|
2393
|
-
recordAsrCompleted(session, chunkIndex)
|
|
2409
|
+
recordAsrCompleted(session, chunkIndex, sessionId)
|
|
2394
2410
|
persistSessionRequired(sessionId)
|
|
2395
2411
|
return emptyChunkResponse(alreadyEmpty, sessionId, chunkIndex)
|
|
2396
2412
|
}
|
|
@@ -2524,7 +2540,7 @@ async function processStreamChunk(opts: {
|
|
|
2524
2540
|
if (labels.segments) labels.segments = labels.segments.map(segment => policyLabel(segment, capturePolicy))
|
|
2525
2541
|
|
|
2526
2542
|
if (!trimmedText) {
|
|
2527
|
-
console.log(`[hallucination] Filtered (${sanitized.fallbackReason || fallbackReason || 'empty'}):
|
|
2543
|
+
console.log(`[hallucination] Filtered (${sanitized.fallbackReason || fallbackReason || 'empty'}): chars=${rawText.length}`)
|
|
2528
2544
|
if (candidate && candidateRecordKey && session.providerCandidates?.[candidateRecordKey]) {
|
|
2529
2545
|
session.providerCandidates[candidateRecordKey].accepted = false
|
|
2530
2546
|
session.providerCandidates[candidateRecordKey].fallbackReason = sanitized.fallbackReason || fallbackReason || 'empty'
|
|
@@ -2543,7 +2559,7 @@ async function processStreamChunk(opts: {
|
|
|
2543
2559
|
}
|
|
2544
2560
|
session.emptyCompletions ??= {}
|
|
2545
2561
|
session.emptyCompletions[String(chunkIndex)] = emptyCompletion
|
|
2546
|
-
recordAsrCompleted(session, chunkIndex)
|
|
2562
|
+
recordAsrCompleted(session, chunkIndex, sessionId)
|
|
2547
2563
|
persistSessionRequired(sessionId)
|
|
2548
2564
|
scheduleSessionProgressiveHq(sessionId)
|
|
2549
2565
|
return emptyChunkResponse(emptyCompletion, sessionId, chunkIndex)
|
|
@@ -2578,12 +2594,12 @@ async function processStreamChunk(opts: {
|
|
|
2578
2594
|
session.providerCandidates[candidateRecordKey].accepted ? undefined : 'canonical_exists'
|
|
2579
2595
|
}
|
|
2580
2596
|
session.lastActivityAt = Date.now()
|
|
2581
|
-
recordAsrCompleted(session, chunkIndex)
|
|
2597
|
+
recordAsrCompleted(session, chunkIndex, sessionId)
|
|
2582
2598
|
persistSessionRequired(sessionId)
|
|
2583
2599
|
return canonicalChunkResponse(finalExisting, sessionId, chunkIndex)
|
|
2584
2600
|
}
|
|
2585
2601
|
session.chunks[chunkIndex] = chunk
|
|
2586
|
-
recordAsrCompleted(session, chunkIndex)
|
|
2602
|
+
recordAsrCompleted(session, chunkIndex, sessionId)
|
|
2587
2603
|
if (candidate && candidateRecordKey) {
|
|
2588
2604
|
if (session.providerCandidates?.[candidateRecordKey]) {
|
|
2589
2605
|
session.providerCandidates[candidateRecordKey].accepted = asrProvider === 'iphone-whisperkit-beta'
|
|
@@ -2608,7 +2624,9 @@ async function processStreamChunk(opts: {
|
|
|
2608
2624
|
// The ASR await above lets a late chunk land after Stop or save; a closed session never feeds cues.
|
|
2609
2625
|
if (!isSessionDeleted(sessionId)) void feedLiveCueTranscript(sessionId, trimmedText, Boolean(fallbackReason)).catch(() => {})
|
|
2610
2626
|
|
|
2611
|
-
|
|
2627
|
+
// 6.85.0: timing and the character count only. This line runs for every chunk, so the meeting
|
|
2628
|
+
// text itself never goes to the server log (transcribe-stream-log-privacy.test.ts).
|
|
2629
|
+
console.log(`[perf] TOTAL request: ${(performance.now() - tReq).toFixed(1)}ms | chunk #${chunkIndex} | ${audioBuffer.length}b | rms=${Math.round(rms)} q=${isQuiet ? 1 : 0} gate=${quiet.gate}${quiet.speechSec === null ? '' : ` speech=${quiet.speechSec.toFixed(2)}s`} fe=${stt.frontEnd} | ${asrProvider} | chars=${trimmedText.length}`)
|
|
2612
2630
|
return canonicalChunkResponse(chunk, sessionId, chunkIndex)
|
|
2613
2631
|
}
|
|
2614
2632
|
|
|
@@ -2688,6 +2706,7 @@ transcribeStreamRouter.post('/transcribe-stream/preview', async (req, res) => {
|
|
|
2688
2706
|
if (!cleaned || sharedIsFullHallucination(cleaned)) {
|
|
2689
2707
|
return void res.status(204).send()
|
|
2690
2708
|
}
|
|
2709
|
+
noteLiveProvisional(sessionId, chunkIndex, previewGen, cleaned, result.model)
|
|
2691
2710
|
return void res.json({
|
|
2692
2711
|
sessionId,
|
|
2693
2712
|
chunkIndex,
|
|
@@ -2706,6 +2725,20 @@ transcribeStreamRouter.post('/transcribe-stream/preview', async (req, res) => {
|
|
|
2706
2725
|
}
|
|
2707
2726
|
})
|
|
2708
2727
|
|
|
2728
|
+
/**
|
|
2729
|
+
* 6.86.0: GET /transcribe-stream/provisional?sessionId=<id>. The newest provisional text of a live meeting, for COS
|
|
2730
|
+
* Control's grey line (live-provisional.ts). Token-authenticated like every /api route; private, no-store. Memory only:
|
|
2731
|
+
* a restart, a save or a settled chunk empties it. `live` is false when this server holds no open session by that id.
|
|
2732
|
+
*/
|
|
2733
|
+
transcribeStreamRouter.get('/transcribe-stream/provisional', (req, res) => {
|
|
2734
|
+
res.set('Cache-Control', 'private, no-store')
|
|
2735
|
+
const sessionId = typeof req.query.sessionId === 'string' ? req.query.sessionId : ''
|
|
2736
|
+
if (!/^[A-Za-z0-9:_-]{3,96}$/.test(sessionId)) return void res.status(400).json({ error: 'invalid sessionId', reason: 'invalid_session_id' })
|
|
2737
|
+
const live = sessions.has(sessionId)
|
|
2738
|
+
const held = live ? liveProvisional().read(sessionId) : null
|
|
2739
|
+
res.json({ v: 1, sessionId, live, provisional: held, serverInstanceId: getServerInstanceId() })
|
|
2740
|
+
})
|
|
2741
|
+
|
|
2709
2742
|
/** Unlike the received ledger, only these canonical completions acknowledge ASR.
|
|
2710
2743
|
* A native stop supplies the total captured chunks, including its last tail. */
|
|
2711
2744
|
export function assertMacCaptureComplete(sessionId: string, expectedChunkCount: unknown): void {
|
|
@@ -2739,6 +2772,9 @@ transcribeStreamRouter.post('/mac-recording/start', (req, res) => {
|
|
|
2739
2772
|
try {
|
|
2740
2773
|
const serverInstanceId = requireMacPin(req)
|
|
2741
2774
|
if (typeof req.body?.captureId !== 'string' || !MAC_CAPTURE_ID.test(req.body.captureId)) throw macError(400, 'invalid_capture_id', 'Invalid Mac capture ID')
|
|
2775
|
+
// 6.86.0: `liveCues: true` asks for live cues on this recording (COS Control's live meeting view). Checked before
|
|
2776
|
+
// anything is written, so a bad body admits nothing.
|
|
2777
|
+
if (req.body.liveCues !== undefined && typeof req.body.liveCues !== 'boolean') throw macError(400, 'invalid_live_cues', 'liveCues must be true or false')
|
|
2742
2778
|
if (isSessionDeleted(req.body.captureId) || getMeetingStore().findBySessionId(req.body.captureId)) throw macError(410, 'session_closed', 'Capture is closed')
|
|
2743
2779
|
// An existing non-Mac session may not acquire new source metadata.
|
|
2744
2780
|
if (sessions.has(req.body.captureId) && !readMacRecording(req.body.captureId)) throw macError(409, 'mac_capture_conflict', 'Capture ID already exists')
|
|
@@ -2747,7 +2783,13 @@ transcribeStreamRouter.post('/mac-recording/start', (req, res) => {
|
|
|
2747
2783
|
const session = getSession(recording.sessionId)
|
|
2748
2784
|
if (!session.chunks.some(Boolean)) session.startTime = recording.startTime
|
|
2749
2785
|
persistSessionRequired(recording.sessionId)
|
|
2750
|
-
|
|
2786
|
+
// Arming happens only after the recording is admitted, and its answer never fails the start (armMacLiveCues returns a
|
|
2787
|
+
// reason code instead of throwing). Without `liveCues: true` the answer is byte for byte what 6.85.0 sent.
|
|
2788
|
+
let liveCues: MacLiveCuesAnswer | undefined
|
|
2789
|
+
if (req.body.liveCues === true) {
|
|
2790
|
+
try { liveCues = armMacLiveCues(recording.sessionId) } catch { liveCues = { armed: false, reason: 'unavailable' } }
|
|
2791
|
+
}
|
|
2792
|
+
res.json({ ...MAC_AUDIO, ...recording, replayed, serverInstanceId, ...(liveCues ? { liveCues } : {}) })
|
|
2751
2793
|
} catch (error) { sendStreamError(res, error) } finally { lease?.release() }
|
|
2752
2794
|
})
|
|
2753
2795
|
const macChunkLocks = new Set<string>()
|
package/server/routes/voice.ts
CHANGED
|
@@ -341,7 +341,7 @@ voiceRouter.get('/voice/ext-audio', (_req, res) => {
|
|
|
341
341
|
ageHours: parseFloat(ageHours),
|
|
342
342
|
expiresIn: `${Math.max(0, 72 - parseFloat(ageHours)).toFixed(1)}h`,
|
|
343
343
|
// 6.45.3 — the chunk indices a reviewer can ask to hear (`?chunk=` on
|
|
344
|
-
// the sample route).
|
|
344
|
+
// the sample route). Grace, 2026-09-12: the Add-a-voice panel let her
|
|
345
345
|
// name a session but not listen to it; naming is a guess without this.
|
|
346
346
|
chunkIndices: listExtAudioChunks(d.name),
|
|
347
347
|
}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// only proxies Open-Meteo / reverse-geocode — it is not "server location".
|
|
4
4
|
|
|
5
5
|
import { Router } from 'express'
|
|
6
|
-
import { callPython } from '../lib/python-bridge.js'
|
|
6
|
+
import { callPython, pythonBridgeAvailable } from '../lib/python-bridge.js'
|
|
7
7
|
|
|
8
8
|
export const welcomeContextRouter = Router()
|
|
9
9
|
|
|
@@ -24,6 +24,8 @@ let lastKnownCity = ''
|
|
|
24
24
|
|
|
25
25
|
let cachedNextEvent: NextEventPayload | null = null
|
|
26
26
|
let calendarFetchedAt = 0
|
|
27
|
+
/** 6.84.1: whether the read that filled `cachedNextEvent` really reached the calendar. */
|
|
28
|
+
let cachedCalendarRead = false
|
|
27
29
|
|
|
28
30
|
// 6.63.1: after a server restart the first forecast lookup can fail and, with nothing cached, the
|
|
29
31
|
// glasses Home got no weather until the app was relaunched. The last good weather and
|
|
@@ -282,9 +284,21 @@ function isGenericNoiseEvent(title: string): boolean {
|
|
|
282
284
|
return /\booo\b|out of office|vacation|\bpto\b/i.test(t)
|
|
283
285
|
}
|
|
284
286
|
|
|
285
|
-
|
|
287
|
+
/**
|
|
288
|
+
* 6.84.1: did the bridge answer with a calendar? False for no bridge (COS_SCRIPTS_DIR unset, or
|
|
289
|
+
* set without the venv and bridge script: callPython then answers an empty no-op), for a
|
|
290
|
+
* missing answer, and for the bridge's own error answer (`calendar_unavailable` on its
|
|
291
|
+
* deadline, or not configured). An empty calendar is a real read.
|
|
292
|
+
*/
|
|
293
|
+
export function calendarAnswerWasRead(bridgeReady: boolean, answer: unknown): boolean {
|
|
294
|
+
if (!bridgeReady) return false
|
|
295
|
+
if (!answer || typeof answer !== 'object' || Array.isArray(answer)) return false
|
|
296
|
+
return !('error' in answer)
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
async function fetchNextEvent(): Promise<{ event: NextEventPayload | null; read: boolean }> {
|
|
286
300
|
if (cachedNextEvent && Date.now() - calendarFetchedAt < CALENDAR_TTL_MS) {
|
|
287
|
-
return cachedNextEvent
|
|
301
|
+
return { event: cachedNextEvent, read: cachedCalendarRead }
|
|
288
302
|
}
|
|
289
303
|
|
|
290
304
|
try {
|
|
@@ -299,6 +313,7 @@ async function fetchNextEvent(): Promise<NextEventPayload | null> {
|
|
|
299
313
|
minutes_until_next?: number
|
|
300
314
|
} | null
|
|
301
315
|
|
|
316
|
+
const read = calendarAnswerWasRead(pythonBridgeAvailable(), cal)
|
|
302
317
|
const remaining = cal?.today_remaining ?? []
|
|
303
318
|
const meaningful = remaining.find((evt) => {
|
|
304
319
|
if (evt.is_all_day) return false
|
|
@@ -327,9 +342,11 @@ async function fetchNextEvent(): Promise<NextEventPayload | null> {
|
|
|
327
342
|
}
|
|
328
343
|
|
|
329
344
|
calendarFetchedAt = Date.now()
|
|
330
|
-
|
|
345
|
+
cachedCalendarRead = read
|
|
346
|
+
return { event: cachedNextEvent, read }
|
|
331
347
|
} catch {
|
|
332
|
-
|
|
348
|
+
// Threw or timed out: whatever is cached was not read just now.
|
|
349
|
+
return { event: cachedNextEvent, read: false }
|
|
333
350
|
}
|
|
334
351
|
}
|
|
335
352
|
|
|
@@ -342,6 +359,7 @@ export function _resetWelcomeContextCachesForTests(): void {
|
|
|
342
359
|
lastKnownCity = ''
|
|
343
360
|
cachedNextEvent = null
|
|
344
361
|
calendarFetchedAt = 0
|
|
362
|
+
cachedCalendarRead = false
|
|
345
363
|
weatherPersistence = null
|
|
346
364
|
weatherRestored = false
|
|
347
365
|
lastFailureLogAt.clear()
|
|
@@ -354,13 +372,18 @@ welcomeContextRouter.get('/welcome-context', async (req, res) => {
|
|
|
354
372
|
const pairLat = lat != null && lon != null ? lat : undefined
|
|
355
373
|
const pairLon = lat != null && lon != null ? lon : undefined
|
|
356
374
|
|
|
357
|
-
const [weather,
|
|
375
|
+
const [weather, calendar] = await Promise.all([
|
|
358
376
|
fetchWeather(pairLat, pairLon),
|
|
359
377
|
fetchNextEvent(),
|
|
360
378
|
])
|
|
379
|
+
const nextEvent = calendar.event
|
|
361
380
|
|
|
362
381
|
res.json({
|
|
363
382
|
...(weather ? { weather } : {}),
|
|
364
383
|
...(nextEvent ? { nextEvent } : {}),
|
|
384
|
+
// 6.84.1: true only when this answer's calendar came from a successful read (an empty
|
|
385
|
+
// calendar included); false when the read threw, timed out or there is no COS pipeline.
|
|
386
|
+
// Lets a client tell "no more events today" from "could not read the calendar".
|
|
387
|
+
calendarRead: calendar.read,
|
|
365
388
|
})
|
|
366
389
|
})
|
package/shared/live-cues-v2.ts
CHANGED
|
@@ -2,13 +2,29 @@
|
|
|
2
2
|
* Live cues v2 wire contract (server 6.75 candidate, companion 6.10.641 candidate; optional capability fields since 6.76.2).
|
|
3
3
|
* Additive to v1: old clients keep receiving `coaching_nudge` and ignore the new named event.
|
|
4
4
|
* The companion keeps a byte-identical copy at src/lib/live-cues-v2-contract.ts.
|
|
5
|
+
* 6.86.0 widens CueKind (additive for every meeting that did not opt in; see CueKind). A companion that has not taken this
|
|
6
|
+
* copy keeps validating the two legacy kinds, which is safe because it is never sent anything else.
|
|
5
7
|
*/
|
|
6
8
|
export const LIVE_CUES_V2_PROTOCOL = 2 as const
|
|
7
9
|
export const CUE_HEADLINE_MAX = 64
|
|
8
10
|
export const CUE_DETAIL_MAX = 220
|
|
9
11
|
export const CUE_FEEDBACK_REASON_MAX = 200
|
|
10
12
|
|
|
11
|
-
|
|
13
|
+
/** 6.86.0: three more kinds for COS Control's live meeting view. A meeting gets them only when its armer opts in (a Mac
|
|
14
|
+
* recording, or POST /live-cues/start with `kinds: 'extended'`). Every other meeting keeps the two legacy kinds, so a
|
|
15
|
+
* companion that validates `kind` against reinforce | challenge never sees a cue it would drop. Readers must treat any
|
|
16
|
+
* kind they do not know as a plain cue, never as an error. */
|
|
17
|
+
export type CueKind = 'reinforce' | 'challenge' | 'decision' | 'promise' | 'followUp'
|
|
18
|
+
export const LEGACY_CUE_KINDS = ['reinforce', 'challenge'] as const
|
|
19
|
+
export const CUE_KINDS = ['reinforce', 'challenge', 'decision', 'promise', 'followUp'] as const
|
|
20
|
+
export type CueKindSet = 'legacy' | 'extended'
|
|
21
|
+
export function isCueKind(value: unknown): value is CueKind {
|
|
22
|
+
return typeof value === 'string' && (CUE_KINDS as readonly string[]).includes(value)
|
|
23
|
+
}
|
|
24
|
+
/** The legacy kind a v1 reader is shown for any kind (the `coaching_nudge` twin and the stop answer's `nudges`). */
|
|
25
|
+
export function legacyCueKind(kind: CueKind): 'reinforce' | 'challenge' {
|
|
26
|
+
return kind === 'reinforce' || kind === 'decision' || kind === 'promise' ? 'reinforce' : 'challenge'
|
|
27
|
+
}
|
|
12
28
|
|
|
13
29
|
/** A past meeting the cue drew on (0 to 2 per cue). */
|
|
14
30
|
export interface CueSourceRef { id: string; date: string; title: string }
|
|
@@ -31,7 +47,7 @@ export interface LiveCueV2 {
|
|
|
31
47
|
}
|
|
32
48
|
|
|
33
49
|
/** v1 `coaching_nudge` as the v2 engine emits it for old clients (headline only). */
|
|
34
|
-
export interface LiveCueV1Compat { nudge: string; type:
|
|
50
|
+
export interface LiveCueV1Compat { nudge: string; type: 'reinforce' | 'challenge'; priority: 2; cueId: string; sessionId: string; protocol: 2 }
|
|
35
51
|
|
|
36
52
|
/** `reason` when unavailable: master_flag_off, api_key_in_env, decision_invalid_provider, decision_not_configured (no key
|
|
37
53
|
* for the selected decision provider; jev_not_configured when Jev was chosen and only a SubQ key exists), claude_not_found,
|
|
@@ -75,7 +91,7 @@ export function isLiveCueV2(value: unknown): value is LiveCueV2 {
|
|
|
75
91
|
return v.protocol === 2 && typeof v.cueId === 'string' && v.cueId.length > 0 && typeof v.sessionId === 'string'
|
|
76
92
|
&& typeof v.headline === 'string' && v.headline.length > 0 && v.headline.length <= CUE_HEADLINE_MAX
|
|
77
93
|
&& typeof v.detail === 'string' && v.detail.length > 0 && v.detail.length <= CUE_DETAIL_MAX
|
|
78
|
-
&& (v.kind
|
|
94
|
+
&& isCueKind(v.kind)
|
|
79
95
|
&& Array.isArray(v.sources) && v.sources.length <= 2
|
|
80
96
|
&& v.sources.every(s => !!s && typeof (s as CueSourceRef).id === 'string' && typeof (s as CueSourceRef).date === 'string' && typeof (s as CueSourceRef).title === 'string')
|
|
81
97
|
&& typeof v.gate === 'number' && typeof v.createdAt === 'number' && typeof v.model === 'string'
|