@cairnvibe/sdk 0.2.13 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-loop.d.ts +113 -0
- package/dist/agent-loop.js +128 -0
- package/dist/cairn-widget.js +2 -2
- package/dist/element-ladder.d.ts +71 -0
- package/dist/element-ladder.js +168 -0
- package/dist/index.d.ts +79 -1
- package/dist/index.js +864 -89
- package/dist/key-rotator.d.ts +28 -0
- package/dist/key-rotator.js +57 -3
- package/dist/memory-sqlite.d.ts +86 -0
- package/dist/memory-sqlite.js +230 -0
- package/dist/realtime-cli.js +22 -1
- package/dist/realtime-server.d.ts +83 -2
- package/dist/realtime-server.js +549 -120
- package/dist/server.d.ts +249 -5
- package/dist/server.js +984 -77
- package/dist/skill-store.d.ts +17 -0
- package/dist/skill-store.js +78 -0
- package/dist/tts-stream.d.ts +25 -0
- package/dist/tts-stream.js +32 -0
- package/dist/vad.d.ts +27 -0
- package/dist/vad.js +128 -0
- package/dist/verb-executor.d.ts +32 -11
- package/dist/verb-executor.js +224 -16
- package/dist/webmcp-client.d.ts +14 -1
- package/dist/webmcp-client.js +22 -1
- package/package.json +3 -1
- package/src/agent-loop.ts +222 -0
- package/src/element-ladder.ts +170 -0
- package/src/index.tsx +914 -93
- package/src/key-rotator.ts +57 -2
- package/src/memory-sqlite.ts +283 -0
- package/src/realtime-cli.ts +24 -1
- package/src/realtime-server.ts +655 -122
- package/src/server.ts +1077 -77
- package/src/skill-store.ts +88 -0
- package/src/tts-stream.ts +30 -0
- package/src/vad.ts +153 -0
- package/src/verb-executor.ts +243 -22
- package/src/web-component.ts +82 -17
- package/src/webmcp-client.ts +30 -2
package/dist/index.js
CHANGED
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
"use client";
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.CONVERSATION_STORAGE_KEY = void 0;
|
|
4
5
|
exports.Copilot = Copilot;
|
|
6
|
+
exports.loadPersistedConversation = loadPersistedConversation;
|
|
7
|
+
exports.savePersistedConversation = savePersistedConversation;
|
|
8
|
+
exports.reconstructHistoryFromPersisted = reconstructHistoryFromPersisted;
|
|
5
9
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
6
10
|
const react_1 = require("react");
|
|
7
11
|
const navigation_1 = require("next/navigation");
|
|
8
12
|
const lucide_react_1 = require("lucide-react");
|
|
9
13
|
const core_1 = require("@cairnvibe/core");
|
|
14
|
+
const agent_loop_1 = require("./agent-loop");
|
|
10
15
|
const context_collector_1 = require("./context-collector");
|
|
11
16
|
const element_ladder_1 = require("./element-ladder");
|
|
12
17
|
const runtime_scan_1 = require("./runtime-scan");
|
|
13
18
|
const webmcp_client_1 = require("./webmcp-client");
|
|
14
19
|
const verb_executor_1 = require("./verb-executor");
|
|
15
|
-
|
|
20
|
+
const vad_1 = require("./vad");
|
|
21
|
+
function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, scopeId, planEndpoint, criticEndpoint, skillsSaveEndpoint, realtimeUrl, persona = "Cairn", }) {
|
|
16
22
|
const pathname = (0, navigation_1.usePathname)() ?? "/";
|
|
17
23
|
// Mirrors `pathname` for use inside long-lived closures (a realtime
|
|
18
24
|
// session's handlers are all created once, when the connection opens —
|
|
@@ -26,6 +32,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
26
32
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
27
33
|
}, [pathname]);
|
|
28
34
|
const router = (0, navigation_1.useRouter)();
|
|
35
|
+
// Starts at the same safe default on both server and client's first
|
|
36
|
+
// render — same hydration-mismatch reason `micSupported` below does this
|
|
37
|
+
// — then, once actually restored from sessionStorage post-mount (see the
|
|
38
|
+
// dedicated restore effect further down), flips to whatever a real page
|
|
39
|
+
// reload (a host app's own mutation handler, e.g. — see
|
|
40
|
+
// loadPersistedConversation's own doc comment) had showing a moment ago,
|
|
41
|
+
// so a real reload never again looks like the conversation simply ended.
|
|
29
42
|
const [open, setOpen] = (0, react_1.useState)(false);
|
|
30
43
|
// Collapsed by default so the panel only ever shows the current exchange
|
|
31
44
|
// — the full archived transcript (built up over a long conversation)
|
|
@@ -34,6 +47,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
34
47
|
const [historyExpanded, setHistoryExpanded] = (0, react_1.useState)(false);
|
|
35
48
|
const [question, setQuestion] = (0, react_1.useState)("");
|
|
36
49
|
const [answer, setAnswer] = (0, react_1.useState)(null);
|
|
50
|
+
// Real, live-reported gap this closes: once a continuing agent-loop step
|
|
51
|
+
// (click/fill/read/call_tool/batch) sets `answer` to its own progress
|
|
52
|
+
// text ("Typing earbuds into the search box"), that text just sat there
|
|
53
|
+
// unchanged for however long the NEXT resolveVerb call took — several
|
|
54
|
+
// real seconds, more under rate-limit retries — with nothing on screen
|
|
55
|
+
// telling the user the agent was still actually doing something. `answer`
|
|
56
|
+
// itself can't double as that signal (a terminal turn's own real,
|
|
57
|
+
// finished answer looks identical to unfinished progress text). This is
|
|
58
|
+
// a separate, explicit flag: true from the moment a continuing step's
|
|
59
|
+
// progress text is shown until the turn actually ends (a terminal verb,
|
|
60
|
+
// an error, or a give-up) — see its own setters below for exactly where.
|
|
61
|
+
const [loopWorking, setLoopWorking] = (0, react_1.useState)(false);
|
|
37
62
|
const [status, setStatus] = (0, react_1.useState)("idle");
|
|
38
63
|
const [caption, setCaption] = (0, react_1.useState)("");
|
|
39
64
|
// The user's own last question, shown as its own floating caption bubble
|
|
@@ -106,6 +131,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
106
131
|
const rtMicMutedRef = (0, react_1.useRef)(false);
|
|
107
132
|
const rtSpeakerMutedRef = (0, react_1.useRef)(false);
|
|
108
133
|
const rtStartingRef = (0, react_1.useRef)(false); // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
|
|
134
|
+
// The generation of the most recent "final" this client has processed —
|
|
135
|
+
// see ServerMessage's own doc comment in realtime-server.ts for the full
|
|
136
|
+
// real, live-found bug this closes: the client's own local barge-in can
|
|
137
|
+
// start a NEW turn (a new "final") before an EARLIER turn's own verb/
|
|
138
|
+
// audio, already in flight on the wire when the server processed that
|
|
139
|
+
// barge-in, actually arrives. WebSocket delivers messages in order, but
|
|
140
|
+
// "in order" isn't "still current" — every verb/speaking_start/
|
|
141
|
+
// audio_chunk/speaking_end/turn_complete message carries the generation
|
|
142
|
+
// it was produced under, and the handler drops it outright if it's
|
|
143
|
+
// older than this ref's value instead of applying it to whatever
|
|
144
|
+
// caption happens to be showing now.
|
|
145
|
+
const rtLastFinalGenerationRef = (0, react_1.useRef)(0);
|
|
109
146
|
// Progressive PCM playback for the buffered (non-realtime) speak endpoint
|
|
110
147
|
// — the same gapless AudioBufferSourceNode scheduling the realtime path
|
|
111
148
|
// uses for its audio_chunk messages (see rtPlaybackCtxRef below), just fed
|
|
@@ -118,6 +155,37 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
118
155
|
const typedPlaybackGainRef = (0, react_1.useRef)(null);
|
|
119
156
|
const typedNextPlayTimeRef = (0, react_1.useRef)(0);
|
|
120
157
|
const typedScheduledSourcesRef = (0, react_1.useRef)([]);
|
|
158
|
+
// Bumped by stopTypedPlayback() every time it runs — playPcmStream's own
|
|
159
|
+
// async reader loop checks this before scheduling each chunk, so a
|
|
160
|
+
// superseded call (two speak()/ask() replies resolving close together)
|
|
161
|
+
// actually STOPS reading and scheduling more audio once a newer call has
|
|
162
|
+
// taken over, instead of continuing to push nodes into the shared
|
|
163
|
+
// playback graph behind the newer call's back. stopTypedPlayback() on
|
|
164
|
+
// its own only ever stopped nodes that already existed at the moment it
|
|
165
|
+
// ran — it never told an in-flight stream reader to stop producing MORE
|
|
166
|
+
// of them, which is exactly what let two replies' audio genuinely
|
|
167
|
+
// overlap (one starting, then a second call's audio starting on top of
|
|
168
|
+
// it moments later) — a real, live-found race, not a guess.
|
|
169
|
+
const typedPlaybackGenerationRef = (0, react_1.useRef)(0);
|
|
170
|
+
// A DIFFERENT real gap the generation counter above doesn't close: it
|
|
171
|
+
// only protects one typed reply's audio against ANOTHER typed reply's
|
|
172
|
+
// audio. A typed ask()/speak() call already in flight — its /api/
|
|
173
|
+
// copilot/speak fetch genuinely takes several seconds under real
|
|
174
|
+
// conditions (rate-limit retries make this worse, not better) — has no
|
|
175
|
+
// way to know a realtime call started WHILE it was still waiting. Its
|
|
176
|
+
// response arrives, and normally-innocent code plays it, seconds after
|
|
177
|
+
// startRealtime() already ran — genuinely overlapping with the live
|
|
178
|
+
// call's own audio, since nothing about the realtime session's own
|
|
179
|
+
// start/mute/barge-in controls have any way to reach a typed reply
|
|
180
|
+
// that hadn't even been scheduled yet when they ran. Found live: two
|
|
181
|
+
// full agent answers, sourced entirely from separate /api/copilot/
|
|
182
|
+
// speak calls, audibly overlapping about a second apart, while a
|
|
183
|
+
// realtime call was the only thing visibly active in the UI the whole
|
|
184
|
+
// time. startRealtime() sets this; endRealtime() clears it; speak()/
|
|
185
|
+
// speakAndWait() check it AFTER their fetch resolves and drop the
|
|
186
|
+
// reply's audio entirely (never call playPcmStream at all) if a
|
|
187
|
+
// realtime call has taken over since the request was made.
|
|
188
|
+
const typedPlaybackSuspendedRef = (0, react_1.useRef)(false);
|
|
121
189
|
// Watchdog for the "rt-thinking" state: started on every "final" transcript,
|
|
122
190
|
// cleared the moment the server responds with anything for that turn
|
|
123
191
|
// (verb/speaking_start/speaking_end/turn_complete/error). If it ever
|
|
@@ -140,6 +208,36 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
140
208
|
// finished arriving, so the mic can't start sending while the agent is
|
|
141
209
|
// still audibly speaking.
|
|
142
210
|
const rtAudioDoneArrivingRef = (0, react_1.useRef)(true);
|
|
211
|
+
// Diagnostic only, not a functional guard — flips to true the first time
|
|
212
|
+
// a real mic packet is actually sent after transitioning to
|
|
213
|
+
// "rt-listening", logged once (not per-packet, which would flood the
|
|
214
|
+
// console). Added specifically so a "status says Listening… but nothing
|
|
215
|
+
// I say gets picked up" report can be told apart, from the log alone,
|
|
216
|
+
// between "the send gate never opened" (this never logs) and "the gate
|
|
217
|
+
// opened and sent real audio, so the problem is somewhere else entirely
|
|
218
|
+
// — Deepgram's own STT, or a real hardware/OS mic issue this app can't
|
|
219
|
+
// see or fix" (this logs once, then goes quiet as expected).
|
|
220
|
+
const micAudioSentSinceListeningRef = (0, react_1.useRef)(false);
|
|
221
|
+
// Safety net for a live realtime call outliving this component instance:
|
|
222
|
+
// without this, unmounting (a parent removing the widget, a route change
|
|
223
|
+
// that remounts it, or — the real, live-hit case — Next.js Fast Refresh
|
|
224
|
+
// remounting the component on every dev-mode source edit while a call is
|
|
225
|
+
// open) left the WebSocket, the open getUserMedia mic stream, and both
|
|
226
|
+
// AudioContexts running completely orphaned: nothing ever called
|
|
227
|
+
// rtSocketRef.current?.close() or stopped the mic tracks. The old,
|
|
228
|
+
// zombie connection then kept transcribing and replying in parallel with
|
|
229
|
+
// whatever the newly-mounted instance does next — the literal
|
|
230
|
+
// "two things running in parallel, the agent answering twice" bug found
|
|
231
|
+
// live. endRealtime() is safe to call from an unmount cleanup even
|
|
232
|
+
// though it also calls React state setters: it only reads stable refs
|
|
233
|
+
// (never a stale closure over props/state), and React 18 silently no-ops
|
|
234
|
+
// a setState call on an already-unmounted component.
|
|
235
|
+
(0, react_1.useEffect)(() => {
|
|
236
|
+
return () => {
|
|
237
|
+
if (rtSocketRef.current || rtCleanupRef.current)
|
|
238
|
+
endRealtime();
|
|
239
|
+
};
|
|
240
|
+
}, []);
|
|
143
241
|
// Starts false on both server and client's first render (avoids a
|
|
144
242
|
// hydration mismatch — `navigator` doesn't exist during SSR), then
|
|
145
243
|
// updated after mount, once we're only ever running in the browser.
|
|
@@ -147,6 +245,36 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
147
245
|
(0, react_1.useEffect)(() => {
|
|
148
246
|
setMicSupported(!!navigator.mediaDevices?.getUserMedia && typeof MediaRecorder !== "undefined");
|
|
149
247
|
}, []);
|
|
248
|
+
// Restores a conversation that survived a real page reload — same
|
|
249
|
+
// hydration-safety reason as `micSupported` just above: `open`/`answer`/
|
|
250
|
+
// `lastQuestion`/`transcript` all start at their normal empty defaults on
|
|
251
|
+
// both server and client's first render (matching what SSR produced), and
|
|
252
|
+
// only flip to whatever sessionStorage actually holds here, post-mount.
|
|
253
|
+
// Real, live-verified bug this closes: a host app's own mutation handler
|
|
254
|
+
// calling `window.location.reload()` (this SDK's own demo app does this
|
|
255
|
+
// after several real actions, e.g. moving a kanban card) tears down the
|
|
256
|
+
// entire React tree, this widget included — every bit of visible
|
|
257
|
+
// conversation state reset to nothing, making an in-progress conversation
|
|
258
|
+
// look like it had simply ended the instant the host page happened to
|
|
259
|
+
// reload, even though nothing about the CONVERSATION itself was over.
|
|
260
|
+
(0, react_1.useEffect)(() => {
|
|
261
|
+
const persisted = loadPersistedConversation();
|
|
262
|
+
if (!persisted)
|
|
263
|
+
return;
|
|
264
|
+
setOpen(persisted.open);
|
|
265
|
+
setAnswer(persisted.answer);
|
|
266
|
+
setLastQuestion(persisted.lastQuestion);
|
|
267
|
+
setTranscript(persisted.transcript);
|
|
268
|
+
historyRef.current = reconstructHistoryFromPersisted(persisted);
|
|
269
|
+
// Starts past the restored transcript's own highest id — otherwise the
|
|
270
|
+
// very next archived entry would reuse an id already on screen, a real
|
|
271
|
+
// duplicate-React-key bug (React silently confuses which DOM node is
|
|
272
|
+
// which when two list items share a key).
|
|
273
|
+
if (persisted.transcript.length > 0) {
|
|
274
|
+
transcriptIdRef.current = Math.max(...persisted.transcript.map((t) => t.id)) + 1;
|
|
275
|
+
}
|
|
276
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
277
|
+
}, []);
|
|
150
278
|
const asking = status === "asking";
|
|
151
279
|
const recording = status === "recording";
|
|
152
280
|
const realtimeActive = status.startsWith("rt-");
|
|
@@ -166,6 +294,25 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
166
294
|
(0, react_1.useEffect)(() => {
|
|
167
295
|
answerRef.current = answer;
|
|
168
296
|
}, [answer]);
|
|
297
|
+
// Persists the visible conversation to sessionStorage on every change, so
|
|
298
|
+
// a host app's own `window.location.reload()` (e.g. after a board-card
|
|
299
|
+
// move — see loadPersistedConversation's own comment for the full story)
|
|
300
|
+
// doesn't make an in-progress conversation look like it simply ended.
|
|
301
|
+
// Skips its own very first (mount) invocation on purpose: that call is
|
|
302
|
+
// always tied to the initial render's plain defaults (open:false,
|
|
303
|
+
// transcript:[], ...) — captured before the restore effect above's
|
|
304
|
+
// setState calls have actually landed — so persisting it would clobber a
|
|
305
|
+
// real, just-restored conversation with empty state for one tick, right
|
|
306
|
+
// before the corrective post-restore render fixes it back. Skipping costs
|
|
307
|
+
// nothing on a genuinely fresh session (there's nothing to persist yet).
|
|
308
|
+
const skippedMountPersistRef = (0, react_1.useRef)(false);
|
|
309
|
+
(0, react_1.useEffect)(() => {
|
|
310
|
+
if (!skippedMountPersistRef.current) {
|
|
311
|
+
skippedMountPersistRef.current = true;
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
savePersistedConversation({ transcript, lastQuestion, answer, open });
|
|
315
|
+
}, [transcript, lastQuestion, answer, open]);
|
|
169
316
|
// Auto-scroll to the newest content whenever the transcript grows or the
|
|
170
317
|
// live (not-yet-archived) bubble's text changes.
|
|
171
318
|
(0, react_1.useEffect)(() => {
|
|
@@ -235,11 +382,38 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
235
382
|
}
|
|
236
383
|
}
|
|
237
384
|
function handleVerb(raw) {
|
|
385
|
+
setLoopWorking(false); // only ever called for a genuinely terminal verb — the turn is over
|
|
238
386
|
(0, verb_executor_1.executeVerbResponse)(raw, pathname, {
|
|
239
387
|
onExplain: (text) => {
|
|
240
388
|
setAnswer(text);
|
|
241
|
-
|
|
242
|
-
|
|
389
|
+
// A REAL, deep, long-standing bug found live — not introduced by
|
|
390
|
+
// today's other fixes, just newly diagnosed: this function is a
|
|
391
|
+
// plain closure defined fresh every render, but when a "verb" WS
|
|
392
|
+
// message arrives it's invoked through ws.onmessage — a callback
|
|
393
|
+
// assigned ONCE, inside startRealtime(), and never reassigned for
|
|
394
|
+
// the rest of that connection's life. `realtimeActive` there is
|
|
395
|
+
// therefore frozen at whatever it was AT THE MOMENT startRealtime()
|
|
396
|
+
// was called — which is BEFORE the click handler's own state
|
|
397
|
+
// updates land, so it reads `false` for literally the entire
|
|
398
|
+
// lifetime of every realtime call. `!realtimeActive` was therefore
|
|
399
|
+
// ALWAYS true here, on every single realtime turn — this ran
|
|
400
|
+
// speak() (the typed HTTP path) in addition to the correct
|
|
401
|
+
// realtime audio_chunk playback, every time. This is the actual,
|
|
402
|
+
// original root cause of "two speakers" — the earlier
|
|
403
|
+
// typedPlaybackSuspendedRef fix only ever suppressed the resulting
|
|
404
|
+
// AUDIO once this was already firing, it never stopped the
|
|
405
|
+
// firing itself (or the wasted LLM/TTS call and quota burn
|
|
406
|
+
// underneath it). rtStateRef mirrors status specifically to avoid
|
|
407
|
+
// this class of bug in a callback like this one (see its own doc
|
|
408
|
+
// comment) — using it here instead of the stale const is the
|
|
409
|
+
// actual fix.
|
|
410
|
+
if (rtStateRef.current.startsWith("rt-")) {
|
|
411
|
+
rtLog("explain: realtime call active, letting the socket's own audio_chunk stream speak this");
|
|
412
|
+
}
|
|
413
|
+
else {
|
|
414
|
+
rtLog("explain: no realtime call active, using the typed speak() HTTP path");
|
|
415
|
+
void speak(text);
|
|
416
|
+
}
|
|
243
417
|
},
|
|
244
418
|
onNavigate: (route) => router.push(route),
|
|
245
419
|
onMiss: reportMiss,
|
|
@@ -267,7 +441,16 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
267
441
|
*/
|
|
268
442
|
async function runTour(steps) {
|
|
269
443
|
const myGeneration = ++tourGenerationRef.current;
|
|
270
|
-
|
|
444
|
+
// Same real stale-closure bug as onExplain's own fix above, and the
|
|
445
|
+
// reason a realtime-triggered tour spoke through the wrong pipeline
|
|
446
|
+
// (or, after that fix suppressed the wrong pipeline's audio, spoke
|
|
447
|
+
// through nothing at all — "tour did not speak anything") and could
|
|
448
|
+
// leave the mic never properly told to resume listening afterward
|
|
449
|
+
// (see the `setRtStatus("rt-listening")` call at the end of this
|
|
450
|
+
// function, fixed the same way). rtStateRef.current is always
|
|
451
|
+
// current, regardless of which render's closure this particular
|
|
452
|
+
// invocation runs inside.
|
|
453
|
+
const wasRealtimeListening = rtStateRef.current.startsWith("rt-");
|
|
271
454
|
touringRef.current = true;
|
|
272
455
|
if (wasRealtimeListening)
|
|
273
456
|
setRtStatus("rt-speaking");
|
|
@@ -350,7 +533,31 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
350
533
|
return;
|
|
351
534
|
setTourStep(null);
|
|
352
535
|
setCaption("");
|
|
353
|
-
|
|
536
|
+
// The most damaging half of this stale-closure bug: this used to
|
|
537
|
+
// read the stale `realtimeActive` const, which meant this call was
|
|
538
|
+
// ALWAYS skipped for a tour reached via realtime — the mic was
|
|
539
|
+
// never explicitly told to resume listening once the tour ended.
|
|
540
|
+
// rtStateRef.current.startsWith("rt-") is what actually reflects
|
|
541
|
+
// whether the connection is still live right now.
|
|
542
|
+
if (wasRealtimeListening && rtStateRef.current.startsWith("rt-"))
|
|
543
|
+
setRtStatus("rt-listening");
|
|
544
|
+
}
|
|
545
|
+
catch (err) {
|
|
546
|
+
// Real, live-found gap: this function had NO catch at all — only
|
|
547
|
+
// try/finally. Any error thrown anywhere in the loop above (a
|
|
548
|
+
// rejected speakOverRealtime/speakAndWait call, a DOM exception
|
|
549
|
+
// from el.click(), a network failure) skipped the resume-listening
|
|
550
|
+
// line above ENTIRELY, leaving the mic stuck exactly where the
|
|
551
|
+
// tour left off — matching "after this it's not listening"
|
|
552
|
+
// reported live. Every other place in this file that can fail
|
|
553
|
+
// mid-turn (ask(), handleDeepgramMessage's own server-side
|
|
554
|
+
// equivalent) already guarantees SOME recovery path; this one
|
|
555
|
+
// didn't have one at all.
|
|
556
|
+
console.error("[cairn] tour failed partway through:", err);
|
|
557
|
+
rtLog("tour failed — forcing the mic back to listening instead of leaving it stuck", { error: String(err) });
|
|
558
|
+
setTourStep(null);
|
|
559
|
+
setCaption("");
|
|
560
|
+
if (wasRealtimeListening && rtStateRef.current.startsWith("rt-"))
|
|
354
561
|
setRtStatus("rt-listening");
|
|
355
562
|
}
|
|
356
563
|
finally {
|
|
@@ -371,13 +578,114 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
371
578
|
await runTypedAgentLoop(q);
|
|
372
579
|
}
|
|
373
580
|
catch {
|
|
374
|
-
|
|
581
|
+
// See typedPlaybackSuspendedRef's own doc comment — this whole
|
|
582
|
+
// fetch can still be in flight when a realtime call starts.
|
|
583
|
+
if (!typedPlaybackSuspendedRef.current)
|
|
584
|
+
setAnswer("Something went wrong reaching the help service — try again in a moment.");
|
|
375
585
|
}
|
|
376
586
|
finally {
|
|
377
|
-
|
|
587
|
+
// The most damaging form of the same race: unconditionally forcing
|
|
588
|
+
// status back to "idle" here, after a realtime call has ALREADY
|
|
589
|
+
// taken over (status is some "rt-*" value), would silently kick the
|
|
590
|
+
// UI out of the live call — hiding its mic/speaker/hangup controls
|
|
591
|
+
// and showing the "start call" screen instead — while the actual
|
|
592
|
+
// WebSocket connection underneath is still fully alive and still
|
|
593
|
+
// talking, now with no visible way to manage it at all. Skipping
|
|
594
|
+
// this reset when suspended is what stops a slow, stale typed
|
|
595
|
+
// request from ever being able to do that.
|
|
596
|
+
if (!typedPlaybackSuspendedRef.current)
|
|
597
|
+
setStatus("idle");
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
/**
|
|
601
|
+
* Architecture Pillar 6 (the safety layer) — the real, working default
|
|
602
|
+
* confirmation UI: a native browser confirm dialog, naming the tool's
|
|
603
|
+
* OWN real name/description (never inventing wording), for a WebMCP
|
|
604
|
+
* tool whose registration declared `riskTier: "confirm"` (a payment, a
|
|
605
|
+
* delete, anything hard to undo). `window.confirm` blocks the calling
|
|
606
|
+
* microtask until the user actually answers — exactly the real,
|
|
607
|
+
* synchronous "get a genuine yes before this runs" behavior needed
|
|
608
|
+
* here, and simple enough to need no new UI component for this to be a
|
|
609
|
+
* real, functioning default rather than just plumbing with nothing on
|
|
610
|
+
* the other end. A host app wanting a nicer in-widget modal can still
|
|
611
|
+
* build one — this function is the only thing that would need
|
|
612
|
+
* replacing to do that.
|
|
613
|
+
*/
|
|
614
|
+
function confirmToolCall(tool) {
|
|
615
|
+
if (typeof window === "undefined" || typeof window.confirm !== "function")
|
|
616
|
+
return Promise.resolve(false);
|
|
617
|
+
const message = tool.description ? `${tool.name}: ${tool.description}\n\nAllow this action?` : `Allow "${tool.name}"?`;
|
|
618
|
+
return Promise.resolve(window.confirm(message));
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* Architecture Pillar 4 — the typed transport's own Planner call,
|
|
622
|
+
* mirroring resolvePlan's real network shape but reached over HTTP
|
|
623
|
+
* (planEndpoint's server-side handler — createPlanHandler in server.ts
|
|
624
|
+
* — is the only place that can hold the real LLM API key). Never
|
|
625
|
+
* throws: any network/parse failure degrades to the exact same single-
|
|
626
|
+
* task fallback plan resolvePlan itself falls back to on an LLM error,
|
|
627
|
+
* so a Planner hiccup never blocks the turn.
|
|
628
|
+
*/
|
|
629
|
+
async function fetchPlan(goal, version = 1) {
|
|
630
|
+
try {
|
|
631
|
+
const res = await fetch(planEndpoint, {
|
|
632
|
+
method: "POST",
|
|
633
|
+
headers: { "content-type": "application/json" },
|
|
634
|
+
body: JSON.stringify({ goal, version }),
|
|
635
|
+
});
|
|
636
|
+
const data = await res.json();
|
|
637
|
+
if (data && typeof data === "object" && Array.isArray(data.tasks) && data.tasks.length > 0)
|
|
638
|
+
return data;
|
|
378
639
|
}
|
|
640
|
+
catch {
|
|
641
|
+
// Falls through to the same fallback plan shape below.
|
|
642
|
+
}
|
|
643
|
+
return { version, goal, facts: [], tasks: [{ id: "t1", description: goal, doneContract: "The stated goal has been achieved.", status: "in_progress" }] };
|
|
644
|
+
}
|
|
645
|
+
/** Same real-network shape as fetchPlan, for criticEndpoint's
|
|
646
|
+
* createCriticHandler — degrades to a safe "continue" verdict on any
|
|
647
|
+
* failure, same resilience discipline as resolveCritic itself. */
|
|
648
|
+
async function fetchCriticVerdict(task, goal, verb, observation) {
|
|
649
|
+
try {
|
|
650
|
+
const res = await fetch(criticEndpoint, {
|
|
651
|
+
method: "POST",
|
|
652
|
+
headers: { "content-type": "application/json" },
|
|
653
|
+
body: JSON.stringify({ task, goal, verb, observation: observation ?? null }),
|
|
654
|
+
});
|
|
655
|
+
const data = await res.json();
|
|
656
|
+
if (data && typeof data.verdict === "string")
|
|
657
|
+
return data;
|
|
658
|
+
}
|
|
659
|
+
catch {
|
|
660
|
+
// Falls through to the safe default below.
|
|
661
|
+
}
|
|
662
|
+
return { verdict: "continue", reasoning: "Critic call failed — defaulting to continue rather than blocking the turn." };
|
|
663
|
+
}
|
|
664
|
+
/**
|
|
665
|
+
* Architecture Pillar 3 (Skill half) — the typed transport's own
|
|
666
|
+
* Formulator save, mirroring realtime-server.ts's own post-turn
|
|
667
|
+
* `compileSkill`+`saveSkill` call. Fire-and-forget (never awaited by
|
|
668
|
+
* the caller, never allowed to affect what the user sees) since saving
|
|
669
|
+
* a Skill is bookkeeping for a FUTURE turn, not part of answering this
|
|
670
|
+
* one — matches the Formulator's own "cheap, runs once per turn, never
|
|
671
|
+
* blocks anything" framing. Classifies the current live page for a
|
|
672
|
+
* best-effort pattern tag the same way resolveVerb's own per-request
|
|
673
|
+
* classification does server-side.
|
|
674
|
+
*/
|
|
675
|
+
function saveSkillIfLearned(goal, learnedFacts) {
|
|
676
|
+
if (!skillsSaveEndpoint || learnedFacts.length === 0)
|
|
677
|
+
return;
|
|
678
|
+
const matches = (0, core_1.classifyUiPattern)((0, core_1.deriveStructureSignals)(liveRegistryRef.current.getSnapshot().elements));
|
|
679
|
+
void fetch(skillsSaveEndpoint, {
|
|
680
|
+
method: "POST",
|
|
681
|
+
headers: { "content-type": "application/json" },
|
|
682
|
+
body: JSON.stringify({ goal, learnedFacts, pattern: matches[0]?.pattern }),
|
|
683
|
+
}).catch(() => {
|
|
684
|
+
// A Skill that fails to save just means the next similar goal
|
|
685
|
+
// starts from scratch again, same as if nothing had been learned
|
|
686
|
+
// this turn — never worth surfacing as a user-visible error.
|
|
687
|
+
});
|
|
379
688
|
}
|
|
380
|
-
const MAX_LOOP_ITERATIONS = 6; // a hard cap, not a target — see runTypedAgentLoop
|
|
381
689
|
/**
|
|
382
690
|
* Drives the agent loop over the stateless HTTP path: ask the server,
|
|
383
691
|
* and if it comes back with a continuing step (click/fill/read/
|
|
@@ -390,58 +698,196 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
390
698
|
* still knows what it was actually asked. `historyRef` (the
|
|
391
699
|
* conversation's real memory) is only ever committed once, at the end —
|
|
392
700
|
* a turn that hits the cap mid-loop doesn't leave partial noise in it.
|
|
701
|
+
* The loop itself (the `for`/TERMINAL_VERBS/iteration-cap shape) lives
|
|
702
|
+
* in agent-loop.ts, shared with the realtime relay's own finalizeTurn —
|
|
703
|
+
* this function owns everything transport-specific: the actual fetch,
|
|
704
|
+
* the raw/untyped response handling a stateless HTTP call needs (unlike
|
|
705
|
+
* realtime's always-valid in-process resolveVerb call), and the real
|
|
706
|
+
* historyRef commit.
|
|
393
707
|
*/
|
|
394
708
|
async function runTypedAgentLoop(q) {
|
|
395
|
-
let loopHistory = historyRef.current;
|
|
396
709
|
const webMcpTools = await (0, webmcp_client_1.discoverWebMcpTools)();
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
710
|
+
let lastRawResponse = null;
|
|
711
|
+
// Architecture Pillar 4 — real Planner/Critic wiring for the typed
|
|
712
|
+
// transport, opt-in via planEndpoint/criticEndpoint (see their own
|
|
713
|
+
// doc comments on CopilotProps) — closes the gap the plan file names
|
|
714
|
+
// directly ("the typed/HTTP path has zero Planner/Critic wiring at
|
|
715
|
+
// all... today explicitly realtime-only by deferral, not by
|
|
716
|
+
// decision"). Mirrors realtime-server.ts's finalizeTurn: an eager
|
|
717
|
+
// Planner kickoff when looksMultiStep(q) already flags a probable
|
|
718
|
+
// compound goal, a lazy fallback kickoff on the first continuing step
|
|
719
|
+
// otherwise, and a genuinely separate Critic pass over each
|
|
720
|
+
// continuing step's real result. Neither endpoint set (the default)
|
|
721
|
+
// means plannerEnabled is false and this whole block is a no-op —
|
|
722
|
+
// the typed loop behaves exactly as it always has.
|
|
723
|
+
let planPromise = null;
|
|
724
|
+
let plan = null;
|
|
725
|
+
let progress = null;
|
|
726
|
+
const STALL_THRESHOLD = 3; // same bounded budget realtime's own Critic wiring uses
|
|
727
|
+
const plannerEnabled = Boolean(planEndpoint && criticEndpoint);
|
|
728
|
+
if (plannerEnabled && (0, agent_loop_1.looksMultiStep)(q))
|
|
729
|
+
planPromise = fetchPlan(q);
|
|
730
|
+
// Architecture Pillar 3 (Skill half) — every real, Critic-verified
|
|
731
|
+
// learnedFact from this turn's steps; saved once the turn concludes,
|
|
732
|
+
// below (saveSkillIfLearned). Empty is the common case, not a gap.
|
|
733
|
+
const learnedFacts = [];
|
|
734
|
+
const result = await (0, agent_loop_1.driveAgentLoop)(historyRef.current, {
|
|
735
|
+
async getNextStep(loopHistory) {
|
|
736
|
+
const liveScan = liveRegistryRef.current.getSnapshot();
|
|
737
|
+
liveMapRef.current = liveScan.byId;
|
|
738
|
+
const res = await fetch(endpoint, {
|
|
739
|
+
method: "POST",
|
|
740
|
+
headers: { "content-type": "application/json" },
|
|
741
|
+
body: JSON.stringify({
|
|
742
|
+
// pathnameRef, not the closed-over `pathname` — a navigate
|
|
743
|
+
// step (now possibly continuing, see isTerminalVerb) can
|
|
744
|
+
// change the real route mid-loop; this whole async function's
|
|
745
|
+
// own `pathname` closure was captured once, at the render
|
|
746
|
+
// that started this turn, and never updates again on its own.
|
|
747
|
+
route: pathnameRef.current,
|
|
748
|
+
question: q,
|
|
749
|
+
visible: (0, context_collector_1.collectVisible)(),
|
|
750
|
+
history: loopHistory,
|
|
751
|
+
liveElements: liveScan.elements,
|
|
752
|
+
webMcpTools,
|
|
753
|
+
scopeId,
|
|
754
|
+
}),
|
|
755
|
+
});
|
|
756
|
+
const data = await res.json().catch(() => null);
|
|
757
|
+
lastRawResponse = data;
|
|
758
|
+
return (0, core_1.safeParseVerbResponse)(data);
|
|
759
|
+
},
|
|
760
|
+
onStep({ verb, terminal }) {
|
|
761
|
+
// A continuing step — show it happening (execution itself
|
|
762
|
+
// happens in executeStep below). Terminal steps are handled once
|
|
763
|
+
// driveAgentLoop returns, via handleVerb — unchanged from before.
|
|
764
|
+
// Same stale-typed-reply guard as the terminal case below — a
|
|
765
|
+
// multi-step typed loop can still be mid-flight when a realtime
|
|
766
|
+
// call starts.
|
|
767
|
+
if (!terminal && !typedPlaybackSuspendedRef.current) {
|
|
768
|
+
setAnswer(summarizeVerbForHistory(verb));
|
|
769
|
+
setLoopWorking(true);
|
|
770
|
+
}
|
|
771
|
+
// The lazy fallback — only fires when looksMultiStep missed
|
|
772
|
+
// (planPromise is still null): a real Plan is still guaranteed
|
|
773
|
+
// before the Critic needs one, just one round trip later.
|
|
774
|
+
if (!terminal && plannerEnabled && !planPromise)
|
|
775
|
+
planPromise = fetchPlan(q);
|
|
776
|
+
return false;
|
|
777
|
+
},
|
|
778
|
+
// Same real, live-found fix as the realtime WS "verb" handler's own
|
|
779
|
+
// executeToolStep call — a fresh scan per step, not the turn's
|
|
780
|
+
// frozen liveMapRef, so a step that reveals new DOM (a click that
|
|
781
|
+
// opens a modal) doesn't leave the NEXT step unable to find
|
|
782
|
+
// anything in it.
|
|
783
|
+
executeStep: (verb) => (0, verb_executor_1.executeToolStep)(verb, pathnameRef.current, liveRegistryRef.current.getSnapshot().byId, (route) => router.push(route), confirmToolCall).then((r) => r?.observation),
|
|
784
|
+
runCritic: plannerEnabled
|
|
785
|
+
? async ({ verb, observation }) => {
|
|
786
|
+
// Real state, not the Executor's self-report — see
|
|
787
|
+
// resolveCritic's own doc comment (server.ts) for why this is
|
|
788
|
+
// a genuinely separate pass, same precedent realtime already
|
|
789
|
+
// established.
|
|
790
|
+
if (!plan) {
|
|
791
|
+
plan = planPromise ? await planPromise : await fetchPlan(q);
|
|
792
|
+
progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
|
|
793
|
+
}
|
|
794
|
+
const currentProgress = progress;
|
|
795
|
+
const currentTask = plan.tasks[currentProgress.currentTaskIndex];
|
|
796
|
+
const verdict = await fetchCriticVerdict(currentTask, q, verb, observation);
|
|
797
|
+
if (verdict.learnedFact)
|
|
798
|
+
learnedFacts.push(verdict.learnedFact);
|
|
799
|
+
if (verdict.verdict === "task_complete") {
|
|
800
|
+
currentTask.status = "done";
|
|
801
|
+
if (currentProgress.currentTaskIndex < plan.tasks.length - 1) {
|
|
802
|
+
// More tasks remain — advance and keep looping instead of
|
|
803
|
+
// ending the turn here.
|
|
804
|
+
currentProgress.currentTaskIndex++;
|
|
805
|
+
plan.tasks[currentProgress.currentTaskIndex].status = "in_progress";
|
|
806
|
+
currentProgress.stallCount = 0;
|
|
807
|
+
return { ...verdict, verdict: "continue" };
|
|
808
|
+
}
|
|
809
|
+
// The last task is genuinely done — end the loop right here
|
|
810
|
+
// instead of asking the model again and hoping it notices.
|
|
811
|
+
return verdict;
|
|
812
|
+
}
|
|
813
|
+
if (verdict.verdict === "replan") {
|
|
814
|
+
plan = await fetchPlan(q, plan.version + 1);
|
|
815
|
+
progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
|
|
816
|
+
return { ...verdict, verdict: "continue" };
|
|
817
|
+
}
|
|
818
|
+
if (verdict.verdict === "give_up")
|
|
819
|
+
return verdict;
|
|
820
|
+
// "continue" — a harness-enforced fail-safe on top of the
|
|
821
|
+
// Critic's own judgment, same Magentic-One-shaped two-tier
|
|
822
|
+
// tolerance realtime already uses.
|
|
823
|
+
currentProgress.stallCount++;
|
|
824
|
+
if (currentProgress.stallCount >= STALL_THRESHOLD) {
|
|
825
|
+
return {
|
|
826
|
+
verdict: "give_up",
|
|
827
|
+
reasoning: `Stuck after ${currentProgress.stallCount} steps with no confirmed progress on "${currentTask.description}" — ${verdict.reasoning}`,
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
return verdict;
|
|
831
|
+
}
|
|
832
|
+
: undefined,
|
|
833
|
+
});
|
|
834
|
+
// Architecture Pillar 3 (Skill half) — the Formulator, once per turn.
|
|
835
|
+
saveSkillIfLearned(q, learnedFacts);
|
|
836
|
+
if (result.outcome === "terminal" || result.outcome === "unparseable" || result.outcome === "critic-complete") {
|
|
837
|
+
// A realtime call can start WHILE this whole typed loop (potentially
|
|
838
|
+
// several real fetches deep) was still in flight — applying this
|
|
839
|
+
// reply now would overwrite the live call's own answer with a
|
|
840
|
+
// stale, orphaned bubble that doesn't correspond to anything the
|
|
841
|
+
// realtime conversation actually said. speak()/speakAndWait() guard
|
|
842
|
+
// the AUDIO half of this same real, live-found race (see
|
|
843
|
+
// typedPlaybackSuspendedRef's own doc comment) — this is the
|
|
844
|
+
// matching guard for the TEXT half, which would otherwise still
|
|
845
|
+
// leak through even with the audio silenced.
|
|
846
|
+
// The Critic independently confirmed the last task's doneContract
|
|
847
|
+
// is satisfied even though the model's own verb never got there —
|
|
848
|
+
// real fix for the diagnosed bug (a batch succeeded and the model
|
|
849
|
+
// kept looping instead of recognizing it). No raw server response
|
|
850
|
+
// exists for this synthesized verb (it never came from `endpoint`
|
|
851
|
+
// at all), so it's built directly from the verdict's own reasoning
|
|
852
|
+
// — same shape realtime-server.ts synthesizes for the same outcome.
|
|
853
|
+
const raw = result.outcome === "critic-complete" ? { verb: "explain", text: result.verdict.reasoning } : lastRawResponse;
|
|
854
|
+
if (typedPlaybackSuspendedRef.current) {
|
|
855
|
+
rtLog("dropping stale typed reply's text — a realtime call started while it was still in flight");
|
|
856
|
+
}
|
|
857
|
+
else {
|
|
858
|
+
handleVerb(raw);
|
|
427
859
|
}
|
|
428
|
-
//
|
|
429
|
-
//
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
860
|
+
// Unlike the realtime relay (one persistent connection, memory
|
|
861
|
+
// lives server-side), each of these POSTs is stateless — the
|
|
862
|
+
// widget itself is what remembers, and resends it above so the
|
|
863
|
+
// model has context for "the first one" / "do that instead" on
|
|
864
|
+
// the next question. Summarized from the RAW response (not
|
|
865
|
+
// driveAgentLoop's typed finalVerb) via this file's own untyped
|
|
866
|
+
// summarizeVerbForHistory — deliberately, since a response that
|
|
867
|
+
// failed schema validation (outcome "unparseable") can still carry
|
|
868
|
+
// real, usable fields (e.g. a stray extra property tripped
|
|
869
|
+
// .strict() while `text` itself was fine) that only the untyped,
|
|
870
|
+
// duck-typed summarizer sees; there is no typed finalVerb at all
|
|
871
|
+
// for that outcome.
|
|
872
|
+
historyRef.current = [
|
|
873
|
+
...result.workingHistory,
|
|
874
|
+
{ role: "user", text: q },
|
|
875
|
+
{ role: "assistant", text: summarizeVerbForHistory(raw) },
|
|
438
876
|
].slice(-MAX_HISTORY_TURNS);
|
|
877
|
+
return;
|
|
439
878
|
}
|
|
440
|
-
|
|
879
|
+
// "gave-up" (iteration cap hit with no terminal verb) OR "critic-give-up"
|
|
880
|
+
// (the Critic/stall fail-safe decided continuing wouldn't help — its
|
|
881
|
+
// own reasoning is a genuinely better message than the generic
|
|
882
|
+
// fallback, same as realtime-server.ts's own finalizeTurn).
|
|
883
|
+
const giveUpText = result.outcome === "critic-give-up" ? result.verdict.reasoning : "I wasn't able to finish that — try asking again or breaking it into smaller steps.";
|
|
884
|
+
const gaveUpSummary = result.outcome === "critic-give-up" ? giveUpText : "(gave up after too many steps)";
|
|
885
|
+
setLoopWorking(false);
|
|
886
|
+
setAnswer(giveUpText);
|
|
441
887
|
historyRef.current = [
|
|
442
|
-
...
|
|
888
|
+
...result.workingHistory,
|
|
443
889
|
{ role: "user", text: q },
|
|
444
|
-
{ role: "assistant", text:
|
|
890
|
+
{ role: "assistant", text: gaveUpSummary },
|
|
445
891
|
].slice(-MAX_HISTORY_TURNS);
|
|
446
892
|
}
|
|
447
893
|
function ensureTypedPlaybackGraph() {
|
|
@@ -456,8 +902,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
456
902
|
}
|
|
457
903
|
/** Stops whatever's currently playing on the typed/mic path's playback
|
|
458
904
|
* graph, so two responses (e.g. a rapid double-click, or two answers
|
|
459
|
-
* resolved close together) can never be heard overlapping.
|
|
905
|
+
* resolved close together) can never be heard overlapping. Also bumps
|
|
906
|
+
* typedPlaybackGenerationRef — see its own doc comment for why that's
|
|
907
|
+
* required for this to actually hold when a NEW reply's audio is still
|
|
908
|
+
* arriving as a stream, not just already fully scheduled. */
|
|
460
909
|
function stopTypedPlayback() {
|
|
910
|
+
typedPlaybackGenerationRef.current++;
|
|
461
911
|
for (const source of typedScheduledSourcesRef.current) {
|
|
462
912
|
source.onended = null;
|
|
463
913
|
try {
|
|
@@ -486,6 +936,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
486
936
|
*/
|
|
487
937
|
function playPcmStream(stream) {
|
|
488
938
|
stopTypedPlayback();
|
|
939
|
+
const myGeneration = typedPlaybackGenerationRef.current;
|
|
489
940
|
const { ctx, gain } = ensureTypedPlaybackGraph();
|
|
490
941
|
void ctx.resume().catch(() => { });
|
|
491
942
|
return new Promise((resolve) => {
|
|
@@ -521,7 +972,16 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
521
972
|
const reader = stream.getReader();
|
|
522
973
|
try {
|
|
523
974
|
for (;;) {
|
|
975
|
+
// A newer call already ran stopTypedPlayback() (bumping the
|
|
976
|
+
// generation) while we were mid-read — stop here instead of
|
|
977
|
+
// scheduling more chunks behind its back. Checked both before
|
|
978
|
+
// AND after the await: a supersede can land at any point while
|
|
979
|
+
// this loop is blocked waiting on the next chunk.
|
|
980
|
+
if (typedPlaybackGenerationRef.current !== myGeneration)
|
|
981
|
+
break;
|
|
524
982
|
const { done, value } = await reader.read();
|
|
983
|
+
if (typedPlaybackGenerationRef.current !== myGeneration)
|
|
984
|
+
break;
|
|
525
985
|
if (done)
|
|
526
986
|
break;
|
|
527
987
|
if (!value || value.length === 0)
|
|
@@ -556,6 +1016,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
556
1016
|
});
|
|
557
1017
|
if (!res.ok || !res.body)
|
|
558
1018
|
return;
|
|
1019
|
+
// A realtime call can start WHILE this fetch was in flight — see
|
|
1020
|
+
// typedPlaybackSuspendedRef's own doc comment for why that's a real,
|
|
1021
|
+
// live-found overlapping-audio case, not a hypothetical one.
|
|
1022
|
+
if (typedPlaybackSuspendedRef.current) {
|
|
1023
|
+
rtLog("dropping stale typed reply's audio — a realtime call started while it was still being fetched");
|
|
1024
|
+
return;
|
|
1025
|
+
}
|
|
559
1026
|
void playPcmStream(res.body);
|
|
560
1027
|
}
|
|
561
1028
|
catch {
|
|
@@ -576,6 +1043,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
576
1043
|
});
|
|
577
1044
|
if (!res.ok || !res.body)
|
|
578
1045
|
return;
|
|
1046
|
+
// See speak()'s own identical check and typedPlaybackSuspendedRef's
|
|
1047
|
+
// doc comment — a realtime call can start while this fetch was in
|
|
1048
|
+
// flight, same real risk here.
|
|
1049
|
+
if (typedPlaybackSuspendedRef.current) {
|
|
1050
|
+
rtLog("dropping stale typed reply's audio — a realtime call started while it was still being fetched");
|
|
1051
|
+
return;
|
|
1052
|
+
}
|
|
579
1053
|
await playPcmStream(res.body);
|
|
580
1054
|
}
|
|
581
1055
|
catch {
|
|
@@ -612,7 +1086,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
612
1086
|
// forever with the mic never resuming — move on instead.
|
|
613
1087
|
setTimeout(() => {
|
|
614
1088
|
if (!settled)
|
|
615
|
-
|
|
1089
|
+
rtLog("tour step audio confirmation timed out after 15s — continuing anyway");
|
|
616
1090
|
finish();
|
|
617
1091
|
}, 15000);
|
|
618
1092
|
ws.send(JSON.stringify({ type: "speak", text }));
|
|
@@ -685,7 +1159,24 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
685
1159
|
// which is exactly what "hearing the agent twice, in parallel" was.
|
|
686
1160
|
if (!realtimeUrl || !micSupported || realtimeActive || rtStartingRef.current)
|
|
687
1161
|
return;
|
|
1162
|
+
rtLog("starting realtime call", { url: realtimeUrl });
|
|
688
1163
|
rtStartingRef.current = true;
|
|
1164
|
+
// A typed/mic-recorded reply's audio can still be mid-playback on its
|
|
1165
|
+
// own separate graph (typedPlaybackGainRef, only ever touched by
|
|
1166
|
+
// stopTypedPlayback/playPcmStream) when the user switches straight into
|
|
1167
|
+
// a live call — endRealtime() already stops it on the way OUT of a
|
|
1168
|
+
// call, but nothing stopped it on the way IN, so it kept playing
|
|
1169
|
+
// completely unaffected by the realtime session's own mute-speaker
|
|
1170
|
+
// button (which only ever touches rtPlaybackGainRef) or by barge-in —
|
|
1171
|
+
// a real, live-found "two independent speakers" bug: muting or saying
|
|
1172
|
+
// "stop" only ever reached the realtime pipeline, while this leftover
|
|
1173
|
+
// typed audio played on regardless until it finished on its own.
|
|
1174
|
+
stopTypedPlayback();
|
|
1175
|
+
// Also blocks any typed reply that's still mid-fetch RIGHT NOW (not
|
|
1176
|
+
// yet playing anything, so stopTypedPlayback() above has nothing to
|
|
1177
|
+
// stop) from playing its audio once it finally arrives, seconds from
|
|
1178
|
+
// now — see typedPlaybackSuspendedRef's own doc comment.
|
|
1179
|
+
typedPlaybackSuspendedRef.current = true;
|
|
689
1180
|
archiveCurrentExchange(); // preserve whatever typed/mic exchange preceded switching into a live call
|
|
690
1181
|
setAnswer(null);
|
|
691
1182
|
setCaption("");
|
|
@@ -717,6 +1208,21 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
717
1208
|
const processor = audioCtx.createScriptProcessor(4096, 1, 1);
|
|
718
1209
|
const silence = audioCtx.createGain();
|
|
719
1210
|
silence.gain.value = 0;
|
|
1211
|
+
const bargeInVad = (0, vad_1.createVadDetector)();
|
|
1212
|
+
// Real, live-reported bug this closes: firing triggerBargeIn() off a
|
|
1213
|
+
// SINGLE ~85-100ms VAD frame meant one cough or door-slam frame that
|
|
1214
|
+
// happened to pass the energy+ZCR gate cut the agent off, permanently
|
|
1215
|
+
// (no server-side "was this real" recovery exists anymore — see
|
|
1216
|
+
// vad.ts's own doc comment for why that was removed instead of kept).
|
|
1217
|
+
// Real research into how production voice-agent platforms solve this
|
|
1218
|
+
// (Pipecat, LiveKit Agents, Vapi, Deepgram's Voice Agent API — see
|
|
1219
|
+
// DEVELOPMENT.md) converges on gating the LOCAL trigger on SUSTAINED
|
|
1220
|
+
// speech across a minimum duration instead — Pipecat's own production
|
|
1221
|
+
// spec cites 250ms, Vapi's stopSpeakingPlan defaults to 0.2s. This
|
|
1222
|
+
// gate does exactly that, entirely client-side (no network round trip
|
|
1223
|
+
// or STT-transcript timing involved, so it can't reintroduce the
|
|
1224
|
+
// removed server-side race).
|
|
1225
|
+
const bargeInGate = (0, vad_1.createBargeInGate)();
|
|
720
1226
|
processor.onaudioprocess = (e) => {
|
|
721
1227
|
if (ws.readyState !== WebSocket.OPEN)
|
|
722
1228
|
return;
|
|
@@ -725,31 +1231,77 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
725
1231
|
// Barge-in: while the agent is speaking a real conversational reply,
|
|
726
1232
|
// still thinking about one, OR mid-tour, keep listening to the mic
|
|
727
1233
|
// locally even though it isn't being sent yet, and cut the agent
|
|
728
|
-
// off
|
|
729
|
-
// them wait — including during a guided
|
|
730
|
-
// rest of the walkthrough on
|
|
731
|
-
// instead of being talked-over-
|
|
732
|
-
// person giving a tour stops when
|
|
733
|
-
// "rt-thinking" half matters just as much
|
|
734
|
-
// turn can easily take a couple of seconds
|
|
735
|
-
// yet, and without this the mic was completely
|
|
736
|
-
// whole window — found live as "not listening
|
|
737
|
-
// interrupting system", not just a missed
|
|
1234
|
+
// off once the user has been sustainedly talking again (bargeInGate,
|
|
1235
|
+
// above) instead of making them wait — including during a guided
|
|
1236
|
+
// tour, which now cancels the rest of the walkthrough on
|
|
1237
|
+
// interruption (see triggerBargeIn) instead of being talked-over-
|
|
1238
|
+
// proof by design, the way a real person giving a tour stops when
|
|
1239
|
+
// you have a question. The "rt-thinking" half matters just as much
|
|
1240
|
+
// as "rt-speaking": an LLM turn can easily take a couple of seconds
|
|
1241
|
+
// with nothing playing yet, and without this the mic was completely
|
|
1242
|
+
// deaf during that whole window — found live as "not listening
|
|
1243
|
+
// while speaking... no interrupting system", not just a missed
|
|
1244
|
+
// nice-to-have.
|
|
738
1245
|
if (rtStateRef.current === "rt-speaking" || rtStateRef.current === "rt-thinking") {
|
|
739
|
-
const
|
|
740
|
-
|
|
1246
|
+
const frame = bargeInVad.process(e.inputBuffer.getChannelData(0));
|
|
1247
|
+
const frameDurationMs = (e.inputBuffer.length / audioCtx.sampleRate) * 1000;
|
|
1248
|
+
if (bargeInGate.update(frame, frameDurationMs))
|
|
741
1249
|
triggerBargeIn();
|
|
742
1250
|
return;
|
|
743
1251
|
}
|
|
1252
|
+
bargeInGate.reset(); // not currently interruptible — don't let stale progress from a moment ago carry into the next speaking/thinking phase
|
|
744
1253
|
if (rtStateRef.current !== "rt-listening")
|
|
745
1254
|
return; // don't send our own mic while the agent is thinking/speaking
|
|
1255
|
+
if (!micAudioSentSinceListeningRef.current) {
|
|
1256
|
+
micAudioSentSinceListeningRef.current = true;
|
|
1257
|
+
rtLog("mic audio actually being sent (send gate is open)");
|
|
1258
|
+
}
|
|
746
1259
|
const pcm = floatTo16BitPCM(downsampleTo16k(e.inputBuffer.getChannelData(0), audioCtx.sampleRate));
|
|
747
1260
|
ws.send(pcm);
|
|
748
1261
|
};
|
|
749
1262
|
source.connect(processor);
|
|
750
1263
|
processor.connect(silence);
|
|
751
1264
|
silence.connect(audioCtx.destination);
|
|
1265
|
+
// Real, live-reported bug this closes: "status says Listening but
|
|
1266
|
+
// nothing gets picked up" — traced to browsers deliberately
|
|
1267
|
+
// suspending an AudioContext that has no active OUTPUT (a real,
|
|
1268
|
+
// documented power-saving policy, not backgrounded-tab-only). This
|
|
1269
|
+
// capture context has no real output at all by design (silence's
|
|
1270
|
+
// gain is 0), making it exactly the shape most likely to get
|
|
1271
|
+
// silently suspended — and once suspended, onaudioprocess simply
|
|
1272
|
+
// stops firing, so nothing inside it can detect or recover from its
|
|
1273
|
+
// own silence. A periodic external health check is the only
|
|
1274
|
+
// reliable way to catch this: resume the context if the browser
|
|
1275
|
+
// suspended it, and — a real, separate failure mode — detect the
|
|
1276
|
+
// mic's OWN MediaStreamTrack actually ending or going muted (device
|
|
1277
|
+
// unplugged, OS-level permission revoked mid-call, another app
|
|
1278
|
+
// taking exclusive access) and surface a real, honest error instead
|
|
1279
|
+
// of silently going deaf with the UI still claiming to listen.
|
|
1280
|
+
const micHealthCheck = setInterval(() => {
|
|
1281
|
+
if (audioCtx.state !== "running") {
|
|
1282
|
+
rtLog("capture AudioContext was suspended — resuming", { state: audioCtx.state });
|
|
1283
|
+
void audioCtx.resume().catch((err) => rtLog("failed to resume capture AudioContext", { error: String(err) }));
|
|
1284
|
+
}
|
|
1285
|
+
const track = stream.getAudioTracks()[0];
|
|
1286
|
+
if (track && (track.readyState === "ended" || track.muted)) {
|
|
1287
|
+
rtLog("mic track is no longer live — ending the call", { readyState: track.readyState, muted: track.muted });
|
|
1288
|
+
setAnswer("The microphone connection was lost — try starting the call again.");
|
|
1289
|
+
endRealtime();
|
|
1290
|
+
}
|
|
1291
|
+
}, 2000);
|
|
1292
|
+
// The track's own "ended" event is the immediate signal (fires the
|
|
1293
|
+
// instant the OS/browser actually kills the track) — the poll above
|
|
1294
|
+
// is the safety net for anything that doesn't fire it reliably
|
|
1295
|
+
// (muted-without-ended has no dedicated event in the spec).
|
|
1296
|
+
const handleMicTrackEnded = () => {
|
|
1297
|
+
rtLog("mic track ended unexpectedly — ending the call");
|
|
1298
|
+
setAnswer("The microphone connection was lost — try starting the call again.");
|
|
1299
|
+
endRealtime();
|
|
1300
|
+
};
|
|
1301
|
+
stream.getAudioTracks().forEach((t) => t.addEventListener("ended", handleMicTrackEnded));
|
|
752
1302
|
rtCleanupRef.current = () => {
|
|
1303
|
+
clearInterval(micHealthCheck);
|
|
1304
|
+
stream.getAudioTracks().forEach((t) => t.removeEventListener("ended", handleMicTrackEnded));
|
|
753
1305
|
processor.disconnect();
|
|
754
1306
|
source.disconnect();
|
|
755
1307
|
stream.getTracks().forEach((t) => t.stop());
|
|
@@ -780,6 +1332,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
780
1332
|
rtTourAudioDoneRef.current = null;
|
|
781
1333
|
return;
|
|
782
1334
|
}
|
|
1335
|
+
rtLog("resumed listening");
|
|
1336
|
+
void audioCtx.resume().catch(() => { }); // don't wait up to 2s for the periodic health check if the browser already suspended capture
|
|
1337
|
+
micAudioSentSinceListeningRef.current = false;
|
|
783
1338
|
setRtStatus("rt-listening");
|
|
784
1339
|
setCaption("");
|
|
785
1340
|
void sendFreshContext(); // refresh before the user starts talking again, not after
|
|
@@ -794,10 +1349,28 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
794
1349
|
disarmThinkingWatchdog();
|
|
795
1350
|
rtThinkingWatchdogRef.current = setTimeout(() => {
|
|
796
1351
|
rtThinkingWatchdogRef.current = null;
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
1352
|
+
rtLog("thinking watchdog fired — server took over 20s, resuming listening and abandoning that turn");
|
|
1353
|
+
// Real, live-found gap: this used to only reset LOCAL state,
|
|
1354
|
+
// never telling the server anything — so a turn that was simply
|
|
1355
|
+
// SLOW (not actually stuck; e.g. retrying a rate-limited call
|
|
1356
|
+
// across every configured key, which can genuinely take longer
|
|
1357
|
+
// than this 20s watchdog) kept running server-side, and its
|
|
1358
|
+
// reply arrived LATE, after the user had already moved on and
|
|
1359
|
+
// started a new turn locally — landing on whatever was now
|
|
1360
|
+
// showing instead of being recognized as stale. triggerBargeIn()
|
|
1361
|
+
// is exactly the fix: it sends the same real barge_in signal a
|
|
1362
|
+
// genuine interruption does, bumping the server's own generation
|
|
1363
|
+
// so that late reply — whenever it finally arrives — carries an
|
|
1364
|
+
// old generation number and gets correctly dropped by the
|
|
1365
|
+
// isStaleRtMessage check above instead of confusingly resuming.
|
|
1366
|
+
triggerBargeIn();
|
|
1367
|
+
setLoopWorking(false);
|
|
1368
|
+
// triggerBargeIn() clears the caption but never touched `answer`
|
|
1369
|
+
// — without this, a timed-out turn gave the user literally
|
|
1370
|
+
// nothing: no reply, no error, just a silent reset back to
|
|
1371
|
+
// "Listening…" that reads as "it heard me and did nothing." A
|
|
1372
|
+
// real, live-found gap, not just a console.warn nobody sees.
|
|
1373
|
+
setAnswer("That's taking longer than expected — try asking again.");
|
|
801
1374
|
}, 20000);
|
|
802
1375
|
}
|
|
803
1376
|
function stopScheduledRtAudio() {
|
|
@@ -818,6 +1391,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
818
1391
|
// now-stale audio_chunk/speaking_end that was already in flight, so a
|
|
819
1392
|
// few straggling chunks can't sneak back in and resume playback.
|
|
820
1393
|
function triggerBargeIn() {
|
|
1394
|
+
rtLog("barge-in triggered", { wasTouring: touringRef.current, discardedAudioChunks: rtScheduledSourcesRef.current.length });
|
|
821
1395
|
disarmThinkingWatchdog();
|
|
822
1396
|
stopScheduledRtAudio();
|
|
823
1397
|
rtAudioDoneArrivingRef.current = true;
|
|
@@ -837,13 +1411,47 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
837
1411
|
if (ws.readyState === WebSocket.OPEN)
|
|
838
1412
|
ws.send(JSON.stringify({ type: "barge_in" }));
|
|
839
1413
|
setRtStatus("rt-listening");
|
|
840
|
-
|
|
1414
|
+
// Real, live-found bug this fix removes: this used to also clear
|
|
1415
|
+
// the caption here (setCaption("")). That's correct-looking for a
|
|
1416
|
+
// REAL, VAD-triggered barge-in (the user's about to say something
|
|
1417
|
+
// new) — the very next "final" already does archiveCurrentExchange()
|
|
1418
|
+
// then overwrites caption with the new utterance, so clearing it
|
|
1419
|
+
// here was always redundant for that path. But this function is
|
|
1420
|
+
// ALSO called by the thinking watchdog on a timeout, where there is
|
|
1421
|
+
// no new utterance coming — clearing the caption there wiped out
|
|
1422
|
+
// the very question that just timed out, an instant before
|
|
1423
|
+
// setAnswer(the timeout message) ran, leaving the live pair as
|
|
1424
|
+
// {caption: "", answer: "That's taking longer..."} — a reply
|
|
1425
|
+
// visibly floating with no question above it, and nothing for
|
|
1426
|
+
// archiveCurrentExchange() to pair it with on the next turn either
|
|
1427
|
+
// (archiveText skips empty text). Simply never clearing it here is
|
|
1428
|
+
// correct for both callers: the barge-in path already gets a fresh
|
|
1429
|
+
// caption from the next "final", and the watchdog path now keeps
|
|
1430
|
+
// the timed-out question correctly paired with its own answer.
|
|
841
1431
|
}
|
|
842
1432
|
ws.onopen = () => {
|
|
1433
|
+
rtLog("connection open");
|
|
843
1434
|
sendFreshContext();
|
|
844
1435
|
setRtStatus("rt-listening");
|
|
845
1436
|
rtStartingRef.current = false;
|
|
846
1437
|
};
|
|
1438
|
+
// True for a verb/speaking_start/audio_chunk/speaking_end/
|
|
1439
|
+
// turn_complete message that belongs to an EARLIER turn than the
|
|
1440
|
+
// most recent "final" this client has seen — see
|
|
1441
|
+
// rtLastFinalGenerationRef's own doc comment for the real race this
|
|
1442
|
+
// closes. A message with no generation field at all (shouldn't
|
|
1443
|
+
// happen against a server running this fix, but a mismatched client/
|
|
1444
|
+
// server version pair during a rolling deploy could) is treated as
|
|
1445
|
+
// current rather than dropped — additive/backward-compatible, same
|
|
1446
|
+
// discipline every other wire-protocol addition in this codebase
|
|
1447
|
+
// follows.
|
|
1448
|
+
function isStaleRtMessage(msg) {
|
|
1449
|
+
const stale = typeof msg.generation === "number" && msg.generation < rtLastFinalGenerationRef.current;
|
|
1450
|
+
if (stale)
|
|
1451
|
+
rtLog("dropped stale message", { type: msg.type, messageGeneration: msg.generation, currentGeneration: rtLastFinalGenerationRef.current });
|
|
1452
|
+
return stale;
|
|
1453
|
+
}
|
|
1454
|
+
let audioChunkCount = 0;
|
|
847
1455
|
ws.onmessage = (event) => {
|
|
848
1456
|
if (typeof event.data !== "string")
|
|
849
1457
|
return; // audio now arrives as base64 inside audio_chunk, not raw binary frames
|
|
@@ -852,14 +1460,35 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
852
1460
|
setCaption(msg.text);
|
|
853
1461
|
}
|
|
854
1462
|
else if (msg.type === "final") {
|
|
1463
|
+
rtLog("final transcript", { text: msg.text, generation: msg.generation });
|
|
1464
|
+
rtLastFinalGenerationRef.current = typeof msg.generation === "number" ? msg.generation : 0;
|
|
855
1465
|
archiveCurrentExchange(); // the previous turn's pair is complete — move it into history before this one starts overwriting caption/answer
|
|
856
1466
|
setCaption(msg.text);
|
|
1467
|
+
// Without this, `answer` still held the PREVIOUS turn's reply
|
|
1468
|
+
// text at the moment this new turn's own reply — if it ever
|
|
1469
|
+
// arrives — would overwrite it. Usually invisible (the previous
|
|
1470
|
+
// reply lands well before the next "final"), but a barge-in can
|
|
1471
|
+
// supersede an in-flight turn before its reply ever arrives (see
|
|
1472
|
+
// realtime-server.ts's onStep generation check) — with no reply
|
|
1473
|
+
// ever coming for THIS caption, the stale previous-turn answer
|
|
1474
|
+
// sat there and got archived alongside the wrong question on the
|
|
1475
|
+
// NEXT final, showing as a mismatched or duplicated-looking
|
|
1476
|
+
// reply. Found live: two turns' worth of the same fallback error
|
|
1477
|
+
// text ("Something went wrong on my end") appearing back to back
|
|
1478
|
+
// with only one visible question between them. Clearing to null
|
|
1479
|
+
// here means an abandoned turn now correctly archives with NO
|
|
1480
|
+
// reply bubble (archiveText skips empty text) instead of someone
|
|
1481
|
+
// else's.
|
|
1482
|
+
setAnswer(null);
|
|
857
1483
|
setRtStatus("rt-thinking");
|
|
858
1484
|
armThinkingWatchdog();
|
|
859
1485
|
}
|
|
860
1486
|
else if (msg.type === "verb") {
|
|
1487
|
+
if (isStaleRtMessage(msg))
|
|
1488
|
+
return; // belongs to a turn a later "final" already superseded
|
|
1489
|
+
rtLog("verb received", { verb: msg.verb?.verb, generation: msg.generation });
|
|
861
1490
|
const parsedStep = (0, core_1.safeParseVerbResponse)(msg.verb);
|
|
862
|
-
if (parsedStep && !core_1.
|
|
1491
|
+
if (parsedStep && !(0, core_1.isTerminalVerb)(parsedStep)) {
|
|
863
1492
|
// A continuing agent-loop step (click/fill/read/call_tool) —
|
|
864
1493
|
// the turn isn't over: execute it for real and report the
|
|
865
1494
|
// result back so the server can decide the next step, instead
|
|
@@ -870,7 +1499,39 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
870
1499
|
// — the server's loop stays quiet between steps on purpose,
|
|
871
1500
|
// to keep it fast.
|
|
872
1501
|
setAnswer(summarizeVerbForHistory(msg.verb));
|
|
873
|
-
|
|
1502
|
+
setLoopWorking(true);
|
|
1503
|
+
// Real, live-found bug: armThinkingWatchdog() only ever fired
|
|
1504
|
+
// once, on the turn's own "final" message, giving the WHOLE
|
|
1505
|
+
// multi-step turn one shared 20s budget — Executor + Planner +
|
|
1506
|
+
// Critic for step 1, then the same again for step 2, and so
|
|
1507
|
+
// on. Directly measured live: one single non-terminal step's
|
|
1508
|
+
// own Executor+Planner+Critic chain alone took ~14s (11.5s +
|
|
1509
|
+
// 1.5s + 1.1s) — a real, multi-step goal needing two or three
|
|
1510
|
+
// such steps blows straight through 20s even though each
|
|
1511
|
+
// individual step is proof of genuine progress, not a stall.
|
|
1512
|
+
// Re-arming here — once per continuing step, not once per
|
|
1513
|
+
// turn — gives every step its own fresh budget, so the
|
|
1514
|
+
// watchdog only ever fires on a step that's ACTUALLY stuck
|
|
1515
|
+
// (no verb/final/speaking_start arriving at all), matching
|
|
1516
|
+
// what its own fallback message ("taking longer than
|
|
1517
|
+
// expected") is supposed to mean.
|
|
1518
|
+
armThinkingWatchdog();
|
|
1519
|
+
// A FRESH scan, not the turn's starting liveMapRef snapshot —
|
|
1520
|
+
// real, live-found bug: a step in THIS SAME multi-step turn
|
|
1521
|
+
// (a "click New Agent" that opens a modal) can reveal DOM a
|
|
1522
|
+
// later step (a "fill" targeting the modal's own input) needs
|
|
1523
|
+
// to find, and liveMapRef is deliberately frozen once per
|
|
1524
|
+
// turn (see its own doc comment — that freeze exists to stop
|
|
1525
|
+
// a background rescan from shifting an id mid-flight during
|
|
1526
|
+
// ONE step's own round trip, not to survive across several
|
|
1527
|
+
// sequential steps that genuinely changed the page).
|
|
1528
|
+
// runTour() already does exactly this for its own steps, for
|
|
1529
|
+
// the identical reason. Without it, "click New Agent, then
|
|
1530
|
+
// type the name" reliably failed every time with "Could not
|
|
1531
|
+
// find that element on the page" — confirmed live, repeated
|
|
1532
|
+
// 5+ times in a row without ever recovering.
|
|
1533
|
+
const freshLiveMap = liveRegistryRef.current.getSnapshot().byId;
|
|
1534
|
+
void (0, verb_executor_1.executeToolStep)(msg.verb, pathnameRef.current, freshLiveMap, (route) => router.push(route), confirmToolCall).then((result) => {
|
|
874
1535
|
if (ws.readyState === WebSocket.OPEN) {
|
|
875
1536
|
ws.send(JSON.stringify({ type: "tool_result", observation: result?.observation ?? "no result" }));
|
|
876
1537
|
}
|
|
@@ -881,11 +1542,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
881
1542
|
handleVerb(msg.verb);
|
|
882
1543
|
}
|
|
883
1544
|
else if (msg.type === "speaking_start") {
|
|
1545
|
+
if (isStaleRtMessage(msg))
|
|
1546
|
+
return;
|
|
1547
|
+
rtLog("speaking start", { generation: msg.generation });
|
|
1548
|
+
audioChunkCount = 0;
|
|
884
1549
|
disarmThinkingWatchdog();
|
|
885
1550
|
rtAudioDoneArrivingRef.current = false;
|
|
886
1551
|
setRtStatus("rt-speaking");
|
|
887
1552
|
}
|
|
888
1553
|
else if (msg.type === "audio_chunk") {
|
|
1554
|
+
if (isStaleRtMessage(msg))
|
|
1555
|
+
return; // the literal "two speakers" case — a chunk from an abandoned turn, already in flight when the barge-in landed
|
|
1556
|
+
audioChunkCount++;
|
|
889
1557
|
const ctx = rtPlaybackCtxRef.current;
|
|
890
1558
|
const gain = rtPlaybackGainRef.current;
|
|
891
1559
|
if (!ctx || !gain)
|
|
@@ -918,6 +1586,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
918
1586
|
};
|
|
919
1587
|
}
|
|
920
1588
|
else if (msg.type === "speaking_end" || msg.type === "turn_complete") {
|
|
1589
|
+
if (isStaleRtMessage(msg))
|
|
1590
|
+
return; // a newer turn's own speaking_end/turn_complete will arrive and resume listening correctly on its own
|
|
1591
|
+
rtLog(msg.type, { audioChunks: audioChunkCount, generation: msg.generation });
|
|
921
1592
|
// turn_complete covers a verb with nothing spoken (a plain
|
|
922
1593
|
// highlight/navigate/do often has no text) — no audio_chunk ever
|
|
923
1594
|
// arrives for it, so rtScheduledSourcesRef is already empty and
|
|
@@ -927,10 +1598,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
927
1598
|
maybeResumeListening();
|
|
928
1599
|
}
|
|
929
1600
|
else if (msg.type === "error") {
|
|
1601
|
+
rtLog("server error", { message: msg.message });
|
|
930
1602
|
// Must actually unstick the turn, not just show the message —
|
|
931
1603
|
// otherwise the mic never resumes and the session is stuck
|
|
932
1604
|
// exactly the way a silently-dropped response used to leave it.
|
|
933
1605
|
disarmThinkingWatchdog();
|
|
1606
|
+
setLoopWorking(false);
|
|
934
1607
|
setAnswer(msg.message ?? "Something went wrong.");
|
|
935
1608
|
if (touringRef.current) {
|
|
936
1609
|
// A tour step's own speakStreamed() failed server-side (see
|
|
@@ -950,10 +1623,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
950
1623
|
}
|
|
951
1624
|
};
|
|
952
1625
|
ws.onerror = () => {
|
|
1626
|
+
rtLog("connection error");
|
|
953
1627
|
setAnswer("Couldn't connect to the realtime voice service.");
|
|
954
1628
|
endRealtime();
|
|
955
1629
|
};
|
|
956
|
-
ws.onclose = () => {
|
|
1630
|
+
ws.onclose = (closeEvent) => {
|
|
1631
|
+
rtLog("connection closed", { code: closeEvent.code, reason: closeEvent.reason, wasIdle: rtStateRef.current === "idle" });
|
|
957
1632
|
if (rtStateRef.current !== "idle")
|
|
958
1633
|
endRealtime();
|
|
959
1634
|
};
|
|
@@ -962,15 +1637,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
962
1637
|
setAnswer("Couldn't access the microphone — check your browser's permission for this site.");
|
|
963
1638
|
setRtStatus("idle");
|
|
964
1639
|
rtStartingRef.current = false;
|
|
1640
|
+
typedPlaybackSuspendedRef.current = false; // the call never actually started — don't leave typed replies permanently silenced
|
|
965
1641
|
}
|
|
966
1642
|
}
|
|
967
1643
|
function endRealtime() {
|
|
1644
|
+
rtLog("ending realtime call", { statusAtEnd: rtStateRef.current });
|
|
968
1645
|
if (rtThinkingWatchdogRef.current) {
|
|
969
1646
|
clearTimeout(rtThinkingWatchdogRef.current);
|
|
970
1647
|
rtThinkingWatchdogRef.current = null;
|
|
971
1648
|
}
|
|
972
1649
|
rtStartingRef.current = false;
|
|
973
1650
|
stopTypedPlayback();
|
|
1651
|
+
typedPlaybackSuspendedRef.current = false; // typed replies work normally again once no live call can race them
|
|
974
1652
|
rtSocketRef.current?.close();
|
|
975
1653
|
rtSocketRef.current = null;
|
|
976
1654
|
rtCleanupRef.current?.();
|
|
@@ -979,6 +1657,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
979
1657
|
setRtSpeakerMuted(false);
|
|
980
1658
|
setCaption("");
|
|
981
1659
|
setRtStatus("idle");
|
|
1660
|
+
setLoopWorking(false); // defensive — a connection dropping mid-loop must never leave the "still working" indicator stuck on
|
|
982
1661
|
tourGenerationRef.current++; // cancel an in-progress tour rather than leaving it stuck waiting to resume rt-listening
|
|
983
1662
|
touringRef.current = false;
|
|
984
1663
|
setTourStep(null);
|
|
@@ -1013,7 +1692,17 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
1013
1692
|
"rt-speaking": "Speaking…",
|
|
1014
1693
|
};
|
|
1015
1694
|
return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.length > 0 && ((0, jsx_runtime_1.jsxs)("button", { type: "button", className: "cairn-history-toggle", onClick: () => setHistoryExpanded((v) => !v), "aria-expanded": historyExpanded, children: [historyExpanded ? (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronUp, { size: 12 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronDown, { size: 12 }), historyExpanded ? "Hide earlier" : `${transcript.length} earlier`] })), historyExpanded &&
|
|
1016
|
-
transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.
|
|
1695
|
+
transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-bubble-text", children: [renderCaptionWords(answer), loopWorking && (
|
|
1696
|
+
// Real, live-reported gap this closes: this bubble
|
|
1697
|
+
// used to go static the instant a continuing step's
|
|
1698
|
+
// own progress text was shown ("Typing earbuds into
|
|
1699
|
+
// the search box"), with nothing telling the user
|
|
1700
|
+
// the agent was still actively working for however
|
|
1701
|
+
// long the next real LLM call took. Appended inline
|
|
1702
|
+
// (not swapped in place of the text, which the
|
|
1703
|
+
// no-answer-yet case below does) so the progress
|
|
1704
|
+
// text stays legible while still showing motion.
|
|
1705
|
+
(0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking cairn-thinking-inline", "aria-label": "Still working", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
|
|
1017
1706
|
e.preventDefault();
|
|
1018
1707
|
const trimmed = question.trim();
|
|
1019
1708
|
if (trimmed)
|
|
@@ -1021,6 +1710,75 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
1021
1710
|
}, children: (0, jsx_runtime_1.jsxs)("div", { className: "cairn-input-row", children: [(0, jsx_runtime_1.jsx)("input", { value: question, onChange: (e) => setQuestion(e.target.value), placeholder: "What do you need help with?", "aria-label": `Ask ${persona} a question`, disabled: recording || touring, autoFocus: true }), realtimeUrl && micSupported && ((0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn", "aria-label": "Start realtime conversation", onClick: () => void startRealtime(), disabled: busy || recording, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneCall, { size: 16 }) })), transcribeEndpoint && micSupported && ((0, jsx_runtime_1.jsx)("button", { type: "button", className: recording ? "cairn-icon-btn cairn-icon-btn-recording" : "cairn-icon-btn", "aria-label": recording ? "Stop recording" : "Ask by voice", onClick: () => (recording ? stopRecording() : void startRecording()), disabled: touring, children: recording ? (0, jsx_runtime_1.jsx)(lucide_react_1.Square, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) })), (0, jsx_runtime_1.jsx)("button", { type: "submit", className: "cairn-send", "aria-label": "Send", disabled: !question.trim() || busy || recording, children: asking ? (0, jsx_runtime_1.jsx)(lucide_react_1.Loader2, { size: 16, className: "cairn-spin" }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Send, { size: 16 }) })] }) }))] }))] }));
|
|
1022
1711
|
}
|
|
1023
1712
|
const MAX_HISTORY_TURNS = 8; // 4 exchanges — matches the same cap the realtime relay uses server-side
|
|
1713
|
+
exports.CONVERSATION_STORAGE_KEY = "cairn:conversation:v1";
|
|
1714
|
+
/**
|
|
1715
|
+
* Real, live-reported bug this closes: a host app's own mutation handler
|
|
1716
|
+
* calling a real `window.location.reload()` — a common, entirely valid
|
|
1717
|
+
* pattern; this SDK's own demo app uses it after several real actions,
|
|
1718
|
+
* e.g. moving a kanban card — tears down the ENTIRE React tree, this
|
|
1719
|
+
* widget included. Every bit of conversation state (the visible
|
|
1720
|
+
* transcript, the current exchange, even whether the panel was open)
|
|
1721
|
+
* reset to nothing, making a real, in-progress conversation look like it
|
|
1722
|
+
* had simply ended the instant a host page happened to reload — even
|
|
1723
|
+
* though nothing about the CONVERSATION itself was actually over.
|
|
1724
|
+
* `sessionStorage`, not `localStorage`, is deliberate: it survives
|
|
1725
|
+
* exactly a reload/navigation within the same tab — the real scope of
|
|
1726
|
+
* "this conversation" — and clears itself once the tab/window actually
|
|
1727
|
+
* closes, never lingering into an unrelated later visit the way
|
|
1728
|
+
* `localStorage` would.
|
|
1729
|
+
*/
|
|
1730
|
+
function loadPersistedConversation() {
|
|
1731
|
+
if (typeof window === "undefined")
|
|
1732
|
+
return null;
|
|
1733
|
+
try {
|
|
1734
|
+
const raw = window.sessionStorage.getItem(exports.CONVERSATION_STORAGE_KEY);
|
|
1735
|
+
if (!raw)
|
|
1736
|
+
return null;
|
|
1737
|
+
const parsed = JSON.parse(raw);
|
|
1738
|
+
if (!parsed || typeof parsed !== "object")
|
|
1739
|
+
return null;
|
|
1740
|
+
const transcript = Array.isArray(parsed.transcript)
|
|
1741
|
+
? parsed.transcript.filter((t) => !!t && typeof t === "object" && typeof t.id === "number" && typeof t.text === "string" && (t.role === "user" || t.role === "agent"))
|
|
1742
|
+
: [];
|
|
1743
|
+
return {
|
|
1744
|
+
transcript,
|
|
1745
|
+
lastQuestion: typeof parsed.lastQuestion === "string" ? parsed.lastQuestion : null,
|
|
1746
|
+
answer: typeof parsed.answer === "string" ? parsed.answer : null,
|
|
1747
|
+
open: Boolean(parsed.open),
|
|
1748
|
+
};
|
|
1749
|
+
}
|
|
1750
|
+
catch {
|
|
1751
|
+
return null; // private browsing, quota, or a genuinely corrupt value — never crash the widget over this
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
function savePersistedConversation(data) {
|
|
1755
|
+
if (typeof window === "undefined")
|
|
1756
|
+
return;
|
|
1757
|
+
try {
|
|
1758
|
+
window.sessionStorage.setItem(exports.CONVERSATION_STORAGE_KEY, JSON.stringify(data));
|
|
1759
|
+
}
|
|
1760
|
+
catch {
|
|
1761
|
+
// Storage unavailable/full — the conversation just won't survive a reload this time, never worth crashing the widget over.
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
/**
|
|
1765
|
+
* Rebuilds a real seed for `historyRef` (what gets sent to the model on
|
|
1766
|
+
* the NEXT typed turn) from a restored transcript — deliberately derived
|
|
1767
|
+
* from the same, already-persisted `transcript` rather than separately
|
|
1768
|
+
* persisting `historyRef`'s own shape: one real source of truth for "what
|
|
1769
|
+
* was actually said," not two that could quietly drift apart. Capped the
|
|
1770
|
+
* same way every other history array in this file already is.
|
|
1771
|
+
*/
|
|
1772
|
+
function reconstructHistoryFromPersisted(persisted) {
|
|
1773
|
+
if (!persisted)
|
|
1774
|
+
return [];
|
|
1775
|
+
const fromTranscript = persisted.transcript.map((t) => ({ role: t.role === "agent" ? "assistant" : "user", text: t.text }));
|
|
1776
|
+
const live = [
|
|
1777
|
+
...(persisted.lastQuestion ? [{ role: "user", text: persisted.lastQuestion }] : []),
|
|
1778
|
+
...(persisted.answer ? [{ role: "assistant", text: persisted.answer }] : []),
|
|
1779
|
+
];
|
|
1780
|
+
return [...fromTranscript, ...live].slice(-MAX_HISTORY_TURNS);
|
|
1781
|
+
}
|
|
1024
1782
|
/** Best-effort text form of a raw (unvalidated) verb response for the
|
|
1025
1783
|
* conversation-history log — not shown to the user, just fed back to the
|
|
1026
1784
|
* model on later turns. Deliberately loose/defensive rather than a full
|
|
@@ -1051,6 +1809,12 @@ function summarizeVerbForHistory(raw) {
|
|
|
1051
1809
|
return `(read ${String(v.target)})`;
|
|
1052
1810
|
case "call_tool":
|
|
1053
1811
|
return `(called ${String(v.name)})`;
|
|
1812
|
+
case "drag":
|
|
1813
|
+
return `(dragged ${String(v.target)} to ${String(v.to)})`;
|
|
1814
|
+
case "select":
|
|
1815
|
+
return `(selected "${String(v.value)}" in ${String(v.target)})`;
|
|
1816
|
+
case "key":
|
|
1817
|
+
return `(pressed ${String(v.key)}${v.target ? ` on ${String(v.target)}` : ""})`;
|
|
1054
1818
|
case "batch":
|
|
1055
1819
|
return Array.isArray(v.actions) ? `(${v.actions.length} steps: ${v.actions.map((a) => a.verb).join(", ")})` : "(batch)";
|
|
1056
1820
|
default:
|
|
@@ -1070,25 +1834,32 @@ function renderCaptionWords(text) {
|
|
|
1070
1834
|
const words = text.split(" ");
|
|
1071
1835
|
return words.map((word, i) => ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-word", style: { animationDelay: `${Math.min(i * 55, 2800)}ms` }, children: [word, i < words.length - 1 ? " " : ""] }, i)));
|
|
1072
1836
|
}
|
|
1837
|
+
// "Waybalance" — three real, irregular stones (an ellipse plus a smaller
|
|
1838
|
+
// bump, not a rectangle), stacked slightly off-center the way a hiker
|
|
1839
|
+
// actually balances a trail cairn, instead of the perfectly centered flat
|
|
1840
|
+
// bars this replaced. Same mark as docs/images/logo.svg and site/index.html's
|
|
1841
|
+
// nav badge, just currentColor here so it inherits the button's own color.
|
|
1073
1842
|
function CairnMark() {
|
|
1074
|
-
return ((0, jsx_runtime_1.jsxs)("svg", { width: "20", height: "20", viewBox: "0 0 20 20", fill: "none", "aria-hidden": "true", children: [(0, jsx_runtime_1.jsx)("
|
|
1843
|
+
return ((0, jsx_runtime_1.jsxs)("svg", { width: "20", height: "20", viewBox: "0 0 20 20", fill: "none", "aria-hidden": "true", children: [(0, jsx_runtime_1.jsx)("ellipse", { cx: "10", cy: "14.6", rx: "6.1", ry: "2.3", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "6.3", cy: "14.1", rx: "2.5", ry: "1.6", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "9.1", cy: "10.4", rx: "4.3", ry: "2", fill: "currentColor", opacity: "0.82" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "6.2", cy: "10", rx: "1.7", ry: "1.2", fill: "currentColor", opacity: "0.82" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "11.7", cy: "6.4", rx: "2.6", ry: "1.6", fill: "currentColor", opacity: "0.6" })] }));
|
|
1844
|
+
}
|
|
1845
|
+
// Real-time lifecycle logging — every message received, every decision made
|
|
1846
|
+
// about it (played, spoken, dropped, and why), every state transition. Added
|
|
1847
|
+
// specifically so a live session's actual behavior is visible in the browser
|
|
1848
|
+
// console instead of only inferable from symptoms after the fact — every bug
|
|
1849
|
+
// found and fixed in this file today was diagnosed from screenshots and
|
|
1850
|
+
// terminal output because nothing like this existed before. `[cairn rt]` is
|
|
1851
|
+
// the tag to filter on. Deliberately excludes per-audio_chunk noise (dozens
|
|
1852
|
+
// of chunks per turn would flood the console) — chunk activity shows up as
|
|
1853
|
+
// a one-line count at speaking_end/turn_complete instead.
|
|
1854
|
+
function rtLog(event, details) {
|
|
1855
|
+
if (details)
|
|
1856
|
+
console.log("[cairn rt]", event, details);
|
|
1857
|
+
else
|
|
1858
|
+
console.log("[cairn rt]", event);
|
|
1075
1859
|
}
|
|
1076
1860
|
// ---------------------------------------------------------------------------
|
|
1077
1861
|
// Audio helpers (real-time PCM16 capture — standard Web Audio API patterns)
|
|
1078
1862
|
// ---------------------------------------------------------------------------
|
|
1079
|
-
// Heuristic energy gate for barge-in: real speech into a laptop/phone mic
|
|
1080
|
-
// typically sits well above this; normal room noise and the mic's own
|
|
1081
|
-
// noise floor typically sit below it. Not calibrated against real hardware
|
|
1082
|
-
// in this environment (no live mic here) — reasonable starting point, may
|
|
1083
|
-
// need tuning against a real device if it proves too trigger-happy or too
|
|
1084
|
-
// insensitive in practice.
|
|
1085
|
-
const BARGE_IN_RMS_THRESHOLD = 0.02;
|
|
1086
|
-
function computeRms(channelData) {
|
|
1087
|
-
let sumSquares = 0;
|
|
1088
|
-
for (let i = 0; i < channelData.length; i++)
|
|
1089
|
-
sumSquares += channelData[i] * channelData[i];
|
|
1090
|
-
return Math.sqrt(sumSquares / channelData.length);
|
|
1091
|
-
}
|
|
1092
1863
|
function downsampleTo16k(input, inputSampleRate) {
|
|
1093
1864
|
const targetRate = 16000;
|
|
1094
1865
|
if (inputSampleRate === targetRate)
|
|
@@ -1300,6 +2071,10 @@ const COPILOT_STYLES = `
|
|
|
1300
2071
|
gap: 4px;
|
|
1301
2072
|
padding: 2px 0;
|
|
1302
2073
|
}
|
|
2074
|
+
.cairn-thinking-inline {
|
|
2075
|
+
margin-left: 6px;
|
|
2076
|
+
vertical-align: middle;
|
|
2077
|
+
}
|
|
1303
2078
|
.cairn-thinking-dot {
|
|
1304
2079
|
width: 5px;
|
|
1305
2080
|
height: 5px;
|