@cairnvibe/sdk 0.2.13 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-loop.d.ts +113 -0
- package/dist/agent-loop.js +128 -0
- package/dist/cairn-widget.js +14 -9
- package/dist/cursor-overlay.d.ts +19 -0
- package/dist/cursor-overlay.js +126 -0
- package/dist/element-ladder.d.ts +71 -0
- package/dist/element-ladder.js +168 -0
- package/dist/index.d.ts +79 -1
- package/dist/index.js +886 -96
- package/dist/key-rotator.d.ts +28 -0
- package/dist/key-rotator.js +57 -3
- package/dist/memory-sqlite.d.ts +86 -0
- package/dist/memory-sqlite.js +230 -0
- package/dist/realtime-cli.js +22 -1
- package/dist/realtime-server.d.ts +83 -2
- package/dist/realtime-server.js +561 -121
- package/dist/server.d.ts +266 -5
- package/dist/server.js +1013 -83
- package/dist/skill-store.d.ts +17 -0
- package/dist/skill-store.js +78 -0
- package/dist/tts-stream.d.ts +25 -0
- package/dist/tts-stream.js +32 -0
- package/dist/vad.d.ts +27 -0
- package/dist/vad.js +128 -0
- package/dist/verb-executor.d.ts +32 -11
- package/dist/verb-executor.js +315 -39
- package/dist/webmcp-client.d.ts +14 -1
- package/dist/webmcp-client.js +22 -1
- package/package.json +3 -1
- package/src/agent-loop.ts +222 -0
- package/src/cursor-overlay.ts +130 -0
- package/src/element-ladder.ts +170 -0
- package/src/index.tsx +935 -100
- package/src/key-rotator.ts +57 -2
- package/src/memory-sqlite.ts +283 -0
- package/src/realtime-cli.ts +24 -1
- package/src/realtime-server.ts +669 -123
- package/src/server.ts +1119 -83
- package/src/skill-store.ts +88 -0
- package/src/tts-stream.ts +30 -0
- package/src/vad.ts +153 -0
- package/src/verb-executor.ts +329 -42
- package/src/web-component.ts +97 -24
- package/src/webmcp-client.ts +30 -2
package/dist/index.js
CHANGED
|
@@ -1,18 +1,25 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
"use client";
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.CONVERSATION_STORAGE_KEY = void 0;
|
|
4
5
|
exports.Copilot = Copilot;
|
|
6
|
+
exports.loadPersistedConversation = loadPersistedConversation;
|
|
7
|
+
exports.savePersistedConversation = savePersistedConversation;
|
|
8
|
+
exports.reconstructHistoryFromPersisted = reconstructHistoryFromPersisted;
|
|
5
9
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
6
10
|
const react_1 = require("react");
|
|
7
11
|
const navigation_1 = require("next/navigation");
|
|
8
12
|
const lucide_react_1 = require("lucide-react");
|
|
9
13
|
const core_1 = require("@cairnvibe/core");
|
|
14
|
+
const agent_loop_1 = require("./agent-loop");
|
|
10
15
|
const context_collector_1 = require("./context-collector");
|
|
16
|
+
const cursor_overlay_1 = require("./cursor-overlay");
|
|
11
17
|
const element_ladder_1 = require("./element-ladder");
|
|
12
18
|
const runtime_scan_1 = require("./runtime-scan");
|
|
13
19
|
const webmcp_client_1 = require("./webmcp-client");
|
|
14
20
|
const verb_executor_1 = require("./verb-executor");
|
|
15
|
-
|
|
21
|
+
const vad_1 = require("./vad");
|
|
22
|
+
function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, scopeId, planEndpoint, criticEndpoint, skillsSaveEndpoint, realtimeUrl, persona = "Cairn", }) {
|
|
16
23
|
const pathname = (0, navigation_1.usePathname)() ?? "/";
|
|
17
24
|
// Mirrors `pathname` for use inside long-lived closures (a realtime
|
|
18
25
|
// session's handlers are all created once, when the connection opens —
|
|
@@ -26,7 +33,21 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
26
33
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
27
34
|
}, [pathname]);
|
|
28
35
|
const router = (0, navigation_1.useRouter)();
|
|
36
|
+
// Starts at the same safe default on both server and client's first
|
|
37
|
+
// render — same hydration-mismatch reason `micSupported` below does this
|
|
38
|
+
// — then, once actually restored from sessionStorage post-mount (see the
|
|
39
|
+
// dedicated restore effect further down), flips to whatever a real page
|
|
40
|
+
// reload (a host app's own mutation handler, e.g. — see
|
|
41
|
+
// loadPersistedConversation's own doc comment) had showing a moment ago,
|
|
42
|
+
// so a real reload never again looks like the conversation simply ended.
|
|
29
43
|
const [open, setOpen] = (0, react_1.useState)(false);
|
|
44
|
+
// Fades the synthetic cursor out once the panel closes (or the widget
|
|
45
|
+
// itself unmounts) instead of leaving it sitting visible on the page.
|
|
46
|
+
(0, react_1.useEffect)(() => {
|
|
47
|
+
if (!open)
|
|
48
|
+
(0, cursor_overlay_1.hideCursor)();
|
|
49
|
+
return () => (0, cursor_overlay_1.hideCursor)();
|
|
50
|
+
}, [open]);
|
|
30
51
|
// Collapsed by default so the panel only ever shows the current exchange
|
|
31
52
|
// — the full archived transcript (built up over a long conversation)
|
|
32
53
|
// stays out of the way behind an explicit toggle instead of always being
|
|
@@ -34,6 +55,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
34
55
|
const [historyExpanded, setHistoryExpanded] = (0, react_1.useState)(false);
|
|
35
56
|
const [question, setQuestion] = (0, react_1.useState)("");
|
|
36
57
|
const [answer, setAnswer] = (0, react_1.useState)(null);
|
|
58
|
+
// Real, live-reported gap this closes: once a continuing agent-loop step
|
|
59
|
+
// (click/fill/read/call_tool/batch) sets `answer` to its own progress
|
|
60
|
+
// text ("Typing earbuds into the search box"), that text just sat there
|
|
61
|
+
// unchanged for however long the NEXT resolveVerb call took — several
|
|
62
|
+
// real seconds, more under rate-limit retries — with nothing on screen
|
|
63
|
+
// telling the user the agent was still actually doing something. `answer`
|
|
64
|
+
// itself can't double as that signal (a terminal turn's own real,
|
|
65
|
+
// finished answer looks identical to unfinished progress text). This is
|
|
66
|
+
// a separate, explicit flag: true from the moment a continuing step's
|
|
67
|
+
// progress text is shown until the turn actually ends (a terminal verb,
|
|
68
|
+
// an error, or a give-up) — see its own setters below for exactly where.
|
|
69
|
+
const [loopWorking, setLoopWorking] = (0, react_1.useState)(false);
|
|
37
70
|
const [status, setStatus] = (0, react_1.useState)("idle");
|
|
38
71
|
const [caption, setCaption] = (0, react_1.useState)("");
|
|
39
72
|
// The user's own last question, shown as its own floating caption bubble
|
|
@@ -106,6 +139,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
106
139
|
const rtMicMutedRef = (0, react_1.useRef)(false);
|
|
107
140
|
const rtSpeakerMutedRef = (0, react_1.useRef)(false);
|
|
108
141
|
const rtStartingRef = (0, react_1.useRef)(false); // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
|
|
142
|
+
// The generation of the most recent "final" this client has processed —
|
|
143
|
+
// see ServerMessage's own doc comment in realtime-server.ts for the full
|
|
144
|
+
// real, live-found bug this closes: the client's own local barge-in can
|
|
145
|
+
// start a NEW turn (a new "final") before an EARLIER turn's own verb/
|
|
146
|
+
// audio, already in flight on the wire when the server processed that
|
|
147
|
+
// barge-in, actually arrives. WebSocket delivers messages in order, but
|
|
148
|
+
// "in order" isn't "still current" — every verb/speaking_start/
|
|
149
|
+
// audio_chunk/speaking_end/turn_complete message carries the generation
|
|
150
|
+
// it was produced under, and the handler drops it outright if it's
|
|
151
|
+
// older than this ref's value instead of applying it to whatever
|
|
152
|
+
// caption happens to be showing now.
|
|
153
|
+
const rtLastFinalGenerationRef = (0, react_1.useRef)(0);
|
|
109
154
|
// Progressive PCM playback for the buffered (non-realtime) speak endpoint
|
|
110
155
|
// — the same gapless AudioBufferSourceNode scheduling the realtime path
|
|
111
156
|
// uses for its audio_chunk messages (see rtPlaybackCtxRef below), just fed
|
|
@@ -118,6 +163,37 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
118
163
|
const typedPlaybackGainRef = (0, react_1.useRef)(null);
|
|
119
164
|
const typedNextPlayTimeRef = (0, react_1.useRef)(0);
|
|
120
165
|
const typedScheduledSourcesRef = (0, react_1.useRef)([]);
|
|
166
|
+
// Bumped by stopTypedPlayback() every time it runs — playPcmStream's own
|
|
167
|
+
// async reader loop checks this before scheduling each chunk, so a
|
|
168
|
+
// superseded call (two speak()/ask() replies resolving close together)
|
|
169
|
+
// actually STOPS reading and scheduling more audio once a newer call has
|
|
170
|
+
// taken over, instead of continuing to push nodes into the shared
|
|
171
|
+
// playback graph behind the newer call's back. stopTypedPlayback() on
|
|
172
|
+
// its own only ever stopped nodes that already existed at the moment it
|
|
173
|
+
// ran — it never told an in-flight stream reader to stop producing MORE
|
|
174
|
+
// of them, which is exactly what let two replies' audio genuinely
|
|
175
|
+
// overlap (one starting, then a second call's audio starting on top of
|
|
176
|
+
// it moments later) — a real, live-found race, not a guess.
|
|
177
|
+
const typedPlaybackGenerationRef = (0, react_1.useRef)(0);
|
|
178
|
+
// A DIFFERENT real gap the generation counter above doesn't close: it
|
|
179
|
+
// only protects one typed reply's audio against ANOTHER typed reply's
|
|
180
|
+
// audio. A typed ask()/speak() call already in flight — its /api/
|
|
181
|
+
// copilot/speak fetch genuinely takes several seconds under real
|
|
182
|
+
// conditions (rate-limit retries make this worse, not better) — has no
|
|
183
|
+
// way to know a realtime call started WHILE it was still waiting. Its
|
|
184
|
+
// response arrives, and normally-innocent code plays it, seconds after
|
|
185
|
+
// startRealtime() already ran — genuinely overlapping with the live
|
|
186
|
+
// call's own audio, since nothing about the realtime session's own
|
|
187
|
+
// start/mute/barge-in controls have any way to reach a typed reply
|
|
188
|
+
// that hadn't even been scheduled yet when they ran. Found live: two
|
|
189
|
+
// full agent answers, sourced entirely from separate /api/copilot/
|
|
190
|
+
// speak calls, audibly overlapping about a second apart, while a
|
|
191
|
+
// realtime call was the only thing visibly active in the UI the whole
|
|
192
|
+
// time. startRealtime() sets this; endRealtime() clears it; speak()/
|
|
193
|
+
// speakAndWait() check it AFTER their fetch resolves and drop the
|
|
194
|
+
// reply's audio entirely (never call playPcmStream at all) if a
|
|
195
|
+
// realtime call has taken over since the request was made.
|
|
196
|
+
const typedPlaybackSuspendedRef = (0, react_1.useRef)(false);
|
|
121
197
|
// Watchdog for the "rt-thinking" state: started on every "final" transcript,
|
|
122
198
|
// cleared the moment the server responds with anything for that turn
|
|
123
199
|
// (verb/speaking_start/speaking_end/turn_complete/error). If it ever
|
|
@@ -140,6 +216,36 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
140
216
|
// finished arriving, so the mic can't start sending while the agent is
|
|
141
217
|
// still audibly speaking.
|
|
142
218
|
const rtAudioDoneArrivingRef = (0, react_1.useRef)(true);
|
|
219
|
+
// Diagnostic only, not a functional guard — flips to true the first time
|
|
220
|
+
// a real mic packet is actually sent after transitioning to
|
|
221
|
+
// "rt-listening", logged once (not per-packet, which would flood the
|
|
222
|
+
// console). Added specifically so a "status says Listening… but nothing
|
|
223
|
+
// I say gets picked up" report can be told apart, from the log alone,
|
|
224
|
+
// between "the send gate never opened" (this never logs) and "the gate
|
|
225
|
+
// opened and sent real audio, so the problem is somewhere else entirely
|
|
226
|
+
// — Deepgram's own STT, or a real hardware/OS mic issue this app can't
|
|
227
|
+
// see or fix" (this logs once, then goes quiet as expected).
|
|
228
|
+
const micAudioSentSinceListeningRef = (0, react_1.useRef)(false);
|
|
229
|
+
// Safety net for a live realtime call outliving this component instance:
|
|
230
|
+
// without this, unmounting (a parent removing the widget, a route change
|
|
231
|
+
// that remounts it, or — the real, live-hit case — Next.js Fast Refresh
|
|
232
|
+
// remounting the component on every dev-mode source edit while a call is
|
|
233
|
+
// open) left the WebSocket, the open getUserMedia mic stream, and both
|
|
234
|
+
// AudioContexts running completely orphaned: nothing ever called
|
|
235
|
+
// rtSocketRef.current?.close() or stopped the mic tracks. The old,
|
|
236
|
+
// zombie connection then kept transcribing and replying in parallel with
|
|
237
|
+
// whatever the newly-mounted instance does next — the literal
|
|
238
|
+
// "two things running in parallel, the agent answering twice" bug found
|
|
239
|
+
// live. endRealtime() is safe to call from an unmount cleanup even
|
|
240
|
+
// though it also calls React state setters: it only reads stable refs
|
|
241
|
+
// (never a stale closure over props/state), and React 18 silently no-ops
|
|
242
|
+
// a setState call on an already-unmounted component.
|
|
243
|
+
(0, react_1.useEffect)(() => {
|
|
244
|
+
return () => {
|
|
245
|
+
if (rtSocketRef.current || rtCleanupRef.current)
|
|
246
|
+
endRealtime();
|
|
247
|
+
};
|
|
248
|
+
}, []);
|
|
143
249
|
// Starts false on both server and client's first render (avoids a
|
|
144
250
|
// hydration mismatch — `navigator` doesn't exist during SSR), then
|
|
145
251
|
// updated after mount, once we're only ever running in the browser.
|
|
@@ -147,6 +253,36 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
147
253
|
(0, react_1.useEffect)(() => {
|
|
148
254
|
setMicSupported(!!navigator.mediaDevices?.getUserMedia && typeof MediaRecorder !== "undefined");
|
|
149
255
|
}, []);
|
|
256
|
+
// Restores a conversation that survived a real page reload — same
|
|
257
|
+
// hydration-safety reason as `micSupported` just above: `open`/`answer`/
|
|
258
|
+
// `lastQuestion`/`transcript` all start at their normal empty defaults on
|
|
259
|
+
// both server and client's first render (matching what SSR produced), and
|
|
260
|
+
// only flip to whatever sessionStorage actually holds here, post-mount.
|
|
261
|
+
// Real, live-verified bug this closes: a host app's own mutation handler
|
|
262
|
+
// calling `window.location.reload()` (this SDK's own demo app does this
|
|
263
|
+
// after several real actions, e.g. moving a kanban card) tears down the
|
|
264
|
+
// entire React tree, this widget included — every bit of visible
|
|
265
|
+
// conversation state reset to nothing, making an in-progress conversation
|
|
266
|
+
// look like it had simply ended the instant the host page happened to
|
|
267
|
+
// reload, even though nothing about the CONVERSATION itself was over.
|
|
268
|
+
(0, react_1.useEffect)(() => {
|
|
269
|
+
const persisted = loadPersistedConversation();
|
|
270
|
+
if (!persisted)
|
|
271
|
+
return;
|
|
272
|
+
setOpen(persisted.open);
|
|
273
|
+
setAnswer(persisted.answer);
|
|
274
|
+
setLastQuestion(persisted.lastQuestion);
|
|
275
|
+
setTranscript(persisted.transcript);
|
|
276
|
+
historyRef.current = reconstructHistoryFromPersisted(persisted);
|
|
277
|
+
// Starts past the restored transcript's own highest id — otherwise the
|
|
278
|
+
// very next archived entry would reuse an id already on screen, a real
|
|
279
|
+
// duplicate-React-key bug (React silently confuses which DOM node is
|
|
280
|
+
// which when two list items share a key).
|
|
281
|
+
if (persisted.transcript.length > 0) {
|
|
282
|
+
transcriptIdRef.current = Math.max(...persisted.transcript.map((t) => t.id)) + 1;
|
|
283
|
+
}
|
|
284
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
285
|
+
}, []);
|
|
150
286
|
const asking = status === "asking";
|
|
151
287
|
const recording = status === "recording";
|
|
152
288
|
const realtimeActive = status.startsWith("rt-");
|
|
@@ -166,6 +302,25 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
166
302
|
(0, react_1.useEffect)(() => {
|
|
167
303
|
answerRef.current = answer;
|
|
168
304
|
}, [answer]);
|
|
305
|
+
// Persists the visible conversation to sessionStorage on every change, so
|
|
306
|
+
// a host app's own `window.location.reload()` (e.g. after a board-card
|
|
307
|
+
// move — see loadPersistedConversation's own comment for the full story)
|
|
308
|
+
// doesn't make an in-progress conversation look like it simply ended.
|
|
309
|
+
// Skips its own very first (mount) invocation on purpose: that call is
|
|
310
|
+
// always tied to the initial render's plain defaults (open:false,
|
|
311
|
+
// transcript:[], ...) — captured before the restore effect above's
|
|
312
|
+
// setState calls have actually landed — so persisting it would clobber a
|
|
313
|
+
// real, just-restored conversation with empty state for one tick, right
|
|
314
|
+
// before the corrective post-restore render fixes it back. Skipping costs
|
|
315
|
+
// nothing on a genuinely fresh session (there's nothing to persist yet).
|
|
316
|
+
const skippedMountPersistRef = (0, react_1.useRef)(false);
|
|
317
|
+
(0, react_1.useEffect)(() => {
|
|
318
|
+
if (!skippedMountPersistRef.current) {
|
|
319
|
+
skippedMountPersistRef.current = true;
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
savePersistedConversation({ transcript, lastQuestion, answer, open });
|
|
323
|
+
}, [transcript, lastQuestion, answer, open]);
|
|
169
324
|
// Auto-scroll to the newest content whenever the transcript grows or the
|
|
170
325
|
// live (not-yet-archived) bubble's text changes.
|
|
171
326
|
(0, react_1.useEffect)(() => {
|
|
@@ -235,11 +390,38 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
235
390
|
}
|
|
236
391
|
}
|
|
237
392
|
function handleVerb(raw) {
|
|
393
|
+
setLoopWorking(false); // only ever called for a genuinely terminal verb — the turn is over
|
|
238
394
|
(0, verb_executor_1.executeVerbResponse)(raw, pathname, {
|
|
239
395
|
onExplain: (text) => {
|
|
240
396
|
setAnswer(text);
|
|
241
|
-
|
|
242
|
-
|
|
397
|
+
// A REAL, deep, long-standing bug found live — not introduced by
|
|
398
|
+
// today's other fixes, just newly diagnosed: this function is a
|
|
399
|
+
// plain closure defined fresh every render, but when a "verb" WS
|
|
400
|
+
// message arrives it's invoked through ws.onmessage — a callback
|
|
401
|
+
// assigned ONCE, inside startRealtime(), and never reassigned for
|
|
402
|
+
// the rest of that connection's life. `realtimeActive` there is
|
|
403
|
+
// therefore frozen at whatever it was AT THE MOMENT startRealtime()
|
|
404
|
+
// was called — which is BEFORE the click handler's own state
|
|
405
|
+
// updates land, so it reads `false` for literally the entire
|
|
406
|
+
// lifetime of every realtime call. `!realtimeActive` was therefore
|
|
407
|
+
// ALWAYS true here, on every single realtime turn — this ran
|
|
408
|
+
// speak() (the typed HTTP path) in addition to the correct
|
|
409
|
+
// realtime audio_chunk playback, every time. This is the actual,
|
|
410
|
+
// original root cause of "two speakers" — the earlier
|
|
411
|
+
// typedPlaybackSuspendedRef fix only ever suppressed the resulting
|
|
412
|
+
// AUDIO once this was already firing, it never stopped the
|
|
413
|
+
// firing itself (or the wasted LLM/TTS call and quota burn
|
|
414
|
+
// underneath it). rtStateRef mirrors status specifically to avoid
|
|
415
|
+
// this class of bug in a callback like this one (see its own doc
|
|
416
|
+
// comment) — using it here instead of the stale const is the
|
|
417
|
+
// actual fix.
|
|
418
|
+
if (rtStateRef.current.startsWith("rt-")) {
|
|
419
|
+
rtLog("explain: realtime call active, letting the socket's own audio_chunk stream speak this");
|
|
420
|
+
}
|
|
421
|
+
else {
|
|
422
|
+
rtLog("explain: no realtime call active, using the typed speak() HTTP path");
|
|
423
|
+
void speak(text);
|
|
424
|
+
}
|
|
243
425
|
},
|
|
244
426
|
onNavigate: (route) => router.push(route),
|
|
245
427
|
onMiss: reportMiss,
|
|
@@ -267,7 +449,16 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
267
449
|
*/
|
|
268
450
|
async function runTour(steps) {
|
|
269
451
|
const myGeneration = ++tourGenerationRef.current;
|
|
270
|
-
|
|
452
|
+
// Same real stale-closure bug as onExplain's own fix above, and the
|
|
453
|
+
// reason a realtime-triggered tour spoke through the wrong pipeline
|
|
454
|
+
// (or, after that fix suppressed the wrong pipeline's audio, spoke
|
|
455
|
+
// through nothing at all — "tour did not speak anything") and could
|
|
456
|
+
// leave the mic never properly told to resume listening afterward
|
|
457
|
+
// (see the `setRtStatus("rt-listening")` call at the end of this
|
|
458
|
+
// function, fixed the same way). rtStateRef.current is always
|
|
459
|
+
// current, regardless of which render's closure this particular
|
|
460
|
+
// invocation runs inside.
|
|
461
|
+
const wasRealtimeListening = rtStateRef.current.startsWith("rt-");
|
|
271
462
|
touringRef.current = true;
|
|
272
463
|
if (wasRealtimeListening)
|
|
273
464
|
setRtStatus("rt-speaking");
|
|
@@ -350,7 +541,31 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
350
541
|
return;
|
|
351
542
|
setTourStep(null);
|
|
352
543
|
setCaption("");
|
|
353
|
-
|
|
544
|
+
// The most damaging half of this stale-closure bug: this used to
|
|
545
|
+
// read the stale `realtimeActive` const, which meant this call was
|
|
546
|
+
// ALWAYS skipped for a tour reached via realtime — the mic was
|
|
547
|
+
// never explicitly told to resume listening once the tour ended.
|
|
548
|
+
// rtStateRef.current.startsWith("rt-") is what actually reflects
|
|
549
|
+
// whether the connection is still live right now.
|
|
550
|
+
if (wasRealtimeListening && rtStateRef.current.startsWith("rt-"))
|
|
551
|
+
setRtStatus("rt-listening");
|
|
552
|
+
}
|
|
553
|
+
catch (err) {
|
|
554
|
+
// Real, live-found gap: this function had NO catch at all — only
|
|
555
|
+
// try/finally. Any error thrown anywhere in the loop above (a
|
|
556
|
+
// rejected speakOverRealtime/speakAndWait call, a DOM exception
|
|
557
|
+
// from el.click(), a network failure) skipped the resume-listening
|
|
558
|
+
// line above ENTIRELY, leaving the mic stuck exactly where the
|
|
559
|
+
// tour left off — matching "after this it's not listening"
|
|
560
|
+
// reported live. Every other place in this file that can fail
|
|
561
|
+
// mid-turn (ask(), handleDeepgramMessage's own server-side
|
|
562
|
+
// equivalent) already guarantees SOME recovery path; this one
|
|
563
|
+
// didn't have one at all.
|
|
564
|
+
console.error("[cairn] tour failed partway through:", err);
|
|
565
|
+
rtLog("tour failed — forcing the mic back to listening instead of leaving it stuck", { error: String(err) });
|
|
566
|
+
setTourStep(null);
|
|
567
|
+
setCaption("");
|
|
568
|
+
if (wasRealtimeListening && rtStateRef.current.startsWith("rt-"))
|
|
354
569
|
setRtStatus("rt-listening");
|
|
355
570
|
}
|
|
356
571
|
finally {
|
|
@@ -371,13 +586,114 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
371
586
|
await runTypedAgentLoop(q);
|
|
372
587
|
}
|
|
373
588
|
catch {
|
|
374
|
-
|
|
589
|
+
// See typedPlaybackSuspendedRef's own doc comment — this whole
|
|
590
|
+
// fetch can still be in flight when a realtime call starts.
|
|
591
|
+
if (!typedPlaybackSuspendedRef.current)
|
|
592
|
+
setAnswer("Something went wrong reaching the help service — try again in a moment.");
|
|
375
593
|
}
|
|
376
594
|
finally {
|
|
377
|
-
|
|
595
|
+
// The most damaging form of the same race: unconditionally forcing
|
|
596
|
+
// status back to "idle" here, after a realtime call has ALREADY
|
|
597
|
+
// taken over (status is some "rt-*" value), would silently kick the
|
|
598
|
+
// UI out of the live call — hiding its mic/speaker/hangup controls
|
|
599
|
+
// and showing the "start call" screen instead — while the actual
|
|
600
|
+
// WebSocket connection underneath is still fully alive and still
|
|
601
|
+
// talking, now with no visible way to manage it at all. Skipping
|
|
602
|
+
// this reset when suspended is what stops a slow, stale typed
|
|
603
|
+
// request from ever being able to do that.
|
|
604
|
+
if (!typedPlaybackSuspendedRef.current)
|
|
605
|
+
setStatus("idle");
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* Architecture Pillar 6 (the safety layer) — the real, working default
|
|
610
|
+
* confirmation UI: a native browser confirm dialog, naming the tool's
|
|
611
|
+
* OWN real name/description (never inventing wording), for a WebMCP
|
|
612
|
+
* tool whose registration declared `riskTier: "confirm"` (a payment, a
|
|
613
|
+
* delete, anything hard to undo). `window.confirm` blocks the calling
|
|
614
|
+
* microtask until the user actually answers — exactly the real,
|
|
615
|
+
* synchronous "get a genuine yes before this runs" behavior needed
|
|
616
|
+
* here, and simple enough to need no new UI component for this to be a
|
|
617
|
+
* real, functioning default rather than just plumbing with nothing on
|
|
618
|
+
* the other end. A host app wanting a nicer in-widget modal can still
|
|
619
|
+
* build one — this function is the only thing that would need
|
|
620
|
+
* replacing to do that.
|
|
621
|
+
*/
|
|
622
|
+
function confirmToolCall(tool) {
|
|
623
|
+
if (typeof window === "undefined" || typeof window.confirm !== "function")
|
|
624
|
+
return Promise.resolve(false);
|
|
625
|
+
const message = tool.description ? `${tool.name}: ${tool.description}\n\nAllow this action?` : `Allow "${tool.name}"?`;
|
|
626
|
+
return Promise.resolve(window.confirm(message));
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* Architecture Pillar 4 — the typed transport's own Planner call,
|
|
630
|
+
* mirroring resolvePlan's real network shape but reached over HTTP
|
|
631
|
+
* (planEndpoint's server-side handler — createPlanHandler in server.ts
|
|
632
|
+
* — is the only place that can hold the real LLM API key). Never
|
|
633
|
+
* throws: any network/parse failure degrades to the exact same single-
|
|
634
|
+
* task fallback plan resolvePlan itself falls back to on an LLM error,
|
|
635
|
+
* so a Planner hiccup never blocks the turn.
|
|
636
|
+
*/
|
|
637
|
+
async function fetchPlan(goal, version = 1) {
|
|
638
|
+
try {
|
|
639
|
+
const res = await fetch(planEndpoint, {
|
|
640
|
+
method: "POST",
|
|
641
|
+
headers: { "content-type": "application/json" },
|
|
642
|
+
body: JSON.stringify({ goal, version }),
|
|
643
|
+
});
|
|
644
|
+
const data = await res.json();
|
|
645
|
+
if (data && typeof data === "object" && Array.isArray(data.tasks) && data.tasks.length > 0)
|
|
646
|
+
return data;
|
|
647
|
+
}
|
|
648
|
+
catch {
|
|
649
|
+
// Falls through to the same fallback plan shape below.
|
|
650
|
+
}
|
|
651
|
+
return { version, goal, facts: [], tasks: [{ id: "t1", description: goal, doneContract: "The stated goal has been achieved.", status: "in_progress" }] };
|
|
652
|
+
}
|
|
653
|
+
/** Same real-network shape as fetchPlan, for criticEndpoint's
|
|
654
|
+
* createCriticHandler — degrades to a safe "continue" verdict on any
|
|
655
|
+
* failure, same resilience discipline as resolveCritic itself. */
|
|
656
|
+
async function fetchCriticVerdict(task, goal, verb, observation) {
|
|
657
|
+
try {
|
|
658
|
+
const res = await fetch(criticEndpoint, {
|
|
659
|
+
method: "POST",
|
|
660
|
+
headers: { "content-type": "application/json" },
|
|
661
|
+
body: JSON.stringify({ task, goal, verb, observation: observation ?? null }),
|
|
662
|
+
});
|
|
663
|
+
const data = await res.json();
|
|
664
|
+
if (data && typeof data.verdict === "string")
|
|
665
|
+
return data;
|
|
666
|
+
}
|
|
667
|
+
catch {
|
|
668
|
+
// Falls through to the safe default below.
|
|
378
669
|
}
|
|
670
|
+
return { verdict: "continue", reasoning: "Critic call failed — defaulting to continue rather than blocking the turn." };
|
|
671
|
+
}
|
|
672
|
+
/**
|
|
673
|
+
* Architecture Pillar 3 (Skill half) — the typed transport's own
|
|
674
|
+
* Formulator save, mirroring realtime-server.ts's own post-turn
|
|
675
|
+
* `compileSkill`+`saveSkill` call. Fire-and-forget (never awaited by
|
|
676
|
+
* the caller, never allowed to affect what the user sees) since saving
|
|
677
|
+
* a Skill is bookkeeping for a FUTURE turn, not part of answering this
|
|
678
|
+
* one — matches the Formulator's own "cheap, runs once per turn, never
|
|
679
|
+
* blocks anything" framing. Classifies the current live page for a
|
|
680
|
+
* best-effort pattern tag the same way resolveVerb's own per-request
|
|
681
|
+
* classification does server-side.
|
|
682
|
+
*/
|
|
683
|
+
function saveSkillIfLearned(goal, learnedFacts) {
|
|
684
|
+
if (!skillsSaveEndpoint || learnedFacts.length === 0)
|
|
685
|
+
return;
|
|
686
|
+
const matches = (0, core_1.classifyUiPattern)((0, core_1.deriveStructureSignals)(liveRegistryRef.current.getSnapshot().elements));
|
|
687
|
+
void fetch(skillsSaveEndpoint, {
|
|
688
|
+
method: "POST",
|
|
689
|
+
headers: { "content-type": "application/json" },
|
|
690
|
+
body: JSON.stringify({ goal, learnedFacts, pattern: matches[0]?.pattern }),
|
|
691
|
+
}).catch(() => {
|
|
692
|
+
// A Skill that fails to save just means the next similar goal
|
|
693
|
+
// starts from scratch again, same as if nothing had been learned
|
|
694
|
+
// this turn — never worth surfacing as a user-visible error.
|
|
695
|
+
});
|
|
379
696
|
}
|
|
380
|
-
const MAX_LOOP_ITERATIONS = 6; // a hard cap, not a target — see runTypedAgentLoop
|
|
381
697
|
/**
|
|
382
698
|
* Drives the agent loop over the stateless HTTP path: ask the server,
|
|
383
699
|
* and if it comes back with a continuing step (click/fill/read/
|
|
@@ -390,58 +706,196 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
390
706
|
* still knows what it was actually asked. `historyRef` (the
|
|
391
707
|
* conversation's real memory) is only ever committed once, at the end —
|
|
392
708
|
* a turn that hits the cap mid-loop doesn't leave partial noise in it.
|
|
709
|
+
* The loop itself (the `for`/TERMINAL_VERBS/iteration-cap shape) lives
|
|
710
|
+
* in agent-loop.ts, shared with the realtime relay's own finalizeTurn —
|
|
711
|
+
* this function owns everything transport-specific: the actual fetch,
|
|
712
|
+
* the raw/untyped response handling a stateless HTTP call needs (unlike
|
|
713
|
+
* realtime's always-valid in-process resolveVerb call), and the real
|
|
714
|
+
* historyRef commit.
|
|
393
715
|
*/
|
|
394
716
|
async function runTypedAgentLoop(q) {
|
|
395
|
-
let loopHistory = historyRef.current;
|
|
396
717
|
const webMcpTools = await (0, webmcp_client_1.discoverWebMcpTools)();
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
718
|
+
let lastRawResponse = null;
|
|
719
|
+
// Architecture Pillar 4 — real Planner/Critic wiring for the typed
|
|
720
|
+
// transport, opt-in via planEndpoint/criticEndpoint (see their own
|
|
721
|
+
// doc comments on CopilotProps) — closes the gap the plan file names
|
|
722
|
+
// directly ("the typed/HTTP path has zero Planner/Critic wiring at
|
|
723
|
+
// all... today explicitly realtime-only by deferral, not by
|
|
724
|
+
// decision"). Mirrors realtime-server.ts's finalizeTurn: an eager
|
|
725
|
+
// Planner kickoff when looksMultiStep(q) already flags a probable
|
|
726
|
+
// compound goal, a lazy fallback kickoff on the first continuing step
|
|
727
|
+
// otherwise, and a genuinely separate Critic pass over each
|
|
728
|
+
// continuing step's real result. Neither endpoint set (the default)
|
|
729
|
+
// means plannerEnabled is false and this whole block is a no-op —
|
|
730
|
+
// the typed loop behaves exactly as it always has.
|
|
731
|
+
let planPromise = null;
|
|
732
|
+
let plan = null;
|
|
733
|
+
let progress = null;
|
|
734
|
+
const STALL_THRESHOLD = 3; // same bounded budget realtime's own Critic wiring uses
|
|
735
|
+
const plannerEnabled = Boolean(planEndpoint && criticEndpoint);
|
|
736
|
+
if (plannerEnabled && (0, agent_loop_1.looksMultiStep)(q))
|
|
737
|
+
planPromise = fetchPlan(q);
|
|
738
|
+
// Architecture Pillar 3 (Skill half) — every real, Critic-verified
|
|
739
|
+
// learnedFact from this turn's steps; saved once the turn concludes,
|
|
740
|
+
// below (saveSkillIfLearned). Empty is the common case, not a gap.
|
|
741
|
+
const learnedFacts = [];
|
|
742
|
+
const result = await (0, agent_loop_1.driveAgentLoop)(historyRef.current, {
|
|
743
|
+
async getNextStep(loopHistory) {
|
|
744
|
+
const liveScan = liveRegistryRef.current.getSnapshot();
|
|
745
|
+
liveMapRef.current = liveScan.byId;
|
|
746
|
+
const res = await fetch(endpoint, {
|
|
747
|
+
method: "POST",
|
|
748
|
+
headers: { "content-type": "application/json" },
|
|
749
|
+
body: JSON.stringify({
|
|
750
|
+
// pathnameRef, not the closed-over `pathname` — a navigate
|
|
751
|
+
// step (now possibly continuing, see isTerminalVerb) can
|
|
752
|
+
// change the real route mid-loop; this whole async function's
|
|
753
|
+
// own `pathname` closure was captured once, at the render
|
|
754
|
+
// that started this turn, and never updates again on its own.
|
|
755
|
+
route: pathnameRef.current,
|
|
756
|
+
question: q,
|
|
757
|
+
visible: (0, context_collector_1.collectVisible)(),
|
|
758
|
+
history: loopHistory,
|
|
759
|
+
liveElements: liveScan.elements,
|
|
760
|
+
webMcpTools,
|
|
761
|
+
scopeId,
|
|
762
|
+
}),
|
|
763
|
+
});
|
|
764
|
+
const data = await res.json().catch(() => null);
|
|
765
|
+
lastRawResponse = data;
|
|
766
|
+
return (0, core_1.safeParseVerbResponse)(data);
|
|
767
|
+
},
|
|
768
|
+
onStep({ verb, terminal }) {
|
|
769
|
+
// A continuing step — show it happening (execution itself
|
|
770
|
+
// happens in executeStep below). Terminal steps are handled once
|
|
771
|
+
// driveAgentLoop returns, via handleVerb — unchanged from before.
|
|
772
|
+
// Same stale-typed-reply guard as the terminal case below — a
|
|
773
|
+
// multi-step typed loop can still be mid-flight when a realtime
|
|
774
|
+
// call starts.
|
|
775
|
+
if (!terminal && !typedPlaybackSuspendedRef.current) {
|
|
776
|
+
setAnswer(summarizeVerbForHistory(verb));
|
|
777
|
+
setLoopWorking(true);
|
|
778
|
+
}
|
|
779
|
+
// The lazy fallback — only fires when looksMultiStep missed
|
|
780
|
+
// (planPromise is still null): a real Plan is still guaranteed
|
|
781
|
+
// before the Critic needs one, just one round trip later.
|
|
782
|
+
if (!terminal && plannerEnabled && !planPromise)
|
|
783
|
+
planPromise = fetchPlan(q);
|
|
784
|
+
return false;
|
|
785
|
+
},
|
|
786
|
+
// Same real, live-found fix as the realtime WS "verb" handler's own
|
|
787
|
+
// executeToolStep call — a fresh scan per step, not the turn's
|
|
788
|
+
// frozen liveMapRef, so a step that reveals new DOM (a click that
|
|
789
|
+
// opens a modal) doesn't leave the NEXT step unable to find
|
|
790
|
+
// anything in it.
|
|
791
|
+
executeStep: (verb) => (0, verb_executor_1.executeToolStep)(verb, pathnameRef.current, liveRegistryRef.current.getSnapshot().byId, (route) => router.push(route), confirmToolCall).then((r) => r?.observation),
|
|
792
|
+
runCritic: plannerEnabled
|
|
793
|
+
? async ({ verb, observation }) => {
|
|
794
|
+
// Real state, not the Executor's self-report — see
|
|
795
|
+
// resolveCritic's own doc comment (server.ts) for why this is
|
|
796
|
+
// a genuinely separate pass, same precedent realtime already
|
|
797
|
+
// established.
|
|
798
|
+
if (!plan) {
|
|
799
|
+
plan = planPromise ? await planPromise : await fetchPlan(q);
|
|
800
|
+
progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
|
|
801
|
+
}
|
|
802
|
+
const currentProgress = progress;
|
|
803
|
+
const currentTask = plan.tasks[currentProgress.currentTaskIndex];
|
|
804
|
+
const verdict = await fetchCriticVerdict(currentTask, q, verb, observation);
|
|
805
|
+
if (verdict.learnedFact)
|
|
806
|
+
learnedFacts.push(verdict.learnedFact);
|
|
807
|
+
if (verdict.verdict === "task_complete") {
|
|
808
|
+
currentTask.status = "done";
|
|
809
|
+
if (currentProgress.currentTaskIndex < plan.tasks.length - 1) {
|
|
810
|
+
// More tasks remain — advance and keep looping instead of
|
|
811
|
+
// ending the turn here.
|
|
812
|
+
currentProgress.currentTaskIndex++;
|
|
813
|
+
plan.tasks[currentProgress.currentTaskIndex].status = "in_progress";
|
|
814
|
+
currentProgress.stallCount = 0;
|
|
815
|
+
return { ...verdict, verdict: "continue" };
|
|
816
|
+
}
|
|
817
|
+
// The last task is genuinely done — end the loop right here
|
|
818
|
+
// instead of asking the model again and hoping it notices.
|
|
819
|
+
return verdict;
|
|
820
|
+
}
|
|
821
|
+
if (verdict.verdict === "replan") {
|
|
822
|
+
plan = await fetchPlan(q, plan.version + 1);
|
|
823
|
+
progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
|
|
824
|
+
return { ...verdict, verdict: "continue" };
|
|
825
|
+
}
|
|
826
|
+
if (verdict.verdict === "give_up")
|
|
827
|
+
return verdict;
|
|
828
|
+
// "continue" — a harness-enforced fail-safe on top of the
|
|
829
|
+
// Critic's own judgment, same Magentic-One-shaped two-tier
|
|
830
|
+
// tolerance realtime already uses.
|
|
831
|
+
currentProgress.stallCount++;
|
|
832
|
+
if (currentProgress.stallCount >= STALL_THRESHOLD) {
|
|
833
|
+
return {
|
|
834
|
+
verdict: "give_up",
|
|
835
|
+
reasoning: `Stuck after ${currentProgress.stallCount} steps with no confirmed progress on "${currentTask.description}" — ${verdict.reasoning}`,
|
|
836
|
+
};
|
|
837
|
+
}
|
|
838
|
+
return verdict;
|
|
839
|
+
}
|
|
840
|
+
: undefined,
|
|
841
|
+
});
|
|
842
|
+
// Architecture Pillar 3 (Skill half) — the Formulator, once per turn.
|
|
843
|
+
saveSkillIfLearned(q, learnedFacts);
|
|
844
|
+
if (result.outcome === "terminal" || result.outcome === "unparseable" || result.outcome === "critic-complete") {
|
|
845
|
+
// A realtime call can start WHILE this whole typed loop (potentially
|
|
846
|
+
// several real fetches deep) was still in flight — applying this
|
|
847
|
+
// reply now would overwrite the live call's own answer with a
|
|
848
|
+
// stale, orphaned bubble that doesn't correspond to anything the
|
|
849
|
+
// realtime conversation actually said. speak()/speakAndWait() guard
|
|
850
|
+
// the AUDIO half of this same real, live-found race (see
|
|
851
|
+
// typedPlaybackSuspendedRef's own doc comment) — this is the
|
|
852
|
+
// matching guard for the TEXT half, which would otherwise still
|
|
853
|
+
// leak through even with the audio silenced.
|
|
854
|
+
// The Critic independently confirmed the last task's doneContract
|
|
855
|
+
// is satisfied even though the model's own verb never got there —
|
|
856
|
+
// real fix for the diagnosed bug (a batch succeeded and the model
|
|
857
|
+
// kept looping instead of recognizing it). No raw server response
|
|
858
|
+
// exists for this synthesized verb (it never came from `endpoint`
|
|
859
|
+
// at all), so it's built directly from the verdict's own reasoning
|
|
860
|
+
// — same shape realtime-server.ts synthesizes for the same outcome.
|
|
861
|
+
const raw = result.outcome === "critic-complete" ? { verb: "explain", text: result.verdict.reasoning } : lastRawResponse;
|
|
862
|
+
if (typedPlaybackSuspendedRef.current) {
|
|
863
|
+
rtLog("dropping stale typed reply's text — a realtime call started while it was still in flight");
|
|
427
864
|
}
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
865
|
+
else {
|
|
866
|
+
handleVerb(raw);
|
|
867
|
+
}
|
|
868
|
+
// Unlike the realtime relay (one persistent connection, memory
|
|
869
|
+
// lives server-side), each of these POSTs is stateless — the
|
|
870
|
+
// widget itself is what remembers, and resends it above so the
|
|
871
|
+
// model has context for "the first one" / "do that instead" on
|
|
872
|
+
// the next question. Summarized from the RAW response (not
|
|
873
|
+
// driveAgentLoop's typed finalVerb) via this file's own untyped
|
|
874
|
+
// summarizeVerbForHistory — deliberately, since a response that
|
|
875
|
+
// failed schema validation (outcome "unparseable") can still carry
|
|
876
|
+
// real, usable fields (e.g. a stray extra property tripped
|
|
877
|
+
// .strict() while `text` itself was fine) that only the untyped,
|
|
878
|
+
// duck-typed summarizer sees; there is no typed finalVerb at all
|
|
879
|
+
// for that outcome.
|
|
880
|
+
historyRef.current = [
|
|
881
|
+
...result.workingHistory,
|
|
882
|
+
{ role: "user", text: q },
|
|
883
|
+
{ role: "assistant", text: summarizeVerbForHistory(raw) },
|
|
438
884
|
].slice(-MAX_HISTORY_TURNS);
|
|
885
|
+
return;
|
|
439
886
|
}
|
|
440
|
-
|
|
887
|
+
// "gave-up" (iteration cap hit with no terminal verb) OR "critic-give-up"
|
|
888
|
+
// (the Critic/stall fail-safe decided continuing wouldn't help — its
|
|
889
|
+
// own reasoning is a genuinely better message than the generic
|
|
890
|
+
// fallback, same as realtime-server.ts's own finalizeTurn).
|
|
891
|
+
const giveUpText = result.outcome === "critic-give-up" ? result.verdict.reasoning : "I wasn't able to finish that — try asking again or breaking it into smaller steps.";
|
|
892
|
+
const gaveUpSummary = result.outcome === "critic-give-up" ? giveUpText : "(gave up after too many steps)";
|
|
893
|
+
setLoopWorking(false);
|
|
894
|
+
setAnswer(giveUpText);
|
|
441
895
|
historyRef.current = [
|
|
442
|
-
...
|
|
896
|
+
...result.workingHistory,
|
|
443
897
|
{ role: "user", text: q },
|
|
444
|
-
{ role: "assistant", text:
|
|
898
|
+
{ role: "assistant", text: gaveUpSummary },
|
|
445
899
|
].slice(-MAX_HISTORY_TURNS);
|
|
446
900
|
}
|
|
447
901
|
function ensureTypedPlaybackGraph() {
|
|
@@ -456,8 +910,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
456
910
|
}
|
|
457
911
|
/** Stops whatever's currently playing on the typed/mic path's playback
|
|
458
912
|
* graph, so two responses (e.g. a rapid double-click, or two answers
|
|
459
|
-
* resolved close together) can never be heard overlapping.
|
|
913
|
+
* resolved close together) can never be heard overlapping. Also bumps
|
|
914
|
+
* typedPlaybackGenerationRef — see its own doc comment for why that's
|
|
915
|
+
* required for this to actually hold when a NEW reply's audio is still
|
|
916
|
+
* arriving as a stream, not just already fully scheduled. */
|
|
460
917
|
function stopTypedPlayback() {
|
|
918
|
+
typedPlaybackGenerationRef.current++;
|
|
461
919
|
for (const source of typedScheduledSourcesRef.current) {
|
|
462
920
|
source.onended = null;
|
|
463
921
|
try {
|
|
@@ -486,6 +944,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
486
944
|
*/
|
|
487
945
|
function playPcmStream(stream) {
|
|
488
946
|
stopTypedPlayback();
|
|
947
|
+
const myGeneration = typedPlaybackGenerationRef.current;
|
|
489
948
|
const { ctx, gain } = ensureTypedPlaybackGraph();
|
|
490
949
|
void ctx.resume().catch(() => { });
|
|
491
950
|
return new Promise((resolve) => {
|
|
@@ -521,7 +980,16 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
521
980
|
const reader = stream.getReader();
|
|
522
981
|
try {
|
|
523
982
|
for (;;) {
|
|
983
|
+
// A newer call already ran stopTypedPlayback() (bumping the
|
|
984
|
+
// generation) while we were mid-read — stop here instead of
|
|
985
|
+
// scheduling more chunks behind its back. Checked both before
|
|
986
|
+
// AND after the await: a supersede can land at any point while
|
|
987
|
+
// this loop is blocked waiting on the next chunk.
|
|
988
|
+
if (typedPlaybackGenerationRef.current !== myGeneration)
|
|
989
|
+
break;
|
|
524
990
|
const { done, value } = await reader.read();
|
|
991
|
+
if (typedPlaybackGenerationRef.current !== myGeneration)
|
|
992
|
+
break;
|
|
525
993
|
if (done)
|
|
526
994
|
break;
|
|
527
995
|
if (!value || value.length === 0)
|
|
@@ -556,6 +1024,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
556
1024
|
});
|
|
557
1025
|
if (!res.ok || !res.body)
|
|
558
1026
|
return;
|
|
1027
|
+
// A realtime call can start WHILE this fetch was in flight — see
|
|
1028
|
+
// typedPlaybackSuspendedRef's own doc comment for why that's a real,
|
|
1029
|
+
// live-found overlapping-audio case, not a hypothetical one.
|
|
1030
|
+
if (typedPlaybackSuspendedRef.current) {
|
|
1031
|
+
rtLog("dropping stale typed reply's audio — a realtime call started while it was still being fetched");
|
|
1032
|
+
return;
|
|
1033
|
+
}
|
|
559
1034
|
void playPcmStream(res.body);
|
|
560
1035
|
}
|
|
561
1036
|
catch {
|
|
@@ -576,6 +1051,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
576
1051
|
});
|
|
577
1052
|
if (!res.ok || !res.body)
|
|
578
1053
|
return;
|
|
1054
|
+
// See speak()'s own identical check and typedPlaybackSuspendedRef's
|
|
1055
|
+
// doc comment — a realtime call can start while this fetch was in
|
|
1056
|
+
// flight, same real risk here.
|
|
1057
|
+
if (typedPlaybackSuspendedRef.current) {
|
|
1058
|
+
rtLog("dropping stale typed reply's audio — a realtime call started while it was still being fetched");
|
|
1059
|
+
return;
|
|
1060
|
+
}
|
|
579
1061
|
await playPcmStream(res.body);
|
|
580
1062
|
}
|
|
581
1063
|
catch {
|
|
@@ -612,7 +1094,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
612
1094
|
// forever with the mic never resuming — move on instead.
|
|
613
1095
|
setTimeout(() => {
|
|
614
1096
|
if (!settled)
|
|
615
|
-
|
|
1097
|
+
rtLog("tour step audio confirmation timed out after 15s — continuing anyway");
|
|
616
1098
|
finish();
|
|
617
1099
|
}, 15000);
|
|
618
1100
|
ws.send(JSON.stringify({ type: "speak", text }));
|
|
@@ -685,7 +1167,24 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
685
1167
|
// which is exactly what "hearing the agent twice, in parallel" was.
|
|
686
1168
|
if (!realtimeUrl || !micSupported || realtimeActive || rtStartingRef.current)
|
|
687
1169
|
return;
|
|
1170
|
+
rtLog("starting realtime call", { url: realtimeUrl });
|
|
688
1171
|
rtStartingRef.current = true;
|
|
1172
|
+
// A typed/mic-recorded reply's audio can still be mid-playback on its
|
|
1173
|
+
// own separate graph (typedPlaybackGainRef, only ever touched by
|
|
1174
|
+
// stopTypedPlayback/playPcmStream) when the user switches straight into
|
|
1175
|
+
// a live call — endRealtime() already stops it on the way OUT of a
|
|
1176
|
+
// call, but nothing stopped it on the way IN, so it kept playing
|
|
1177
|
+
// completely unaffected by the realtime session's own mute-speaker
|
|
1178
|
+
// button (which only ever touches rtPlaybackGainRef) or by barge-in —
|
|
1179
|
+
// a real, live-found "two independent speakers" bug: muting or saying
|
|
1180
|
+
// "stop" only ever reached the realtime pipeline, while this leftover
|
|
1181
|
+
// typed audio played on regardless until it finished on its own.
|
|
1182
|
+
stopTypedPlayback();
|
|
1183
|
+
// Also blocks any typed reply that's still mid-fetch RIGHT NOW (not
|
|
1184
|
+
// yet playing anything, so stopTypedPlayback() above has nothing to
|
|
1185
|
+
// stop) from playing its audio once it finally arrives, seconds from
|
|
1186
|
+
// now — see typedPlaybackSuspendedRef's own doc comment.
|
|
1187
|
+
typedPlaybackSuspendedRef.current = true;
|
|
689
1188
|
archiveCurrentExchange(); // preserve whatever typed/mic exchange preceded switching into a live call
|
|
690
1189
|
setAnswer(null);
|
|
691
1190
|
setCaption("");
|
|
@@ -717,6 +1216,21 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
717
1216
|
const processor = audioCtx.createScriptProcessor(4096, 1, 1);
|
|
718
1217
|
const silence = audioCtx.createGain();
|
|
719
1218
|
silence.gain.value = 0;
|
|
1219
|
+
const bargeInVad = (0, vad_1.createVadDetector)();
|
|
1220
|
+
// Real, live-reported bug this closes: firing triggerBargeIn() off a
|
|
1221
|
+
// SINGLE ~85-100ms VAD frame meant one cough or door-slam frame that
|
|
1222
|
+
// happened to pass the energy+ZCR gate cut the agent off, permanently
|
|
1223
|
+
// (no server-side "was this real" recovery exists anymore — see
|
|
1224
|
+
// vad.ts's own doc comment for why that was removed instead of kept).
|
|
1225
|
+
// Real research into how production voice-agent platforms solve this
|
|
1226
|
+
// (Pipecat, LiveKit Agents, Vapi, Deepgram's Voice Agent API — see
|
|
1227
|
+
// DEVELOPMENT.md) converges on gating the LOCAL trigger on SUSTAINED
|
|
1228
|
+
// speech across a minimum duration instead — Pipecat's own production
|
|
1229
|
+
// spec cites 250ms, Vapi's stopSpeakingPlan defaults to 0.2s. This
|
|
1230
|
+
// gate does exactly that, entirely client-side (no network round trip
|
|
1231
|
+
// or STT-transcript timing involved, so it can't reintroduce the
|
|
1232
|
+
// removed server-side race).
|
|
1233
|
+
const bargeInGate = (0, vad_1.createBargeInGate)();
|
|
720
1234
|
processor.onaudioprocess = (e) => {
|
|
721
1235
|
if (ws.readyState !== WebSocket.OPEN)
|
|
722
1236
|
return;
|
|
@@ -725,31 +1239,77 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
725
1239
|
// Barge-in: while the agent is speaking a real conversational reply,
|
|
726
1240
|
// still thinking about one, OR mid-tour, keep listening to the mic
|
|
727
1241
|
// locally even though it isn't being sent yet, and cut the agent
|
|
728
|
-
// off
|
|
729
|
-
// them wait — including during a guided
|
|
730
|
-
// rest of the walkthrough on
|
|
731
|
-
// instead of being talked-over-
|
|
732
|
-
// person giving a tour stops when
|
|
733
|
-
// "rt-thinking" half matters just as much
|
|
734
|
-
// turn can easily take a couple of seconds
|
|
735
|
-
// yet, and without this the mic was completely
|
|
736
|
-
// whole window — found live as "not listening
|
|
737
|
-
// interrupting system", not just a missed
|
|
1242
|
+
// off once the user has been sustainedly talking again (bargeInGate,
|
|
1243
|
+
// above) instead of making them wait — including during a guided
|
|
1244
|
+
// tour, which now cancels the rest of the walkthrough on
|
|
1245
|
+
// interruption (see triggerBargeIn) instead of being talked-over-
|
|
1246
|
+
// proof by design, the way a real person giving a tour stops when
|
|
1247
|
+
// you have a question. The "rt-thinking" half matters just as much
|
|
1248
|
+
// as "rt-speaking": an LLM turn can easily take a couple of seconds
|
|
1249
|
+
// with nothing playing yet, and without this the mic was completely
|
|
1250
|
+
// deaf during that whole window — found live as "not listening
|
|
1251
|
+
// while speaking... no interrupting system", not just a missed
|
|
1252
|
+
// nice-to-have.
|
|
738
1253
|
if (rtStateRef.current === "rt-speaking" || rtStateRef.current === "rt-thinking") {
|
|
739
|
-
const
|
|
740
|
-
|
|
1254
|
+
const frame = bargeInVad.process(e.inputBuffer.getChannelData(0));
|
|
1255
|
+
const frameDurationMs = (e.inputBuffer.length / audioCtx.sampleRate) * 1000;
|
|
1256
|
+
if (bargeInGate.update(frame, frameDurationMs))
|
|
741
1257
|
triggerBargeIn();
|
|
742
1258
|
return;
|
|
743
1259
|
}
|
|
1260
|
+
bargeInGate.reset(); // not currently interruptible — don't let stale progress from a moment ago carry into the next speaking/thinking phase
|
|
744
1261
|
if (rtStateRef.current !== "rt-listening")
|
|
745
1262
|
return; // don't send our own mic while the agent is thinking/speaking
|
|
1263
|
+
if (!micAudioSentSinceListeningRef.current) {
|
|
1264
|
+
micAudioSentSinceListeningRef.current = true;
|
|
1265
|
+
rtLog("mic audio actually being sent (send gate is open)");
|
|
1266
|
+
}
|
|
746
1267
|
const pcm = floatTo16BitPCM(downsampleTo16k(e.inputBuffer.getChannelData(0), audioCtx.sampleRate));
|
|
747
1268
|
ws.send(pcm);
|
|
748
1269
|
};
|
|
749
1270
|
source.connect(processor);
|
|
750
1271
|
processor.connect(silence);
|
|
751
1272
|
silence.connect(audioCtx.destination);
|
|
1273
|
+
// Real, live-reported bug this closes: "status says Listening but
|
|
1274
|
+
// nothing gets picked up" — traced to browsers deliberately
|
|
1275
|
+
// suspending an AudioContext that has no active OUTPUT (a real,
|
|
1276
|
+
// documented power-saving policy, not backgrounded-tab-only). This
|
|
1277
|
+
// capture context has no real output at all by design (silence's
|
|
1278
|
+
// gain is 0), making it exactly the shape most likely to get
|
|
1279
|
+
// silently suspended — and once suspended, onaudioprocess simply
|
|
1280
|
+
// stops firing, so nothing inside it can detect or recover from its
|
|
1281
|
+
// own silence. A periodic external health check is the only
|
|
1282
|
+
// reliable way to catch this: resume the context if the browser
|
|
1283
|
+
// suspended it, and — a real, separate failure mode — detect the
|
|
1284
|
+
// mic's OWN MediaStreamTrack actually ending or going muted (device
|
|
1285
|
+
// unplugged, OS-level permission revoked mid-call, another app
|
|
1286
|
+
// taking exclusive access) and surface a real, honest error instead
|
|
1287
|
+
// of silently going deaf with the UI still claiming to listen.
|
|
1288
|
+
const micHealthCheck = setInterval(() => {
|
|
1289
|
+
if (audioCtx.state !== "running") {
|
|
1290
|
+
rtLog("capture AudioContext was suspended — resuming", { state: audioCtx.state });
|
|
1291
|
+
void audioCtx.resume().catch((err) => rtLog("failed to resume capture AudioContext", { error: String(err) }));
|
|
1292
|
+
}
|
|
1293
|
+
const track = stream.getAudioTracks()[0];
|
|
1294
|
+
if (track && (track.readyState === "ended" || track.muted)) {
|
|
1295
|
+
rtLog("mic track is no longer live — ending the call", { readyState: track.readyState, muted: track.muted });
|
|
1296
|
+
setAnswer("The microphone connection was lost — try starting the call again.");
|
|
1297
|
+
endRealtime();
|
|
1298
|
+
}
|
|
1299
|
+
}, 2000);
|
|
1300
|
+
// The track's own "ended" event is the immediate signal (fires the
|
|
1301
|
+
// instant the OS/browser actually kills the track) — the poll above
|
|
1302
|
+
// is the safety net for anything that doesn't fire it reliably
|
|
1303
|
+
// (muted-without-ended has no dedicated event in the spec).
|
|
1304
|
+
const handleMicTrackEnded = () => {
|
|
1305
|
+
rtLog("mic track ended unexpectedly — ending the call");
|
|
1306
|
+
setAnswer("The microphone connection was lost — try starting the call again.");
|
|
1307
|
+
endRealtime();
|
|
1308
|
+
};
|
|
1309
|
+
stream.getAudioTracks().forEach((t) => t.addEventListener("ended", handleMicTrackEnded));
|
|
752
1310
|
rtCleanupRef.current = () => {
|
|
1311
|
+
clearInterval(micHealthCheck);
|
|
1312
|
+
stream.getAudioTracks().forEach((t) => t.removeEventListener("ended", handleMicTrackEnded));
|
|
753
1313
|
processor.disconnect();
|
|
754
1314
|
source.disconnect();
|
|
755
1315
|
stream.getTracks().forEach((t) => t.stop());
|
|
@@ -780,6 +1340,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
780
1340
|
rtTourAudioDoneRef.current = null;
|
|
781
1341
|
return;
|
|
782
1342
|
}
|
|
1343
|
+
rtLog("resumed listening");
|
|
1344
|
+
void audioCtx.resume().catch(() => { }); // don't wait up to 2s for the periodic health check if the browser already suspended capture
|
|
1345
|
+
micAudioSentSinceListeningRef.current = false;
|
|
783
1346
|
setRtStatus("rt-listening");
|
|
784
1347
|
setCaption("");
|
|
785
1348
|
void sendFreshContext(); // refresh before the user starts talking again, not after
|
|
@@ -794,10 +1357,28 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
794
1357
|
disarmThinkingWatchdog();
|
|
795
1358
|
rtThinkingWatchdogRef.current = setTimeout(() => {
|
|
796
1359
|
rtThinkingWatchdogRef.current = null;
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
1360
|
+
rtLog("thinking watchdog fired — server took over 20s, resuming listening and abandoning that turn");
|
|
1361
|
+
// Real, live-found gap: this used to only reset LOCAL state,
|
|
1362
|
+
// never telling the server anything — so a turn that was simply
|
|
1363
|
+
// SLOW (not actually stuck; e.g. retrying a rate-limited call
|
|
1364
|
+
// across every configured key, which can genuinely take longer
|
|
1365
|
+
// than this 20s watchdog) kept running server-side, and its
|
|
1366
|
+
// reply arrived LATE, after the user had already moved on and
|
|
1367
|
+
// started a new turn locally — landing on whatever was now
|
|
1368
|
+
// showing instead of being recognized as stale. triggerBargeIn()
|
|
1369
|
+
// is exactly the fix: it sends the same real barge_in signal a
|
|
1370
|
+
// genuine interruption does, bumping the server's own generation
|
|
1371
|
+
// so that late reply — whenever it finally arrives — carries an
|
|
1372
|
+
// old generation number and gets correctly dropped by the
|
|
1373
|
+
// isStaleRtMessage check above instead of confusingly resuming.
|
|
1374
|
+
triggerBargeIn();
|
|
1375
|
+
setLoopWorking(false);
|
|
1376
|
+
// triggerBargeIn() clears the caption but never touched `answer`
|
|
1377
|
+
// — without this, a timed-out turn gave the user literally
|
|
1378
|
+
// nothing: no reply, no error, just a silent reset back to
|
|
1379
|
+
// "Listening…" that reads as "it heard me and did nothing." A
|
|
1380
|
+
// real, live-found gap, not just a console.warn nobody sees.
|
|
1381
|
+
setAnswer("That's taking longer than expected — try asking again.");
|
|
801
1382
|
}, 20000);
|
|
802
1383
|
}
|
|
803
1384
|
function stopScheduledRtAudio() {
|
|
@@ -818,6 +1399,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
818
1399
|
// now-stale audio_chunk/speaking_end that was already in flight, so a
|
|
819
1400
|
// few straggling chunks can't sneak back in and resume playback.
|
|
820
1401
|
function triggerBargeIn() {
|
|
1402
|
+
rtLog("barge-in triggered", { wasTouring: touringRef.current, discardedAudioChunks: rtScheduledSourcesRef.current.length });
|
|
821
1403
|
disarmThinkingWatchdog();
|
|
822
1404
|
stopScheduledRtAudio();
|
|
823
1405
|
rtAudioDoneArrivingRef.current = true;
|
|
@@ -837,13 +1419,47 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
837
1419
|
if (ws.readyState === WebSocket.OPEN)
|
|
838
1420
|
ws.send(JSON.stringify({ type: "barge_in" }));
|
|
839
1421
|
setRtStatus("rt-listening");
|
|
840
|
-
|
|
1422
|
+
// Real, live-found bug this fix removes: this used to also clear
|
|
1423
|
+
// the caption here (setCaption("")). That's correct-looking for a
|
|
1424
|
+
// REAL, VAD-triggered barge-in (the user's about to say something
|
|
1425
|
+
// new) — the very next "final" already does archiveCurrentExchange()
|
|
1426
|
+
// then overwrites caption with the new utterance, so clearing it
|
|
1427
|
+
// here was always redundant for that path. But this function is
|
|
1428
|
+
// ALSO called by the thinking watchdog on a timeout, where there is
|
|
1429
|
+
// no new utterance coming — clearing the caption there wiped out
|
|
1430
|
+
// the very question that just timed out, an instant before
|
|
1431
|
+
// setAnswer(the timeout message) ran, leaving the live pair as
|
|
1432
|
+
// {caption: "", answer: "That's taking longer..."} — a reply
|
|
1433
|
+
// visibly floating with no question above it, and nothing for
|
|
1434
|
+
// archiveCurrentExchange() to pair it with on the next turn either
|
|
1435
|
+
// (archiveText skips empty text). Simply never clearing it here is
|
|
1436
|
+
// correct for both callers: the barge-in path already gets a fresh
|
|
1437
|
+
// caption from the next "final", and the watchdog path now keeps
|
|
1438
|
+
// the timed-out question correctly paired with its own answer.
|
|
841
1439
|
}
|
|
842
1440
|
ws.onopen = () => {
|
|
1441
|
+
rtLog("connection open");
|
|
843
1442
|
sendFreshContext();
|
|
844
1443
|
setRtStatus("rt-listening");
|
|
845
1444
|
rtStartingRef.current = false;
|
|
846
1445
|
};
|
|
1446
|
+
// True for a verb/speaking_start/audio_chunk/speaking_end/
|
|
1447
|
+
// turn_complete message that belongs to an EARLIER turn than the
|
|
1448
|
+
// most recent "final" this client has seen — see
|
|
1449
|
+
// rtLastFinalGenerationRef's own doc comment for the real race this
|
|
1450
|
+
// closes. A message with no generation field at all (shouldn't
|
|
1451
|
+
// happen against a server running this fix, but a mismatched client/
|
|
1452
|
+
// server version pair during a rolling deploy could) is treated as
|
|
1453
|
+
// current rather than dropped — additive/backward-compatible, same
|
|
1454
|
+
// discipline every other wire-protocol addition in this codebase
|
|
1455
|
+
// follows.
|
|
1456
|
+
function isStaleRtMessage(msg) {
|
|
1457
|
+
const stale = typeof msg.generation === "number" && msg.generation < rtLastFinalGenerationRef.current;
|
|
1458
|
+
if (stale)
|
|
1459
|
+
rtLog("dropped stale message", { type: msg.type, messageGeneration: msg.generation, currentGeneration: rtLastFinalGenerationRef.current });
|
|
1460
|
+
return stale;
|
|
1461
|
+
}
|
|
1462
|
+
let audioChunkCount = 0;
|
|
847
1463
|
ws.onmessage = (event) => {
|
|
848
1464
|
if (typeof event.data !== "string")
|
|
849
1465
|
return; // audio now arrives as base64 inside audio_chunk, not raw binary frames
|
|
@@ -852,14 +1468,35 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
852
1468
|
setCaption(msg.text);
|
|
853
1469
|
}
|
|
854
1470
|
else if (msg.type === "final") {
|
|
1471
|
+
rtLog("final transcript", { text: msg.text, generation: msg.generation });
|
|
1472
|
+
rtLastFinalGenerationRef.current = typeof msg.generation === "number" ? msg.generation : 0;
|
|
855
1473
|
archiveCurrentExchange(); // the previous turn's pair is complete — move it into history before this one starts overwriting caption/answer
|
|
856
1474
|
setCaption(msg.text);
|
|
1475
|
+
// Without this, `answer` still held the PREVIOUS turn's reply
|
|
1476
|
+
// text at the moment this new turn's own reply — if it ever
|
|
1477
|
+
// arrives — would overwrite it. Usually invisible (the previous
|
|
1478
|
+
// reply lands well before the next "final"), but a barge-in can
|
|
1479
|
+
// supersede an in-flight turn before its reply ever arrives (see
|
|
1480
|
+
// realtime-server.ts's onStep generation check) — with no reply
|
|
1481
|
+
// ever coming for THIS caption, the stale previous-turn answer
|
|
1482
|
+
// sat there and got archived alongside the wrong question on the
|
|
1483
|
+
// NEXT final, showing as a mismatched or duplicated-looking
|
|
1484
|
+
// reply. Found live: two turns' worth of the same fallback error
|
|
1485
|
+
// text ("Something went wrong on my end") appearing back to back
|
|
1486
|
+
// with only one visible question between them. Clearing to null
|
|
1487
|
+
// here means an abandoned turn now correctly archives with NO
|
|
1488
|
+
// reply bubble (archiveText skips empty text) instead of someone
|
|
1489
|
+
// else's.
|
|
1490
|
+
setAnswer(null);
|
|
857
1491
|
setRtStatus("rt-thinking");
|
|
858
1492
|
armThinkingWatchdog();
|
|
859
1493
|
}
|
|
860
1494
|
else if (msg.type === "verb") {
|
|
1495
|
+
if (isStaleRtMessage(msg))
|
|
1496
|
+
return; // belongs to a turn a later "final" already superseded
|
|
1497
|
+
rtLog("verb received", { verb: msg.verb?.verb, generation: msg.generation });
|
|
861
1498
|
const parsedStep = (0, core_1.safeParseVerbResponse)(msg.verb);
|
|
862
|
-
if (parsedStep && !core_1.
|
|
1499
|
+
if (parsedStep && !(0, core_1.isTerminalVerb)(parsedStep)) {
|
|
863
1500
|
// A continuing agent-loop step (click/fill/read/call_tool) —
|
|
864
1501
|
// the turn isn't over: execute it for real and report the
|
|
865
1502
|
// result back so the server can decide the next step, instead
|
|
@@ -870,7 +1507,39 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
870
1507
|
// — the server's loop stays quiet between steps on purpose,
|
|
871
1508
|
// to keep it fast.
|
|
872
1509
|
setAnswer(summarizeVerbForHistory(msg.verb));
|
|
873
|
-
|
|
1510
|
+
setLoopWorking(true);
|
|
1511
|
+
// Real, live-found bug: armThinkingWatchdog() only ever fired
|
|
1512
|
+
// once, on the turn's own "final" message, giving the WHOLE
|
|
1513
|
+
// multi-step turn one shared 20s budget — Executor + Planner +
|
|
1514
|
+
// Critic for step 1, then the same again for step 2, and so
|
|
1515
|
+
// on. Directly measured live: one single non-terminal step's
|
|
1516
|
+
// own Executor+Planner+Critic chain alone took ~14s (11.5s +
|
|
1517
|
+
// 1.5s + 1.1s) — a real, multi-step goal needing two or three
|
|
1518
|
+
// such steps blows straight through 20s even though each
|
|
1519
|
+
// individual step is proof of genuine progress, not a stall.
|
|
1520
|
+
// Re-arming here — once per continuing step, not once per
|
|
1521
|
+
// turn — gives every step its own fresh budget, so the
|
|
1522
|
+
// watchdog only ever fires on a step that's ACTUALLY stuck
|
|
1523
|
+
// (no verb/final/speaking_start arriving at all), matching
|
|
1524
|
+
// what its own fallback message ("taking longer than
|
|
1525
|
+
// expected") is supposed to mean.
|
|
1526
|
+
armThinkingWatchdog();
|
|
1527
|
+
// A FRESH scan, not the turn's starting liveMapRef snapshot —
|
|
1528
|
+
// real, live-found bug: a step in THIS SAME multi-step turn
|
|
1529
|
+
// (a "click New Agent" that opens a modal) can reveal DOM a
|
|
1530
|
+
// later step (a "fill" targeting the modal's own input) needs
|
|
1531
|
+
// to find, and liveMapRef is deliberately frozen once per
|
|
1532
|
+
// turn (see its own doc comment — that freeze exists to stop
|
|
1533
|
+
// a background rescan from shifting an id mid-flight during
|
|
1534
|
+
// ONE step's own round trip, not to survive across several
|
|
1535
|
+
// sequential steps that genuinely changed the page).
|
|
1536
|
+
// runTour() already does exactly this for its own steps, for
|
|
1537
|
+
// the identical reason. Without it, "click New Agent, then
|
|
1538
|
+
// type the name" reliably failed every time with "Could not
|
|
1539
|
+
// find that element on the page" — confirmed live, repeated
|
|
1540
|
+
// 5+ times in a row without ever recovering.
|
|
1541
|
+
const freshLiveMap = liveRegistryRef.current.getSnapshot().byId;
|
|
1542
|
+
void (0, verb_executor_1.executeToolStep)(msg.verb, pathnameRef.current, freshLiveMap, (route) => router.push(route), confirmToolCall).then((result) => {
|
|
874
1543
|
if (ws.readyState === WebSocket.OPEN) {
|
|
875
1544
|
ws.send(JSON.stringify({ type: "tool_result", observation: result?.observation ?? "no result" }));
|
|
876
1545
|
}
|
|
@@ -881,11 +1550,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
881
1550
|
handleVerb(msg.verb);
|
|
882
1551
|
}
|
|
883
1552
|
else if (msg.type === "speaking_start") {
|
|
1553
|
+
if (isStaleRtMessage(msg))
|
|
1554
|
+
return;
|
|
1555
|
+
rtLog("speaking start", { generation: msg.generation });
|
|
1556
|
+
audioChunkCount = 0;
|
|
884
1557
|
disarmThinkingWatchdog();
|
|
885
1558
|
rtAudioDoneArrivingRef.current = false;
|
|
886
1559
|
setRtStatus("rt-speaking");
|
|
887
1560
|
}
|
|
888
1561
|
else if (msg.type === "audio_chunk") {
|
|
1562
|
+
if (isStaleRtMessage(msg))
|
|
1563
|
+
return; // the literal "two speakers" case — a chunk from an abandoned turn, already in flight when the barge-in landed
|
|
1564
|
+
audioChunkCount++;
|
|
889
1565
|
const ctx = rtPlaybackCtxRef.current;
|
|
890
1566
|
const gain = rtPlaybackGainRef.current;
|
|
891
1567
|
if (!ctx || !gain)
|
|
@@ -918,6 +1594,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
918
1594
|
};
|
|
919
1595
|
}
|
|
920
1596
|
else if (msg.type === "speaking_end" || msg.type === "turn_complete") {
|
|
1597
|
+
if (isStaleRtMessage(msg))
|
|
1598
|
+
return; // a newer turn's own speaking_end/turn_complete will arrive and resume listening correctly on its own
|
|
1599
|
+
rtLog(msg.type, { audioChunks: audioChunkCount, generation: msg.generation });
|
|
921
1600
|
// turn_complete covers a verb with nothing spoken (a plain
|
|
922
1601
|
// highlight/navigate/do often has no text) — no audio_chunk ever
|
|
923
1602
|
// arrives for it, so rtScheduledSourcesRef is already empty and
|
|
@@ -927,10 +1606,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
927
1606
|
maybeResumeListening();
|
|
928
1607
|
}
|
|
929
1608
|
else if (msg.type === "error") {
|
|
1609
|
+
rtLog("server error", { message: msg.message });
|
|
930
1610
|
// Must actually unstick the turn, not just show the message —
|
|
931
1611
|
// otherwise the mic never resumes and the session is stuck
|
|
932
1612
|
// exactly the way a silently-dropped response used to leave it.
|
|
933
1613
|
disarmThinkingWatchdog();
|
|
1614
|
+
setLoopWorking(false);
|
|
934
1615
|
setAnswer(msg.message ?? "Something went wrong.");
|
|
935
1616
|
if (touringRef.current) {
|
|
936
1617
|
// A tour step's own speakStreamed() failed server-side (see
|
|
@@ -950,10 +1631,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
950
1631
|
}
|
|
951
1632
|
};
|
|
952
1633
|
ws.onerror = () => {
|
|
1634
|
+
rtLog("connection error");
|
|
953
1635
|
setAnswer("Couldn't connect to the realtime voice service.");
|
|
954
1636
|
endRealtime();
|
|
955
1637
|
};
|
|
956
|
-
ws.onclose = () => {
|
|
1638
|
+
ws.onclose = (closeEvent) => {
|
|
1639
|
+
rtLog("connection closed", { code: closeEvent.code, reason: closeEvent.reason, wasIdle: rtStateRef.current === "idle" });
|
|
957
1640
|
if (rtStateRef.current !== "idle")
|
|
958
1641
|
endRealtime();
|
|
959
1642
|
};
|
|
@@ -962,15 +1645,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
962
1645
|
setAnswer("Couldn't access the microphone — check your browser's permission for this site.");
|
|
963
1646
|
setRtStatus("idle");
|
|
964
1647
|
rtStartingRef.current = false;
|
|
1648
|
+
typedPlaybackSuspendedRef.current = false; // the call never actually started — don't leave typed replies permanently silenced
|
|
965
1649
|
}
|
|
966
1650
|
}
|
|
967
1651
|
function endRealtime() {
|
|
1652
|
+
rtLog("ending realtime call", { statusAtEnd: rtStateRef.current });
|
|
968
1653
|
if (rtThinkingWatchdogRef.current) {
|
|
969
1654
|
clearTimeout(rtThinkingWatchdogRef.current);
|
|
970
1655
|
rtThinkingWatchdogRef.current = null;
|
|
971
1656
|
}
|
|
972
1657
|
rtStartingRef.current = false;
|
|
973
1658
|
stopTypedPlayback();
|
|
1659
|
+
typedPlaybackSuspendedRef.current = false; // typed replies work normally again once no live call can race them
|
|
974
1660
|
rtSocketRef.current?.close();
|
|
975
1661
|
rtSocketRef.current = null;
|
|
976
1662
|
rtCleanupRef.current?.();
|
|
@@ -979,6 +1665,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
979
1665
|
setRtSpeakerMuted(false);
|
|
980
1666
|
setCaption("");
|
|
981
1667
|
setRtStatus("idle");
|
|
1668
|
+
setLoopWorking(false); // defensive — a connection dropping mid-loop must never leave the "still working" indicator stuck on
|
|
982
1669
|
tourGenerationRef.current++; // cancel an in-progress tour rather than leaving it stuck waiting to resume rt-listening
|
|
983
1670
|
touringRef.current = false;
|
|
984
1671
|
setTourStep(null);
|
|
@@ -1013,7 +1700,17 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
1013
1700
|
"rt-speaking": "Speaking…",
|
|
1014
1701
|
};
|
|
1015
1702
|
return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.length > 0 && ((0, jsx_runtime_1.jsxs)("button", { type: "button", className: "cairn-history-toggle", onClick: () => setHistoryExpanded((v) => !v), "aria-expanded": historyExpanded, children: [historyExpanded ? (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronUp, { size: 12 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronDown, { size: 12 }), historyExpanded ? "Hide earlier" : `${transcript.length} earlier`] })), historyExpanded &&
|
|
1016
|
-
transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.
|
|
1703
|
+
transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-bubble-text", children: [renderCaptionWords(answer), loopWorking && (
|
|
1704
|
+
// Real, live-reported gap this closes: this bubble
|
|
1705
|
+
// used to go static the instant a continuing step's
|
|
1706
|
+
// own progress text was shown ("Typing earbuds into
|
|
1707
|
+
// the search box"), with nothing telling the user
|
|
1708
|
+
// the agent was still actively working for however
|
|
1709
|
+
// long the next real LLM call took. Appended inline
|
|
1710
|
+
// (not swapped in place of the text, which the
|
|
1711
|
+
// no-answer-yet case below does) so the progress
|
|
1712
|
+
// text stays legible while still showing motion.
|
|
1713
|
+
(0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking cairn-thinking-inline", "aria-label": "Still working", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
|
|
1017
1714
|
e.preventDefault();
|
|
1018
1715
|
const trimmed = question.trim();
|
|
1019
1716
|
if (trimmed)
|
|
@@ -1021,6 +1718,75 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
|
|
|
1021
1718
|
}, children: (0, jsx_runtime_1.jsxs)("div", { className: "cairn-input-row", children: [(0, jsx_runtime_1.jsx)("input", { value: question, onChange: (e) => setQuestion(e.target.value), placeholder: "What do you need help with?", "aria-label": `Ask ${persona} a question`, disabled: recording || touring, autoFocus: true }), realtimeUrl && micSupported && ((0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn", "aria-label": "Start realtime conversation", onClick: () => void startRealtime(), disabled: busy || recording, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneCall, { size: 16 }) })), transcribeEndpoint && micSupported && ((0, jsx_runtime_1.jsx)("button", { type: "button", className: recording ? "cairn-icon-btn cairn-icon-btn-recording" : "cairn-icon-btn", "aria-label": recording ? "Stop recording" : "Ask by voice", onClick: () => (recording ? stopRecording() : void startRecording()), disabled: touring, children: recording ? (0, jsx_runtime_1.jsx)(lucide_react_1.Square, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) })), (0, jsx_runtime_1.jsx)("button", { type: "submit", className: "cairn-send", "aria-label": "Send", disabled: !question.trim() || busy || recording, children: asking ? (0, jsx_runtime_1.jsx)(lucide_react_1.Loader2, { size: 16, className: "cairn-spin" }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Send, { size: 16 }) })] }) }))] }))] }));
|
|
1022
1719
|
}
|
|
1023
1720
|
const MAX_HISTORY_TURNS = 8; // 4 exchanges — matches the same cap the realtime relay uses server-side
|
|
1721
|
+
exports.CONVERSATION_STORAGE_KEY = "cairn:conversation:v1";
|
|
1722
|
+
/**
|
|
1723
|
+
* Real, live-reported bug this closes: a host app's own mutation handler
|
|
1724
|
+
* calling a real `window.location.reload()` — a common, entirely valid
|
|
1725
|
+
* pattern; this SDK's own demo app uses it after several real actions,
|
|
1726
|
+
* e.g. moving a kanban card — tears down the ENTIRE React tree, this
|
|
1727
|
+
* widget included. Every bit of conversation state (the visible
|
|
1728
|
+
* transcript, the current exchange, even whether the panel was open)
|
|
1729
|
+
* reset to nothing, making a real, in-progress conversation look like it
|
|
1730
|
+
* had simply ended the instant a host page happened to reload — even
|
|
1731
|
+
* though nothing about the CONVERSATION itself was actually over.
|
|
1732
|
+
* `sessionStorage`, not `localStorage`, is deliberate: it survives
|
|
1733
|
+
* exactly a reload/navigation within the same tab — the real scope of
|
|
1734
|
+
* "this conversation" — and clears itself once the tab/window actually
|
|
1735
|
+
* closes, never lingering into an unrelated later visit the way
|
|
1736
|
+
* `localStorage` would.
|
|
1737
|
+
*/
|
|
1738
|
+
function loadPersistedConversation() {
|
|
1739
|
+
if (typeof window === "undefined")
|
|
1740
|
+
return null;
|
|
1741
|
+
try {
|
|
1742
|
+
const raw = window.sessionStorage.getItem(exports.CONVERSATION_STORAGE_KEY);
|
|
1743
|
+
if (!raw)
|
|
1744
|
+
return null;
|
|
1745
|
+
const parsed = JSON.parse(raw);
|
|
1746
|
+
if (!parsed || typeof parsed !== "object")
|
|
1747
|
+
return null;
|
|
1748
|
+
const transcript = Array.isArray(parsed.transcript)
|
|
1749
|
+
? parsed.transcript.filter((t) => !!t && typeof t === "object" && typeof t.id === "number" && typeof t.text === "string" && (t.role === "user" || t.role === "agent"))
|
|
1750
|
+
: [];
|
|
1751
|
+
return {
|
|
1752
|
+
transcript,
|
|
1753
|
+
lastQuestion: typeof parsed.lastQuestion === "string" ? parsed.lastQuestion : null,
|
|
1754
|
+
answer: typeof parsed.answer === "string" ? parsed.answer : null,
|
|
1755
|
+
open: Boolean(parsed.open),
|
|
1756
|
+
};
|
|
1757
|
+
}
|
|
1758
|
+
catch {
|
|
1759
|
+
return null; // private browsing, quota, or a genuinely corrupt value — never crash the widget over this
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
function savePersistedConversation(data) {
|
|
1763
|
+
if (typeof window === "undefined")
|
|
1764
|
+
return;
|
|
1765
|
+
try {
|
|
1766
|
+
window.sessionStorage.setItem(exports.CONVERSATION_STORAGE_KEY, JSON.stringify(data));
|
|
1767
|
+
}
|
|
1768
|
+
catch {
|
|
1769
|
+
// Storage unavailable/full — the conversation just won't survive a reload this time, never worth crashing the widget over.
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
/**
|
|
1773
|
+
* Rebuilds a real seed for `historyRef` (what gets sent to the model on
|
|
1774
|
+
* the NEXT typed turn) from a restored transcript — deliberately derived
|
|
1775
|
+
* from the same, already-persisted `transcript` rather than separately
|
|
1776
|
+
* persisting `historyRef`'s own shape: one real source of truth for "what
|
|
1777
|
+
* was actually said," not two that could quietly drift apart. Capped the
|
|
1778
|
+
* same way every other history array in this file already is.
|
|
1779
|
+
*/
|
|
1780
|
+
function reconstructHistoryFromPersisted(persisted) {
|
|
1781
|
+
if (!persisted)
|
|
1782
|
+
return [];
|
|
1783
|
+
const fromTranscript = persisted.transcript.map((t) => ({ role: t.role === "agent" ? "assistant" : "user", text: t.text }));
|
|
1784
|
+
const live = [
|
|
1785
|
+
...(persisted.lastQuestion ? [{ role: "user", text: persisted.lastQuestion }] : []),
|
|
1786
|
+
...(persisted.answer ? [{ role: "assistant", text: persisted.answer }] : []),
|
|
1787
|
+
];
|
|
1788
|
+
return [...fromTranscript, ...live].slice(-MAX_HISTORY_TURNS);
|
|
1789
|
+
}
|
|
1024
1790
|
/** Best-effort text form of a raw (unvalidated) verb response for the
|
|
1025
1791
|
* conversation-history log — not shown to the user, just fed back to the
|
|
1026
1792
|
* model on later turns. Deliberately loose/defensive rather than a full
|
|
@@ -1051,6 +1817,12 @@ function summarizeVerbForHistory(raw) {
|
|
|
1051
1817
|
return `(read ${String(v.target)})`;
|
|
1052
1818
|
case "call_tool":
|
|
1053
1819
|
return `(called ${String(v.name)})`;
|
|
1820
|
+
case "drag":
|
|
1821
|
+
return `(dragged ${String(v.target)} to ${String(v.to)})`;
|
|
1822
|
+
case "select":
|
|
1823
|
+
return `(selected "${String(v.value)}" in ${String(v.target)})`;
|
|
1824
|
+
case "key":
|
|
1825
|
+
return `(pressed ${String(v.key)}${v.target ? ` on ${String(v.target)}` : ""})`;
|
|
1054
1826
|
case "batch":
|
|
1055
1827
|
return Array.isArray(v.actions) ? `(${v.actions.length} steps: ${v.actions.map((a) => a.verb).join(", ")})` : "(batch)";
|
|
1056
1828
|
default:
|
|
@@ -1070,25 +1842,32 @@ function renderCaptionWords(text) {
|
|
|
1070
1842
|
const words = text.split(" ");
|
|
1071
1843
|
return words.map((word, i) => ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-word", style: { animationDelay: `${Math.min(i * 55, 2800)}ms` }, children: [word, i < words.length - 1 ? " " : ""] }, i)));
|
|
1072
1844
|
}
|
|
1845
|
+
// "Waybalance" — three real, irregular stones (an ellipse plus a smaller
|
|
1846
|
+
// bump, not a rectangle), stacked slightly off-center the way a hiker
|
|
1847
|
+
// actually balances a trail cairn, instead of the perfectly centered flat
|
|
1848
|
+
// bars this replaced. Same mark as docs/images/logo.svg and site/index.html's
|
|
1849
|
+
// nav badge, just currentColor here so it inherits the button's own color.
|
|
1073
1850
|
function CairnMark() {
|
|
1074
|
-
return ((0, jsx_runtime_1.jsxs)("svg", { width: "20", height: "20", viewBox: "0 0 20 20", fill: "none", "aria-hidden": "true", children: [(0, jsx_runtime_1.jsx)("
|
|
1851
|
+
return ((0, jsx_runtime_1.jsxs)("svg", { width: "20", height: "20", viewBox: "0 0 20 20", fill: "none", "aria-hidden": "true", children: [(0, jsx_runtime_1.jsx)("ellipse", { cx: "10", cy: "14.6", rx: "6.1", ry: "2.3", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "6.3", cy: "14.1", rx: "2.5", ry: "1.6", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "9.1", cy: "10.4", rx: "4.3", ry: "2", fill: "currentColor", opacity: "0.82" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "6.2", cy: "10", rx: "1.7", ry: "1.2", fill: "currentColor", opacity: "0.82" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "11.7", cy: "6.4", rx: "2.6", ry: "1.6", fill: "currentColor", opacity: "0.6" })] }));
|
|
1852
|
+
}
|
|
1853
|
+
// Real-time lifecycle logging — every message received, every decision made
|
|
1854
|
+
// about it (played, spoken, dropped, and why), every state transition. Added
|
|
1855
|
+
// specifically so a live session's actual behavior is visible in the browser
|
|
1856
|
+
// console instead of only inferable from symptoms after the fact — every bug
|
|
1857
|
+
// found and fixed in this file today was diagnosed from screenshots and
|
|
1858
|
+
// terminal output because nothing like this existed before. `[cairn rt]` is
|
|
1859
|
+
// the tag to filter on. Deliberately excludes per-audio_chunk noise (dozens
|
|
1860
|
+
// of chunks per turn would flood the console) — chunk activity shows up as
|
|
1861
|
+
// a one-line count at speaking_end/turn_complete instead.
|
|
1862
|
+
function rtLog(event, details) {
|
|
1863
|
+
if (details)
|
|
1864
|
+
console.log("[cairn rt]", event, details);
|
|
1865
|
+
else
|
|
1866
|
+
console.log("[cairn rt]", event);
|
|
1075
1867
|
}
|
|
1076
1868
|
// ---------------------------------------------------------------------------
|
|
1077
1869
|
// Audio helpers (real-time PCM16 capture — standard Web Audio API patterns)
|
|
1078
1870
|
// ---------------------------------------------------------------------------
|
|
1079
|
-
// Heuristic energy gate for barge-in: real speech into a laptop/phone mic
|
|
1080
|
-
// typically sits well above this; normal room noise and the mic's own
|
|
1081
|
-
// noise floor typically sit below it. Not calibrated against real hardware
|
|
1082
|
-
// in this environment (no live mic here) — reasonable starting point, may
|
|
1083
|
-
// need tuning against a real device if it proves too trigger-happy or too
|
|
1084
|
-
// insensitive in practice.
|
|
1085
|
-
const BARGE_IN_RMS_THRESHOLD = 0.02;
|
|
1086
|
-
function computeRms(channelData) {
|
|
1087
|
-
let sumSquares = 0;
|
|
1088
|
-
for (let i = 0; i < channelData.length; i++)
|
|
1089
|
-
sumSquares += channelData[i] * channelData[i];
|
|
1090
|
-
return Math.sqrt(sumSquares / channelData.length);
|
|
1091
|
-
}
|
|
1092
1871
|
function downsampleTo16k(input, inputSampleRate) {
|
|
1093
1872
|
const targetRate = 16000;
|
|
1094
1873
|
if (inputSampleRate === targetRate)
|
|
@@ -1130,9 +1909,13 @@ const COPILOT_STYLES = `
|
|
|
1130
1909
|
0%, 100% { box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.4); }
|
|
1131
1910
|
70% { box-shadow: 0 0 0 10px rgba(16, 185, 129, 0); }
|
|
1132
1911
|
}
|
|
1133
|
-
@keyframes cairn-pulse-
|
|
1134
|
-
0%, 100% { box-shadow: 0 0 0 0 rgba(
|
|
1135
|
-
70% { box-shadow: 0 0 0 10px rgba(
|
|
1912
|
+
@keyframes cairn-pulse-ember {
|
|
1913
|
+
0%, 100% { box-shadow: 0 0 0 0 rgba(224, 122, 63, 0.4); }
|
|
1914
|
+
70% { box-shadow: 0 0 0 10px rgba(224, 122, 63, 0); }
|
|
1915
|
+
}
|
|
1916
|
+
@keyframes cairn-cursor-arrive {
|
|
1917
|
+
0% { box-shadow: 0 0 0 0 rgba(224, 122, 63, 0.55); }
|
|
1918
|
+
100% { box-shadow: 0 0 0 9px rgba(224, 122, 63, 0); }
|
|
1136
1919
|
}
|
|
1137
1920
|
@keyframes cairn-spin {
|
|
1138
1921
|
from { transform: rotate(0deg); }
|
|
@@ -1148,7 +1931,7 @@ const COPILOT_STYLES = `
|
|
|
1148
1931
|
}
|
|
1149
1932
|
@keyframes cairn-word-sweep {
|
|
1150
1933
|
0% { opacity: 0.35; text-shadow: none; }
|
|
1151
|
-
35% { opacity: 1; color: #
|
|
1934
|
+
35% { opacity: 1; color: #E07A3F; text-shadow: 0 0 10px rgba(224, 122, 63, 0.45); }
|
|
1152
1935
|
100% { opacity: 1; color: inherit; text-shadow: none; }
|
|
1153
1936
|
}
|
|
1154
1937
|
@keyframes cairn-thinking-bounce {
|
|
@@ -1156,16 +1939,19 @@ const COPILOT_STYLES = `
|
|
|
1156
1939
|
40% { opacity: 0.9; transform: translateY(-3px); }
|
|
1157
1940
|
}
|
|
1158
1941
|
.cairn-glow {
|
|
1159
|
-
animation: cairn-pulse-
|
|
1160
|
-
outline: 2px solid #
|
|
1942
|
+
animation: cairn-pulse-ember 1.1s ease-out 2;
|
|
1943
|
+
outline: 2px solid #E07A3F;
|
|
1161
1944
|
outline-offset: 3px;
|
|
1162
1945
|
border-radius: 8px;
|
|
1163
1946
|
}
|
|
1947
|
+
.cairn-cursor-hover {
|
|
1948
|
+
animation: cairn-cursor-arrive 0.3s ease-out;
|
|
1949
|
+
}
|
|
1164
1950
|
.cairn-spin {
|
|
1165
1951
|
animation: cairn-spin 0.8s linear infinite;
|
|
1166
1952
|
}
|
|
1167
1953
|
@media (prefers-reduced-motion: reduce) {
|
|
1168
|
-
.cairn-fab, .cairn-panel, .cairn-bubble, .cairn-word, .cairn-thinking-dot {
|
|
1954
|
+
.cairn-fab, .cairn-panel, .cairn-bubble, .cairn-word, .cairn-thinking-dot, #cairn-cursor {
|
|
1169
1955
|
animation: none !important;
|
|
1170
1956
|
transition: none !important;
|
|
1171
1957
|
}
|
|
@@ -1300,6 +2086,10 @@ const COPILOT_STYLES = `
|
|
|
1300
2086
|
gap: 4px;
|
|
1301
2087
|
padding: 2px 0;
|
|
1302
2088
|
}
|
|
2089
|
+
.cairn-thinking-inline {
|
|
2090
|
+
margin-left: 6px;
|
|
2091
|
+
vertical-align: middle;
|
|
2092
|
+
}
|
|
1303
2093
|
.cairn-thinking-dot {
|
|
1304
2094
|
width: 5px;
|
|
1305
2095
|
height: 5px;
|