@cairnvibe/sdk 0.2.13 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/agent-loop.d.ts +113 -0
  2. package/dist/agent-loop.js +128 -0
  3. package/dist/cairn-widget.js +14 -9
  4. package/dist/cursor-overlay.d.ts +19 -0
  5. package/dist/cursor-overlay.js +126 -0
  6. package/dist/element-ladder.d.ts +71 -0
  7. package/dist/element-ladder.js +168 -0
  8. package/dist/index.d.ts +79 -1
  9. package/dist/index.js +886 -96
  10. package/dist/key-rotator.d.ts +28 -0
  11. package/dist/key-rotator.js +57 -3
  12. package/dist/memory-sqlite.d.ts +86 -0
  13. package/dist/memory-sqlite.js +230 -0
  14. package/dist/realtime-cli.js +22 -1
  15. package/dist/realtime-server.d.ts +83 -2
  16. package/dist/realtime-server.js +561 -121
  17. package/dist/server.d.ts +266 -5
  18. package/dist/server.js +1013 -83
  19. package/dist/skill-store.d.ts +17 -0
  20. package/dist/skill-store.js +78 -0
  21. package/dist/tts-stream.d.ts +25 -0
  22. package/dist/tts-stream.js +32 -0
  23. package/dist/vad.d.ts +27 -0
  24. package/dist/vad.js +128 -0
  25. package/dist/verb-executor.d.ts +32 -11
  26. package/dist/verb-executor.js +315 -39
  27. package/dist/webmcp-client.d.ts +14 -1
  28. package/dist/webmcp-client.js +22 -1
  29. package/package.json +3 -1
  30. package/src/agent-loop.ts +222 -0
  31. package/src/cursor-overlay.ts +130 -0
  32. package/src/element-ladder.ts +170 -0
  33. package/src/index.tsx +935 -100
  34. package/src/key-rotator.ts +57 -2
  35. package/src/memory-sqlite.ts +283 -0
  36. package/src/realtime-cli.ts +24 -1
  37. package/src/realtime-server.ts +669 -123
  38. package/src/server.ts +1119 -83
  39. package/src/skill-store.ts +88 -0
  40. package/src/tts-stream.ts +30 -0
  41. package/src/vad.ts +153 -0
  42. package/src/verb-executor.ts +329 -42
  43. package/src/web-component.ts +97 -24
  44. package/src/webmcp-client.ts +30 -2
package/dist/index.js CHANGED
@@ -1,18 +1,25 @@
1
1
  "use strict";
2
2
  "use client";
3
3
  Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.CONVERSATION_STORAGE_KEY = void 0;
4
5
  exports.Copilot = Copilot;
6
+ exports.loadPersistedConversation = loadPersistedConversation;
7
+ exports.savePersistedConversation = savePersistedConversation;
8
+ exports.reconstructHistoryFromPersisted = reconstructHistoryFromPersisted;
5
9
  const jsx_runtime_1 = require("react/jsx-runtime");
6
10
  const react_1 = require("react");
7
11
  const navigation_1 = require("next/navigation");
8
12
  const lucide_react_1 = require("lucide-react");
9
13
  const core_1 = require("@cairnvibe/core");
14
+ const agent_loop_1 = require("./agent-loop");
10
15
  const context_collector_1 = require("./context-collector");
16
+ const cursor_overlay_1 = require("./cursor-overlay");
11
17
  const element_ladder_1 = require("./element-ladder");
12
18
  const runtime_scan_1 = require("./runtime-scan");
13
19
  const webmcp_client_1 = require("./webmcp-client");
14
20
  const verb_executor_1 = require("./verb-executor");
15
- function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, realtimeUrl, persona = "Cairn", }) {
21
+ const vad_1 = require("./vad");
22
+ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, scopeId, planEndpoint, criticEndpoint, skillsSaveEndpoint, realtimeUrl, persona = "Cairn", }) {
16
23
  const pathname = (0, navigation_1.usePathname)() ?? "/";
17
24
  // Mirrors `pathname` for use inside long-lived closures (a realtime
18
25
  // session's handlers are all created once, when the connection opens —
@@ -26,7 +33,21 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
26
33
  // eslint-disable-next-line react-hooks/exhaustive-deps
27
34
  }, [pathname]);
28
35
  const router = (0, navigation_1.useRouter)();
36
+ // Starts at the same safe default on both server and client's first
37
+ // render — same hydration-mismatch reason `micSupported` below does this
38
+ // — then, once actually restored from sessionStorage post-mount (see the
39
+ // dedicated restore effect further down), flips to whatever a real page
40
+ // reload (a host app's own mutation handler, e.g. — see
41
+ // loadPersistedConversation's own doc comment) had showing a moment ago,
42
+ // so a real reload never again looks like the conversation simply ended.
29
43
  const [open, setOpen] = (0, react_1.useState)(false);
44
+ // Fades the synthetic cursor out once the panel closes (or the widget
45
+ // itself unmounts) instead of leaving it sitting visible on the page.
46
+ (0, react_1.useEffect)(() => {
47
+ if (!open)
48
+ (0, cursor_overlay_1.hideCursor)();
49
+ return () => (0, cursor_overlay_1.hideCursor)();
50
+ }, [open]);
30
51
  // Collapsed by default so the panel only ever shows the current exchange
31
52
  // — the full archived transcript (built up over a long conversation)
32
53
  // stays out of the way behind an explicit toggle instead of always being
@@ -34,6 +55,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
34
55
  const [historyExpanded, setHistoryExpanded] = (0, react_1.useState)(false);
35
56
  const [question, setQuestion] = (0, react_1.useState)("");
36
57
  const [answer, setAnswer] = (0, react_1.useState)(null);
58
+ // Real, live-reported gap this closes: once a continuing agent-loop step
59
+ // (click/fill/read/call_tool/batch) sets `answer` to its own progress
60
+ // text ("Typing earbuds into the search box"), that text just sat there
61
+ // unchanged for however long the NEXT resolveVerb call took — several
62
+ // real seconds, more under rate-limit retries — with nothing on screen
63
+ // telling the user the agent was still actually doing something. `answer`
64
+ // itself can't double as that signal (a terminal turn's own real,
65
+ // finished answer looks identical to unfinished progress text). This is
66
+ // a separate, explicit flag: true from the moment a continuing step's
67
+ // progress text is shown until the turn actually ends (a terminal verb,
68
+ // an error, or a give-up) — see its own setters below for exactly where.
69
+ const [loopWorking, setLoopWorking] = (0, react_1.useState)(false);
37
70
  const [status, setStatus] = (0, react_1.useState)("idle");
38
71
  const [caption, setCaption] = (0, react_1.useState)("");
39
72
  // The user's own last question, shown as its own floating caption bubble
@@ -106,6 +139,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
106
139
  const rtMicMutedRef = (0, react_1.useRef)(false);
107
140
  const rtSpeakerMutedRef = (0, react_1.useRef)(false);
108
141
  const rtStartingRef = (0, react_1.useRef)(false); // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
142
+ // The generation of the most recent "final" this client has processed —
143
+ // see ServerMessage's own doc comment in realtime-server.ts for the full
144
+ // real, live-found bug this closes: the client's own local barge-in can
145
+ // start a NEW turn (a new "final") before an EARLIER turn's own verb/
146
+ // audio, already in flight on the wire when the server processed that
147
+ // barge-in, actually arrives. WebSocket delivers messages in order, but
148
+ // "in order" isn't "still current" — every verb/speaking_start/
149
+ // audio_chunk/speaking_end/turn_complete message carries the generation
150
+ // it was produced under, and the handler drops it outright if it's
151
+ // older than this ref's value instead of applying it to whatever
152
+ // caption happens to be showing now.
153
+ const rtLastFinalGenerationRef = (0, react_1.useRef)(0);
109
154
  // Progressive PCM playback for the buffered (non-realtime) speak endpoint
110
155
  // — the same gapless AudioBufferSourceNode scheduling the realtime path
111
156
  // uses for its audio_chunk messages (see rtPlaybackCtxRef below), just fed
@@ -118,6 +163,37 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
118
163
  const typedPlaybackGainRef = (0, react_1.useRef)(null);
119
164
  const typedNextPlayTimeRef = (0, react_1.useRef)(0);
120
165
  const typedScheduledSourcesRef = (0, react_1.useRef)([]);
166
+ // Bumped by stopTypedPlayback() every time it runs — playPcmStream's own
167
+ // async reader loop checks this before scheduling each chunk, so a
168
+ // superseded call (two speak()/ask() replies resolving close together)
169
+ // actually STOPS reading and scheduling more audio once a newer call has
170
+ // taken over, instead of continuing to push nodes into the shared
171
+ // playback graph behind the newer call's back. stopTypedPlayback() on
172
+ // its own only ever stopped nodes that already existed at the moment it
173
+ // ran — it never told an in-flight stream reader to stop producing MORE
174
+ // of them, which is exactly what let two replies' audio genuinely
175
+ // overlap (one starting, then a second call's audio starting on top of
176
+ // it moments later) — a real, live-found race, not a guess.
177
+ const typedPlaybackGenerationRef = (0, react_1.useRef)(0);
178
+ // A DIFFERENT real gap the generation counter above doesn't close: it
179
+ // only protects one typed reply's audio against ANOTHER typed reply's
180
+ // audio. A typed ask()/speak() call already in flight — its /api/
181
+ // copilot/speak fetch genuinely takes several seconds under real
182
+ // conditions (rate-limit retries make this worse, not better) — has no
183
+ // way to know a realtime call started WHILE it was still waiting. Its
184
+ // response arrives, and normally-innocent code plays it, seconds after
185
+ // startRealtime() already ran — genuinely overlapping with the live
186
+ // call's own audio, since nothing about the realtime session's own
187
+ // start/mute/barge-in controls have any way to reach a typed reply
188
+ // that hadn't even been scheduled yet when they ran. Found live: two
189
+ // full agent answers, sourced entirely from separate /api/copilot/
190
+ // speak calls, audibly overlapping about a second apart, while a
191
+ // realtime call was the only thing visibly active in the UI the whole
192
+ // time. startRealtime() sets this; endRealtime() clears it; speak()/
193
+ // speakAndWait() check it AFTER their fetch resolves and drop the
194
+ // reply's audio entirely (never call playPcmStream at all) if a
195
+ // realtime call has taken over since the request was made.
196
+ const typedPlaybackSuspendedRef = (0, react_1.useRef)(false);
121
197
  // Watchdog for the "rt-thinking" state: started on every "final" transcript,
122
198
  // cleared the moment the server responds with anything for that turn
123
199
  // (verb/speaking_start/speaking_end/turn_complete/error). If it ever
@@ -140,6 +216,36 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
140
216
  // finished arriving, so the mic can't start sending while the agent is
141
217
  // still audibly speaking.
142
218
  const rtAudioDoneArrivingRef = (0, react_1.useRef)(true);
219
+ // Diagnostic only, not a functional guard — flips to true the first time
220
+ // a real mic packet is actually sent after transitioning to
221
+ // "rt-listening", logged once (not per-packet, which would flood the
222
+ // console). Added specifically so a "status says Listening… but nothing
223
+ // I say gets picked up" report can be told apart, from the log alone,
224
+ // between "the send gate never opened" (this never logs) and "the gate
225
+ // opened and sent real audio, so the problem is somewhere else entirely
226
+ // — Deepgram's own STT, or a real hardware/OS mic issue this app can't
227
+ // see or fix" (this logs once, then goes quiet as expected).
228
+ const micAudioSentSinceListeningRef = (0, react_1.useRef)(false);
229
+ // Safety net for a live realtime call outliving this component instance:
230
+ // without this, unmounting (a parent removing the widget, a route change
231
+ // that remounts it, or — the real, live-hit case — Next.js Fast Refresh
232
+ // remounting the component on every dev-mode source edit while a call is
233
+ // open) left the WebSocket, the open getUserMedia mic stream, and both
234
+ // AudioContexts running completely orphaned: nothing ever called
235
+ // rtSocketRef.current?.close() or stopped the mic tracks. The old,
236
+ // zombie connection then kept transcribing and replying in parallel with
237
+ // whatever the newly-mounted instance does next — the literal
238
+ // "two things running in parallel, the agent answering twice" bug found
239
+ // live. endRealtime() is safe to call from an unmount cleanup even
240
+ // though it also calls React state setters: it only reads stable refs
241
+ // (never a stale closure over props/state), and React 18 silently no-ops
242
+ // a setState call on an already-unmounted component.
243
+ (0, react_1.useEffect)(() => {
244
+ return () => {
245
+ if (rtSocketRef.current || rtCleanupRef.current)
246
+ endRealtime();
247
+ };
248
+ }, []);
143
249
  // Starts false on both server and client's first render (avoids a
144
250
  // hydration mismatch — `navigator` doesn't exist during SSR), then
145
251
  // updated after mount, once we're only ever running in the browser.
@@ -147,6 +253,36 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
147
253
  (0, react_1.useEffect)(() => {
148
254
  setMicSupported(!!navigator.mediaDevices?.getUserMedia && typeof MediaRecorder !== "undefined");
149
255
  }, []);
256
+ // Restores a conversation that survived a real page reload — same
257
+ // hydration-safety reason as `micSupported` just above: `open`/`answer`/
258
+ // `lastQuestion`/`transcript` all start at their normal empty defaults on
259
+ // both server and client's first render (matching what SSR produced), and
260
+ // only flip to whatever sessionStorage actually holds here, post-mount.
261
+ // Real, live-verified bug this closes: a host app's own mutation handler
262
+ // calling `window.location.reload()` (this SDK's own demo app does this
263
+ // after several real actions, e.g. moving a kanban card) tears down the
264
+ // entire React tree, this widget included — every bit of visible
265
+ // conversation state reset to nothing, making an in-progress conversation
266
+ // look like it had simply ended the instant the host page happened to
267
+ // reload, even though nothing about the CONVERSATION itself was over.
268
+ (0, react_1.useEffect)(() => {
269
+ const persisted = loadPersistedConversation();
270
+ if (!persisted)
271
+ return;
272
+ setOpen(persisted.open);
273
+ setAnswer(persisted.answer);
274
+ setLastQuestion(persisted.lastQuestion);
275
+ setTranscript(persisted.transcript);
276
+ historyRef.current = reconstructHistoryFromPersisted(persisted);
277
+ // Starts past the restored transcript's own highest id — otherwise the
278
+ // very next archived entry would reuse an id already on screen, a real
279
+ // duplicate-React-key bug (React silently confuses which DOM node is
280
+ // which when two list items share a key).
281
+ if (persisted.transcript.length > 0) {
282
+ transcriptIdRef.current = Math.max(...persisted.transcript.map((t) => t.id)) + 1;
283
+ }
284
+ // eslint-disable-next-line react-hooks/exhaustive-deps
285
+ }, []);
150
286
  const asking = status === "asking";
151
287
  const recording = status === "recording";
152
288
  const realtimeActive = status.startsWith("rt-");
@@ -166,6 +302,25 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
166
302
  (0, react_1.useEffect)(() => {
167
303
  answerRef.current = answer;
168
304
  }, [answer]);
305
+ // Persists the visible conversation to sessionStorage on every change, so
306
+ // a host app's own `window.location.reload()` (e.g. after a board-card
307
+ // move — see loadPersistedConversation's own comment for the full story)
308
+ // doesn't make an in-progress conversation look like it simply ended.
309
+ // Skips its own very first (mount) invocation on purpose: that call is
310
+ // always tied to the initial render's plain defaults (open:false,
311
+ // transcript:[], ...) — captured before the restore effect above's
312
+ // setState calls have actually landed — so persisting it would clobber a
313
+ // real, just-restored conversation with empty state for one tick, right
314
+ // before the corrective post-restore render fixes it back. Skipping costs
315
+ // nothing on a genuinely fresh session (there's nothing to persist yet).
316
+ const skippedMountPersistRef = (0, react_1.useRef)(false);
317
+ (0, react_1.useEffect)(() => {
318
+ if (!skippedMountPersistRef.current) {
319
+ skippedMountPersistRef.current = true;
320
+ return;
321
+ }
322
+ savePersistedConversation({ transcript, lastQuestion, answer, open });
323
+ }, [transcript, lastQuestion, answer, open]);
169
324
  // Auto-scroll to the newest content whenever the transcript grows or the
170
325
  // live (not-yet-archived) bubble's text changes.
171
326
  (0, react_1.useEffect)(() => {
@@ -235,11 +390,38 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
235
390
  }
236
391
  }
237
392
  function handleVerb(raw) {
393
+ setLoopWorking(false); // only ever called for a genuinely terminal verb — the turn is over
238
394
  (0, verb_executor_1.executeVerbResponse)(raw, pathname, {
239
395
  onExplain: (text) => {
240
396
  setAnswer(text);
241
- if (!realtimeActive)
242
- void speak(text); // realtime mode gets audio over the socket instead
397
+ // A REAL, deep, long-standing bug found live — not introduced by
398
+ // today's other fixes, just newly diagnosed: this function is a
399
+ // plain closure defined fresh every render, but when a "verb" WS
400
+ // message arrives it's invoked through ws.onmessage — a callback
401
+ // assigned ONCE, inside startRealtime(), and never reassigned for
402
+ // the rest of that connection's life. `realtimeActive` there is
403
+ // therefore frozen at whatever it was AT THE MOMENT startRealtime()
404
+ // was called — which is BEFORE the click handler's own state
405
+ // updates land, so it reads `false` for literally the entire
406
+ // lifetime of every realtime call. `!realtimeActive` was therefore
407
+ // ALWAYS true here, on every single realtime turn — this ran
408
+ // speak() (the typed HTTP path) in addition to the correct
409
+ // realtime audio_chunk playback, every time. This is the actual,
410
+ // original root cause of "two speakers" — the earlier
411
+ // typedPlaybackSuspendedRef fix only ever suppressed the resulting
412
+ // AUDIO once this was already firing, it never stopped the
413
+ // firing itself (or the wasted LLM/TTS call and quota burn
414
+ // underneath it). rtStateRef mirrors status specifically to avoid
415
+ // this class of bug in a callback like this one (see its own doc
416
+ // comment) — using it here instead of the stale const is the
417
+ // actual fix.
418
+ if (rtStateRef.current.startsWith("rt-")) {
419
+ rtLog("explain: realtime call active, letting the socket's own audio_chunk stream speak this");
420
+ }
421
+ else {
422
+ rtLog("explain: no realtime call active, using the typed speak() HTTP path");
423
+ void speak(text);
424
+ }
243
425
  },
244
426
  onNavigate: (route) => router.push(route),
245
427
  onMiss: reportMiss,
@@ -267,7 +449,16 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
267
449
  */
268
450
  async function runTour(steps) {
269
451
  const myGeneration = ++tourGenerationRef.current;
270
- const wasRealtimeListening = realtimeActive;
452
+ // Same real stale-closure bug as onExplain's own fix above, and the
453
+ // reason a realtime-triggered tour spoke through the wrong pipeline
454
+ // (or, after that fix suppressed the wrong pipeline's audio, spoke
455
+ // through nothing at all — "tour did not speak anything") and could
456
+ // leave the mic never properly told to resume listening afterward
457
+ // (see the `setRtStatus("rt-listening")` call at the end of this
458
+ // function, fixed the same way). rtStateRef.current is always
459
+ // current, regardless of which render's closure this particular
460
+ // invocation runs inside.
461
+ const wasRealtimeListening = rtStateRef.current.startsWith("rt-");
271
462
  touringRef.current = true;
272
463
  if (wasRealtimeListening)
273
464
  setRtStatus("rt-speaking");
@@ -350,7 +541,31 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
350
541
  return;
351
542
  setTourStep(null);
352
543
  setCaption("");
353
- if (wasRealtimeListening && realtimeActive)
544
+ // The most damaging half of this stale-closure bug: this used to
545
+ // read the stale `realtimeActive` const, which meant this call was
546
+ // ALWAYS skipped for a tour reached via realtime — the mic was
547
+ // never explicitly told to resume listening once the tour ended.
548
+ // rtStateRef.current.startsWith("rt-") is what actually reflects
549
+ // whether the connection is still live right now.
550
+ if (wasRealtimeListening && rtStateRef.current.startsWith("rt-"))
551
+ setRtStatus("rt-listening");
552
+ }
553
+ catch (err) {
554
+ // Real, live-found gap: this function had NO catch at all — only
555
+ // try/finally. Any error thrown anywhere in the loop above (a
556
+ // rejected speakOverRealtime/speakAndWait call, a DOM exception
557
+ // from el.click(), a network failure) skipped the resume-listening
558
+ // line above ENTIRELY, leaving the mic stuck exactly where the
559
+ // tour left off — matching "after this it's not listening"
560
+ // reported live. Every other place in this file that can fail
561
+ // mid-turn (ask(), handleDeepgramMessage's own server-side
562
+ // equivalent) already guarantees SOME recovery path; this one
563
+ // didn't have one at all.
564
+ console.error("[cairn] tour failed partway through:", err);
565
+ rtLog("tour failed — forcing the mic back to listening instead of leaving it stuck", { error: String(err) });
566
+ setTourStep(null);
567
+ setCaption("");
568
+ if (wasRealtimeListening && rtStateRef.current.startsWith("rt-"))
354
569
  setRtStatus("rt-listening");
355
570
  }
356
571
  finally {
@@ -371,13 +586,114 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
371
586
  await runTypedAgentLoop(q);
372
587
  }
373
588
  catch {
374
- setAnswer("Something went wrong reaching the help service try again in a moment.");
589
+ // See typedPlaybackSuspendedRef's own doc commentthis whole
590
+ // fetch can still be in flight when a realtime call starts.
591
+ if (!typedPlaybackSuspendedRef.current)
592
+ setAnswer("Something went wrong reaching the help service — try again in a moment.");
375
593
  }
376
594
  finally {
377
- setStatus("idle");
595
+ // The most damaging form of the same race: unconditionally forcing
596
+ // status back to "idle" here, after a realtime call has ALREADY
597
+ // taken over (status is some "rt-*" value), would silently kick the
598
+ // UI out of the live call — hiding its mic/speaker/hangup controls
599
+ // and showing the "start call" screen instead — while the actual
600
+ // WebSocket connection underneath is still fully alive and still
601
+ // talking, now with no visible way to manage it at all. Skipping
602
+ // this reset when suspended is what stops a slow, stale typed
603
+ // request from ever being able to do that.
604
+ if (!typedPlaybackSuspendedRef.current)
605
+ setStatus("idle");
606
+ }
607
+ }
608
+ /**
609
+ * Architecture Pillar 6 (the safety layer) — the real, working default
610
+ * confirmation UI: a native browser confirm dialog, naming the tool's
611
+ * OWN real name/description (never inventing wording), for a WebMCP
612
+ * tool whose registration declared `riskTier: "confirm"` (a payment, a
613
+ * delete, anything hard to undo). `window.confirm` blocks the calling
614
+ * microtask until the user actually answers — exactly the real,
615
+ * synchronous "get a genuine yes before this runs" behavior needed
616
+ * here, and simple enough to need no new UI component for this to be a
617
+ * real, functioning default rather than just plumbing with nothing on
618
+ * the other end. A host app wanting a nicer in-widget modal can still
619
+ * build one — this function is the only thing that would need
620
+ * replacing to do that.
621
+ */
622
+ function confirmToolCall(tool) {
623
+ if (typeof window === "undefined" || typeof window.confirm !== "function")
624
+ return Promise.resolve(false);
625
+ const message = tool.description ? `${tool.name}: ${tool.description}\n\nAllow this action?` : `Allow "${tool.name}"?`;
626
+ return Promise.resolve(window.confirm(message));
627
+ }
628
+ /**
629
+ * Architecture Pillar 4 — the typed transport's own Planner call,
630
+ * mirroring resolvePlan's real network shape but reached over HTTP
631
+ * (planEndpoint's server-side handler — createPlanHandler in server.ts
632
+ * — is the only place that can hold the real LLM API key). Never
633
+ * throws: any network/parse failure degrades to the exact same single-
634
+ * task fallback plan resolvePlan itself falls back to on an LLM error,
635
+ * so a Planner hiccup never blocks the turn.
636
+ */
637
+ async function fetchPlan(goal, version = 1) {
638
+ try {
639
+ const res = await fetch(planEndpoint, {
640
+ method: "POST",
641
+ headers: { "content-type": "application/json" },
642
+ body: JSON.stringify({ goal, version }),
643
+ });
644
+ const data = await res.json();
645
+ if (data && typeof data === "object" && Array.isArray(data.tasks) && data.tasks.length > 0)
646
+ return data;
647
+ }
648
+ catch {
649
+ // Falls through to the same fallback plan shape below.
650
+ }
651
+ return { version, goal, facts: [], tasks: [{ id: "t1", description: goal, doneContract: "The stated goal has been achieved.", status: "in_progress" }] };
652
+ }
653
+ /** Same real-network shape as fetchPlan, for criticEndpoint's
654
+ * createCriticHandler — degrades to a safe "continue" verdict on any
655
+ * failure, same resilience discipline as resolveCritic itself. */
656
+ async function fetchCriticVerdict(task, goal, verb, observation) {
657
+ try {
658
+ const res = await fetch(criticEndpoint, {
659
+ method: "POST",
660
+ headers: { "content-type": "application/json" },
661
+ body: JSON.stringify({ task, goal, verb, observation: observation ?? null }),
662
+ });
663
+ const data = await res.json();
664
+ if (data && typeof data.verdict === "string")
665
+ return data;
666
+ }
667
+ catch {
668
+ // Falls through to the safe default below.
378
669
  }
670
+ return { verdict: "continue", reasoning: "Critic call failed — defaulting to continue rather than blocking the turn." };
671
+ }
672
+ /**
673
+ * Architecture Pillar 3 (Skill half) — the typed transport's own
674
+ * Formulator save, mirroring realtime-server.ts's own post-turn
675
+ * `compileSkill`+`saveSkill` call. Fire-and-forget (never awaited by
676
+ * the caller, never allowed to affect what the user sees) since saving
677
+ * a Skill is bookkeeping for a FUTURE turn, not part of answering this
678
+ * one — matches the Formulator's own "cheap, runs once per turn, never
679
+ * blocks anything" framing. Classifies the current live page for a
680
+ * best-effort pattern tag the same way resolveVerb's own per-request
681
+ * classification does server-side.
682
+ */
683
+ function saveSkillIfLearned(goal, learnedFacts) {
684
+ if (!skillsSaveEndpoint || learnedFacts.length === 0)
685
+ return;
686
+ const matches = (0, core_1.classifyUiPattern)((0, core_1.deriveStructureSignals)(liveRegistryRef.current.getSnapshot().elements));
687
+ void fetch(skillsSaveEndpoint, {
688
+ method: "POST",
689
+ headers: { "content-type": "application/json" },
690
+ body: JSON.stringify({ goal, learnedFacts, pattern: matches[0]?.pattern }),
691
+ }).catch(() => {
692
+ // A Skill that fails to save just means the next similar goal
693
+ // starts from scratch again, same as if nothing had been learned
694
+ // this turn — never worth surfacing as a user-visible error.
695
+ });
379
696
  }
380
- const MAX_LOOP_ITERATIONS = 6; // a hard cap, not a target — see runTypedAgentLoop
381
697
  /**
382
698
  * Drives the agent loop over the stateless HTTP path: ask the server,
383
699
  * and if it comes back with a continuing step (click/fill/read/
@@ -390,58 +706,196 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
390
706
  * still knows what it was actually asked. `historyRef` (the
391
707
  * conversation's real memory) is only ever committed once, at the end —
392
708
  * a turn that hits the cap mid-loop doesn't leave partial noise in it.
709
+ * The loop itself (the `for`/TERMINAL_VERBS/iteration-cap shape) lives
710
+ * in agent-loop.ts, shared with the realtime relay's own finalizeTurn —
711
+ * this function owns everything transport-specific: the actual fetch,
712
+ * the raw/untyped response handling a stateless HTTP call needs (unlike
713
+ * realtime's always-valid in-process resolveVerb call), and the real
714
+ * historyRef commit.
393
715
  */
394
716
  async function runTypedAgentLoop(q) {
395
- let loopHistory = historyRef.current;
396
717
  const webMcpTools = await (0, webmcp_client_1.discoverWebMcpTools)();
397
- for (let i = 0; i < MAX_LOOP_ITERATIONS; i++) {
398
- const liveScan = liveRegistryRef.current.getSnapshot();
399
- liveMapRef.current = liveScan.byId;
400
- const res = await fetch(endpoint, {
401
- method: "POST",
402
- headers: { "content-type": "application/json" },
403
- body: JSON.stringify({
404
- route: pathname,
405
- question: q,
406
- visible: (0, context_collector_1.collectVisible)(),
407
- history: loopHistory,
408
- liveElements: liveScan.elements,
409
- webMcpTools,
410
- }),
411
- });
412
- const data = await res.json().catch(() => null);
413
- const parsed = (0, core_1.safeParseVerbResponse)(data);
414
- if (!parsed || core_1.TERMINAL_VERBS.has(parsed.verb)) {
415
- handleVerb(data);
416
- // Unlike the realtime relay (one persistent connection, memory
417
- // lives server-side), each of these POSTs is stateless — the
418
- // widget itself is what remembers, and resends it above so the
419
- // model has context for "the first one" / "do that instead" on
420
- // the next question.
421
- historyRef.current = [
422
- ...loopHistory,
423
- { role: "user", text: q },
424
- { role: "assistant", text: summarizeVerbForHistory(data) },
425
- ].slice(-MAX_HISTORY_TURNS);
426
- return;
718
+ let lastRawResponse = null;
719
+ // Architecture Pillar 4 — real Planner/Critic wiring for the typed
720
+ // transport, opt-in via planEndpoint/criticEndpoint (see their own
721
+ // doc comments on CopilotProps) — closes the gap the plan file names
722
+ // directly ("the typed/HTTP path has zero Planner/Critic wiring at
723
+ // all... today explicitly realtime-only by deferral, not by
724
+ // decision"). Mirrors realtime-server.ts's finalizeTurn: an eager
725
+ // Planner kickoff when looksMultiStep(q) already flags a probable
726
+ // compound goal, a lazy fallback kickoff on the first continuing step
727
+ // otherwise, and a genuinely separate Critic pass over each
728
+ // continuing step's real result. Neither endpoint set (the default)
729
+ // means plannerEnabled is false and this whole block is a no-op —
730
+ // the typed loop behaves exactly as it always has.
731
+ let planPromise = null;
732
+ let plan = null;
733
+ let progress = null;
734
+ const STALL_THRESHOLD = 3; // same bounded budget realtime's own Critic wiring uses
735
+ const plannerEnabled = Boolean(planEndpoint && criticEndpoint);
736
+ if (plannerEnabled && (0, agent_loop_1.looksMultiStep)(q))
737
+ planPromise = fetchPlan(q);
738
+ // Architecture Pillar 3 (Skill half) every real, Critic-verified
739
+ // learnedFact from this turn's steps; saved once the turn concludes,
740
+ // below (saveSkillIfLearned). Empty is the common case, not a gap.
741
+ const learnedFacts = [];
742
+ const result = await (0, agent_loop_1.driveAgentLoop)(historyRef.current, {
743
+ async getNextStep(loopHistory) {
744
+ const liveScan = liveRegistryRef.current.getSnapshot();
745
+ liveMapRef.current = liveScan.byId;
746
+ const res = await fetch(endpoint, {
747
+ method: "POST",
748
+ headers: { "content-type": "application/json" },
749
+ body: JSON.stringify({
750
+ // pathnameRef, not the closed-over `pathname` — a navigate
751
+ // step (now possibly continuing, see isTerminalVerb) can
752
+ // change the real route mid-loop; this whole async function's
753
+ // own `pathname` closure was captured once, at the render
754
+ // that started this turn, and never updates again on its own.
755
+ route: pathnameRef.current,
756
+ question: q,
757
+ visible: (0, context_collector_1.collectVisible)(),
758
+ history: loopHistory,
759
+ liveElements: liveScan.elements,
760
+ webMcpTools,
761
+ scopeId,
762
+ }),
763
+ });
764
+ const data = await res.json().catch(() => null);
765
+ lastRawResponse = data;
766
+ return (0, core_1.safeParseVerbResponse)(data);
767
+ },
768
+ onStep({ verb, terminal }) {
769
+ // A continuing step — show it happening (execution itself
770
+ // happens in executeStep below). Terminal steps are handled once
771
+ // driveAgentLoop returns, via handleVerb — unchanged from before.
772
+ // Same stale-typed-reply guard as the terminal case below — a
773
+ // multi-step typed loop can still be mid-flight when a realtime
774
+ // call starts.
775
+ if (!terminal && !typedPlaybackSuspendedRef.current) {
776
+ setAnswer(summarizeVerbForHistory(verb));
777
+ setLoopWorking(true);
778
+ }
779
+ // The lazy fallback — only fires when looksMultiStep missed
780
+ // (planPromise is still null): a real Plan is still guaranteed
781
+ // before the Critic needs one, just one round trip later.
782
+ if (!terminal && plannerEnabled && !planPromise)
783
+ planPromise = fetchPlan(q);
784
+ return false;
785
+ },
786
+ // Same real, live-found fix as the realtime WS "verb" handler's own
787
+ // executeToolStep call — a fresh scan per step, not the turn's
788
+ // frozen liveMapRef, so a step that reveals new DOM (a click that
789
+ // opens a modal) doesn't leave the NEXT step unable to find
790
+ // anything in it.
791
+ executeStep: (verb) => (0, verb_executor_1.executeToolStep)(verb, pathnameRef.current, liveRegistryRef.current.getSnapshot().byId, (route) => router.push(route), confirmToolCall).then((r) => r?.observation),
792
+ runCritic: plannerEnabled
793
+ ? async ({ verb, observation }) => {
794
+ // Real state, not the Executor's self-report — see
795
+ // resolveCritic's own doc comment (server.ts) for why this is
796
+ // a genuinely separate pass, same precedent realtime already
797
+ // established.
798
+ if (!plan) {
799
+ plan = planPromise ? await planPromise : await fetchPlan(q);
800
+ progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
801
+ }
802
+ const currentProgress = progress;
803
+ const currentTask = plan.tasks[currentProgress.currentTaskIndex];
804
+ const verdict = await fetchCriticVerdict(currentTask, q, verb, observation);
805
+ if (verdict.learnedFact)
806
+ learnedFacts.push(verdict.learnedFact);
807
+ if (verdict.verdict === "task_complete") {
808
+ currentTask.status = "done";
809
+ if (currentProgress.currentTaskIndex < plan.tasks.length - 1) {
810
+ // More tasks remain — advance and keep looping instead of
811
+ // ending the turn here.
812
+ currentProgress.currentTaskIndex++;
813
+ plan.tasks[currentProgress.currentTaskIndex].status = "in_progress";
814
+ currentProgress.stallCount = 0;
815
+ return { ...verdict, verdict: "continue" };
816
+ }
817
+ // The last task is genuinely done — end the loop right here
818
+ // instead of asking the model again and hoping it notices.
819
+ return verdict;
820
+ }
821
+ if (verdict.verdict === "replan") {
822
+ plan = await fetchPlan(q, plan.version + 1);
823
+ progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
824
+ return { ...verdict, verdict: "continue" };
825
+ }
826
+ if (verdict.verdict === "give_up")
827
+ return verdict;
828
+ // "continue" — a harness-enforced fail-safe on top of the
829
+ // Critic's own judgment, same Magentic-One-shaped two-tier
830
+ // tolerance realtime already uses.
831
+ currentProgress.stallCount++;
832
+ if (currentProgress.stallCount >= STALL_THRESHOLD) {
833
+ return {
834
+ verdict: "give_up",
835
+ reasoning: `Stuck after ${currentProgress.stallCount} steps with no confirmed progress on "${currentTask.description}" — ${verdict.reasoning}`,
836
+ };
837
+ }
838
+ return verdict;
839
+ }
840
+ : undefined,
841
+ });
842
+ // Architecture Pillar 3 (Skill half) — the Formulator, once per turn.
843
+ saveSkillIfLearned(q, learnedFacts);
844
+ if (result.outcome === "terminal" || result.outcome === "unparseable" || result.outcome === "critic-complete") {
845
+ // A realtime call can start WHILE this whole typed loop (potentially
846
+ // several real fetches deep) was still in flight — applying this
847
+ // reply now would overwrite the live call's own answer with a
848
+ // stale, orphaned bubble that doesn't correspond to anything the
849
+ // realtime conversation actually said. speak()/speakAndWait() guard
850
+ // the AUDIO half of this same real, live-found race (see
851
+ // typedPlaybackSuspendedRef's own doc comment) — this is the
852
+ // matching guard for the TEXT half, which would otherwise still
853
+ // leak through even with the audio silenced.
854
+ // The Critic independently confirmed the last task's doneContract
855
+ // is satisfied even though the model's own verb never got there —
856
+ // real fix for the diagnosed bug (a batch succeeded and the model
857
+ // kept looping instead of recognizing it). No raw server response
858
+ // exists for this synthesized verb (it never came from `endpoint`
859
+ // at all), so it's built directly from the verdict's own reasoning
860
+ // — same shape realtime-server.ts synthesizes for the same outcome.
861
+ const raw = result.outcome === "critic-complete" ? { verb: "explain", text: result.verdict.reasoning } : lastRawResponse;
862
+ if (typedPlaybackSuspendedRef.current) {
863
+ rtLog("dropping stale typed reply's text — a realtime call started while it was still in flight");
427
864
  }
428
- // A continuing step — show it happening, execute it for real, and
429
- // go around again with the real result instead of ending the turn.
430
- setAnswer(summarizeVerbForHistory(data));
431
- const stepResult = await (0, verb_executor_1.executeToolStep)(data, pathname, liveMapRef.current);
432
- loopHistory = [
433
- ...loopHistory,
434
- {
435
- role: "assistant",
436
- text: `${summarizeVerbForHistory(data)}. Result: ${stepResult?.observation ?? "no result"}`,
437
- },
865
+ else {
866
+ handleVerb(raw);
867
+ }
868
+ // Unlike the realtime relay (one persistent connection, memory
869
+ // lives server-side), each of these POSTs is stateless — the
870
+ // widget itself is what remembers, and resends it above so the
871
+ // model has context for "the first one" / "do that instead" on
872
+ // the next question. Summarized from the RAW response (not
873
+ // driveAgentLoop's typed finalVerb) via this file's own untyped
874
+ // summarizeVerbForHistory — deliberately, since a response that
875
+ // failed schema validation (outcome "unparseable") can still carry
876
+ // real, usable fields (e.g. a stray extra property tripped
877
+ // .strict() while `text` itself was fine) that only the untyped,
878
+ // duck-typed summarizer sees; there is no typed finalVerb at all
879
+ // for that outcome.
880
+ historyRef.current = [
881
+ ...result.workingHistory,
882
+ { role: "user", text: q },
883
+ { role: "assistant", text: summarizeVerbForHistory(raw) },
438
884
  ].slice(-MAX_HISTORY_TURNS);
885
+ return;
439
886
  }
440
- setAnswer("I wasn't able to finish that try asking again or breaking it into smaller steps.");
887
+ // "gave-up" (iteration cap hit with no terminal verb) OR "critic-give-up"
888
+ // (the Critic/stall fail-safe decided continuing wouldn't help — its
889
+ // own reasoning is a genuinely better message than the generic
890
+ // fallback, same as realtime-server.ts's own finalizeTurn).
891
+ const giveUpText = result.outcome === "critic-give-up" ? result.verdict.reasoning : "I wasn't able to finish that — try asking again or breaking it into smaller steps.";
892
+ const gaveUpSummary = result.outcome === "critic-give-up" ? giveUpText : "(gave up after too many steps)";
893
+ setLoopWorking(false);
894
+ setAnswer(giveUpText);
441
895
  historyRef.current = [
442
- ...loopHistory,
896
+ ...result.workingHistory,
443
897
  { role: "user", text: q },
444
- { role: "assistant", text: "(gave up after too many steps)" },
898
+ { role: "assistant", text: gaveUpSummary },
445
899
  ].slice(-MAX_HISTORY_TURNS);
446
900
  }
447
901
  function ensureTypedPlaybackGraph() {
@@ -456,8 +910,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
456
910
  }
457
911
  /** Stops whatever's currently playing on the typed/mic path's playback
458
912
  * graph, so two responses (e.g. a rapid double-click, or two answers
459
- * resolved close together) can never be heard overlapping. */
913
+ * resolved close together) can never be heard overlapping. Also bumps
914
+ * typedPlaybackGenerationRef — see its own doc comment for why that's
915
+ * required for this to actually hold when a NEW reply's audio is still
916
+ * arriving as a stream, not just already fully scheduled. */
460
917
  function stopTypedPlayback() {
918
+ typedPlaybackGenerationRef.current++;
461
919
  for (const source of typedScheduledSourcesRef.current) {
462
920
  source.onended = null;
463
921
  try {
@@ -486,6 +944,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
486
944
  */
487
945
  function playPcmStream(stream) {
488
946
  stopTypedPlayback();
947
+ const myGeneration = typedPlaybackGenerationRef.current;
489
948
  const { ctx, gain } = ensureTypedPlaybackGraph();
490
949
  void ctx.resume().catch(() => { });
491
950
  return new Promise((resolve) => {
@@ -521,7 +980,16 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
521
980
  const reader = stream.getReader();
522
981
  try {
523
982
  for (;;) {
983
+ // A newer call already ran stopTypedPlayback() (bumping the
984
+ // generation) while we were mid-read — stop here instead of
985
+ // scheduling more chunks behind its back. Checked both before
986
+ // AND after the await: a supersede can land at any point while
987
+ // this loop is blocked waiting on the next chunk.
988
+ if (typedPlaybackGenerationRef.current !== myGeneration)
989
+ break;
524
990
  const { done, value } = await reader.read();
991
+ if (typedPlaybackGenerationRef.current !== myGeneration)
992
+ break;
525
993
  if (done)
526
994
  break;
527
995
  if (!value || value.length === 0)
@@ -556,6 +1024,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
556
1024
  });
557
1025
  if (!res.ok || !res.body)
558
1026
  return;
1027
+ // A realtime call can start WHILE this fetch was in flight — see
1028
+ // typedPlaybackSuspendedRef's own doc comment for why that's a real,
1029
+ // live-found overlapping-audio case, not a hypothetical one.
1030
+ if (typedPlaybackSuspendedRef.current) {
1031
+ rtLog("dropping stale typed reply's audio — a realtime call started while it was still being fetched");
1032
+ return;
1033
+ }
559
1034
  void playPcmStream(res.body);
560
1035
  }
561
1036
  catch {
@@ -576,6 +1051,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
576
1051
  });
577
1052
  if (!res.ok || !res.body)
578
1053
  return;
1054
+ // See speak()'s own identical check and typedPlaybackSuspendedRef's
1055
+ // doc comment — a realtime call can start while this fetch was in
1056
+ // flight, same real risk here.
1057
+ if (typedPlaybackSuspendedRef.current) {
1058
+ rtLog("dropping stale typed reply's audio — a realtime call started while it was still being fetched");
1059
+ return;
1060
+ }
579
1061
  await playPcmStream(res.body);
580
1062
  }
581
1063
  catch {
@@ -612,7 +1094,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
612
1094
  // forever with the mic never resuming — move on instead.
613
1095
  setTimeout(() => {
614
1096
  if (!settled)
615
- console.warn("[cairn] tour step audio confirmation timed out — continuing");
1097
+ rtLog("tour step audio confirmation timed out after 15s — continuing anyway");
616
1098
  finish();
617
1099
  }, 15000);
618
1100
  ws.send(JSON.stringify({ type: "speak", text }));
@@ -685,7 +1167,24 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
685
1167
  // which is exactly what "hearing the agent twice, in parallel" was.
686
1168
  if (!realtimeUrl || !micSupported || realtimeActive || rtStartingRef.current)
687
1169
  return;
1170
+ rtLog("starting realtime call", { url: realtimeUrl });
688
1171
  rtStartingRef.current = true;
1172
+ // A typed/mic-recorded reply's audio can still be mid-playback on its
1173
+ // own separate graph (typedPlaybackGainRef, only ever touched by
1174
+ // stopTypedPlayback/playPcmStream) when the user switches straight into
1175
+ // a live call — endRealtime() already stops it on the way OUT of a
1176
+ // call, but nothing stopped it on the way IN, so it kept playing
1177
+ // completely unaffected by the realtime session's own mute-speaker
1178
+ // button (which only ever touches rtPlaybackGainRef) or by barge-in —
1179
+ // a real, live-found "two independent speakers" bug: muting or saying
1180
+ // "stop" only ever reached the realtime pipeline, while this leftover
1181
+ // typed audio played on regardless until it finished on its own.
1182
+ stopTypedPlayback();
1183
+ // Also blocks any typed reply that's still mid-fetch RIGHT NOW (not
1184
+ // yet playing anything, so stopTypedPlayback() above has nothing to
1185
+ // stop) from playing its audio once it finally arrives, seconds from
1186
+ // now — see typedPlaybackSuspendedRef's own doc comment.
1187
+ typedPlaybackSuspendedRef.current = true;
689
1188
  archiveCurrentExchange(); // preserve whatever typed/mic exchange preceded switching into a live call
690
1189
  setAnswer(null);
691
1190
  setCaption("");
@@ -717,6 +1216,21 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
717
1216
  const processor = audioCtx.createScriptProcessor(4096, 1, 1);
718
1217
  const silence = audioCtx.createGain();
719
1218
  silence.gain.value = 0;
1219
+ const bargeInVad = (0, vad_1.createVadDetector)();
1220
+ // Real, live-reported bug this closes: firing triggerBargeIn() off a
1221
+ // SINGLE ~85-100ms VAD frame meant one cough or door-slam frame that
1222
+ // happened to pass the energy+ZCR gate cut the agent off, permanently
1223
+ // (no server-side "was this real" recovery exists anymore — see
1224
+ // vad.ts's own doc comment for why that was removed instead of kept).
1225
+ // Real research into how production voice-agent platforms solve this
1226
+ // (Pipecat, LiveKit Agents, Vapi, Deepgram's Voice Agent API — see
1227
+ // DEVELOPMENT.md) converges on gating the LOCAL trigger on SUSTAINED
1228
+ // speech across a minimum duration instead — Pipecat's own production
1229
+ // spec cites 250ms, Vapi's stopSpeakingPlan defaults to 0.2s. This
1230
+ // gate does exactly that, entirely client-side (no network round trip
1231
+ // or STT-transcript timing involved, so it can't reintroduce the
1232
+ // removed server-side race).
1233
+ const bargeInGate = (0, vad_1.createBargeInGate)();
720
1234
  processor.onaudioprocess = (e) => {
721
1235
  if (ws.readyState !== WebSocket.OPEN)
722
1236
  return;
@@ -725,31 +1239,77 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
725
1239
  // Barge-in: while the agent is speaking a real conversational reply,
726
1240
  // still thinking about one, OR mid-tour, keep listening to the mic
727
1241
  // locally even though it isn't being sent yet, and cut the agent
728
- // off the instant the user starts talking again instead of making
729
- // them wait — including during a guided tour, which now cancels the
730
- // rest of the walkthrough on interruption (see triggerBargeIn)
731
- // instead of being talked-over-proof by design, the way a real
732
- // person giving a tour stops when you have a question. The
733
- // "rt-thinking" half matters just as much as "rt-speaking": an LLM
734
- // turn can easily take a couple of seconds with nothing playing
735
- // yet, and without this the mic was completely deaf during that
736
- // whole window — found live as "not listening while speaking... no
737
- // interrupting system", not just a missed nice-to-have.
1242
+ // off once the user has been sustainedly talking again (bargeInGate,
1243
+ // above) instead of making them wait — including during a guided
1244
+ // tour, which now cancels the rest of the walkthrough on
1245
+ // interruption (see triggerBargeIn) instead of being talked-over-
1246
+ // proof by design, the way a real person giving a tour stops when
1247
+ // you have a question. The "rt-thinking" half matters just as much
1248
+ // as "rt-speaking": an LLM turn can easily take a couple of seconds
1249
+ // with nothing playing yet, and without this the mic was completely
1250
+ // deaf during that whole window — found live as "not listening
1251
+ // while speaking... no interrupting system", not just a missed
1252
+ // nice-to-have.
738
1253
  if (rtStateRef.current === "rt-speaking" || rtStateRef.current === "rt-thinking") {
739
- const rms = computeRms(e.inputBuffer.getChannelData(0));
740
- if (rms > BARGE_IN_RMS_THRESHOLD)
1254
+ const frame = bargeInVad.process(e.inputBuffer.getChannelData(0));
1255
+ const frameDurationMs = (e.inputBuffer.length / audioCtx.sampleRate) * 1000;
1256
+ if (bargeInGate.update(frame, frameDurationMs))
741
1257
  triggerBargeIn();
742
1258
  return;
743
1259
  }
1260
+ bargeInGate.reset(); // not currently interruptible — don't let stale progress from a moment ago carry into the next speaking/thinking phase
744
1261
  if (rtStateRef.current !== "rt-listening")
745
1262
  return; // don't send our own mic while the agent is thinking/speaking
1263
+ if (!micAudioSentSinceListeningRef.current) {
1264
+ micAudioSentSinceListeningRef.current = true;
1265
+ rtLog("mic audio actually being sent (send gate is open)");
1266
+ }
746
1267
  const pcm = floatTo16BitPCM(downsampleTo16k(e.inputBuffer.getChannelData(0), audioCtx.sampleRate));
747
1268
  ws.send(pcm);
748
1269
  };
749
1270
  source.connect(processor);
750
1271
  processor.connect(silence);
751
1272
  silence.connect(audioCtx.destination);
1273
+ // Real, live-reported bug this closes: "status says Listening but
1274
+ // nothing gets picked up" — traced to browsers deliberately
1275
+ // suspending an AudioContext that has no active OUTPUT (a real,
1276
+ // documented power-saving policy, not backgrounded-tab-only). This
1277
+ // capture context has no real output at all by design (silence's
1278
+ // gain is 0), making it exactly the shape most likely to get
1279
+ // silently suspended — and once suspended, onaudioprocess simply
1280
+ // stops firing, so nothing inside it can detect or recover from its
1281
+ // own silence. A periodic external health check is the only
1282
+ // reliable way to catch this: resume the context if the browser
1283
+ // suspended it, and — a real, separate failure mode — detect the
1284
+ // mic's OWN MediaStreamTrack actually ending or going muted (device
1285
+ // unplugged, OS-level permission revoked mid-call, another app
1286
+ // taking exclusive access) and surface a real, honest error instead
1287
+ // of silently going deaf with the UI still claiming to listen.
1288
+ const micHealthCheck = setInterval(() => {
1289
+ if (audioCtx.state !== "running") {
1290
+ rtLog("capture AudioContext was suspended — resuming", { state: audioCtx.state });
1291
+ void audioCtx.resume().catch((err) => rtLog("failed to resume capture AudioContext", { error: String(err) }));
1292
+ }
1293
+ const track = stream.getAudioTracks()[0];
1294
+ if (track && (track.readyState === "ended" || track.muted)) {
1295
+ rtLog("mic track is no longer live — ending the call", { readyState: track.readyState, muted: track.muted });
1296
+ setAnswer("The microphone connection was lost — try starting the call again.");
1297
+ endRealtime();
1298
+ }
1299
+ }, 2000);
1300
+ // The track's own "ended" event is the immediate signal (fires the
1301
+ // instant the OS/browser actually kills the track) — the poll above
1302
+ // is the safety net for anything that doesn't fire it reliably
1303
+ // (muted-without-ended has no dedicated event in the spec).
1304
+ const handleMicTrackEnded = () => {
1305
+ rtLog("mic track ended unexpectedly — ending the call");
1306
+ setAnswer("The microphone connection was lost — try starting the call again.");
1307
+ endRealtime();
1308
+ };
1309
+ stream.getAudioTracks().forEach((t) => t.addEventListener("ended", handleMicTrackEnded));
752
1310
  rtCleanupRef.current = () => {
1311
+ clearInterval(micHealthCheck);
1312
+ stream.getAudioTracks().forEach((t) => t.removeEventListener("ended", handleMicTrackEnded));
753
1313
  processor.disconnect();
754
1314
  source.disconnect();
755
1315
  stream.getTracks().forEach((t) => t.stop());
@@ -780,6 +1340,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
780
1340
  rtTourAudioDoneRef.current = null;
781
1341
  return;
782
1342
  }
1343
+ rtLog("resumed listening");
1344
+ void audioCtx.resume().catch(() => { }); // don't wait up to 2s for the periodic health check if the browser already suspended capture
1345
+ micAudioSentSinceListeningRef.current = false;
783
1346
  setRtStatus("rt-listening");
784
1347
  setCaption("");
785
1348
  void sendFreshContext(); // refresh before the user starts talking again, not after
@@ -794,10 +1357,28 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
794
1357
  disarmThinkingWatchdog();
795
1358
  rtThinkingWatchdogRef.current = setTimeout(() => {
796
1359
  rtThinkingWatchdogRef.current = null;
797
- console.warn("[cairn] realtime turn timed out waiting on the server resuming listening");
798
- rtAudioDoneArrivingRef.current = true;
799
- setRtStatus("rt-listening");
800
- setCaption("");
1360
+ rtLog("thinking watchdog fired server took over 20s, resuming listening and abandoning that turn");
1361
+ // Real, live-found gap: this used to only reset LOCAL state,
1362
+ // never telling the server anything — so a turn that was simply
1363
+ // SLOW (not actually stuck; e.g. retrying a rate-limited call
1364
+ // across every configured key, which can genuinely take longer
1365
+ // than this 20s watchdog) kept running server-side, and its
1366
+ // reply arrived LATE, after the user had already moved on and
1367
+ // started a new turn locally — landing on whatever was now
1368
+ // showing instead of being recognized as stale. triggerBargeIn()
1369
+ // is exactly the fix: it sends the same real barge_in signal a
1370
+ // genuine interruption does, bumping the server's own generation
1371
+ // so that late reply — whenever it finally arrives — carries an
1372
+ // old generation number and gets correctly dropped by the
1373
+ // isStaleRtMessage check above instead of confusingly resuming.
1374
+ triggerBargeIn();
1375
+ setLoopWorking(false);
1376
+ // triggerBargeIn() clears the caption but never touched `answer`
1377
+ // — without this, a timed-out turn gave the user literally
1378
+ // nothing: no reply, no error, just a silent reset back to
1379
+ // "Listening…" that reads as "it heard me and did nothing." A
1380
+ // real, live-found gap, not just a console.warn nobody sees.
1381
+ setAnswer("That's taking longer than expected — try asking again.");
801
1382
  }, 20000);
802
1383
  }
803
1384
  function stopScheduledRtAudio() {
@@ -818,6 +1399,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
818
1399
  // now-stale audio_chunk/speaking_end that was already in flight, so a
819
1400
  // few straggling chunks can't sneak back in and resume playback.
820
1401
  function triggerBargeIn() {
1402
+ rtLog("barge-in triggered", { wasTouring: touringRef.current, discardedAudioChunks: rtScheduledSourcesRef.current.length });
821
1403
  disarmThinkingWatchdog();
822
1404
  stopScheduledRtAudio();
823
1405
  rtAudioDoneArrivingRef.current = true;
@@ -837,13 +1419,47 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
837
1419
  if (ws.readyState === WebSocket.OPEN)
838
1420
  ws.send(JSON.stringify({ type: "barge_in" }));
839
1421
  setRtStatus("rt-listening");
840
- setCaption("");
1422
+ // Real, live-found bug this fix removes: this used to also clear
1423
+ // the caption here (setCaption("")). That's correct-looking for a
1424
+ // REAL, VAD-triggered barge-in (the user's about to say something
1425
+ // new) — the very next "final" already does archiveCurrentExchange()
1426
+ // then overwrites caption with the new utterance, so clearing it
1427
+ // here was always redundant for that path. But this function is
1428
+ // ALSO called by the thinking watchdog on a timeout, where there is
1429
+ // no new utterance coming — clearing the caption there wiped out
1430
+ // the very question that just timed out, an instant before
1431
+ // setAnswer(the timeout message) ran, leaving the live pair as
1432
+ // {caption: "", answer: "That's taking longer..."} — a reply
1433
+ // visibly floating with no question above it, and nothing for
1434
+ // archiveCurrentExchange() to pair it with on the next turn either
1435
+ // (archiveText skips empty text). Simply never clearing it here is
1436
+ // correct for both callers: the barge-in path already gets a fresh
1437
+ // caption from the next "final", and the watchdog path now keeps
1438
+ // the timed-out question correctly paired with its own answer.
841
1439
  }
842
1440
  ws.onopen = () => {
1441
+ rtLog("connection open");
843
1442
  sendFreshContext();
844
1443
  setRtStatus("rt-listening");
845
1444
  rtStartingRef.current = false;
846
1445
  };
1446
+ // True for a verb/speaking_start/audio_chunk/speaking_end/
1447
+ // turn_complete message that belongs to an EARLIER turn than the
1448
+ // most recent "final" this client has seen — see
1449
+ // rtLastFinalGenerationRef's own doc comment for the real race this
1450
+ // closes. A message with no generation field at all (shouldn't
1451
+ // happen against a server running this fix, but a mismatched client/
1452
+ // server version pair during a rolling deploy could) is treated as
1453
+ // current rather than dropped — additive/backward-compatible, same
1454
+ // discipline every other wire-protocol addition in this codebase
1455
+ // follows.
1456
+ function isStaleRtMessage(msg) {
1457
+ const stale = typeof msg.generation === "number" && msg.generation < rtLastFinalGenerationRef.current;
1458
+ if (stale)
1459
+ rtLog("dropped stale message", { type: msg.type, messageGeneration: msg.generation, currentGeneration: rtLastFinalGenerationRef.current });
1460
+ return stale;
1461
+ }
1462
+ let audioChunkCount = 0;
847
1463
  ws.onmessage = (event) => {
848
1464
  if (typeof event.data !== "string")
849
1465
  return; // audio now arrives as base64 inside audio_chunk, not raw binary frames
@@ -852,14 +1468,35 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
852
1468
  setCaption(msg.text);
853
1469
  }
854
1470
  else if (msg.type === "final") {
1471
+ rtLog("final transcript", { text: msg.text, generation: msg.generation });
1472
+ rtLastFinalGenerationRef.current = typeof msg.generation === "number" ? msg.generation : 0;
855
1473
  archiveCurrentExchange(); // the previous turn's pair is complete — move it into history before this one starts overwriting caption/answer
856
1474
  setCaption(msg.text);
1475
+ // Without this, `answer` still held the PREVIOUS turn's reply
1476
+ // text at the moment this new turn's own reply — if it ever
1477
+ // arrives — would overwrite it. Usually invisible (the previous
1478
+ // reply lands well before the next "final"), but a barge-in can
1479
+ // supersede an in-flight turn before its reply ever arrives (see
1480
+ // realtime-server.ts's onStep generation check) — with no reply
1481
+ // ever coming for THIS caption, the stale previous-turn answer
1482
+ // sat there and got archived alongside the wrong question on the
1483
+ // NEXT final, showing as a mismatched or duplicated-looking
1484
+ // reply. Found live: two turns' worth of the same fallback error
1485
+ // text ("Something went wrong on my end") appearing back to back
1486
+ // with only one visible question between them. Clearing to null
1487
+ // here means an abandoned turn now correctly archives with NO
1488
+ // reply bubble (archiveText skips empty text) instead of someone
1489
+ // else's.
1490
+ setAnswer(null);
857
1491
  setRtStatus("rt-thinking");
858
1492
  armThinkingWatchdog();
859
1493
  }
860
1494
  else if (msg.type === "verb") {
1495
+ if (isStaleRtMessage(msg))
1496
+ return; // belongs to a turn a later "final" already superseded
1497
+ rtLog("verb received", { verb: msg.verb?.verb, generation: msg.generation });
861
1498
  const parsedStep = (0, core_1.safeParseVerbResponse)(msg.verb);
862
- if (parsedStep && !core_1.TERMINAL_VERBS.has(parsedStep.verb)) {
1499
+ if (parsedStep && !(0, core_1.isTerminalVerb)(parsedStep)) {
863
1500
  // A continuing agent-loop step (click/fill/read/call_tool) —
864
1501
  // the turn isn't over: execute it for real and report the
865
1502
  // result back so the server can decide the next step, instead
@@ -870,7 +1507,39 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
870
1507
  // — the server's loop stays quiet between steps on purpose,
871
1508
  // to keep it fast.
872
1509
  setAnswer(summarizeVerbForHistory(msg.verb));
873
- void (0, verb_executor_1.executeToolStep)(msg.verb, pathnameRef.current, liveMapRef.current).then((result) => {
1510
+ setLoopWorking(true);
1511
+ // Real, live-found bug: armThinkingWatchdog() only ever fired
1512
+ // once, on the turn's own "final" message, giving the WHOLE
1513
+ // multi-step turn one shared 20s budget — Executor + Planner +
1514
+ // Critic for step 1, then the same again for step 2, and so
1515
+ // on. Directly measured live: one single non-terminal step's
1516
+ // own Executor+Planner+Critic chain alone took ~14s (11.5s +
1517
+ // 1.5s + 1.1s) — a real, multi-step goal needing two or three
1518
+ // such steps blows straight through 20s even though each
1519
+ // individual step is proof of genuine progress, not a stall.
1520
+ // Re-arming here — once per continuing step, not once per
1521
+ // turn — gives every step its own fresh budget, so the
1522
+ // watchdog only ever fires on a step that's ACTUALLY stuck
1523
+ // (no verb/final/speaking_start arriving at all), matching
1524
+ // what its own fallback message ("taking longer than
1525
+ // expected") is supposed to mean.
1526
+ armThinkingWatchdog();
1527
+ // A FRESH scan, not the turn's starting liveMapRef snapshot —
1528
+ // real, live-found bug: a step in THIS SAME multi-step turn
1529
+ // (a "click New Agent" that opens a modal) can reveal DOM a
1530
+ // later step (a "fill" targeting the modal's own input) needs
1531
+ // to find, and liveMapRef is deliberately frozen once per
1532
+ // turn (see its own doc comment — that freeze exists to stop
1533
+ // a background rescan from shifting an id mid-flight during
1534
+ // ONE step's own round trip, not to survive across several
1535
+ // sequential steps that genuinely changed the page).
1536
+ // runTour() already does exactly this for its own steps, for
1537
+ // the identical reason. Without it, "click New Agent, then
1538
+ // type the name" reliably failed every time with "Could not
1539
+ // find that element on the page" — confirmed live, repeated
1540
+ // 5+ times in a row without ever recovering.
1541
+ const freshLiveMap = liveRegistryRef.current.getSnapshot().byId;
1542
+ void (0, verb_executor_1.executeToolStep)(msg.verb, pathnameRef.current, freshLiveMap, (route) => router.push(route), confirmToolCall).then((result) => {
874
1543
  if (ws.readyState === WebSocket.OPEN) {
875
1544
  ws.send(JSON.stringify({ type: "tool_result", observation: result?.observation ?? "no result" }));
876
1545
  }
@@ -881,11 +1550,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
881
1550
  handleVerb(msg.verb);
882
1551
  }
883
1552
  else if (msg.type === "speaking_start") {
1553
+ if (isStaleRtMessage(msg))
1554
+ return;
1555
+ rtLog("speaking start", { generation: msg.generation });
1556
+ audioChunkCount = 0;
884
1557
  disarmThinkingWatchdog();
885
1558
  rtAudioDoneArrivingRef.current = false;
886
1559
  setRtStatus("rt-speaking");
887
1560
  }
888
1561
  else if (msg.type === "audio_chunk") {
1562
+ if (isStaleRtMessage(msg))
1563
+ return; // the literal "two speakers" case — a chunk from an abandoned turn, already in flight when the barge-in landed
1564
+ audioChunkCount++;
889
1565
  const ctx = rtPlaybackCtxRef.current;
890
1566
  const gain = rtPlaybackGainRef.current;
891
1567
  if (!ctx || !gain)
@@ -918,6 +1594,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
918
1594
  };
919
1595
  }
920
1596
  else if (msg.type === "speaking_end" || msg.type === "turn_complete") {
1597
+ if (isStaleRtMessage(msg))
1598
+ return; // a newer turn's own speaking_end/turn_complete will arrive and resume listening correctly on its own
1599
+ rtLog(msg.type, { audioChunks: audioChunkCount, generation: msg.generation });
921
1600
  // turn_complete covers a verb with nothing spoken (a plain
922
1601
  // highlight/navigate/do often has no text) — no audio_chunk ever
923
1602
  // arrives for it, so rtScheduledSourcesRef is already empty and
@@ -927,10 +1606,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
927
1606
  maybeResumeListening();
928
1607
  }
929
1608
  else if (msg.type === "error") {
1609
+ rtLog("server error", { message: msg.message });
930
1610
  // Must actually unstick the turn, not just show the message —
931
1611
  // otherwise the mic never resumes and the session is stuck
932
1612
  // exactly the way a silently-dropped response used to leave it.
933
1613
  disarmThinkingWatchdog();
1614
+ setLoopWorking(false);
934
1615
  setAnswer(msg.message ?? "Something went wrong.");
935
1616
  if (touringRef.current) {
936
1617
  // A tour step's own speakStreamed() failed server-side (see
@@ -950,10 +1631,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
950
1631
  }
951
1632
  };
952
1633
  ws.onerror = () => {
1634
+ rtLog("connection error");
953
1635
  setAnswer("Couldn't connect to the realtime voice service.");
954
1636
  endRealtime();
955
1637
  };
956
- ws.onclose = () => {
1638
+ ws.onclose = (closeEvent) => {
1639
+ rtLog("connection closed", { code: closeEvent.code, reason: closeEvent.reason, wasIdle: rtStateRef.current === "idle" });
957
1640
  if (rtStateRef.current !== "idle")
958
1641
  endRealtime();
959
1642
  };
@@ -962,15 +1645,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
962
1645
  setAnswer("Couldn't access the microphone — check your browser's permission for this site.");
963
1646
  setRtStatus("idle");
964
1647
  rtStartingRef.current = false;
1648
+ typedPlaybackSuspendedRef.current = false; // the call never actually started — don't leave typed replies permanently silenced
965
1649
  }
966
1650
  }
967
1651
  function endRealtime() {
1652
+ rtLog("ending realtime call", { statusAtEnd: rtStateRef.current });
968
1653
  if (rtThinkingWatchdogRef.current) {
969
1654
  clearTimeout(rtThinkingWatchdogRef.current);
970
1655
  rtThinkingWatchdogRef.current = null;
971
1656
  }
972
1657
  rtStartingRef.current = false;
973
1658
  stopTypedPlayback();
1659
+ typedPlaybackSuspendedRef.current = false; // typed replies work normally again once no live call can race them
974
1660
  rtSocketRef.current?.close();
975
1661
  rtSocketRef.current = null;
976
1662
  rtCleanupRef.current?.();
@@ -979,6 +1665,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
979
1665
  setRtSpeakerMuted(false);
980
1666
  setCaption("");
981
1667
  setRtStatus("idle");
1668
+ setLoopWorking(false); // defensive — a connection dropping mid-loop must never leave the "still working" indicator stuck on
982
1669
  tourGenerationRef.current++; // cancel an in-progress tour rather than leaving it stuck waiting to resume rt-listening
983
1670
  touringRef.current = false;
984
1671
  setTourStep(null);
@@ -1013,7 +1700,17 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
1013
1700
  "rt-speaking": "Speaking…",
1014
1701
  };
1015
1702
  return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.length > 0 && ((0, jsx_runtime_1.jsxs)("button", { type: "button", className: "cairn-history-toggle", onClick: () => setHistoryExpanded((v) => !v), "aria-expanded": historyExpanded, children: [historyExpanded ? (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronUp, { size: 12 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronDown, { size: 12 }), historyExpanded ? "Hide earlier" : `${transcript.length} earlier`] })), historyExpanded &&
1016
- transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: renderCaptionWords(answer) })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
1703
+ transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-bubble-text", children: [renderCaptionWords(answer), loopWorking && (
1704
+ // Real, live-reported gap this closes: this bubble
1705
+ // used to go static the instant a continuing step's
1706
+ // own progress text was shown ("Typing earbuds into
1707
+ // the search box"), with nothing telling the user
1708
+ // the agent was still actively working for however
1709
+ // long the next real LLM call took. Appended inline
1710
+ // (not swapped in place of the text, which the
1711
+ // no-answer-yet case below does) so the progress
1712
+ // text stays legible while still showing motion.
1713
+ (0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking cairn-thinking-inline", "aria-label": "Still working", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
1017
1714
  e.preventDefault();
1018
1715
  const trimmed = question.trim();
1019
1716
  if (trimmed)
@@ -1021,6 +1718,75 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
1021
1718
  }, children: (0, jsx_runtime_1.jsxs)("div", { className: "cairn-input-row", children: [(0, jsx_runtime_1.jsx)("input", { value: question, onChange: (e) => setQuestion(e.target.value), placeholder: "What do you need help with?", "aria-label": `Ask ${persona} a question`, disabled: recording || touring, autoFocus: true }), realtimeUrl && micSupported && ((0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn", "aria-label": "Start realtime conversation", onClick: () => void startRealtime(), disabled: busy || recording, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneCall, { size: 16 }) })), transcribeEndpoint && micSupported && ((0, jsx_runtime_1.jsx)("button", { type: "button", className: recording ? "cairn-icon-btn cairn-icon-btn-recording" : "cairn-icon-btn", "aria-label": recording ? "Stop recording" : "Ask by voice", onClick: () => (recording ? stopRecording() : void startRecording()), disabled: touring, children: recording ? (0, jsx_runtime_1.jsx)(lucide_react_1.Square, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) })), (0, jsx_runtime_1.jsx)("button", { type: "submit", className: "cairn-send", "aria-label": "Send", disabled: !question.trim() || busy || recording, children: asking ? (0, jsx_runtime_1.jsx)(lucide_react_1.Loader2, { size: 16, className: "cairn-spin" }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Send, { size: 16 }) })] }) }))] }))] }));
1022
1719
  }
1023
1720
  const MAX_HISTORY_TURNS = 8; // 4 exchanges — matches the same cap the realtime relay uses server-side
1721
+ exports.CONVERSATION_STORAGE_KEY = "cairn:conversation:v1";
1722
+ /**
1723
+ * Real, live-reported bug this closes: a host app's own mutation handler
1724
+ * calling a real `window.location.reload()` — a common, entirely valid
1725
+ * pattern; this SDK's own demo app uses it after several real actions,
1726
+ * e.g. moving a kanban card — tears down the ENTIRE React tree, this
1727
+ * widget included. Every bit of conversation state (the visible
1728
+ * transcript, the current exchange, even whether the panel was open)
1729
+ * reset to nothing, making a real, in-progress conversation look like it
1730
+ * had simply ended the instant a host page happened to reload — even
1731
+ * though nothing about the CONVERSATION itself was actually over.
1732
+ * `sessionStorage`, not `localStorage`, is deliberate: it survives
1733
+ * exactly a reload/navigation within the same tab — the real scope of
1734
+ * "this conversation" — and clears itself once the tab/window actually
1735
+ * closes, never lingering into an unrelated later visit the way
1736
+ * `localStorage` would.
1737
+ */
1738
+ function loadPersistedConversation() {
1739
+ if (typeof window === "undefined")
1740
+ return null;
1741
+ try {
1742
+ const raw = window.sessionStorage.getItem(exports.CONVERSATION_STORAGE_KEY);
1743
+ if (!raw)
1744
+ return null;
1745
+ const parsed = JSON.parse(raw);
1746
+ if (!parsed || typeof parsed !== "object")
1747
+ return null;
1748
+ const transcript = Array.isArray(parsed.transcript)
1749
+ ? parsed.transcript.filter((t) => !!t && typeof t === "object" && typeof t.id === "number" && typeof t.text === "string" && (t.role === "user" || t.role === "agent"))
1750
+ : [];
1751
+ return {
1752
+ transcript,
1753
+ lastQuestion: typeof parsed.lastQuestion === "string" ? parsed.lastQuestion : null,
1754
+ answer: typeof parsed.answer === "string" ? parsed.answer : null,
1755
+ open: Boolean(parsed.open),
1756
+ };
1757
+ }
1758
+ catch {
1759
+ return null; // private browsing, quota, or a genuinely corrupt value — never crash the widget over this
1760
+ }
1761
+ }
1762
+ function savePersistedConversation(data) {
1763
+ if (typeof window === "undefined")
1764
+ return;
1765
+ try {
1766
+ window.sessionStorage.setItem(exports.CONVERSATION_STORAGE_KEY, JSON.stringify(data));
1767
+ }
1768
+ catch {
1769
+ // Storage unavailable/full — the conversation just won't survive a reload this time, never worth crashing the widget over.
1770
+ }
1771
+ }
1772
+ /**
1773
+ * Rebuilds a real seed for `historyRef` (what gets sent to the model on
1774
+ * the NEXT typed turn) from a restored transcript — deliberately derived
1775
+ * from the same, already-persisted `transcript` rather than separately
1776
+ * persisting `historyRef`'s own shape: one real source of truth for "what
1777
+ * was actually said," not two that could quietly drift apart. Capped the
1778
+ * same way every other history array in this file already is.
1779
+ */
1780
+ function reconstructHistoryFromPersisted(persisted) {
1781
+ if (!persisted)
1782
+ return [];
1783
+ const fromTranscript = persisted.transcript.map((t) => ({ role: t.role === "agent" ? "assistant" : "user", text: t.text }));
1784
+ const live = [
1785
+ ...(persisted.lastQuestion ? [{ role: "user", text: persisted.lastQuestion }] : []),
1786
+ ...(persisted.answer ? [{ role: "assistant", text: persisted.answer }] : []),
1787
+ ];
1788
+ return [...fromTranscript, ...live].slice(-MAX_HISTORY_TURNS);
1789
+ }
1024
1790
  /** Best-effort text form of a raw (unvalidated) verb response for the
1025
1791
  * conversation-history log — not shown to the user, just fed back to the
1026
1792
  * model on later turns. Deliberately loose/defensive rather than a full
@@ -1051,6 +1817,12 @@ function summarizeVerbForHistory(raw) {
1051
1817
  return `(read ${String(v.target)})`;
1052
1818
  case "call_tool":
1053
1819
  return `(called ${String(v.name)})`;
1820
+ case "drag":
1821
+ return `(dragged ${String(v.target)} to ${String(v.to)})`;
1822
+ case "select":
1823
+ return `(selected "${String(v.value)}" in ${String(v.target)})`;
1824
+ case "key":
1825
+ return `(pressed ${String(v.key)}${v.target ? ` on ${String(v.target)}` : ""})`;
1054
1826
  case "batch":
1055
1827
  return Array.isArray(v.actions) ? `(${v.actions.length} steps: ${v.actions.map((a) => a.verb).join(", ")})` : "(batch)";
1056
1828
  default:
@@ -1070,25 +1842,32 @@ function renderCaptionWords(text) {
1070
1842
  const words = text.split(" ");
1071
1843
  return words.map((word, i) => ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-word", style: { animationDelay: `${Math.min(i * 55, 2800)}ms` }, children: [word, i < words.length - 1 ? " " : ""] }, i)));
1072
1844
  }
1845
+ // "Waybalance" — three real, irregular stones (an ellipse plus a smaller
1846
+ // bump, not a rectangle), stacked slightly off-center the way a hiker
1847
+ // actually balances a trail cairn, instead of the perfectly centered flat
1848
+ // bars this replaced. Same mark as docs/images/logo.svg and site/index.html's
1849
+ // nav badge, just currentColor here so it inherits the button's own color.
1073
1850
  function CairnMark() {
1074
- return ((0, jsx_runtime_1.jsxs)("svg", { width: "20", height: "20", viewBox: "0 0 20 20", fill: "none", "aria-hidden": "true", children: [(0, jsx_runtime_1.jsx)("rect", { x: "7", y: "12.5", width: "6", height: "2.6", rx: "0.5", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("rect", { x: "4.5", y: "8.5", width: "11", height: "2.6", rx: "0.5", fill: "currentColor", opacity: "0.75" }), (0, jsx_runtime_1.jsx)("rect", { x: "8.2", y: "4.5", width: "3.6", height: "2.6", rx: "0.5", fill: "currentColor", opacity: "0.5" })] }));
1851
+ return ((0, jsx_runtime_1.jsxs)("svg", { width: "20", height: "20", viewBox: "0 0 20 20", fill: "none", "aria-hidden": "true", children: [(0, jsx_runtime_1.jsx)("ellipse", { cx: "10", cy: "14.6", rx: "6.1", ry: "2.3", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "6.3", cy: "14.1", rx: "2.5", ry: "1.6", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "9.1", cy: "10.4", rx: "4.3", ry: "2", fill: "currentColor", opacity: "0.82" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "6.2", cy: "10", rx: "1.7", ry: "1.2", fill: "currentColor", opacity: "0.82" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "11.7", cy: "6.4", rx: "2.6", ry: "1.6", fill: "currentColor", opacity: "0.6" })] }));
1852
+ }
1853
+ // Real-time lifecycle logging — every message received, every decision made
1854
+ // about it (played, spoken, dropped, and why), every state transition. Added
1855
+ // specifically so a live session's actual behavior is visible in the browser
1856
+ // console instead of only inferable from symptoms after the fact — every bug
1857
+ // found and fixed in this file today was diagnosed from screenshots and
1858
+ // terminal output because nothing like this existed before. `[cairn rt]` is
1859
+ // the tag to filter on. Deliberately excludes per-audio_chunk noise (dozens
1860
+ // of chunks per turn would flood the console) — chunk activity shows up as
1861
+ // a one-line count at speaking_end/turn_complete instead.
1862
+ function rtLog(event, details) {
1863
+ if (details)
1864
+ console.log("[cairn rt]", event, details);
1865
+ else
1866
+ console.log("[cairn rt]", event);
1075
1867
  }
1076
1868
  // ---------------------------------------------------------------------------
1077
1869
  // Audio helpers (real-time PCM16 capture — standard Web Audio API patterns)
1078
1870
  // ---------------------------------------------------------------------------
1079
- // Heuristic energy gate for barge-in: real speech into a laptop/phone mic
1080
- // typically sits well above this; normal room noise and the mic's own
1081
- // noise floor typically sit below it. Not calibrated against real hardware
1082
- // in this environment (no live mic here) — reasonable starting point, may
1083
- // need tuning against a real device if it proves too trigger-happy or too
1084
- // insensitive in practice.
1085
- const BARGE_IN_RMS_THRESHOLD = 0.02;
1086
- function computeRms(channelData) {
1087
- let sumSquares = 0;
1088
- for (let i = 0; i < channelData.length; i++)
1089
- sumSquares += channelData[i] * channelData[i];
1090
- return Math.sqrt(sumSquares / channelData.length);
1091
- }
1092
1871
  function downsampleTo16k(input, inputSampleRate) {
1093
1872
  const targetRate = 16000;
1094
1873
  if (inputSampleRate === targetRate)
@@ -1130,9 +1909,13 @@ const COPILOT_STYLES = `
1130
1909
  0%, 100% { box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.4); }
1131
1910
  70% { box-shadow: 0 0 0 10px rgba(16, 185, 129, 0); }
1132
1911
  }
1133
- @keyframes cairn-pulse-indigo {
1134
- 0%, 100% { box-shadow: 0 0 0 0 rgba(99, 102, 241, 0.4); }
1135
- 70% { box-shadow: 0 0 0 10px rgba(99, 102, 241, 0); }
1912
+ @keyframes cairn-pulse-ember {
1913
+ 0%, 100% { box-shadow: 0 0 0 0 rgba(224, 122, 63, 0.4); }
1914
+ 70% { box-shadow: 0 0 0 10px rgba(224, 122, 63, 0); }
1915
+ }
1916
+ @keyframes cairn-cursor-arrive {
1917
+ 0% { box-shadow: 0 0 0 0 rgba(224, 122, 63, 0.55); }
1918
+ 100% { box-shadow: 0 0 0 9px rgba(224, 122, 63, 0); }
1136
1919
  }
1137
1920
  @keyframes cairn-spin {
1138
1921
  from { transform: rotate(0deg); }
@@ -1148,7 +1931,7 @@ const COPILOT_STYLES = `
1148
1931
  }
1149
1932
  @keyframes cairn-word-sweep {
1150
1933
  0% { opacity: 0.35; text-shadow: none; }
1151
- 35% { opacity: 1; color: #4f46e5; text-shadow: 0 0 10px rgba(99, 102, 241, 0.45); }
1934
+ 35% { opacity: 1; color: #E07A3F; text-shadow: 0 0 10px rgba(224, 122, 63, 0.45); }
1152
1935
  100% { opacity: 1; color: inherit; text-shadow: none; }
1153
1936
  }
1154
1937
  @keyframes cairn-thinking-bounce {
@@ -1156,16 +1939,19 @@ const COPILOT_STYLES = `
1156
1939
  40% { opacity: 0.9; transform: translateY(-3px); }
1157
1940
  }
1158
1941
  .cairn-glow {
1159
- animation: cairn-pulse-indigo 1.1s ease-out 2;
1160
- outline: 2px solid #6366f1;
1942
+ animation: cairn-pulse-ember 1.1s ease-out 2;
1943
+ outline: 2px solid #E07A3F;
1161
1944
  outline-offset: 3px;
1162
1945
  border-radius: 8px;
1163
1946
  }
1947
+ .cairn-cursor-hover {
1948
+ animation: cairn-cursor-arrive 0.3s ease-out;
1949
+ }
1164
1950
  .cairn-spin {
1165
1951
  animation: cairn-spin 0.8s linear infinite;
1166
1952
  }
1167
1953
  @media (prefers-reduced-motion: reduce) {
1168
- .cairn-fab, .cairn-panel, .cairn-bubble, .cairn-word, .cairn-thinking-dot {
1954
+ .cairn-fab, .cairn-panel, .cairn-bubble, .cairn-word, .cairn-thinking-dot, #cairn-cursor {
1169
1955
  animation: none !important;
1170
1956
  transition: none !important;
1171
1957
  }
@@ -1300,6 +2086,10 @@ const COPILOT_STYLES = `
1300
2086
  gap: 4px;
1301
2087
  padding: 2px 0;
1302
2088
  }
2089
+ .cairn-thinking-inline {
2090
+ margin-left: 6px;
2091
+ vertical-align: middle;
2092
+ }
1303
2093
  .cairn-thinking-dot {
1304
2094
  width: 5px;
1305
2095
  height: 5px;