@cairnvibe/sdk 0.2.13 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,18 +1,24 @@
1
1
  "use strict";
2
2
  "use client";
3
3
  Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.CONVERSATION_STORAGE_KEY = void 0;
4
5
  exports.Copilot = Copilot;
6
+ exports.loadPersistedConversation = loadPersistedConversation;
7
+ exports.savePersistedConversation = savePersistedConversation;
8
+ exports.reconstructHistoryFromPersisted = reconstructHistoryFromPersisted;
5
9
  const jsx_runtime_1 = require("react/jsx-runtime");
6
10
  const react_1 = require("react");
7
11
  const navigation_1 = require("next/navigation");
8
12
  const lucide_react_1 = require("lucide-react");
9
13
  const core_1 = require("@cairnvibe/core");
14
+ const agent_loop_1 = require("./agent-loop");
10
15
  const context_collector_1 = require("./context-collector");
11
16
  const element_ladder_1 = require("./element-ladder");
12
17
  const runtime_scan_1 = require("./runtime-scan");
13
18
  const webmcp_client_1 = require("./webmcp-client");
14
19
  const verb_executor_1 = require("./verb-executor");
15
- function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, realtimeUrl, persona = "Cairn", }) {
20
+ const vad_1 = require("./vad");
21
+ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, scopeId, planEndpoint, criticEndpoint, skillsSaveEndpoint, realtimeUrl, persona = "Cairn", }) {
16
22
  const pathname = (0, navigation_1.usePathname)() ?? "/";
17
23
  // Mirrors `pathname` for use inside long-lived closures (a realtime
18
24
  // session's handlers are all created once, when the connection opens —
@@ -26,6 +32,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
26
32
  // eslint-disable-next-line react-hooks/exhaustive-deps
27
33
  }, [pathname]);
28
34
  const router = (0, navigation_1.useRouter)();
35
+ // Starts at the same safe default on both server and client's first
36
+ // render — same hydration-mismatch reason `micSupported` below does this
37
+ // — then, once actually restored from sessionStorage post-mount (see the
38
+ // dedicated restore effect further down), flips to whatever a real page
39
+ // reload (a host app's own mutation handler, e.g. — see
40
+ // loadPersistedConversation's own doc comment) had showing a moment ago,
41
+ // so a real reload never again looks like the conversation simply ended.
29
42
  const [open, setOpen] = (0, react_1.useState)(false);
30
43
  // Collapsed by default so the panel only ever shows the current exchange
31
44
  // — the full archived transcript (built up over a long conversation)
@@ -34,6 +47,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
34
47
  const [historyExpanded, setHistoryExpanded] = (0, react_1.useState)(false);
35
48
  const [question, setQuestion] = (0, react_1.useState)("");
36
49
  const [answer, setAnswer] = (0, react_1.useState)(null);
50
+ // Real, live-reported gap this closes: once a continuing agent-loop step
51
+ // (click/fill/read/call_tool/batch) sets `answer` to its own progress
52
+ // text ("Typing earbuds into the search box"), that text just sat there
53
+ // unchanged for however long the NEXT resolveVerb call took — several
54
+ // real seconds, more under rate-limit retries — with nothing on screen
55
+ // telling the user the agent was still actually doing something. `answer`
56
+ // itself can't double as that signal (a terminal turn's own real,
57
+ // finished answer looks identical to unfinished progress text). This is
58
+ // a separate, explicit flag: true from the moment a continuing step's
59
+ // progress text is shown until the turn actually ends (a terminal verb,
60
+ // an error, or a give-up) — see its own setters below for exactly where.
61
+ const [loopWorking, setLoopWorking] = (0, react_1.useState)(false);
37
62
  const [status, setStatus] = (0, react_1.useState)("idle");
38
63
  const [caption, setCaption] = (0, react_1.useState)("");
39
64
  // The user's own last question, shown as its own floating caption bubble
@@ -106,6 +131,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
106
131
  const rtMicMutedRef = (0, react_1.useRef)(false);
107
132
  const rtSpeakerMutedRef = (0, react_1.useRef)(false);
108
133
  const rtStartingRef = (0, react_1.useRef)(false); // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
134
+ // The generation of the most recent "final" this client has processed —
135
+ // see ServerMessage's own doc comment in realtime-server.ts for the full
136
+ // real, live-found bug this closes: the client's own local barge-in can
137
+ // start a NEW turn (a new "final") before an EARLIER turn's own verb/
138
+ // audio, already in flight on the wire when the server processed that
139
+ // barge-in, actually arrives. WebSocket delivers messages in order, but
140
+ // "in order" isn't "still current" — every verb/speaking_start/
141
+ // audio_chunk/speaking_end/turn_complete message carries the generation
142
+ // it was produced under, and the handler drops it outright if it's
143
+ // older than this ref's value instead of applying it to whatever
144
+ // caption happens to be showing now.
145
+ const rtLastFinalGenerationRef = (0, react_1.useRef)(0);
109
146
  // Progressive PCM playback for the buffered (non-realtime) speak endpoint
110
147
  // — the same gapless AudioBufferSourceNode scheduling the realtime path
111
148
  // uses for its audio_chunk messages (see rtPlaybackCtxRef below), just fed
@@ -118,6 +155,37 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
118
155
  const typedPlaybackGainRef = (0, react_1.useRef)(null);
119
156
  const typedNextPlayTimeRef = (0, react_1.useRef)(0);
120
157
  const typedScheduledSourcesRef = (0, react_1.useRef)([]);
158
+ // Bumped by stopTypedPlayback() every time it runs — playPcmStream's own
159
+ // async reader loop checks this before scheduling each chunk, so a
160
+ // superseded call (two speak()/ask() replies resolving close together)
161
+ // actually STOPS reading and scheduling more audio once a newer call has
162
+ // taken over, instead of continuing to push nodes into the shared
163
+ // playback graph behind the newer call's back. stopTypedPlayback() on
164
+ // its own only ever stopped nodes that already existed at the moment it
165
+ // ran — it never told an in-flight stream reader to stop producing MORE
166
+ // of them, which is exactly what let two replies' audio genuinely
167
+ // overlap (one starting, then a second call's audio starting on top of
168
+ // it moments later) — a real, live-found race, not a guess.
169
+ const typedPlaybackGenerationRef = (0, react_1.useRef)(0);
170
+ // A DIFFERENT real gap the generation counter above doesn't close: it
171
+ // only protects one typed reply's audio against ANOTHER typed reply's
172
+ // audio. A typed ask()/speak() call already in flight — its /api/
173
+ // copilot/speak fetch genuinely takes several seconds under real
174
+ // conditions (rate-limit retries make this worse, not better) — has no
175
+ // way to know a realtime call started WHILE it was still waiting. Its
176
+ // response arrives, and normally-innocent code plays it, seconds after
177
+ // startRealtime() already ran — genuinely overlapping with the live
178
+ // call's own audio, since nothing about the realtime session's own
179
+ // start/mute/barge-in controls have any way to reach a typed reply
180
+ // that hadn't even been scheduled yet when they ran. Found live: two
181
+ // full agent answers, sourced entirely from separate /api/copilot/
182
+ // speak calls, audibly overlapping about a second apart, while a
183
+ // realtime call was the only thing visibly active in the UI the whole
184
+ // time. startRealtime() sets this; endRealtime() clears it; speak()/
185
+ // speakAndWait() check it AFTER their fetch resolves and drop the
186
+ // reply's audio entirely (never call playPcmStream at all) if a
187
+ // realtime call has taken over since the request was made.
188
+ const typedPlaybackSuspendedRef = (0, react_1.useRef)(false);
121
189
  // Watchdog for the "rt-thinking" state: started on every "final" transcript,
122
190
  // cleared the moment the server responds with anything for that turn
123
191
  // (verb/speaking_start/speaking_end/turn_complete/error). If it ever
@@ -140,6 +208,36 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
140
208
  // finished arriving, so the mic can't start sending while the agent is
141
209
  // still audibly speaking.
142
210
  const rtAudioDoneArrivingRef = (0, react_1.useRef)(true);
211
+ // Diagnostic only, not a functional guard — flips to true the first time
212
+ // a real mic packet is actually sent after transitioning to
213
+ // "rt-listening", logged once (not per-packet, which would flood the
214
+ // console). Added specifically so a "status says Listening… but nothing
215
+ // I say gets picked up" report can be told apart, from the log alone,
216
+ // between "the send gate never opened" (this never logs) and "the gate
217
+ // opened and sent real audio, so the problem is somewhere else entirely
218
+ // — Deepgram's own STT, or a real hardware/OS mic issue this app can't
219
+ // see or fix" (this logs once, then goes quiet as expected).
220
+ const micAudioSentSinceListeningRef = (0, react_1.useRef)(false);
221
+ // Safety net for a live realtime call outliving this component instance:
222
+ // without this, unmounting (a parent removing the widget, a route change
223
+ // that remounts it, or — the real, live-hit case — Next.js Fast Refresh
224
+ // remounting the component on every dev-mode source edit while a call is
225
+ // open) left the WebSocket, the open getUserMedia mic stream, and both
226
+ // AudioContexts running completely orphaned: nothing ever called
227
+ // rtSocketRef.current?.close() or stopped the mic tracks. The old,
228
+ // zombie connection then kept transcribing and replying in parallel with
229
+ // whatever the newly-mounted instance does next — the literal
230
+ // "two things running in parallel, the agent answering twice" bug found
231
+ // live. endRealtime() is safe to call from an unmount cleanup even
232
+ // though it also calls React state setters: it only reads stable refs
233
+ // (never a stale closure over props/state), and React 18 silently no-ops
234
+ // a setState call on an already-unmounted component.
235
+ (0, react_1.useEffect)(() => {
236
+ return () => {
237
+ if (rtSocketRef.current || rtCleanupRef.current)
238
+ endRealtime();
239
+ };
240
+ }, []);
143
241
  // Starts false on both server and client's first render (avoids a
144
242
  // hydration mismatch — `navigator` doesn't exist during SSR), then
145
243
  // updated after mount, once we're only ever running in the browser.
@@ -147,6 +245,36 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
147
245
  (0, react_1.useEffect)(() => {
148
246
  setMicSupported(!!navigator.mediaDevices?.getUserMedia && typeof MediaRecorder !== "undefined");
149
247
  }, []);
248
+ // Restores a conversation that survived a real page reload — same
249
+ // hydration-safety reason as `micSupported` just above: `open`/`answer`/
250
+ // `lastQuestion`/`transcript` all start at their normal empty defaults on
251
+ // both server and client's first render (matching what SSR produced), and
252
+ // only flip to whatever sessionStorage actually holds here, post-mount.
253
+ // Real, live-verified bug this closes: a host app's own mutation handler
254
+ // calling `window.location.reload()` (this SDK's own demo app does this
255
+ // after several real actions, e.g. moving a kanban card) tears down the
256
+ // entire React tree, this widget included — every bit of visible
257
+ // conversation state reset to nothing, making an in-progress conversation
258
+ // look like it had simply ended the instant the host page happened to
259
+ // reload, even though nothing about the CONVERSATION itself was over.
260
+ (0, react_1.useEffect)(() => {
261
+ const persisted = loadPersistedConversation();
262
+ if (!persisted)
263
+ return;
264
+ setOpen(persisted.open);
265
+ setAnswer(persisted.answer);
266
+ setLastQuestion(persisted.lastQuestion);
267
+ setTranscript(persisted.transcript);
268
+ historyRef.current = reconstructHistoryFromPersisted(persisted);
269
+ // Starts past the restored transcript's own highest id — otherwise the
270
+ // very next archived entry would reuse an id already on screen, a real
271
+ // duplicate-React-key bug (React silently confuses which DOM node is
272
+ // which when two list items share a key).
273
+ if (persisted.transcript.length > 0) {
274
+ transcriptIdRef.current = Math.max(...persisted.transcript.map((t) => t.id)) + 1;
275
+ }
276
+ // eslint-disable-next-line react-hooks/exhaustive-deps
277
+ }, []);
150
278
  const asking = status === "asking";
151
279
  const recording = status === "recording";
152
280
  const realtimeActive = status.startsWith("rt-");
@@ -166,6 +294,25 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
166
294
  (0, react_1.useEffect)(() => {
167
295
  answerRef.current = answer;
168
296
  }, [answer]);
297
+ // Persists the visible conversation to sessionStorage on every change, so
298
+ // a host app's own `window.location.reload()` (e.g. after a board-card
299
+ // move — see loadPersistedConversation's own comment for the full story)
300
+ // doesn't make an in-progress conversation look like it simply ended.
301
+ // Skips its own very first (mount) invocation on purpose: that call is
302
+ // always tied to the initial render's plain defaults (open:false,
303
+ // transcript:[], ...) — captured before the restore effect above's
304
+ // setState calls have actually landed — so persisting it would clobber a
305
+ // real, just-restored conversation with empty state for one tick, right
306
+ // before the corrective post-restore render fixes it back. Skipping costs
307
+ // nothing on a genuinely fresh session (there's nothing to persist yet).
308
+ const skippedMountPersistRef = (0, react_1.useRef)(false);
309
+ (0, react_1.useEffect)(() => {
310
+ if (!skippedMountPersistRef.current) {
311
+ skippedMountPersistRef.current = true;
312
+ return;
313
+ }
314
+ savePersistedConversation({ transcript, lastQuestion, answer, open });
315
+ }, [transcript, lastQuestion, answer, open]);
169
316
  // Auto-scroll to the newest content whenever the transcript grows or the
170
317
  // live (not-yet-archived) bubble's text changes.
171
318
  (0, react_1.useEffect)(() => {
@@ -235,11 +382,38 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
235
382
  }
236
383
  }
237
384
  function handleVerb(raw) {
385
+ setLoopWorking(false); // only ever called for a genuinely terminal verb — the turn is over
238
386
  (0, verb_executor_1.executeVerbResponse)(raw, pathname, {
239
387
  onExplain: (text) => {
240
388
  setAnswer(text);
241
- if (!realtimeActive)
242
- void speak(text); // realtime mode gets audio over the socket instead
389
+ // A REAL, deep, long-standing bug found live — not introduced by
390
+ // today's other fixes, just newly diagnosed: this function is a
391
+ // plain closure defined fresh every render, but when a "verb" WS
392
+ // message arrives it's invoked through ws.onmessage — a callback
393
+ // assigned ONCE, inside startRealtime(), and never reassigned for
394
+ // the rest of that connection's life. `realtimeActive` there is
395
+ // therefore frozen at whatever it was AT THE MOMENT startRealtime()
396
+ // was called — which is BEFORE the click handler's own state
397
+ // updates land, so it reads `false` for literally the entire
398
+ // lifetime of every realtime call. `!realtimeActive` was therefore
399
+ // ALWAYS true here, on every single realtime turn — this ran
400
+ // speak() (the typed HTTP path) in addition to the correct
401
+ // realtime audio_chunk playback, every time. This is the actual,
402
+ // original root cause of "two speakers" — the earlier
403
+ // typedPlaybackSuspendedRef fix only ever suppressed the resulting
404
+ // AUDIO once this was already firing, it never stopped the
405
+ // firing itself (or the wasted LLM/TTS call and quota burn
406
+ // underneath it). rtStateRef mirrors status specifically to avoid
407
+ // this class of bug in a callback like this one (see its own doc
408
+ // comment) — using it here instead of the stale const is the
409
+ // actual fix.
410
+ if (rtStateRef.current.startsWith("rt-")) {
411
+ rtLog("explain: realtime call active, letting the socket's own audio_chunk stream speak this");
412
+ }
413
+ else {
414
+ rtLog("explain: no realtime call active, using the typed speak() HTTP path");
415
+ void speak(text);
416
+ }
243
417
  },
244
418
  onNavigate: (route) => router.push(route),
245
419
  onMiss: reportMiss,
@@ -267,7 +441,16 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
267
441
  */
268
442
  async function runTour(steps) {
269
443
  const myGeneration = ++tourGenerationRef.current;
270
- const wasRealtimeListening = realtimeActive;
444
+ // Same real stale-closure bug as onExplain's own fix above, and the
445
+ // reason a realtime-triggered tour spoke through the wrong pipeline
446
+ // (or, after that fix suppressed the wrong pipeline's audio, spoke
447
+ // through nothing at all — "tour did not speak anything") and could
448
+ // leave the mic never properly told to resume listening afterward
449
+ // (see the `setRtStatus("rt-listening")` call at the end of this
450
+ // function, fixed the same way). rtStateRef.current is always
451
+ // current, regardless of which render's closure this particular
452
+ // invocation runs inside.
453
+ const wasRealtimeListening = rtStateRef.current.startsWith("rt-");
271
454
  touringRef.current = true;
272
455
  if (wasRealtimeListening)
273
456
  setRtStatus("rt-speaking");
@@ -350,7 +533,31 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
350
533
  return;
351
534
  setTourStep(null);
352
535
  setCaption("");
353
- if (wasRealtimeListening && realtimeActive)
536
+ // The most damaging half of this stale-closure bug: this used to
537
+ // read the stale `realtimeActive` const, which meant this call was
538
+ // ALWAYS skipped for a tour reached via realtime — the mic was
539
+ // never explicitly told to resume listening once the tour ended.
540
+ // rtStateRef.current.startsWith("rt-") is what actually reflects
541
+ // whether the connection is still live right now.
542
+ if (wasRealtimeListening && rtStateRef.current.startsWith("rt-"))
543
+ setRtStatus("rt-listening");
544
+ }
545
+ catch (err) {
546
+ // Real, live-found gap: this function had NO catch at all — only
547
+ // try/finally. Any error thrown anywhere in the loop above (a
548
+ // rejected speakOverRealtime/speakAndWait call, a DOM exception
549
+ // from el.click(), a network failure) skipped the resume-listening
550
+ // line above ENTIRELY, leaving the mic stuck exactly where the
551
+ // tour left off — matching "after this it's not listening"
552
+ // reported live. Every other place in this file that can fail
553
+ // mid-turn (ask(), handleDeepgramMessage's own server-side
554
+ // equivalent) already guarantees SOME recovery path; this one
555
+ // didn't have one at all.
556
+ console.error("[cairn] tour failed partway through:", err);
557
+ rtLog("tour failed — forcing the mic back to listening instead of leaving it stuck", { error: String(err) });
558
+ setTourStep(null);
559
+ setCaption("");
560
+ if (wasRealtimeListening && rtStateRef.current.startsWith("rt-"))
354
561
  setRtStatus("rt-listening");
355
562
  }
356
563
  finally {
@@ -371,13 +578,114 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
371
578
  await runTypedAgentLoop(q);
372
579
  }
373
580
  catch {
374
- setAnswer("Something went wrong reaching the help service try again in a moment.");
581
+ // See typedPlaybackSuspendedRef's own doc commentthis whole
582
+ // fetch can still be in flight when a realtime call starts.
583
+ if (!typedPlaybackSuspendedRef.current)
584
+ setAnswer("Something went wrong reaching the help service — try again in a moment.");
375
585
  }
376
586
  finally {
377
- setStatus("idle");
587
+ // The most damaging form of the same race: unconditionally forcing
588
+ // status back to "idle" here, after a realtime call has ALREADY
589
+ // taken over (status is some "rt-*" value), would silently kick the
590
+ // UI out of the live call — hiding its mic/speaker/hangup controls
591
+ // and showing the "start call" screen instead — while the actual
592
+ // WebSocket connection underneath is still fully alive and still
593
+ // talking, now with no visible way to manage it at all. Skipping
594
+ // this reset when suspended is what stops a slow, stale typed
595
+ // request from ever being able to do that.
596
+ if (!typedPlaybackSuspendedRef.current)
597
+ setStatus("idle");
598
+ }
599
+ }
600
+ /**
601
+ * Architecture Pillar 6 (the safety layer) — the real, working default
602
+ * confirmation UI: a native browser confirm dialog, naming the tool's
603
+ * OWN real name/description (never inventing wording), for a WebMCP
604
+ * tool whose registration declared `riskTier: "confirm"` (a payment, a
605
+ * delete, anything hard to undo). `window.confirm` blocks the calling
606
+ * microtask until the user actually answers — exactly the real,
607
+ * synchronous "get a genuine yes before this runs" behavior needed
608
+ * here, and simple enough to need no new UI component for this to be a
609
+ * real, functioning default rather than just plumbing with nothing on
610
+ * the other end. A host app wanting a nicer in-widget modal can still
611
+ * build one — this function is the only thing that would need
612
+ * replacing to do that.
613
+ */
614
+ function confirmToolCall(tool) {
615
+ if (typeof window === "undefined" || typeof window.confirm !== "function")
616
+ return Promise.resolve(false);
617
+ const message = tool.description ? `${tool.name}: ${tool.description}\n\nAllow this action?` : `Allow "${tool.name}"?`;
618
+ return Promise.resolve(window.confirm(message));
619
+ }
620
+ /**
621
+ * Architecture Pillar 4 — the typed transport's own Planner call,
622
+ * mirroring resolvePlan's real network shape but reached over HTTP
623
+ * (planEndpoint's server-side handler — createPlanHandler in server.ts
624
+ * — is the only place that can hold the real LLM API key). Never
625
+ * throws: any network/parse failure degrades to the exact same single-
626
+ * task fallback plan resolvePlan itself falls back to on an LLM error,
627
+ * so a Planner hiccup never blocks the turn.
628
+ */
629
+ async function fetchPlan(goal, version = 1) {
630
+ try {
631
+ const res = await fetch(planEndpoint, {
632
+ method: "POST",
633
+ headers: { "content-type": "application/json" },
634
+ body: JSON.stringify({ goal, version }),
635
+ });
636
+ const data = await res.json();
637
+ if (data && typeof data === "object" && Array.isArray(data.tasks) && data.tasks.length > 0)
638
+ return data;
378
639
  }
640
+ catch {
641
+ // Falls through to the same fallback plan shape below.
642
+ }
643
+ return { version, goal, facts: [], tasks: [{ id: "t1", description: goal, doneContract: "The stated goal has been achieved.", status: "in_progress" }] };
644
+ }
645
+ /** Same real-network shape as fetchPlan, for criticEndpoint's
646
+ * createCriticHandler — degrades to a safe "continue" verdict on any
647
+ * failure, same resilience discipline as resolveCritic itself. */
648
+ async function fetchCriticVerdict(task, goal, verb, observation) {
649
+ try {
650
+ const res = await fetch(criticEndpoint, {
651
+ method: "POST",
652
+ headers: { "content-type": "application/json" },
653
+ body: JSON.stringify({ task, goal, verb, observation: observation ?? null }),
654
+ });
655
+ const data = await res.json();
656
+ if (data && typeof data.verdict === "string")
657
+ return data;
658
+ }
659
+ catch {
660
+ // Falls through to the safe default below.
661
+ }
662
+ return { verdict: "continue", reasoning: "Critic call failed — defaulting to continue rather than blocking the turn." };
663
+ }
664
+ /**
665
+ * Architecture Pillar 3 (Skill half) — the typed transport's own
666
+ * Formulator save, mirroring realtime-server.ts's own post-turn
667
+ * `compileSkill`+`saveSkill` call. Fire-and-forget (never awaited by
668
+ * the caller, never allowed to affect what the user sees) since saving
669
+ * a Skill is bookkeeping for a FUTURE turn, not part of answering this
670
+ * one — matches the Formulator's own "cheap, runs once per turn, never
671
+ * blocks anything" framing. Classifies the current live page for a
672
+ * best-effort pattern tag the same way resolveVerb's own per-request
673
+ * classification does server-side.
674
+ */
675
+ function saveSkillIfLearned(goal, learnedFacts) {
676
+ if (!skillsSaveEndpoint || learnedFacts.length === 0)
677
+ return;
678
+ const matches = (0, core_1.classifyUiPattern)((0, core_1.deriveStructureSignals)(liveRegistryRef.current.getSnapshot().elements));
679
+ void fetch(skillsSaveEndpoint, {
680
+ method: "POST",
681
+ headers: { "content-type": "application/json" },
682
+ body: JSON.stringify({ goal, learnedFacts, pattern: matches[0]?.pattern }),
683
+ }).catch(() => {
684
+ // A Skill that fails to save just means the next similar goal
685
+ // starts from scratch again, same as if nothing had been learned
686
+ // this turn — never worth surfacing as a user-visible error.
687
+ });
379
688
  }
380
- const MAX_LOOP_ITERATIONS = 6; // a hard cap, not a target — see runTypedAgentLoop
381
689
  /**
382
690
  * Drives the agent loop over the stateless HTTP path: ask the server,
383
691
  * and if it comes back with a continuing step (click/fill/read/
@@ -390,58 +698,196 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
390
698
  * still knows what it was actually asked. `historyRef` (the
391
699
  * conversation's real memory) is only ever committed once, at the end —
392
700
  * a turn that hits the cap mid-loop doesn't leave partial noise in it.
701
+ * The loop itself (the `for`/TERMINAL_VERBS/iteration-cap shape) lives
702
+ * in agent-loop.ts, shared with the realtime relay's own finalizeTurn —
703
+ * this function owns everything transport-specific: the actual fetch,
704
+ * the raw/untyped response handling a stateless HTTP call needs (unlike
705
+ * realtime's always-valid in-process resolveVerb call), and the real
706
+ * historyRef commit.
393
707
  */
394
708
  async function runTypedAgentLoop(q) {
395
- let loopHistory = historyRef.current;
396
709
  const webMcpTools = await (0, webmcp_client_1.discoverWebMcpTools)();
397
- for (let i = 0; i < MAX_LOOP_ITERATIONS; i++) {
398
- const liveScan = liveRegistryRef.current.getSnapshot();
399
- liveMapRef.current = liveScan.byId;
400
- const res = await fetch(endpoint, {
401
- method: "POST",
402
- headers: { "content-type": "application/json" },
403
- body: JSON.stringify({
404
- route: pathname,
405
- question: q,
406
- visible: (0, context_collector_1.collectVisible)(),
407
- history: loopHistory,
408
- liveElements: liveScan.elements,
409
- webMcpTools,
410
- }),
411
- });
412
- const data = await res.json().catch(() => null);
413
- const parsed = (0, core_1.safeParseVerbResponse)(data);
414
- if (!parsed || core_1.TERMINAL_VERBS.has(parsed.verb)) {
415
- handleVerb(data);
416
- // Unlike the realtime relay (one persistent connection, memory
417
- // lives server-side), each of these POSTs is stateless — the
418
- // widget itself is what remembers, and resends it above so the
419
- // model has context for "the first one" / "do that instead" on
420
- // the next question.
421
- historyRef.current = [
422
- ...loopHistory,
423
- { role: "user", text: q },
424
- { role: "assistant", text: summarizeVerbForHistory(data) },
425
- ].slice(-MAX_HISTORY_TURNS);
426
- return;
710
+ let lastRawResponse = null;
711
+ // Architecture Pillar 4 — real Planner/Critic wiring for the typed
712
+ // transport, opt-in via planEndpoint/criticEndpoint (see their own
713
+ // doc comments on CopilotProps) — closes the gap the plan file names
714
+ // directly ("the typed/HTTP path has zero Planner/Critic wiring at
715
+ // all... today explicitly realtime-only by deferral, not by
716
+ // decision"). Mirrors realtime-server.ts's finalizeTurn: an eager
717
+ // Planner kickoff when looksMultiStep(q) already flags a probable
718
+ // compound goal, a lazy fallback kickoff on the first continuing step
719
+ // otherwise, and a genuinely separate Critic pass over each
720
+ // continuing step's real result. Neither endpoint set (the default)
721
+ // means plannerEnabled is false and this whole block is a no-op —
722
+ // the typed loop behaves exactly as it always has.
723
+ let planPromise = null;
724
+ let plan = null;
725
+ let progress = null;
726
+ const STALL_THRESHOLD = 3; // same bounded budget realtime's own Critic wiring uses
727
+ const plannerEnabled = Boolean(planEndpoint && criticEndpoint);
728
+ if (plannerEnabled && (0, agent_loop_1.looksMultiStep)(q))
729
+ planPromise = fetchPlan(q);
730
+ // Architecture Pillar 3 (Skill half) every real, Critic-verified
731
+ // learnedFact from this turn's steps; saved once the turn concludes,
732
+ // below (saveSkillIfLearned). Empty is the common case, not a gap.
733
+ const learnedFacts = [];
734
+ const result = await (0, agent_loop_1.driveAgentLoop)(historyRef.current, {
735
+ async getNextStep(loopHistory) {
736
+ const liveScan = liveRegistryRef.current.getSnapshot();
737
+ liveMapRef.current = liveScan.byId;
738
+ const res = await fetch(endpoint, {
739
+ method: "POST",
740
+ headers: { "content-type": "application/json" },
741
+ body: JSON.stringify({
742
+ // pathnameRef, not the closed-over `pathname` — a navigate
743
+ // step (now possibly continuing, see isTerminalVerb) can
744
+ // change the real route mid-loop; this whole async function's
745
+ // own `pathname` closure was captured once, at the render
746
+ // that started this turn, and never updates again on its own.
747
+ route: pathnameRef.current,
748
+ question: q,
749
+ visible: (0, context_collector_1.collectVisible)(),
750
+ history: loopHistory,
751
+ liveElements: liveScan.elements,
752
+ webMcpTools,
753
+ scopeId,
754
+ }),
755
+ });
756
+ const data = await res.json().catch(() => null);
757
+ lastRawResponse = data;
758
+ return (0, core_1.safeParseVerbResponse)(data);
759
+ },
760
+ onStep({ verb, terminal }) {
761
+ // A continuing step — show it happening (execution itself
762
+ // happens in executeStep below). Terminal steps are handled once
763
+ // driveAgentLoop returns, via handleVerb — unchanged from before.
764
+ // Same stale-typed-reply guard as the terminal case below — a
765
+ // multi-step typed loop can still be mid-flight when a realtime
766
+ // call starts.
767
+ if (!terminal && !typedPlaybackSuspendedRef.current) {
768
+ setAnswer(summarizeVerbForHistory(verb));
769
+ setLoopWorking(true);
770
+ }
771
+ // The lazy fallback — only fires when looksMultiStep missed
772
+ // (planPromise is still null): a real Plan is still guaranteed
773
+ // before the Critic needs one, just one round trip later.
774
+ if (!terminal && plannerEnabled && !planPromise)
775
+ planPromise = fetchPlan(q);
776
+ return false;
777
+ },
778
+ // Same real, live-found fix as the realtime WS "verb" handler's own
779
+ // executeToolStep call — a fresh scan per step, not the turn's
780
+ // frozen liveMapRef, so a step that reveals new DOM (a click that
781
+ // opens a modal) doesn't leave the NEXT step unable to find
782
+ // anything in it.
783
+ executeStep: (verb) => (0, verb_executor_1.executeToolStep)(verb, pathnameRef.current, liveRegistryRef.current.getSnapshot().byId, (route) => router.push(route), confirmToolCall).then((r) => r?.observation),
784
+ runCritic: plannerEnabled
785
+ ? async ({ verb, observation }) => {
786
+ // Real state, not the Executor's self-report — see
787
+ // resolveCritic's own doc comment (server.ts) for why this is
788
+ // a genuinely separate pass, same precedent realtime already
789
+ // established.
790
+ if (!plan) {
791
+ plan = planPromise ? await planPromise : await fetchPlan(q);
792
+ progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
793
+ }
794
+ const currentProgress = progress;
795
+ const currentTask = plan.tasks[currentProgress.currentTaskIndex];
796
+ const verdict = await fetchCriticVerdict(currentTask, q, verb, observation);
797
+ if (verdict.learnedFact)
798
+ learnedFacts.push(verdict.learnedFact);
799
+ if (verdict.verdict === "task_complete") {
800
+ currentTask.status = "done";
801
+ if (currentProgress.currentTaskIndex < plan.tasks.length - 1) {
802
+ // More tasks remain — advance and keep looping instead of
803
+ // ending the turn here.
804
+ currentProgress.currentTaskIndex++;
805
+ plan.tasks[currentProgress.currentTaskIndex].status = "in_progress";
806
+ currentProgress.stallCount = 0;
807
+ return { ...verdict, verdict: "continue" };
808
+ }
809
+ // The last task is genuinely done — end the loop right here
810
+ // instead of asking the model again and hoping it notices.
811
+ return verdict;
812
+ }
813
+ if (verdict.verdict === "replan") {
814
+ plan = await fetchPlan(q, plan.version + 1);
815
+ progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
816
+ return { ...verdict, verdict: "continue" };
817
+ }
818
+ if (verdict.verdict === "give_up")
819
+ return verdict;
820
+ // "continue" — a harness-enforced fail-safe on top of the
821
+ // Critic's own judgment, same Magentic-One-shaped two-tier
822
+ // tolerance realtime already uses.
823
+ currentProgress.stallCount++;
824
+ if (currentProgress.stallCount >= STALL_THRESHOLD) {
825
+ return {
826
+ verdict: "give_up",
827
+ reasoning: `Stuck after ${currentProgress.stallCount} steps with no confirmed progress on "${currentTask.description}" — ${verdict.reasoning}`,
828
+ };
829
+ }
830
+ return verdict;
831
+ }
832
+ : undefined,
833
+ });
834
+ // Architecture Pillar 3 (Skill half) — the Formulator, once per turn.
835
+ saveSkillIfLearned(q, learnedFacts);
836
+ if (result.outcome === "terminal" || result.outcome === "unparseable" || result.outcome === "critic-complete") {
837
+ // A realtime call can start WHILE this whole typed loop (potentially
838
+ // several real fetches deep) was still in flight — applying this
839
+ // reply now would overwrite the live call's own answer with a
840
+ // stale, orphaned bubble that doesn't correspond to anything the
841
+ // realtime conversation actually said. speak()/speakAndWait() guard
842
+ // the AUDIO half of this same real, live-found race (see
843
+ // typedPlaybackSuspendedRef's own doc comment) — this is the
844
+ // matching guard for the TEXT half, which would otherwise still
845
+ // leak through even with the audio silenced.
846
+ // The Critic independently confirmed the last task's doneContract
847
+ // is satisfied even though the model's own verb never got there —
848
+ // real fix for the diagnosed bug (a batch succeeded and the model
849
+ // kept looping instead of recognizing it). No raw server response
850
+ // exists for this synthesized verb (it never came from `endpoint`
851
+ // at all), so it's built directly from the verdict's own reasoning
852
+ // — same shape realtime-server.ts synthesizes for the same outcome.
853
+ const raw = result.outcome === "critic-complete" ? { verb: "explain", text: result.verdict.reasoning } : lastRawResponse;
854
+ if (typedPlaybackSuspendedRef.current) {
855
+ rtLog("dropping stale typed reply's text — a realtime call started while it was still in flight");
856
+ }
857
+ else {
858
+ handleVerb(raw);
427
859
  }
428
- // A continuing step show it happening, execute it for real, and
429
- // go around again with the real result instead of ending the turn.
430
- setAnswer(summarizeVerbForHistory(data));
431
- const stepResult = await (0, verb_executor_1.executeToolStep)(data, pathname, liveMapRef.current);
432
- loopHistory = [
433
- ...loopHistory,
434
- {
435
- role: "assistant",
436
- text: `${summarizeVerbForHistory(data)}. Result: ${stepResult?.observation ?? "no result"}`,
437
- },
860
+ // Unlike the realtime relay (one persistent connection, memory
861
+ // lives server-side), each of these POSTs is stateless the
862
+ // widget itself is what remembers, and resends it above so the
863
+ // model has context for "the first one" / "do that instead" on
864
+ // the next question. Summarized from the RAW response (not
865
+ // driveAgentLoop's typed finalVerb) via this file's own untyped
866
+ // summarizeVerbForHistory — deliberately, since a response that
867
+ // failed schema validation (outcome "unparseable") can still carry
868
+ // real, usable fields (e.g. a stray extra property tripped
869
+ // .strict() while `text` itself was fine) that only the untyped,
870
+ // duck-typed summarizer sees; there is no typed finalVerb at all
871
+ // for that outcome.
872
+ historyRef.current = [
873
+ ...result.workingHistory,
874
+ { role: "user", text: q },
875
+ { role: "assistant", text: summarizeVerbForHistory(raw) },
438
876
  ].slice(-MAX_HISTORY_TURNS);
877
+ return;
439
878
  }
440
- setAnswer("I wasn't able to finish that try asking again or breaking it into smaller steps.");
879
+ // "gave-up" (iteration cap hit with no terminal verb) OR "critic-give-up"
880
+ // (the Critic/stall fail-safe decided continuing wouldn't help — its
881
+ // own reasoning is a genuinely better message than the generic
882
+ // fallback, same as realtime-server.ts's own finalizeTurn).
883
+ const giveUpText = result.outcome === "critic-give-up" ? result.verdict.reasoning : "I wasn't able to finish that — try asking again or breaking it into smaller steps.";
884
+ const gaveUpSummary = result.outcome === "critic-give-up" ? giveUpText : "(gave up after too many steps)";
885
+ setLoopWorking(false);
886
+ setAnswer(giveUpText);
441
887
  historyRef.current = [
442
- ...loopHistory,
888
+ ...result.workingHistory,
443
889
  { role: "user", text: q },
444
- { role: "assistant", text: "(gave up after too many steps)" },
890
+ { role: "assistant", text: gaveUpSummary },
445
891
  ].slice(-MAX_HISTORY_TURNS);
446
892
  }
447
893
  function ensureTypedPlaybackGraph() {
@@ -456,8 +902,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
456
902
  }
457
903
  /** Stops whatever's currently playing on the typed/mic path's playback
458
904
  * graph, so two responses (e.g. a rapid double-click, or two answers
459
- * resolved close together) can never be heard overlapping. */
905
+ * resolved close together) can never be heard overlapping. Also bumps
906
+ * typedPlaybackGenerationRef — see its own doc comment for why that's
907
+ * required for this to actually hold when a NEW reply's audio is still
908
+ * arriving as a stream, not just already fully scheduled. */
460
909
  function stopTypedPlayback() {
910
+ typedPlaybackGenerationRef.current++;
461
911
  for (const source of typedScheduledSourcesRef.current) {
462
912
  source.onended = null;
463
913
  try {
@@ -486,6 +936,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
486
936
  */
487
937
  function playPcmStream(stream) {
488
938
  stopTypedPlayback();
939
+ const myGeneration = typedPlaybackGenerationRef.current;
489
940
  const { ctx, gain } = ensureTypedPlaybackGraph();
490
941
  void ctx.resume().catch(() => { });
491
942
  return new Promise((resolve) => {
@@ -521,7 +972,16 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
521
972
  const reader = stream.getReader();
522
973
  try {
523
974
  for (;;) {
975
+ // A newer call already ran stopTypedPlayback() (bumping the
976
+ // generation) while we were mid-read — stop here instead of
977
+ // scheduling more chunks behind its back. Checked both before
978
+ // AND after the await: a supersede can land at any point while
979
+ // this loop is blocked waiting on the next chunk.
980
+ if (typedPlaybackGenerationRef.current !== myGeneration)
981
+ break;
524
982
  const { done, value } = await reader.read();
983
+ if (typedPlaybackGenerationRef.current !== myGeneration)
984
+ break;
525
985
  if (done)
526
986
  break;
527
987
  if (!value || value.length === 0)
@@ -556,6 +1016,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
556
1016
  });
557
1017
  if (!res.ok || !res.body)
558
1018
  return;
1019
+ // A realtime call can start WHILE this fetch was in flight — see
1020
+ // typedPlaybackSuspendedRef's own doc comment for why that's a real,
1021
+ // live-found overlapping-audio case, not a hypothetical one.
1022
+ if (typedPlaybackSuspendedRef.current) {
1023
+ rtLog("dropping stale typed reply's audio — a realtime call started while it was still being fetched");
1024
+ return;
1025
+ }
559
1026
  void playPcmStream(res.body);
560
1027
  }
561
1028
  catch {
@@ -576,6 +1043,13 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
576
1043
  });
577
1044
  if (!res.ok || !res.body)
578
1045
  return;
1046
+ // See speak()'s own identical check and typedPlaybackSuspendedRef's
1047
+ // doc comment — a realtime call can start while this fetch was in
1048
+ // flight, same real risk here.
1049
+ if (typedPlaybackSuspendedRef.current) {
1050
+ rtLog("dropping stale typed reply's audio — a realtime call started while it was still being fetched");
1051
+ return;
1052
+ }
579
1053
  await playPcmStream(res.body);
580
1054
  }
581
1055
  catch {
@@ -612,7 +1086,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
612
1086
  // forever with the mic never resuming — move on instead.
613
1087
  setTimeout(() => {
614
1088
  if (!settled)
615
- console.warn("[cairn] tour step audio confirmation timed out — continuing");
1089
+ rtLog("tour step audio confirmation timed out after 15s — continuing anyway");
616
1090
  finish();
617
1091
  }, 15000);
618
1092
  ws.send(JSON.stringify({ type: "speak", text }));
@@ -685,7 +1159,24 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
685
1159
  // which is exactly what "hearing the agent twice, in parallel" was.
686
1160
  if (!realtimeUrl || !micSupported || realtimeActive || rtStartingRef.current)
687
1161
  return;
1162
+ rtLog("starting realtime call", { url: realtimeUrl });
688
1163
  rtStartingRef.current = true;
1164
+ // A typed/mic-recorded reply's audio can still be mid-playback on its
1165
+ // own separate graph (typedPlaybackGainRef, only ever touched by
1166
+ // stopTypedPlayback/playPcmStream) when the user switches straight into
1167
+ // a live call — endRealtime() already stops it on the way OUT of a
1168
+ // call, but nothing stopped it on the way IN, so it kept playing
1169
+ // completely unaffected by the realtime session's own mute-speaker
1170
+ // button (which only ever touches rtPlaybackGainRef) or by barge-in —
1171
+ // a real, live-found "two independent speakers" bug: muting or saying
1172
+ // "stop" only ever reached the realtime pipeline, while this leftover
1173
+ // typed audio played on regardless until it finished on its own.
1174
+ stopTypedPlayback();
1175
+ // Also blocks any typed reply that's still mid-fetch RIGHT NOW (not
1176
+ // yet playing anything, so stopTypedPlayback() above has nothing to
1177
+ // stop) from playing its audio once it finally arrives, seconds from
1178
+ // now — see typedPlaybackSuspendedRef's own doc comment.
1179
+ typedPlaybackSuspendedRef.current = true;
689
1180
  archiveCurrentExchange(); // preserve whatever typed/mic exchange preceded switching into a live call
690
1181
  setAnswer(null);
691
1182
  setCaption("");
@@ -717,6 +1208,21 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
717
1208
  const processor = audioCtx.createScriptProcessor(4096, 1, 1);
718
1209
  const silence = audioCtx.createGain();
719
1210
  silence.gain.value = 0;
1211
+ const bargeInVad = (0, vad_1.createVadDetector)();
1212
+ // Real, live-reported bug this closes: firing triggerBargeIn() off a
1213
+ // SINGLE ~85-100ms VAD frame meant one cough or door-slam frame that
1214
+ // happened to pass the energy+ZCR gate cut the agent off, permanently
1215
+ // (no server-side "was this real" recovery exists anymore — see
1216
+ // vad.ts's own doc comment for why that was removed instead of kept).
1217
+ // Real research into how production voice-agent platforms solve this
1218
+ // (Pipecat, LiveKit Agents, Vapi, Deepgram's Voice Agent API — see
1219
+ // DEVELOPMENT.md) converges on gating the LOCAL trigger on SUSTAINED
1220
+ // speech across a minimum duration instead — Pipecat's own production
1221
+ // spec cites 250ms, Vapi's stopSpeakingPlan defaults to 0.2s. This
1222
+ // gate does exactly that, entirely client-side (no network round trip
1223
+ // or STT-transcript timing involved, so it can't reintroduce the
1224
+ // removed server-side race).
1225
+ const bargeInGate = (0, vad_1.createBargeInGate)();
720
1226
  processor.onaudioprocess = (e) => {
721
1227
  if (ws.readyState !== WebSocket.OPEN)
722
1228
  return;
@@ -725,31 +1231,77 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
725
1231
  // Barge-in: while the agent is speaking a real conversational reply,
726
1232
  // still thinking about one, OR mid-tour, keep listening to the mic
727
1233
  // locally even though it isn't being sent yet, and cut the agent
728
- // off the instant the user starts talking again instead of making
729
- // them wait — including during a guided tour, which now cancels the
730
- // rest of the walkthrough on interruption (see triggerBargeIn)
731
- // instead of being talked-over-proof by design, the way a real
732
- // person giving a tour stops when you have a question. The
733
- // "rt-thinking" half matters just as much as "rt-speaking": an LLM
734
- // turn can easily take a couple of seconds with nothing playing
735
- // yet, and without this the mic was completely deaf during that
736
- // whole window — found live as "not listening while speaking... no
737
- // interrupting system", not just a missed nice-to-have.
1234
+ // off once the user has been sustainedly talking again (bargeInGate,
1235
+ // above) instead of making them wait — including during a guided
1236
+ // tour, which now cancels the rest of the walkthrough on
1237
+ // interruption (see triggerBargeIn) instead of being talked-over-
1238
+ // proof by design, the way a real person giving a tour stops when
1239
+ // you have a question. The "rt-thinking" half matters just as much
1240
+ // as "rt-speaking": an LLM turn can easily take a couple of seconds
1241
+ // with nothing playing yet, and without this the mic was completely
1242
+ // deaf during that whole window — found live as "not listening
1243
+ // while speaking... no interrupting system", not just a missed
1244
+ // nice-to-have.
738
1245
  if (rtStateRef.current === "rt-speaking" || rtStateRef.current === "rt-thinking") {
739
- const rms = computeRms(e.inputBuffer.getChannelData(0));
740
- if (rms > BARGE_IN_RMS_THRESHOLD)
1246
+ const frame = bargeInVad.process(e.inputBuffer.getChannelData(0));
1247
+ const frameDurationMs = (e.inputBuffer.length / audioCtx.sampleRate) * 1000;
1248
+ if (bargeInGate.update(frame, frameDurationMs))
741
1249
  triggerBargeIn();
742
1250
  return;
743
1251
  }
1252
+ bargeInGate.reset(); // not currently interruptible — don't let stale progress from a moment ago carry into the next speaking/thinking phase
744
1253
  if (rtStateRef.current !== "rt-listening")
745
1254
  return; // don't send our own mic while the agent is thinking/speaking
1255
+ if (!micAudioSentSinceListeningRef.current) {
1256
+ micAudioSentSinceListeningRef.current = true;
1257
+ rtLog("mic audio actually being sent (send gate is open)");
1258
+ }
746
1259
  const pcm = floatTo16BitPCM(downsampleTo16k(e.inputBuffer.getChannelData(0), audioCtx.sampleRate));
747
1260
  ws.send(pcm);
748
1261
  };
749
1262
  source.connect(processor);
750
1263
  processor.connect(silence);
751
1264
  silence.connect(audioCtx.destination);
1265
+ // Real, live-reported bug this closes: "status says Listening but
1266
+ // nothing gets picked up" — traced to browsers deliberately
1267
+ // suspending an AudioContext that has no active OUTPUT (a real,
1268
+ // documented power-saving policy, not backgrounded-tab-only). This
1269
+ // capture context has no real output at all by design (silence's
1270
+ // gain is 0), making it exactly the shape most likely to get
1271
+ // silently suspended — and once suspended, onaudioprocess simply
1272
+ // stops firing, so nothing inside it can detect or recover from its
1273
+ // own silence. A periodic external health check is the only
1274
+ // reliable way to catch this: resume the context if the browser
1275
+ // suspended it, and — a real, separate failure mode — detect the
1276
+ // mic's OWN MediaStreamTrack actually ending or going muted (device
1277
+ // unplugged, OS-level permission revoked mid-call, another app
1278
+ // taking exclusive access) and surface a real, honest error instead
1279
+ // of silently going deaf with the UI still claiming to listen.
1280
+ const micHealthCheck = setInterval(() => {
1281
+ if (audioCtx.state !== "running") {
1282
+ rtLog("capture AudioContext was suspended — resuming", { state: audioCtx.state });
1283
+ void audioCtx.resume().catch((err) => rtLog("failed to resume capture AudioContext", { error: String(err) }));
1284
+ }
1285
+ const track = stream.getAudioTracks()[0];
1286
+ if (track && (track.readyState === "ended" || track.muted)) {
1287
+ rtLog("mic track is no longer live — ending the call", { readyState: track.readyState, muted: track.muted });
1288
+ setAnswer("The microphone connection was lost — try starting the call again.");
1289
+ endRealtime();
1290
+ }
1291
+ }, 2000);
1292
+ // The track's own "ended" event is the immediate signal (fires the
1293
+ // instant the OS/browser actually kills the track) — the poll above
1294
+ // is the safety net for anything that doesn't fire it reliably
1295
+ // (muted-without-ended has no dedicated event in the spec).
1296
+ const handleMicTrackEnded = () => {
1297
+ rtLog("mic track ended unexpectedly — ending the call");
1298
+ setAnswer("The microphone connection was lost — try starting the call again.");
1299
+ endRealtime();
1300
+ };
1301
+ stream.getAudioTracks().forEach((t) => t.addEventListener("ended", handleMicTrackEnded));
752
1302
  rtCleanupRef.current = () => {
1303
+ clearInterval(micHealthCheck);
1304
+ stream.getAudioTracks().forEach((t) => t.removeEventListener("ended", handleMicTrackEnded));
753
1305
  processor.disconnect();
754
1306
  source.disconnect();
755
1307
  stream.getTracks().forEach((t) => t.stop());
@@ -780,6 +1332,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
780
1332
  rtTourAudioDoneRef.current = null;
781
1333
  return;
782
1334
  }
1335
+ rtLog("resumed listening");
1336
+ void audioCtx.resume().catch(() => { }); // don't wait up to 2s for the periodic health check if the browser already suspended capture
1337
+ micAudioSentSinceListeningRef.current = false;
783
1338
  setRtStatus("rt-listening");
784
1339
  setCaption("");
785
1340
  void sendFreshContext(); // refresh before the user starts talking again, not after
@@ -794,10 +1349,28 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
794
1349
  disarmThinkingWatchdog();
795
1350
  rtThinkingWatchdogRef.current = setTimeout(() => {
796
1351
  rtThinkingWatchdogRef.current = null;
797
- console.warn("[cairn] realtime turn timed out waiting on the server resuming listening");
798
- rtAudioDoneArrivingRef.current = true;
799
- setRtStatus("rt-listening");
800
- setCaption("");
1352
+ rtLog("thinking watchdog fired server took over 20s, resuming listening and abandoning that turn");
1353
+ // Real, live-found gap: this used to only reset LOCAL state,
1354
+ // never telling the server anything — so a turn that was simply
1355
+ // SLOW (not actually stuck; e.g. retrying a rate-limited call
1356
+ // across every configured key, which can genuinely take longer
1357
+ // than this 20s watchdog) kept running server-side, and its
1358
+ // reply arrived LATE, after the user had already moved on and
1359
+ // started a new turn locally — landing on whatever was now
1360
+ // showing instead of being recognized as stale. triggerBargeIn()
1361
+ // is exactly the fix: it sends the same real barge_in signal a
1362
+ // genuine interruption does, bumping the server's own generation
1363
+ // so that late reply — whenever it finally arrives — carries an
1364
+ // old generation number and gets correctly dropped by the
1365
+ // isStaleRtMessage check above instead of confusingly resuming.
1366
+ triggerBargeIn();
1367
+ setLoopWorking(false);
1368
+ // triggerBargeIn() clears the caption but never touched `answer`
1369
+ // — without this, a timed-out turn gave the user literally
1370
+ // nothing: no reply, no error, just a silent reset back to
1371
+ // "Listening…" that reads as "it heard me and did nothing." A
1372
+ // real, live-found gap, not just a console.warn nobody sees.
1373
+ setAnswer("That's taking longer than expected — try asking again.");
801
1374
  }, 20000);
802
1375
  }
803
1376
  function stopScheduledRtAudio() {
@@ -818,6 +1391,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
818
1391
  // now-stale audio_chunk/speaking_end that was already in flight, so a
819
1392
  // few straggling chunks can't sneak back in and resume playback.
820
1393
  function triggerBargeIn() {
1394
+ rtLog("barge-in triggered", { wasTouring: touringRef.current, discardedAudioChunks: rtScheduledSourcesRef.current.length });
821
1395
  disarmThinkingWatchdog();
822
1396
  stopScheduledRtAudio();
823
1397
  rtAudioDoneArrivingRef.current = true;
@@ -837,13 +1411,47 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
837
1411
  if (ws.readyState === WebSocket.OPEN)
838
1412
  ws.send(JSON.stringify({ type: "barge_in" }));
839
1413
  setRtStatus("rt-listening");
840
- setCaption("");
1414
+ // Real, live-found bug this fix removes: this used to also clear
1415
+ // the caption here (setCaption("")). That's correct-looking for a
1416
+ // REAL, VAD-triggered barge-in (the user's about to say something
1417
+ // new) — the very next "final" already does archiveCurrentExchange()
1418
+ // then overwrites caption with the new utterance, so clearing it
1419
+ // here was always redundant for that path. But this function is
1420
+ // ALSO called by the thinking watchdog on a timeout, where there is
1421
+ // no new utterance coming — clearing the caption there wiped out
1422
+ // the very question that just timed out, an instant before
1423
+ // setAnswer(the timeout message) ran, leaving the live pair as
1424
+ // {caption: "", answer: "That's taking longer..."} — a reply
1425
+ // visibly floating with no question above it, and nothing for
1426
+ // archiveCurrentExchange() to pair it with on the next turn either
1427
+ // (archiveText skips empty text). Simply never clearing it here is
1428
+ // correct for both callers: the barge-in path already gets a fresh
1429
+ // caption from the next "final", and the watchdog path now keeps
1430
+ // the timed-out question correctly paired with its own answer.
841
1431
  }
842
1432
  ws.onopen = () => {
1433
+ rtLog("connection open");
843
1434
  sendFreshContext();
844
1435
  setRtStatus("rt-listening");
845
1436
  rtStartingRef.current = false;
846
1437
  };
1438
+ // True for a verb/speaking_start/audio_chunk/speaking_end/
1439
+ // turn_complete message that belongs to an EARLIER turn than the
1440
+ // most recent "final" this client has seen — see
1441
+ // rtLastFinalGenerationRef's own doc comment for the real race this
1442
+ // closes. A message with no generation field at all (shouldn't
1443
+ // happen against a server running this fix, but a mismatched client/
1444
+ // server version pair during a rolling deploy could) is treated as
1445
+ // current rather than dropped — additive/backward-compatible, same
1446
+ // discipline every other wire-protocol addition in this codebase
1447
+ // follows.
1448
+ function isStaleRtMessage(msg) {
1449
+ const stale = typeof msg.generation === "number" && msg.generation < rtLastFinalGenerationRef.current;
1450
+ if (stale)
1451
+ rtLog("dropped stale message", { type: msg.type, messageGeneration: msg.generation, currentGeneration: rtLastFinalGenerationRef.current });
1452
+ return stale;
1453
+ }
1454
+ let audioChunkCount = 0;
847
1455
  ws.onmessage = (event) => {
848
1456
  if (typeof event.data !== "string")
849
1457
  return; // audio now arrives as base64 inside audio_chunk, not raw binary frames
@@ -852,14 +1460,35 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
852
1460
  setCaption(msg.text);
853
1461
  }
854
1462
  else if (msg.type === "final") {
1463
+ rtLog("final transcript", { text: msg.text, generation: msg.generation });
1464
+ rtLastFinalGenerationRef.current = typeof msg.generation === "number" ? msg.generation : 0;
855
1465
  archiveCurrentExchange(); // the previous turn's pair is complete — move it into history before this one starts overwriting caption/answer
856
1466
  setCaption(msg.text);
1467
+ // Without this, `answer` still held the PREVIOUS turn's reply
1468
+ // text at the moment this new turn's own reply — if it ever
1469
+ // arrives — would overwrite it. Usually invisible (the previous
1470
+ // reply lands well before the next "final"), but a barge-in can
1471
+ // supersede an in-flight turn before its reply ever arrives (see
1472
+ // realtime-server.ts's onStep generation check) — with no reply
1473
+ // ever coming for THIS caption, the stale previous-turn answer
1474
+ // sat there and got archived alongside the wrong question on the
1475
+ // NEXT final, showing as a mismatched or duplicated-looking
1476
+ // reply. Found live: two turns' worth of the same fallback error
1477
+ // text ("Something went wrong on my end") appearing back to back
1478
+ // with only one visible question between them. Clearing to null
1479
+ // here means an abandoned turn now correctly archives with NO
1480
+ // reply bubble (archiveText skips empty text) instead of someone
1481
+ // else's.
1482
+ setAnswer(null);
857
1483
  setRtStatus("rt-thinking");
858
1484
  armThinkingWatchdog();
859
1485
  }
860
1486
  else if (msg.type === "verb") {
1487
+ if (isStaleRtMessage(msg))
1488
+ return; // belongs to a turn a later "final" already superseded
1489
+ rtLog("verb received", { verb: msg.verb?.verb, generation: msg.generation });
861
1490
  const parsedStep = (0, core_1.safeParseVerbResponse)(msg.verb);
862
- if (parsedStep && !core_1.TERMINAL_VERBS.has(parsedStep.verb)) {
1491
+ if (parsedStep && !(0, core_1.isTerminalVerb)(parsedStep)) {
863
1492
  // A continuing agent-loop step (click/fill/read/call_tool) —
864
1493
  // the turn isn't over: execute it for real and report the
865
1494
  // result back so the server can decide the next step, instead
@@ -870,7 +1499,39 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
870
1499
  // — the server's loop stays quiet between steps on purpose,
871
1500
  // to keep it fast.
872
1501
  setAnswer(summarizeVerbForHistory(msg.verb));
873
- void (0, verb_executor_1.executeToolStep)(msg.verb, pathnameRef.current, liveMapRef.current).then((result) => {
1502
+ setLoopWorking(true);
1503
+ // Real, live-found bug: armThinkingWatchdog() only ever fired
1504
+ // once, on the turn's own "final" message, giving the WHOLE
1505
+ // multi-step turn one shared 20s budget — Executor + Planner +
1506
+ // Critic for step 1, then the same again for step 2, and so
1507
+ // on. Directly measured live: one single non-terminal step's
1508
+ // own Executor+Planner+Critic chain alone took ~14s (11.5s +
1509
+ // 1.5s + 1.1s) — a real, multi-step goal needing two or three
1510
+ // such steps blows straight through 20s even though each
1511
+ // individual step is proof of genuine progress, not a stall.
1512
+ // Re-arming here — once per continuing step, not once per
1513
+ // turn — gives every step its own fresh budget, so the
1514
+ // watchdog only ever fires on a step that's ACTUALLY stuck
1515
+ // (no verb/final/speaking_start arriving at all), matching
1516
+ // what its own fallback message ("taking longer than
1517
+ // expected") is supposed to mean.
1518
+ armThinkingWatchdog();
1519
+ // A FRESH scan, not the turn's starting liveMapRef snapshot —
1520
+ // real, live-found bug: a step in THIS SAME multi-step turn
1521
+ // (a "click New Agent" that opens a modal) can reveal DOM a
1522
+ // later step (a "fill" targeting the modal's own input) needs
1523
+ // to find, and liveMapRef is deliberately frozen once per
1524
+ // turn (see its own doc comment — that freeze exists to stop
1525
+ // a background rescan from shifting an id mid-flight during
1526
+ // ONE step's own round trip, not to survive across several
1527
+ // sequential steps that genuinely changed the page).
1528
+ // runTour() already does exactly this for its own steps, for
1529
+ // the identical reason. Without it, "click New Agent, then
1530
+ // type the name" reliably failed every time with "Could not
1531
+ // find that element on the page" — confirmed live, repeated
1532
+ // 5+ times in a row without ever recovering.
1533
+ const freshLiveMap = liveRegistryRef.current.getSnapshot().byId;
1534
+ void (0, verb_executor_1.executeToolStep)(msg.verb, pathnameRef.current, freshLiveMap, (route) => router.push(route), confirmToolCall).then((result) => {
874
1535
  if (ws.readyState === WebSocket.OPEN) {
875
1536
  ws.send(JSON.stringify({ type: "tool_result", observation: result?.observation ?? "no result" }));
876
1537
  }
@@ -881,11 +1542,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
881
1542
  handleVerb(msg.verb);
882
1543
  }
883
1544
  else if (msg.type === "speaking_start") {
1545
+ if (isStaleRtMessage(msg))
1546
+ return;
1547
+ rtLog("speaking start", { generation: msg.generation });
1548
+ audioChunkCount = 0;
884
1549
  disarmThinkingWatchdog();
885
1550
  rtAudioDoneArrivingRef.current = false;
886
1551
  setRtStatus("rt-speaking");
887
1552
  }
888
1553
  else if (msg.type === "audio_chunk") {
1554
+ if (isStaleRtMessage(msg))
1555
+ return; // the literal "two speakers" case — a chunk from an abandoned turn, already in flight when the barge-in landed
1556
+ audioChunkCount++;
889
1557
  const ctx = rtPlaybackCtxRef.current;
890
1558
  const gain = rtPlaybackGainRef.current;
891
1559
  if (!ctx || !gain)
@@ -918,6 +1586,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
918
1586
  };
919
1587
  }
920
1588
  else if (msg.type === "speaking_end" || msg.type === "turn_complete") {
1589
+ if (isStaleRtMessage(msg))
1590
+ return; // a newer turn's own speaking_end/turn_complete will arrive and resume listening correctly on its own
1591
+ rtLog(msg.type, { audioChunks: audioChunkCount, generation: msg.generation });
921
1592
  // turn_complete covers a verb with nothing spoken (a plain
922
1593
  // highlight/navigate/do often has no text) — no audio_chunk ever
923
1594
  // arrives for it, so rtScheduledSourcesRef is already empty and
@@ -927,10 +1598,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
927
1598
  maybeResumeListening();
928
1599
  }
929
1600
  else if (msg.type === "error") {
1601
+ rtLog("server error", { message: msg.message });
930
1602
  // Must actually unstick the turn, not just show the message —
931
1603
  // otherwise the mic never resumes and the session is stuck
932
1604
  // exactly the way a silently-dropped response used to leave it.
933
1605
  disarmThinkingWatchdog();
1606
+ setLoopWorking(false);
934
1607
  setAnswer(msg.message ?? "Something went wrong.");
935
1608
  if (touringRef.current) {
936
1609
  // A tour step's own speakStreamed() failed server-side (see
@@ -950,10 +1623,12 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
950
1623
  }
951
1624
  };
952
1625
  ws.onerror = () => {
1626
+ rtLog("connection error");
953
1627
  setAnswer("Couldn't connect to the realtime voice service.");
954
1628
  endRealtime();
955
1629
  };
956
- ws.onclose = () => {
1630
+ ws.onclose = (closeEvent) => {
1631
+ rtLog("connection closed", { code: closeEvent.code, reason: closeEvent.reason, wasIdle: rtStateRef.current === "idle" });
957
1632
  if (rtStateRef.current !== "idle")
958
1633
  endRealtime();
959
1634
  };
@@ -962,15 +1637,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
962
1637
  setAnswer("Couldn't access the microphone — check your browser's permission for this site.");
963
1638
  setRtStatus("idle");
964
1639
  rtStartingRef.current = false;
1640
+ typedPlaybackSuspendedRef.current = false; // the call never actually started — don't leave typed replies permanently silenced
965
1641
  }
966
1642
  }
967
1643
  function endRealtime() {
1644
+ rtLog("ending realtime call", { statusAtEnd: rtStateRef.current });
968
1645
  if (rtThinkingWatchdogRef.current) {
969
1646
  clearTimeout(rtThinkingWatchdogRef.current);
970
1647
  rtThinkingWatchdogRef.current = null;
971
1648
  }
972
1649
  rtStartingRef.current = false;
973
1650
  stopTypedPlayback();
1651
+ typedPlaybackSuspendedRef.current = false; // typed replies work normally again once no live call can race them
974
1652
  rtSocketRef.current?.close();
975
1653
  rtSocketRef.current = null;
976
1654
  rtCleanupRef.current?.();
@@ -979,6 +1657,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
979
1657
  setRtSpeakerMuted(false);
980
1658
  setCaption("");
981
1659
  setRtStatus("idle");
1660
+ setLoopWorking(false); // defensive — a connection dropping mid-loop must never leave the "still working" indicator stuck on
982
1661
  tourGenerationRef.current++; // cancel an in-progress tour rather than leaving it stuck waiting to resume rt-listening
983
1662
  touringRef.current = false;
984
1663
  setTourStep(null);
@@ -1013,7 +1692,17 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
1013
1692
  "rt-speaking": "Speaking…",
1014
1693
  };
1015
1694
  return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.length > 0 && ((0, jsx_runtime_1.jsxs)("button", { type: "button", className: "cairn-history-toggle", onClick: () => setHistoryExpanded((v) => !v), "aria-expanded": historyExpanded, children: [historyExpanded ? (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronUp, { size: 12 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronDown, { size: 12 }), historyExpanded ? "Hide earlier" : `${transcript.length} earlier`] })), historyExpanded &&
1016
- transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: renderCaptionWords(answer) })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
1695
+ transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-bubble-text", children: [renderCaptionWords(answer), loopWorking && (
1696
+ // Real, live-reported gap this closes: this bubble
1697
+ // used to go static the instant a continuing step's
1698
+ // own progress text was shown ("Typing earbuds into
1699
+ // the search box"), with nothing telling the user
1700
+ // the agent was still actively working for however
1701
+ // long the next real LLM call took. Appended inline
1702
+ // (not swapped in place of the text, which the
1703
+ // no-answer-yet case below does) so the progress
1704
+ // text stays legible while still showing motion.
1705
+ (0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking cairn-thinking-inline", "aria-label": "Still working", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
1017
1706
  e.preventDefault();
1018
1707
  const trimmed = question.trim();
1019
1708
  if (trimmed)
@@ -1021,6 +1710,75 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
1021
1710
  }, children: (0, jsx_runtime_1.jsxs)("div", { className: "cairn-input-row", children: [(0, jsx_runtime_1.jsx)("input", { value: question, onChange: (e) => setQuestion(e.target.value), placeholder: "What do you need help with?", "aria-label": `Ask ${persona} a question`, disabled: recording || touring, autoFocus: true }), realtimeUrl && micSupported && ((0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn", "aria-label": "Start realtime conversation", onClick: () => void startRealtime(), disabled: busy || recording, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneCall, { size: 16 }) })), transcribeEndpoint && micSupported && ((0, jsx_runtime_1.jsx)("button", { type: "button", className: recording ? "cairn-icon-btn cairn-icon-btn-recording" : "cairn-icon-btn", "aria-label": recording ? "Stop recording" : "Ask by voice", onClick: () => (recording ? stopRecording() : void startRecording()), disabled: touring, children: recording ? (0, jsx_runtime_1.jsx)(lucide_react_1.Square, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) })), (0, jsx_runtime_1.jsx)("button", { type: "submit", className: "cairn-send", "aria-label": "Send", disabled: !question.trim() || busy || recording, children: asking ? (0, jsx_runtime_1.jsx)(lucide_react_1.Loader2, { size: 16, className: "cairn-spin" }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Send, { size: 16 }) })] }) }))] }))] }));
1022
1711
  }
1023
1712
  const MAX_HISTORY_TURNS = 8; // 4 exchanges — matches the same cap the realtime relay uses server-side
1713
+ exports.CONVERSATION_STORAGE_KEY = "cairn:conversation:v1";
1714
+ /**
1715
+ * Real, live-reported bug this closes: a host app's own mutation handler
1716
+ * calling a real `window.location.reload()` — a common, entirely valid
1717
+ * pattern; this SDK's own demo app uses it after several real actions,
1718
+ * e.g. moving a kanban card — tears down the ENTIRE React tree, this
1719
+ * widget included. Every bit of conversation state (the visible
1720
+ * transcript, the current exchange, even whether the panel was open)
1721
+ * reset to nothing, making a real, in-progress conversation look like it
1722
+ * had simply ended the instant a host page happened to reload — even
1723
+ * though nothing about the CONVERSATION itself was actually over.
1724
+ * `sessionStorage`, not `localStorage`, is deliberate: it survives
1725
+ * exactly a reload/navigation within the same tab — the real scope of
1726
+ * "this conversation" — and clears itself once the tab/window actually
1727
+ * closes, never lingering into an unrelated later visit the way
1728
+ * `localStorage` would.
1729
+ */
1730
+ function loadPersistedConversation() {
1731
+ if (typeof window === "undefined")
1732
+ return null;
1733
+ try {
1734
+ const raw = window.sessionStorage.getItem(exports.CONVERSATION_STORAGE_KEY);
1735
+ if (!raw)
1736
+ return null;
1737
+ const parsed = JSON.parse(raw);
1738
+ if (!parsed || typeof parsed !== "object")
1739
+ return null;
1740
+ const transcript = Array.isArray(parsed.transcript)
1741
+ ? parsed.transcript.filter((t) => !!t && typeof t === "object" && typeof t.id === "number" && typeof t.text === "string" && (t.role === "user" || t.role === "agent"))
1742
+ : [];
1743
+ return {
1744
+ transcript,
1745
+ lastQuestion: typeof parsed.lastQuestion === "string" ? parsed.lastQuestion : null,
1746
+ answer: typeof parsed.answer === "string" ? parsed.answer : null,
1747
+ open: Boolean(parsed.open),
1748
+ };
1749
+ }
1750
+ catch {
1751
+ return null; // private browsing, quota, or a genuinely corrupt value — never crash the widget over this
1752
+ }
1753
+ }
1754
+ function savePersistedConversation(data) {
1755
+ if (typeof window === "undefined")
1756
+ return;
1757
+ try {
1758
+ window.sessionStorage.setItem(exports.CONVERSATION_STORAGE_KEY, JSON.stringify(data));
1759
+ }
1760
+ catch {
1761
+ // Storage unavailable/full — the conversation just won't survive a reload this time, never worth crashing the widget over.
1762
+ }
1763
+ }
1764
+ /**
1765
+ * Rebuilds a real seed for `historyRef` (what gets sent to the model on
1766
+ * the NEXT typed turn) from a restored transcript — deliberately derived
1767
+ * from the same, already-persisted `transcript` rather than separately
1768
+ * persisting `historyRef`'s own shape: one real source of truth for "what
1769
+ * was actually said," not two that could quietly drift apart. Capped the
1770
+ * same way every other history array in this file already is.
1771
+ */
1772
+ function reconstructHistoryFromPersisted(persisted) {
1773
+ if (!persisted)
1774
+ return [];
1775
+ const fromTranscript = persisted.transcript.map((t) => ({ role: t.role === "agent" ? "assistant" : "user", text: t.text }));
1776
+ const live = [
1777
+ ...(persisted.lastQuestion ? [{ role: "user", text: persisted.lastQuestion }] : []),
1778
+ ...(persisted.answer ? [{ role: "assistant", text: persisted.answer }] : []),
1779
+ ];
1780
+ return [...fromTranscript, ...live].slice(-MAX_HISTORY_TURNS);
1781
+ }
1024
1782
  /** Best-effort text form of a raw (unvalidated) verb response for the
1025
1783
  * conversation-history log — not shown to the user, just fed back to the
1026
1784
  * model on later turns. Deliberately loose/defensive rather than a full
@@ -1051,6 +1809,12 @@ function summarizeVerbForHistory(raw) {
1051
1809
  return `(read ${String(v.target)})`;
1052
1810
  case "call_tool":
1053
1811
  return `(called ${String(v.name)})`;
1812
+ case "drag":
1813
+ return `(dragged ${String(v.target)} to ${String(v.to)})`;
1814
+ case "select":
1815
+ return `(selected "${String(v.value)}" in ${String(v.target)})`;
1816
+ case "key":
1817
+ return `(pressed ${String(v.key)}${v.target ? ` on ${String(v.target)}` : ""})`;
1054
1818
  case "batch":
1055
1819
  return Array.isArray(v.actions) ? `(${v.actions.length} steps: ${v.actions.map((a) => a.verb).join(", ")})` : "(batch)";
1056
1820
  default:
@@ -1070,25 +1834,32 @@ function renderCaptionWords(text) {
1070
1834
  const words = text.split(" ");
1071
1835
  return words.map((word, i) => ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-word", style: { animationDelay: `${Math.min(i * 55, 2800)}ms` }, children: [word, i < words.length - 1 ? " " : ""] }, i)));
1072
1836
  }
1837
+ // "Waybalance" — three real, irregular stones (an ellipse plus a smaller
1838
+ // bump, not a rectangle), stacked slightly off-center the way a hiker
1839
+ // actually balances a trail cairn, instead of the perfectly centered flat
1840
+ // bars this replaced. Same mark as docs/images/logo.svg and site/index.html's
1841
+ // nav badge, just currentColor here so it inherits the button's own color.
1073
1842
  function CairnMark() {
1074
- return ((0, jsx_runtime_1.jsxs)("svg", { width: "20", height: "20", viewBox: "0 0 20 20", fill: "none", "aria-hidden": "true", children: [(0, jsx_runtime_1.jsx)("rect", { x: "7", y: "12.5", width: "6", height: "2.6", rx: "0.5", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("rect", { x: "4.5", y: "8.5", width: "11", height: "2.6", rx: "0.5", fill: "currentColor", opacity: "0.75" }), (0, jsx_runtime_1.jsx)("rect", { x: "8.2", y: "4.5", width: "3.6", height: "2.6", rx: "0.5", fill: "currentColor", opacity: "0.5" })] }));
1843
+ return ((0, jsx_runtime_1.jsxs)("svg", { width: "20", height: "20", viewBox: "0 0 20 20", fill: "none", "aria-hidden": "true", children: [(0, jsx_runtime_1.jsx)("ellipse", { cx: "10", cy: "14.6", rx: "6.1", ry: "2.3", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "6.3", cy: "14.1", rx: "2.5", ry: "1.6", fill: "currentColor" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "9.1", cy: "10.4", rx: "4.3", ry: "2", fill: "currentColor", opacity: "0.82" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "6.2", cy: "10", rx: "1.7", ry: "1.2", fill: "currentColor", opacity: "0.82" }), (0, jsx_runtime_1.jsx)("ellipse", { cx: "11.7", cy: "6.4", rx: "2.6", ry: "1.6", fill: "currentColor", opacity: "0.6" })] }));
1844
+ }
1845
+ // Real-time lifecycle logging — every message received, every decision made
1846
+ // about it (played, spoken, dropped, and why), every state transition. Added
1847
+ // specifically so a live session's actual behavior is visible in the browser
1848
+ // console instead of only inferable from symptoms after the fact — every bug
1849
+ // found and fixed in this file today was diagnosed from screenshots and
1850
+ // terminal output because nothing like this existed before. `[cairn rt]` is
1851
+ // the tag to filter on. Deliberately excludes per-audio_chunk noise (dozens
1852
+ // of chunks per turn would flood the console) — chunk activity shows up as
1853
+ // a one-line count at speaking_end/turn_complete instead.
1854
+ function rtLog(event, details) {
1855
+ if (details)
1856
+ console.log("[cairn rt]", event, details);
1857
+ else
1858
+ console.log("[cairn rt]", event);
1075
1859
  }
1076
1860
  // ---------------------------------------------------------------------------
1077
1861
  // Audio helpers (real-time PCM16 capture — standard Web Audio API patterns)
1078
1862
  // ---------------------------------------------------------------------------
1079
- // Heuristic energy gate for barge-in: real speech into a laptop/phone mic
1080
- // typically sits well above this; normal room noise and the mic's own
1081
- // noise floor typically sit below it. Not calibrated against real hardware
1082
- // in this environment (no live mic here) — reasonable starting point, may
1083
- // need tuning against a real device if it proves too trigger-happy or too
1084
- // insensitive in practice.
1085
- const BARGE_IN_RMS_THRESHOLD = 0.02;
1086
- function computeRms(channelData) {
1087
- let sumSquares = 0;
1088
- for (let i = 0; i < channelData.length; i++)
1089
- sumSquares += channelData[i] * channelData[i];
1090
- return Math.sqrt(sumSquares / channelData.length);
1091
- }
1092
1863
  function downsampleTo16k(input, inputSampleRate) {
1093
1864
  const targetRate = 16000;
1094
1865
  if (inputSampleRate === targetRate)
@@ -1300,6 +2071,10 @@ const COPILOT_STYLES = `
1300
2071
  gap: 4px;
1301
2072
  padding: 2px 0;
1302
2073
  }
2074
+ .cairn-thinking-inline {
2075
+ margin-left: 6px;
2076
+ vertical-align: middle;
2077
+ }
1303
2078
  .cairn-thinking-dot {
1304
2079
  width: 5px;
1305
2080
  height: 5px;