@cairnvibe/sdk 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/LICENSE +21 -0
  2. package/dist/cairn-widget.js +228 -0
  3. package/dist/context-collector.d.ts +1 -0
  4. package/dist/context-collector.js +23 -0
  5. package/dist/dashboard-sqlite.d.ts +8 -0
  6. package/dist/dashboard-sqlite.js +50 -0
  7. package/dist/dashboard.d.ts +39 -0
  8. package/dist/dashboard.js +60 -0
  9. package/dist/element-ladder.d.ts +7 -0
  10. package/dist/element-ladder.js +60 -0
  11. package/dist/index.d.ts +31 -0
  12. package/dist/index.js +1069 -0
  13. package/dist/key-rotator.d.ts +7 -0
  14. package/dist/key-rotator.js +31 -0
  15. package/dist/package.json +1 -0
  16. package/dist/realtime-cli.d.ts +2 -0
  17. package/dist/realtime-cli.js +59 -0
  18. package/dist/realtime-server.d.ts +10 -0
  19. package/dist/realtime-server.js +291 -0
  20. package/dist/server.d.ts +95 -0
  21. package/dist/server.js +298 -0
  22. package/dist/speak-server.d.ts +16 -0
  23. package/dist/speak-server.js +41 -0
  24. package/dist/transcribe-server.d.ts +14 -0
  25. package/dist/transcribe-server.js +47 -0
  26. package/dist/tts-stream.d.ts +33 -0
  27. package/dist/tts-stream.js +124 -0
  28. package/dist/verb-executor.d.ts +17 -0
  29. package/dist/verb-executor.js +67 -0
  30. package/package.json +56 -0
  31. package/src/context-collector.ts +21 -0
  32. package/src/dashboard-sqlite.ts +52 -0
  33. package/src/dashboard.ts +82 -0
  34. package/src/element-ladder.ts +67 -0
  35. package/src/index.tsx +1250 -0
  36. package/src/key-rotator.ts +29 -0
  37. package/src/realtime-cli.ts +62 -0
  38. package/src/realtime-server.ts +342 -0
  39. package/src/server.ts +386 -0
  40. package/src/speak-server.ts +56 -0
  41. package/src/transcribe-server.ts +68 -0
  42. package/src/tts-stream.ts +140 -0
  43. package/src/verb-executor.ts +84 -0
  44. package/src/web-component.ts +1252 -0
@@ -0,0 +1,1252 @@
1
+ // Framework-agnostic delivery for the Cairn widget — a plain Web Component
2
+ // (Custom Element + Shadow DOM), usable via a single <script> tag on any
3
+ // page: Vue, Angular, Svelte, a static HTML file, or Next.js/React (which
4
+ // also gets the <Copilot/> wrapper in index.tsx, for convenience).
5
+ //
6
+ // Reuses the exact same framework-neutral engine the React widget does —
7
+ // executeVerbResponse, findElement/highlightElement, collectVisible — none
8
+ // of that changes here. What's different is *rendering*: real DOM nodes
9
+ // created once and mutated in place, not JSX re-rendered on every state
10
+ // change (a naive innerHTML-per-update approach would drop input focus on
11
+ // every keystroke — see ROADMAP.md for why this matters), and *state*:
12
+ // plain instance fields instead of React state/refs. One genuine
13
+ // simplification vanilla gets for free — no ref-vs-state split needed for
14
+ // "read the current value inside an async callback without a stale
15
+ // closure" the way React's rtStateRef/rtMicMutedRef/etc. existed purely to
16
+ // solve; a class field read inside any closure on `this` is always current.
17
+ //
18
+ // Full feature parity with <Copilot/> (index.tsx), including live realtime
19
+ // voice conversation (streaming TTS, barge-in) — ported directly from that
20
+ // file's implementation, same protocol, same audio pipeline, same
21
+ // barge-in heuristic. See ROADMAP.md Phase 1 for how this was staged (a
22
+ // typed-Q&A-only MVP shipped first, live-verified, before this).
23
+
24
+ import type { HistoryTurn as HistoryEntry, TourStep } from "@cairnvibe/core";
25
+ import { collectVisible } from "./context-collector";
26
+ import { findElement, highlightElement, logMiss, type MissContext } from "./element-ladder";
27
+ import { executeVerbResponse } from "./verb-executor";
28
+
29
+ type Status = "idle" | "asking" | "recording" | "rt-connecting" | "rt-listening" | "rt-thinking" | "rt-speaking";
30
+
31
+ const STATUS_LABEL: Record<Status, string> = {
32
+ idle: "",
33
+ asking: "Thinking…",
34
+ recording: "Listening — transcribing live…",
35
+ "rt-connecting": "Connecting…",
36
+ "rt-listening": "Listening…",
37
+ "rt-thinking": "Thinking…",
38
+ "rt-speaking": "Speaking…",
39
+ };
40
+
41
+ const MAX_HISTORY_TURNS = 8; // 4 exchanges — matches the realtime relay's own server-side cap
42
+
43
+ const STYLES = `
44
+ :host {
45
+ all: initial;
46
+ font: 13.5px/1.5 -apple-system, BlinkMacSystemFont, "SF Pro Text", system-ui, "Segoe UI", sans-serif;
47
+ color: #0b0d12;
48
+ }
49
+ * { box-sizing: border-box; }
50
+ @keyframes cairn-pulse {
51
+ 0%, 100% { box-shadow: 0 0 0 0 rgba(239, 68, 68, 0.4); }
52
+ 70% { box-shadow: 0 0 0 10px rgba(239, 68, 68, 0); }
53
+ }
54
+ @keyframes cairn-pulse-green {
55
+ 0%, 100% { box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.4); }
56
+ 70% { box-shadow: 0 0 0 10px rgba(16, 185, 129, 0); }
57
+ }
58
+ @keyframes cairn-pulse-indigo {
59
+ 0%, 100% { box-shadow: 0 0 0 0 rgba(99, 102, 241, 0.4); }
60
+ 70% { box-shadow: 0 0 0 10px rgba(99, 102, 241, 0); }
61
+ }
62
+ @keyframes cairn-spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } }
63
+ @keyframes cairn-rt-dot {
64
+ 0%, 100% { opacity: 0.5; transform: scale(0.85); }
65
+ 50% { opacity: 1; transform: scale(1.15); }
66
+ }
67
+ @keyframes cairn-bubble-in {
68
+ from { opacity: 0; transform: translateY(6px); }
69
+ to { opacity: 1; transform: translateY(0); }
70
+ }
71
+ @keyframes cairn-panel-in {
72
+ from { opacity: 0; transform: translateY(8px) scale(0.98); }
73
+ to { opacity: 1; transform: translateY(0) scale(1); }
74
+ }
75
+ @keyframes cairn-word-sweep {
76
+ 0% { opacity: 0.35; text-shadow: none; }
77
+ 35% { opacity: 1; color: #4f46e5; text-shadow: 0 0 10px rgba(99, 102, 241, 0.45); }
78
+ 100% { opacity: 1; color: inherit; text-shadow: none; }
79
+ }
80
+ @keyframes cairn-thinking-bounce {
81
+ 0%, 80%, 100% { opacity: 0.3; transform: translateY(0); }
82
+ 40% { opacity: 0.9; transform: translateY(-3px); }
83
+ }
84
+ .cairn-glow {
85
+ animation: cairn-pulse-indigo 1.1s ease-out 2;
86
+ outline: 2px solid #6366f1;
87
+ outline-offset: 3px;
88
+ border-radius: 8px;
89
+ }
90
+ .cairn-spin { animation: cairn-spin 0.8s linear infinite; }
91
+ @media (prefers-reduced-motion: reduce) {
92
+ .cairn-fab, .cairn-panel, .cairn-bubble, .cairn-word, .cairn-thinking-dot {
93
+ animation: none !important;
94
+ transition: none !important;
95
+ }
96
+ }
97
+
98
+ .cairn-fab {
99
+ position: fixed;
100
+ right: 20px;
101
+ bottom: 20px;
102
+ z-index: 2147483000;
103
+ width: 52px;
104
+ height: 52px;
105
+ border-radius: 999px;
106
+ border: none;
107
+ display: flex;
108
+ align-items: center;
109
+ justify-content: center;
110
+ background: #14151b;
111
+ color: white;
112
+ cursor: pointer;
113
+ box-shadow: 0 6px 20px rgba(0, 0, 0, 0.25);
114
+ transition: transform 0.15s ease, box-shadow 0.15s ease;
115
+ }
116
+ .cairn-fab:hover { transform: translateY(-1px); box-shadow: 0 8px 24px rgba(0, 0, 0, 0.3); }
117
+ .cairn-fab-speaking {
118
+ box-shadow: 0 0 0 4px rgba(16, 185, 129, 0.22), 0 6px 20px rgba(0, 0, 0, 0.25);
119
+ animation: cairn-pulse-green 1.2s ease-out infinite;
120
+ }
121
+
122
+ /* No shared background/border/shadow on the panel itself — every piece
123
+ below (title, bubbles, input, buttons) floats independently on its own
124
+ minimal glass, directly over the host page, the way a caption track
125
+ floats over a video rather than sitting in a drawn box. */
126
+ /* One unified card — title, conversation, and input all live inside the
127
+ same bounded, padded container instead of floating as independent
128
+ fixed-position pieces. That "everything floats separately" approach
129
+ kept producing new collisions (title vs input, send button vs the
130
+ close FAB) every time one piece's position changed; grouping them
131
+ under one panel with real internal spacing removes that whole class
132
+ of bug at the source. */
133
+ .cairn-panel {
134
+ position: fixed;
135
+ right: 20px;
136
+ bottom: 92px;
137
+ z-index: 2147483000;
138
+ width: min(340px, calc(100vw - 40px));
139
+ max-height: 480px;
140
+ overflow-y: auto;
141
+ overflow-x: hidden;
142
+ flex-direction: column;
143
+ gap: 14px;
144
+ padding: 18px;
145
+ background: rgba(255, 255, 255, 0.96);
146
+ backdrop-filter: blur(24px) saturate(160%);
147
+ -webkit-backdrop-filter: blur(24px) saturate(160%);
148
+ border-radius: 20px;
149
+ box-shadow: 0 20px 50px rgba(15, 15, 25, 0.16), 0 2px 8px rgba(15, 15, 25, 0.06);
150
+ display: none;
151
+ }
152
+ .cairn-panel.cairn-open { display: flex; animation: cairn-panel-in 0.2s cubic-bezier(0.16, 1, 0.3, 1); }
153
+ .cairn-panel::-webkit-scrollbar { width: 0; }
154
+
155
+ .cairn-stack { display: flex; flex-direction: column; gap: 10px; }
156
+ .cairn-bubble {
157
+ max-width: 92%;
158
+ font-size: 13.5px;
159
+ line-height: 1.5;
160
+ color: #0b0d12;
161
+ animation: cairn-bubble-in 0.2s ease-out;
162
+ }
163
+ .cairn-bubble-user {
164
+ align-self: flex-end;
165
+ text-align: right;
166
+ color: #33384a;
167
+ }
168
+ .cairn-bubble-agent {
169
+ align-self: flex-start;
170
+ display: flex;
171
+ flex-direction: column;
172
+ gap: 4px;
173
+ }
174
+ .cairn-bubble-text { white-space: pre-wrap; }
175
+ .cairn-word { display: inline-block; animation: cairn-word-sweep 0.4s ease forwards; }
176
+ .cairn-chip {
177
+ align-self: flex-start;
178
+ font-size: 10.5px;
179
+ font-weight: 700;
180
+ letter-spacing: 0.07em;
181
+ text-transform: uppercase;
182
+ color: rgba(11, 13, 18, 0.48);
183
+ }
184
+ .cairn-thinking { display: inline-flex; gap: 4px; padding: 2px 0; }
185
+ .cairn-thinking-dot {
186
+ width: 5px;
187
+ height: 5px;
188
+ border-radius: 999px;
189
+ background: rgba(11, 13, 18, 0.4);
190
+ animation: cairn-thinking-bounce 1.1s ease-in-out infinite;
191
+ }
192
+ .cairn-thinking-dot:nth-child(2) { animation-delay: 0.15s; }
193
+ .cairn-thinking-dot:nth-child(3) { animation-delay: 0.3s; }
194
+
195
+ .cairn-input-row { display: flex; gap: 7px; align-items: center; }
196
+ .cairn-input-row input {
197
+ flex: 1;
198
+ min-width: 0;
199
+ border: none;
200
+ border-radius: 999px;
201
+ padding: 10px 14px;
202
+ font: inherit;
203
+ background: rgba(11, 13, 18, 0.045);
204
+ color: #0b0d12;
205
+ transition: background 0.15s ease, box-shadow 0.15s ease;
206
+ }
207
+ .cairn-input-row input::placeholder { color: rgba(11, 13, 18, 0.4); }
208
+ .cairn-input-row input:disabled { opacity: 0.55; }
209
+ .cairn-input-row input:focus {
210
+ outline: none;
211
+ background: rgba(11, 13, 18, 0.06);
212
+ box-shadow: 0 0 0 3px rgba(99, 102, 241, 0.16);
213
+ }
214
+ .cairn-icon-btn, .cairn-send {
215
+ flex-shrink: 0;
216
+ width: 36px;
217
+ height: 36px;
218
+ display: flex;
219
+ align-items: center;
220
+ justify-content: center;
221
+ border-radius: 999px;
222
+ border: none;
223
+ background: rgba(11, 13, 18, 0.045);
224
+ color: #33384a;
225
+ cursor: pointer;
226
+ transition: background 0.15s ease, transform 0.15s ease;
227
+ }
228
+ .cairn-icon-btn:hover { background: rgba(11, 13, 18, 0.09); transform: translateY(-1px); }
229
+ .cairn-send {
230
+ border: none;
231
+ background: #14151b;
232
+ color: white;
233
+ box-shadow: 0 4px 14px rgba(0, 0, 0, 0.25);
234
+ }
235
+ .cairn-send:hover:not(:disabled) { transform: translateY(-1px); }
236
+ .cairn-send:disabled, .cairn-icon-btn:disabled { opacity: 0.5; cursor: not-allowed; box-shadow: none; }
237
+ .cairn-icon-btn-recording {
238
+ background: #ef4444;
239
+ border-color: #ef4444;
240
+ color: white;
241
+ animation: cairn-pulse 1.4s ease-out infinite;
242
+ }
243
+ .cairn-icon-btn-speaking {
244
+ background: #10b981;
245
+ border-color: #10b981;
246
+ color: white;
247
+ animation: cairn-pulse-green 1.2s ease-out infinite;
248
+ }
249
+ .cairn-icon-btn-end { background: #ef4444; border-color: #ef4444; color: white; }
250
+ .cairn-rt-bar {
251
+ display: flex;
252
+ align-items: center;
253
+ gap: 8px;
254
+ padding: 8px 12px;
255
+ border-radius: 999px;
256
+ background: rgba(11, 13, 18, 0.045);
257
+ }
258
+ .cairn-rt-dot {
259
+ width: 8px;
260
+ height: 8px;
261
+ border-radius: 999px;
262
+ background: #6366f1;
263
+ animation: cairn-rt-dot 1.2s ease-in-out infinite;
264
+ flex-shrink: 0;
265
+ }
266
+ .cairn-rt-dot-rt-speaking { background: #10b981; }
267
+ .cairn-rt-dot-rt-thinking { background: #f59e0b; }
268
+ .cairn-rt-label { flex: 1; font-size: 12.5px; color: #33384a; }
269
+ .cairn-rt-controls { display: flex; gap: 6px; }
270
+ `;
271
+
272
+ const SEND_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="m22 2-7 20-4-9-9-4Z"/><path d="M22 2 11 13"/></svg>`;
273
+ const MIC_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 2a3 3 0 0 0-3 3v6a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z"/><path d="M19 10v1a7 7 0 0 1-14 0v-1"/><line x1="12" x2="12" y1="18" y2="22"/></svg>`;
274
+ const MIC_OFF_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><line x1="2" y1="2" x2="22" y2="22"/><path d="M9 9v3a3 3 0 0 0 5.12 2.12M15 9.34V5a3 3 0 0 0-5.94-.6"/><path d="M17 16.95A7 7 0 0 1 5 12v-2m14 0v2a7 7 0 0 1-.11 1.23"/><line x1="12" y1="19" x2="12" y2="22"/></svg>`;
275
+ const SQUARE_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="14" height="14" x="5" y="5" rx="2"/></svg>`;
276
+ const SPINNER_ICON = `<svg class="cairn-spin" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M21 12a9 9 0 1 1-6.219-8.56"/></svg>`;
277
+ const CLOSE_ICON = `<svg width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 6 6 18"/><path d="m6 6 12 12"/></svg>`;
278
+ const MARK_ICON = `<svg width="20" height="20" viewBox="0 0 20 20" fill="none" aria-hidden="true"><rect x="7" y="12.5" width="6" height="2.6" rx="0.5" fill="currentColor"/><rect x="4.5" y="8.5" width="11" height="2.6" rx="0.5" fill="currentColor" opacity="0.75"/><rect x="8.2" y="4.5" width="3.6" height="2.6" rx="0.5" fill="currentColor" opacity="0.5"/></svg>`;
279
+ const PHONE_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M13.832 16.568a1 1 0 0 0 1.213-.303l.355-.465A2 2 0 0 1 17 15h3a2 2 0 0 1 2 2v3a2 2 0 0 1-2 2A18 18 0 0 1 2 4a2 2 0 0 1 2-2h3a2 2 0 0 1 2 2v3a2 2 0 0 1-.8 1.6l-.468.351a1 1 0 0 0-.292 1.233 14 14 0 0 0 6.392 6.384"/></svg>`;
280
+ const PHONE_OFF_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M13.83 16.57a1 1 0 0 0 1.21-.3l.36-.47A2 2 0 0 1 17 15h3a2 2 0 0 1 2 2v3a2 2 0 0 1-2 2A17.87 17.87 0 0 1 9.3 17.6"/><path d="M4.27 5.34C3.5 6.44 4 8 4 8a17.9 17.9 0 0 0 2.14 6.6"/><path d="M2 2v0"/><line x1="2" y1="2" x2="22" y2="22"/></svg>`;
281
+ const VOLUME_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5"/><path d="M15.54 8.46a5 5 0 0 1 0 7.07"/><path d="M19.07 4.93a10 10 0 0 1 0 14.14"/></svg>`;
282
+ const VOLUME_OFF_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5"/><line x1="22" y1="9" x2="16" y2="15"/><line x1="16" y1="9" x2="22" y2="15"/></svg>`;
283
+
284
+ /**
285
+ * `<cairn-widget endpoint="/api/copilot" persona="Cairn" realtime-url="ws://..." ...>`
286
+ * — see README.md's "Voice & conversation" / install sections for the full
287
+ * attribute list and framework-specific examples.
288
+ */
289
+ export class CairnWidgetElement extends HTMLElement {
290
+ private shadow!: ShadowRoot;
291
+ private fab!: HTMLButtonElement;
292
+ private panel!: HTMLDivElement;
293
+ private stackEl!: HTMLDivElement;
294
+ private userBubbleEl!: HTMLDivElement;
295
+ private agentBubbleEl!: HTMLDivElement;
296
+ private chipEl!: HTMLSpanElement;
297
+ private agentTextEl!: HTMLSpanElement;
298
+ private thinkingEl!: HTMLSpanElement;
299
+ private formEl!: HTMLFormElement;
300
+ private inputEl!: HTMLInputElement;
301
+ private sendBtn!: HTMLButtonElement;
302
+ private micBtn: HTMLButtonElement | null = null;
303
+ private phoneBtn: HTMLButtonElement | null = null;
304
+ private rtBar!: HTMLDivElement;
305
+ private rtDot!: HTMLSpanElement;
306
+ private rtLabel!: HTMLSpanElement;
307
+ private rtMicBtn!: HTMLButtonElement;
308
+ private rtSpeakerBtn!: HTMLButtonElement;
309
+ private rtEndBtn!: HTMLButtonElement;
310
+
311
+ private isOpen = false;
312
+ private status: Status = "idle";
313
+ private recording = false;
314
+ private touringActive = false;
315
+ private caption = "";
316
+ private answer: string | null = null;
317
+ // The user's own last question, shown as its own floating caption bubble
318
+ // alongside the agent's — set once per ask() call, not cleared on
319
+ // completion, so the exchange stays paired on screen the way a caption
320
+ // track shows the current line, not a scrolling transcript.
321
+ private lastQuestion: string | null = null;
322
+ private tourGeneration = 0;
323
+ private mediaRecorder: MediaRecorder | null = null;
324
+ private audioChunks: Blob[] = [];
325
+ private transcribeInFlight = false;
326
+ private activeAudio: HTMLAudioElement | null = null;
327
+ /** Conversation memory for the typed/mic path — the realtime path keeps its own history server-side (that connection is already stateful). */
328
+ private history: HistoryEntry[] = [];
329
+
330
+ // --- realtime voice state ------------------------------------------------
331
+ private rtSocket: WebSocket | null = null;
332
+ private rtCleanup: (() => void) | null = null;
333
+ private rtMicMuted = false;
334
+ private rtSpeakerMuted = false;
335
+ private rtStarting = false; // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
336
+ private rtPlaybackCtx: AudioContext | null = null;
337
+ private rtPlaybackGain: GainNode | null = null;
338
+ private rtNextPlayTime = 0;
339
+ private rtScheduledSources: AudioBufferSourceNode[] = [];
340
+ // Watchdog for the "rt-thinking" state: started on every "final"
341
+ // transcript, cleared the moment the server responds with anything for
342
+ // that turn (verb/speaking_start/speaking_end/turn_complete/error). If it
343
+ // ever fires, the server went silent for this turn — force the mic back
344
+ // to listening instead of leaving the session stuck showing "Thinking…"
345
+ // forever with no way to speak again short of ending the call.
346
+ private rtThinkingWatchdog: ReturnType<typeof setTimeout> | null = null;
347
+ /** True once the server says no more audio_chunks are coming for the current turn — listening only resumes once this AND every scheduled chunk has actually finished playing. */
348
+ private rtAudioDoneArriving = true;
349
+ /** Resolver for "this tour step's audio has fully finished playing" when narrating over an already-open realtime session. */
350
+ private rtTourAudioDoneResolve: (() => void) | null = null;
351
+
352
+ static get observedAttributes() {
353
+ return ["persona"];
354
+ }
355
+
356
+ connectedCallback() {
357
+ if (this.shadow) return; // already connected once — don't rebuild on re-attach
358
+ this.shadow = this.attachShadow({ mode: "open" });
359
+ this.buildDom();
360
+ }
361
+
362
+ attributeChangedCallback(name: string) {
363
+ if (name === "persona" && this.panel) this.panel.setAttribute("aria-label", `${this.persona} help panel`);
364
+ }
365
+
366
+ // --- attributes -----------------------------------------------------
367
+ private get endpoint(): string { return this.getAttribute("endpoint") ?? "/api/copilot"; }
368
+ private get speakEndpoint(): string | null { return this.getAttribute("speak-endpoint"); }
369
+ private get transcribeEndpoint(): string | null { return this.getAttribute("transcribe-endpoint"); }
370
+ private get reportMissesEndpoint(): string | null { return this.getAttribute("report-misses-endpoint"); }
371
+ private get realtimeUrl(): string | null { return this.getAttribute("realtime-url"); }
372
+ private get persona(): string { return this.getAttribute("persona") ?? "Cairn"; }
373
+ private get registeredActions(): string[] {
374
+ return (this.getAttribute("registered-actions") ?? "").split(",").map((a) => a.trim()).filter(Boolean);
375
+ }
376
+
377
+ private get realtimeActive(): boolean {
378
+ return this.status.startsWith("rt-");
379
+ }
380
+
381
+ private get busy(): boolean {
382
+ return this.status === "asking" || this.status === "rt-thinking" || this.touringActive;
383
+ }
384
+
385
+ private micSupported(): boolean {
386
+ return typeof navigator !== "undefined" && !!navigator.mediaDevices?.getUserMedia && typeof MediaRecorder !== "undefined";
387
+ }
388
+
389
+ // --- DOM construction (once) -----------------------------------------
390
+ private buildDom() {
391
+ const style = document.createElement("style");
392
+ style.textContent = STYLES;
393
+ this.shadow.appendChild(style);
394
+
395
+ this.fab = document.createElement("button");
396
+ this.fab.className = "cairn-fab";
397
+ this.fab.innerHTML = MARK_ICON;
398
+ this.fab.setAttribute("aria-label", `Open ${this.persona} help`);
399
+ this.fab.addEventListener("click", () => this.toggleOpen());
400
+ this.shadow.appendChild(this.fab);
401
+
402
+ this.panel = document.createElement("div");
403
+ this.panel.className = "cairn-panel";
404
+ this.panel.setAttribute("role", "dialog");
405
+ this.panel.setAttribute("aria-label", `${this.persona} help panel`);
406
+
407
+ this.stackEl = document.createElement("div");
408
+ this.stackEl.className = "cairn-stack";
409
+ this.stackEl.style.display = "none";
410
+
411
+ this.userBubbleEl = document.createElement("div");
412
+ this.userBubbleEl.className = "cairn-bubble cairn-bubble-user";
413
+ this.userBubbleEl.style.display = "none";
414
+ this.stackEl.appendChild(this.userBubbleEl);
415
+
416
+ this.agentBubbleEl = document.createElement("div");
417
+ this.agentBubbleEl.className = "cairn-bubble cairn-bubble-agent";
418
+ this.agentBubbleEl.style.display = "none";
419
+
420
+ this.chipEl = document.createElement("span");
421
+ this.chipEl.className = "cairn-chip";
422
+ this.chipEl.style.display = "none";
423
+ this.agentBubbleEl.appendChild(this.chipEl);
424
+
425
+ this.agentTextEl = document.createElement("span");
426
+ this.agentTextEl.className = "cairn-bubble-text";
427
+ this.agentBubbleEl.appendChild(this.agentTextEl);
428
+
429
+ this.thinkingEl = document.createElement("span");
430
+ this.thinkingEl.className = "cairn-thinking";
431
+ this.thinkingEl.setAttribute("aria-label", "Thinking");
432
+ this.thinkingEl.style.display = "none";
433
+ this.thinkingEl.innerHTML =
434
+ '<span class="cairn-thinking-dot"></span><span class="cairn-thinking-dot"></span><span class="cairn-thinking-dot"></span>';
435
+ this.agentBubbleEl.appendChild(this.thinkingEl);
436
+
437
+ this.stackEl.appendChild(this.agentBubbleEl);
438
+ this.panel.appendChild(this.stackEl);
439
+
440
+ // --- realtime control bar (shown instead of the form while a live call is active) ---
441
+ this.rtBar = document.createElement("div");
442
+ this.rtBar.className = "cairn-rt-bar";
443
+ this.rtBar.style.display = "none";
444
+
445
+ this.rtDot = document.createElement("span");
446
+ this.rtDot.className = "cairn-rt-dot";
447
+ this.rtBar.appendChild(this.rtDot);
448
+
449
+ this.rtLabel = document.createElement("span");
450
+ this.rtLabel.className = "cairn-rt-label";
451
+ this.rtBar.appendChild(this.rtLabel);
452
+
453
+ const rtControls = document.createElement("div");
454
+ rtControls.className = "cairn-rt-controls";
455
+
456
+ this.rtMicBtn = document.createElement("button");
457
+ this.rtMicBtn.type = "button";
458
+ this.rtMicBtn.className = "cairn-icon-btn";
459
+ this.rtMicBtn.innerHTML = MIC_ICON;
460
+ this.rtMicBtn.setAttribute("aria-label", "Mute microphone");
461
+ this.rtMicBtn.addEventListener("click", () => this.toggleRtMic());
462
+ rtControls.appendChild(this.rtMicBtn);
463
+
464
+ this.rtSpeakerBtn = document.createElement("button");
465
+ this.rtSpeakerBtn.type = "button";
466
+ this.rtSpeakerBtn.className = "cairn-icon-btn";
467
+ this.rtSpeakerBtn.innerHTML = VOLUME_ICON;
468
+ this.rtSpeakerBtn.setAttribute("aria-label", "Mute speaker");
469
+ this.rtSpeakerBtn.addEventListener("click", () => this.toggleRtSpeaker());
470
+ rtControls.appendChild(this.rtSpeakerBtn);
471
+
472
+ this.rtEndBtn = document.createElement("button");
473
+ this.rtEndBtn.type = "button";
474
+ this.rtEndBtn.className = "cairn-icon-btn cairn-icon-btn-end";
475
+ this.rtEndBtn.innerHTML = PHONE_OFF_ICON;
476
+ this.rtEndBtn.setAttribute("aria-label", "End conversation");
477
+ this.rtEndBtn.addEventListener("click", () => this.endRealtime());
478
+ rtControls.appendChild(this.rtEndBtn);
479
+
480
+ this.rtBar.appendChild(rtControls);
481
+ this.panel.appendChild(this.rtBar);
482
+
483
+ // --- typed question form ---
484
+ this.formEl = document.createElement("form");
485
+ const row = document.createElement("div");
486
+ row.className = "cairn-input-row";
487
+
488
+ this.inputEl = document.createElement("input");
489
+ this.inputEl.placeholder = "What do you need help with?";
490
+ this.inputEl.setAttribute("aria-label", `Ask ${this.persona} a question`);
491
+ // The send button's disabled state depends on whether there's text —
492
+ // native/uncontrolled input, so nothing re-evaluates that on its own
493
+ // without this: typing would leave "Send" permanently disabled from
494
+ // its initial (empty-input) state.
495
+ this.inputEl.addEventListener("input", () => this.updateBusyState());
496
+ row.appendChild(this.inputEl);
497
+
498
+ if (this.realtimeUrl && this.micSupported()) {
499
+ this.phoneBtn = document.createElement("button");
500
+ this.phoneBtn.type = "button";
501
+ this.phoneBtn.className = "cairn-icon-btn";
502
+ this.phoneBtn.innerHTML = PHONE_ICON;
503
+ this.phoneBtn.setAttribute("aria-label", "Start realtime conversation");
504
+ this.phoneBtn.addEventListener("click", () => void this.startRealtime());
505
+ row.appendChild(this.phoneBtn);
506
+ }
507
+
508
+ if (this.transcribeEndpoint && this.micSupported()) {
509
+ this.micBtn = document.createElement("button");
510
+ this.micBtn.type = "button";
511
+ this.micBtn.className = "cairn-icon-btn";
512
+ this.micBtn.innerHTML = MIC_ICON;
513
+ this.micBtn.setAttribute("aria-label", "Ask by voice");
514
+ this.micBtn.addEventListener("click", () => (this.recording ? this.stopRecording() : void this.startRecording()));
515
+ row.appendChild(this.micBtn);
516
+ }
517
+
518
+ this.sendBtn = document.createElement("button");
519
+ this.sendBtn.type = "submit";
520
+ this.sendBtn.className = "cairn-send";
521
+ this.sendBtn.innerHTML = SEND_ICON;
522
+ this.sendBtn.setAttribute("aria-label", "Send");
523
+ row.appendChild(this.sendBtn);
524
+
525
+ this.formEl.appendChild(row);
526
+ this.formEl.addEventListener("submit", (e) => {
527
+ e.preventDefault();
528
+ const q = this.inputEl.value.trim();
529
+ if (q) void this.ask(q);
530
+ });
531
+ this.panel.appendChild(this.formEl);
532
+
533
+ this.shadow.appendChild(this.panel);
534
+ this.render();
535
+ }
536
+
537
+ // --- open/close -------------------------------------------------------
538
+ private toggleOpen() {
539
+ this.isOpen = !this.isOpen;
540
+ this.panel.classList.toggle("cairn-open", this.isOpen);
541
+ this.fab.innerHTML = this.isOpen ? CLOSE_ICON : MARK_ICON;
542
+ this.fab.setAttribute("aria-label", this.isOpen ? `Close ${this.persona} help` : `Open ${this.persona} help`);
543
+ if (this.isOpen && !this.realtimeActive) this.inputEl.focus();
544
+ }
545
+
546
+ // --- rendering ------------------------------------------------------
547
+ /** Single place that reconciles all status-derived UI — called on every status/caption/touring change, targeted DOM mutation only (never rebuilds nodes, so focus/scroll position/etc. are never disturbed). */
548
+ private render() {
549
+ const realtimeActive = this.realtimeActive;
550
+
551
+ this.fab.classList.toggle("cairn-fab-speaking", this.status === "rt-speaking");
552
+
553
+ this.rtBar.style.display = realtimeActive ? "flex" : "none";
554
+ this.formEl.style.display = realtimeActive ? "none" : "block";
555
+
556
+ if (realtimeActive) {
557
+ this.rtDot.className = `cairn-rt-dot cairn-rt-dot-${this.status}`;
558
+ this.rtLabel.textContent = STATUS_LABEL[this.status];
559
+ const speaking = this.status === "rt-speaking";
560
+ this.rtMicBtn.className = speaking ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn";
561
+ this.rtSpeakerBtn.className = speaking ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn";
562
+ this.rtMicBtn.innerHTML = this.rtMicMuted ? MIC_OFF_ICON : MIC_ICON;
563
+ this.rtMicBtn.setAttribute("aria-label", this.rtMicMuted ? "Unmute microphone" : "Mute microphone");
564
+ this.rtSpeakerBtn.innerHTML = this.rtSpeakerMuted ? VOLUME_OFF_ICON : VOLUME_ICON;
565
+ this.rtSpeakerBtn.setAttribute("aria-label", this.rtSpeakerMuted ? "Unmute speaker" : "Mute speaker");
566
+ }
567
+
568
+ // `caption` is overloaded by design (see its setters elsewhere): during
569
+ // a tour it's a step-progress label ("Step 1 of 2"), not user speech,
570
+ // so it reads as a small chip over the agent's bubble instead. While
571
+ // actively recording or on a live realtime call it's the user's own
572
+ // live/last transcript, so it reads as the user's floating bubble;
573
+ // otherwise that slot falls back to the last typed question.
574
+ const tourChip = this.touringActive ? this.caption : "";
575
+ const userCaption = !this.touringActive && (this.recording || realtimeActive) ? this.caption : this.lastQuestion ?? "";
576
+ const showAgent = !!this.answer || this.busy;
577
+
578
+ this.stackEl.style.display = userCaption || showAgent ? "flex" : "none";
579
+
580
+ this.userBubbleEl.style.display = userCaption ? "block" : "none";
581
+ this.userBubbleEl.textContent = userCaption;
582
+
583
+ this.agentBubbleEl.style.display = showAgent ? "flex" : "none";
584
+ // Only the tour step counter shows here — generic realtime status
585
+ // (listening/thinking/speaking) already has its own place in the
586
+ // rt-bar below; showing it a second time here was a real duplication.
587
+ this.chipEl.style.display = tourChip ? "inline-block" : "none";
588
+ this.chipEl.textContent = tourChip;
589
+
590
+ if (this.answer) {
591
+ this.agentTextEl.style.display = "inline";
592
+ this.renderCaptionWords(this.agentTextEl, this.answer);
593
+ this.thinkingEl.style.display = "none";
594
+ } else {
595
+ this.agentTextEl.style.display = "none";
596
+ this.agentTextEl.textContent = "";
597
+ this.thinkingEl.style.display = this.busy ? "inline-flex" : "none";
598
+ }
599
+
600
+ if (this.phoneBtn) this.phoneBtn.disabled = this.busy || this.recording;
601
+ this.updateBusyState();
602
+ }
603
+
604
+ private setStatus(next: Status) {
605
+ this.status = next;
606
+ this.render();
607
+ }
608
+
609
+ private setCaption(text: string) {
610
+ this.caption = text;
611
+ this.render();
612
+ }
613
+
614
+ private updateBusyState() {
615
+ this.inputEl.disabled = this.recording || this.touringActive;
616
+ this.sendBtn.disabled = this.busy || this.recording || !this.inputEl.value.trim();
617
+ this.sendBtn.innerHTML = this.busy ? SPINNER_ICON : SEND_ICON;
618
+ if (this.micBtn) this.micBtn.disabled = this.touringActive;
619
+ }
620
+
621
+ private setAnswer(text: string | null) {
622
+ this.answer = text;
623
+ this.render();
624
+ }
625
+
626
+ /**
627
+ * Renders text as a sequence of spans that light up in order — a caption
628
+ * "sweep" that reads like the agent is speaking it, whether or not audio
629
+ * is actually playing right now. This is a pacing *estimate* (staggered
630
+ * by word position, capped so long answers don't take forever), not
631
+ * synced to real TTS word timestamps — Deepgram's streaming API doesn't
632
+ * hand those to the client today. Builds real DOM nodes with textContent
633
+ * (never innerHTML on the actual words) so LLM-produced text can never
634
+ * be interpreted as markup.
635
+ */
636
+ private renderCaptionWords(container: HTMLElement, text: string) {
637
+ container.innerHTML = "";
638
+ const words = text.split(" ");
639
+ words.forEach((word, i) => {
640
+ const span = document.createElement("span");
641
+ span.className = "cairn-word";
642
+ span.style.animationDelay = `${Math.min(i * 55, 2800)}ms`;
643
+ span.textContent = i < words.length - 1 ? word + " " : word;
644
+ container.appendChild(span);
645
+ });
646
+ }
647
+
648
+ private reportMiss(context: MissContext) {
649
+ logMiss(context);
650
+ const endpoint = this.reportMissesEndpoint;
651
+ if (endpoint) {
652
+ fetch(endpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(context) }).catch(() => {});
653
+ }
654
+ }
655
+
656
+ // --- ask / verb execution ------------------------------------------------
657
+ private async ask(question: string) {
658
+ this.setStatus("asking");
659
+ this.setAnswer(null);
660
+ this.lastQuestion = question;
661
+ this.inputEl.value = "";
662
+ this.render();
663
+ try {
664
+ const res = await fetch(this.endpoint, {
665
+ method: "POST",
666
+ headers: { "content-type": "application/json" },
667
+ body: JSON.stringify({ route: location.pathname, question, visible: collectVisible(), history: this.history }),
668
+ });
669
+ const data = await res.json().catch(() => null);
670
+ this.handleVerb(data);
671
+ // Unlike the realtime relay (one persistent connection, memory lives
672
+ // server-side), each of these POSTs is stateless — the widget itself
673
+ // is what remembers, and resends it above so the model has context
674
+ // for "the first one" / "do that instead" on the next question.
675
+ this.history = [
676
+ ...this.history,
677
+ { role: "user", text: question } satisfies HistoryEntry,
678
+ { role: "assistant", text: summarizeVerbForHistory(data) } satisfies HistoryEntry,
679
+ ].slice(-MAX_HISTORY_TURNS);
680
+ } catch {
681
+ this.setAnswer("Something went wrong reaching the help service — try again in a moment.");
682
+ } finally {
683
+ this.setStatus("idle");
684
+ }
685
+ }
686
+
687
+ private handleVerb(raw: unknown) {
688
+ executeVerbResponse(raw, location.pathname, {
689
+ onExplain: (text) => {
690
+ this.setAnswer(text);
691
+ if (!this.realtimeActive) void this.speak(text); // realtime mode gets audio over the socket instead
692
+ },
693
+ onNavigate: (route) => {
694
+ location.assign(route);
695
+ },
696
+ onMiss: (ctx) => this.reportMiss(ctx),
697
+ onDo: (action, target) => {
698
+ this.dispatchEvent(new CustomEvent("cairn-do", { detail: { action, target }, bubbles: true, composed: true }));
699
+ },
700
+ onTour: (steps) => void this.runTour(steps),
701
+ registeredActions: this.registeredActions,
702
+ });
703
+ }
704
+
705
+ /**
706
+ * Walks a "tour" verb's steps one at a time: highlight this step's
707
+ * target (if any), speak/show its text, wait for that to finish, then
708
+ * move on. Always narrates via speakEndpoint (or the open realtime
709
+ * socket, if there is one), even mid realtime-call — a tour is a
710
+ * distinct guided walkthrough, not a conversational turn.
711
+ */
712
+ private async runTour(steps: TourStep[]) {
713
+ const myGeneration = ++this.tourGeneration;
714
+ const wasRealtimeListening = this.realtimeActive;
715
+ this.touringActive = true;
716
+ if (wasRealtimeListening) this.setStatus("rt-speaking");
717
+ this.setAnswer(null);
718
+ // Tracked locally rather than reading location.pathname each time — a
719
+ // step below can navigate mid-tour, and this keeps the miss-report
720
+ // route accurate for every step after that.
721
+ let currentRoute = location.pathname;
722
+
723
+ try {
724
+ for (let i = 0; i < steps.length; i++) {
725
+ if (this.tourGeneration !== myGeneration) return; // superseded — e.g. widget closed or a new question came in
726
+ const step = steps[i];
727
+ this.setCaption(`Step ${i + 1} of ${steps.length}`);
728
+ this.setAnswer(step.text);
729
+
730
+ if (step.route && step.route !== currentRoute) {
731
+ // No client-side router here (framework-agnostic — there isn't
732
+ // one to assume) — a full page navigation is the honest
733
+ // universal fallback, and it ends this tour run since the new
734
+ // page has no memory of it (documented limitation, see ROADMAP).
735
+ location.assign(step.route);
736
+ return;
737
+ }
738
+
739
+ if (step.target) {
740
+ const el = findElement(step.target);
741
+ if (el) highlightElement(el);
742
+ else this.reportMiss({ attempted: step.target, route: currentRoute });
743
+ }
744
+
745
+ if (wasRealtimeListening && this.rtSocket?.readyState === WebSocket.OPEN) {
746
+ // Already have a live streaming connection open — reuse it
747
+ // (same Speak WS, same gapless PCM scheduling a normal reply
748
+ // uses) instead of falling back to a separate buffered REST call.
749
+ await this.speakOverRealtime(step.text);
750
+ } else if (this.speakEndpoint) {
751
+ await this.speakAndWait(step.text);
752
+ } else {
753
+ // No TTS configured — pace by an estimate of reading time instead of racing through every step instantly.
754
+ await new Promise((resolve) => setTimeout(resolve, Math.max(1200, step.text.length * 45)));
755
+ }
756
+ if (this.tourGeneration !== myGeneration) return;
757
+ }
758
+
759
+ if (this.tourGeneration !== myGeneration) return;
760
+ this.setCaption("");
761
+ if (wasRealtimeListening && this.realtimeActive) this.setStatus("rt-listening");
762
+ } finally {
763
+ if (this.tourGeneration === myGeneration) {
764
+ this.touringActive = false;
765
+ this.render();
766
+ }
767
+ }
768
+ }
769
+
770
+ // --- speech -------------------------------------------------------------
771
+ /** Stops whatever's currently playing first, so two responses can never be heard overlapping. Used by both the typed/mic path and tours. */
772
+ private playResponseAudio(blob: Blob): Promise<void> {
773
+ if (this.activeAudio) {
774
+ this.activeAudio.pause();
775
+ this.activeAudio.currentTime = 0;
776
+ }
777
+ const url = URL.createObjectURL(blob);
778
+ const audio = new Audio(url);
779
+ this.activeAudio = audio;
780
+ return new Promise((resolve) => {
781
+ const clear = () => {
782
+ URL.revokeObjectURL(url);
783
+ if (this.activeAudio === audio) this.activeAudio = null;
784
+ resolve();
785
+ };
786
+ audio.onended = clear;
787
+ audio.play().catch(clear);
788
+ });
789
+ }
790
+
791
+ private async speak(text: string): Promise<void> {
792
+ if (!this.speakEndpoint || !text.trim()) return;
793
+ try {
794
+ const res = await fetch(this.speakEndpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ text }) });
795
+ if (!res.ok) return;
796
+ void this.playResponseAudio(await res.blob());
797
+ } catch {
798
+ // best-effort — never let speech playback break the widget
799
+ }
800
+ }
801
+
802
+ /** Like speak(), but resolves once playback actually finishes — used by runTour() so each step's highlight stays up for exactly as long as its narration takes. */
803
+ private async speakAndWait(text: string): Promise<void> {
804
+ if (!this.speakEndpoint || !text.trim()) return;
805
+ try {
806
+ const res = await fetch(this.speakEndpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ text }) });
807
+ if (!res.ok) return;
808
+ await this.playResponseAudio(await res.blob());
809
+ } catch {
810
+ // best-effort — never hang the tour
811
+ }
812
+ }
813
+
814
+ /** Narrates over an already-open realtime WebSocket instead of a separate REST call — resolved by maybeResumeListening() inside startRealtime() once this step's audio has both fully arrived and fully finished playing. */
815
+ private speakOverRealtime(text: string): Promise<void> {
816
+ return new Promise((resolve) => {
817
+ const ws = this.rtSocket;
818
+ if (!ws || ws.readyState !== WebSocket.OPEN) {
819
+ resolve();
820
+ return;
821
+ }
822
+ this.rtAudioDoneArriving = false;
823
+ let settled = false;
824
+ const finish = () => {
825
+ if (settled) return;
826
+ settled = true;
827
+ this.rtTourAudioDoneResolve = null;
828
+ resolve();
829
+ };
830
+ this.rtTourAudioDoneResolve = finish;
831
+ // Safety net: if the server's "this step's audio is fully done"
832
+ // confirmation is ever dropped (a flaky Deepgram Flushed event, a
833
+ // closed connection mid-turn), don't let the tour hang on this step
834
+ // forever with the mic never resuming — move on instead.
835
+ setTimeout(() => {
836
+ if (!settled) console.warn("[cairn] tour step audio confirmation timed out — continuing");
837
+ finish();
838
+ }, 15000);
839
+ ws.send(JSON.stringify({ type: "speak", text }));
840
+ });
841
+ }
842
+
843
+ // --- push-to-talk mic -----------------------------------------------------
844
+ private async startRecording() {
845
+ if (!this.transcribeEndpoint || !this.micSupported() || this.realtimeActive) return;
846
+ try {
847
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
848
+ const recorder = new MediaRecorder(stream);
849
+ this.audioChunks = [];
850
+ this.setCaption("");
851
+ recorder.ondataavailable = (e) => {
852
+ if (e.data.size === 0) return;
853
+ this.audioChunks.push(e.data);
854
+ void this.transcribeSoFar(recorder.mimeType || "audio/webm", true);
855
+ };
856
+ recorder.onstop = () => void this.transcribeSoFar(recorder.mimeType || "audio/webm", false);
857
+ this.mediaRecorder = recorder;
858
+ recorder.start(2000);
859
+ this.recording = true;
860
+ if (this.micBtn) {
861
+ this.micBtn.innerHTML = SQUARE_ICON;
862
+ this.micBtn.classList.add("cairn-icon-btn-recording");
863
+ this.micBtn.setAttribute("aria-label", "Stop recording");
864
+ }
865
+ this.render();
866
+ } catch {
867
+ this.setAnswer("Couldn't access the microphone — check your browser's permission for this site.");
868
+ }
869
+ }
870
+
871
+ private stopRecording() {
872
+ const stream = this.mediaRecorder?.stream;
873
+ this.mediaRecorder?.stop();
874
+ stream?.getTracks().forEach((t) => t.stop());
875
+ this.recording = false;
876
+ if (this.micBtn) {
877
+ this.micBtn.innerHTML = MIC_ICON;
878
+ this.micBtn.classList.remove("cairn-icon-btn-recording");
879
+ this.micBtn.setAttribute("aria-label", "Ask by voice");
880
+ }
881
+ this.render();
882
+ }
883
+
884
+ private async transcribeSoFar(mimeType: string, isProgressive: boolean) {
885
+ if (!this.transcribeEndpoint) return;
886
+ if (isProgressive && this.transcribeInFlight) return;
887
+ this.transcribeInFlight = true;
888
+ try {
889
+ const blob = new Blob(this.audioChunks, { type: mimeType });
890
+ const res = await fetch(this.transcribeEndpoint, { method: "POST", headers: { "content-type": mimeType }, body: blob });
891
+ const data = await res.json().catch(() => null);
892
+ if (data?.text) {
893
+ this.inputEl.value = data.text;
894
+ this.setCaption(data.text);
895
+ this.updateBusyState();
896
+ } else if (!isProgressive) {
897
+ this.setAnswer("Couldn't make that out — try typing instead.");
898
+ }
899
+ } catch {
900
+ if (!isProgressive) this.setAnswer("Couldn't reach the transcription service.");
901
+ } finally {
902
+ this.transcribeInFlight = false;
903
+ }
904
+ }
905
+
906
+ // ---------------------------------------------------------------------
907
+ // Real-time voice conversation
908
+ // ---------------------------------------------------------------------
909
+
910
+ private async startRealtime() {
911
+ // rtStarting closes the gap between click and the first status update
912
+ // landing — without it a rapid double-click could race past the
913
+ // realtimeActive check twice and open two sessions.
914
+ if (!this.realtimeUrl || !this.micSupported() || this.realtimeActive || this.rtStarting) return;
915
+ this.rtStarting = true;
916
+ this.setAnswer(null);
917
+ this.setCaption("");
918
+ this.setStatus("rt-connecting");
919
+
920
+ try {
921
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
922
+ const ws = new WebSocket(this.realtimeUrl);
923
+ ws.binaryType = "arraybuffer";
924
+ this.rtSocket = ws;
925
+
926
+ // Separate AudioContext from the mic capture graph below — one for
927
+ // capture, one for playback, independently lifecycled (playback
928
+ // keeps scheduling audio after a turn while the mic graph is
929
+ // simultaneously idle, and vice versa).
930
+ const playbackCtx = new AudioContext();
931
+ const playbackGain = playbackCtx.createGain();
932
+ playbackGain.gain.value = this.rtSpeakerMuted ? 0 : 1;
933
+ playbackGain.connect(playbackCtx.destination);
934
+ this.rtPlaybackCtx = playbackCtx;
935
+ this.rtPlaybackGain = playbackGain;
936
+ this.rtNextPlayTime = 0;
937
+ this.rtScheduledSources = [];
938
+ this.rtAudioDoneArriving = true;
939
+
940
+ const audioCtx = new AudioContext();
941
+ const source = audioCtx.createMediaStreamSource(stream);
942
+ // ScriptProcessorNode is deprecated in favor of AudioWorklet, but
943
+ // needs no separate worklet file to serve — fine for this scope,
944
+ // still supported everywhere. Routed through a silent gain (not
945
+ // straight to destination) so the mic input is never audibly looped back.
946
+ const processor = audioCtx.createScriptProcessor(4096, 1, 1);
947
+ const silence = audioCtx.createGain();
948
+ silence.gain.value = 0;
949
+
950
+ // Only flips back to "listening" (and lets the mic resume sending)
951
+ // once BOTH the server has said no more audio is coming for this
952
+ // turn AND every chunk already scheduled has actually finished
953
+ // playing — from playback completion, not the server's speaking_end
954
+ // alone, which is what stops the mic picking up the tail end of the
955
+ // agent's own voice. Shared with speakOverRealtime(): while touring,
956
+ // this same "audio fully drained" condition resolves the current
957
+ // step's wait instead of touching status/caption.
958
+ const maybeResumeListening = () => {
959
+ if (!this.rtAudioDoneArriving) return;
960
+ if (this.rtScheduledSources.length > 0) return;
961
+ if (this.touringActive) {
962
+ this.rtTourAudioDoneResolve?.();
963
+ this.rtTourAudioDoneResolve = null;
964
+ return;
965
+ }
966
+ this.setStatus("rt-listening");
967
+ this.setCaption("");
968
+ };
969
+
970
+ const disarmThinkingWatchdog = () => {
971
+ if (this.rtThinkingWatchdog) {
972
+ clearTimeout(this.rtThinkingWatchdog);
973
+ this.rtThinkingWatchdog = null;
974
+ }
975
+ };
976
+
977
+ const armThinkingWatchdog = () => {
978
+ disarmThinkingWatchdog();
979
+ this.rtThinkingWatchdog = setTimeout(() => {
980
+ this.rtThinkingWatchdog = null;
981
+ console.warn("[cairn] realtime turn timed out waiting on the server — resuming listening");
982
+ this.rtAudioDoneArriving = true;
983
+ this.setStatus("rt-listening");
984
+ this.setCaption("");
985
+ }, 20000);
986
+ };
987
+
988
+ const stopScheduledRtAudio = () => {
989
+ for (const node of this.rtScheduledSources) {
990
+ try {
991
+ node.stop();
992
+ } catch {
993
+ // may have already finished naturally
994
+ }
995
+ }
996
+ this.rtScheduledSources = [];
997
+ this.rtNextPlayTime = this.rtPlaybackCtx?.currentTime ?? 0;
998
+ };
999
+
1000
+ // Stops the agent immediately (locally) and tells the server to
1001
+ // discard whatever it's still synthesizing/sending for this turn —
1002
+ // the server tags every turn with a generation number and drops any
1003
+ // now-stale audio_chunk/speaking_end already in flight.
1004
+ const triggerBargeIn = () => {
1005
+ disarmThinkingWatchdog();
1006
+ stopScheduledRtAudio();
1007
+ this.rtAudioDoneArriving = true;
1008
+ if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "barge_in" }));
1009
+ this.setStatus("rt-listening");
1010
+ this.setCaption("");
1011
+ };
1012
+
1013
+ processor.onaudioprocess = (e) => {
1014
+ if (ws.readyState !== WebSocket.OPEN) return;
1015
+ if (this.rtMicMuted) return;
1016
+
1017
+ // Barge-in: while the agent is speaking a real conversational
1018
+ // reply (not touring — a tour deliberately can't be talked over),
1019
+ // keep listening to the mic locally even though it isn't being
1020
+ // sent yet, and cut the agent off the instant the user starts
1021
+ // talking over it instead of making them wait for it to finish.
1022
+ if (this.status === "rt-speaking" && !this.touringActive) {
1023
+ const rms = computeRms(e.inputBuffer.getChannelData(0));
1024
+ if (rms > BARGE_IN_RMS_THRESHOLD) triggerBargeIn();
1025
+ return;
1026
+ }
1027
+
1028
+ if (this.status !== "rt-listening") return; // don't send our own mic while the agent is thinking/speaking
1029
+ const pcm = floatTo16BitPCM(downsampleTo16k(e.inputBuffer.getChannelData(0), audioCtx.sampleRate));
1030
+ ws.send(pcm);
1031
+ };
1032
+ source.connect(processor);
1033
+ processor.connect(silence);
1034
+ silence.connect(audioCtx.destination);
1035
+
1036
+ this.rtCleanup = () => {
1037
+ processor.disconnect();
1038
+ source.disconnect();
1039
+ stream.getTracks().forEach((t) => t.stop());
1040
+ void audioCtx.close();
1041
+ stopScheduledRtAudio();
1042
+ void playbackCtx.close();
1043
+ this.rtPlaybackCtx = null;
1044
+ this.rtPlaybackGain = null;
1045
+ };
1046
+
1047
+ ws.onopen = () => {
1048
+ ws.send(JSON.stringify({ type: "context", route: location.pathname, visible: collectVisible() }));
1049
+ this.setStatus("rt-listening");
1050
+ this.rtStarting = false;
1051
+ };
1052
+
1053
+ ws.onmessage = (event) => {
1054
+ if (typeof event.data !== "string") return; // audio arrives as base64 inside audio_chunk, not raw binary frames
1055
+ const msg = JSON.parse(event.data);
1056
+ if (msg.type === "interim") {
1057
+ this.setCaption(msg.text);
1058
+ } else if (msg.type === "final") {
1059
+ this.setCaption(msg.text);
1060
+ this.setStatus("rt-thinking");
1061
+ armThinkingWatchdog();
1062
+ } else if (msg.type === "verb") {
1063
+ disarmThinkingWatchdog();
1064
+ this.handleVerb(msg.verb);
1065
+ } else if (msg.type === "speaking_start") {
1066
+ disarmThinkingWatchdog();
1067
+ this.rtAudioDoneArriving = false;
1068
+ this.setStatus("rt-speaking");
1069
+ } else if (msg.type === "audio_chunk") {
1070
+ const ctx = this.rtPlaybackCtx;
1071
+ const gain = this.rtPlaybackGain;
1072
+ if (!ctx || !gain) return;
1073
+ void ctx.resume().catch(() => {});
1074
+
1075
+ // Decode base64 linear16 PCM -> Float32 samples in [-1, 1], then
1076
+ // schedule gapless-appended after whatever's already queued —
1077
+ // this is what lets playback start on the first chunk instead
1078
+ // of waiting for the whole reply.
1079
+ const bytes = Uint8Array.from(atob(msg.audio), (c) => c.charCodeAt(0));
1080
+ const sampleCount = bytes.length / 2;
1081
+ const float32 = new Float32Array(sampleCount);
1082
+ const view = new DataView(bytes.buffer);
1083
+ for (let i = 0; i < sampleCount; i++) {
1084
+ float32[i] = view.getInt16(i * 2, true) / 32768;
1085
+ }
1086
+ const sampleRate = typeof msg.sampleRate === "number" ? msg.sampleRate : 24000;
1087
+ const buffer = ctx.createBuffer(1, sampleCount, sampleRate);
1088
+ buffer.copyToChannel(float32, 0);
1089
+
1090
+ const bufferSource = ctx.createBufferSource();
1091
+ bufferSource.buffer = buffer;
1092
+ bufferSource.connect(gain);
1093
+
1094
+ const startAt = Math.max(ctx.currentTime, this.rtNextPlayTime);
1095
+ bufferSource.start(startAt);
1096
+ this.rtNextPlayTime = startAt + buffer.duration;
1097
+
1098
+ this.rtScheduledSources.push(bufferSource);
1099
+ bufferSource.onended = () => {
1100
+ this.rtScheduledSources = this.rtScheduledSources.filter((n) => n !== bufferSource);
1101
+ maybeResumeListening();
1102
+ };
1103
+ } else if (msg.type === "speaking_end" || msg.type === "turn_complete") {
1104
+ // turn_complete covers a verb with nothing spoken — no audio_chunk
1105
+ // ever arrives for it, so rtScheduledSources is already empty and
1106
+ // maybeResumeListening() resumes immediately.
1107
+ disarmThinkingWatchdog();
1108
+ this.rtAudioDoneArriving = true;
1109
+ maybeResumeListening();
1110
+ } else if (msg.type === "error") {
1111
+ // Must actually unstick the turn, not just show the message —
1112
+ // otherwise the mic never resumes and the session is stuck
1113
+ // exactly the way a silently-dropped response used to leave it.
1114
+ disarmThinkingWatchdog();
1115
+ this.setAnswer(msg.message ?? "Something went wrong.");
1116
+ if (!this.touringActive) {
1117
+ this.setStatus("rt-listening");
1118
+ this.setCaption("");
1119
+ }
1120
+ }
1121
+ };
1122
+
1123
+ ws.onerror = () => {
1124
+ this.setAnswer("Couldn't connect to the realtime voice service.");
1125
+ this.endRealtime();
1126
+ };
1127
+ ws.onclose = () => {
1128
+ if (this.status !== "idle") this.endRealtime();
1129
+ };
1130
+ } catch {
1131
+ this.setAnswer("Couldn't access the microphone — check your browser's permission for this site.");
1132
+ this.setStatus("idle");
1133
+ this.rtStarting = false;
1134
+ }
1135
+ }
1136
+
1137
+ private endRealtime() {
1138
+ if (this.rtThinkingWatchdog) {
1139
+ clearTimeout(this.rtThinkingWatchdog);
1140
+ this.rtThinkingWatchdog = null;
1141
+ }
1142
+ this.rtStarting = false;
1143
+ this.activeAudio?.pause();
1144
+ this.activeAudio = null;
1145
+ this.rtSocket?.close();
1146
+ this.rtSocket = null;
1147
+ this.rtCleanup?.();
1148
+ this.rtCleanup = null;
1149
+ this.rtMicMuted = false;
1150
+ this.rtSpeakerMuted = false;
1151
+ this.setCaption("");
1152
+ this.setStatus("idle");
1153
+ this.tourGeneration++; // cancel an in-progress tour rather than leaving it stuck waiting to resume rt-listening
1154
+ this.touringActive = false;
1155
+ // Unstick a tour step mid-narration over realtime — the socket above is
1156
+ // already closed, so nothing will ever deliver the audio_chunk/
1157
+ // speaking_end that would normally resolve this.
1158
+ this.rtTourAudioDoneResolve?.();
1159
+ this.rtTourAudioDoneResolve = null;
1160
+ this.render();
1161
+ }
1162
+
1163
+ private toggleRtMic() {
1164
+ this.rtMicMuted = !this.rtMicMuted;
1165
+ this.render();
1166
+ }
1167
+
1168
+ private toggleRtSpeaker() {
1169
+ this.rtSpeakerMuted = !this.rtSpeakerMuted;
1170
+ // Zeroing the shared gain node silences output immediately, including
1171
+ // whatever's mid-playback right now, and applies to every future
1172
+ // scheduled chunk automatically — no per-chunk check needed.
1173
+ if (this.rtPlaybackGain) this.rtPlaybackGain.gain.value = this.rtSpeakerMuted ? 0 : 1;
1174
+ this.render();
1175
+ }
1176
+ }
1177
+
1178
+ /** Best-effort text form of a raw (unvalidated) verb response for the
1179
+ * conversation-history log — not shown to the user, just fed back to the
1180
+ * model on later turns. */
1181
+ function summarizeVerbForHistory(raw: unknown): string {
1182
+ if (!raw || typeof raw !== "object") return "(no response)";
1183
+ const v = raw as Record<string, unknown>;
1184
+ if (typeof v.text === "string" && v.text) return v.text;
1185
+ switch (v.verb) {
1186
+ case "highlight":
1187
+ case "open":
1188
+ return `(highlighted ${String(v.target)})`;
1189
+ case "navigate":
1190
+ return `(navigated to ${String(v.route)})`;
1191
+ case "do":
1192
+ return `(ran ${String(v.action)}${v.target ? ` on ${String(v.target)}` : ""})`;
1193
+ case "tour":
1194
+ return Array.isArray(v.steps) ? v.steps.map((s: { text?: string }) => s.text ?? "").join(" ") : "(tour)";
1195
+ default:
1196
+ return "(no response)";
1197
+ }
1198
+ }
1199
+
1200
+ // ---------------------------------------------------------------------------
1201
+ // Audio helpers (real-time PCM16 capture — standard Web Audio API patterns,
1202
+ // identical to index.tsx's — same protocol, same math, ported directly)
1203
+ // ---------------------------------------------------------------------------
1204
+
1205
+ // Heuristic energy gate for barge-in: real speech into a laptop/phone mic
1206
+ // typically sits well above this; normal room noise and the mic's own
1207
+ // noise floor typically sit below it. Same threshold as the React widget —
1208
+ // not independently recalibrated, since it's the same audio pipeline.
1209
+ const BARGE_IN_RMS_THRESHOLD = 0.02;
1210
+
1211
+ function computeRms(channelData: Float32Array): number {
1212
+ let sumSquares = 0;
1213
+ for (let i = 0; i < channelData.length; i++) sumSquares += channelData[i] * channelData[i];
1214
+ return Math.sqrt(sumSquares / channelData.length);
1215
+ }
1216
+
1217
+ function downsampleTo16k(input: Float32Array, inputSampleRate: number): Float32Array {
1218
+ const targetRate = 16000;
1219
+ if (inputSampleRate === targetRate) return input;
1220
+ const ratio = inputSampleRate / targetRate;
1221
+ const outLength = Math.round(input.length / ratio);
1222
+ const result = new Float32Array(outLength);
1223
+ let offsetResult = 0;
1224
+ let offsetInput = 0;
1225
+ while (offsetResult < outLength) {
1226
+ const nextOffsetInput = Math.round((offsetResult + 1) * ratio);
1227
+ let accum = 0;
1228
+ let count = 0;
1229
+ for (let i = offsetInput; i < nextOffsetInput && i < input.length; i++) {
1230
+ accum += input[i];
1231
+ count++;
1232
+ }
1233
+ result[offsetResult] = count > 0 ? accum / count : 0;
1234
+ offsetResult++;
1235
+ offsetInput = nextOffsetInput;
1236
+ }
1237
+ return result;
1238
+ }
1239
+
1240
+ function floatTo16BitPCM(input: Float32Array): ArrayBuffer {
1241
+ const buffer = new ArrayBuffer(input.length * 2);
1242
+ const view = new DataView(buffer);
1243
+ for (let i = 0; i < input.length; i++) {
1244
+ const s = Math.max(-1, Math.min(1, input[i]));
1245
+ view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);
1246
+ }
1247
+ return buffer;
1248
+ }
1249
+
1250
+ if (typeof customElements !== "undefined" && !customElements.get("cairn-widget")) {
1251
+ customElements.define("cairn-widget", CairnWidgetElement);
1252
+ }