@cairnvibe/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/cairn-widget.js +228 -0
- package/dist/context-collector.d.ts +1 -0
- package/dist/context-collector.js +23 -0
- package/dist/dashboard-sqlite.d.ts +8 -0
- package/dist/dashboard-sqlite.js +50 -0
- package/dist/dashboard.d.ts +39 -0
- package/dist/dashboard.js +60 -0
- package/dist/element-ladder.d.ts +7 -0
- package/dist/element-ladder.js +60 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.js +1069 -0
- package/dist/key-rotator.d.ts +7 -0
- package/dist/key-rotator.js +31 -0
- package/dist/package.json +1 -0
- package/dist/realtime-cli.d.ts +2 -0
- package/dist/realtime-cli.js +59 -0
- package/dist/realtime-server.d.ts +10 -0
- package/dist/realtime-server.js +291 -0
- package/dist/server.d.ts +95 -0
- package/dist/server.js +298 -0
- package/dist/speak-server.d.ts +16 -0
- package/dist/speak-server.js +41 -0
- package/dist/transcribe-server.d.ts +14 -0
- package/dist/transcribe-server.js +47 -0
- package/dist/tts-stream.d.ts +33 -0
- package/dist/tts-stream.js +124 -0
- package/dist/verb-executor.d.ts +17 -0
- package/dist/verb-executor.js +67 -0
- package/package.json +56 -0
- package/src/context-collector.ts +21 -0
- package/src/dashboard-sqlite.ts +52 -0
- package/src/dashboard.ts +82 -0
- package/src/element-ladder.ts +67 -0
- package/src/index.tsx +1250 -0
- package/src/key-rotator.ts +29 -0
- package/src/realtime-cli.ts +62 -0
- package/src/realtime-server.ts +342 -0
- package/src/server.ts +386 -0
- package/src/speak-server.ts +56 -0
- package/src/transcribe-server.ts +68 -0
- package/src/tts-stream.ts +140 -0
- package/src/verb-executor.ts +84 -0
- package/src/web-component.ts +1252 -0
|
@@ -0,0 +1,1252 @@
|
|
|
1
|
+
// Framework-agnostic delivery for the Cairn widget — a plain Web Component
|
|
2
|
+
// (Custom Element + Shadow DOM), usable via a single <script> tag on any
|
|
3
|
+
// page: Vue, Angular, Svelte, a static HTML file, or Next.js/React (which
|
|
4
|
+
// also gets the <Copilot/> wrapper in index.tsx, for convenience).
|
|
5
|
+
//
|
|
6
|
+
// Reuses the exact same framework-neutral engine the React widget does —
|
|
7
|
+
// executeVerbResponse, findElement/highlightElement, collectVisible — none
|
|
8
|
+
// of that changes here. What's different is *rendering*: real DOM nodes
|
|
9
|
+
// created once and mutated in place, not JSX re-rendered on every state
|
|
10
|
+
// change (a naive innerHTML-per-update approach would drop input focus on
|
|
11
|
+
// every keystroke — see ROADMAP.md for why this matters), and *state*:
|
|
12
|
+
// plain instance fields instead of React state/refs. One genuine
|
|
13
|
+
// simplification vanilla gets for free — no ref-vs-state split needed for
|
|
14
|
+
// "read the current value inside an async callback without a stale
|
|
15
|
+
// closure" the way React's rtStateRef/rtMicMutedRef/etc. existed purely to
|
|
16
|
+
// solve; a class field read inside any closure on `this` is always current.
|
|
17
|
+
//
|
|
18
|
+
// Full feature parity with <Copilot/> (index.tsx), including live realtime
|
|
19
|
+
// voice conversation (streaming TTS, barge-in) — ported directly from that
|
|
20
|
+
// file's implementation, same protocol, same audio pipeline, same
|
|
21
|
+
// barge-in heuristic. See ROADMAP.md Phase 1 for how this was staged (a
|
|
22
|
+
// typed-Q&A-only MVP shipped first, live-verified, before this).
|
|
23
|
+
|
|
24
|
+
import type { HistoryTurn as HistoryEntry, TourStep } from "@cairnvibe/core";
|
|
25
|
+
import { collectVisible } from "./context-collector";
|
|
26
|
+
import { findElement, highlightElement, logMiss, type MissContext } from "./element-ladder";
|
|
27
|
+
import { executeVerbResponse } from "./verb-executor";
|
|
28
|
+
|
|
29
|
+
type Status = "idle" | "asking" | "recording" | "rt-connecting" | "rt-listening" | "rt-thinking" | "rt-speaking";
|
|
30
|
+
|
|
31
|
+
const STATUS_LABEL: Record<Status, string> = {
|
|
32
|
+
idle: "",
|
|
33
|
+
asking: "Thinking…",
|
|
34
|
+
recording: "Listening — transcribing live…",
|
|
35
|
+
"rt-connecting": "Connecting…",
|
|
36
|
+
"rt-listening": "Listening…",
|
|
37
|
+
"rt-thinking": "Thinking…",
|
|
38
|
+
"rt-speaking": "Speaking…",
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
const MAX_HISTORY_TURNS = 8; // 4 exchanges — matches the realtime relay's own server-side cap
|
|
42
|
+
|
|
43
|
+
const STYLES = `
|
|
44
|
+
:host {
|
|
45
|
+
all: initial;
|
|
46
|
+
font: 13.5px/1.5 -apple-system, BlinkMacSystemFont, "SF Pro Text", system-ui, "Segoe UI", sans-serif;
|
|
47
|
+
color: #0b0d12;
|
|
48
|
+
}
|
|
49
|
+
* { box-sizing: border-box; }
|
|
50
|
+
@keyframes cairn-pulse {
|
|
51
|
+
0%, 100% { box-shadow: 0 0 0 0 rgba(239, 68, 68, 0.4); }
|
|
52
|
+
70% { box-shadow: 0 0 0 10px rgba(239, 68, 68, 0); }
|
|
53
|
+
}
|
|
54
|
+
@keyframes cairn-pulse-green {
|
|
55
|
+
0%, 100% { box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.4); }
|
|
56
|
+
70% { box-shadow: 0 0 0 10px rgba(16, 185, 129, 0); }
|
|
57
|
+
}
|
|
58
|
+
@keyframes cairn-pulse-indigo {
|
|
59
|
+
0%, 100% { box-shadow: 0 0 0 0 rgba(99, 102, 241, 0.4); }
|
|
60
|
+
70% { box-shadow: 0 0 0 10px rgba(99, 102, 241, 0); }
|
|
61
|
+
}
|
|
62
|
+
@keyframes cairn-spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } }
|
|
63
|
+
@keyframes cairn-rt-dot {
|
|
64
|
+
0%, 100% { opacity: 0.5; transform: scale(0.85); }
|
|
65
|
+
50% { opacity: 1; transform: scale(1.15); }
|
|
66
|
+
}
|
|
67
|
+
@keyframes cairn-bubble-in {
|
|
68
|
+
from { opacity: 0; transform: translateY(6px); }
|
|
69
|
+
to { opacity: 1; transform: translateY(0); }
|
|
70
|
+
}
|
|
71
|
+
@keyframes cairn-panel-in {
|
|
72
|
+
from { opacity: 0; transform: translateY(8px) scale(0.98); }
|
|
73
|
+
to { opacity: 1; transform: translateY(0) scale(1); }
|
|
74
|
+
}
|
|
75
|
+
@keyframes cairn-word-sweep {
|
|
76
|
+
0% { opacity: 0.35; text-shadow: none; }
|
|
77
|
+
35% { opacity: 1; color: #4f46e5; text-shadow: 0 0 10px rgba(99, 102, 241, 0.45); }
|
|
78
|
+
100% { opacity: 1; color: inherit; text-shadow: none; }
|
|
79
|
+
}
|
|
80
|
+
@keyframes cairn-thinking-bounce {
|
|
81
|
+
0%, 80%, 100% { opacity: 0.3; transform: translateY(0); }
|
|
82
|
+
40% { opacity: 0.9; transform: translateY(-3px); }
|
|
83
|
+
}
|
|
84
|
+
.cairn-glow {
|
|
85
|
+
animation: cairn-pulse-indigo 1.1s ease-out 2;
|
|
86
|
+
outline: 2px solid #6366f1;
|
|
87
|
+
outline-offset: 3px;
|
|
88
|
+
border-radius: 8px;
|
|
89
|
+
}
|
|
90
|
+
.cairn-spin { animation: cairn-spin 0.8s linear infinite; }
|
|
91
|
+
@media (prefers-reduced-motion: reduce) {
|
|
92
|
+
.cairn-fab, .cairn-panel, .cairn-bubble, .cairn-word, .cairn-thinking-dot {
|
|
93
|
+
animation: none !important;
|
|
94
|
+
transition: none !important;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
.cairn-fab {
|
|
99
|
+
position: fixed;
|
|
100
|
+
right: 20px;
|
|
101
|
+
bottom: 20px;
|
|
102
|
+
z-index: 2147483000;
|
|
103
|
+
width: 52px;
|
|
104
|
+
height: 52px;
|
|
105
|
+
border-radius: 999px;
|
|
106
|
+
border: none;
|
|
107
|
+
display: flex;
|
|
108
|
+
align-items: center;
|
|
109
|
+
justify-content: center;
|
|
110
|
+
background: #14151b;
|
|
111
|
+
color: white;
|
|
112
|
+
cursor: pointer;
|
|
113
|
+
box-shadow: 0 6px 20px rgba(0, 0, 0, 0.25);
|
|
114
|
+
transition: transform 0.15s ease, box-shadow 0.15s ease;
|
|
115
|
+
}
|
|
116
|
+
.cairn-fab:hover { transform: translateY(-1px); box-shadow: 0 8px 24px rgba(0, 0, 0, 0.3); }
|
|
117
|
+
.cairn-fab-speaking {
|
|
118
|
+
box-shadow: 0 0 0 4px rgba(16, 185, 129, 0.22), 0 6px 20px rgba(0, 0, 0, 0.25);
|
|
119
|
+
animation: cairn-pulse-green 1.2s ease-out infinite;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/* No shared background/border/shadow on the panel itself — every piece
|
|
123
|
+
below (title, bubbles, input, buttons) floats independently on its own
|
|
124
|
+
minimal glass, directly over the host page, the way a caption track
|
|
125
|
+
floats over a video rather than sitting in a drawn box. */
|
|
126
|
+
/* One unified card — title, conversation, and input all live inside the
|
|
127
|
+
same bounded, padded container instead of floating as independent
|
|
128
|
+
fixed-position pieces. That "everything floats separately" approach
|
|
129
|
+
kept producing new collisions (title vs input, send button vs the
|
|
130
|
+
close FAB) every time one piece's position changed; grouping them
|
|
131
|
+
under one panel with real internal spacing removes that whole class
|
|
132
|
+
of bug at the source. */
|
|
133
|
+
.cairn-panel {
|
|
134
|
+
position: fixed;
|
|
135
|
+
right: 20px;
|
|
136
|
+
bottom: 92px;
|
|
137
|
+
z-index: 2147483000;
|
|
138
|
+
width: min(340px, calc(100vw - 40px));
|
|
139
|
+
max-height: 480px;
|
|
140
|
+
overflow-y: auto;
|
|
141
|
+
overflow-x: hidden;
|
|
142
|
+
flex-direction: column;
|
|
143
|
+
gap: 14px;
|
|
144
|
+
padding: 18px;
|
|
145
|
+
background: rgba(255, 255, 255, 0.96);
|
|
146
|
+
backdrop-filter: blur(24px) saturate(160%);
|
|
147
|
+
-webkit-backdrop-filter: blur(24px) saturate(160%);
|
|
148
|
+
border-radius: 20px;
|
|
149
|
+
box-shadow: 0 20px 50px rgba(15, 15, 25, 0.16), 0 2px 8px rgba(15, 15, 25, 0.06);
|
|
150
|
+
display: none;
|
|
151
|
+
}
|
|
152
|
+
.cairn-panel.cairn-open { display: flex; animation: cairn-panel-in 0.2s cubic-bezier(0.16, 1, 0.3, 1); }
|
|
153
|
+
.cairn-panel::-webkit-scrollbar { width: 0; }
|
|
154
|
+
|
|
155
|
+
.cairn-stack { display: flex; flex-direction: column; gap: 10px; }
|
|
156
|
+
.cairn-bubble {
|
|
157
|
+
max-width: 92%;
|
|
158
|
+
font-size: 13.5px;
|
|
159
|
+
line-height: 1.5;
|
|
160
|
+
color: #0b0d12;
|
|
161
|
+
animation: cairn-bubble-in 0.2s ease-out;
|
|
162
|
+
}
|
|
163
|
+
.cairn-bubble-user {
|
|
164
|
+
align-self: flex-end;
|
|
165
|
+
text-align: right;
|
|
166
|
+
color: #33384a;
|
|
167
|
+
}
|
|
168
|
+
.cairn-bubble-agent {
|
|
169
|
+
align-self: flex-start;
|
|
170
|
+
display: flex;
|
|
171
|
+
flex-direction: column;
|
|
172
|
+
gap: 4px;
|
|
173
|
+
}
|
|
174
|
+
.cairn-bubble-text { white-space: pre-wrap; }
|
|
175
|
+
.cairn-word { display: inline-block; animation: cairn-word-sweep 0.4s ease forwards; }
|
|
176
|
+
.cairn-chip {
|
|
177
|
+
align-self: flex-start;
|
|
178
|
+
font-size: 10.5px;
|
|
179
|
+
font-weight: 700;
|
|
180
|
+
letter-spacing: 0.07em;
|
|
181
|
+
text-transform: uppercase;
|
|
182
|
+
color: rgba(11, 13, 18, 0.48);
|
|
183
|
+
}
|
|
184
|
+
.cairn-thinking { display: inline-flex; gap: 4px; padding: 2px 0; }
|
|
185
|
+
.cairn-thinking-dot {
|
|
186
|
+
width: 5px;
|
|
187
|
+
height: 5px;
|
|
188
|
+
border-radius: 999px;
|
|
189
|
+
background: rgba(11, 13, 18, 0.4);
|
|
190
|
+
animation: cairn-thinking-bounce 1.1s ease-in-out infinite;
|
|
191
|
+
}
|
|
192
|
+
.cairn-thinking-dot:nth-child(2) { animation-delay: 0.15s; }
|
|
193
|
+
.cairn-thinking-dot:nth-child(3) { animation-delay: 0.3s; }
|
|
194
|
+
|
|
195
|
+
.cairn-input-row { display: flex; gap: 7px; align-items: center; }
|
|
196
|
+
.cairn-input-row input {
|
|
197
|
+
flex: 1;
|
|
198
|
+
min-width: 0;
|
|
199
|
+
border: none;
|
|
200
|
+
border-radius: 999px;
|
|
201
|
+
padding: 10px 14px;
|
|
202
|
+
font: inherit;
|
|
203
|
+
background: rgba(11, 13, 18, 0.045);
|
|
204
|
+
color: #0b0d12;
|
|
205
|
+
transition: background 0.15s ease, box-shadow 0.15s ease;
|
|
206
|
+
}
|
|
207
|
+
.cairn-input-row input::placeholder { color: rgba(11, 13, 18, 0.4); }
|
|
208
|
+
.cairn-input-row input:disabled { opacity: 0.55; }
|
|
209
|
+
.cairn-input-row input:focus {
|
|
210
|
+
outline: none;
|
|
211
|
+
background: rgba(11, 13, 18, 0.06);
|
|
212
|
+
box-shadow: 0 0 0 3px rgba(99, 102, 241, 0.16);
|
|
213
|
+
}
|
|
214
|
+
.cairn-icon-btn, .cairn-send {
|
|
215
|
+
flex-shrink: 0;
|
|
216
|
+
width: 36px;
|
|
217
|
+
height: 36px;
|
|
218
|
+
display: flex;
|
|
219
|
+
align-items: center;
|
|
220
|
+
justify-content: center;
|
|
221
|
+
border-radius: 999px;
|
|
222
|
+
border: none;
|
|
223
|
+
background: rgba(11, 13, 18, 0.045);
|
|
224
|
+
color: #33384a;
|
|
225
|
+
cursor: pointer;
|
|
226
|
+
transition: background 0.15s ease, transform 0.15s ease;
|
|
227
|
+
}
|
|
228
|
+
.cairn-icon-btn:hover { background: rgba(11, 13, 18, 0.09); transform: translateY(-1px); }
|
|
229
|
+
.cairn-send {
|
|
230
|
+
border: none;
|
|
231
|
+
background: #14151b;
|
|
232
|
+
color: white;
|
|
233
|
+
box-shadow: 0 4px 14px rgba(0, 0, 0, 0.25);
|
|
234
|
+
}
|
|
235
|
+
.cairn-send:hover:not(:disabled) { transform: translateY(-1px); }
|
|
236
|
+
.cairn-send:disabled, .cairn-icon-btn:disabled { opacity: 0.5; cursor: not-allowed; box-shadow: none; }
|
|
237
|
+
.cairn-icon-btn-recording {
|
|
238
|
+
background: #ef4444;
|
|
239
|
+
border-color: #ef4444;
|
|
240
|
+
color: white;
|
|
241
|
+
animation: cairn-pulse 1.4s ease-out infinite;
|
|
242
|
+
}
|
|
243
|
+
.cairn-icon-btn-speaking {
|
|
244
|
+
background: #10b981;
|
|
245
|
+
border-color: #10b981;
|
|
246
|
+
color: white;
|
|
247
|
+
animation: cairn-pulse-green 1.2s ease-out infinite;
|
|
248
|
+
}
|
|
249
|
+
.cairn-icon-btn-end { background: #ef4444; border-color: #ef4444; color: white; }
|
|
250
|
+
.cairn-rt-bar {
|
|
251
|
+
display: flex;
|
|
252
|
+
align-items: center;
|
|
253
|
+
gap: 8px;
|
|
254
|
+
padding: 8px 12px;
|
|
255
|
+
border-radius: 999px;
|
|
256
|
+
background: rgba(11, 13, 18, 0.045);
|
|
257
|
+
}
|
|
258
|
+
.cairn-rt-dot {
|
|
259
|
+
width: 8px;
|
|
260
|
+
height: 8px;
|
|
261
|
+
border-radius: 999px;
|
|
262
|
+
background: #6366f1;
|
|
263
|
+
animation: cairn-rt-dot 1.2s ease-in-out infinite;
|
|
264
|
+
flex-shrink: 0;
|
|
265
|
+
}
|
|
266
|
+
.cairn-rt-dot-rt-speaking { background: #10b981; }
|
|
267
|
+
.cairn-rt-dot-rt-thinking { background: #f59e0b; }
|
|
268
|
+
.cairn-rt-label { flex: 1; font-size: 12.5px; color: #33384a; }
|
|
269
|
+
.cairn-rt-controls { display: flex; gap: 6px; }
|
|
270
|
+
`;
|
|
271
|
+
|
|
272
|
+
const SEND_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="m22 2-7 20-4-9-9-4Z"/><path d="M22 2 11 13"/></svg>`;
|
|
273
|
+
const MIC_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 2a3 3 0 0 0-3 3v6a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z"/><path d="M19 10v1a7 7 0 0 1-14 0v-1"/><line x1="12" x2="12" y1="18" y2="22"/></svg>`;
|
|
274
|
+
const MIC_OFF_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><line x1="2" y1="2" x2="22" y2="22"/><path d="M9 9v3a3 3 0 0 0 5.12 2.12M15 9.34V5a3 3 0 0 0-5.94-.6"/><path d="M17 16.95A7 7 0 0 1 5 12v-2m14 0v2a7 7 0 0 1-.11 1.23"/><line x1="12" y1="19" x2="12" y2="22"/></svg>`;
|
|
275
|
+
const SQUARE_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="14" height="14" x="5" y="5" rx="2"/></svg>`;
|
|
276
|
+
const SPINNER_ICON = `<svg class="cairn-spin" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M21 12a9 9 0 1 1-6.219-8.56"/></svg>`;
|
|
277
|
+
const CLOSE_ICON = `<svg width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 6 6 18"/><path d="m6 6 12 12"/></svg>`;
|
|
278
|
+
const MARK_ICON = `<svg width="20" height="20" viewBox="0 0 20 20" fill="none" aria-hidden="true"><rect x="7" y="12.5" width="6" height="2.6" rx="0.5" fill="currentColor"/><rect x="4.5" y="8.5" width="11" height="2.6" rx="0.5" fill="currentColor" opacity="0.75"/><rect x="8.2" y="4.5" width="3.6" height="2.6" rx="0.5" fill="currentColor" opacity="0.5"/></svg>`;
|
|
279
|
+
const PHONE_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M13.832 16.568a1 1 0 0 0 1.213-.303l.355-.465A2 2 0 0 1 17 15h3a2 2 0 0 1 2 2v3a2 2 0 0 1-2 2A18 18 0 0 1 2 4a2 2 0 0 1 2-2h3a2 2 0 0 1 2 2v3a2 2 0 0 1-.8 1.6l-.468.351a1 1 0 0 0-.292 1.233 14 14 0 0 0 6.392 6.384"/></svg>`;
|
|
280
|
+
const PHONE_OFF_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M13.83 16.57a1 1 0 0 0 1.21-.3l.36-.47A2 2 0 0 1 17 15h3a2 2 0 0 1 2 2v3a2 2 0 0 1-2 2A17.87 17.87 0 0 1 9.3 17.6"/><path d="M4.27 5.34C3.5 6.44 4 8 4 8a17.9 17.9 0 0 0 2.14 6.6"/><path d="M2 2v0"/><line x1="2" y1="2" x2="22" y2="22"/></svg>`;
|
|
281
|
+
const VOLUME_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5"/><path d="M15.54 8.46a5 5 0 0 1 0 7.07"/><path d="M19.07 4.93a10 10 0 0 1 0 14.14"/></svg>`;
|
|
282
|
+
const VOLUME_OFF_ICON = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5"/><line x1="22" y1="9" x2="16" y2="15"/><line x1="16" y1="9" x2="22" y2="15"/></svg>`;
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* `<cairn-widget endpoint="/api/copilot" persona="Cairn" realtime-url="ws://..." ...>`
|
|
286
|
+
* — see README.md's "Voice & conversation" / install sections for the full
|
|
287
|
+
* attribute list and framework-specific examples.
|
|
288
|
+
*/
|
|
289
|
+
export class CairnWidgetElement extends HTMLElement {
|
|
290
|
+
private shadow!: ShadowRoot;
|
|
291
|
+
private fab!: HTMLButtonElement;
|
|
292
|
+
private panel!: HTMLDivElement;
|
|
293
|
+
private stackEl!: HTMLDivElement;
|
|
294
|
+
private userBubbleEl!: HTMLDivElement;
|
|
295
|
+
private agentBubbleEl!: HTMLDivElement;
|
|
296
|
+
private chipEl!: HTMLSpanElement;
|
|
297
|
+
private agentTextEl!: HTMLSpanElement;
|
|
298
|
+
private thinkingEl!: HTMLSpanElement;
|
|
299
|
+
private formEl!: HTMLFormElement;
|
|
300
|
+
private inputEl!: HTMLInputElement;
|
|
301
|
+
private sendBtn!: HTMLButtonElement;
|
|
302
|
+
private micBtn: HTMLButtonElement | null = null;
|
|
303
|
+
private phoneBtn: HTMLButtonElement | null = null;
|
|
304
|
+
private rtBar!: HTMLDivElement;
|
|
305
|
+
private rtDot!: HTMLSpanElement;
|
|
306
|
+
private rtLabel!: HTMLSpanElement;
|
|
307
|
+
private rtMicBtn!: HTMLButtonElement;
|
|
308
|
+
private rtSpeakerBtn!: HTMLButtonElement;
|
|
309
|
+
private rtEndBtn!: HTMLButtonElement;
|
|
310
|
+
|
|
311
|
+
private isOpen = false;
|
|
312
|
+
private status: Status = "idle";
|
|
313
|
+
private recording = false;
|
|
314
|
+
private touringActive = false;
|
|
315
|
+
private caption = "";
|
|
316
|
+
private answer: string | null = null;
|
|
317
|
+
// The user's own last question, shown as its own floating caption bubble
|
|
318
|
+
// alongside the agent's — set once per ask() call, not cleared on
|
|
319
|
+
// completion, so the exchange stays paired on screen the way a caption
|
|
320
|
+
// track shows the current line, not a scrolling transcript.
|
|
321
|
+
private lastQuestion: string | null = null;
|
|
322
|
+
private tourGeneration = 0;
|
|
323
|
+
private mediaRecorder: MediaRecorder | null = null;
|
|
324
|
+
private audioChunks: Blob[] = [];
|
|
325
|
+
private transcribeInFlight = false;
|
|
326
|
+
private activeAudio: HTMLAudioElement | null = null;
|
|
327
|
+
/** Conversation memory for the typed/mic path — the realtime path keeps its own history server-side (that connection is already stateful). */
|
|
328
|
+
private history: HistoryEntry[] = [];
|
|
329
|
+
|
|
330
|
+
// --- realtime voice state ------------------------------------------------
|
|
331
|
+
private rtSocket: WebSocket | null = null;
|
|
332
|
+
private rtCleanup: (() => void) | null = null;
|
|
333
|
+
private rtMicMuted = false;
|
|
334
|
+
private rtSpeakerMuted = false;
|
|
335
|
+
private rtStarting = false; // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
|
|
336
|
+
private rtPlaybackCtx: AudioContext | null = null;
|
|
337
|
+
private rtPlaybackGain: GainNode | null = null;
|
|
338
|
+
private rtNextPlayTime = 0;
|
|
339
|
+
private rtScheduledSources: AudioBufferSourceNode[] = [];
|
|
340
|
+
// Watchdog for the "rt-thinking" state: started on every "final"
|
|
341
|
+
// transcript, cleared the moment the server responds with anything for
|
|
342
|
+
// that turn (verb/speaking_start/speaking_end/turn_complete/error). If it
|
|
343
|
+
// ever fires, the server went silent for this turn — force the mic back
|
|
344
|
+
// to listening instead of leaving the session stuck showing "Thinking…"
|
|
345
|
+
// forever with no way to speak again short of ending the call.
|
|
346
|
+
private rtThinkingWatchdog: ReturnType<typeof setTimeout> | null = null;
|
|
347
|
+
/** True once the server says no more audio_chunks are coming for the current turn — listening only resumes once this AND every scheduled chunk has actually finished playing. */
|
|
348
|
+
private rtAudioDoneArriving = true;
|
|
349
|
+
/** Resolver for "this tour step's audio has fully finished playing" when narrating over an already-open realtime session. */
|
|
350
|
+
private rtTourAudioDoneResolve: (() => void) | null = null;
|
|
351
|
+
|
|
352
|
+
static get observedAttributes() {
|
|
353
|
+
return ["persona"];
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
connectedCallback() {
|
|
357
|
+
if (this.shadow) return; // already connected once — don't rebuild on re-attach
|
|
358
|
+
this.shadow = this.attachShadow({ mode: "open" });
|
|
359
|
+
this.buildDom();
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
attributeChangedCallback(name: string) {
|
|
363
|
+
if (name === "persona" && this.panel) this.panel.setAttribute("aria-label", `${this.persona} help panel`);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// --- attributes -----------------------------------------------------
|
|
367
|
+
private get endpoint(): string { return this.getAttribute("endpoint") ?? "/api/copilot"; }
|
|
368
|
+
private get speakEndpoint(): string | null { return this.getAttribute("speak-endpoint"); }
|
|
369
|
+
private get transcribeEndpoint(): string | null { return this.getAttribute("transcribe-endpoint"); }
|
|
370
|
+
private get reportMissesEndpoint(): string | null { return this.getAttribute("report-misses-endpoint"); }
|
|
371
|
+
private get realtimeUrl(): string | null { return this.getAttribute("realtime-url"); }
|
|
372
|
+
private get persona(): string { return this.getAttribute("persona") ?? "Cairn"; }
|
|
373
|
+
private get registeredActions(): string[] {
|
|
374
|
+
return (this.getAttribute("registered-actions") ?? "").split(",").map((a) => a.trim()).filter(Boolean);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
private get realtimeActive(): boolean {
|
|
378
|
+
return this.status.startsWith("rt-");
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
private get busy(): boolean {
|
|
382
|
+
return this.status === "asking" || this.status === "rt-thinking" || this.touringActive;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
private micSupported(): boolean {
|
|
386
|
+
return typeof navigator !== "undefined" && !!navigator.mediaDevices?.getUserMedia && typeof MediaRecorder !== "undefined";
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// --- DOM construction (once) -----------------------------------------
|
|
390
|
+
private buildDom() {
|
|
391
|
+
const style = document.createElement("style");
|
|
392
|
+
style.textContent = STYLES;
|
|
393
|
+
this.shadow.appendChild(style);
|
|
394
|
+
|
|
395
|
+
this.fab = document.createElement("button");
|
|
396
|
+
this.fab.className = "cairn-fab";
|
|
397
|
+
this.fab.innerHTML = MARK_ICON;
|
|
398
|
+
this.fab.setAttribute("aria-label", `Open ${this.persona} help`);
|
|
399
|
+
this.fab.addEventListener("click", () => this.toggleOpen());
|
|
400
|
+
this.shadow.appendChild(this.fab);
|
|
401
|
+
|
|
402
|
+
this.panel = document.createElement("div");
|
|
403
|
+
this.panel.className = "cairn-panel";
|
|
404
|
+
this.panel.setAttribute("role", "dialog");
|
|
405
|
+
this.panel.setAttribute("aria-label", `${this.persona} help panel`);
|
|
406
|
+
|
|
407
|
+
this.stackEl = document.createElement("div");
|
|
408
|
+
this.stackEl.className = "cairn-stack";
|
|
409
|
+
this.stackEl.style.display = "none";
|
|
410
|
+
|
|
411
|
+
this.userBubbleEl = document.createElement("div");
|
|
412
|
+
this.userBubbleEl.className = "cairn-bubble cairn-bubble-user";
|
|
413
|
+
this.userBubbleEl.style.display = "none";
|
|
414
|
+
this.stackEl.appendChild(this.userBubbleEl);
|
|
415
|
+
|
|
416
|
+
this.agentBubbleEl = document.createElement("div");
|
|
417
|
+
this.agentBubbleEl.className = "cairn-bubble cairn-bubble-agent";
|
|
418
|
+
this.agentBubbleEl.style.display = "none";
|
|
419
|
+
|
|
420
|
+
this.chipEl = document.createElement("span");
|
|
421
|
+
this.chipEl.className = "cairn-chip";
|
|
422
|
+
this.chipEl.style.display = "none";
|
|
423
|
+
this.agentBubbleEl.appendChild(this.chipEl);
|
|
424
|
+
|
|
425
|
+
this.agentTextEl = document.createElement("span");
|
|
426
|
+
this.agentTextEl.className = "cairn-bubble-text";
|
|
427
|
+
this.agentBubbleEl.appendChild(this.agentTextEl);
|
|
428
|
+
|
|
429
|
+
this.thinkingEl = document.createElement("span");
|
|
430
|
+
this.thinkingEl.className = "cairn-thinking";
|
|
431
|
+
this.thinkingEl.setAttribute("aria-label", "Thinking");
|
|
432
|
+
this.thinkingEl.style.display = "none";
|
|
433
|
+
this.thinkingEl.innerHTML =
|
|
434
|
+
'<span class="cairn-thinking-dot"></span><span class="cairn-thinking-dot"></span><span class="cairn-thinking-dot"></span>';
|
|
435
|
+
this.agentBubbleEl.appendChild(this.thinkingEl);
|
|
436
|
+
|
|
437
|
+
this.stackEl.appendChild(this.agentBubbleEl);
|
|
438
|
+
this.panel.appendChild(this.stackEl);
|
|
439
|
+
|
|
440
|
+
// --- realtime control bar (shown instead of the form while a live call is active) ---
|
|
441
|
+
this.rtBar = document.createElement("div");
|
|
442
|
+
this.rtBar.className = "cairn-rt-bar";
|
|
443
|
+
this.rtBar.style.display = "none";
|
|
444
|
+
|
|
445
|
+
this.rtDot = document.createElement("span");
|
|
446
|
+
this.rtDot.className = "cairn-rt-dot";
|
|
447
|
+
this.rtBar.appendChild(this.rtDot);
|
|
448
|
+
|
|
449
|
+
this.rtLabel = document.createElement("span");
|
|
450
|
+
this.rtLabel.className = "cairn-rt-label";
|
|
451
|
+
this.rtBar.appendChild(this.rtLabel);
|
|
452
|
+
|
|
453
|
+
const rtControls = document.createElement("div");
|
|
454
|
+
rtControls.className = "cairn-rt-controls";
|
|
455
|
+
|
|
456
|
+
this.rtMicBtn = document.createElement("button");
|
|
457
|
+
this.rtMicBtn.type = "button";
|
|
458
|
+
this.rtMicBtn.className = "cairn-icon-btn";
|
|
459
|
+
this.rtMicBtn.innerHTML = MIC_ICON;
|
|
460
|
+
this.rtMicBtn.setAttribute("aria-label", "Mute microphone");
|
|
461
|
+
this.rtMicBtn.addEventListener("click", () => this.toggleRtMic());
|
|
462
|
+
rtControls.appendChild(this.rtMicBtn);
|
|
463
|
+
|
|
464
|
+
this.rtSpeakerBtn = document.createElement("button");
|
|
465
|
+
this.rtSpeakerBtn.type = "button";
|
|
466
|
+
this.rtSpeakerBtn.className = "cairn-icon-btn";
|
|
467
|
+
this.rtSpeakerBtn.innerHTML = VOLUME_ICON;
|
|
468
|
+
this.rtSpeakerBtn.setAttribute("aria-label", "Mute speaker");
|
|
469
|
+
this.rtSpeakerBtn.addEventListener("click", () => this.toggleRtSpeaker());
|
|
470
|
+
rtControls.appendChild(this.rtSpeakerBtn);
|
|
471
|
+
|
|
472
|
+
this.rtEndBtn = document.createElement("button");
|
|
473
|
+
this.rtEndBtn.type = "button";
|
|
474
|
+
this.rtEndBtn.className = "cairn-icon-btn cairn-icon-btn-end";
|
|
475
|
+
this.rtEndBtn.innerHTML = PHONE_OFF_ICON;
|
|
476
|
+
this.rtEndBtn.setAttribute("aria-label", "End conversation");
|
|
477
|
+
this.rtEndBtn.addEventListener("click", () => this.endRealtime());
|
|
478
|
+
rtControls.appendChild(this.rtEndBtn);
|
|
479
|
+
|
|
480
|
+
this.rtBar.appendChild(rtControls);
|
|
481
|
+
this.panel.appendChild(this.rtBar);
|
|
482
|
+
|
|
483
|
+
// --- typed question form ---
|
|
484
|
+
this.formEl = document.createElement("form");
|
|
485
|
+
const row = document.createElement("div");
|
|
486
|
+
row.className = "cairn-input-row";
|
|
487
|
+
|
|
488
|
+
this.inputEl = document.createElement("input");
|
|
489
|
+
this.inputEl.placeholder = "What do you need help with?";
|
|
490
|
+
this.inputEl.setAttribute("aria-label", `Ask ${this.persona} a question`);
|
|
491
|
+
// The send button's disabled state depends on whether there's text —
|
|
492
|
+
// native/uncontrolled input, so nothing re-evaluates that on its own
|
|
493
|
+
// without this: typing would leave "Send" permanently disabled from
|
|
494
|
+
// its initial (empty-input) state.
|
|
495
|
+
this.inputEl.addEventListener("input", () => this.updateBusyState());
|
|
496
|
+
row.appendChild(this.inputEl);
|
|
497
|
+
|
|
498
|
+
if (this.realtimeUrl && this.micSupported()) {
|
|
499
|
+
this.phoneBtn = document.createElement("button");
|
|
500
|
+
this.phoneBtn.type = "button";
|
|
501
|
+
this.phoneBtn.className = "cairn-icon-btn";
|
|
502
|
+
this.phoneBtn.innerHTML = PHONE_ICON;
|
|
503
|
+
this.phoneBtn.setAttribute("aria-label", "Start realtime conversation");
|
|
504
|
+
this.phoneBtn.addEventListener("click", () => void this.startRealtime());
|
|
505
|
+
row.appendChild(this.phoneBtn);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
if (this.transcribeEndpoint && this.micSupported()) {
|
|
509
|
+
this.micBtn = document.createElement("button");
|
|
510
|
+
this.micBtn.type = "button";
|
|
511
|
+
this.micBtn.className = "cairn-icon-btn";
|
|
512
|
+
this.micBtn.innerHTML = MIC_ICON;
|
|
513
|
+
this.micBtn.setAttribute("aria-label", "Ask by voice");
|
|
514
|
+
this.micBtn.addEventListener("click", () => (this.recording ? this.stopRecording() : void this.startRecording()));
|
|
515
|
+
row.appendChild(this.micBtn);
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
this.sendBtn = document.createElement("button");
|
|
519
|
+
this.sendBtn.type = "submit";
|
|
520
|
+
this.sendBtn.className = "cairn-send";
|
|
521
|
+
this.sendBtn.innerHTML = SEND_ICON;
|
|
522
|
+
this.sendBtn.setAttribute("aria-label", "Send");
|
|
523
|
+
row.appendChild(this.sendBtn);
|
|
524
|
+
|
|
525
|
+
this.formEl.appendChild(row);
|
|
526
|
+
this.formEl.addEventListener("submit", (e) => {
|
|
527
|
+
e.preventDefault();
|
|
528
|
+
const q = this.inputEl.value.trim();
|
|
529
|
+
if (q) void this.ask(q);
|
|
530
|
+
});
|
|
531
|
+
this.panel.appendChild(this.formEl);
|
|
532
|
+
|
|
533
|
+
this.shadow.appendChild(this.panel);
|
|
534
|
+
this.render();
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// --- open/close -------------------------------------------------------
|
|
538
|
+
private toggleOpen() {
|
|
539
|
+
this.isOpen = !this.isOpen;
|
|
540
|
+
this.panel.classList.toggle("cairn-open", this.isOpen);
|
|
541
|
+
this.fab.innerHTML = this.isOpen ? CLOSE_ICON : MARK_ICON;
|
|
542
|
+
this.fab.setAttribute("aria-label", this.isOpen ? `Close ${this.persona} help` : `Open ${this.persona} help`);
|
|
543
|
+
if (this.isOpen && !this.realtimeActive) this.inputEl.focus();
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
// --- rendering ------------------------------------------------------
|
|
547
|
+
/** Single place that reconciles all status-derived UI — called on every status/caption/touring change, targeted DOM mutation only (never rebuilds nodes, so focus/scroll position/etc. are never disturbed). */
|
|
548
|
+
private render() {
|
|
549
|
+
const realtimeActive = this.realtimeActive;
|
|
550
|
+
|
|
551
|
+
this.fab.classList.toggle("cairn-fab-speaking", this.status === "rt-speaking");
|
|
552
|
+
|
|
553
|
+
this.rtBar.style.display = realtimeActive ? "flex" : "none";
|
|
554
|
+
this.formEl.style.display = realtimeActive ? "none" : "block";
|
|
555
|
+
|
|
556
|
+
if (realtimeActive) {
|
|
557
|
+
this.rtDot.className = `cairn-rt-dot cairn-rt-dot-${this.status}`;
|
|
558
|
+
this.rtLabel.textContent = STATUS_LABEL[this.status];
|
|
559
|
+
const speaking = this.status === "rt-speaking";
|
|
560
|
+
this.rtMicBtn.className = speaking ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn";
|
|
561
|
+
this.rtSpeakerBtn.className = speaking ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn";
|
|
562
|
+
this.rtMicBtn.innerHTML = this.rtMicMuted ? MIC_OFF_ICON : MIC_ICON;
|
|
563
|
+
this.rtMicBtn.setAttribute("aria-label", this.rtMicMuted ? "Unmute microphone" : "Mute microphone");
|
|
564
|
+
this.rtSpeakerBtn.innerHTML = this.rtSpeakerMuted ? VOLUME_OFF_ICON : VOLUME_ICON;
|
|
565
|
+
this.rtSpeakerBtn.setAttribute("aria-label", this.rtSpeakerMuted ? "Unmute speaker" : "Mute speaker");
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// `caption` is overloaded by design (see its setters elsewhere): during
|
|
569
|
+
// a tour it's a step-progress label ("Step 1 of 2"), not user speech,
|
|
570
|
+
// so it reads as a small chip over the agent's bubble instead. While
|
|
571
|
+
// actively recording or on a live realtime call it's the user's own
|
|
572
|
+
// live/last transcript, so it reads as the user's floating bubble;
|
|
573
|
+
// otherwise that slot falls back to the last typed question.
|
|
574
|
+
const tourChip = this.touringActive ? this.caption : "";
|
|
575
|
+
const userCaption = !this.touringActive && (this.recording || realtimeActive) ? this.caption : this.lastQuestion ?? "";
|
|
576
|
+
const showAgent = !!this.answer || this.busy;
|
|
577
|
+
|
|
578
|
+
this.stackEl.style.display = userCaption || showAgent ? "flex" : "none";
|
|
579
|
+
|
|
580
|
+
this.userBubbleEl.style.display = userCaption ? "block" : "none";
|
|
581
|
+
this.userBubbleEl.textContent = userCaption;
|
|
582
|
+
|
|
583
|
+
this.agentBubbleEl.style.display = showAgent ? "flex" : "none";
|
|
584
|
+
// Only the tour step counter shows here — generic realtime status
|
|
585
|
+
// (listening/thinking/speaking) already has its own place in the
|
|
586
|
+
// rt-bar below; showing it a second time here was a real duplication.
|
|
587
|
+
this.chipEl.style.display = tourChip ? "inline-block" : "none";
|
|
588
|
+
this.chipEl.textContent = tourChip;
|
|
589
|
+
|
|
590
|
+
if (this.answer) {
|
|
591
|
+
this.agentTextEl.style.display = "inline";
|
|
592
|
+
this.renderCaptionWords(this.agentTextEl, this.answer);
|
|
593
|
+
this.thinkingEl.style.display = "none";
|
|
594
|
+
} else {
|
|
595
|
+
this.agentTextEl.style.display = "none";
|
|
596
|
+
this.agentTextEl.textContent = "";
|
|
597
|
+
this.thinkingEl.style.display = this.busy ? "inline-flex" : "none";
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
if (this.phoneBtn) this.phoneBtn.disabled = this.busy || this.recording;
|
|
601
|
+
this.updateBusyState();
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
private setStatus(next: Status) {
|
|
605
|
+
this.status = next;
|
|
606
|
+
this.render();
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
private setCaption(text: string) {
|
|
610
|
+
this.caption = text;
|
|
611
|
+
this.render();
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
private updateBusyState() {
|
|
615
|
+
this.inputEl.disabled = this.recording || this.touringActive;
|
|
616
|
+
this.sendBtn.disabled = this.busy || this.recording || !this.inputEl.value.trim();
|
|
617
|
+
this.sendBtn.innerHTML = this.busy ? SPINNER_ICON : SEND_ICON;
|
|
618
|
+
if (this.micBtn) this.micBtn.disabled = this.touringActive;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
private setAnswer(text: string | null) {
|
|
622
|
+
this.answer = text;
|
|
623
|
+
this.render();
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* Renders text as a sequence of spans that light up in order — a caption
|
|
628
|
+
* "sweep" that reads like the agent is speaking it, whether or not audio
|
|
629
|
+
* is actually playing right now. This is a pacing *estimate* (staggered
|
|
630
|
+
* by word position, capped so long answers don't take forever), not
|
|
631
|
+
* synced to real TTS word timestamps — Deepgram's streaming API doesn't
|
|
632
|
+
* hand those to the client today. Builds real DOM nodes with textContent
|
|
633
|
+
* (never innerHTML on the actual words) so LLM-produced text can never
|
|
634
|
+
* be interpreted as markup.
|
|
635
|
+
*/
|
|
636
|
+
private renderCaptionWords(container: HTMLElement, text: string) {
|
|
637
|
+
container.innerHTML = "";
|
|
638
|
+
const words = text.split(" ");
|
|
639
|
+
words.forEach((word, i) => {
|
|
640
|
+
const span = document.createElement("span");
|
|
641
|
+
span.className = "cairn-word";
|
|
642
|
+
span.style.animationDelay = `${Math.min(i * 55, 2800)}ms`;
|
|
643
|
+
span.textContent = i < words.length - 1 ? word + " " : word;
|
|
644
|
+
container.appendChild(span);
|
|
645
|
+
});
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
private reportMiss(context: MissContext) {
|
|
649
|
+
logMiss(context);
|
|
650
|
+
const endpoint = this.reportMissesEndpoint;
|
|
651
|
+
if (endpoint) {
|
|
652
|
+
fetch(endpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(context) }).catch(() => {});
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
// --- ask / verb execution ------------------------------------------------
|
|
657
|
+
private async ask(question: string) {
|
|
658
|
+
this.setStatus("asking");
|
|
659
|
+
this.setAnswer(null);
|
|
660
|
+
this.lastQuestion = question;
|
|
661
|
+
this.inputEl.value = "";
|
|
662
|
+
this.render();
|
|
663
|
+
try {
|
|
664
|
+
const res = await fetch(this.endpoint, {
|
|
665
|
+
method: "POST",
|
|
666
|
+
headers: { "content-type": "application/json" },
|
|
667
|
+
body: JSON.stringify({ route: location.pathname, question, visible: collectVisible(), history: this.history }),
|
|
668
|
+
});
|
|
669
|
+
const data = await res.json().catch(() => null);
|
|
670
|
+
this.handleVerb(data);
|
|
671
|
+
// Unlike the realtime relay (one persistent connection, memory lives
|
|
672
|
+
// server-side), each of these POSTs is stateless — the widget itself
|
|
673
|
+
// is what remembers, and resends it above so the model has context
|
|
674
|
+
// for "the first one" / "do that instead" on the next question.
|
|
675
|
+
this.history = [
|
|
676
|
+
...this.history,
|
|
677
|
+
{ role: "user", text: question } satisfies HistoryEntry,
|
|
678
|
+
{ role: "assistant", text: summarizeVerbForHistory(data) } satisfies HistoryEntry,
|
|
679
|
+
].slice(-MAX_HISTORY_TURNS);
|
|
680
|
+
} catch {
|
|
681
|
+
this.setAnswer("Something went wrong reaching the help service — try again in a moment.");
|
|
682
|
+
} finally {
|
|
683
|
+
this.setStatus("idle");
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
private handleVerb(raw: unknown) {
|
|
688
|
+
executeVerbResponse(raw, location.pathname, {
|
|
689
|
+
onExplain: (text) => {
|
|
690
|
+
this.setAnswer(text);
|
|
691
|
+
if (!this.realtimeActive) void this.speak(text); // realtime mode gets audio over the socket instead
|
|
692
|
+
},
|
|
693
|
+
onNavigate: (route) => {
|
|
694
|
+
location.assign(route);
|
|
695
|
+
},
|
|
696
|
+
onMiss: (ctx) => this.reportMiss(ctx),
|
|
697
|
+
onDo: (action, target) => {
|
|
698
|
+
this.dispatchEvent(new CustomEvent("cairn-do", { detail: { action, target }, bubbles: true, composed: true }));
|
|
699
|
+
},
|
|
700
|
+
onTour: (steps) => void this.runTour(steps),
|
|
701
|
+
registeredActions: this.registeredActions,
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
/**
|
|
706
|
+
* Walks a "tour" verb's steps one at a time: highlight this step's
|
|
707
|
+
* target (if any), speak/show its text, wait for that to finish, then
|
|
708
|
+
* move on. Always narrates via speakEndpoint (or the open realtime
|
|
709
|
+
* socket, if there is one), even mid realtime-call — a tour is a
|
|
710
|
+
* distinct guided walkthrough, not a conversational turn.
|
|
711
|
+
*/
|
|
712
|
+
private async runTour(steps: TourStep[]) {
|
|
713
|
+
const myGeneration = ++this.tourGeneration;
|
|
714
|
+
const wasRealtimeListening = this.realtimeActive;
|
|
715
|
+
this.touringActive = true;
|
|
716
|
+
if (wasRealtimeListening) this.setStatus("rt-speaking");
|
|
717
|
+
this.setAnswer(null);
|
|
718
|
+
// Tracked locally rather than reading location.pathname each time — a
|
|
719
|
+
// step below can navigate mid-tour, and this keeps the miss-report
|
|
720
|
+
// route accurate for every step after that.
|
|
721
|
+
let currentRoute = location.pathname;
|
|
722
|
+
|
|
723
|
+
try {
|
|
724
|
+
for (let i = 0; i < steps.length; i++) {
|
|
725
|
+
if (this.tourGeneration !== myGeneration) return; // superseded — e.g. widget closed or a new question came in
|
|
726
|
+
const step = steps[i];
|
|
727
|
+
this.setCaption(`Step ${i + 1} of ${steps.length}`);
|
|
728
|
+
this.setAnswer(step.text);
|
|
729
|
+
|
|
730
|
+
if (step.route && step.route !== currentRoute) {
|
|
731
|
+
// No client-side router here (framework-agnostic — there isn't
|
|
732
|
+
// one to assume) — a full page navigation is the honest
|
|
733
|
+
// universal fallback, and it ends this tour run since the new
|
|
734
|
+
// page has no memory of it (documented limitation, see ROADMAP).
|
|
735
|
+
location.assign(step.route);
|
|
736
|
+
return;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
if (step.target) {
|
|
740
|
+
const el = findElement(step.target);
|
|
741
|
+
if (el) highlightElement(el);
|
|
742
|
+
else this.reportMiss({ attempted: step.target, route: currentRoute });
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
if (wasRealtimeListening && this.rtSocket?.readyState === WebSocket.OPEN) {
|
|
746
|
+
// Already have a live streaming connection open — reuse it
|
|
747
|
+
// (same Speak WS, same gapless PCM scheduling a normal reply
|
|
748
|
+
// uses) instead of falling back to a separate buffered REST call.
|
|
749
|
+
await this.speakOverRealtime(step.text);
|
|
750
|
+
} else if (this.speakEndpoint) {
|
|
751
|
+
await this.speakAndWait(step.text);
|
|
752
|
+
} else {
|
|
753
|
+
// No TTS configured — pace by an estimate of reading time instead of racing through every step instantly.
|
|
754
|
+
await new Promise((resolve) => setTimeout(resolve, Math.max(1200, step.text.length * 45)));
|
|
755
|
+
}
|
|
756
|
+
if (this.tourGeneration !== myGeneration) return;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
if (this.tourGeneration !== myGeneration) return;
|
|
760
|
+
this.setCaption("");
|
|
761
|
+
if (wasRealtimeListening && this.realtimeActive) this.setStatus("rt-listening");
|
|
762
|
+
} finally {
|
|
763
|
+
if (this.tourGeneration === myGeneration) {
|
|
764
|
+
this.touringActive = false;
|
|
765
|
+
this.render();
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// --- speech -------------------------------------------------------------
|
|
771
|
+
/** Stops whatever's currently playing first, so two responses can never be heard overlapping. Used by both the typed/mic path and tours. */
|
|
772
|
+
private playResponseAudio(blob: Blob): Promise<void> {
|
|
773
|
+
if (this.activeAudio) {
|
|
774
|
+
this.activeAudio.pause();
|
|
775
|
+
this.activeAudio.currentTime = 0;
|
|
776
|
+
}
|
|
777
|
+
const url = URL.createObjectURL(blob);
|
|
778
|
+
const audio = new Audio(url);
|
|
779
|
+
this.activeAudio = audio;
|
|
780
|
+
return new Promise((resolve) => {
|
|
781
|
+
const clear = () => {
|
|
782
|
+
URL.revokeObjectURL(url);
|
|
783
|
+
if (this.activeAudio === audio) this.activeAudio = null;
|
|
784
|
+
resolve();
|
|
785
|
+
};
|
|
786
|
+
audio.onended = clear;
|
|
787
|
+
audio.play().catch(clear);
|
|
788
|
+
});
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
private async speak(text: string): Promise<void> {
|
|
792
|
+
if (!this.speakEndpoint || !text.trim()) return;
|
|
793
|
+
try {
|
|
794
|
+
const res = await fetch(this.speakEndpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ text }) });
|
|
795
|
+
if (!res.ok) return;
|
|
796
|
+
void this.playResponseAudio(await res.blob());
|
|
797
|
+
} catch {
|
|
798
|
+
// best-effort — never let speech playback break the widget
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/** Like speak(), but resolves once playback actually finishes — used by runTour() so each step's highlight stays up for exactly as long as its narration takes. */
|
|
803
|
+
private async speakAndWait(text: string): Promise<void> {
|
|
804
|
+
if (!this.speakEndpoint || !text.trim()) return;
|
|
805
|
+
try {
|
|
806
|
+
const res = await fetch(this.speakEndpoint, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ text }) });
|
|
807
|
+
if (!res.ok) return;
|
|
808
|
+
await this.playResponseAudio(await res.blob());
|
|
809
|
+
} catch {
|
|
810
|
+
// best-effort — never hang the tour
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
/** Narrates over an already-open realtime WebSocket instead of a separate REST call — resolved by maybeResumeListening() inside startRealtime() once this step's audio has both fully arrived and fully finished playing. */
|
|
815
|
+
private speakOverRealtime(text: string): Promise<void> {
|
|
816
|
+
return new Promise((resolve) => {
|
|
817
|
+
const ws = this.rtSocket;
|
|
818
|
+
if (!ws || ws.readyState !== WebSocket.OPEN) {
|
|
819
|
+
resolve();
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
this.rtAudioDoneArriving = false;
|
|
823
|
+
let settled = false;
|
|
824
|
+
const finish = () => {
|
|
825
|
+
if (settled) return;
|
|
826
|
+
settled = true;
|
|
827
|
+
this.rtTourAudioDoneResolve = null;
|
|
828
|
+
resolve();
|
|
829
|
+
};
|
|
830
|
+
this.rtTourAudioDoneResolve = finish;
|
|
831
|
+
// Safety net: if the server's "this step's audio is fully done"
|
|
832
|
+
// confirmation is ever dropped (a flaky Deepgram Flushed event, a
|
|
833
|
+
// closed connection mid-turn), don't let the tour hang on this step
|
|
834
|
+
// forever with the mic never resuming — move on instead.
|
|
835
|
+
setTimeout(() => {
|
|
836
|
+
if (!settled) console.warn("[cairn] tour step audio confirmation timed out — continuing");
|
|
837
|
+
finish();
|
|
838
|
+
}, 15000);
|
|
839
|
+
ws.send(JSON.stringify({ type: "speak", text }));
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
// --- push-to-talk mic -----------------------------------------------------
|
|
844
|
+
private async startRecording() {
|
|
845
|
+
if (!this.transcribeEndpoint || !this.micSupported() || this.realtimeActive) return;
|
|
846
|
+
try {
|
|
847
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
848
|
+
const recorder = new MediaRecorder(stream);
|
|
849
|
+
this.audioChunks = [];
|
|
850
|
+
this.setCaption("");
|
|
851
|
+
recorder.ondataavailable = (e) => {
|
|
852
|
+
if (e.data.size === 0) return;
|
|
853
|
+
this.audioChunks.push(e.data);
|
|
854
|
+
void this.transcribeSoFar(recorder.mimeType || "audio/webm", true);
|
|
855
|
+
};
|
|
856
|
+
recorder.onstop = () => void this.transcribeSoFar(recorder.mimeType || "audio/webm", false);
|
|
857
|
+
this.mediaRecorder = recorder;
|
|
858
|
+
recorder.start(2000);
|
|
859
|
+
this.recording = true;
|
|
860
|
+
if (this.micBtn) {
|
|
861
|
+
this.micBtn.innerHTML = SQUARE_ICON;
|
|
862
|
+
this.micBtn.classList.add("cairn-icon-btn-recording");
|
|
863
|
+
this.micBtn.setAttribute("aria-label", "Stop recording");
|
|
864
|
+
}
|
|
865
|
+
this.render();
|
|
866
|
+
} catch {
|
|
867
|
+
this.setAnswer("Couldn't access the microphone — check your browser's permission for this site.");
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
private stopRecording() {
|
|
872
|
+
const stream = this.mediaRecorder?.stream;
|
|
873
|
+
this.mediaRecorder?.stop();
|
|
874
|
+
stream?.getTracks().forEach((t) => t.stop());
|
|
875
|
+
this.recording = false;
|
|
876
|
+
if (this.micBtn) {
|
|
877
|
+
this.micBtn.innerHTML = MIC_ICON;
|
|
878
|
+
this.micBtn.classList.remove("cairn-icon-btn-recording");
|
|
879
|
+
this.micBtn.setAttribute("aria-label", "Ask by voice");
|
|
880
|
+
}
|
|
881
|
+
this.render();
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
private async transcribeSoFar(mimeType: string, isProgressive: boolean) {
|
|
885
|
+
if (!this.transcribeEndpoint) return;
|
|
886
|
+
if (isProgressive && this.transcribeInFlight) return;
|
|
887
|
+
this.transcribeInFlight = true;
|
|
888
|
+
try {
|
|
889
|
+
const blob = new Blob(this.audioChunks, { type: mimeType });
|
|
890
|
+
const res = await fetch(this.transcribeEndpoint, { method: "POST", headers: { "content-type": mimeType }, body: blob });
|
|
891
|
+
const data = await res.json().catch(() => null);
|
|
892
|
+
if (data?.text) {
|
|
893
|
+
this.inputEl.value = data.text;
|
|
894
|
+
this.setCaption(data.text);
|
|
895
|
+
this.updateBusyState();
|
|
896
|
+
} else if (!isProgressive) {
|
|
897
|
+
this.setAnswer("Couldn't make that out — try typing instead.");
|
|
898
|
+
}
|
|
899
|
+
} catch {
|
|
900
|
+
if (!isProgressive) this.setAnswer("Couldn't reach the transcription service.");
|
|
901
|
+
} finally {
|
|
902
|
+
this.transcribeInFlight = false;
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
// ---------------------------------------------------------------------
|
|
907
|
+
// Real-time voice conversation
|
|
908
|
+
// ---------------------------------------------------------------------
|
|
909
|
+
|
|
910
|
+
private async startRealtime() {
|
|
911
|
+
// rtStarting closes the gap between click and the first status update
|
|
912
|
+
// landing — without it a rapid double-click could race past the
|
|
913
|
+
// realtimeActive check twice and open two sessions.
|
|
914
|
+
if (!this.realtimeUrl || !this.micSupported() || this.realtimeActive || this.rtStarting) return;
|
|
915
|
+
this.rtStarting = true;
|
|
916
|
+
this.setAnswer(null);
|
|
917
|
+
this.setCaption("");
|
|
918
|
+
this.setStatus("rt-connecting");
|
|
919
|
+
|
|
920
|
+
try {
|
|
921
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
922
|
+
const ws = new WebSocket(this.realtimeUrl);
|
|
923
|
+
ws.binaryType = "arraybuffer";
|
|
924
|
+
this.rtSocket = ws;
|
|
925
|
+
|
|
926
|
+
// Separate AudioContext from the mic capture graph below — one for
|
|
927
|
+
// capture, one for playback, independently lifecycled (playback
|
|
928
|
+
// keeps scheduling audio after a turn while the mic graph is
|
|
929
|
+
// simultaneously idle, and vice versa).
|
|
930
|
+
const playbackCtx = new AudioContext();
|
|
931
|
+
const playbackGain = playbackCtx.createGain();
|
|
932
|
+
playbackGain.gain.value = this.rtSpeakerMuted ? 0 : 1;
|
|
933
|
+
playbackGain.connect(playbackCtx.destination);
|
|
934
|
+
this.rtPlaybackCtx = playbackCtx;
|
|
935
|
+
this.rtPlaybackGain = playbackGain;
|
|
936
|
+
this.rtNextPlayTime = 0;
|
|
937
|
+
this.rtScheduledSources = [];
|
|
938
|
+
this.rtAudioDoneArriving = true;
|
|
939
|
+
|
|
940
|
+
const audioCtx = new AudioContext();
|
|
941
|
+
const source = audioCtx.createMediaStreamSource(stream);
|
|
942
|
+
// ScriptProcessorNode is deprecated in favor of AudioWorklet, but
|
|
943
|
+
// needs no separate worklet file to serve — fine for this scope,
|
|
944
|
+
// still supported everywhere. Routed through a silent gain (not
|
|
945
|
+
// straight to destination) so the mic input is never audibly looped back.
|
|
946
|
+
const processor = audioCtx.createScriptProcessor(4096, 1, 1);
|
|
947
|
+
const silence = audioCtx.createGain();
|
|
948
|
+
silence.gain.value = 0;
|
|
949
|
+
|
|
950
|
+
// Only flips back to "listening" (and lets the mic resume sending)
|
|
951
|
+
// once BOTH the server has said no more audio is coming for this
|
|
952
|
+
// turn AND every chunk already scheduled has actually finished
|
|
953
|
+
// playing — from playback completion, not the server's speaking_end
|
|
954
|
+
// alone, which is what stops the mic picking up the tail end of the
|
|
955
|
+
// agent's own voice. Shared with speakOverRealtime(): while touring,
|
|
956
|
+
// this same "audio fully drained" condition resolves the current
|
|
957
|
+
// step's wait instead of touching status/caption.
|
|
958
|
+
const maybeResumeListening = () => {
|
|
959
|
+
if (!this.rtAudioDoneArriving) return;
|
|
960
|
+
if (this.rtScheduledSources.length > 0) return;
|
|
961
|
+
if (this.touringActive) {
|
|
962
|
+
this.rtTourAudioDoneResolve?.();
|
|
963
|
+
this.rtTourAudioDoneResolve = null;
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
this.setStatus("rt-listening");
|
|
967
|
+
this.setCaption("");
|
|
968
|
+
};
|
|
969
|
+
|
|
970
|
+
const disarmThinkingWatchdog = () => {
|
|
971
|
+
if (this.rtThinkingWatchdog) {
|
|
972
|
+
clearTimeout(this.rtThinkingWatchdog);
|
|
973
|
+
this.rtThinkingWatchdog = null;
|
|
974
|
+
}
|
|
975
|
+
};
|
|
976
|
+
|
|
977
|
+
const armThinkingWatchdog = () => {
|
|
978
|
+
disarmThinkingWatchdog();
|
|
979
|
+
this.rtThinkingWatchdog = setTimeout(() => {
|
|
980
|
+
this.rtThinkingWatchdog = null;
|
|
981
|
+
console.warn("[cairn] realtime turn timed out waiting on the server — resuming listening");
|
|
982
|
+
this.rtAudioDoneArriving = true;
|
|
983
|
+
this.setStatus("rt-listening");
|
|
984
|
+
this.setCaption("");
|
|
985
|
+
}, 20000);
|
|
986
|
+
};
|
|
987
|
+
|
|
988
|
+
const stopScheduledRtAudio = () => {
|
|
989
|
+
for (const node of this.rtScheduledSources) {
|
|
990
|
+
try {
|
|
991
|
+
node.stop();
|
|
992
|
+
} catch {
|
|
993
|
+
// may have already finished naturally
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
this.rtScheduledSources = [];
|
|
997
|
+
this.rtNextPlayTime = this.rtPlaybackCtx?.currentTime ?? 0;
|
|
998
|
+
};
|
|
999
|
+
|
|
1000
|
+
// Stops the agent immediately (locally) and tells the server to
|
|
1001
|
+
// discard whatever it's still synthesizing/sending for this turn —
|
|
1002
|
+
// the server tags every turn with a generation number and drops any
|
|
1003
|
+
// now-stale audio_chunk/speaking_end already in flight.
|
|
1004
|
+
const triggerBargeIn = () => {
|
|
1005
|
+
disarmThinkingWatchdog();
|
|
1006
|
+
stopScheduledRtAudio();
|
|
1007
|
+
this.rtAudioDoneArriving = true;
|
|
1008
|
+
if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "barge_in" }));
|
|
1009
|
+
this.setStatus("rt-listening");
|
|
1010
|
+
this.setCaption("");
|
|
1011
|
+
};
|
|
1012
|
+
|
|
1013
|
+
processor.onaudioprocess = (e) => {
|
|
1014
|
+
if (ws.readyState !== WebSocket.OPEN) return;
|
|
1015
|
+
if (this.rtMicMuted) return;
|
|
1016
|
+
|
|
1017
|
+
// Barge-in: while the agent is speaking a real conversational
|
|
1018
|
+
// reply (not touring — a tour deliberately can't be talked over),
|
|
1019
|
+
// keep listening to the mic locally even though it isn't being
|
|
1020
|
+
// sent yet, and cut the agent off the instant the user starts
|
|
1021
|
+
// talking over it instead of making them wait for it to finish.
|
|
1022
|
+
if (this.status === "rt-speaking" && !this.touringActive) {
|
|
1023
|
+
const rms = computeRms(e.inputBuffer.getChannelData(0));
|
|
1024
|
+
if (rms > BARGE_IN_RMS_THRESHOLD) triggerBargeIn();
|
|
1025
|
+
return;
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
if (this.status !== "rt-listening") return; // don't send our own mic while the agent is thinking/speaking
|
|
1029
|
+
const pcm = floatTo16BitPCM(downsampleTo16k(e.inputBuffer.getChannelData(0), audioCtx.sampleRate));
|
|
1030
|
+
ws.send(pcm);
|
|
1031
|
+
};
|
|
1032
|
+
source.connect(processor);
|
|
1033
|
+
processor.connect(silence);
|
|
1034
|
+
silence.connect(audioCtx.destination);
|
|
1035
|
+
|
|
1036
|
+
this.rtCleanup = () => {
|
|
1037
|
+
processor.disconnect();
|
|
1038
|
+
source.disconnect();
|
|
1039
|
+
stream.getTracks().forEach((t) => t.stop());
|
|
1040
|
+
void audioCtx.close();
|
|
1041
|
+
stopScheduledRtAudio();
|
|
1042
|
+
void playbackCtx.close();
|
|
1043
|
+
this.rtPlaybackCtx = null;
|
|
1044
|
+
this.rtPlaybackGain = null;
|
|
1045
|
+
};
|
|
1046
|
+
|
|
1047
|
+
ws.onopen = () => {
|
|
1048
|
+
ws.send(JSON.stringify({ type: "context", route: location.pathname, visible: collectVisible() }));
|
|
1049
|
+
this.setStatus("rt-listening");
|
|
1050
|
+
this.rtStarting = false;
|
|
1051
|
+
};
|
|
1052
|
+
|
|
1053
|
+
ws.onmessage = (event) => {
|
|
1054
|
+
if (typeof event.data !== "string") return; // audio arrives as base64 inside audio_chunk, not raw binary frames
|
|
1055
|
+
const msg = JSON.parse(event.data);
|
|
1056
|
+
if (msg.type === "interim") {
|
|
1057
|
+
this.setCaption(msg.text);
|
|
1058
|
+
} else if (msg.type === "final") {
|
|
1059
|
+
this.setCaption(msg.text);
|
|
1060
|
+
this.setStatus("rt-thinking");
|
|
1061
|
+
armThinkingWatchdog();
|
|
1062
|
+
} else if (msg.type === "verb") {
|
|
1063
|
+
disarmThinkingWatchdog();
|
|
1064
|
+
this.handleVerb(msg.verb);
|
|
1065
|
+
} else if (msg.type === "speaking_start") {
|
|
1066
|
+
disarmThinkingWatchdog();
|
|
1067
|
+
this.rtAudioDoneArriving = false;
|
|
1068
|
+
this.setStatus("rt-speaking");
|
|
1069
|
+
} else if (msg.type === "audio_chunk") {
|
|
1070
|
+
const ctx = this.rtPlaybackCtx;
|
|
1071
|
+
const gain = this.rtPlaybackGain;
|
|
1072
|
+
if (!ctx || !gain) return;
|
|
1073
|
+
void ctx.resume().catch(() => {});
|
|
1074
|
+
|
|
1075
|
+
// Decode base64 linear16 PCM -> Float32 samples in [-1, 1], then
|
|
1076
|
+
// schedule gapless-appended after whatever's already queued —
|
|
1077
|
+
// this is what lets playback start on the first chunk instead
|
|
1078
|
+
// of waiting for the whole reply.
|
|
1079
|
+
const bytes = Uint8Array.from(atob(msg.audio), (c) => c.charCodeAt(0));
|
|
1080
|
+
const sampleCount = bytes.length / 2;
|
|
1081
|
+
const float32 = new Float32Array(sampleCount);
|
|
1082
|
+
const view = new DataView(bytes.buffer);
|
|
1083
|
+
for (let i = 0; i < sampleCount; i++) {
|
|
1084
|
+
float32[i] = view.getInt16(i * 2, true) / 32768;
|
|
1085
|
+
}
|
|
1086
|
+
const sampleRate = typeof msg.sampleRate === "number" ? msg.sampleRate : 24000;
|
|
1087
|
+
const buffer = ctx.createBuffer(1, sampleCount, sampleRate);
|
|
1088
|
+
buffer.copyToChannel(float32, 0);
|
|
1089
|
+
|
|
1090
|
+
const bufferSource = ctx.createBufferSource();
|
|
1091
|
+
bufferSource.buffer = buffer;
|
|
1092
|
+
bufferSource.connect(gain);
|
|
1093
|
+
|
|
1094
|
+
const startAt = Math.max(ctx.currentTime, this.rtNextPlayTime);
|
|
1095
|
+
bufferSource.start(startAt);
|
|
1096
|
+
this.rtNextPlayTime = startAt + buffer.duration;
|
|
1097
|
+
|
|
1098
|
+
this.rtScheduledSources.push(bufferSource);
|
|
1099
|
+
bufferSource.onended = () => {
|
|
1100
|
+
this.rtScheduledSources = this.rtScheduledSources.filter((n) => n !== bufferSource);
|
|
1101
|
+
maybeResumeListening();
|
|
1102
|
+
};
|
|
1103
|
+
} else if (msg.type === "speaking_end" || msg.type === "turn_complete") {
|
|
1104
|
+
// turn_complete covers a verb with nothing spoken — no audio_chunk
|
|
1105
|
+
// ever arrives for it, so rtScheduledSources is already empty and
|
|
1106
|
+
// maybeResumeListening() resumes immediately.
|
|
1107
|
+
disarmThinkingWatchdog();
|
|
1108
|
+
this.rtAudioDoneArriving = true;
|
|
1109
|
+
maybeResumeListening();
|
|
1110
|
+
} else if (msg.type === "error") {
|
|
1111
|
+
// Must actually unstick the turn, not just show the message —
|
|
1112
|
+
// otherwise the mic never resumes and the session is stuck
|
|
1113
|
+
// exactly the way a silently-dropped response used to leave it.
|
|
1114
|
+
disarmThinkingWatchdog();
|
|
1115
|
+
this.setAnswer(msg.message ?? "Something went wrong.");
|
|
1116
|
+
if (!this.touringActive) {
|
|
1117
|
+
this.setStatus("rt-listening");
|
|
1118
|
+
this.setCaption("");
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
};
|
|
1122
|
+
|
|
1123
|
+
ws.onerror = () => {
|
|
1124
|
+
this.setAnswer("Couldn't connect to the realtime voice service.");
|
|
1125
|
+
this.endRealtime();
|
|
1126
|
+
};
|
|
1127
|
+
ws.onclose = () => {
|
|
1128
|
+
if (this.status !== "idle") this.endRealtime();
|
|
1129
|
+
};
|
|
1130
|
+
} catch {
|
|
1131
|
+
this.setAnswer("Couldn't access the microphone — check your browser's permission for this site.");
|
|
1132
|
+
this.setStatus("idle");
|
|
1133
|
+
this.rtStarting = false;
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
private endRealtime() {
|
|
1138
|
+
if (this.rtThinkingWatchdog) {
|
|
1139
|
+
clearTimeout(this.rtThinkingWatchdog);
|
|
1140
|
+
this.rtThinkingWatchdog = null;
|
|
1141
|
+
}
|
|
1142
|
+
this.rtStarting = false;
|
|
1143
|
+
this.activeAudio?.pause();
|
|
1144
|
+
this.activeAudio = null;
|
|
1145
|
+
this.rtSocket?.close();
|
|
1146
|
+
this.rtSocket = null;
|
|
1147
|
+
this.rtCleanup?.();
|
|
1148
|
+
this.rtCleanup = null;
|
|
1149
|
+
this.rtMicMuted = false;
|
|
1150
|
+
this.rtSpeakerMuted = false;
|
|
1151
|
+
this.setCaption("");
|
|
1152
|
+
this.setStatus("idle");
|
|
1153
|
+
this.tourGeneration++; // cancel an in-progress tour rather than leaving it stuck waiting to resume rt-listening
|
|
1154
|
+
this.touringActive = false;
|
|
1155
|
+
// Unstick a tour step mid-narration over realtime — the socket above is
|
|
1156
|
+
// already closed, so nothing will ever deliver the audio_chunk/
|
|
1157
|
+
// speaking_end that would normally resolve this.
|
|
1158
|
+
this.rtTourAudioDoneResolve?.();
|
|
1159
|
+
this.rtTourAudioDoneResolve = null;
|
|
1160
|
+
this.render();
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
private toggleRtMic() {
|
|
1164
|
+
this.rtMicMuted = !this.rtMicMuted;
|
|
1165
|
+
this.render();
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
private toggleRtSpeaker() {
|
|
1169
|
+
this.rtSpeakerMuted = !this.rtSpeakerMuted;
|
|
1170
|
+
// Zeroing the shared gain node silences output immediately, including
|
|
1171
|
+
// whatever's mid-playback right now, and applies to every future
|
|
1172
|
+
// scheduled chunk automatically — no per-chunk check needed.
|
|
1173
|
+
if (this.rtPlaybackGain) this.rtPlaybackGain.gain.value = this.rtSpeakerMuted ? 0 : 1;
|
|
1174
|
+
this.render();
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
/** Best-effort text form of a raw (unvalidated) verb response for the
|
|
1179
|
+
* conversation-history log — not shown to the user, just fed back to the
|
|
1180
|
+
* model on later turns. */
|
|
1181
|
+
function summarizeVerbForHistory(raw: unknown): string {
|
|
1182
|
+
if (!raw || typeof raw !== "object") return "(no response)";
|
|
1183
|
+
const v = raw as Record<string, unknown>;
|
|
1184
|
+
if (typeof v.text === "string" && v.text) return v.text;
|
|
1185
|
+
switch (v.verb) {
|
|
1186
|
+
case "highlight":
|
|
1187
|
+
case "open":
|
|
1188
|
+
return `(highlighted ${String(v.target)})`;
|
|
1189
|
+
case "navigate":
|
|
1190
|
+
return `(navigated to ${String(v.route)})`;
|
|
1191
|
+
case "do":
|
|
1192
|
+
return `(ran ${String(v.action)}${v.target ? ` on ${String(v.target)}` : ""})`;
|
|
1193
|
+
case "tour":
|
|
1194
|
+
return Array.isArray(v.steps) ? v.steps.map((s: { text?: string }) => s.text ?? "").join(" ") : "(tour)";
|
|
1195
|
+
default:
|
|
1196
|
+
return "(no response)";
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
// ---------------------------------------------------------------------------
|
|
1201
|
+
// Audio helpers (real-time PCM16 capture — standard Web Audio API patterns,
|
|
1202
|
+
// identical to index.tsx's — same protocol, same math, ported directly)
|
|
1203
|
+
// ---------------------------------------------------------------------------
|
|
1204
|
+
|
|
1205
|
+
// Heuristic energy gate for barge-in: real speech into a laptop/phone mic
|
|
1206
|
+
// typically sits well above this; normal room noise and the mic's own
|
|
1207
|
+
// noise floor typically sit below it. Same threshold as the React widget —
|
|
1208
|
+
// not independently recalibrated, since it's the same audio pipeline.
|
|
1209
|
+
const BARGE_IN_RMS_THRESHOLD = 0.02;
|
|
1210
|
+
|
|
1211
|
+
function computeRms(channelData: Float32Array): number {
|
|
1212
|
+
let sumSquares = 0;
|
|
1213
|
+
for (let i = 0; i < channelData.length; i++) sumSquares += channelData[i] * channelData[i];
|
|
1214
|
+
return Math.sqrt(sumSquares / channelData.length);
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
function downsampleTo16k(input: Float32Array, inputSampleRate: number): Float32Array {
|
|
1218
|
+
const targetRate = 16000;
|
|
1219
|
+
if (inputSampleRate === targetRate) return input;
|
|
1220
|
+
const ratio = inputSampleRate / targetRate;
|
|
1221
|
+
const outLength = Math.round(input.length / ratio);
|
|
1222
|
+
const result = new Float32Array(outLength);
|
|
1223
|
+
let offsetResult = 0;
|
|
1224
|
+
let offsetInput = 0;
|
|
1225
|
+
while (offsetResult < outLength) {
|
|
1226
|
+
const nextOffsetInput = Math.round((offsetResult + 1) * ratio);
|
|
1227
|
+
let accum = 0;
|
|
1228
|
+
let count = 0;
|
|
1229
|
+
for (let i = offsetInput; i < nextOffsetInput && i < input.length; i++) {
|
|
1230
|
+
accum += input[i];
|
|
1231
|
+
count++;
|
|
1232
|
+
}
|
|
1233
|
+
result[offsetResult] = count > 0 ? accum / count : 0;
|
|
1234
|
+
offsetResult++;
|
|
1235
|
+
offsetInput = nextOffsetInput;
|
|
1236
|
+
}
|
|
1237
|
+
return result;
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
function floatTo16BitPCM(input: Float32Array): ArrayBuffer {
|
|
1241
|
+
const buffer = new ArrayBuffer(input.length * 2);
|
|
1242
|
+
const view = new DataView(buffer);
|
|
1243
|
+
for (let i = 0; i < input.length; i++) {
|
|
1244
|
+
const s = Math.max(-1, Math.min(1, input[i]));
|
|
1245
|
+
view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
|
1246
|
+
}
|
|
1247
|
+
return buffer;
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
if (typeof customElements !== "undefined" && !customElements.get("cairn-widget")) {
|
|
1251
|
+
customElements.define("cairn-widget", CairnWidgetElement);
|
|
1252
|
+
}
|