@mihe-dev/js 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +114 -2
- package/dist/audio.d.ts +121 -0
- package/dist/audio.d.ts.map +1 -0
- package/dist/audio.js +377 -0
- package/dist/audio.js.map +1 -0
- package/dist/chart.d.ts +15 -0
- package/dist/chart.d.ts.map +1 -0
- package/dist/chart.js +58 -0
- package/dist/chart.js.map +1 -0
- package/dist/client.d.ts +226 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +358 -0
- package/dist/client.js.map +1 -0
- package/dist/commands.d.ts +70 -0
- package/dist/commands.d.ts.map +1 -0
- package/dist/commands.js +295 -0
- package/dist/commands.js.map +1 -0
- package/dist/debug.d.ts +17 -0
- package/dist/debug.d.ts.map +1 -0
- package/dist/debug.js +29 -0
- package/dist/debug.js.map +1 -0
- package/dist/declarations.d.ts +160 -0
- package/dist/declarations.d.ts.map +1 -0
- package/dist/declarations.js +171 -0
- package/dist/declarations.js.map +1 -0
- package/dist/highlight.d.ts +77 -0
- package/dist/highlight.d.ts.map +1 -0
- package/dist/highlight.js +233 -0
- package/dist/highlight.js.map +1 -0
- package/dist/index.d.ts +39 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +31 -0
- package/dist/index.js.map +1 -0
- package/dist/mihe.d.ts +343 -0
- package/dist/mihe.d.ts.map +1 -0
- package/dist/mihe.iife.min.js +206 -0
- package/dist/mihe.iife.min.js.map +7 -0
- package/dist/mihe.js +720 -0
- package/dist/mihe.js.map +1 -0
- package/dist/mihe.min.js +206 -0
- package/dist/mihe.min.js.map +7 -0
- package/dist/store.d.ts +175 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +208 -0
- package/dist/store.js.map +1 -0
- package/dist/voice.d.ts +192 -0
- package/dist/voice.d.ts.map +1 -0
- package/dist/voice.js +443 -0
- package/dist/voice.js.map +1 -0
- package/dist/widget/card.d.ts +15 -0
- package/dist/widget/card.d.ts.map +1 -0
- package/dist/widget/card.js +28 -0
- package/dist/widget/card.js.map +1 -0
- package/dist/widget/dom.d.ts +24 -0
- package/dist/widget/dom.d.ts.map +1 -0
- package/dist/widget/dom.js +49 -0
- package/dist/widget/dom.js.map +1 -0
- package/dist/widget/element.d.ts +88 -0
- package/dist/widget/element.d.ts.map +1 -0
- package/dist/widget/element.js +285 -0
- package/dist/widget/element.js.map +1 -0
- package/dist/widget/format.d.ts +24 -0
- package/dist/widget/format.d.ts.map +1 -0
- package/dist/widget/format.js +64 -0
- package/dist/widget/format.js.map +1 -0
- package/dist/widget/icons.d.ts +5 -0
- package/dist/widget/icons.d.ts.map +1 -0
- package/dist/widget/icons.js +19 -0
- package/dist/widget/icons.js.map +1 -0
- package/dist/widget/launcher.d.ts +13 -0
- package/dist/widget/launcher.d.ts.map +1 -0
- package/dist/widget/launcher.js +27 -0
- package/dist/widget/launcher.js.map +1 -0
- package/dist/widget/mark.d.ts +10 -0
- package/dist/widget/mark.d.ts.map +1 -0
- package/dist/widget/mark.js +20 -0
- package/dist/widget/mark.js.map +1 -0
- package/dist/widget/orb.d.ts +91 -0
- package/dist/widget/orb.d.ts.map +1 -0
- package/dist/widget/orb.js +497 -0
- package/dist/widget/orb.js.map +1 -0
- package/dist/widget/panel.d.ts +40 -0
- package/dist/widget/panel.d.ts.map +1 -0
- package/dist/widget/panel.js +239 -0
- package/dist/widget/panel.js.map +1 -0
- package/dist/widget/styles.d.ts +22 -0
- package/dist/widget/styles.d.ts.map +1 -0
- package/dist/widget/styles.js +181 -0
- package/dist/widget/styles.js.map +1 -0
- package/package.json +43 -4
package/dist/voice.js
ADDED
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The voice session (doc 03, phase 5a).
|
|
3
|
+
*
|
|
4
|
+
* A second, short-lived WebSocket carrying audio only. Commands and
|
|
5
|
+
* confirmation cards keep arriving on the chat socket, which is the whole
|
|
6
|
+
* design: voice is an input modality, not a second command channel, so a
|
|
7
|
+
* confirmation card is the same card whether the user typed or spoke and there
|
|
8
|
+
* is exactly one place that draws it.
|
|
9
|
+
*
|
|
10
|
+
* Push-to-talk, deliberately, before hands-free. It has no VAD edge cases, it
|
|
11
|
+
* works on a phone the day it ships, and the button press doubles as the user
|
|
12
|
+
* gesture iOS Safari requires before any audio can start. Hands-free is 5b.
|
|
13
|
+
*/
|
|
14
|
+
import { VOICE_SAMPLE_RATE, } from "@mihe-dev/protocol";
|
|
15
|
+
import { MicrophoneCapture, SpeakerQueue, pcmLevel } from "./audio.js";
|
|
16
|
+
export class VoiceSession {
|
|
17
|
+
options;
|
|
18
|
+
socket = null;
|
|
19
|
+
state = "idle";
|
|
20
|
+
mic;
|
|
21
|
+
speaker;
|
|
22
|
+
/**
|
|
23
|
+
* Audio captured since a held press began, not yet sent.
|
|
24
|
+
*
|
|
25
|
+
* A press on a draggable orb is ambiguous for its first moment: it may be
|
|
26
|
+
* the start of speech or the start of a drag. Holding the frames here means
|
|
27
|
+
* neither answer costs anything. Confirmed, they go up in order and the first
|
|
28
|
+
* word survives. Discarded, they never leave the browser, because the voice
|
|
29
|
+
* socket has no way to take back audio it has been sent.
|
|
30
|
+
*/
|
|
31
|
+
held = null;
|
|
32
|
+
stopped = false;
|
|
33
|
+
/**
|
|
34
|
+
* Milliseconds of audio captured since the last commit.
|
|
35
|
+
*
|
|
36
|
+
* Counted rather than timed from the button press, because those differ:
|
|
37
|
+
* the microphone graph delivers in render quanta, so a press held for
|
|
38
|
+
* exactly the floor produces slightly less audio than the clock suggests.
|
|
39
|
+
* The provider measures the buffer, so we measure the buffer too.
|
|
40
|
+
*/
|
|
41
|
+
utteranceMs = 0;
|
|
42
|
+
/** The assistant's line so far, joined from streamed transcript deltas. */
|
|
43
|
+
assistantLine = "";
|
|
44
|
+
/** Reports playback position while speaking, for barge-in truncation. */
|
|
45
|
+
playbackTimer = null;
|
|
46
|
+
/**
|
|
47
|
+
* The provider's shortest acceptable utterance, learned at `ready`.
|
|
48
|
+
*
|
|
49
|
+
* Defaults high enough to be safe before the runtime has told us, since
|
|
50
|
+
* guessing low means the first press of a session is the one that fails.
|
|
51
|
+
*/
|
|
52
|
+
minUtteranceMs = 150;
|
|
53
|
+
constructor(options) {
|
|
54
|
+
this.options = options;
|
|
55
|
+
this.speaker = new SpeakerQueue((level) => options.onLevel?.("assistant", level));
|
|
56
|
+
this.mic = new MicrophoneCapture((pcm) => this.sendAudio(pcm));
|
|
57
|
+
}
|
|
58
|
+
get currentState() {
|
|
59
|
+
return this.state;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Open the socket and the audio devices.
|
|
63
|
+
*
|
|
64
|
+
* Must be called from a user gesture. Both the microphone permission prompt
|
|
65
|
+
* and iOS Safari's audio unlock require one, and a session started from an
|
|
66
|
+
* effect will connect happily and then be silent in both directions with no
|
|
67
|
+
* error anywhere.
|
|
68
|
+
*/
|
|
69
|
+
async start() {
|
|
70
|
+
if (this.socket)
|
|
71
|
+
return;
|
|
72
|
+
this.stopped = false;
|
|
73
|
+
this.setState("connecting");
|
|
74
|
+
try {
|
|
75
|
+
// Devices before the socket: a user who declines the microphone prompt
|
|
76
|
+
// should not have consumed their single-use voice token, because the
|
|
77
|
+
// next attempt would then fail for a second, unrelated reason.
|
|
78
|
+
await this.mic.start();
|
|
79
|
+
await this.speaker.start();
|
|
80
|
+
}
|
|
81
|
+
catch (cause) {
|
|
82
|
+
this.emit({
|
|
83
|
+
type: "error",
|
|
84
|
+
code: "microphone_unavailable",
|
|
85
|
+
message: cause instanceof Error && cause.name === "NotAllowedError"
|
|
86
|
+
? "Microphone access was declined, so voice cannot start."
|
|
87
|
+
: "No microphone is available.",
|
|
88
|
+
// No microphone means no voice at all, so this one really is the end.
|
|
89
|
+
recoverable: false,
|
|
90
|
+
fallbackToChat: true,
|
|
91
|
+
});
|
|
92
|
+
this.setState("error");
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
const factory = this.options.socketFactory ??
|
|
96
|
+
((url) => new WebSocket(url));
|
|
97
|
+
const socket = factory(this.options.url);
|
|
98
|
+
socket.binaryType = "arraybuffer";
|
|
99
|
+
this.socket = socket;
|
|
100
|
+
socket.onmessage = (event) => {
|
|
101
|
+
if (typeof event.data === "string") {
|
|
102
|
+
this.handleControl(event.data);
|
|
103
|
+
}
|
|
104
|
+
else if (event.data instanceof ArrayBuffer) {
|
|
105
|
+
this.speaker.enqueue(event.data);
|
|
106
|
+
// The runtime never says `speaking`; the first audio of a reply is how
|
|
107
|
+
// the client knows. Without this a push-to-talk session sat in
|
|
108
|
+
// `thinking` after its first answer, `speech_done` only moves
|
|
109
|
+
// `speaking` on, and every later press was refused.
|
|
110
|
+
if (this.state === "thinking")
|
|
111
|
+
this.setState("speaking");
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
socket.onclose = () => {
|
|
115
|
+
// No reconnect, unlike the chat socket. A voice token is single use and
|
|
116
|
+
// a provider session is not resumable, so reconnecting would mean a new
|
|
117
|
+
// session with none of this one's context — better to end cleanly and
|
|
118
|
+
// let the user press the button again.
|
|
119
|
+
void this.stop();
|
|
120
|
+
};
|
|
121
|
+
socket.onerror = () => {
|
|
122
|
+
// `onclose` always follows.
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* The user pressed the button. Push-to-talk only.
|
|
127
|
+
*
|
|
128
|
+
* With `hold`, capture starts but nothing is sent, and the assistant is not
|
|
129
|
+
* interrupted, until `confirmUtterance`. `discardUtterance` drops it
|
|
130
|
+
* instead. For a control where a press might turn out to be a drag.
|
|
131
|
+
*/
|
|
132
|
+
beginUtterance(options) {
|
|
133
|
+
if (this.options.handsFree)
|
|
134
|
+
return;
|
|
135
|
+
if (this.state !== "listening" && this.state !== "speaking")
|
|
136
|
+
return;
|
|
137
|
+
this.utteranceMs = 0;
|
|
138
|
+
if (options?.hold) {
|
|
139
|
+
this.held = [];
|
|
140
|
+
this.mic.unmute();
|
|
141
|
+
this.setState("recording");
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
this.interruptPlayback();
|
|
145
|
+
this.mic.unmute();
|
|
146
|
+
this.setState("recording");
|
|
147
|
+
}
|
|
148
|
+
/** A held press was speech after all: send what was captured, and carry on. */
|
|
149
|
+
confirmUtterance() {
|
|
150
|
+
const held = this.held;
|
|
151
|
+
if (!held)
|
|
152
|
+
return;
|
|
153
|
+
this.held = null;
|
|
154
|
+
this.interruptPlayback();
|
|
155
|
+
for (const pcm of held)
|
|
156
|
+
this.sendAudio(pcm);
|
|
157
|
+
}
|
|
158
|
+
/** A held press was a drag: drop what was captured, and send nothing. */
|
|
159
|
+
discardUtterance() {
|
|
160
|
+
if (!this.held)
|
|
161
|
+
return;
|
|
162
|
+
this.held = null;
|
|
163
|
+
this.mic.mute();
|
|
164
|
+
this.utteranceMs = 0;
|
|
165
|
+
this.options.onLevel?.("user", 0);
|
|
166
|
+
this.setState(this.speaker.speaking ? "speaking" : "listening");
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Pressing while the assistant is talking is an interruption. Split out so a
|
|
170
|
+
* held press does not cut the assistant off before it is known to be speech.
|
|
171
|
+
*/
|
|
172
|
+
interruptPlayback() {
|
|
173
|
+
// Stop the playback the user has clearly stopped listening to, and tell
|
|
174
|
+
// the runtime how much of it they actually heard so the model's history
|
|
175
|
+
// matches their memory of the conversation.
|
|
176
|
+
if (this.speaker.speaking) {
|
|
177
|
+
this.sendControl({
|
|
178
|
+
type: "playback",
|
|
179
|
+
data: { ms: this.speaker.heardMs() },
|
|
180
|
+
});
|
|
181
|
+
this.speaker.flush();
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
/** The user released the button. Push-to-talk only. */
|
|
185
|
+
endUtterance() {
|
|
186
|
+
if (this.options.handsFree)
|
|
187
|
+
return;
|
|
188
|
+
if (this.state !== "recording")
|
|
189
|
+
return;
|
|
190
|
+
// Released before anyone decided: a press that never moved is speech.
|
|
191
|
+
this.confirmUtterance();
|
|
192
|
+
this.mic.mute();
|
|
193
|
+
this.options.onLevel?.("user", 0);
|
|
194
|
+
// A tap, not speech. Committing this comes back from the provider as
|
|
195
|
+
// "buffer too small", which is a true statement about something the user
|
|
196
|
+
// should never have been able to send — so it is caught here instead, and
|
|
197
|
+
// again in the runtime, which is the authority for clients that are not
|
|
198
|
+
// this one.
|
|
199
|
+
if (this.utteranceMs < this.minUtteranceMs) {
|
|
200
|
+
this.emit({
|
|
201
|
+
type: "error",
|
|
202
|
+
code: "utterance_too_short",
|
|
203
|
+
message: "Hold the button while you speak.",
|
|
204
|
+
recoverable: true,
|
|
205
|
+
fallbackToChat: false,
|
|
206
|
+
});
|
|
207
|
+
this.setState("listening");
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
this.sendControl({ type: "commit" });
|
|
211
|
+
this.setState("thinking");
|
|
212
|
+
}
|
|
213
|
+
async stop() {
|
|
214
|
+
if (this.stopped)
|
|
215
|
+
return;
|
|
216
|
+
this.stopped = true;
|
|
217
|
+
if (this.playbackTimer !== null) {
|
|
218
|
+
clearInterval(this.playbackTimer);
|
|
219
|
+
this.playbackTimer = null;
|
|
220
|
+
}
|
|
221
|
+
this.sendControl({ type: "end" });
|
|
222
|
+
this.socket?.close();
|
|
223
|
+
this.socket = null;
|
|
224
|
+
await Promise.all([this.mic.stop(), this.speaker.stop()]);
|
|
225
|
+
this.setState("idle");
|
|
226
|
+
}
|
|
227
|
+
// ------------------------------------------------------------- internals
|
|
228
|
+
handleControl(raw) {
|
|
229
|
+
let message;
|
|
230
|
+
try {
|
|
231
|
+
message = JSON.parse(raw);
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
switch (message.type) {
|
|
237
|
+
case "ready":
|
|
238
|
+
// Tolerating an absent value keeps a newer SDK working against an older
|
|
239
|
+
// runtime, where guessing low would fail the first press of a session.
|
|
240
|
+
this.minUtteranceMs = message.data.minUtteranceMs ?? this.minUtteranceMs;
|
|
241
|
+
// Hands-free opens the microphone here and not in `start`, and the
|
|
242
|
+
// difference is a whole class of bug. Unmuting early meant capturing
|
|
243
|
+
// from before the socket existed until the provider accepted its
|
|
244
|
+
// configuration — a second or two of the room — and every frame after
|
|
245
|
+
// the socket opened was sent, queued behind a relay still sitting in
|
|
246
|
+
// `wait_for_ready`. It arrived in one burst the moment the loop began
|
|
247
|
+
// reading, the provider's VAD read the burst as an utterance, and the
|
|
248
|
+
// session answered something nobody said. Reliably at the start of a
|
|
249
|
+
// session and never afterwards, because it can only happen once.
|
|
250
|
+
//
|
|
251
|
+
// The runtime already holds this rule for its own upstream: nothing is
|
|
252
|
+
// sent until the provider has accepted the configuration. This is the
|
|
253
|
+
// client keeping the same rule.
|
|
254
|
+
//
|
|
255
|
+
// Still open for the whole session once it opens — there is no press
|
|
256
|
+
// to open it on, and opening per detected utterance would clip the
|
|
257
|
+
// first word, which is usually the one carrying the intent.
|
|
258
|
+
if (this.options.handsFree)
|
|
259
|
+
this.mic.unmute();
|
|
260
|
+
this.setState("listening");
|
|
261
|
+
break;
|
|
262
|
+
case "transcript": {
|
|
263
|
+
// The runtime forwards the provider's deltas as they arrive, a few
|
|
264
|
+
// characters each, so the line so far is built here. Emitting the raw
|
|
265
|
+
// delta made the caption show only the latest fragment, which read as
|
|
266
|
+
// gibberish flickering over the orb.
|
|
267
|
+
let text = message.data.text;
|
|
268
|
+
if (message.data.role === "assistant") {
|
|
269
|
+
if (message.data.final) {
|
|
270
|
+
this.assistantLine = "";
|
|
271
|
+
}
|
|
272
|
+
else {
|
|
273
|
+
this.assistantLine += text;
|
|
274
|
+
text = this.assistantLine;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
this.emit({
|
|
278
|
+
type: "transcript",
|
|
279
|
+
role: message.data.role,
|
|
280
|
+
text,
|
|
281
|
+
final: message.data.final,
|
|
282
|
+
});
|
|
283
|
+
break;
|
|
284
|
+
}
|
|
285
|
+
case "status":
|
|
286
|
+
// A status of `listening` means one thing: the user interrupted. The
|
|
287
|
+
// queued audio has to go, or the assistant carries on out of the
|
|
288
|
+
// speaker after being cut off — and the runtime has already truncated
|
|
289
|
+
// its history to what was heard, so the two would disagree about the
|
|
290
|
+
// conversation.
|
|
291
|
+
//
|
|
292
|
+
// This is safe only because a *finished* reply arrives as
|
|
293
|
+
// `speech_done` instead. The relay used to send both, and flushing on
|
|
294
|
+
// a completed response threw away everything still queued — heard as
|
|
295
|
+
// every answer breaking off part-way through.
|
|
296
|
+
if (message.data.state === "listening" &&
|
|
297
|
+
this.speaker.speaking) {
|
|
298
|
+
this.speaker.flush();
|
|
299
|
+
}
|
|
300
|
+
// An interrupted line may never be marked final, and its leftovers
|
|
301
|
+
// must not prefix the next reply's caption.
|
|
302
|
+
if (message.data.state === "listening")
|
|
303
|
+
this.assistantLine = "";
|
|
304
|
+
// The server's view of the session, except while the button is down —
|
|
305
|
+
// the client knows about a press before the server does, and letting a
|
|
306
|
+
// status frame overwrite `recording` makes the button flicker.
|
|
307
|
+
if (this.state !== "recording") {
|
|
308
|
+
this.setState(message.data.state === "reconnecting"
|
|
309
|
+
? "connecting"
|
|
310
|
+
: message.data.state);
|
|
311
|
+
}
|
|
312
|
+
break;
|
|
313
|
+
case "speech_done":
|
|
314
|
+
// From `thinking` too: a reply can finish with no audio at all, a tool
|
|
315
|
+
// call and nothing said, and the session must still take the next
|
|
316
|
+
// press.
|
|
317
|
+
if (this.state === "speaking" || this.state === "thinking") {
|
|
318
|
+
this.setState("listening");
|
|
319
|
+
}
|
|
320
|
+
break;
|
|
321
|
+
case "ending_soon":
|
|
322
|
+
this.emit({
|
|
323
|
+
type: "ending_soon",
|
|
324
|
+
secondsRemaining: message.data.secondsRemaining,
|
|
325
|
+
});
|
|
326
|
+
break;
|
|
327
|
+
case "ended":
|
|
328
|
+
this.emit({
|
|
329
|
+
type: "ended",
|
|
330
|
+
reason: message.data.reason,
|
|
331
|
+
message: message.data.message,
|
|
332
|
+
});
|
|
333
|
+
// `stop` leaves the state `idle`, which is the point: the microphone
|
|
334
|
+
// button goes back to its resting look rather than the failed one, and
|
|
335
|
+
// pressing it again starts a fresh session on the same conversation.
|
|
336
|
+
void this.stop();
|
|
337
|
+
break;
|
|
338
|
+
case "error":
|
|
339
|
+
this.emit({
|
|
340
|
+
type: "error",
|
|
341
|
+
code: message.data.code,
|
|
342
|
+
message: message.data.message,
|
|
343
|
+
recoverable: message.data.recoverable,
|
|
344
|
+
fallbackToChat: message.data.fallbackToChat,
|
|
345
|
+
});
|
|
346
|
+
// A recoverable error is one operation failing, not the call ending.
|
|
347
|
+
// Tearing the session down here was the bug: a mistimed press hung up
|
|
348
|
+
// on the user and made them start over.
|
|
349
|
+
if (!message.data.recoverable) {
|
|
350
|
+
this.setState("error");
|
|
351
|
+
void this.stop();
|
|
352
|
+
}
|
|
353
|
+
break;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
sendAudio(pcm) {
|
|
357
|
+
// Hands-free sends throughout: the provider's VAD is what decides where an
|
|
358
|
+
// utterance begins, and it cannot do that on audio it never receives.
|
|
359
|
+
if (!this.options.handsFree && this.state !== "recording")
|
|
360
|
+
return;
|
|
361
|
+
if (this.held) {
|
|
362
|
+
this.held.push(pcm);
|
|
363
|
+
this.options.onLevel?.("user", pcmLevel(pcm));
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
this.options.onLevel?.("user", pcmLevel(pcm));
|
|
367
|
+
// Two bytes per sample, one channel.
|
|
368
|
+
this.utteranceMs += (pcm.byteLength * 1000) / (2 * VOICE_SAMPLE_RATE);
|
|
369
|
+
try {
|
|
370
|
+
this.socket?.send(pcm);
|
|
371
|
+
}
|
|
372
|
+
catch {
|
|
373
|
+
// A send on a closing socket throws; `onclose` handles the rest.
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
sendControl(message) {
|
|
377
|
+
try {
|
|
378
|
+
this.socket?.send(JSON.stringify(message));
|
|
379
|
+
}
|
|
380
|
+
catch {
|
|
381
|
+
// Same.
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
setState(state) {
|
|
385
|
+
if (this.state === state)
|
|
386
|
+
return;
|
|
387
|
+
this.state = state;
|
|
388
|
+
this.reportPlaybackWhileSpeaking();
|
|
389
|
+
this.options.onEvent({ type: "state", state });
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* Tell the runtime how much of the current reply has actually been heard,
|
|
393
|
+
* a few times a second, for as long as the assistant is speaking.
|
|
394
|
+
*
|
|
395
|
+
* Push-to-talk does not need this: the interruption *is* a button press, so
|
|
396
|
+
* the position can be measured at exactly that moment. Hands-free has no
|
|
397
|
+
* such moment on this side — the provider notices the user talking, upstream
|
|
398
|
+
* — so the runtime has to already hold a recent figure when it truncates.
|
|
399
|
+
*
|
|
400
|
+
* Without it, truncation falls back to nothing and the model's history keeps
|
|
401
|
+
* the whole reply, including the part the user cut off. It then answers
|
|
402
|
+
* follow-ups about words nobody heard.
|
|
403
|
+
*
|
|
404
|
+
* Four frames a second of small JSON, against fifty of audio.
|
|
405
|
+
*/
|
|
406
|
+
reportPlaybackWhileSpeaking() {
|
|
407
|
+
const shouldReport = Boolean(this.options.handsFree) && this.state === "speaking";
|
|
408
|
+
if (!shouldReport) {
|
|
409
|
+
if (this.playbackTimer !== null) {
|
|
410
|
+
clearInterval(this.playbackTimer);
|
|
411
|
+
this.playbackTimer = null;
|
|
412
|
+
}
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
if (this.playbackTimer !== null)
|
|
416
|
+
return;
|
|
417
|
+
this.playbackTimer = setInterval(() => {
|
|
418
|
+
if (!this.speaker.speaking)
|
|
419
|
+
return;
|
|
420
|
+
this.sendControl({
|
|
421
|
+
type: "playback",
|
|
422
|
+
data: { ms: this.speaker.heardMs() },
|
|
423
|
+
});
|
|
424
|
+
}, 250);
|
|
425
|
+
}
|
|
426
|
+
emit(event) {
|
|
427
|
+
this.options.onEvent(event);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Derive the voice socket URL from the chat socket URL.
|
|
432
|
+
*
|
|
433
|
+
* One `url` prop for the whole SDK rather than two: the two sockets are always
|
|
434
|
+
* served by the same runtime, and a second prop would be a second thing to get
|
|
435
|
+
* wrong in every integration for no configurability anyone wants.
|
|
436
|
+
*/
|
|
437
|
+
export function voiceUrl(chatUrl, token, handsFree = false) {
|
|
438
|
+
const base = chatUrl.replace(/\/v1\/ws\/?$/, "/v1/voice");
|
|
439
|
+
const separator = base.includes("?") ? "&" : "?";
|
|
440
|
+
const mode = handsFree ? "&handsFree=true" : "";
|
|
441
|
+
return `${base}${separator}token=${encodeURIComponent(token)}${mode}`;
|
|
442
|
+
}
|
|
443
|
+
//# sourceMappingURL=voice.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice.js","sourceRoot":"","sources":["../../../js/src/voice.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACL,iBAAiB,GAIlB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AA8EpE,MAAM,OAAO,YAAY;IAqCM;IApCrB,MAAM,GAA2B,IAAI,CAAC;IACtC,KAAK,GAAe,MAAM,CAAC;IAC3B,GAAG,CAAoB;IACvB,OAAO,CAAe;IAC9B;;;;;;;;OAQG;IACK,IAAI,GAAyB,IAAI,CAAC;IAClC,OAAO,GAAG,KAAK,CAAC;IACxB;;;;;;;OAOG;IACK,WAAW,GAAG,CAAC,CAAC;IACxB,2EAA2E;IACnE,aAAa,GAAG,EAAE,CAAC;IAC3B,yEAAyE;IACjE,aAAa,GAA0C,IAAI,CAAC;IACpE;;;;;OAKG;IACK,cAAc,GAAG,GAAG,CAAC;IAE7B,YAA6B,OAA4B;QAA5B,YAAO,GAAP,OAAO,CAAqB;QACvD,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,IAAI,CAAC,GAAG,GAAG,IAAI,iBAAiB,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE5B,IAAI,CAAC;YACH,uEAAuE;YACvE,qEAAqE;YACrE,+DAA+D;YAC/D,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC;gBACR,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,wBAAwB;gBAC9B,OAAO,EACL,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB;oBACxD,CAAC,CAAC,wDAAwD;oBAC1D,CAAC,CAAC,6BAA6B;gBACnC,sEAAsE;gBACtE,WAAW,EAAE,KAAK;gBAClB,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GACX,IAAI,CAAC,OAAO,CAAC,aAAa;YAC1B,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,GAAG,CAA+B,CAAC,CAAC;QAEtE,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,CAAC,UAAU,GAAG,aAAa,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE;YAC3B,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACnC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,YAAY,WAAW,EAAE,CAAC;gBAC7C,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjC,uEAAuE;gBACvE,+DAA+D;gBAC/D,8DAA8D;gBAC9D,oDAAoD;gBACpD,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU;oBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE;YACpB,wEAAwE;YACxE,wEAAwE;YACxE,sEAAsE;YACtE,uCAAuC;YACvC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;QACnB,CAAC,CAAC;QAEF,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE;YACpB,4BAA4B;QAC9B,CAAC,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACH,cAAc,CAAC,OAA4B;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS;YAAE,OAAO;QACnC,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU;YAAE,OAAO;QAEpE,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QAErB,IAAI,OAAO,EAAE,IAAI,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;YACf,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QAClB,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC7B,CAAC;IAED,+EAA+E;IAC/E,gBAAgB;QACd,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,KAAK,MAAM,GAAG,IAAI,IAAI;YAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;IAED,yEAAyE;IACzE,gBAAgB;QACd,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAChB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAClE,CAAC;IAED;;;OAGG;IACK,iBAAiB;QACvB,wEAAwE;QACxE,wEAAwE;QACxE,4CAA4C;QAC5C,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1B,IAAI,CAAC,WAAW,CAAC;gBACf,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE;aACrC,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,YAAY;QACV,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS;YAAE,OAAO;QACnC,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW;YAAE,OAAO;QACvC,sEAAsE;QACtE,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAChB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAElC,qEAAqE;QACrE,yEAAyE;QACzE,0EAA0E;QAC1E,wEAAwE;QACxE,YAAY;QACZ,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3C,IAAI,CAAC,IAAI,CAAC;gBACR,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,qBAAqB;gBAC3B,OAAO,EAAE,kCAAkC;gBAC3C,WAAW,EAAE,IAAI;gBACjB,cAAc,EAAE,KAAK;aACtB,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAChC,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAED,0EAA0E;IAElE,aAAa,CAAC,GAAW;QAC/B,IAAI,OAA2B,CAAC;QAChC,IAAI,CAAC;YACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAuB,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QAED,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,OAAO;gBACV,wEAAwE;gBACxE,uEAAuE;gBACvE,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC;gBAEzE,mEAAmE;gBACnE,qEAAqE;gBACrE,iEAAiE;gBACjE,sEAAsE;gBACtE,qEAAqE;gBACrE,sEAAsE;gBACtE,sEAAsE;gBACtE,qEAAqE;gBACrE,iEAAiE;gBACjE,EAAE;gBACF,uEAAuE;gBACvE,sEAAsE;gBACtE,gCAAgC;gBAChC,EAAE;gBACF,qEAAqE;gBACrE,mEAAmE;gBACnE,4DAA4D;gBAC5D,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS;oBAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;gBAE9C,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC3B,MAAM;YAER,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,mEAAmE;gBACnE,sEAAsE;gBACtE,sEAAsE;gBACtE,qCAAqC;gBACrC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC7B,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACtC,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;wBACvB,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;oBAC1B,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC;wBAC3B,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC;oBAC5B,CAAC;gBACH,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC;oBACR,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI;oBACvB,IAAI;oBACJ,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK;iBAC1B,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAED,KAAK,QAAQ;gBACX,qEAAqE;gBACrE,iEAAiE;gBACjE,sEAAsE;gBACtE,qEAAqE;gBACrE,gBAAgB;gBAChB,EAAE;gBACF,0DAA0D;gBAC1D,sEAAsE;gBACtE,qEAAqE;gBACrE,8CAA8C;gBAC9C,IACE,OAAO,CAAC,IAAI,CAAC,KAAK,KAAK,WAAW;oBAClC,IAAI,CAAC,OAAO,CAAC,QAAQ,EACrB,CAAC;oBACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACvB,CAAC;gBACD,mEAAmE;gBACnE,4CAA4C;gBAC5C,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,KAAK,WAAW;oBAAE,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;gBAEhE,sEAAsE;gBACtE,uEAAuE;gBACvE,+DAA+D;gBAC/D,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;oBAC/B,IAAI,CAAC,QAAQ,CACX,OAAO,CAAC,IAAI,CAAC,KAAK,KAAK,cAAc;wBACnC,CAAC,CAAC,YAAY;wBACd,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CACvB,CAAC;gBACJ,CAAC;gBACD,MAAM;YAER,KAAK,aAAa;gBAChB,uEAAuE;gBACvE,kEAAkE;gBAClE,SAAS;gBACT,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;oBAC3D,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC7B,CAAC;gBACD,MAAM;YAER,KAAK,aAAa;gBAChB,IAAI,CAAC,IAAI,CAAC;oBACR,IAAI,EAAE,aAAa;oBACnB,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,gBAAgB;iBAChD,CAAC,CAAC;gBACH,MAAM;YAER,KAAK,OAAO;gBACV,IAAI,CAAC,IAAI,CAAC;oBACR,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM;oBAC3B,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO;iBAC9B,CAAC,CAAC;gBACH,qEAAqE;gBACrE,uEAAuE;gBACvE,qEAAqE;gBACrE,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM;YAER,KAAK,OAAO;gBACV,IAAI,CAAC,IAAI,CAAC;oBACR,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI;oBACvB,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO;oBAC7B,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW;oBACrC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,cAAc;iBAC5C,CAAC,CAAC;gBACH,qEAAqE;gBACrE,sEAAsE;gBACtE,wCAAwC;gBACxC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;oBAC9B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;oBACvB,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBACnB,CAAC;gBACD,MAAM;QACV,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,GAAgB;QAChC,2EAA2E;QAC3E,sEAAsE;QACtE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW;YAAE,OAAO;QAClE,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9C,qCAAqC;QACrC,IAAI,CAAC,WAAW,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACP,iEAAiE;QACnE,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,OAA2B;QAC7C,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ;QACV,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,KAAiB;QAChC,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK;YAAE,OAAO;QACjC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,2BAA2B,EAAE,CAAC;QACnC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACK,2BAA2B;QACjC,MAAM,YAAY,GAChB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC;QAE/D,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;gBAChC,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC5B,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI;YAAE,OAAO;QAExC,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;YACpC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,OAAO;YACnC,IAAI,CAAC,WAAW,CAAC;gBACf,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE;aACrC,CAAC,CAAC;QACL,CAAC,EAAE,GAAG,CAAC,CAAC;IACV,CAAC;IAEO,IAAI,CAAC,KAAiB;QAC5B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CACtB,OAAe,EACf,KAAa,EACb,SAAS,GAAG,KAAK;IAEjB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACjD,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;IAChD,OAAO,GAAG,IAAI,GAAG,SAAS,SAAS,kBAAkB,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,CAAC;AACxE,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The approval surface for a proposed action, shared by the panel and the orb.
|
|
3
|
+
*
|
|
4
|
+
* It renders `label`, `description` and `input` from the proposal payload,
|
|
5
|
+
* which the runtime copies from its registry, never from the assistant's
|
|
6
|
+
* prose. Text smuggled into a product name or a document therefore cannot
|
|
7
|
+
* change what a user is shown they are about to approve, and it is added as
|
|
8
|
+
* text, so markup in it is shown rather than run.
|
|
9
|
+
*
|
|
10
|
+
* High-risk actions require this tap — the runtime refuses a spoken or typed
|
|
11
|
+
* "yes" for them — so this is a security boundary, not decoration.
|
|
12
|
+
*/
|
|
13
|
+
import type { ProposalPayload } from "../client.js";
|
|
14
|
+
export declare function confirmationCard(proposal: ProposalPayload, onRespond: (proposalId: string, confirmed: boolean) => void): HTMLElement;
|
|
15
|
+
//# sourceMappingURL=card.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"card.d.ts","sourceRoot":"","sources":["../../../../js/src/widget/card.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAIjD,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,eAAe,EACzB,SAAS,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,KAAK,IAAI,GAC1D,WAAW,CA0Cb"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The approval surface for a proposed action, shared by the panel and the orb.
|
|
3
|
+
*
|
|
4
|
+
* It renders `label`, `description` and `input` from the proposal payload,
|
|
5
|
+
* which the runtime copies from its registry, never from the assistant's
|
|
6
|
+
* prose. Text smuggled into a product name or a document therefore cannot
|
|
7
|
+
* change what a user is shown they are about to approve, and it is added as
|
|
8
|
+
* text, so markup in it is shown rather than run.
|
|
9
|
+
*
|
|
10
|
+
* High-risk actions require this tap — the runtime refuses a spoken or typed
|
|
11
|
+
* "yes" for them — so this is a security boundary, not decoration.
|
|
12
|
+
*/
|
|
13
|
+
import { h } from "./dom.js";
|
|
14
|
+
import { formatValue, humanizeKey } from "./format.js";
|
|
15
|
+
export function confirmationCard(proposal, onRespond) {
|
|
16
|
+
const entries = Object.entries(proposal.input ?? {});
|
|
17
|
+
const high = proposal.risk === "high";
|
|
18
|
+
return h("div", { class: "card", attrs: { role: "group", "aria-label": `Confirm: ${proposal.label}` } }, h("div", { class: "card-header" }, h("strong", {}, proposal.label), h("span", { class: high ? "risk risk-high" : "risk" }, high ? "Needs care" : "Confirm")), h("p", { class: "card-description" }, proposal.description), entries.length > 0 &&
|
|
19
|
+
h("dl", { class: "card-input" }, ...entries.map(([key, value]) => h("div", { class: "card-row" }, h("dt", { class: "card-key" }, humanizeKey(key)), h("dd", { class: "card-value" }, inputValue(value))))), h("div", { class: "card-actions" }, h("button", { class: "button-secondary", attrs: { type: "button" }, on: { click: () => onRespond(proposal.proposalId, false) } }, "Cancel"), h("button", { class: "button-primary", attrs: { type: "button" }, on: { click: () => onRespond(proposal.proposalId, true) } }, "Confirm")));
|
|
20
|
+
}
|
|
21
|
+
/** Lists stay separate so a user can check each line before approving. */
|
|
22
|
+
function inputValue(value) {
|
|
23
|
+
if (Array.isArray(value) && value.length > 0) {
|
|
24
|
+
return h("ol", { class: "input-list" }, ...value.map((item) => h("li", {}, inputValue(item))));
|
|
25
|
+
}
|
|
26
|
+
return h("span", { class: "input-text" }, formatValue(value));
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=card.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"card.js","sourceRoot":"","sources":["../../../../js/src/widget/card.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,EAAE,CAAC,EAAE,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEpD,MAAM,UAAU,gBAAgB,CAC9B,QAAyB,EACzB,SAA2D;IAE3D,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC;IAEtC,OAAO,CAAC,CACN,KAAK,EACL,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,QAAQ,CAAC,KAAK,EAAE,EAAE,EAAE,EACvF,CAAC,CACC,KAAK,EACL,EAAE,KAAK,EAAE,aAAa,EAAE,EACxB,CAAC,CAAC,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,EAC/B,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CACxF,EACD,CAAC,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,kBAAkB,EAAE,EAAE,QAAQ,CAAC,WAAW,CAAC,EAC3D,OAAO,CAAC,MAAM,GAAG,CAAC;QAChB,CAAC,CACC,IAAI,EACJ,EAAE,KAAK,EAAE,YAAY,EAAE,EACvB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAC9B,CAAC,CACC,KAAK,EACL,EAAE,KAAK,EAAE,UAAU,EAAE,EACrB,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,EAChD,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CACpD,CACF,CACF,EACH,CAAC,CACC,KAAK,EACL,EAAE,KAAK,EAAE,cAAc,EAAE,EACzB,CAAC,CACC,QAAQ,EACR,EAAE,KAAK,EAAE,kBAAkB,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,EAAE,EACpH,QAAQ,CACT,EACD,CAAC,CACC,QAAQ,EACR,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,EAAE,EACjH,SAAS,CACV,CACF,CACF,CAAC;AACJ,CAAC;AAED,0EAA0E;AAC1E,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACjG,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Building DOM without a framework, and without ever parsing a string.
|
|
3
|
+
*
|
|
4
|
+
* Text goes in as text nodes, so a product name holding `<img onerror=…>`
|
|
5
|
+
* is shown as those characters and never runs: the card's firebreak depends
|
|
6
|
+
* on it. Nothing in the widget sets `innerHTML`.
|
|
7
|
+
*/
|
|
8
|
+
type Child = Node | string | null | undefined | false;
|
|
9
|
+
export interface Props {
|
|
10
|
+
class?: string;
|
|
11
|
+
/** Set with `setAttribute`; `false` and `undefined` leave it off. */
|
|
12
|
+
attrs?: Record<string, string | number | boolean | undefined>;
|
|
13
|
+
on?: {
|
|
14
|
+
[K in keyof HTMLElementEventMap]?: (event: HTMLElementEventMap[K]) => void;
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
export declare function h<K extends keyof HTMLElementTagNameMap>(tag: K, props?: Props, ...children: Child[]): HTMLElementTagNameMap[K];
|
|
18
|
+
export declare function svg(tag: string, attrs: Record<string, string | number>, ...children: Node[]): SVGElement;
|
|
19
|
+
export declare function setAttrs(element: Element, attrs: Record<string, string | number | boolean | undefined>): void;
|
|
20
|
+
export declare function append(element: Node, children: Child[]): void;
|
|
21
|
+
/** Replace an element's children. */
|
|
22
|
+
export declare function replace(element: Node, ...children: Child[]): void;
|
|
23
|
+
export {};
|
|
24
|
+
//# sourceMappingURL=dom.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dom.d.ts","sourceRoot":"","sources":["../../../../js/src/widget/dom.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,KAAK,KAAK,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,KAAK,CAAC;AAEtD,MAAM,WAAW,KAAK;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC,CAAC;IAC9D,EAAE,CAAC,EAAE;SAAG,CAAC,IAAI,MAAM,mBAAmB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC,KAAK,IAAI;KAAE,CAAC;CACrF;AAED,wBAAgB,CAAC,CAAC,CAAC,SAAS,MAAM,qBAAqB,EACrD,GAAG,EAAE,CAAC,EACN,KAAK,GAAE,KAAU,EACjB,GAAG,QAAQ,EAAE,KAAK,EAAE,GACnB,qBAAqB,CAAC,CAAC,CAAC,CAS1B;AAID,wBAAgB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,GAAG,UAAU,CAKxG;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC,GAAG,IAAI,CAK7G;AAED,wBAAgB,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAI,CAK7D;AAED,qCAAqC;AACrC,wBAAgB,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAI,CAGjE"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Building DOM without a framework, and without ever parsing a string.
|
|
3
|
+
*
|
|
4
|
+
* Text goes in as text nodes, so a product name holding `<img onerror=…>`
|
|
5
|
+
* is shown as those characters and never runs: the card's firebreak depends
|
|
6
|
+
* on it. Nothing in the widget sets `innerHTML`.
|
|
7
|
+
*/
|
|
8
|
+
export function h(tag, props = {}, ...children) {
|
|
9
|
+
const element = document.createElement(tag);
|
|
10
|
+
if (props.class)
|
|
11
|
+
element.className = props.class;
|
|
12
|
+
setAttrs(element, props.attrs ?? {});
|
|
13
|
+
for (const [type, listener] of Object.entries(props.on ?? {})) {
|
|
14
|
+
element.addEventListener(type, listener);
|
|
15
|
+
}
|
|
16
|
+
append(element, children);
|
|
17
|
+
return element;
|
|
18
|
+
}
|
|
19
|
+
const SVG = "http://www.w3.org/2000/svg";
|
|
20
|
+
export function svg(tag, attrs, ...children) {
|
|
21
|
+
const element = document.createElementNS(SVG, tag);
|
|
22
|
+
for (const [name, value] of Object.entries(attrs))
|
|
23
|
+
element.setAttribute(name, String(value));
|
|
24
|
+
for (const child of children)
|
|
25
|
+
element.appendChild(child);
|
|
26
|
+
return element;
|
|
27
|
+
}
|
|
28
|
+
export function setAttrs(element, attrs) {
|
|
29
|
+
for (const [name, value] of Object.entries(attrs)) {
|
|
30
|
+
if (value === undefined || value === false)
|
|
31
|
+
element.removeAttribute(name);
|
|
32
|
+
else
|
|
33
|
+
element.setAttribute(name, value === true ? "" : String(value));
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export function append(element, children) {
|
|
37
|
+
for (const child of children) {
|
|
38
|
+
if (child === null || child === undefined || child === false)
|
|
39
|
+
continue;
|
|
40
|
+
element.appendChild(typeof child === "string" ? document.createTextNode(child) : child);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/** Replace an element's children. */
|
|
44
|
+
export function replace(element, ...children) {
|
|
45
|
+
while (element.firstChild)
|
|
46
|
+
element.removeChild(element.firstChild);
|
|
47
|
+
append(element, children);
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=dom.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dom.js","sourceRoot":"","sources":["../../../../js/src/widget/dom.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAWH,MAAM,UAAU,CAAC,CACf,GAAM,EACN,QAAe,EAAE,EACjB,GAAG,QAAiB;IAEpB,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,KAAK,CAAC,KAAK;QAAE,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QAC9D,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,QAAyB,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC1B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,GAAG,GAAG,4BAA4B,CAAC;AAEzC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,KAAsC,EAAE,GAAG,QAAgB;IAC1F,MAAM,OAAO,GAAG,QAAQ,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACnD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7F,KAAK,MAAM,KAAK,IAAI,QAAQ;QAAE,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IACzD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,OAAgB,EAAE,KAA4D;IACrG,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,KAAK;YAAE,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;;YACrE,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,OAAa,EAAE,QAAiB;IACrD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,KAAK;YAAE,SAAS;QACvE,OAAO,CAAC,WAAW,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1F,CAAC;AACH,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,OAAO,CAAC,OAAa,EAAE,GAAG,QAAiB;IACzD,OAAO,OAAO,CAAC,UAAU;QAAE,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACnE,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC5B,CAAC"}
|