@alexkroman1/aai-ui 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_module-url-C_4gRVL0.js +15 -0
- package/dist/audio.d.ts +42 -6
- package/dist/audio.js +79 -69
- package/dist/{chat-view-DKFhxMAT.js → chat-view-C1oJxsWz.js} +4 -4
- package/dist/client-config.d.ts +5 -6
- package/dist/components/chat-view.js +1 -1
- package/dist/components/console-shell.d.ts +2 -2
- package/dist/components/controls.js +1 -1
- package/dist/components/url-chips.d.ts +0 -3
- package/dist/context.d.ts +1 -1
- package/dist/context.js +1 -4
- package/dist/{controls-4OJoekj6.js → controls-DV368uhb.js} +1 -4
- package/dist/default-client/assets/_module-url-BX0RuRU2.js +1 -0
- package/dist/default-client/assets/audio-BGHiiDY_.js +1 -0
- package/dist/default-client/assets/{capture-processor-DLHxAIfT.js → capture-processor-BLPsqnGl.js} +4 -4
- package/dist/default-client/assets/index-DXf7-M0Q.js +244 -0
- package/dist/default-client/assets/index-Dv-Q5VRL.css +2 -0
- package/dist/default-client/assets/{playback-processor-bMTdFp-8.js → playback-processor-B_T_1KQP.js} +28 -21
- package/dist/default-client/index.html +2 -5
- package/dist/define-client-Cyf1jqAl.js +150 -0
- package/dist/define-client.js +1 -1
- package/dist/index.d.ts +1 -3
- package/dist/index.js +7 -6
- package/dist/{session-core-BLhiQ18c.js → session-core-BM3WHeoY.js} +30 -128
- package/dist/session-core-messages.d.ts +0 -4
- package/dist/session-core-types.d.ts +1 -27
- package/dist/session-core.js +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.js +2 -2
- package/dist/worklets/_module-url.d.ts +10 -0
- package/dist/worklets/capture-processor.d.ts +2 -2
- package/dist/worklets/capture-processor.js +5 -5
- package/dist/worklets/playback-processor.d.ts +3 -3
- package/dist/worklets/playback-processor.js +31 -24
- package/package.json +2 -2
- package/dist/components/workflow-view.d.ts +0 -13
- package/dist/default-client/assets/audio-Cgviqo9t.js +0 -1
- package/dist/default-client/assets/index-BgbIWnfG.css +0 -2
- package/dist/default-client/assets/index-DbKKR3UE.js +0 -62
- package/dist/default-client/assets/rolldown-runtime-BpQH8Ho1.js +0 -1
- package/dist/default-client/assets/types-Bpg3ZIZK.js +0 -64
- package/dist/define-client-yoGybEYR.js +0 -710
- package/dist/session-core-upload.d.ts +0 -16
- package/dist/sync-mic.d.ts +0 -61
- package/dist/sync-session.d.ts +0 -49
|
@@ -1,710 +0,0 @@
|
|
|
1
|
-
import { SessionProvider, ThemeProvider, useTheme } from "./context.js";
|
|
2
|
-
import { Button } from "./components/button.js";
|
|
3
|
-
import { t as ERROR_COLOR } from "./_colors-DYX7XRTr.js";
|
|
4
|
-
import { n as ConsoleShell, t as ChatView } from "./chat-view-DKFhxMAT.js";
|
|
5
|
-
import { a as UrlChip } from "./controls-4OJoekj6.js";
|
|
6
|
-
import { n as ToolConfigContext, t as ToolCallBlock } from "./tool-call-block-DIxpG8GM.js";
|
|
7
|
-
import { ThinkingDots } from "./components/message-list.js";
|
|
8
|
-
import { SidebarLayout } from "./components/sidebar-layout.js";
|
|
9
|
-
import { StartScreen } from "./components/start-screen.js";
|
|
10
|
-
import { VOICE_CAPTURE_CONSTRAINTS } from "./types.js";
|
|
11
|
-
import { decodeAudioToPcm16 } from "./audio.js";
|
|
12
|
-
import { t as createSessionCore } from "./session-core-BLhiQ18c.js";
|
|
13
|
-
import { CLIENT_CONFIG_PATH, ClientConfigResponseSchema, SyncTurnResponseSchema } from "@alexkroman1/aai/protocol";
|
|
14
|
-
import clsx from "clsx";
|
|
15
|
-
import { createElement, useEffect, useRef, useState } from "react";
|
|
16
|
-
import { jsx, jsxs } from "react/jsx-runtime";
|
|
17
|
-
import { DEFAULT_MAX_HISTORY, errorMessage, safeJsonParse } from "@alexkroman1/aai";
|
|
18
|
-
import { flushSync } from "react-dom";
|
|
19
|
-
import { createRoot } from "react-dom/client";
|
|
20
|
-
//#region client-config.ts
|
|
21
|
-
/**
|
|
22
|
-
* Pre-connection client-config lookup.
|
|
23
|
-
*
|
|
24
|
-
* `GET client-config` (relative to the agent's base URL — see
|
|
25
|
-
* `sdk/client-config.ts` in `@alexkroman1/aai`) tells the default client
|
|
26
|
-
* what kind of app the agent is before any connection exists — a
|
|
27
|
-
* conversational agent (WebSocket chat shell) or a workflow (one-shot run
|
|
28
|
-
* surface). Every failure path — network error, 404 from an older server,
|
|
29
|
-
* malformed body — degrades to the agent default, so this lookup can never
|
|
30
|
-
* break an existing agent.
|
|
31
|
-
*/
|
|
32
|
-
/** Resolve a relative endpoint path against the agent's base URL. */
|
|
33
|
-
function buildAgentUrl(platformUrl, endpointPath) {
|
|
34
|
-
return new URL(endpointPath, platformUrl.endsWith("/") ? platformUrl : `${platformUrl}/`);
|
|
35
|
-
}
|
|
36
|
-
const AGENT_DEFAULT = { kind: "agent" };
|
|
37
|
-
/** Fetch the agent's client config; any failure yields the agent default. */
|
|
38
|
-
async function fetchClientConfig(platformUrl, fetchFn) {
|
|
39
|
-
const doFetch = fetchFn ?? ((input, init) => globalThis.fetch(input, init));
|
|
40
|
-
try {
|
|
41
|
-
const resp = await doFetch(buildAgentUrl(platformUrl, CLIENT_CONFIG_PATH).href);
|
|
42
|
-
if (!resp.ok) return AGENT_DEFAULT;
|
|
43
|
-
const parsed = ClientConfigResponseSchema.safeParse(await resp.json());
|
|
44
|
-
return parsed.success ? parsed.data : AGENT_DEFAULT;
|
|
45
|
-
} catch {
|
|
46
|
-
return AGENT_DEFAULT;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
//#endregion
|
|
50
|
-
//#region sync-mic.ts
|
|
51
|
-
/**
|
|
52
|
-
* WebRTC push-to-talk capture for the workflow run surface.
|
|
53
|
-
*
|
|
54
|
-
* Captures voice through `getUserMedia` under
|
|
55
|
-
* {@link VOICE_CAPTURE_CONSTRAINTS} — echo cancellation only — and runs an
|
|
56
|
-
* AudioWorklet that batches raw frames to the main thread; the caller's
|
|
57
|
-
* button is the endpointing. Each recording becomes one `POST /sync` run.
|
|
58
|
-
* No WebSocket anywhere on the path.
|
|
59
|
-
*
|
|
60
|
-
* The worklet module ships inline as a blob URL (same pattern as the
|
|
61
|
-
* WebSocket path's worklets), so this path needs no separately-served
|
|
62
|
-
* processor file. A blob URL rather than a data URI because the agent
|
|
63
|
-
* page's CSP allows `script-src blob:` but not `data:` — a data-URI
|
|
64
|
-
* module fails `addModule` with "Unable to load a worklet's module".
|
|
65
|
-
*/
|
|
66
|
-
/** Default capture rate — what the STT providers expect. */
|
|
67
|
-
const DEFAULT_SYNC_MIC_SAMPLE_RATE = 16e3;
|
|
68
|
-
/** ~128 ms at 16 kHz: few messages per second, fine-enough VAD granularity. */
|
|
69
|
-
const CAPTURE_BATCH_SAMPLES = 2048;
|
|
70
|
-
/**
|
|
71
|
-
* The capture processor: coalesces 128-sample render quanta into
|
|
72
|
-
* {@link CAPTURE_BATCH_SAMPLES} batches and posts them (transferred, so no
|
|
73
|
-
* per-batch copy). Inlined as source because it must be stringified into a
|
|
74
|
-
* blob URL.
|
|
75
|
-
*
|
|
76
|
-
* `batch` is held as a field rather than re-read from the posted view:
|
|
77
|
-
* `postMessage` with a transfer list detaches the buffer, so `out.length` is
|
|
78
|
-
* 0 by the time the next buffer is allocated. Allocating a zero-length `buf`
|
|
79
|
-
* from it made `n` 0 forever, so `read` stopped advancing and the render
|
|
80
|
-
* thread spun inside `process()` posting empty chunks — the mic went
|
|
81
|
-
* permanently deaf on its first flush.
|
|
82
|
-
*
|
|
83
|
-
* Exported for the worklet unit tests (`sync-mic-worklet.test.ts`), which
|
|
84
|
-
* evaluate this source directly; it is not part of the package surface.
|
|
85
|
-
*/
|
|
86
|
-
const CAPTURE_PROCESSOR_SRC = `
|
|
87
|
-
registerProcessor("aai-sync-capture", class extends AudioWorkletProcessor {
|
|
88
|
-
constructor(options) {
|
|
89
|
-
super();
|
|
90
|
-
const batch = (options && options.processorOptions && options.processorOptions.batchSamples) || ${CAPTURE_BATCH_SAMPLES};
|
|
91
|
-
this.batch = batch;
|
|
92
|
-
this.buf = new Float32Array(batch);
|
|
93
|
-
this.len = 0;
|
|
94
|
-
}
|
|
95
|
-
process(inputs) {
|
|
96
|
-
const ch = inputs[0] && inputs[0][0];
|
|
97
|
-
if (!ch) return true;
|
|
98
|
-
let read = 0;
|
|
99
|
-
while (read < ch.length) {
|
|
100
|
-
const n = Math.min(ch.length - read, this.buf.length - this.len);
|
|
101
|
-
this.buf.set(ch.subarray(read, read + n), this.len);
|
|
102
|
-
this.len += n;
|
|
103
|
-
read += n;
|
|
104
|
-
if (this.len === this.batch) {
|
|
105
|
-
const out = this.buf;
|
|
106
|
-
// Size the next buffer from this.batch, never from \`out\`: the
|
|
107
|
-
// transfer below detaches out.buffer, so out.length reads 0 here.
|
|
108
|
-
this.buf = new Float32Array(this.batch);
|
|
109
|
-
this.len = 0;
|
|
110
|
-
this.port.postMessage({ event: "chunk", samples: out }, [out.buffer]);
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
return true;
|
|
114
|
-
}
|
|
115
|
-
});
|
|
116
|
-
`;
|
|
117
|
-
/**
|
|
118
|
-
* Blob-URL module for the capture processor (no served asset). Satisfies the
|
|
119
|
-
* agent page's `script-src blob:` CSP, which rejects data-URI modules.
|
|
120
|
-
*/
|
|
121
|
-
const CAPTURE_WORKLET_MODULE_URL = URL.createObjectURL(new Blob([CAPTURE_PROCESSOR_SRC], { type: "application/javascript" }));
|
|
122
|
-
/** Clamp-and-convert one Float32 capture batch to PCM16. */
|
|
123
|
-
function floatToPcm16(samples) {
|
|
124
|
-
const pcm = new Int16Array(samples.length);
|
|
125
|
-
let i = 0;
|
|
126
|
-
for (const sample of samples) {
|
|
127
|
-
const s = Math.max(-1, Math.min(1, sample));
|
|
128
|
-
pcm[i++] = s < 0 ? s * 32768 : s * 32767;
|
|
129
|
-
}
|
|
130
|
-
return pcm;
|
|
131
|
-
}
|
|
132
|
-
/**
|
|
133
|
-
* Push-to-talk recorder: `getUserMedia` voice capture feeding the capture
|
|
134
|
-
* worklet, with the caller's button as the endpointing. Recording runs
|
|
135
|
-
* exactly between `start()` and `stop()`; the mic stays open across presses
|
|
136
|
-
* until `close()`.
|
|
137
|
-
*
|
|
138
|
-
* @public
|
|
139
|
-
*/
|
|
140
|
-
function createPttRecorder(sampleRate = DEFAULT_SYNC_MIC_SAMPLE_RATE) {
|
|
141
|
-
let ctx = null;
|
|
142
|
-
let stream = null;
|
|
143
|
-
let node = null;
|
|
144
|
-
let chunks = [];
|
|
145
|
-
let recording = false;
|
|
146
|
-
async function ensureOpen() {
|
|
147
|
-
if (ctx) return;
|
|
148
|
-
const streamPromise = navigator.mediaDevices.getUserMedia({ audio: VOICE_CAPTURE_CONSTRAINTS });
|
|
149
|
-
const audioCtx = new AudioContext({
|
|
150
|
-
sampleRate,
|
|
151
|
-
latencyHint: "interactive"
|
|
152
|
-
});
|
|
153
|
-
try {
|
|
154
|
-
const [media] = await Promise.all([
|
|
155
|
-
streamPromise,
|
|
156
|
-
audioCtx.resume(),
|
|
157
|
-
audioCtx.audioWorklet.addModule(CAPTURE_WORKLET_MODULE_URL)
|
|
158
|
-
]);
|
|
159
|
-
stream = media;
|
|
160
|
-
} catch (err) {
|
|
161
|
-
streamPromise.then((s) => {
|
|
162
|
-
for (const t of s.getTracks()) t.stop();
|
|
163
|
-
}).catch(() => {});
|
|
164
|
-
await audioCtx.close().catch(() => {});
|
|
165
|
-
throw err;
|
|
166
|
-
}
|
|
167
|
-
const workletNode = new AudioWorkletNode(audioCtx, "aai-sync-capture", {
|
|
168
|
-
channelCount: 1,
|
|
169
|
-
channelCountMode: "explicit",
|
|
170
|
-
processorOptions: { batchSamples: CAPTURE_BATCH_SAMPLES }
|
|
171
|
-
});
|
|
172
|
-
workletNode.port.onmessage = (e) => {
|
|
173
|
-
const data = e.data;
|
|
174
|
-
if (recording && data.event === "chunk" && data.samples) chunks.push(data.samples);
|
|
175
|
-
};
|
|
176
|
-
audioCtx.createMediaStreamSource(stream).connect(workletNode);
|
|
177
|
-
ctx = audioCtx;
|
|
178
|
-
node = workletNode;
|
|
179
|
-
}
|
|
180
|
-
return {
|
|
181
|
-
async start() {
|
|
182
|
-
await ensureOpen();
|
|
183
|
-
chunks = [];
|
|
184
|
-
recording = true;
|
|
185
|
-
},
|
|
186
|
-
async stop() {
|
|
187
|
-
await new Promise((r) => setTimeout(r, 150));
|
|
188
|
-
recording = false;
|
|
189
|
-
const total = chunks.reduce((n, c) => n + c.length, 0);
|
|
190
|
-
const all = new Float32Array(total);
|
|
191
|
-
let offset = 0;
|
|
192
|
-
for (const c of chunks) {
|
|
193
|
-
all.set(c, offset);
|
|
194
|
-
offset += c.length;
|
|
195
|
-
}
|
|
196
|
-
chunks = [];
|
|
197
|
-
return floatToPcm16(all);
|
|
198
|
-
},
|
|
199
|
-
async close() {
|
|
200
|
-
recording = false;
|
|
201
|
-
node?.disconnect();
|
|
202
|
-
if (stream) for (const t of stream.getTracks()) t.stop();
|
|
203
|
-
await ctx?.close().catch(() => {});
|
|
204
|
-
ctx = null;
|
|
205
|
-
node = null;
|
|
206
|
-
stream = null;
|
|
207
|
-
}
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
|
-
//#endregion
|
|
211
|
-
//#region sync-session.ts
|
|
212
|
-
/**
|
|
213
|
-
* Sync-mode browser session — HTTP turns, no WebSocket.
|
|
214
|
-
*
|
|
215
|
-
* The client half of the server's `POST /sync` endpoint (see
|
|
216
|
-
* `host/sync-turn.ts` in `@alexkroman1/aai`): each turn is one request
|
|
217
|
-
* carrying committed text or one endpointed utterance of PCM16 audio plus
|
|
218
|
-
* the conversation history, answered with the transcript, the reply text,
|
|
219
|
-
* and (when the agent's TTS provider supports one-shot synthesis) the
|
|
220
|
-
* spoken reply. The server holds no session state — this object owns the
|
|
221
|
-
* history and replays it every turn.
|
|
222
|
-
*
|
|
223
|
-
* Microphone capture and utterance endpointing live in `sync-mic.ts` /
|
|
224
|
-
* `sync-vad.ts`; this module is transport only, so it also runs in
|
|
225
|
-
* non-browser clients that bring their own audio.
|
|
226
|
-
*/
|
|
227
|
-
/** Base64-encode PCM16 samples (chunked — `btoa` takes a binary string). */
|
|
228
|
-
function pcm16ToBase64(pcm) {
|
|
229
|
-
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
|
230
|
-
let binary = "";
|
|
231
|
-
const CHUNK = 32768;
|
|
232
|
-
for (let i = 0; i < bytes.length; i += CHUNK) binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
|
|
233
|
-
return btoa(binary);
|
|
234
|
-
}
|
|
235
|
-
/** Decode base64 PCM16LE (the sync response's `audio` field) into samples. */
|
|
236
|
-
function base64ToPcm16(base64) {
|
|
237
|
-
const binary = atob(base64);
|
|
238
|
-
const samples = binary.length >> 1;
|
|
239
|
-
const bytes = new Uint8Array(samples * 2);
|
|
240
|
-
for (let i = 0; i < bytes.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
241
|
-
return new Int16Array(bytes.buffer, 0, samples);
|
|
242
|
-
}
|
|
243
|
-
/** Create a {@link SyncSession} against a sync-mode agent server. */
|
|
244
|
-
function createSyncSession(opts) {
|
|
245
|
-
const fetchFn = opts.fetch ?? ((input, init) => globalThis.fetch(input, init));
|
|
246
|
-
const history = [];
|
|
247
|
-
let queue = Promise.resolve();
|
|
248
|
-
async function parseTurn(resp) {
|
|
249
|
-
if (!resp.ok) {
|
|
250
|
-
const detail = safeJsonParse(await resp.text().catch(() => "")) ?? {};
|
|
251
|
-
throw new Error(`Sync turn failed: HTTP ${resp.status}${detail.error ? ` (${detail.error})` : ""}`);
|
|
252
|
-
}
|
|
253
|
-
const parsed = SyncTurnResponseSchema.safeParse(await resp.json());
|
|
254
|
-
if (!parsed.success) throw new Error("Sync turn failed: malformed server response");
|
|
255
|
-
return parsed.data;
|
|
256
|
-
}
|
|
257
|
-
async function runTurn(body) {
|
|
258
|
-
try {
|
|
259
|
-
const turn = await parseTurn(await fetchFn(opts.url, {
|
|
260
|
-
method: "POST",
|
|
261
|
-
headers: { "Content-Type": "application/json" },
|
|
262
|
-
body: JSON.stringify({
|
|
263
|
-
...body,
|
|
264
|
-
history: [...history]
|
|
265
|
-
})
|
|
266
|
-
}));
|
|
267
|
-
history.push({
|
|
268
|
-
role: "user",
|
|
269
|
-
content: turn.transcript
|
|
270
|
-
});
|
|
271
|
-
if (turn.reply.length > 0) history.push({
|
|
272
|
-
role: "assistant",
|
|
273
|
-
content: turn.reply
|
|
274
|
-
});
|
|
275
|
-
if (history.length > DEFAULT_MAX_HISTORY) history.splice(0, history.length - DEFAULT_MAX_HISTORY);
|
|
276
|
-
const result = {
|
|
277
|
-
...turn,
|
|
278
|
-
pcm: turn.audio !== void 0 ? base64ToPcm16(turn.audio) : null
|
|
279
|
-
};
|
|
280
|
-
opts.onTurn?.(result);
|
|
281
|
-
return result;
|
|
282
|
-
} catch (err) {
|
|
283
|
-
const wrapped = err instanceof Error ? err : new Error(errorMessage(err));
|
|
284
|
-
opts.onError?.(wrapped);
|
|
285
|
-
throw wrapped;
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
function enqueue(body) {
|
|
289
|
-
const next = queue.then(() => runTurn(body), () => runTurn(body));
|
|
290
|
-
queue = next.catch(() => void 0);
|
|
291
|
-
return next;
|
|
292
|
-
}
|
|
293
|
-
return {
|
|
294
|
-
get history() {
|
|
295
|
-
return history;
|
|
296
|
-
},
|
|
297
|
-
sendText(text) {
|
|
298
|
-
return enqueue({ text });
|
|
299
|
-
},
|
|
300
|
-
sendPcm16(pcm, sampleRate) {
|
|
301
|
-
return enqueue({
|
|
302
|
-
audio: pcm16ToBase64(pcm),
|
|
303
|
-
sampleRate
|
|
304
|
-
});
|
|
305
|
-
},
|
|
306
|
-
reset() {
|
|
307
|
-
history.length = 0;
|
|
308
|
-
}
|
|
309
|
-
};
|
|
310
|
-
}
|
|
311
|
-
//#endregion
|
|
312
|
-
//#region components/workflow-view.tsx
|
|
313
|
-
/** @jsxImportSource react */
|
|
314
|
-
/**
|
|
315
|
-
* Default shell for workflows (`workflow()` definitions) — the SDK's
|
|
316
|
-
* audio-in / action-out mode.
|
|
317
|
-
*
|
|
318
|
-
* Where {@link ChatView} is a *conversation*, this surface is
|
|
319
|
-
* a one-shot *run*: hold the talk button (or upload an audio file) to stage
|
|
320
|
-
* one instruction, press **Go**, and the whole clip goes out as a single
|
|
321
|
-
* `POST /sync` request with **no history** — the server transcribes it, the
|
|
322
|
-
* agentic loop executes the actions with the workflow's tools, and the view
|
|
323
|
-
* shows what was heard and the tool calls that ran. Deliberately **no
|
|
324
|
-
* greeting and no assistant prose** — a workflow is an execution surface,
|
|
325
|
-
* not a chat, so the record of a run is the transcript plus the actions
|
|
326
|
-
* taken. Each run is independent; staging a new clip clears the last
|
|
327
|
-
* result.
|
|
328
|
-
*
|
|
329
|
-
* Built on the same {@link ConsoleShell} chrome as the chat shell, so the
|
|
330
|
-
* two app modes share one visual language by construction. Rendered by
|
|
331
|
-
* `client()` when `GET /client-config` declares
|
|
332
|
-
* `kind: "workflow"`; exported for custom clients that want the stock run
|
|
333
|
-
* surface with their own chrome.
|
|
334
|
-
*/
|
|
335
|
-
/**
|
|
336
|
-
* The sync response's completed tool records, as the shared
|
|
337
|
-
* {@link ToolCallBlock} rows render them. A sync turn only reports once it
|
|
338
|
-
* has fully run, so every call is `done`; the synthetic fallback id keeps
|
|
339
|
-
* render keys stable when a provider omits `toolCallId`.
|
|
340
|
-
*/
|
|
341
|
-
function toToolCallInfo(turn) {
|
|
342
|
-
return (turn.toolCalls ?? []).map((call, i) => ({
|
|
343
|
-
callId: call.toolCallId.length > 0 ? call.toolCallId : `run-call-${i}`,
|
|
344
|
-
name: call.toolName,
|
|
345
|
-
args: call.args,
|
|
346
|
-
status: "done",
|
|
347
|
-
result: call.result,
|
|
348
|
-
seq: i,
|
|
349
|
-
afterMessageId: -1
|
|
350
|
-
}));
|
|
351
|
-
}
|
|
352
|
-
function clipSeconds(clip) {
|
|
353
|
-
return clip.pcm.length / clip.sampleRate;
|
|
354
|
-
}
|
|
355
|
-
/**
|
|
356
|
-
* Map the run lifecycle onto the shared eyebrow states: recording reads as
|
|
357
|
-
* "listening", an in-flight run as "thinking", a finished run as "ready".
|
|
358
|
-
*/
|
|
359
|
-
function workflowState(opts) {
|
|
360
|
-
if (opts.running) return "thinking";
|
|
361
|
-
if (opts.recording) return "listening";
|
|
362
|
-
if (opts.error) return "error";
|
|
363
|
-
return "ready";
|
|
364
|
-
}
|
|
365
|
-
/** The output card's body: instruction line, staged clip, then the run result. */
|
|
366
|
-
function RunCard({ clip, running, result }) {
|
|
367
|
-
const theme = useTheme();
|
|
368
|
-
return /* @__PURE__ */ jsxs("div", {
|
|
369
|
-
className: "flex flex-col gap-5 p-7",
|
|
370
|
-
children: [
|
|
371
|
-
clip === null && result === null && !running && /* @__PURE__ */ jsx("p", {
|
|
372
|
-
className: "text-sm",
|
|
373
|
-
style: { color: "#57534B" },
|
|
374
|
-
children: "Hold to talk or upload an audio file, then press Go. The whole clip runs as one instruction — the actions it executed appear here."
|
|
375
|
-
}),
|
|
376
|
-
clip && !running && /* @__PURE__ */ jsxs("p", {
|
|
377
|
-
className: "text-sm",
|
|
378
|
-
"data-testid": "staged-clip",
|
|
379
|
-
style: { color: "#57534B" },
|
|
380
|
-
children: [
|
|
381
|
-
clip.source === "recording" ? "Recording" : clip.source,
|
|
382
|
-
" staged (",
|
|
383
|
-
clipSeconds(clip).toFixed(1),
|
|
384
|
-
"s) — press Go to run it."
|
|
385
|
-
]
|
|
386
|
-
}),
|
|
387
|
-
running && /* @__PURE__ */ jsx("div", {
|
|
388
|
-
"data-testid": "running",
|
|
389
|
-
children: /* @__PURE__ */ jsx(ThinkingDots, {})
|
|
390
|
-
}),
|
|
391
|
-
result && /* @__PURE__ */ jsxs("div", {
|
|
392
|
-
className: "flex flex-col gap-1.5",
|
|
393
|
-
"data-testid": "run-result",
|
|
394
|
-
children: [
|
|
395
|
-
/* @__PURE__ */ jsx("span", {
|
|
396
|
-
className: "text-[10px] font-medium tracking-[1.2px] uppercase leading-none",
|
|
397
|
-
style: { color: "#6F6A60" },
|
|
398
|
-
children: "Heard"
|
|
399
|
-
}),
|
|
400
|
-
/* @__PURE__ */ jsx("p", {
|
|
401
|
-
className: "text-[15px] leading-[22px]",
|
|
402
|
-
style: { color: theme.text },
|
|
403
|
-
children: result.heard
|
|
404
|
-
}),
|
|
405
|
-
/* @__PURE__ */ jsx("span", {
|
|
406
|
-
className: "text-[10px] font-medium tracking-[1.2px] uppercase leading-none mt-1.5",
|
|
407
|
-
style: { color: "#6F6A60" },
|
|
408
|
-
children: "Actions"
|
|
409
|
-
}),
|
|
410
|
-
result.toolCalls.length === 0 ? /* @__PURE__ */ jsx("p", {
|
|
411
|
-
className: "text-sm",
|
|
412
|
-
style: { color: "#57534B" },
|
|
413
|
-
children: "No actions were taken."
|
|
414
|
-
}) : /* @__PURE__ */ jsx("div", {
|
|
415
|
-
className: "flex flex-col gap-2",
|
|
416
|
-
"data-testid": "run-tool-calls",
|
|
417
|
-
children: result.toolCalls.map((call) => /* @__PURE__ */ jsx(ToolCallBlock, { toolCall: call }, call.callId))
|
|
418
|
-
})
|
|
419
|
-
]
|
|
420
|
-
})
|
|
421
|
-
]
|
|
422
|
-
});
|
|
423
|
-
}
|
|
424
|
-
/**
|
|
425
|
-
* Workflow run surface: push-to-talk or audio upload stages one instruction
|
|
426
|
-
* clip; **Go** runs it as a single sync turn and shows what was heard plus
|
|
427
|
-
* the tool calls that executed — no greeting, no assistant messages.
|
|
428
|
-
*
|
|
429
|
-
* @public
|
|
430
|
-
*/
|
|
431
|
-
function WorkflowView({ syncUrl, title }) {
|
|
432
|
-
const [clip, setClip] = useState(null);
|
|
433
|
-
const [result, setResult] = useState(null);
|
|
434
|
-
const [recording, setRecording] = useState(false);
|
|
435
|
-
const [running, setRunning] = useState(false);
|
|
436
|
-
const [error, setError] = useState(null);
|
|
437
|
-
const recorder = useRef(null);
|
|
438
|
-
const fileInput = useRef(null);
|
|
439
|
-
const disposed = useRef(false);
|
|
440
|
-
const sessionRef = useRef(createSyncSession({ url: syncUrl }));
|
|
441
|
-
useEffect(() => () => {
|
|
442
|
-
disposed.current = true;
|
|
443
|
-
recorder.current?.close();
|
|
444
|
-
recorder.current = null;
|
|
445
|
-
}, []);
|
|
446
|
-
async function holdStart() {
|
|
447
|
-
if (running || recording) return;
|
|
448
|
-
try {
|
|
449
|
-
recorder.current ??= createPttRecorder();
|
|
450
|
-
await recorder.current.start();
|
|
451
|
-
if (disposed.current) return;
|
|
452
|
-
setRecording(true);
|
|
453
|
-
setError(null);
|
|
454
|
-
} catch (err) {
|
|
455
|
-
setError(err instanceof Error ? err.message : String(err));
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
|
-
async function holdEnd() {
|
|
459
|
-
if (!recording) return;
|
|
460
|
-
setRecording(false);
|
|
461
|
-
try {
|
|
462
|
-
const pcm = await recorder.current?.stop();
|
|
463
|
-
if (disposed.current || !pcm) return;
|
|
464
|
-
if (pcm.length === 0) {
|
|
465
|
-
setError("Nothing was recorded — hold the button while you speak.");
|
|
466
|
-
return;
|
|
467
|
-
}
|
|
468
|
-
setClip({
|
|
469
|
-
pcm,
|
|
470
|
-
sampleRate: DEFAULT_SYNC_MIC_SAMPLE_RATE,
|
|
471
|
-
source: "recording"
|
|
472
|
-
});
|
|
473
|
-
setResult(null);
|
|
474
|
-
} catch (err) {
|
|
475
|
-
setError(err instanceof Error ? err.message : String(err));
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
async function stageFile(file) {
|
|
479
|
-
try {
|
|
480
|
-
const pcm = await decodeAudioToPcm16(await file.arrayBuffer(), DEFAULT_SYNC_MIC_SAMPLE_RATE);
|
|
481
|
-
if (disposed.current) return;
|
|
482
|
-
if (pcm.length === 0) {
|
|
483
|
-
setError("That file decoded to no audio.");
|
|
484
|
-
return;
|
|
485
|
-
}
|
|
486
|
-
setClip({
|
|
487
|
-
pcm,
|
|
488
|
-
sampleRate: DEFAULT_SYNC_MIC_SAMPLE_RATE,
|
|
489
|
-
source: file.name
|
|
490
|
-
});
|
|
491
|
-
setResult(null);
|
|
492
|
-
setError(null);
|
|
493
|
-
} catch {
|
|
494
|
-
setError("Could not decode that file as audio.");
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
|
-
async function go() {
|
|
498
|
-
if (!clip || running) return;
|
|
499
|
-
setRunning(true);
|
|
500
|
-
setError(null);
|
|
501
|
-
sessionRef.current.reset();
|
|
502
|
-
const outcome = await sessionRef.current.sendPcm16(clip.pcm, clip.sampleRate).then((turn) => ({ turn }), (err) => ({ message: err instanceof Error ? err.message : String(err) }));
|
|
503
|
-
if (disposed.current) return;
|
|
504
|
-
setRunning(false);
|
|
505
|
-
if ("turn" in outcome) {
|
|
506
|
-
setResult({
|
|
507
|
-
heard: outcome.turn.transcript,
|
|
508
|
-
toolCalls: toToolCallInfo(outcome.turn)
|
|
509
|
-
});
|
|
510
|
-
setClip(null);
|
|
511
|
-
} else setError(outcome.message);
|
|
512
|
-
}
|
|
513
|
-
const state = workflowState({
|
|
514
|
-
error,
|
|
515
|
-
recording,
|
|
516
|
-
running
|
|
517
|
-
});
|
|
518
|
-
const controls = /* @__PURE__ */ jsxs("div", {
|
|
519
|
-
className: "flex items-center gap-2 shrink-0",
|
|
520
|
-
children: [
|
|
521
|
-
/* @__PURE__ */ jsxs(Button, {
|
|
522
|
-
size: "lg",
|
|
523
|
-
variant: recording ? "default" : "secondary",
|
|
524
|
-
className: "select-none touch-none",
|
|
525
|
-
style: recording ? {
|
|
526
|
-
background: ERROR_COLOR,
|
|
527
|
-
borderColor: "transparent"
|
|
528
|
-
} : void 0,
|
|
529
|
-
disabled: running,
|
|
530
|
-
onPointerDown: () => void holdStart(),
|
|
531
|
-
onPointerUp: () => void holdEnd(),
|
|
532
|
-
onPointerLeave: () => void holdEnd(),
|
|
533
|
-
"aria-pressed": recording,
|
|
534
|
-
title: "Hold to record your instructions",
|
|
535
|
-
children: [/* @__PURE__ */ jsx("span", {
|
|
536
|
-
className: clsx("w-2 h-2 rounded-full mr-2", recording && "animate-pulse"),
|
|
537
|
-
style: { background: recording ? "#fff" : ERROR_COLOR }
|
|
538
|
-
}), recording ? "Release to stage" : "Hold to talk"]
|
|
539
|
-
}),
|
|
540
|
-
/* @__PURE__ */ jsx("input", {
|
|
541
|
-
ref: fileInput,
|
|
542
|
-
type: "file",
|
|
543
|
-
accept: "audio/*",
|
|
544
|
-
className: "hidden",
|
|
545
|
-
"data-testid": "workflow-file-input",
|
|
546
|
-
onChange: (e) => {
|
|
547
|
-
const file = e.target.files?.[0];
|
|
548
|
-
e.target.value = "";
|
|
549
|
-
if (file) stageFile(file);
|
|
550
|
-
}
|
|
551
|
-
}),
|
|
552
|
-
/* @__PURE__ */ jsx(Button, {
|
|
553
|
-
size: "lg",
|
|
554
|
-
variant: "secondary",
|
|
555
|
-
disabled: running || recording,
|
|
556
|
-
onClick: () => fileInput.current?.click(),
|
|
557
|
-
title: "Upload an audio file of your instructions",
|
|
558
|
-
children: "Upload audio"
|
|
559
|
-
}),
|
|
560
|
-
/* @__PURE__ */ jsx(Button, {
|
|
561
|
-
size: "lg",
|
|
562
|
-
disabled: !clip || running || recording,
|
|
563
|
-
onClick: () => void go(),
|
|
564
|
-
"data-testid": "workflow-go",
|
|
565
|
-
title: "Run the staged instructions",
|
|
566
|
-
children: running ? "Running…" : "Go"
|
|
567
|
-
}),
|
|
568
|
-
/* @__PURE__ */ jsx(UrlChip, {
|
|
569
|
-
label: "Run",
|
|
570
|
-
url: syncUrl,
|
|
571
|
-
hint: "Each run is one POST to this endpoint",
|
|
572
|
-
testId: "sync-url-chip",
|
|
573
|
-
className: "ml-auto min-w-0 max-w-[40%]"
|
|
574
|
-
})
|
|
575
|
-
]
|
|
576
|
-
});
|
|
577
|
-
return /* @__PURE__ */ jsx(ConsoleShell, {
|
|
578
|
-
title: title ?? "Workflow",
|
|
579
|
-
state,
|
|
580
|
-
pulsing: recording || running,
|
|
581
|
-
error,
|
|
582
|
-
footer: controls,
|
|
583
|
-
children: /* @__PURE__ */ jsx("div", {
|
|
584
|
-
role: "log",
|
|
585
|
-
className: "flex-1 overflow-y-auto [scrollbar-width:none]",
|
|
586
|
-
children: /* @__PURE__ */ jsx(RunCard, {
|
|
587
|
-
clip,
|
|
588
|
-
running,
|
|
589
|
-
result
|
|
590
|
-
})
|
|
591
|
-
})
|
|
592
|
-
});
|
|
593
|
-
}
|
|
594
|
-
//#endregion
|
|
595
|
-
//#region define-client.tsx
|
|
596
|
-
/** @jsxImportSource react */
|
|
597
|
-
function resolveContainer(target = "#app") {
|
|
598
|
-
if (typeof target !== "string") return target;
|
|
599
|
-
const el = document.querySelector(target);
|
|
600
|
-
if (!el) throw new Error(`Element not found: ${target}`);
|
|
601
|
-
return el;
|
|
602
|
-
}
|
|
603
|
-
/**
|
|
604
|
-
* Default shell rendered in config tier.
|
|
605
|
-
* Wraps StartScreen → (SidebarLayout →) ChatView.
|
|
606
|
-
*/
|
|
607
|
-
function DefaultShell({ name, Sidebar, sidebarWidth }) {
|
|
608
|
-
const chat = /* @__PURE__ */ jsx(ChatView, { title: name });
|
|
609
|
-
return /* @__PURE__ */ jsx(StartScreen, {
|
|
610
|
-
title: name,
|
|
611
|
-
children: Sidebar ? /* @__PURE__ */ jsx(SidebarLayout, {
|
|
612
|
-
sidebar: /* @__PURE__ */ jsx(Sidebar, {}),
|
|
613
|
-
sidebarWidth,
|
|
614
|
-
children: chat
|
|
615
|
-
}) : chat
|
|
616
|
-
});
|
|
617
|
-
}
|
|
618
|
-
/**
|
|
619
|
-
* Config-tier root: resolves the app kind via the server's
|
|
620
|
-
* `GET client-config` and renders the chat shell or the workflow surface.
|
|
621
|
-
*
|
|
622
|
-
* The chat shell renders immediately — optimistically — while the lookup is
|
|
623
|
-
* in flight, then swaps if the agent is a workflow. That keeps mounting
|
|
624
|
-
* synchronous and keeps agents on servers without the endpoint (every
|
|
625
|
-
* lookup failure resolves to the agent kind) exactly as before.
|
|
626
|
-
*/
|
|
627
|
-
function DefaultRoot({ platformUrl, name, Sidebar, sidebarWidth }) {
|
|
628
|
-
const [resolved, setResolved] = useState(null);
|
|
629
|
-
useEffect(() => {
|
|
630
|
-
let cancelled = false;
|
|
631
|
-
fetchClientConfig(platformUrl).then((cfg) => {
|
|
632
|
-
if (!cancelled) setResolved(cfg);
|
|
633
|
-
});
|
|
634
|
-
return () => {
|
|
635
|
-
cancelled = true;
|
|
636
|
-
};
|
|
637
|
-
}, [platformUrl]);
|
|
638
|
-
if (resolved?.kind === "workflow") return /* @__PURE__ */ jsx(WorkflowView, {
|
|
639
|
-
syncUrl: buildAgentUrl(platformUrl, "sync").href,
|
|
640
|
-
title: name ?? resolved.name
|
|
641
|
-
});
|
|
642
|
-
return /* @__PURE__ */ jsx(DefaultShell, {
|
|
643
|
-
name: name ?? resolved?.name,
|
|
644
|
-
Sidebar,
|
|
645
|
-
sidebarWidth
|
|
646
|
-
});
|
|
647
|
-
}
|
|
648
|
-
/**
|
|
649
|
-
* Define and mount a client UI for a voice agent.
|
|
650
|
-
*
|
|
651
|
-
* **Tier 1 (config-only):** Pass options without `component` to get the
|
|
652
|
-
* default shell (StartScreen + ChatView, optional sidebar).
|
|
653
|
-
*
|
|
654
|
-
* **Tier 2 (custom component):** Pass `component` to render a fully custom
|
|
655
|
-
* root component inside the providers.
|
|
656
|
-
*
|
|
657
|
-
* @example Tier 1
|
|
658
|
-
* ```tsx
|
|
659
|
-
* client({
|
|
660
|
-
* name: "Pizza Ordering",
|
|
661
|
-
* theme: { bg: "#1a1a1a", primary: "#e55" },
|
|
662
|
-
* sidebar: OrderPanel,
|
|
663
|
-
* tools: { add_pizza: { icon: "🍕", label: "Adding pizza" } },
|
|
664
|
-
* });
|
|
665
|
-
* ```
|
|
666
|
-
*
|
|
667
|
-
* @example Tier 2
|
|
668
|
-
* ```tsx
|
|
669
|
-
* client({ component: MyCustomApp });
|
|
670
|
-
* ```
|
|
671
|
-
*
|
|
672
|
-
* @returns A {@link ClientHandle} for cleanup.
|
|
673
|
-
* @throws If the target element is not found in the DOM.
|
|
674
|
-
*
|
|
675
|
-
* @public
|
|
676
|
-
*/
|
|
677
|
-
function client(config) {
|
|
678
|
-
const container = resolveContainer(config.target);
|
|
679
|
-
const platformUrl = config.platformUrl ?? globalThis.location.origin + globalThis.location.pathname;
|
|
680
|
-
const session = createSessionCore({
|
|
681
|
-
platformUrl,
|
|
682
|
-
onSessionId: config.onSessionId,
|
|
683
|
-
resumeSessionId: config.resumeSessionId,
|
|
684
|
-
WebSocket: config.WebSocket
|
|
685
|
-
});
|
|
686
|
-
const rootNode = config.component ? createElement(config.component) : createElement(DefaultRoot, {
|
|
687
|
-
platformUrl,
|
|
688
|
-
name: config.name,
|
|
689
|
-
Sidebar: config.sidebar,
|
|
690
|
-
sidebarWidth: config.sidebarWidth
|
|
691
|
-
});
|
|
692
|
-
const toolConfig = config.tools ?? {};
|
|
693
|
-
const root = createRoot(container);
|
|
694
|
-
flushSync(() => {
|
|
695
|
-
root.render(createElement(ToolConfigContext.Provider, { value: toolConfig }, createElement(ThemeProvider, { value: config.theme }, createElement(SessionProvider, { value: session }, rootNode))));
|
|
696
|
-
});
|
|
697
|
-
const handle = {
|
|
698
|
-
session,
|
|
699
|
-
dispose() {
|
|
700
|
-
root.unmount();
|
|
701
|
-
session[Symbol.dispose]();
|
|
702
|
-
},
|
|
703
|
-
[Symbol.dispose]() {
|
|
704
|
-
handle.dispose();
|
|
705
|
-
}
|
|
706
|
-
};
|
|
707
|
-
return handle;
|
|
708
|
-
}
|
|
709
|
-
//#endregion
|
|
710
|
-
export { pcm16ToBase64 as a, createPttRecorder as c, fetchClientConfig as d, createSyncSession as i, floatToPcm16 as l, WorkflowView as n, CAPTURE_WORKLET_MODULE_URL as o, base64ToPcm16 as r, DEFAULT_SYNC_MIC_SAMPLE_RATE as s, client as t, buildAgentUrl as u };
|