@runtypelabs/voice 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -6
- package/dist/index.cjs +211 -17
- package/dist/index.d.cts +24 -2
- package/dist/index.d.ts +24 -2
- package/dist/index.js +211 -17
- package/dist/persona.cjs +272 -21
- package/dist/persona.d.cts +23 -4
- package/dist/persona.d.ts +23 -4
- package/dist/persona.js +272 -21
- package/dist/react.cjs +261 -24
- package/dist/react.d.cts +7 -4
- package/dist/react.d.ts +7 -4
- package/dist/react.js +262 -25
- package/dist/types-DaXyPeiV.d.cts +95 -0
- package/dist/types-DaXyPeiV.d.ts +95 -0
- package/package.json +9 -10
- package/dist/types-9SfA7oHc.d.cts +0 -40
- package/dist/types-9SfA7oHc.d.ts +0 -40
package/dist/react.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use client";
|
|
2
2
|
|
|
3
3
|
// src/react.ts
|
|
4
|
-
import { useEffect,
|
|
4
|
+
import { useEffect, useRef, useState, useSyncExternalStore } from "react";
|
|
5
5
|
|
|
6
6
|
// src/pcm-player.ts
|
|
7
7
|
var PCM_SAMPLE_RATE = 24e3;
|
|
@@ -148,9 +148,58 @@ function initialSnapshot() {
|
|
|
148
148
|
error: null,
|
|
149
149
|
errorDetails: void 0,
|
|
150
150
|
interruptionMode: "none",
|
|
151
|
-
canCancel: false
|
|
151
|
+
canCancel: false,
|
|
152
|
+
artifacts: [],
|
|
153
|
+
session: null
|
|
152
154
|
};
|
|
153
155
|
}
|
|
156
|
+
function readString(value) {
|
|
157
|
+
return typeof value === "string" && value ? value : void 0;
|
|
158
|
+
}
|
|
159
|
+
function readArtifactFile(value) {
|
|
160
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
161
|
+
const file = value;
|
|
162
|
+
const path = readString(file.path);
|
|
163
|
+
const mimeType = readString(file.mimeType);
|
|
164
|
+
if (!path || !mimeType) return void 0;
|
|
165
|
+
const language = readString(file.language);
|
|
166
|
+
return { path, mimeType, ...language ? { language } : {} };
|
|
167
|
+
}
|
|
168
|
+
function applyArtifactFrame(existing, frame, identity) {
|
|
169
|
+
if (frame.type === "artifact_start") {
|
|
170
|
+
const title = readString(frame.title);
|
|
171
|
+
const component = readString(frame.component);
|
|
172
|
+
const file = readArtifactFile(frame.file);
|
|
173
|
+
return {
|
|
174
|
+
...identity,
|
|
175
|
+
artifactType: frame.artifactType === "component" ? "component" : "markdown",
|
|
176
|
+
...title ? { title } : {},
|
|
177
|
+
...file ? { file } : {},
|
|
178
|
+
...component ? { component } : {},
|
|
179
|
+
content: "",
|
|
180
|
+
status: "streaming"
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
if (!existing) return void 0;
|
|
184
|
+
if (frame.type === "artifact_delta") {
|
|
185
|
+
const delta = typeof frame.delta === "string" ? frame.delta : "";
|
|
186
|
+
return delta ? { ...existing, content: existing.content + delta } : void 0;
|
|
187
|
+
}
|
|
188
|
+
if (frame.type === "artifact_update") {
|
|
189
|
+
const component = readString(frame.component) ?? existing.component;
|
|
190
|
+
const props = frame.props && typeof frame.props === "object" && !Array.isArray(frame.props) ? frame.props : existing.props;
|
|
191
|
+
return {
|
|
192
|
+
...existing,
|
|
193
|
+
...component ? { component } : {},
|
|
194
|
+
...props ? { props } : {}
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
if (frame.type === "artifact_complete") return { ...existing, status: "complete" };
|
|
198
|
+
return void 0;
|
|
199
|
+
}
|
|
200
|
+
function readTurnId(msg) {
|
|
201
|
+
return typeof msg.turnId === "string" && msg.turnId ? msg.turnId : null;
|
|
202
|
+
}
|
|
154
203
|
var VoiceClient = class {
|
|
155
204
|
constructor(options) {
|
|
156
205
|
this.options = options;
|
|
@@ -171,6 +220,16 @@ var VoiceClient = class {
|
|
|
171
220
|
stoppedResponse = false;
|
|
172
221
|
responseEnded = true;
|
|
173
222
|
hasPendingAudio = false;
|
|
223
|
+
/**
|
|
224
|
+
* The assistant entry built from clause captions for the turn being spoken. The
|
|
225
|
+
* authoritative transcript replaces it; a stop commits it, because it captions
|
|
226
|
+
* audio the caller already heard.
|
|
227
|
+
*/
|
|
228
|
+
pendingAssistant = null;
|
|
229
|
+
/** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
|
|
230
|
+
committedAssistantTurnId = null;
|
|
231
|
+
activeTurnId = null;
|
|
232
|
+
suppressedArtifactTurnId = null;
|
|
174
233
|
getSnapshot = () => this.snapshot;
|
|
175
234
|
subscribe = (listener) => {
|
|
176
235
|
this.listeners.add(listener);
|
|
@@ -192,7 +251,7 @@ var VoiceClient = class {
|
|
|
192
251
|
if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
|
|
193
252
|
this.cleanup();
|
|
194
253
|
const generation = this.generation;
|
|
195
|
-
this.update({ ...initialSnapshot(), status: "connecting" });
|
|
254
|
+
this.update({ ...initialSnapshot(), session: this.snapshot.session, status: "connecting" });
|
|
196
255
|
try {
|
|
197
256
|
const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
|
|
198
257
|
if (generation !== this.generation) return;
|
|
@@ -207,6 +266,20 @@ var VoiceClient = class {
|
|
|
207
266
|
url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
|
|
208
267
|
url.search = "";
|
|
209
268
|
url.searchParams.set("voiceProtocol", "runtype-browser-v1");
|
|
269
|
+
const capabilities = this.options.artifacts ? "partial_transcript,artifacts" : "partial_transcript";
|
|
270
|
+
url.searchParams.set("clientCapabilities", capabilities);
|
|
271
|
+
const sessionId = this.options.sessionId ?? this.snapshot.session?.sessionId;
|
|
272
|
+
if (sessionId) url.searchParams.set("sessionId", sessionId);
|
|
273
|
+
const visitorToken = typeof this.options.visitorToken === "function" ? await this.options.visitorToken() : this.options.visitorToken;
|
|
274
|
+
if (generation !== this.generation) return;
|
|
275
|
+
if (this.options.visitorToken !== void 0 && !visitorToken) {
|
|
276
|
+
throw new Error(
|
|
277
|
+
"Voice visitor credential unavailable. Initialize the client session first."
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
281
|
+
throw new Error("Invalid voice visitor credential.");
|
|
282
|
+
}
|
|
210
283
|
url.hash = "";
|
|
211
284
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
212
285
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
@@ -230,7 +303,11 @@ var VoiceClient = class {
|
|
|
230
303
|
return;
|
|
231
304
|
}
|
|
232
305
|
this.player = player;
|
|
233
|
-
const socket = new WebSocket(url.toString(), [
|
|
306
|
+
const socket = new WebSocket(url.toString(), [
|
|
307
|
+
"runtype.bearer",
|
|
308
|
+
token,
|
|
309
|
+
...visitorToken ? [visitorToken] : []
|
|
310
|
+
]);
|
|
234
311
|
socket.binaryType = "arraybuffer";
|
|
235
312
|
this.socket = socket;
|
|
236
313
|
socket.onopen = () => {
|
|
@@ -261,6 +338,11 @@ var VoiceClient = class {
|
|
|
261
338
|
this.cleanup();
|
|
262
339
|
this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
|
|
263
340
|
};
|
|
341
|
+
/** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
|
|
342
|
+
resetSession = () => {
|
|
343
|
+
if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
|
|
344
|
+
this.update({ session: null });
|
|
345
|
+
};
|
|
264
346
|
toggleMute = () => {
|
|
265
347
|
this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
|
|
266
348
|
};
|
|
@@ -275,9 +357,11 @@ var VoiceClient = class {
|
|
|
275
357
|
this.clearPlayback();
|
|
276
358
|
if (this.snapshot.interruptionMode === "none") {
|
|
277
359
|
this.stoppedResponse = !this.responseEnded;
|
|
360
|
+
this.commitClauseCaptions();
|
|
278
361
|
this.setStatus("listening");
|
|
279
362
|
return;
|
|
280
363
|
}
|
|
364
|
+
this.suppressedArtifactTurnId = this.activeTurnId;
|
|
281
365
|
this.awaitingClear = true;
|
|
282
366
|
this.setStatus("listening");
|
|
283
367
|
this.socket.send(JSON.stringify({ type: "cancel" }));
|
|
@@ -319,6 +403,82 @@ var VoiceClient = class {
|
|
|
319
403
|
this.awaitingClear = false;
|
|
320
404
|
this.stoppedResponse = false;
|
|
321
405
|
this.responseEnded = true;
|
|
406
|
+
this.commitClauseCaptions();
|
|
407
|
+
this.committedAssistantTurnId = null;
|
|
408
|
+
this.activeTurnId = null;
|
|
409
|
+
this.suppressedArtifactTurnId = null;
|
|
410
|
+
const settledArtifacts = this.snapshot.artifacts.filter(
|
|
411
|
+
(artifact) => artifact.status === "complete"
|
|
412
|
+
);
|
|
413
|
+
if (settledArtifacts.length !== this.snapshot.artifacts.length)
|
|
414
|
+
this.update({ artifacts: settledArtifacts });
|
|
415
|
+
}
|
|
416
|
+
/**
|
|
417
|
+
* Folds an `artifact` message into the snapshot. Frames of a turn the caller
|
|
418
|
+
* cancelled are dropped; a playback-only stop leaves the turn running, so its
|
|
419
|
+
* artifacts keep arriving and are kept.
|
|
420
|
+
*/
|
|
421
|
+
applyArtifactMessage(msg) {
|
|
422
|
+
if (this.awaitingClear) return;
|
|
423
|
+
const frame = msg.event;
|
|
424
|
+
if (!frame || typeof frame !== "object" || Array.isArray(frame)) return;
|
|
425
|
+
const event = frame;
|
|
426
|
+
const id = readString(event.id);
|
|
427
|
+
if (!id) return;
|
|
428
|
+
const turnId = readTurnId(msg);
|
|
429
|
+
if (turnId !== null && turnId === this.suppressedArtifactTurnId) return;
|
|
430
|
+
if (turnId !== null) this.activeTurnId = turnId;
|
|
431
|
+
const artifacts = this.snapshot.artifacts;
|
|
432
|
+
const index = artifacts.findIndex((artifact) => artifact.id === id);
|
|
433
|
+
const executionId = readString(msg.executionId);
|
|
434
|
+
const next = applyArtifactFrame(index === -1 ? void 0 : artifacts[index], event, {
|
|
435
|
+
id,
|
|
436
|
+
turnId,
|
|
437
|
+
...executionId ? { executionId } : {}
|
|
438
|
+
});
|
|
439
|
+
if (!next) return;
|
|
440
|
+
this.update({
|
|
441
|
+
artifacts: index === -1 ? [...artifacts, next] : [...artifacts.slice(0, index), next, ...artifacts.slice(index + 1)]
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
/** Settles the spoken clauses in place and ends the turn's claim on them. */
|
|
445
|
+
commitClauseCaptions() {
|
|
446
|
+
this.committedAssistantTurnId = this.pendingAssistant?.turnId ?? null;
|
|
447
|
+
const settled = this.pendingAssistant !== null;
|
|
448
|
+
this.pendingAssistant = null;
|
|
449
|
+
if (!settled) return;
|
|
450
|
+
const transcript = this.snapshot.transcript;
|
|
451
|
+
const last = transcript[transcript.length - 1];
|
|
452
|
+
if (last?.role !== "assistant" || last.partial !== true) return;
|
|
453
|
+
const { partial: _partial, ...committed } = last;
|
|
454
|
+
this.update({ transcript: [...transcript.slice(0, -1), committed] });
|
|
455
|
+
}
|
|
456
|
+
/** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
|
|
457
|
+
appendClauseCaption(text, turnId) {
|
|
458
|
+
const transcript = this.snapshot.transcript;
|
|
459
|
+
const last = transcript[transcript.length - 1];
|
|
460
|
+
if (this.pendingAssistant?.turnId === turnId && last?.role === "assistant") {
|
|
461
|
+
return {
|
|
462
|
+
interimTranscript: null,
|
|
463
|
+
transcript: [...transcript.slice(0, -1), { ...last, content: last.content + text }],
|
|
464
|
+
status: "speaking"
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
this.pendingAssistant = { turnId };
|
|
468
|
+
return {
|
|
469
|
+
interimTranscript: null,
|
|
470
|
+
transcript: [
|
|
471
|
+
...transcript,
|
|
472
|
+
{
|
|
473
|
+
role: "assistant",
|
|
474
|
+
content: text,
|
|
475
|
+
timestamp: Date.now(),
|
|
476
|
+
partial: true,
|
|
477
|
+
...turnId ? { turnId } : {}
|
|
478
|
+
}
|
|
479
|
+
],
|
|
480
|
+
status: "speaking"
|
|
481
|
+
};
|
|
322
482
|
}
|
|
323
483
|
clearPlayback() {
|
|
324
484
|
this.playbackRevision += 1;
|
|
@@ -353,32 +513,61 @@ var VoiceClient = class {
|
|
|
353
513
|
return;
|
|
354
514
|
}
|
|
355
515
|
switch (msg.type) {
|
|
356
|
-
case "session_config":
|
|
516
|
+
case "session_config": {
|
|
357
517
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
358
518
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
359
519
|
}
|
|
520
|
+
const sessionId = readString(msg.sessionId);
|
|
521
|
+
const conversationId = readString(msg.conversationId);
|
|
522
|
+
if (sessionId && conversationId) {
|
|
523
|
+
const session = { sessionId, conversationId };
|
|
524
|
+
this.update({ session });
|
|
525
|
+
this.options.onSession?.(session);
|
|
526
|
+
}
|
|
527
|
+
break;
|
|
528
|
+
}
|
|
529
|
+
case "artifact":
|
|
530
|
+
this.applyArtifactMessage(msg);
|
|
360
531
|
break;
|
|
361
532
|
case "transcript_interim":
|
|
362
533
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
363
534
|
break;
|
|
364
|
-
case "
|
|
535
|
+
case "transcript_partial": {
|
|
536
|
+
if (msg.role !== "assistant" || typeof msg.text !== "string" || !msg.text) break;
|
|
537
|
+
this.activeTurnId = readTurnId(msg) ?? this.activeTurnId;
|
|
538
|
+
if (this.awaitingClear || this.stoppedResponse) break;
|
|
539
|
+
this.responseEnded = false;
|
|
540
|
+
this.update(this.appendClauseCaption(msg.text, readTurnId(msg)));
|
|
541
|
+
break;
|
|
542
|
+
}
|
|
543
|
+
case "transcript_final": {
|
|
365
544
|
if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
|
|
366
|
-
|
|
545
|
+
const role = msg.role;
|
|
546
|
+
const text = msg.text;
|
|
547
|
+
if (role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
|
|
548
|
+
const turnId = readTurnId(msg);
|
|
549
|
+
if (turnId !== null) this.activeTurnId = turnId;
|
|
550
|
+
if (role === "assistant" && turnId !== null && turnId === this.committedAssistantTurnId)
|
|
551
|
+
break;
|
|
367
552
|
this.responseEnded = false;
|
|
553
|
+
const transcript = this.snapshot.transcript;
|
|
554
|
+
const last = transcript[transcript.length - 1];
|
|
555
|
+
const supersedes = role === "assistant" && this.pendingAssistant !== null && last?.role === "assistant" && (turnId === null || this.pendingAssistant.turnId === null || this.pendingAssistant.turnId === turnId);
|
|
556
|
+
this.pendingAssistant = null;
|
|
557
|
+
const entry = {
|
|
558
|
+
role,
|
|
559
|
+
content: text,
|
|
560
|
+
// INVARIANT: reuse the caption's timestamp so a keyed list does not remount the bubble.
|
|
561
|
+
timestamp: supersedes && last ? last.timestamp : Date.now(),
|
|
562
|
+
...turnId ? { turnId } : {}
|
|
563
|
+
};
|
|
368
564
|
this.update({
|
|
369
565
|
interimTranscript: null,
|
|
370
|
-
transcript: [
|
|
371
|
-
|
|
372
|
-
{
|
|
373
|
-
role: msg.role,
|
|
374
|
-
content: msg.text,
|
|
375
|
-
timestamp: Date.now(),
|
|
376
|
-
...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
|
|
377
|
-
}
|
|
378
|
-
],
|
|
379
|
-
status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
|
|
566
|
+
transcript: supersedes ? [...transcript.slice(0, -1), entry] : [...transcript, entry],
|
|
567
|
+
status: this.awaitingClear ? this.snapshot.status : role === "user" ? "thinking" : "speaking"
|
|
380
568
|
});
|
|
381
569
|
break;
|
|
570
|
+
}
|
|
382
571
|
case "audio_end": {
|
|
383
572
|
this.responseEnded = true;
|
|
384
573
|
if (this.stoppedResponse) {
|
|
@@ -395,10 +584,12 @@ var VoiceClient = class {
|
|
|
395
584
|
break;
|
|
396
585
|
}
|
|
397
586
|
case "audio_clear":
|
|
587
|
+
this.suppressedArtifactTurnId = this.activeTurnId ?? this.suppressedArtifactTurnId;
|
|
398
588
|
this.clearPlayback();
|
|
399
589
|
this.awaitingClear = false;
|
|
400
590
|
this.stoppedResponse = false;
|
|
401
591
|
this.responseEnded = true;
|
|
592
|
+
this.commitClauseCaptions();
|
|
402
593
|
this.setStatus("listening");
|
|
403
594
|
break;
|
|
404
595
|
case "metrics": {
|
|
@@ -411,8 +602,11 @@ var VoiceClient = class {
|
|
|
411
602
|
// @snake-case-ok: Existing voice wire contract.
|
|
412
603
|
firstAudioMs: number(msg.first_audio_ms),
|
|
413
604
|
// @snake-case-ok: Existing voice wire contract.
|
|
414
|
-
totalMs: number(msg.total_ms)
|
|
605
|
+
totalMs: number(msg.total_ms),
|
|
606
|
+
// @snake-case-ok: Existing voice wire contract.
|
|
607
|
+
firstSynthesisMs: number(msg.first_synthesis_ms),
|
|
415
608
|
// @snake-case-ok: Existing voice wire contract.
|
|
609
|
+
incremental: msg.incremental === true
|
|
416
610
|
}
|
|
417
611
|
});
|
|
418
612
|
break;
|
|
@@ -467,25 +661,68 @@ var VoiceClient = class {
|
|
|
467
661
|
};
|
|
468
662
|
|
|
469
663
|
// src/react.ts
|
|
470
|
-
function useVoiceClient({
|
|
664
|
+
function useVoiceClient({
|
|
665
|
+
agentId,
|
|
666
|
+
apiUrl,
|
|
667
|
+
clientToken,
|
|
668
|
+
artifacts,
|
|
669
|
+
sessionId,
|
|
670
|
+
visitorToken,
|
|
671
|
+
onSession
|
|
672
|
+
}) {
|
|
673
|
+
const sessionRef = useRef(sessionId);
|
|
674
|
+
useEffect(() => {
|
|
675
|
+
sessionRef.current = sessionId;
|
|
676
|
+
}, [sessionId]);
|
|
471
677
|
const tokenRef = useRef(clientToken);
|
|
678
|
+
const visitorRef = useRef(visitorToken);
|
|
679
|
+
const sessionCallbackRef = useRef(onSession);
|
|
680
|
+
useEffect(() => {
|
|
681
|
+
visitorRef.current = visitorToken;
|
|
682
|
+
sessionCallbackRef.current = onSession;
|
|
683
|
+
}, [visitorToken, onSession]);
|
|
472
684
|
useEffect(() => {
|
|
473
685
|
tokenRef.current = clientToken;
|
|
474
686
|
}, [clientToken]);
|
|
475
|
-
const
|
|
476
|
-
|
|
687
|
+
const createClient = () => new VoiceClient({
|
|
688
|
+
agentId,
|
|
689
|
+
apiUrl,
|
|
690
|
+
artifacts,
|
|
691
|
+
get sessionId() {
|
|
692
|
+
return sessionRef.current;
|
|
693
|
+
},
|
|
694
|
+
get visitorToken() {
|
|
695
|
+
return visitorRef.current;
|
|
696
|
+
},
|
|
697
|
+
onSession: (session) => sessionCallbackRef.current?.(session),
|
|
698
|
+
clientToken: () => typeof tokenRef.current === "function" ? tokenRef.current() : tokenRef.current
|
|
699
|
+
});
|
|
700
|
+
const [binding, setBinding] = useState(() => ({
|
|
701
|
+
agentId,
|
|
702
|
+
apiUrl,
|
|
703
|
+
artifacts,
|
|
704
|
+
sessionId,
|
|
705
|
+
client: createClient()
|
|
706
|
+
}));
|
|
707
|
+
const configurationChanged = binding.agentId !== agentId || binding.apiUrl !== apiUrl || binding.artifacts !== artifacts;
|
|
708
|
+
if (configurationChanged || binding.sessionId !== sessionId) {
|
|
709
|
+
const acknowledgesSession = sessionId !== void 0 && sessionId === binding.client.getSnapshot().session?.sessionId;
|
|
710
|
+
setBinding({
|
|
477
711
|
agentId,
|
|
478
712
|
apiUrl,
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
713
|
+
artifacts,
|
|
714
|
+
sessionId,
|
|
715
|
+
client: !configurationChanged && acknowledgesSession ? binding.client : createClient()
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
const client = binding.client;
|
|
483
719
|
const snapshot = useSyncExternalStore(client.subscribe, client.getSnapshot, client.getSnapshot);
|
|
484
720
|
useEffect(() => () => client.endCall(), [client]);
|
|
485
721
|
return {
|
|
486
722
|
...snapshot,
|
|
487
723
|
startCall: client.startCall,
|
|
488
724
|
endCall: client.endCall,
|
|
725
|
+
resetSession: client.resetSession,
|
|
489
726
|
toggleMute: client.toggleMute,
|
|
490
727
|
cancelResponse: client.cancelResponse
|
|
491
728
|
};
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
type VoiceStatus = 'idle' | 'connecting' | 'listening' | 'thinking' | 'speaking' | 'error';
|
|
2
|
+
type InterruptionMode = 'none' | 'cancel' | 'barge-in';
|
|
3
|
+
interface TranscriptEntry {
|
|
4
|
+
readonly role: 'user' | 'assistant';
|
|
5
|
+
readonly content: string;
|
|
6
|
+
readonly timestamp: number;
|
|
7
|
+
readonly turnId?: string;
|
|
8
|
+
/**
|
|
9
|
+
* The reply is still being spoken, so `content` holds the clauses committed so
|
|
10
|
+
* far. A later snapshot replaces this entry with the authoritative transcript,
|
|
11
|
+
* or clears the flag in place when the caller stops the reply.
|
|
12
|
+
*/
|
|
13
|
+
readonly partial?: boolean;
|
|
14
|
+
}
|
|
15
|
+
interface VoiceMetrics {
|
|
16
|
+
readonly llmMs?: number;
|
|
17
|
+
readonly ttsMs?: number;
|
|
18
|
+
readonly firstAudioMs?: number;
|
|
19
|
+
readonly totalMs?: number;
|
|
20
|
+
/**
|
|
21
|
+
* Turn start to the first synthesis request. Below `llmMs` when the engine spoke
|
|
22
|
+
* clauses while the reply was still generating, and equal to it otherwise.
|
|
23
|
+
*/
|
|
24
|
+
readonly firstSynthesisMs?: number;
|
|
25
|
+
/** True when the reply was synthesized clause by clause, so `ttsMs` covers overlapped work. */
|
|
26
|
+
readonly incremental?: boolean;
|
|
27
|
+
}
|
|
28
|
+
/** File metadata for a previewable artifact, mirroring the unified `artifact_start.file` field. */
|
|
29
|
+
interface VoiceArtifactFile {
|
|
30
|
+
readonly path: string;
|
|
31
|
+
readonly mimeType: string;
|
|
32
|
+
readonly language?: string;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* An artifact the agent produced during a call, assembled from the additive
|
|
36
|
+
* `artifact` wire message. Artifacts arrive beside speech and are never spoken.
|
|
37
|
+
*/
|
|
38
|
+
interface VoiceArtifact {
|
|
39
|
+
readonly id: string;
|
|
40
|
+
readonly turnId: string | null;
|
|
41
|
+
readonly executionId?: string;
|
|
42
|
+
readonly artifactType: 'markdown' | 'component';
|
|
43
|
+
readonly title?: string;
|
|
44
|
+
readonly file?: VoiceArtifactFile;
|
|
45
|
+
readonly content: string;
|
|
46
|
+
readonly component?: string;
|
|
47
|
+
readonly props?: Record<string, unknown>;
|
|
48
|
+
readonly status: 'streaming' | 'complete';
|
|
49
|
+
}
|
|
50
|
+
/** The conversation this call is attached to, reported by the server once the call opens. */
|
|
51
|
+
interface VoiceSession {
|
|
52
|
+
readonly sessionId: string;
|
|
53
|
+
readonly conversationId: string;
|
|
54
|
+
}
|
|
55
|
+
interface VoiceSnapshot {
|
|
56
|
+
readonly status: VoiceStatus;
|
|
57
|
+
readonly transcript: readonly TranscriptEntry[];
|
|
58
|
+
readonly interimTranscript: string | null;
|
|
59
|
+
readonly metrics: VoiceMetrics | null;
|
|
60
|
+
readonly audioLevel: number;
|
|
61
|
+
readonly isMuted: boolean;
|
|
62
|
+
readonly error: string | null;
|
|
63
|
+
readonly errorDetails?: {
|
|
64
|
+
code: string;
|
|
65
|
+
serverId?: string;
|
|
66
|
+
serverName: string;
|
|
67
|
+
diagnosticId: string;
|
|
68
|
+
};
|
|
69
|
+
readonly interruptionMode: InterruptionMode;
|
|
70
|
+
readonly canCancel: boolean;
|
|
71
|
+
readonly artifacts: readonly VoiceArtifact[];
|
|
72
|
+
/** Survives `endCall` so a host can reopen the same conversation; null until the server reports it. */
|
|
73
|
+
readonly session: VoiceSession | null;
|
|
74
|
+
}
|
|
75
|
+
interface VoiceClientOptions {
|
|
76
|
+
agentId: string;
|
|
77
|
+
/** Browser client token, or a getter called once per call to obtain a fresh token. */
|
|
78
|
+
clientToken: string | (() => string | Promise<string>);
|
|
79
|
+
/** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
|
|
80
|
+
apiUrl?: string;
|
|
81
|
+
/**
|
|
82
|
+
* Declares the `artifacts` capability, so the server sends `artifact` messages
|
|
83
|
+
* and the client assembles `snapshot.artifacts`. Off by default, which keeps an
|
|
84
|
+
* existing embed's wire byte-identical.
|
|
85
|
+
*/
|
|
86
|
+
artifacts?: boolean;
|
|
87
|
+
/** Reuses an existing client session's conversation. Overrides the id remembered from a previous call. */
|
|
88
|
+
sessionId?: string;
|
|
89
|
+
/** Proof minted by client/init; resolve from the host's token-scoped visitor store on every call. */
|
|
90
|
+
visitorToken?: string | (() => string | undefined | Promise<string | undefined>);
|
|
91
|
+
/** Observe the server's conversation so a host can reuse it for text. Never contains visitor credentials. */
|
|
92
|
+
onSession?: (session: VoiceSession) => void;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export type { InterruptionMode as I, TranscriptEntry as T, VoiceClientOptions as V, VoiceSnapshot as a, VoiceArtifact as b, VoiceArtifactFile as c, VoiceMetrics as d, VoiceSession as e, VoiceStatus as f };
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
type VoiceStatus = 'idle' | 'connecting' | 'listening' | 'thinking' | 'speaking' | 'error';
|
|
2
|
+
type InterruptionMode = 'none' | 'cancel' | 'barge-in';
|
|
3
|
+
interface TranscriptEntry {
|
|
4
|
+
readonly role: 'user' | 'assistant';
|
|
5
|
+
readonly content: string;
|
|
6
|
+
readonly timestamp: number;
|
|
7
|
+
readonly turnId?: string;
|
|
8
|
+
/**
|
|
9
|
+
* The reply is still being spoken, so `content` holds the clauses committed so
|
|
10
|
+
* far. A later snapshot replaces this entry with the authoritative transcript,
|
|
11
|
+
* or clears the flag in place when the caller stops the reply.
|
|
12
|
+
*/
|
|
13
|
+
readonly partial?: boolean;
|
|
14
|
+
}
|
|
15
|
+
interface VoiceMetrics {
|
|
16
|
+
readonly llmMs?: number;
|
|
17
|
+
readonly ttsMs?: number;
|
|
18
|
+
readonly firstAudioMs?: number;
|
|
19
|
+
readonly totalMs?: number;
|
|
20
|
+
/**
|
|
21
|
+
* Turn start to the first synthesis request. Below `llmMs` when the engine spoke
|
|
22
|
+
* clauses while the reply was still generating, and equal to it otherwise.
|
|
23
|
+
*/
|
|
24
|
+
readonly firstSynthesisMs?: number;
|
|
25
|
+
/** True when the reply was synthesized clause by clause, so `ttsMs` covers overlapped work. */
|
|
26
|
+
readonly incremental?: boolean;
|
|
27
|
+
}
|
|
28
|
+
/** File metadata for a previewable artifact, mirroring the unified `artifact_start.file` field. */
|
|
29
|
+
interface VoiceArtifactFile {
|
|
30
|
+
readonly path: string;
|
|
31
|
+
readonly mimeType: string;
|
|
32
|
+
readonly language?: string;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* An artifact the agent produced during a call, assembled from the additive
|
|
36
|
+
* `artifact` wire message. Artifacts arrive beside speech and are never spoken.
|
|
37
|
+
*/
|
|
38
|
+
interface VoiceArtifact {
|
|
39
|
+
readonly id: string;
|
|
40
|
+
readonly turnId: string | null;
|
|
41
|
+
readonly executionId?: string;
|
|
42
|
+
readonly artifactType: 'markdown' | 'component';
|
|
43
|
+
readonly title?: string;
|
|
44
|
+
readonly file?: VoiceArtifactFile;
|
|
45
|
+
readonly content: string;
|
|
46
|
+
readonly component?: string;
|
|
47
|
+
readonly props?: Record<string, unknown>;
|
|
48
|
+
readonly status: 'streaming' | 'complete';
|
|
49
|
+
}
|
|
50
|
+
/** The conversation this call is attached to, reported by the server once the call opens. */
|
|
51
|
+
interface VoiceSession {
|
|
52
|
+
readonly sessionId: string;
|
|
53
|
+
readonly conversationId: string;
|
|
54
|
+
}
|
|
55
|
+
interface VoiceSnapshot {
|
|
56
|
+
readonly status: VoiceStatus;
|
|
57
|
+
readonly transcript: readonly TranscriptEntry[];
|
|
58
|
+
readonly interimTranscript: string | null;
|
|
59
|
+
readonly metrics: VoiceMetrics | null;
|
|
60
|
+
readonly audioLevel: number;
|
|
61
|
+
readonly isMuted: boolean;
|
|
62
|
+
readonly error: string | null;
|
|
63
|
+
readonly errorDetails?: {
|
|
64
|
+
code: string;
|
|
65
|
+
serverId?: string;
|
|
66
|
+
serverName: string;
|
|
67
|
+
diagnosticId: string;
|
|
68
|
+
};
|
|
69
|
+
readonly interruptionMode: InterruptionMode;
|
|
70
|
+
readonly canCancel: boolean;
|
|
71
|
+
readonly artifacts: readonly VoiceArtifact[];
|
|
72
|
+
/** Survives `endCall` so a host can reopen the same conversation; null until the server reports it. */
|
|
73
|
+
readonly session: VoiceSession | null;
|
|
74
|
+
}
|
|
75
|
+
interface VoiceClientOptions {
|
|
76
|
+
agentId: string;
|
|
77
|
+
/** Browser client token, or a getter called once per call to obtain a fresh token. */
|
|
78
|
+
clientToken: string | (() => string | Promise<string>);
|
|
79
|
+
/** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
|
|
80
|
+
apiUrl?: string;
|
|
81
|
+
/**
|
|
82
|
+
* Declares the `artifacts` capability, so the server sends `artifact` messages
|
|
83
|
+
* and the client assembles `snapshot.artifacts`. Off by default, which keeps an
|
|
84
|
+
* existing embed's wire byte-identical.
|
|
85
|
+
*/
|
|
86
|
+
artifacts?: boolean;
|
|
87
|
+
/** Reuses an existing client session's conversation. Overrides the id remembered from a previous call. */
|
|
88
|
+
sessionId?: string;
|
|
89
|
+
/** Proof minted by client/init; resolve from the host's token-scoped visitor store on every call. */
|
|
90
|
+
visitorToken?: string | (() => string | undefined | Promise<string | undefined>);
|
|
91
|
+
/** Observe the server's conversation so a host can reuse it for text. Never contains visitor credentials. */
|
|
92
|
+
onSession?: (session: VoiceSession) => void;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export type { InterruptionMode as I, TranscriptEntry as T, VoiceClientOptions as V, VoiceSnapshot as a, VoiceArtifact as b, VoiceArtifactFile as c, VoiceMetrics as d, VoiceSession as e, VoiceStatus as f };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@runtypelabs/voice",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Runtype browser voice client with React and Persona adapters",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -30,14 +30,6 @@
|
|
|
30
30
|
"README.md",
|
|
31
31
|
"LICENSE"
|
|
32
32
|
],
|
|
33
|
-
"scripts": {
|
|
34
|
-
"build": "tsup",
|
|
35
|
-
"dev": "tsup --watch",
|
|
36
|
-
"test": "vitest run",
|
|
37
|
-
"typecheck": "tsgo --noEmit",
|
|
38
|
-
"lint": "eslint .",
|
|
39
|
-
"prepublishOnly": "pnpm build"
|
|
40
|
-
},
|
|
41
33
|
"peerDependencies": {
|
|
42
34
|
"react": ">=18 <20",
|
|
43
35
|
"@runtypelabs/persona": ">=4.22.0 <5"
|
|
@@ -69,5 +61,12 @@
|
|
|
69
61
|
},
|
|
70
62
|
"publishConfig": {
|
|
71
63
|
"access": "public"
|
|
64
|
+
},
|
|
65
|
+
"scripts": {
|
|
66
|
+
"build": "tsup",
|
|
67
|
+
"dev": "tsup --watch",
|
|
68
|
+
"test": "vitest run",
|
|
69
|
+
"typecheck": "tsgo --noEmit",
|
|
70
|
+
"lint": "eslint ."
|
|
72
71
|
}
|
|
73
|
-
}
|
|
72
|
+
}
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
type VoiceStatus = 'idle' | 'connecting' | 'listening' | 'thinking' | 'speaking' | 'error';
|
|
2
|
-
type InterruptionMode = 'none' | 'cancel' | 'barge-in';
|
|
3
|
-
interface TranscriptEntry {
|
|
4
|
-
readonly role: 'user' | 'assistant';
|
|
5
|
-
readonly content: string;
|
|
6
|
-
readonly timestamp: number;
|
|
7
|
-
readonly turnId?: string;
|
|
8
|
-
}
|
|
9
|
-
interface VoiceMetrics {
|
|
10
|
-
readonly llmMs?: number;
|
|
11
|
-
readonly ttsMs?: number;
|
|
12
|
-
readonly firstAudioMs?: number;
|
|
13
|
-
readonly totalMs?: number;
|
|
14
|
-
}
|
|
15
|
-
interface VoiceSnapshot {
|
|
16
|
-
readonly status: VoiceStatus;
|
|
17
|
-
readonly transcript: readonly TranscriptEntry[];
|
|
18
|
-
readonly interimTranscript: string | null;
|
|
19
|
-
readonly metrics: VoiceMetrics | null;
|
|
20
|
-
readonly audioLevel: number;
|
|
21
|
-
readonly isMuted: boolean;
|
|
22
|
-
readonly error: string | null;
|
|
23
|
-
readonly errorDetails?: {
|
|
24
|
-
code: string;
|
|
25
|
-
serverId?: string;
|
|
26
|
-
serverName: string;
|
|
27
|
-
diagnosticId: string;
|
|
28
|
-
};
|
|
29
|
-
readonly interruptionMode: InterruptionMode;
|
|
30
|
-
readonly canCancel: boolean;
|
|
31
|
-
}
|
|
32
|
-
interface VoiceClientOptions {
|
|
33
|
-
agentId: string;
|
|
34
|
-
/** Browser client token, or a getter called once per call to obtain a fresh token. */
|
|
35
|
-
clientToken: string | (() => string | Promise<string>);
|
|
36
|
-
/** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
|
|
37
|
-
apiUrl?: string;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export type { InterruptionMode as I, TranscriptEntry as T, VoiceClientOptions as V, VoiceSnapshot as a, VoiceMetrics as b, VoiceStatus as c };
|