@runtypelabs/voice 0.2.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -4
- package/dist/index.cjs +257 -25
- package/dist/index.d.cts +25 -2
- package/dist/index.d.ts +25 -2
- package/dist/index.js +257 -25
- package/dist/persona.cjs +319 -30
- package/dist/persona.d.cts +23 -4
- package/dist/persona.d.ts +23 -4
- package/dist/persona.js +319 -30
- package/dist/react.cjs +320 -32
- package/dist/react.d.cts +7 -4
- package/dist/react.d.ts +7 -4
- package/dist/react.js +321 -33
- package/dist/types-Dkh4LxEA.d.cts +100 -0
- package/dist/types-Dkh4LxEA.d.ts +100 -0
- package/package.json +1 -1
- package/dist/types-9SfA7oHc.d.cts +0 -40
- package/dist/types-9SfA7oHc.d.ts +0 -40
package/dist/persona.cjs
CHANGED
|
@@ -20,6 +20,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
20
20
|
// src/persona.ts
|
|
21
21
|
var persona_exports = {};
|
|
22
22
|
__export(persona_exports, {
|
|
23
|
+
bindVoiceArtifactsToPersona: () => bindVoiceArtifactsToPersona,
|
|
23
24
|
createPersonaVoiceProvider: () => createPersonaVoiceProvider
|
|
24
25
|
});
|
|
25
26
|
module.exports = __toCommonJS(persona_exports);
|
|
@@ -35,13 +36,16 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
35
36
|
this.readOffset = 0
|
|
36
37
|
this.buffered = 0
|
|
37
38
|
this.waiting = true
|
|
38
|
-
// INVARIANT:
|
|
39
|
+
// INVARIANT: Pipeline replies need EOS; continuous sessions report queue drain without a provider turn boundary.
|
|
39
40
|
this.eosSeen = false
|
|
41
|
+
this.continuous = false
|
|
40
42
|
this.revision = 0
|
|
41
43
|
this.port.onmessage = (e) => {
|
|
42
44
|
const msg = e.data
|
|
43
45
|
this.revision = msg.revision
|
|
44
|
-
if (msg.type === '
|
|
46
|
+
if (msg.type === 'continuous') {
|
|
47
|
+
this.continuous = msg.enabled
|
|
48
|
+
} else if (msg.type === 'push') {
|
|
45
49
|
this.eosSeen = false
|
|
46
50
|
this.chunks.push(msg.samples)
|
|
47
51
|
this.buffered += msg.samples.length
|
|
@@ -81,7 +85,7 @@ class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
|
|
|
81
85
|
}
|
|
82
86
|
if (this.buffered === 0) {
|
|
83
87
|
this.waiting = true // mid-reply underrun: re-buffer silently
|
|
84
|
-
if (this.eosSeen) {
|
|
88
|
+
if (this.eosSeen || this.continuous) {
|
|
85
89
|
this.eosSeen = false
|
|
86
90
|
this.port.postMessage({ type: 'drained', revision: this.revision })
|
|
87
91
|
}
|
|
@@ -125,6 +129,9 @@ async function createPcmPlayer(onDrained) {
|
|
|
125
129
|
if (samples.length === 0) return;
|
|
126
130
|
node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
|
|
127
131
|
},
|
|
132
|
+
setContinuousMode(enabled) {
|
|
133
|
+
node.port.postMessage({ type: "continuous", enabled, revision });
|
|
134
|
+
},
|
|
128
135
|
endOfStream() {
|
|
129
136
|
node.port.postMessage({ type: "eos", revision });
|
|
130
137
|
},
|
|
@@ -169,9 +176,58 @@ function initialSnapshot() {
|
|
|
169
176
|
error: null,
|
|
170
177
|
errorDetails: void 0,
|
|
171
178
|
interruptionMode: "none",
|
|
172
|
-
canCancel: false
|
|
179
|
+
canCancel: false,
|
|
180
|
+
artifacts: [],
|
|
181
|
+
session: null
|
|
173
182
|
};
|
|
174
183
|
}
|
|
184
|
+
function readString(value) {
|
|
185
|
+
return typeof value === "string" && value ? value : void 0;
|
|
186
|
+
}
|
|
187
|
+
function readArtifactFile(value) {
|
|
188
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
189
|
+
const file = value;
|
|
190
|
+
const path = readString(file.path);
|
|
191
|
+
const mimeType = readString(file.mimeType);
|
|
192
|
+
if (!path || !mimeType) return void 0;
|
|
193
|
+
const language = readString(file.language);
|
|
194
|
+
return { path, mimeType, ...language ? { language } : {} };
|
|
195
|
+
}
|
|
196
|
+
function applyArtifactFrame(existing, frame, identity) {
|
|
197
|
+
if (frame.type === "artifact_start") {
|
|
198
|
+
const title = readString(frame.title);
|
|
199
|
+
const component = readString(frame.component);
|
|
200
|
+
const file = readArtifactFile(frame.file);
|
|
201
|
+
return {
|
|
202
|
+
...identity,
|
|
203
|
+
artifactType: frame.artifactType === "component" ? "component" : "markdown",
|
|
204
|
+
...title ? { title } : {},
|
|
205
|
+
...file ? { file } : {},
|
|
206
|
+
...component ? { component } : {},
|
|
207
|
+
content: "",
|
|
208
|
+
status: "streaming"
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
if (!existing) return void 0;
|
|
212
|
+
if (frame.type === "artifact_delta") {
|
|
213
|
+
const delta = typeof frame.delta === "string" ? frame.delta : "";
|
|
214
|
+
return delta ? { ...existing, content: existing.content + delta } : void 0;
|
|
215
|
+
}
|
|
216
|
+
if (frame.type === "artifact_update") {
|
|
217
|
+
const component = readString(frame.component) ?? existing.component;
|
|
218
|
+
const props = frame.props && typeof frame.props === "object" && !Array.isArray(frame.props) ? frame.props : existing.props;
|
|
219
|
+
return {
|
|
220
|
+
...existing,
|
|
221
|
+
...component ? { component } : {},
|
|
222
|
+
...props ? { props } : {}
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
if (frame.type === "artifact_complete") return { ...existing, status: "complete" };
|
|
226
|
+
return void 0;
|
|
227
|
+
}
|
|
228
|
+
function readTurnId(msg) {
|
|
229
|
+
return typeof msg.turnId === "string" && msg.turnId ? msg.turnId : null;
|
|
230
|
+
}
|
|
175
231
|
var VoiceClient = class {
|
|
176
232
|
constructor(options) {
|
|
177
233
|
this.options = options;
|
|
@@ -192,6 +248,17 @@ var VoiceClient = class {
|
|
|
192
248
|
stoppedResponse = false;
|
|
193
249
|
responseEnded = true;
|
|
194
250
|
hasPendingAudio = false;
|
|
251
|
+
/**
|
|
252
|
+
* The assistant entry built from clause captions for the turn being spoken. The
|
|
253
|
+
* authoritative transcript replaces it; a stop commits it, because it captions
|
|
254
|
+
* audio the caller already heard.
|
|
255
|
+
*/
|
|
256
|
+
pendingAssistant = null;
|
|
257
|
+
/** Turn whose clause captions were already committed by a stop, so its authoritative text is redundant. */
|
|
258
|
+
committedAssistantTurnId = null;
|
|
259
|
+
activeTurnId = null;
|
|
260
|
+
suppressedArtifactTurnId = null;
|
|
261
|
+
fullDuplex = false;
|
|
195
262
|
getSnapshot = () => this.snapshot;
|
|
196
263
|
subscribe = (listener) => {
|
|
197
264
|
this.listeners.add(listener);
|
|
@@ -213,7 +280,7 @@ var VoiceClient = class {
|
|
|
213
280
|
if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
|
|
214
281
|
this.cleanup();
|
|
215
282
|
const generation = this.generation;
|
|
216
|
-
this.update({ ...initialSnapshot(), status: "connecting" });
|
|
283
|
+
this.update({ ...initialSnapshot(), session: this.snapshot.session, status: "connecting" });
|
|
217
284
|
try {
|
|
218
285
|
const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
|
|
219
286
|
if (generation !== this.generation) return;
|
|
@@ -228,10 +295,26 @@ var VoiceClient = class {
|
|
|
228
295
|
url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
|
|
229
296
|
url.search = "";
|
|
230
297
|
url.searchParams.set("voiceProtocol", "runtype-browser-v1");
|
|
298
|
+
const capabilities = this.options.artifacts ? "partial_transcript,artifacts" : "partial_transcript";
|
|
299
|
+
url.searchParams.set("clientCapabilities", capabilities);
|
|
300
|
+
const sessionId = this.options.sessionId ?? this.snapshot.session?.sessionId;
|
|
301
|
+
if (sessionId) url.searchParams.set("sessionId", sessionId);
|
|
302
|
+
const visitorToken = typeof this.options.visitorToken === "function" ? await this.options.visitorToken() : this.options.visitorToken;
|
|
303
|
+
if (generation !== this.generation) return;
|
|
304
|
+
if (this.options.visitorToken !== void 0 && !visitorToken) {
|
|
305
|
+
throw new Error(
|
|
306
|
+
"Voice visitor credential unavailable. Initialize the client session first."
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
if (visitorToken && !/^cvt_[A-Za-z0-9_-]+$/.test(visitorToken)) {
|
|
310
|
+
throw new Error("Invalid voice visitor credential.");
|
|
311
|
+
}
|
|
312
|
+
if (this.options.fullDuplex !== false)
|
|
313
|
+
url.searchParams.set("voiceCapabilities", "full-duplex-v1");
|
|
231
314
|
url.hash = "";
|
|
232
|
-
const stream = await navigator.mediaDevices.getUserMedia({
|
|
315
|
+
const stream = await (this.options.audioSource?.() ?? navigator.mediaDevices.getUserMedia({
|
|
233
316
|
audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
|
|
234
|
-
});
|
|
317
|
+
}));
|
|
235
318
|
if (generation !== this.generation) {
|
|
236
319
|
stream.getTracks().forEach((track) => track.stop());
|
|
237
320
|
return;
|
|
@@ -251,7 +334,11 @@ var VoiceClient = class {
|
|
|
251
334
|
return;
|
|
252
335
|
}
|
|
253
336
|
this.player = player;
|
|
254
|
-
const socket = new WebSocket(url.toString(), [
|
|
337
|
+
const socket = new WebSocket(url.toString(), [
|
|
338
|
+
"runtype.bearer",
|
|
339
|
+
token,
|
|
340
|
+
...visitorToken ? [visitorToken] : []
|
|
341
|
+
]);
|
|
255
342
|
socket.binaryType = "arraybuffer";
|
|
256
343
|
this.socket = socket;
|
|
257
344
|
socket.onopen = () => {
|
|
@@ -282,6 +369,11 @@ var VoiceClient = class {
|
|
|
282
369
|
this.cleanup();
|
|
283
370
|
this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
|
|
284
371
|
};
|
|
372
|
+
/** Forget the remembered conversation so the next startCall opens a new one. No-op during a call. */
|
|
373
|
+
resetSession = () => {
|
|
374
|
+
if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
|
|
375
|
+
this.update({ session: null });
|
|
376
|
+
};
|
|
285
377
|
toggleMute = () => {
|
|
286
378
|
this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
|
|
287
379
|
};
|
|
@@ -296,9 +388,11 @@ var VoiceClient = class {
|
|
|
296
388
|
this.clearPlayback();
|
|
297
389
|
if (this.snapshot.interruptionMode === "none") {
|
|
298
390
|
this.stoppedResponse = !this.responseEnded;
|
|
391
|
+
this.commitClauseCaptions();
|
|
299
392
|
this.setStatus("listening");
|
|
300
393
|
return;
|
|
301
394
|
}
|
|
395
|
+
this.suppressedArtifactTurnId = this.activeTurnId;
|
|
302
396
|
this.awaitingClear = true;
|
|
303
397
|
this.setStatus("listening");
|
|
304
398
|
this.socket.send(JSON.stringify({ type: "cancel" }));
|
|
@@ -315,6 +409,7 @@ var VoiceClient = class {
|
|
|
315
409
|
});
|
|
316
410
|
}
|
|
317
411
|
cleanup() {
|
|
412
|
+
this.fullDuplex = false;
|
|
318
413
|
this.generation += 1;
|
|
319
414
|
if (this.processor) this.processor.onaudioprocess = null;
|
|
320
415
|
this.processor?.disconnect();
|
|
@@ -340,6 +435,82 @@ var VoiceClient = class {
|
|
|
340
435
|
this.awaitingClear = false;
|
|
341
436
|
this.stoppedResponse = false;
|
|
342
437
|
this.responseEnded = true;
|
|
438
|
+
this.commitClauseCaptions();
|
|
439
|
+
this.committedAssistantTurnId = null;
|
|
440
|
+
this.activeTurnId = null;
|
|
441
|
+
this.suppressedArtifactTurnId = null;
|
|
442
|
+
const settledArtifacts = this.snapshot.artifacts.filter(
|
|
443
|
+
(artifact) => artifact.status === "complete"
|
|
444
|
+
);
|
|
445
|
+
if (settledArtifacts.length !== this.snapshot.artifacts.length)
|
|
446
|
+
this.update({ artifacts: settledArtifacts });
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* Folds an `artifact` message into the snapshot. Frames of a turn the caller
|
|
450
|
+
* cancelled are dropped; a playback-only stop leaves the turn running, so its
|
|
451
|
+
* artifacts keep arriving and are kept.
|
|
452
|
+
*/
|
|
453
|
+
applyArtifactMessage(msg) {
|
|
454
|
+
if (this.awaitingClear) return;
|
|
455
|
+
const frame = msg.event;
|
|
456
|
+
if (!frame || typeof frame !== "object" || Array.isArray(frame)) return;
|
|
457
|
+
const event = frame;
|
|
458
|
+
const id = readString(event.id);
|
|
459
|
+
if (!id) return;
|
|
460
|
+
const turnId = readTurnId(msg);
|
|
461
|
+
if (turnId !== null && turnId === this.suppressedArtifactTurnId) return;
|
|
462
|
+
if (turnId !== null) this.activeTurnId = turnId;
|
|
463
|
+
const artifacts = this.snapshot.artifacts;
|
|
464
|
+
const index = artifacts.findIndex((artifact) => artifact.id === id);
|
|
465
|
+
const executionId = readString(msg.executionId);
|
|
466
|
+
const next = applyArtifactFrame(index === -1 ? void 0 : artifacts[index], event, {
|
|
467
|
+
id,
|
|
468
|
+
turnId,
|
|
469
|
+
...executionId ? { executionId } : {}
|
|
470
|
+
});
|
|
471
|
+
if (!next) return;
|
|
472
|
+
this.update({
|
|
473
|
+
artifacts: index === -1 ? [...artifacts, next] : [...artifacts.slice(0, index), next, ...artifacts.slice(index + 1)]
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
/** Settles the spoken clauses in place and ends the turn's claim on them. */
|
|
477
|
+
commitClauseCaptions() {
|
|
478
|
+
this.committedAssistantTurnId = this.pendingAssistant?.turnId ?? null;
|
|
479
|
+
const settled = this.pendingAssistant !== null;
|
|
480
|
+
this.pendingAssistant = null;
|
|
481
|
+
if (!settled) return;
|
|
482
|
+
const transcript = this.snapshot.transcript;
|
|
483
|
+
const last = transcript[transcript.length - 1];
|
|
484
|
+
if (last?.role !== "assistant" || last.partial !== true) return;
|
|
485
|
+
const { partial: _partial, ...committed } = last;
|
|
486
|
+
this.update({ transcript: [...transcript.slice(0, -1), committed] });
|
|
487
|
+
}
|
|
488
|
+
/** Extends the turn's spoken-so-far assistant entry, or opens one for a new turn. */
|
|
489
|
+
appendClauseCaption(text, turnId) {
|
|
490
|
+
const transcript = this.snapshot.transcript;
|
|
491
|
+
const last = transcript[transcript.length - 1];
|
|
492
|
+
if (this.pendingAssistant?.turnId === turnId && last?.role === "assistant") {
|
|
493
|
+
return {
|
|
494
|
+
interimTranscript: null,
|
|
495
|
+
transcript: [...transcript.slice(0, -1), { ...last, content: last.content + text }],
|
|
496
|
+
status: "speaking"
|
|
497
|
+
};
|
|
498
|
+
}
|
|
499
|
+
this.pendingAssistant = { turnId };
|
|
500
|
+
return {
|
|
501
|
+
interimTranscript: null,
|
|
502
|
+
transcript: [
|
|
503
|
+
...transcript,
|
|
504
|
+
{
|
|
505
|
+
role: "assistant",
|
|
506
|
+
content: text,
|
|
507
|
+
timestamp: Date.now(),
|
|
508
|
+
partial: true,
|
|
509
|
+
...turnId ? { turnId } : {}
|
|
510
|
+
}
|
|
511
|
+
],
|
|
512
|
+
status: "speaking"
|
|
513
|
+
};
|
|
343
514
|
}
|
|
344
515
|
clearPlayback() {
|
|
345
516
|
this.playbackRevision += 1;
|
|
@@ -374,32 +545,89 @@ var VoiceClient = class {
|
|
|
374
545
|
return;
|
|
375
546
|
}
|
|
376
547
|
switch (msg.type) {
|
|
377
|
-
case "session_config":
|
|
548
|
+
case "session_config": {
|
|
549
|
+
this.fullDuplex = msg.speechMode === "speech_to_speech";
|
|
550
|
+
if (this.fullDuplex) player.setContinuousMode?.(true);
|
|
378
551
|
if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
|
|
379
552
|
this.update({ interruptionMode: msg.interruptionMode });
|
|
380
553
|
}
|
|
554
|
+
const sessionId = readString(msg.sessionId);
|
|
555
|
+
const conversationId = readString(msg.conversationId);
|
|
556
|
+
if (sessionId && conversationId) {
|
|
557
|
+
const session = { sessionId, conversationId };
|
|
558
|
+
this.update({ session });
|
|
559
|
+
this.options.onSession?.(session);
|
|
560
|
+
}
|
|
561
|
+
break;
|
|
562
|
+
}
|
|
563
|
+
case "artifact":
|
|
564
|
+
this.applyArtifactMessage(msg);
|
|
565
|
+
break;
|
|
566
|
+
case "transcript_update": {
|
|
567
|
+
if (typeof msg.text !== "string" || typeof msg.turnId !== "string" || msg.role !== "user" && msg.role !== "assistant")
|
|
568
|
+
break;
|
|
569
|
+
if (msg.role === "assistant" && this.awaitingClear) break;
|
|
570
|
+
const entries = [...this.snapshot.transcript];
|
|
571
|
+
const index = entries.findIndex(
|
|
572
|
+
(entry2) => entry2.turnId === msg.turnId && entry2.role === msg.role
|
|
573
|
+
);
|
|
574
|
+
const entry = {
|
|
575
|
+
role: msg.role,
|
|
576
|
+
content: msg.text,
|
|
577
|
+
turnId: msg.turnId,
|
|
578
|
+
isFinal: msg.final === true,
|
|
579
|
+
timestamp: index >= 0 ? entries[index].timestamp : Date.now()
|
|
580
|
+
};
|
|
581
|
+
if (index >= 0) entries[index] = entry;
|
|
582
|
+
else entries.push(entry);
|
|
583
|
+
this.update({ transcript: entries, interimTranscript: null });
|
|
584
|
+
break;
|
|
585
|
+
}
|
|
586
|
+
case "delegation_started":
|
|
587
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("thinking");
|
|
588
|
+
break;
|
|
589
|
+
case "delegation_completed":
|
|
590
|
+
if (!this.hasPendingAudio && !this.awaitingClear) this.setStatus("listening");
|
|
381
591
|
break;
|
|
382
592
|
case "transcript_interim":
|
|
383
593
|
this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
|
|
384
594
|
break;
|
|
385
|
-
case "
|
|
595
|
+
case "transcript_partial": {
|
|
596
|
+
if (msg.role !== "assistant" || typeof msg.text !== "string" || !msg.text) break;
|
|
597
|
+
this.activeTurnId = readTurnId(msg) ?? this.activeTurnId;
|
|
598
|
+
if (this.awaitingClear || this.stoppedResponse) break;
|
|
599
|
+
this.responseEnded = false;
|
|
600
|
+
this.update(this.appendClauseCaption(msg.text, readTurnId(msg)));
|
|
601
|
+
break;
|
|
602
|
+
}
|
|
603
|
+
case "transcript_final": {
|
|
386
604
|
if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
|
|
387
|
-
|
|
605
|
+
const role = msg.role;
|
|
606
|
+
const text = msg.text;
|
|
607
|
+
if (role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
|
|
608
|
+
const turnId = readTurnId(msg);
|
|
609
|
+
if (turnId !== null) this.activeTurnId = turnId;
|
|
610
|
+
if (role === "assistant" && turnId !== null && turnId === this.committedAssistantTurnId)
|
|
611
|
+
break;
|
|
388
612
|
this.responseEnded = false;
|
|
613
|
+
const transcript = this.snapshot.transcript;
|
|
614
|
+
const last = transcript[transcript.length - 1];
|
|
615
|
+
const supersedes = role === "assistant" && this.pendingAssistant !== null && last?.role === "assistant" && (turnId === null || this.pendingAssistant.turnId === null || this.pendingAssistant.turnId === turnId);
|
|
616
|
+
this.pendingAssistant = null;
|
|
617
|
+
const entry = {
|
|
618
|
+
role,
|
|
619
|
+
content: text,
|
|
620
|
+
// INVARIANT: reuse the caption's timestamp so a keyed list does not remount the bubble.
|
|
621
|
+
timestamp: supersedes && last ? last.timestamp : Date.now(),
|
|
622
|
+
...turnId ? { turnId } : {}
|
|
623
|
+
};
|
|
389
624
|
this.update({
|
|
390
625
|
interimTranscript: null,
|
|
391
|
-
transcript: [
|
|
392
|
-
|
|
393
|
-
{
|
|
394
|
-
role: msg.role,
|
|
395
|
-
content: msg.text,
|
|
396
|
-
timestamp: Date.now(),
|
|
397
|
-
...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
|
|
398
|
-
}
|
|
399
|
-
],
|
|
400
|
-
status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
|
|
626
|
+
transcript: supersedes ? [...transcript.slice(0, -1), entry] : [...transcript, entry],
|
|
627
|
+
status: this.awaitingClear ? this.snapshot.status : role === "user" ? "thinking" : "speaking"
|
|
401
628
|
});
|
|
402
629
|
break;
|
|
630
|
+
}
|
|
403
631
|
case "audio_end": {
|
|
404
632
|
this.responseEnded = true;
|
|
405
633
|
if (this.stoppedResponse) {
|
|
@@ -416,10 +644,12 @@ var VoiceClient = class {
|
|
|
416
644
|
break;
|
|
417
645
|
}
|
|
418
646
|
case "audio_clear":
|
|
647
|
+
this.suppressedArtifactTurnId = this.activeTurnId ?? this.suppressedArtifactTurnId;
|
|
419
648
|
this.clearPlayback();
|
|
420
649
|
this.awaitingClear = false;
|
|
421
650
|
this.stoppedResponse = false;
|
|
422
651
|
this.responseEnded = true;
|
|
652
|
+
this.commitClauseCaptions();
|
|
423
653
|
this.setStatus("listening");
|
|
424
654
|
break;
|
|
425
655
|
case "metrics": {
|
|
@@ -432,8 +662,11 @@ var VoiceClient = class {
|
|
|
432
662
|
// @snake-case-ok: Existing voice wire contract.
|
|
433
663
|
firstAudioMs: number(msg.first_audio_ms),
|
|
434
664
|
// @snake-case-ok: Existing voice wire contract.
|
|
435
|
-
totalMs: number(msg.total_ms)
|
|
665
|
+
totalMs: number(msg.total_ms),
|
|
436
666
|
// @snake-case-ok: Existing voice wire contract.
|
|
667
|
+
firstSynthesisMs: number(msg.first_synthesis_ms),
|
|
668
|
+
// @snake-case-ok: Existing voice wire contract.
|
|
669
|
+
incremental: msg.incremental === true
|
|
437
670
|
}
|
|
438
671
|
});
|
|
439
672
|
break;
|
|
@@ -465,14 +698,14 @@ var VoiceClient = class {
|
|
|
465
698
|
const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
|
|
466
699
|
this.processor = processor;
|
|
467
700
|
processor.onaudioprocess = (event) => {
|
|
468
|
-
if (generation !== this.generation || this.snapshot.isMuted) return;
|
|
701
|
+
if (generation !== this.generation || this.snapshot.isMuted && !this.fullDuplex) return;
|
|
469
702
|
const input = event.inputBuffer.getChannelData(0);
|
|
470
703
|
let sum = 0;
|
|
471
704
|
for (const sample of input) sum += sample * sample;
|
|
472
|
-
this.update({ audioLevel: Math.sqrt(sum / input.length) });
|
|
705
|
+
this.update({ audioLevel: this.snapshot.isMuted ? 0 : Math.sqrt(sum / input.length) });
|
|
473
706
|
if (socket.readyState !== WebSocket.OPEN) return;
|
|
474
707
|
const pcm = new Int16Array(input.length);
|
|
475
|
-
if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
708
|
+
if (this.snapshot.isMuted || this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
|
|
476
709
|
socket.send(pcm.buffer);
|
|
477
710
|
return;
|
|
478
711
|
}
|
|
@@ -489,12 +722,13 @@ var VoiceClient = class {
|
|
|
489
722
|
|
|
490
723
|
// src/persona.ts
|
|
491
724
|
function createPersonaVoiceProvider(options) {
|
|
492
|
-
const client = new VoiceClient(options);
|
|
725
|
+
const client = new VoiceClient({ ...options, fullDuplex: false });
|
|
493
726
|
const statusCallbacks = /* @__PURE__ */ new Set();
|
|
494
727
|
const errorCallbacks = /* @__PURE__ */ new Set();
|
|
495
728
|
const transcriptCallbacks = /* @__PURE__ */ new Set();
|
|
496
729
|
const metricsCallbacks = /* @__PURE__ */ new Set();
|
|
497
730
|
const levelCallbacks = /* @__PURE__ */ new Set();
|
|
731
|
+
const artifactCallbacks = /* @__PURE__ */ new Set();
|
|
498
732
|
let previous = client.getSnapshot();
|
|
499
733
|
let unsubscribe = null;
|
|
500
734
|
const onSnapshot = () => {
|
|
@@ -518,12 +752,18 @@ function createPersonaVoiceProvider(options) {
|
|
|
518
752
|
for (const callback of transcriptCallbacks)
|
|
519
753
|
callback("user", snapshot.interimTranscript, false);
|
|
520
754
|
}
|
|
521
|
-
|
|
755
|
+
snapshot.artifacts.forEach((artifact, index) => {
|
|
756
|
+
if (artifact === before.artifacts[index]) return;
|
|
757
|
+
for (const callback of artifactCallbacks) callback(artifact);
|
|
758
|
+
});
|
|
759
|
+
snapshot.transcript.forEach((entry, index) => {
|
|
760
|
+
if (entry === before.transcript[index]) return;
|
|
761
|
+
const isFinal = entry.partial !== true && entry.isFinal !== false;
|
|
522
762
|
for (const callback of transcriptCallbacks) {
|
|
523
|
-
if (entry.turnId) callback(entry.role, entry.content,
|
|
524
|
-
else callback(entry.role, entry.content,
|
|
763
|
+
if (entry.turnId) callback(entry.role, entry.content, isFinal, { turnId: entry.turnId });
|
|
764
|
+
else callback(entry.role, entry.content, isFinal);
|
|
525
765
|
}
|
|
526
|
-
}
|
|
766
|
+
});
|
|
527
767
|
};
|
|
528
768
|
const subscribe = () => {
|
|
529
769
|
if (unsubscribe) return;
|
|
@@ -544,6 +784,7 @@ function createPersonaVoiceProvider(options) {
|
|
|
544
784
|
transcriptCallbacks.clear();
|
|
545
785
|
metricsCallbacks.clear();
|
|
546
786
|
levelCallbacks.clear();
|
|
787
|
+
artifactCallbacks.clear();
|
|
547
788
|
},
|
|
548
789
|
startListening: async () => {
|
|
549
790
|
subscribe();
|
|
@@ -570,6 +811,9 @@ function createPersonaVoiceProvider(options) {
|
|
|
570
811
|
onLevel: (callback) => {
|
|
571
812
|
levelCallbacks.add(callback);
|
|
572
813
|
},
|
|
814
|
+
onArtifact: (callback) => {
|
|
815
|
+
artifactCallbacks.add(callback);
|
|
816
|
+
},
|
|
573
817
|
getInterruptionMode: () => client.getSnapshot().interruptionMode,
|
|
574
818
|
isBargeInActive: () => !["idle", "error"].includes(client.getSnapshot().status),
|
|
575
819
|
deactivateBargeIn: async () => {
|
|
@@ -578,3 +822,48 @@ function createPersonaVoiceProvider(options) {
|
|
|
578
822
|
stopPlayback: client.stopPlayback
|
|
579
823
|
};
|
|
580
824
|
}
|
|
825
|
+
function toManualUpsert(artifact) {
|
|
826
|
+
const title = artifact.title ?? artifact.file?.path;
|
|
827
|
+
const transcript = artifact.status === "complete";
|
|
828
|
+
if (artifact.artifactType === "component") {
|
|
829
|
+
if (!artifact.component) return null;
|
|
830
|
+
return {
|
|
831
|
+
id: artifact.id,
|
|
832
|
+
artifactType: "component",
|
|
833
|
+
component: artifact.component,
|
|
834
|
+
...title ? { title } : {},
|
|
835
|
+
...artifact.props ? { props: artifact.props } : {},
|
|
836
|
+
transcript
|
|
837
|
+
};
|
|
838
|
+
}
|
|
839
|
+
return {
|
|
840
|
+
id: artifact.id,
|
|
841
|
+
artifactType: "markdown",
|
|
842
|
+
content: artifact.content,
|
|
843
|
+
...title ? { title } : {},
|
|
844
|
+
...artifact.file ? {
|
|
845
|
+
file: {
|
|
846
|
+
path: artifact.file.path,
|
|
847
|
+
mimeType: artifact.file.mimeType,
|
|
848
|
+
...artifact.file.language ? { language: artifact.file.language } : {}
|
|
849
|
+
}
|
|
850
|
+
} : {},
|
|
851
|
+
transcript
|
|
852
|
+
};
|
|
853
|
+
}
|
|
854
|
+
function bindVoiceArtifactsToPersona(provider, widget, options) {
|
|
855
|
+
let released = false;
|
|
856
|
+
let shown = false;
|
|
857
|
+
provider.onArtifact((artifact) => {
|
|
858
|
+
if (released) return;
|
|
859
|
+
const manual = toManualUpsert(artifact);
|
|
860
|
+
if (!manual) return;
|
|
861
|
+
widget.upsertArtifact(manual);
|
|
862
|
+
if (shown || options?.showOnFirst === false) return;
|
|
863
|
+
shown = true;
|
|
864
|
+
widget.showArtifacts();
|
|
865
|
+
});
|
|
866
|
+
return () => {
|
|
867
|
+
released = true;
|
|
868
|
+
};
|
|
869
|
+
}
|
package/dist/persona.d.cts
CHANGED
|
@@ -1,7 +1,26 @@
|
|
|
1
|
-
import { VoiceProvider } from '@runtypelabs/persona';
|
|
2
|
-
import { V as VoiceClientOptions } from './types-
|
|
1
|
+
import { AgentWidgetInitHandle, VoiceProvider } from '@runtypelabs/persona';
|
|
2
|
+
import { b as VoiceArtifact, V as VoiceClientOptions } from './types-Dkh4LxEA.cjs';
|
|
3
3
|
|
|
4
|
+
type ArtifactCallback = (artifact: VoiceArtifact) => void;
|
|
5
|
+
/**
|
|
6
|
+
* Persona 4.22 has no artifact hook on `VoiceProvider`, so the adapter adds its
|
|
7
|
+
* own `onArtifact`. Delivery to the widget is the host's, through the widget
|
|
8
|
+
* handle's `upsertArtifact` (see `bindVoiceArtifactsToPersona`).
|
|
9
|
+
*/
|
|
10
|
+
type RuntypePersonaVoiceProvider = VoiceProvider & {
|
|
11
|
+
onArtifact(callback: ArtifactCallback): void;
|
|
12
|
+
};
|
|
13
|
+
/** The part of Persona's widget handle an artifact binding needs. */
|
|
14
|
+
type PersonaArtifactHost = Pick<AgentWidgetInitHandle, 'upsertArtifact' | 'showArtifacts'>;
|
|
4
15
|
/** Adapt the shared client to Persona 4.22's custom voice provider API. */
|
|
5
|
-
declare function createPersonaVoiceProvider(options: VoiceClientOptions):
|
|
16
|
+
declare function createPersonaVoiceProvider(options: VoiceClientOptions): RuntypePersonaVoiceProvider;
|
|
17
|
+
/**
|
|
18
|
+
* Streams a provider's artifacts into an already-initialized Persona widget. The
|
|
19
|
+
* artifact id is the upsert id, so each frame updates the same record in place,
|
|
20
|
+
* and the pane opens once on the first artifact. Returns a release function.
|
|
21
|
+
*/
|
|
22
|
+
declare function bindVoiceArtifactsToPersona(provider: RuntypePersonaVoiceProvider, widget: PersonaArtifactHost, options?: {
|
|
23
|
+
showOnFirst?: boolean;
|
|
24
|
+
}): () => void;
|
|
6
25
|
|
|
7
|
-
export { createPersonaVoiceProvider };
|
|
26
|
+
export { type PersonaArtifactHost, type RuntypePersonaVoiceProvider, bindVoiceArtifactsToPersona, createPersonaVoiceProvider };
|
package/dist/persona.d.ts
CHANGED
|
@@ -1,7 +1,26 @@
|
|
|
1
|
-
import { VoiceProvider } from '@runtypelabs/persona';
|
|
2
|
-
import { V as VoiceClientOptions } from './types-
|
|
1
|
+
import { AgentWidgetInitHandle, VoiceProvider } from '@runtypelabs/persona';
|
|
2
|
+
import { b as VoiceArtifact, V as VoiceClientOptions } from './types-Dkh4LxEA.js';
|
|
3
3
|
|
|
4
|
+
type ArtifactCallback = (artifact: VoiceArtifact) => void;
|
|
5
|
+
/**
|
|
6
|
+
* Persona 4.22 has no artifact hook on `VoiceProvider`, so the adapter adds its
|
|
7
|
+
* own `onArtifact`. Delivery to the widget is the host's, through the widget
|
|
8
|
+
* handle's `upsertArtifact` (see `bindVoiceArtifactsToPersona`).
|
|
9
|
+
*/
|
|
10
|
+
type RuntypePersonaVoiceProvider = VoiceProvider & {
|
|
11
|
+
onArtifact(callback: ArtifactCallback): void;
|
|
12
|
+
};
|
|
13
|
+
/** The part of Persona's widget handle an artifact binding needs. */
|
|
14
|
+
type PersonaArtifactHost = Pick<AgentWidgetInitHandle, 'upsertArtifact' | 'showArtifacts'>;
|
|
4
15
|
/** Adapt the shared client to Persona 4.22's custom voice provider API. */
|
|
5
|
-
declare function createPersonaVoiceProvider(options: VoiceClientOptions):
|
|
16
|
+
declare function createPersonaVoiceProvider(options: VoiceClientOptions): RuntypePersonaVoiceProvider;
|
|
17
|
+
/**
|
|
18
|
+
* Streams a provider's artifacts into an already-initialized Persona widget. The
|
|
19
|
+
* artifact id is the upsert id, so each frame updates the same record in place,
|
|
20
|
+
* and the pane opens once on the first artifact. Returns a release function.
|
|
21
|
+
*/
|
|
22
|
+
declare function bindVoiceArtifactsToPersona(provider: RuntypePersonaVoiceProvider, widget: PersonaArtifactHost, options?: {
|
|
23
|
+
showOnFirst?: boolean;
|
|
24
|
+
}): () => void;
|
|
6
25
|
|
|
7
|
-
export { createPersonaVoiceProvider };
|
|
26
|
+
export { type PersonaArtifactHost, type RuntypePersonaVoiceProvider, bindVoiceArtifactsToPersona, createPersonaVoiceProvider };
|