@runtypelabs/persona 4.26.1 β 4.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/approval-ui.d.cts +18 -0
- package/dist/approval-ui.d.ts +18 -0
- package/dist/artifacts-ui.d.cts +18 -0
- package/dist/artifacts-ui.d.ts +18 -0
- package/dist/chunk-AALCT54S.js +1 -0
- package/dist/{client-history-GOZWBDA5.js β client-history-LV3KHEFA.js} +1 -1
- package/dist/client-history.js +1 -1
- package/dist/client-stream-N22JHTLE.js +4 -0
- package/dist/client-stream.js +3 -3
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions.d.cts +18 -0
- package/dist/context-mentions.d.ts +18 -0
- package/dist/event-stream-view.d.cts +18 -0
- package/dist/event-stream-view.d.ts +18 -0
- package/dist/forms-ui.d.cts +26 -0
- package/dist/forms-ui.d.ts +26 -0
- package/dist/index.cjs +57 -57
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +34 -0
- package/dist/index.d.ts +34 -0
- package/dist/index.global.js +1 -1
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +58 -58
- package/dist/index.js.map +1 -1
- package/dist/session-reconnect.d.cts +18 -0
- package/dist/session-reconnect.d.ts +18 -0
- package/dist/smart-dom-reader.d.cts +18 -0
- package/dist/smart-dom-reader.d.ts +18 -0
- package/dist/theme-editor-preview.cjs +66 -66
- package/dist/theme-editor-preview.d.cts +18 -0
- package/dist/theme-editor-preview.d.ts +18 -0
- package/dist/theme-editor-preview.js +31 -31
- package/dist/theme-editor.d.cts +18 -0
- package/dist/theme-editor.d.ts +18 -0
- package/dist/voice-runtime-LVUMPUDS.js +17 -0
- package/dist/voice-runtime.cjs +7 -7
- package/dist/voice-runtime.d.cts +33 -0
- package/dist/voice-runtime.d.ts +33 -0
- package/dist/voice-runtime.js +7 -7
- package/package.json +1 -1
- package/src/client-stream.ts +10 -3
- package/src/client.interrupt.test.ts +18 -0
- package/src/client.test.ts +20 -0
- package/src/client.ts +7 -0
- package/src/session.ts +16 -3
- package/src/session.voice.test.ts +199 -3
- package/src/types.ts +24 -0
- package/src/ui.composer-defer-one.test.ts +17 -0
- package/src/ui.ts +49 -12
- package/src/voice/__fixtures__/voice-wire/server/session_end-provider-error.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/session_end-unknown-reason.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/session_end.json +5 -0
- package/src/voice/runtype-voice-provider.ts +205 -21
- package/src/voice/voice-clause-buffer.test.ts +143 -0
- package/src/voice/voice-clause-buffer.ts +187 -0
- package/src/voice/voice-delegation.ts +111 -13
- package/src/voice/voice.test.ts +433 -2
- package/dist/chunk-GKPCIQPB.js +0 -1
- package/dist/client-stream-42FOI3EC.js +0 -4
- package/dist/voice-runtime-XLU6YK2N.js +0 -17
|
@@ -64,7 +64,25 @@
|
|
|
64
64
|
// agent as conversation; a user bubble becomes conversation only when it is
|
|
65
65
|
// submitted. With `context` negotiated, one `context{text}` frame (chat history
|
|
66
66
|
// as of call start plus the host's `callContext`) is held until the visitor's
|
|
67
|
-
// first final transcript or first delegation
|
|
67
|
+
// first final transcript or first delegation; a delegation's frames (deltas
|
|
68
|
+
// and result) wait for it (up to 2 s, then it is dropped).
|
|
69
|
+
//
|
|
70
|
+
// Streaming (`delegation_stream` and `delegation_read_back` both negotiated,
|
|
71
|
+
// with client delegation; the read-back tags are what keep a streamed answer
|
|
72
|
+
// spoken as several utterances folded): while a
|
|
73
|
+
// delegated turn renders, its assistant answer goes out as ordered
|
|
74
|
+
// `delegation_delta{delegationId, text}` pieces (clause-sized, see
|
|
75
|
+
// voice-clause-buffer.ts; approval cards, tool bubbles and captions never
|
|
76
|
+
// stream), all before the delegation's first `delegation_update` or
|
|
77
|
+
// `delegation_result`. That frame then carries `streamedChars`: how many
|
|
78
|
+
// leading characters of its `text` the deltas already sent (an approval
|
|
79
|
+
// script appended after the answer is the unstreamed rest). Absent or 0: none.
|
|
80
|
+
// The server reports each spoken chunk with `delegation_progress{delegationId,
|
|
81
|
+
// text}`; the first one of a spoken phase folds the read-back as
|
|
82
|
+
// `delegation_completed{speak:true}` does (status stays "processing" until
|
|
83
|
+
// completed), and that phase's completion doesn't fold another utterance.
|
|
84
|
+
// Delegated sends carry `voice: { spoken: true }` on the chat request, so the
|
|
85
|
+
// agent answers in speakable sentences.
|
|
68
86
|
|
|
69
87
|
import type {
|
|
70
88
|
VoiceProvider,
|
|
@@ -99,6 +117,8 @@ const LEVEL_RMS_SCALE = 4;
|
|
|
99
117
|
const RIFF_MAGIC = 0x52494646; // "RIFF"
|
|
100
118
|
/** `prewarm()` warms at most once per this window per provider instance. */
|
|
101
119
|
const PREWARM_THROTTLE_MS = 30_000;
|
|
120
|
+
/** How long a call start waits for a suspended capture context to run. */
|
|
121
|
+
const AUDIO_RESUME_TIMEOUT_MS = 1_000;
|
|
102
122
|
/** How long `startListening()` waits on an attach still handshaking. */
|
|
103
123
|
const ATTACH_HANDSHAKE_TIMEOUT_MS = 5_000;
|
|
104
124
|
/** Attached-socket idle window: server default and clamp range. */
|
|
@@ -142,6 +162,17 @@ const VOICE_PROTOCOL = "runtype-browser-v1";
|
|
|
142
162
|
/** `approvalTimeoutMs` cap: below core's delegation deadline (600 s, restarted by each update). */
|
|
143
163
|
const APPROVAL_TIMEOUT_MAX_MS = 540_000;
|
|
144
164
|
const DISCLOSURE_TEXT = "You're talking to an AI assistant. Voice is processed by OpenAI.";
|
|
165
|
+
/** Status-line text per `session_end.reason` (Amendment 6); unlisted reasons show nothing. */
|
|
166
|
+
const SESSION_END_TEXT = new Map([
|
|
167
|
+
["idle_timeout", "Voice call ended after a quiet period."],
|
|
168
|
+
["max_duration", "This voice call reached its time limit."],
|
|
169
|
+
["provider_ended", "The voice session ended."],
|
|
170
|
+
["quota", "Voice is unavailable right now."],
|
|
171
|
+
["auth_expired", "Voice session expired. Tap the mic to reconnect."],
|
|
172
|
+
]);
|
|
173
|
+
/** Reasons that get one automatic reconnect. */
|
|
174
|
+
const RECONNECT_REASONS = ["provider_error", "server_restart"];
|
|
175
|
+
const CONNECTION_LOST = "Voice connection lost.";
|
|
145
176
|
/** Call-start context frame: total cap, host share, history window, per-message cap. */
|
|
146
177
|
const CONTEXT_MAX_CHARS = 8000;
|
|
147
178
|
const CONTEXT_HOST_MAX_CHARS = 4000;
|
|
@@ -271,6 +302,14 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
271
302
|
#cancelledDelegations = new Set<string>();
|
|
272
303
|
// session_config.callId: the server's id for this call, for logs.
|
|
273
304
|
#callId: string | undefined;
|
|
305
|
+
// Amendment 6: why the server ended this call (its last frame), if it said.
|
|
306
|
+
#sessionEnd: { reason: string; message?: string } | null = null;
|
|
307
|
+
// Status-line text for the UI, read once (`takeNotice`).
|
|
308
|
+
#notice: string | null = null;
|
|
309
|
+
// One automatic reconnect per call the visitor started: pending, and spent.
|
|
310
|
+
#reconnectTimer: ReturnType<typeof setTimeout> | undefined;
|
|
311
|
+
#reconnected = false;
|
|
312
|
+
#autoReconnect = false;
|
|
274
313
|
// Call-start context, built at session_config and held until the visitor's
|
|
275
314
|
// first final transcript (sent earlier, the voice model tends to answer it).
|
|
276
315
|
#contextSent = false;
|
|
@@ -278,8 +317,13 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
278
317
|
// The released context's send; settles once it went out (or never will).
|
|
279
318
|
#contextSend: { done: Promise<void>; drop: () => void } | null = null;
|
|
280
319
|
#delegations: Promise<void> = Promise.resolve();
|
|
281
|
-
// Delegations answered ok
|
|
320
|
+
// Delegations answered ok (or that streamed part of their answer): their
|
|
321
|
+
// spoken read-back is folded.
|
|
282
322
|
#answered = new Set<string>();
|
|
323
|
+
// Delegations whose current spoken phase already folded on `delegation_progress`.
|
|
324
|
+
#progressed = new Set<string>();
|
|
325
|
+
// The delegation whose read-back `#foldReadback` is waiting for.
|
|
326
|
+
#foldArmedBy: string | null = null;
|
|
283
327
|
#foldReadback = false;
|
|
284
328
|
#assistantTurns = new Set<string>();
|
|
285
329
|
#foldedTurns = new Set<string>();
|
|
@@ -452,7 +496,7 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
452
496
|
#voiceSocketUrl(host: string, agentId: string, attach?: URLSearchParams): string {
|
|
453
497
|
const capabilities = ["partial_transcript", "context"];
|
|
454
498
|
if (this.bridge && this.config?.clientDelegation !== false)
|
|
455
|
-
capabilities.push("client_delegation", "delegation_update", "delegation_read_back");
|
|
499
|
+
capabilities.push("client_delegation", "delegation_update", "delegation_read_back", "delegation_stream");
|
|
456
500
|
if (attach) capabilities.unshift("attach");
|
|
457
501
|
const params = new URLSearchParams({
|
|
458
502
|
voiceProtocol: VOICE_PROTOCOL,
|
|
@@ -467,6 +511,13 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
467
511
|
/** Start the call: acquire mic, open the WS, stream PCM until hang-up. */
|
|
468
512
|
async startListening(): Promise<void> {
|
|
469
513
|
if (this.#callLive) return; // idempotent: a call is already live
|
|
514
|
+
// A call the visitor starts gets a fresh reconnect; the automatic one doesn't.
|
|
515
|
+
const auto = this.#autoReconnect;
|
|
516
|
+
if (!auto) {
|
|
517
|
+
this.#reconnected = false;
|
|
518
|
+
this.#notice = null;
|
|
519
|
+
}
|
|
520
|
+
this.#autoReconnect = false;
|
|
470
521
|
|
|
471
522
|
const agentId = this.config?.agentId;
|
|
472
523
|
const token = this.config?.clientToken;
|
|
@@ -499,10 +550,22 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
499
550
|
const captureContext: AudioContext = new AudioCtx({
|
|
500
551
|
sampleRate: CAPTURE_SAMPLE_RATE,
|
|
501
552
|
});
|
|
553
|
+
this.#captureContext = captureContext;
|
|
502
554
|
if (captureContext.state === "suspended") {
|
|
503
|
-
|
|
555
|
+
// Outside a click (the automatic reconnect, or a call restored after a
|
|
556
|
+
// page navigation) the autoplay policy can leave audio suspended and
|
|
557
|
+
// `resume()` pending forever: fail fast rather than hang or open a dead call.
|
|
558
|
+
await Promise.race([
|
|
559
|
+
captureContext.resume().catch(() => {}),
|
|
560
|
+
new Promise((resolve) => setTimeout(resolve, auto ? 500 : AUDIO_RESUME_TIMEOUT_MS)),
|
|
561
|
+
]);
|
|
562
|
+
if (generation !== this.#callGeneration) return; // hung up meanwhile
|
|
563
|
+
if ((captureContext.state as string) !== "running") {
|
|
564
|
+
if (auto) throw new Error((this.#notice = `${CONNECTION_LOST} Tap the mic to reconnect.`));
|
|
565
|
+
// A restored call: the widget asks for a tap to resume.
|
|
566
|
+
throw new Error("Audio blocked until a tap");
|
|
567
|
+
}
|
|
504
568
|
}
|
|
505
|
-
this.#captureContext = captureContext;
|
|
506
569
|
|
|
507
570
|
const engine = this.config?.createPlaybackEngine
|
|
508
571
|
? await this.config.createPlaybackEngine()
|
|
@@ -560,7 +623,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
560
623
|
ws.onmessage = (event) => this.#handleMessage(event, generation);
|
|
561
624
|
|
|
562
625
|
ws.onerror = () => {
|
|
563
|
-
|
|
626
|
+
// After `session_end`, the close that follows reports the server's reason.
|
|
627
|
+
if (generation !== this.#callGeneration || this.#sessionEnd) return;
|
|
564
628
|
this.#emitError(new Error("Voice connection failed"));
|
|
565
629
|
this.#emitStatus("error");
|
|
566
630
|
this.#cleanup();
|
|
@@ -572,17 +636,68 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
572
636
|
return;
|
|
573
637
|
}
|
|
574
638
|
if (generation !== this.#callGeneration) return;
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
639
|
+
// Why the server ended it: its `session_end`, or the attach idle close.
|
|
640
|
+
const end = this.#sessionEnd ?? (evt.code === 4408 ? { reason: "idle_timeout" } : null);
|
|
641
|
+
if (!end) {
|
|
642
|
+
// No reason given (a deploy, a dropped connection): as before.
|
|
643
|
+
if (evt.code !== 1000) {
|
|
644
|
+
const codeMsg = evt.code ? ` (code ${evt.code})` : "";
|
|
645
|
+
this.#emitError(new Error(`Voice connection closed${codeMsg}`));
|
|
646
|
+
this.#emitStatus("error");
|
|
647
|
+
} else {
|
|
648
|
+
this.#emitStatus("idle");
|
|
649
|
+
}
|
|
650
|
+
this.#cleanup();
|
|
651
|
+
return;
|
|
581
652
|
}
|
|
582
|
-
this.#
|
|
653
|
+
this.#endCall(end.reason, end.message);
|
|
583
654
|
};
|
|
584
655
|
}
|
|
585
656
|
|
|
657
|
+
/**
|
|
658
|
+
* The server ended the call with a reason (`session_end`, Amendment 6): show
|
|
659
|
+
* its text, and reconnect once after a provider error or a restart. Unknown
|
|
660
|
+
* reasons are an ordinary end (`ended_by_server`).
|
|
661
|
+
*/
|
|
662
|
+
#endCall(reason: string, message?: string): void {
|
|
663
|
+
const retry = RECONNECT_REASONS.includes(reason) && !this.#reconnected;
|
|
664
|
+
this.#cleanup();
|
|
665
|
+
if (retry) {
|
|
666
|
+
// Keep the call "live" while it waits, so the mic button hangs up (and cancels).
|
|
667
|
+
this.#reconnected = true;
|
|
668
|
+
this.#callLive = true;
|
|
669
|
+
this.#notice = reason === "provider_error" ? `${CONNECTION_LOST} Reconnectingβ¦` : "Reconnectingβ¦";
|
|
670
|
+
// A hang-up meanwhile runs #cleanup, which clears this timer.
|
|
671
|
+
this.#reconnectTimer = setTimeout(
|
|
672
|
+
() => {
|
|
673
|
+
this.#callLive = false;
|
|
674
|
+
this.#autoReconnect = true;
|
|
675
|
+
void this.startListening().catch(() => {});
|
|
676
|
+
},
|
|
677
|
+
1000 + Math.random() * 1000,
|
|
678
|
+
);
|
|
679
|
+
this.#emitStatus("idle");
|
|
680
|
+
return;
|
|
681
|
+
}
|
|
682
|
+
if (RECONNECT_REASONS.includes(reason)) {
|
|
683
|
+
// The reconnect already ran: this end is the second failure.
|
|
684
|
+
this.#notice = CONNECTION_LOST;
|
|
685
|
+
this.#emitError(new Error(CONNECTION_LOST));
|
|
686
|
+
return this.#emitStatus("error");
|
|
687
|
+
}
|
|
688
|
+
// `error`: the preceding `error` frame already reported it.
|
|
689
|
+
if (reason === "error") return this.#emitStatus("error");
|
|
690
|
+
this.#notice = (reason === "quota" && message) || SESSION_END_TEXT.get(reason) || null;
|
|
691
|
+
this.#emitStatus("idle");
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
/** Status-line text for the call (why it ended, or that it is reconnecting), once. */
|
|
695
|
+
takeNotice(): string | null {
|
|
696
|
+
const notice = this.#notice;
|
|
697
|
+
this.#notice = null;
|
|
698
|
+
return notice;
|
|
699
|
+
}
|
|
700
|
+
|
|
586
701
|
/** The AI-disclosure notice for a live speech-to-speech call (`disclosureText`; `false` hides it). */
|
|
587
702
|
getDisclosure(): string | null {
|
|
588
703
|
const text = this.config?.disclosureText;
|
|
@@ -757,6 +872,13 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
757
872
|
const tag = msg.delegationId;
|
|
758
873
|
if (typeof tag === "string" && this.#answered.has(tag) && !this.#assistantTurns.has(turnId)) {
|
|
759
874
|
this.#foldedTurns.add(turnId);
|
|
875
|
+
this.#foldReadback = false;
|
|
876
|
+
} else if (this.#foldReadback && !this.#assistantTurns.has(turnId)) {
|
|
877
|
+
// Armed by `delegation_progress`: the server may leave the
|
|
878
|
+
// read-back untagged (it continued a filler it cut), so the
|
|
879
|
+
// first new utterance after it is the read-back.
|
|
880
|
+
this.#foldedTurns.add(turnId);
|
|
881
|
+
this.#foldReadback = false;
|
|
760
882
|
}
|
|
761
883
|
} else if (this.#foldReadback && !this.#assistantTurns.has(turnId)) {
|
|
762
884
|
this.#foldedTurns.add(turnId);
|
|
@@ -799,8 +921,13 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
799
921
|
} else if (msg.speak !== false && this.#answered.has(delegationId)) {
|
|
800
922
|
// Every spoken phase (the approval ask, the late result) of an answer
|
|
801
923
|
// the chat shows is read back. A refusal of one that never started
|
|
802
|
-
// here has no chat answer: it renders. `final` may never come.
|
|
803
|
-
|
|
924
|
+
// here has no chat answer: it renders. `final` may never come. A
|
|
925
|
+
// phase whose first `delegation_progress` already armed the fold
|
|
926
|
+
// ends here without arming it again. With `delegation_read_back`
|
|
927
|
+
// the tag marks the read-back, so completion arms nothing (only
|
|
928
|
+
// progress does, for a read-back the server left untagged).
|
|
929
|
+
if (!this.#progressed.delete(delegationId) && !this.#capabilities.has("delegation_read_back"))
|
|
930
|
+
this.#armFold(delegationId);
|
|
804
931
|
}
|
|
805
932
|
this.#delegating = msg.type === "delegation_started";
|
|
806
933
|
if (!this.#isSpeaking && !this.#isCancelling()) {
|
|
@@ -809,6 +936,23 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
809
936
|
break;
|
|
810
937
|
}
|
|
811
938
|
|
|
939
|
+
case "delegation_progress": {
|
|
940
|
+
// The server handed a spoken chunk of this delegation to the voice
|
|
941
|
+
// model: its read-back is starting. The first one of a phase folds it,
|
|
942
|
+
// as `delegation_completed{speak:true}` does; later ones (and that
|
|
943
|
+
// phase's completion) don't fold another utterance. Still delegating.
|
|
944
|
+
const delegationId = String(msg.delegationId);
|
|
945
|
+
if (
|
|
946
|
+
this.#answered.has(delegationId) &&
|
|
947
|
+
!this.#progressed.has(delegationId) &&
|
|
948
|
+
!this.#cancelledDelegations.has(delegationId)
|
|
949
|
+
) {
|
|
950
|
+
this.#progressed.add(delegationId);
|
|
951
|
+
this.#armFold(delegationId);
|
|
952
|
+
}
|
|
953
|
+
break;
|
|
954
|
+
}
|
|
955
|
+
|
|
812
956
|
case "delegation_cancelled":
|
|
813
957
|
// The server gave up on it: no more frames for it, and its approval
|
|
814
958
|
// timers stop. The chat turn still renders. Past its deadline (or for
|
|
@@ -872,6 +1016,14 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
872
1016
|
this.#emitError(new Error(msg.error || "Voice error"));
|
|
873
1017
|
this.#emitStatus("error");
|
|
874
1018
|
break;
|
|
1019
|
+
|
|
1020
|
+
case "session_end":
|
|
1021
|
+
// The server's last frame before it closes: the close acts on it.
|
|
1022
|
+
this.#sessionEnd = {
|
|
1023
|
+
reason: String(msg.reason),
|
|
1024
|
+
...(typeof msg.message === "string" && { message: msg.message }),
|
|
1025
|
+
};
|
|
1026
|
+
break;
|
|
875
1027
|
}
|
|
876
1028
|
}
|
|
877
1029
|
|
|
@@ -921,9 +1073,19 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
921
1073
|
if (timedOut) send.drop();
|
|
922
1074
|
}
|
|
923
1075
|
|
|
1076
|
+
/** The next new assistant utterance is this delegation's read-back: fold it. */
|
|
1077
|
+
#armFold(delegationId: string): void {
|
|
1078
|
+
this.#foldReadback = true;
|
|
1079
|
+
this.#foldArmedBy = delegationId;
|
|
1080
|
+
}
|
|
1081
|
+
|
|
924
1082
|
/** No more frames for this delegation; its chat answer goes back to browser TTS. */
|
|
925
1083
|
#dropDelegation(delegationId: string, expired = false): void {
|
|
926
1084
|
this.#cancelledDelegations.add(delegationId);
|
|
1085
|
+
// A fold it armed whose read-back never started would hide the next
|
|
1086
|
+
// unrelated utterance: disarm it.
|
|
1087
|
+
if (this.#foldArmedBy === delegationId) this.#foldReadback = false;
|
|
1088
|
+
this.#progressed.delete(delegationId);
|
|
927
1089
|
this.bridge?.dropDelegation?.(delegationId, expired);
|
|
928
1090
|
this.#followUps.get(delegationId)?.abort();
|
|
929
1091
|
}
|
|
@@ -932,28 +1094,46 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
932
1094
|
#runDelegation(delegationId: string, request: VoiceDelegationRequest, generation: number): void {
|
|
933
1095
|
const bridge = this.bridge!;
|
|
934
1096
|
/** Send a frame for this delegation, unless the call or the delegation ended. */
|
|
935
|
-
const send = (type
|
|
1097
|
+
const send = ({ type, ...fields }: Record<string, unknown> & { type: string }) => {
|
|
936
1098
|
const ws = this.#ws;
|
|
937
1099
|
if (generation !== this.#callGeneration || ws?.readyState !== WebSocket.OPEN) return false;
|
|
938
1100
|
if (this.#cancelledDelegations.has(delegationId)) return false;
|
|
939
|
-
ws.send(JSON.stringify({ type, delegationId,
|
|
1101
|
+
ws.send(JSON.stringify({ type, delegationId, ...fields }));
|
|
940
1102
|
return true;
|
|
941
1103
|
};
|
|
1104
|
+
const answer = (type: "delegation_update" | "delegation_result", status: string, text: string, streamedChars = 0) =>
|
|
1105
|
+
send({ type, status, text, ...(streamedChars > 0 && { streamedChars }) });
|
|
942
1106
|
this.#delegations = this.#delegations.then(async () => {
|
|
943
1107
|
// Cancelled while queued behind another turn: never start it.
|
|
944
1108
|
if (generation !== this.#callGeneration || this.#cancelledDelegations.has(delegationId)) return;
|
|
1109
|
+
// Streaming (`delegation_stream`): the answer goes out as
|
|
1110
|
+
// `delegation_delta` pieces while the chat renders it, after the call
|
|
1111
|
+
// context, and all of them before the result.
|
|
1112
|
+
let deltas: Promise<void> | null = null;
|
|
1113
|
+
// Only with read-back tags: a streamed answer can be spoken as several
|
|
1114
|
+
// utterances, and the tags are what keep all of them folded.
|
|
1115
|
+
const streaming = this.#capabilities.has("delegation_stream") && this.#capabilities.has("delegation_read_back");
|
|
1116
|
+
const stream = (text: string) => {
|
|
1117
|
+
deltas = (deltas ?? this.#awaitContextSend()).then(() => {
|
|
1118
|
+
// The chat shows what was streamed: its read-back is folded.
|
|
1119
|
+
if (send({ type: "delegation_delta", text })) this.#answered.add(delegationId);
|
|
1120
|
+
});
|
|
1121
|
+
};
|
|
945
1122
|
const result: VoiceDelegationResult = await bridge
|
|
946
|
-
.runDelegatedTurn(request)
|
|
1123
|
+
.runDelegatedTurn(streaming ? { ...request, stream } : request)
|
|
947
1124
|
.catch(() => ({ status: "failed", text: "" }));
|
|
948
1125
|
await this.#awaitContextSend();
|
|
1126
|
+
await deltas;
|
|
1127
|
+
const streamedChars = streaming ? (result.streamedChars ?? 0) : 0;
|
|
949
1128
|
// Parked on an approval: ask now (non-terminal), answer once the visitor
|
|
950
1129
|
// decides. A server that can't take an update gets the ask as the result.
|
|
951
1130
|
const parked = !!result.followUp;
|
|
952
1131
|
const update = parked && this.#capabilities.has("delegation_update");
|
|
953
|
-
const sent =
|
|
1132
|
+
const sent = answer(
|
|
954
1133
|
update ? "delegation_update" : "delegation_result",
|
|
955
1134
|
parked && !update ? "completed" : result.status,
|
|
956
1135
|
result.text,
|
|
1136
|
+
streamedChars,
|
|
957
1137
|
);
|
|
958
1138
|
if (!sent) return;
|
|
959
1139
|
if (result.status !== "failed" && result.inChat !== false) this.#answered.add(delegationId);
|
|
@@ -968,12 +1148,12 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
968
1148
|
approvalTimeoutMs: this.config?.approvalTimeoutMs && Math.min(this.config.approvalTimeoutMs, APPROVAL_TIMEOUT_MAX_MS),
|
|
969
1149
|
readBack: update,
|
|
970
1150
|
// Another gated tool in the same turn: another (non-terminal) update.
|
|
971
|
-
onUpdate: (text) =>
|
|
1151
|
+
onUpdate: (text) => answer("delegation_update", "pending_approval", text),
|
|
972
1152
|
})
|
|
973
1153
|
.then((followUp) => {
|
|
974
1154
|
this.#followUps.delete(delegationId);
|
|
975
1155
|
// Hang-up and cancellation send nothing.
|
|
976
|
-
if (update && followUp && !abort.signal.aborted)
|
|
1156
|
+
if (update && followUp && !abort.signal.aborted) answer("delegation_result", followUp.status, followUp.text);
|
|
977
1157
|
});
|
|
978
1158
|
});
|
|
979
1159
|
}
|
|
@@ -1063,11 +1243,15 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
1063
1243
|
this.#followUps.clear();
|
|
1064
1244
|
this.#cancelledDelegations.clear();
|
|
1065
1245
|
this.#callId = undefined;
|
|
1246
|
+
this.#sessionEnd = null;
|
|
1247
|
+
clearTimeout(this.#reconnectTimer);
|
|
1066
1248
|
this.#contextSent = false;
|
|
1067
1249
|
this.#pendingContext = null;
|
|
1068
1250
|
this.#contextSend = null;
|
|
1069
1251
|
this.#delegations = Promise.resolve();
|
|
1070
1252
|
this.#answered.clear();
|
|
1253
|
+
this.#progressed.clear();
|
|
1254
|
+
this.#foldArmedBy = null;
|
|
1071
1255
|
this.#foldReadback = false;
|
|
1072
1256
|
this.#assistantTurns.clear();
|
|
1073
1257
|
this.#foldedTurns.clear();
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { CLAUSE_MAX_CHARS, ClauseBuffer } from "./voice-clause-buffer";
|
|
3
|
+
|
|
4
|
+
/** Push `chunks` one by one, then flush; returns every piece in order. */
|
|
5
|
+
function run(chunks: string[]): string[] {
|
|
6
|
+
const buffer = new ClauseBuffer();
|
|
7
|
+
const pieces = chunks.flatMap((chunk) => buffer.push(chunk));
|
|
8
|
+
return [...pieces, ...buffer.flush()];
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** Push character by character (the worst case for boundary detection). */
|
|
12
|
+
const byChar = (text: string) => run([...text]);
|
|
13
|
+
|
|
14
|
+
describe("ClauseBuffer", () => {
|
|
15
|
+
it("emits the first piece at a clause mark, then whole sentences", () => {
|
|
16
|
+
expect(byChar("Sure, we open at eight. On Sundays we are closed. Anything else?")).toEqual([
|
|
17
|
+
"Sure, ",
|
|
18
|
+
"we open at eight. ",
|
|
19
|
+
"On Sundays we are closed. ",
|
|
20
|
+
"Anything else?",
|
|
21
|
+
]);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it("does not split later pieces at clause marks", () => {
|
|
25
|
+
expect(byChar("Yes. Monday, Tuesday, and Friday work; Sunday doesn't. Done")).toEqual([
|
|
26
|
+
"Yes. ",
|
|
27
|
+
"Monday, Tuesday, and Friday work; Sunday doesn't. ",
|
|
28
|
+
"Done",
|
|
29
|
+
]);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("emits the first piece after ~30 characters or 6 words without punctuation", () => {
|
|
33
|
+
const buffer = new ClauseBuffer();
|
|
34
|
+
expect(buffer.push("We bake sourdough rye ")).toEqual([]);
|
|
35
|
+
expect(buffer.push("and a seeded loaf every")).toEqual(["We bake sourdough rye and a seeded loaf "]);
|
|
36
|
+
expect(buffer.push(" morning")).toEqual([]);
|
|
37
|
+
expect(buffer.flush()).toEqual(["every morning"]);
|
|
38
|
+
|
|
39
|
+
const words = new ClauseBuffer();
|
|
40
|
+
expect(words.push("a b c d e f ")).toEqual(["a b c d e f "]);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it("keeps a whole first sentence together when it arrives before its trailing space", () => {
|
|
44
|
+
const buffer = new ClauseBuffer();
|
|
45
|
+
expect(buffer.push("I can place that order for you.")).toEqual([]);
|
|
46
|
+
expect(buffer.flush()).toEqual(["I can place that order for you."]);
|
|
47
|
+
expect(run(["I can place that order for you.", " Shall I?"])).toEqual(["I can place that order for you. ", "Shall I?"]);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it("keeps abbreviations, initials, decimals and list numbers inside a sentence", () => {
|
|
51
|
+
expect(
|
|
52
|
+
byChar("Hi there. Dr. Smith and J. R. Lee arrive at 9 a.m. today, e.g. for 3.5 hours. Then we close."),
|
|
53
|
+
).toEqual(["Hi there. ", "Dr. Smith and J. R. Lee arrive at 9 a.m. today, e.g. for 3.5 hours. ", "Then we close."]);
|
|
54
|
+
expect(byChar("Steps:\n1. Mix the flour.\n2. Bake it.")).toEqual([
|
|
55
|
+
"Steps:\n",
|
|
56
|
+
"1. Mix the flour.\n",
|
|
57
|
+
"2. Bake it.",
|
|
58
|
+
]);
|
|
59
|
+
expect(byChar("Please call 911. It is urgent.")).toEqual(["Please call 911. ", "It is urgent."]);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it("treats line breaks (Markdown blocks and list items) as boundaries", () => {
|
|
63
|
+
expect(byChar("**Opening hours**\n\n- Monday to Friday: 8am to 6pm\n- Sunday: closed")).toEqual([
|
|
64
|
+
"**Opening hours**\n\n",
|
|
65
|
+
"- Monday to Friday: 8am to 6pm\n",
|
|
66
|
+
"- Sunday: closed",
|
|
67
|
+
]);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("keeps closing quotes and emphasis with their sentence", () => {
|
|
71
|
+
expect(byChar('He said "go." Then **we left.** Fine')).toEqual(['He said "go." ', "Then **we left.** ", "Fine"]);
|
|
72
|
+
expect(byChar("Really?! Yes")).toEqual(["Really?! ", "Yes"]);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it("waits for whitespace after a period before cutting", () => {
|
|
76
|
+
const buffer = new ClauseBuffer();
|
|
77
|
+
expect(buffer.push("It costs 3.")).toEqual([]);
|
|
78
|
+
expect(buffer.push("50 dollars. And")).toEqual(["It costs 3.50 dollars. "]);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("caps pieces at the max length, cutting at a clause mark, then a word, then hard", () => {
|
|
82
|
+
const clauses = `${"word ".repeat(30)}and then, ${"more ".repeat(30)}end.`;
|
|
83
|
+
const buffer = new ClauseBuffer();
|
|
84
|
+
expect(buffer.push("Hi. ")).toEqual([]); // the whitespace may still grow
|
|
85
|
+
const [hi, ...pieces] = [...buffer.push(clauses), ...buffer.flush()];
|
|
86
|
+
expect(hi).toBe("Hi. ");
|
|
87
|
+
expect(pieces.every((p) => p.length <= CLAUSE_MAX_CHARS)).toBe(true);
|
|
88
|
+
expect(pieces[0]).toMatch(/and then, $/);
|
|
89
|
+
expect(pieces.join("")).toBe(clauses);
|
|
90
|
+
|
|
91
|
+
const words = run(["Ok. ", "abcd ".repeat(100)]);
|
|
92
|
+
expect(words.slice(1).every((p) => p.length <= CLAUSE_MAX_CHARS && /\s$/.test(p))).toBe(true);
|
|
93
|
+
|
|
94
|
+
const solid = run(["x".repeat(500)]);
|
|
95
|
+
expect(solid.map((p) => p.length)).toEqual([CLAUSE_MAX_CHARS, CLAUSE_MAX_CHARS, 60]);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it("never cuts inside a surrogate pair", () => {
|
|
99
|
+
const text = `${"x".repeat(CLAUSE_MAX_CHARS - 1)}π${"y".repeat(10)}`;
|
|
100
|
+
const pieces = run([text]);
|
|
101
|
+
expect(pieces.join("")).toBe(text);
|
|
102
|
+
expect(pieces[0]).toBe("x".repeat(CLAUSE_MAX_CHARS - 1));
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it("pieces always concatenate to the input, however it is chunked", () => {
|
|
106
|
+
const text =
|
|
107
|
+
"**Opening hours**\n\n- Monday to Friday: 8am to 6pm\n- Saturday: 9am to 4pm\n\nWe're on Main St. near the U.S. post office, e.g. by the park. Call us! ";
|
|
108
|
+
for (const size of [1, 2, 3, 7, 13, 40, text.length]) {
|
|
109
|
+
const chunks: string[] = [];
|
|
110
|
+
for (let i = 0; i < text.length; i += size) chunks.push(text.slice(i, i + size));
|
|
111
|
+
const pieces = run(chunks);
|
|
112
|
+
const joined = pieces.join("");
|
|
113
|
+
expect(text.startsWith(joined)).toBe(true);
|
|
114
|
+
expect(joined.trimEnd()).toBe(text.trimEnd());
|
|
115
|
+
expect(pieces.every((p) => p.trim().length > 0 && p.length <= CLAUSE_MAX_CHARS)).toBe(true);
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it("idle() releases held text that ends in a terminator, and nothing else", () => {
|
|
120
|
+
const buffer = new ClauseBuffer();
|
|
121
|
+
expect(buffer.push("We're open Monday to Friday from 8am to 6pm.")).toEqual([]);
|
|
122
|
+
expect(buffer.idle()).toEqual(["We're open Monday to Friday from 8am to 6pm."]);
|
|
123
|
+
expect(buffer.idle()).toEqual([]);
|
|
124
|
+
expect(buffer.push(" On Saturdays we open")).toEqual([]);
|
|
125
|
+
expect(buffer.idle()).toEqual([]); // no terminator: keep waiting
|
|
126
|
+
expect(buffer.push(" at 9 a.m.")).toEqual([]);
|
|
127
|
+
expect(buffer.idle()).toEqual([" On Saturdays we open at 9 a.m."]); // no guards on a pause
|
|
128
|
+
for (const ending of ["Really?!", 'He said "go."', "**Done.**", "Waitβ¦", "Line\n"]) {
|
|
129
|
+
const b = new ClauseBuffer();
|
|
130
|
+
b.push(ending);
|
|
131
|
+
expect(b.idle()).toEqual([ending]);
|
|
132
|
+
}
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it("emits nothing for whitespace only, and flush empties the buffer", () => {
|
|
136
|
+
const buffer = new ClauseBuffer();
|
|
137
|
+
expect(buffer.push(" ")).toEqual([]);
|
|
138
|
+
expect(buffer.flush()).toEqual([]);
|
|
139
|
+
expect(buffer.push("Hello")).toEqual([]);
|
|
140
|
+
expect(buffer.flush()).toEqual(["Hello"]);
|
|
141
|
+
expect(buffer.flush()).toEqual([]);
|
|
142
|
+
});
|
|
143
|
+
});
|