@talk2view/sdk 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -16
- package/dist/chat/views/settings.d.ts.map +1 -1
- package/dist/chat/views/settings.js +15 -13
- package/dist/chat/views/settings.js.map +1 -1
- package/dist/chat/voice-button.d.ts.map +1 -1
- package/dist/chat/voice-button.js +46 -7
- package/dist/chat/voice-button.js.map +1 -1
- package/dist/chat-css.js +1 -1
- package/dist/chat.css +1 -1
- package/dist/chat.isolated.css +21 -1
- package/dist/chat.raw.css +21 -1
- package/dist/constants.d.ts +10 -4
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +10 -4
- package/dist/constants.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/sessions.d.ts +29 -6
- package/dist/sessions.d.ts.map +1 -1
- package/dist/sessions.js +27 -11
- package/dist/sessions.js.map +1 -1
- package/dist/types.d.ts +55 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +2 -2
- package/dist/version.js +1 -1
- package/dist/voice-elevenlabs.d.ts +197 -0
- package/dist/voice-elevenlabs.d.ts.map +1 -0
- package/dist/voice-elevenlabs.js +358 -0
- package/dist/voice-elevenlabs.js.map +1 -0
- package/dist/voice-facade.js +1 -1
- package/dist/voice-facade.js.map +1 -1
- package/dist/voice-fence.d.ts +37 -0
- package/dist/voice-fence.d.ts.map +1 -0
- package/dist/voice-fence.js +44 -0
- package/dist/voice-fence.js.map +1 -0
- package/dist/voice-turn.d.ts +119 -0
- package/dist/voice-turn.d.ts.map +1 -0
- package/dist/voice-turn.js +207 -0
- package/dist/voice-turn.js.map +1 -0
- package/dist/voice.d.ts +131 -8
- package/dist/voice.d.ts.map +1 -1
- package/dist/voice.js +492 -114
- package/dist/voice.js.map +1 -1
- package/docs/chat.md +13 -7
- package/docs/voice.md +157 -33
- package/package.json +2 -1
package/dist/voice-facade.js
CHANGED
|
@@ -10,7 +10,7 @@ import { TypedEventEmitter } from './event-emitter.js';
|
|
|
10
10
|
import { VoiceStartError } from './voice-error.js';
|
|
11
11
|
const loadVoiceController = async () => (await import('./voice.js')).T2VVoiceController;
|
|
12
12
|
/** Every event the controller emits; the facade forwards each one. */
|
|
13
|
-
const FORWARDED = ['transcript', 'toolCall', 'agentState', 'approvalChange', 'error', 'ended'];
|
|
13
|
+
const FORWARDED = ['transcript', 'toolCall', 'agentState', 'approvalChange', 'error', 'ended', 'note', 'callStarted'];
|
|
14
14
|
export class T2VVoice {
|
|
15
15
|
options;
|
|
16
16
|
emitter = new TypedEventEmitter();
|
package/dist/voice-facade.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voice-facade.js","sourceRoot":"","sources":["../src/voice-facade.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAGvD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAWnD,MAAM,mBAAmB,GAA0B,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,kBAAkB,CAAC;AAE/G,sEAAsE;AACtE,MAAM,SAAS,GAAG,CAAC,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,OAAO,CAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"voice-facade.js","sourceRoot":"","sources":["../src/voice-facade.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAGvD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAWnD,MAAM,mBAAmB,GAA0B,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,kBAAkB,CAAC;AAE/G,sEAAsE;AACtE,MAAM,SAAS,GAAG,CAAC,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,CAAU,CAAC;AAE/H,MAAM,OAAO,QAAQ;IAUU;IATZ,OAAO,GAAG,IAAI,iBAAiB,EAAiB,CAAC;IAC1D,MAAM,GAAe,MAAM,CAAC;IAC5B,UAAU,GAA8B,IAAI,CAAC;IAC7C,OAAO,GAAuC,IAAI,CAAC;IAC3D,4EAA4E;IACpE,KAAK,GAAG,CAAC,CAAC;IAClB,2DAA2D;IACnD,YAAY,GAAG,KAAK,CAAC;IAE7B,YAA6B,OAAwB;QAAxB,YAAO,GAAP,OAAO,CAAiB;IAAG,CAAC;IAEzD,gEAAgE;IAChE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,EAAE,CACA,KAAQ,EACR,QAA6C;QAE7C,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAED,GAAG,CAAyC,KAAQ,EAAE,QAA6C;QACjG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,uFAAuF;IACvF,KAAK,CAAC,KAAK,CAAC,UAA6B,EAAE;QACzC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjF,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QAC9B,MAAM,KAAK,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5B,IAAI,UAA8B,CAAC;QACnC,IAAI,CAAC;YACH,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,yEAAyE;YACzE,sEAAsE;YACtE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,qCAAqC;YAC1D,IAAI,KAAK,KAAK,IAAI,CAAC,KAAK;gBAAE,OAAO;YACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,MAAM,OAAO,GAAG,4DAA4D,CAAC;YAC7E,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvB,MAAM,IAAI,eAAe,CAAC,aAAa,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,wEAAwE;QACxE,IAAI,KAAK,KAAK,IAAI,CAAC,KAAK;YAAE,OAAO;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,OAAO,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACH,OAAO;QACL,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CACP,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EACpC,GAAG,EAAE;YACH,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;gBAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACjD,CAAC,CACF,CAAC;IACJ,CAAC;IAED,kFAAkF;IAClF,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAChB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAClC,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM;QACV,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAChB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,sDAAsD,EAAE,CAAC,CAAC;YAClH,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvB,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YACvC,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAClC,CAAC;IAEO,QAAQ,CAAC,KAAiB;QAChC,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK;YAAE,OAAO;QAClC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IACtC,CAAC;IAED;;;OAGG;IACK,QAAQ,CAAyC,KAAQ,EAAE,GAAG,IAAsB;QAC1F,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,wBAAwB,KAAK,kBAAkB,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAEO,IAAI;QACV,IAAI,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,mBAAmB,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE;YAC1F,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;YAC1D,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;YACxC,UAAU,CAAC,EAAE,CAAC,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9D,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;gBAC9B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,GAAG,IAAe,EAAE,EAAE,CACzC,IAAI,CAAC,QAAiD,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,CACnF,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;YAC7B,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fence guard for an ElevenLabs call's instructions.
|
|
3
|
+
*
|
|
4
|
+
* The engine puts the partner prompt, the page prompt and the app's
|
|
5
|
+
* capabilities between two delimiters in the agent's instructions. An
|
|
6
|
+
* ElevenLabs call's page prompt and capabilities reach the agent as dynamic
|
|
7
|
+
* variables inside that fence, so the SDK guards them as the engine does:
|
|
8
|
+
* nothing from the page or a tool can close the fence early. Each function
|
|
9
|
+
* gives the engine's text for the same input (`t2v/core/voice_rules.py`), so
|
|
10
|
+
* the instructions read the same whichever voice model takes the call, with
|
|
11
|
+
* one difference: `formatCapabilities([])` gives `(none)` where the engine's
|
|
12
|
+
* `format_capabilities` gives `""`, because the dynamic variable needs a
|
|
13
|
+
* value and the agent's placeholder is `(none)`.
|
|
14
|
+
*/
|
|
15
|
+
import type { VoiceCapability } from './types.js';
|
|
16
|
+
/** The delimiter that opens the fence. */
|
|
17
|
+
export declare const PARTNER_PROMPT_BEGIN = "<<<BEGIN APP DESCRIPTION>>>";
|
|
18
|
+
/** The delimiter that closes the fence. */
|
|
19
|
+
export declare const PARTNER_PROMPT_END = "<<<END APP DESCRIPTION>>>";
|
|
20
|
+
/** What a delimiter inside the fenced text is replaced with. */
|
|
21
|
+
export declare const DELIMITER_REMOVED = "[delimiter removed]";
|
|
22
|
+
/** What an empty page prompt or capability list reads as: a dynamic variable needs a value. */
|
|
23
|
+
export declare const NONE_PLACEHOLDER = "(none)";
|
|
24
|
+
/**
|
|
25
|
+
* `text` with each delimiter replaced by {@link DELIMITER_REMOVED}: every
|
|
26
|
+
* occurrence, left to right, as the engine's `str.replace` does. Replaced,
|
|
27
|
+
* never deleted: deleting lets nested fragments such as
|
|
28
|
+
* `<<<END APP<<<END APP DESCRIPTION>>> DESCRIPTION>>>` rejoin into a real fence.
|
|
29
|
+
*/
|
|
30
|
+
export declare function unfence(text: string): string;
|
|
31
|
+
/**
|
|
32
|
+
* The app's capabilities as `- name: description` lines (`- name` when the
|
|
33
|
+
* description is empty), every name and description unfenced; or
|
|
34
|
+
* {@link NONE_PLACEHOLDER} when there are none.
|
|
35
|
+
*/
|
|
36
|
+
export declare function formatCapabilities(capabilities: VoiceCapability[]): string;
|
|
37
|
+
//# sourceMappingURL=voice-fence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-fence.d.ts","sourceRoot":"","sources":["../src/voice-fence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,0CAA0C;AAC1C,eAAO,MAAM,oBAAoB,gCAAgC,CAAC;AAClE,2CAA2C;AAC3C,eAAO,MAAM,kBAAkB,8BAA8B,CAAC;AAC9D,gEAAgE;AAChE,eAAO,MAAM,iBAAiB,wBAAwB,CAAC;AACvD,+FAA+F;AAC/F,eAAO,MAAM,gBAAgB,WAAW,CAAC;AAEzC;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE5C;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,eAAe,EAAE,GAAG,MAAM,CAO1E"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fence guard for an ElevenLabs call's instructions.
|
|
3
|
+
*
|
|
4
|
+
* The engine puts the partner prompt, the page prompt and the app's
|
|
5
|
+
* capabilities between two delimiters in the agent's instructions. An
|
|
6
|
+
* ElevenLabs call's page prompt and capabilities reach the agent as dynamic
|
|
7
|
+
* variables inside that fence, so the SDK guards them as the engine does:
|
|
8
|
+
* nothing from the page or a tool can close the fence early. Each function
|
|
9
|
+
* gives the engine's text for the same input (`t2v/core/voice_rules.py`), so
|
|
10
|
+
* the instructions read the same whichever voice model takes the call, with
|
|
11
|
+
* one difference: `formatCapabilities([])` gives `(none)` where the engine's
|
|
12
|
+
* `format_capabilities` gives `""`, because the dynamic variable needs a
|
|
13
|
+
* value and the agent's placeholder is `(none)`.
|
|
14
|
+
*/
|
|
15
|
+
/** The delimiter that opens the fence. */
|
|
16
|
+
export const PARTNER_PROMPT_BEGIN = '<<<BEGIN APP DESCRIPTION>>>';
|
|
17
|
+
/** The delimiter that closes the fence. */
|
|
18
|
+
export const PARTNER_PROMPT_END = '<<<END APP DESCRIPTION>>>';
|
|
19
|
+
/** What a delimiter inside the fenced text is replaced with. */
|
|
20
|
+
export const DELIMITER_REMOVED = '[delimiter removed]';
|
|
21
|
+
/** What an empty page prompt or capability list reads as: a dynamic variable needs a value. */
|
|
22
|
+
export const NONE_PLACEHOLDER = '(none)';
|
|
23
|
+
/**
|
|
24
|
+
* `text` with each delimiter replaced by {@link DELIMITER_REMOVED}: every
|
|
25
|
+
* occurrence, left to right, as the engine's `str.replace` does. Replaced,
|
|
26
|
+
* never deleted: deleting lets nested fragments such as
|
|
27
|
+
* `<<<END APP<<<END APP DESCRIPTION>>> DESCRIPTION>>>` rejoin into a real fence.
|
|
28
|
+
*/
|
|
29
|
+
export function unfence(text) {
|
|
30
|
+
return text.split(PARTNER_PROMPT_BEGIN).join(DELIMITER_REMOVED).split(PARTNER_PROMPT_END).join(DELIMITER_REMOVED);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* The app's capabilities as `- name: description` lines (`- name` when the
|
|
34
|
+
* description is empty), every name and description unfenced; or
|
|
35
|
+
* {@link NONE_PLACEHOLDER} when there are none.
|
|
36
|
+
*/
|
|
37
|
+
export function formatCapabilities(capabilities) {
|
|
38
|
+
if (capabilities.length === 0)
|
|
39
|
+
return NONE_PLACEHOLDER;
|
|
40
|
+
return capabilities
|
|
41
|
+
.map(({ name, description }) => description ? `- ${unfence(name)}: ${unfence(description)}` : `- ${unfence(name)}`)
|
|
42
|
+
.join('\n');
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=voice-fence.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-fence.js","sourceRoot":"","sources":["../src/voice-fence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH,0CAA0C;AAC1C,MAAM,CAAC,MAAM,oBAAoB,GAAG,6BAA6B,CAAC;AAClE,2CAA2C;AAC3C,MAAM,CAAC,MAAM,kBAAkB,GAAG,2BAA2B,CAAC;AAC9D,gEAAgE;AAChE,MAAM,CAAC,MAAM,iBAAiB,GAAG,qBAAqB,CAAC;AACvD,+FAA+F;AAC/F,MAAM,CAAC,MAAM,gBAAgB,GAAG,QAAQ,CAAC;AAEzC;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AACpH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,YAA+B;IAChE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,gBAAgB,CAAC;IACvD,OAAO,YAAY;SAChB,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAC7B,WAAW,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,EAAE,CACnF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The voice turn runner: each `ask_talk2view` request of an ElevenLabs voice
|
|
3
|
+
* call becomes an engine turn on the end-user's chat session, one at a time,
|
|
4
|
+
* as the voice service's handoff does for Gemini Live (`va/handoff.py`).
|
|
5
|
+
*
|
|
6
|
+
* An identical request already in flight or queued is merged
|
|
7
|
+
* ({@link ALREADY_WORKING}); a different one waits its turn. Every `ask()`
|
|
8
|
+
* resolves with plain text for the agent to speak.
|
|
9
|
+
*
|
|
10
|
+
* Keep this module out of core's up-front bundle: only the voice controller,
|
|
11
|
+
* which core loads with `import()`, may import it.
|
|
12
|
+
*/
|
|
13
|
+
import type { T2VSession, TurnOptions } from './sessions.js';
|
|
14
|
+
import type { HumanDecision } from './types.js';
|
|
15
|
+
/** The answer to a request identical to one already in flight or queued. */
|
|
16
|
+
export declare const ALREADY_WORKING = "status: already_working (still on the previous request)";
|
|
17
|
+
/** The answer when a turn ends without any text. */
|
|
18
|
+
export declare const NO_TEXT = "(the agent returned no text)";
|
|
19
|
+
/** The answer when a turn could not be run, or ran past its limit. */
|
|
20
|
+
export declare const TURN_FAILED = "error: I couldn\u2019t reach the assistant just now.";
|
|
21
|
+
/**
|
|
22
|
+
* How long a turn may run, its resumes and any wait on a card included,
|
|
23
|
+
* before it is ended with {@link TURN_FAILED}. It is the voice service's
|
|
24
|
+
* `engine_turn_timeout_s` (`va/engine_bridge.py`), and the agent's
|
|
25
|
+
* `ask_talk2view` tool stops waiting for an answer at the same 120 seconds
|
|
26
|
+
* (`response_timeout_secs`).
|
|
27
|
+
*/
|
|
28
|
+
export declare const TURN_TIMEOUT_MS = 120000;
|
|
29
|
+
/**
|
|
30
|
+
* What the engine is told about a voice turn: the reply is spoken, and the
|
|
31
|
+
* call collects no approvals and shows no platform-tool activity.
|
|
32
|
+
*/
|
|
33
|
+
export declare const VOICE_TURN: TurnOptions;
|
|
34
|
+
/**
|
|
35
|
+
* What makes two requests the same: lower-cased, whitespace collapsed to
|
|
36
|
+
* single spaces, and trailing `?`, `.`, `!` and spaces dropped, as the voice
|
|
37
|
+
* service's `normalise_request` (`va/handoff.py`).
|
|
38
|
+
*/
|
|
39
|
+
export declare function normaliseRequest(text: string): string;
|
|
40
|
+
export interface VoiceTurnDeps {
|
|
41
|
+
/** The end-user's chat session the mint names: every turn runs on it. */
|
|
42
|
+
session: T2VSession;
|
|
43
|
+
/** The page prompt, sent as each turn's system message. */
|
|
44
|
+
pagePrompt?: string;
|
|
45
|
+
/**
|
|
46
|
+
* Shows the card for a client tool call that needs the end-user's go-ahead,
|
|
47
|
+
* and resolves with their decision, or with `endedBecause` when the card
|
|
48
|
+
* closed unanswered. The caller remembers an `always`.
|
|
49
|
+
*/
|
|
50
|
+
onApproval: (toolCallId: string, toolName: string, args: Record<string, unknown>) => Promise<{
|
|
51
|
+
decision: HumanDecision;
|
|
52
|
+
endedBecause?: string;
|
|
53
|
+
}>;
|
|
54
|
+
/** Whether the end-user has said `always` to this tool: it runs without a card. */
|
|
55
|
+
isAlwaysAllowed: (toolName: string) => boolean;
|
|
56
|
+
/** `working` as a turn starts, `idle` once it settles. */
|
|
57
|
+
onState: (state: 'working' | 'idle') => void;
|
|
58
|
+
/** A turn found the end-user's budget spent: the call says goodbye. */
|
|
59
|
+
onBudgetExceeded: () => void;
|
|
60
|
+
/** How long a turn may run before it is ended. Default: {@link TURN_TIMEOUT_MS}. */
|
|
61
|
+
turnTimeoutMs?: number;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Runs a voice call's requests as engine turns, one at a time. `abort()`
|
|
65
|
+
* answers every pending `ask()` at once, as the call ends.
|
|
66
|
+
*/
|
|
67
|
+
export declare class VoiceTurnRunner {
|
|
68
|
+
private readonly deps;
|
|
69
|
+
/** Normalised requests in flight or queued. */
|
|
70
|
+
private readonly inFlight;
|
|
71
|
+
/** The last turn queued: the next one starts when it settles. */
|
|
72
|
+
private last;
|
|
73
|
+
/** The controller of the turn in flight. */
|
|
74
|
+
private controller;
|
|
75
|
+
/** Set by {@link abort}: from then on every `ask()` resolves with it. */
|
|
76
|
+
private reason;
|
|
77
|
+
/** Resolves with the reason on {@link abort}; every `ask()` races it. */
|
|
78
|
+
private readonly aborted;
|
|
79
|
+
private resolveAborted;
|
|
80
|
+
constructor(deps: VoiceTurnDeps);
|
|
81
|
+
/** A turn is in flight or queued. */
|
|
82
|
+
get busy(): boolean;
|
|
83
|
+
/**
|
|
84
|
+
* Run `request` as an engine turn. Resolves with the turn's text, trimmed,
|
|
85
|
+
* or {@link NO_TEXT}; `error: <message>` when the engine reports an error;
|
|
86
|
+
* {@link TURN_FAILED} when the turn could not be run or ran past its limit;
|
|
87
|
+
* {@link ALREADY_WORKING} when the same request is already in flight or
|
|
88
|
+
* queued; once aborted, the abort reason.
|
|
89
|
+
*/
|
|
90
|
+
ask(request: string): Promise<string>;
|
|
91
|
+
/**
|
|
92
|
+
* End the runner for good: abort the turn in flight, resolve its `ask()`
|
|
93
|
+
* and every queued one with `reason`, and resolve later ones with it at once.
|
|
94
|
+
*/
|
|
95
|
+
abort(reason: string): void;
|
|
96
|
+
/**
|
|
97
|
+
* One turn, unless the runner was aborted while it waited. A turn still
|
|
98
|
+
* running at its limit is aborted and answers {@link TURN_FAILED}, whether
|
|
99
|
+
* or not its stream has noticed, and the runner moves on: the request
|
|
100
|
+
* leaves the in-flight set and the next turn starts.
|
|
101
|
+
*/
|
|
102
|
+
private run;
|
|
103
|
+
/**
|
|
104
|
+
* Tell the caller the runner's state. A throw from `onState` is swallowed,
|
|
105
|
+
* and nothing is logged: it must not lose the turn's answer, nor stop the
|
|
106
|
+
* turns queued behind it.
|
|
107
|
+
*/
|
|
108
|
+
private reportState;
|
|
109
|
+
/** The turn's answer, from its stream and the stream of each resume. */
|
|
110
|
+
private turn;
|
|
111
|
+
/**
|
|
112
|
+
* The resume of a client tool call that needs the end-user's go-ahead. A
|
|
113
|
+
* tool they always allow runs at once. Otherwise the card decides: the
|
|
114
|
+
* resume carries their decision, or the reason the card closed without one.
|
|
115
|
+
* Nothing runs until the resume is read.
|
|
116
|
+
*/
|
|
117
|
+
private resume;
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=voice-turn.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-turn.d.ts","sourceRoot":"","sources":["../src/voice-turn.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,KAAK,EAAa,aAAa,EAAmB,MAAM,YAAY,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,eAAe,4DAA4D,CAAC;AAEzF,oDAAoD;AACpD,eAAO,MAAM,OAAO,iCAAiC,CAAC;AAEtD,sEAAsE;AACtE,eAAO,MAAM,WAAW,yDAAoD,CAAC;AAE7E;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,SAAU,CAAC;AAEvC;;;GAGG;AACH,eAAO,MAAM,UAAU,EAAE,WAAsF,CAAC;AAEhH;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKrD;AAED,MAAM,WAAW,aAAa;IAC5B,yEAAyE;IACzE,OAAO,EAAE,UAAU,CAAC;IACpB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,UAAU,EAAE,CACV,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC1B,OAAO,CAAC;QAAE,QAAQ,EAAE,aAAa,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mFAAmF;IACnF,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;IAC/C,0DAA0D;IAC1D,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,GAAG,MAAM,KAAK,IAAI,CAAC;IAC7C,uEAAuE;IACvE,gBAAgB,EAAE,MAAM,IAAI,CAAC;IAC7B,oFAAoF;IACpF,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,qBAAa,eAAe;IAad,OAAO,CAAC,QAAQ,CAAC,IAAI;IAZjC,+CAA+C;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,iEAAiE;IACjE,OAAO,CAAC,IAAI,CAAuC;IACnD,4CAA4C;IAC5C,OAAO,CAAC,UAAU,CAAgC;IAClD,yEAAyE;IACzE,OAAO,CAAC,MAAM,CAAuB;IACrC,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,cAAc,CAA4B;gBAErB,IAAI,EAAE,aAAa;IAMhD,qCAAqC;IACrC,IAAI,IAAI,IAAI,OAAO,CAElB;IAED;;;;;;OAMG;IACH,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAcrC;;;OAGG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAQ3B;;;;;OAKG;YACW,GAAG;IAwBjB;;;;OAIG;IACH,OAAO,CAAC,WAAW;IAQnB,wEAAwE;YAC1D,IAAI;IAgClB;;;;;OAKG;YACW,MAAM;CAerB"}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The voice turn runner: each `ask_talk2view` request of an ElevenLabs voice
|
|
3
|
+
* call becomes an engine turn on the end-user's chat session, one at a time,
|
|
4
|
+
* as the voice service's handoff does for Gemini Live (`va/handoff.py`).
|
|
5
|
+
*
|
|
6
|
+
* An identical request already in flight or queued is merged
|
|
7
|
+
* ({@link ALREADY_WORKING}); a different one waits its turn. Every `ask()`
|
|
8
|
+
* resolves with plain text for the agent to speak.
|
|
9
|
+
*
|
|
10
|
+
* Keep this module out of core's up-front bundle: only the voice controller,
|
|
11
|
+
* which core loads with `import()`, may import it.
|
|
12
|
+
*/
|
|
13
|
+
/** The answer to a request identical to one already in flight or queued. */
|
|
14
|
+
export const ALREADY_WORKING = 'status: already_working (still on the previous request)';
|
|
15
|
+
/** The answer when a turn ends without any text. */
|
|
16
|
+
export const NO_TEXT = '(the agent returned no text)';
|
|
17
|
+
/** The answer when a turn could not be run, or ran past its limit. */
|
|
18
|
+
export const TURN_FAILED = 'error: I couldn’t reach the assistant just now.';
|
|
19
|
+
/**
|
|
20
|
+
* How long a turn may run, its resumes and any wait on a card included,
|
|
21
|
+
* before it is ended with {@link TURN_FAILED}. It is the voice service's
|
|
22
|
+
* `engine_turn_timeout_s` (`va/engine_bridge.py`), and the agent's
|
|
23
|
+
* `ask_talk2view` tool stops waiting for an answer at the same 120 seconds
|
|
24
|
+
* (`response_timeout_secs`).
|
|
25
|
+
*/
|
|
26
|
+
export const TURN_TIMEOUT_MS = 120_000;
|
|
27
|
+
/**
|
|
28
|
+
* What the engine is told about a voice turn: the reply is spoken, and the
|
|
29
|
+
* call collects no approvals and shows no platform-tool activity.
|
|
30
|
+
*/
|
|
31
|
+
export const VOICE_TURN = { surface: 'voice', supportsApproval: false, supportsToolEvents: false };
|
|
32
|
+
/**
|
|
33
|
+
* What makes two requests the same: lower-cased, whitespace collapsed to
|
|
34
|
+
* single spaces, and trailing `?`, `.`, `!` and spaces dropped, as the voice
|
|
35
|
+
* service's `normalise_request` (`va/handoff.py`).
|
|
36
|
+
*/
|
|
37
|
+
export function normaliseRequest(text) {
|
|
38
|
+
const collapsed = text.toLowerCase().trim().split(/\s+/).join(' ');
|
|
39
|
+
let end = collapsed.length;
|
|
40
|
+
while (end > 0 && '?.! '.includes(collapsed.charAt(end - 1)))
|
|
41
|
+
end -= 1;
|
|
42
|
+
return collapsed.slice(0, end);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Runs a voice call's requests as engine turns, one at a time. `abort()`
|
|
46
|
+
* answers every pending `ask()` at once, as the call ends.
|
|
47
|
+
*/
|
|
48
|
+
export class VoiceTurnRunner {
|
|
49
|
+
deps;
|
|
50
|
+
/** Normalised requests in flight or queued. */
|
|
51
|
+
inFlight = new Set();
|
|
52
|
+
/** The last turn queued: the next one starts when it settles. */
|
|
53
|
+
last = Promise.resolve();
|
|
54
|
+
/** The controller of the turn in flight. */
|
|
55
|
+
controller = null;
|
|
56
|
+
/** Set by {@link abort}: from then on every `ask()` resolves with it. */
|
|
57
|
+
reason = null;
|
|
58
|
+
/** Resolves with the reason on {@link abort}; every `ask()` races it. */
|
|
59
|
+
aborted;
|
|
60
|
+
resolveAborted;
|
|
61
|
+
constructor(deps) {
|
|
62
|
+
this.deps = deps;
|
|
63
|
+
this.aborted = new Promise((resolve) => {
|
|
64
|
+
this.resolveAborted = resolve;
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
/** A turn is in flight or queued. */
|
|
68
|
+
get busy() {
|
|
69
|
+
return this.inFlight.size > 0;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Run `request` as an engine turn. Resolves with the turn's text, trimmed,
|
|
73
|
+
* or {@link NO_TEXT}; `error: <message>` when the engine reports an error;
|
|
74
|
+
* {@link TURN_FAILED} when the turn could not be run or ran past its limit;
|
|
75
|
+
* {@link ALREADY_WORKING} when the same request is already in flight or
|
|
76
|
+
* queued; once aborted, the abort reason.
|
|
77
|
+
*/
|
|
78
|
+
ask(request) {
|
|
79
|
+
if (this.reason !== null)
|
|
80
|
+
return Promise.resolve(this.reason);
|
|
81
|
+
const key = normaliseRequest(request);
|
|
82
|
+
if (this.inFlight.has(key))
|
|
83
|
+
return Promise.resolve(ALREADY_WORKING);
|
|
84
|
+
// One turn at a time, in order. With none in flight or queued this one
|
|
85
|
+
// starts now, so the engine has it at once.
|
|
86
|
+
const start = () => this.run(request);
|
|
87
|
+
const idle = !this.busy;
|
|
88
|
+
this.inFlight.add(key);
|
|
89
|
+
const turn = (idle ? start() : this.last.then(start)).finally(() => this.inFlight.delete(key));
|
|
90
|
+
this.last = turn;
|
|
91
|
+
return Promise.race([turn, this.aborted]);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* End the runner for good: abort the turn in flight, resolve its `ask()`
|
|
95
|
+
* and every queued one with `reason`, and resolve later ones with it at once.
|
|
96
|
+
*/
|
|
97
|
+
abort(reason) {
|
|
98
|
+
if (this.reason !== null)
|
|
99
|
+
return;
|
|
100
|
+
this.reason = reason;
|
|
101
|
+
this.inFlight.clear();
|
|
102
|
+
this.controller?.abort();
|
|
103
|
+
this.resolveAborted(reason);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* One turn, unless the runner was aborted while it waited. A turn still
|
|
107
|
+
* running at its limit is aborted and answers {@link TURN_FAILED}, whether
|
|
108
|
+
* or not its stream has noticed, and the runner moves on: the request
|
|
109
|
+
* leaves the in-flight set and the next turn starts.
|
|
110
|
+
*/
|
|
111
|
+
async run(request) {
|
|
112
|
+
if (this.reason !== null)
|
|
113
|
+
return this.reason;
|
|
114
|
+
const controller = new AbortController();
|
|
115
|
+
this.controller = controller;
|
|
116
|
+
let expire;
|
|
117
|
+
const expired = new Promise((resolve) => {
|
|
118
|
+
expire = resolve;
|
|
119
|
+
});
|
|
120
|
+
const timer = setTimeout(() => {
|
|
121
|
+
controller.abort();
|
|
122
|
+
expire(TURN_FAILED);
|
|
123
|
+
}, this.deps.turnTimeoutMs ?? TURN_TIMEOUT_MS);
|
|
124
|
+
try {
|
|
125
|
+
this.reportState('working');
|
|
126
|
+
return await Promise.race([this.turn(request, controller.signal), expired]);
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
return TURN_FAILED;
|
|
130
|
+
}
|
|
131
|
+
finally {
|
|
132
|
+
clearTimeout(timer);
|
|
133
|
+
this.controller = null;
|
|
134
|
+
this.reportState('idle');
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Tell the caller the runner's state. A throw from `onState` is swallowed,
|
|
139
|
+
* and nothing is logged: it must not lose the turn's answer, nor stop the
|
|
140
|
+
* turns queued behind it.
|
|
141
|
+
*/
|
|
142
|
+
reportState(state) {
|
|
143
|
+
try {
|
|
144
|
+
this.deps.onState(state);
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
// Swallowed, as above.
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
/** The turn's answer, from its stream and the stream of each resume. */
|
|
151
|
+
async turn(request, signal) {
|
|
152
|
+
let text = '';
|
|
153
|
+
let events = this.deps.session.sendMessage(request, { systemPrompt: this.deps.pagePrompt, signal, ...VOICE_TURN });
|
|
154
|
+
for (;;) {
|
|
155
|
+
let approval = null;
|
|
156
|
+
for await (const event of events) {
|
|
157
|
+
// Aborted (the call ended, or the turn ran past its limit): the ask has
|
|
158
|
+
// its answer already, so whatever arrives after that is dropped.
|
|
159
|
+
if (signal.aborted)
|
|
160
|
+
return TURN_FAILED;
|
|
161
|
+
if (event.type === 'text') {
|
|
162
|
+
text += event.content;
|
|
163
|
+
}
|
|
164
|
+
else if (event.type === 'approval_required') {
|
|
165
|
+
approval = {
|
|
166
|
+
toolCallId: event.toolCallId,
|
|
167
|
+
toolName: event.toolName,
|
|
168
|
+
arguments: event.arguments,
|
|
169
|
+
description: event.description,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
else if (event.type === 'error') {
|
|
173
|
+
if (event.errorType === 'budget_exceeded')
|
|
174
|
+
this.deps.onBudgetExceeded();
|
|
175
|
+
return `error: ${event.message}`;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
if (approval === null)
|
|
179
|
+
return text.trim() || NO_TEXT;
|
|
180
|
+
// Nor, once aborted, does a card open, a tool run or a resume go out:
|
|
181
|
+
// checked here, after the tool permission, and below, after the card.
|
|
182
|
+
if (signal.aborted)
|
|
183
|
+
return TURN_FAILED;
|
|
184
|
+
events = await this.resume(approval, signal);
|
|
185
|
+
if (signal.aborted)
|
|
186
|
+
return TURN_FAILED;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* The resume of a client tool call that needs the end-user's go-ahead. A
|
|
191
|
+
* tool they always allow runs at once. Otherwise the card decides: the
|
|
192
|
+
* resume carries their decision, or the reason the card closed without one.
|
|
193
|
+
* Nothing runs until the resume is read.
|
|
194
|
+
*/
|
|
195
|
+
async resume(approval, signal) {
|
|
196
|
+
const { session } = this.deps;
|
|
197
|
+
if (this.deps.isAlwaysAllowed(approval.toolName)) {
|
|
198
|
+
return session.respondToApproval(approval, { action: 'once' }, signal, VOICE_TURN);
|
|
199
|
+
}
|
|
200
|
+
const { decision, endedBecause } = await this.deps.onApproval(approval.toolCallId, approval.toolName, approval.arguments);
|
|
201
|
+
if (endedBecause) {
|
|
202
|
+
return session.resumeToolCall(approval.toolCallId, JSON.stringify({ error: endedBecause }), true, signal, VOICE_TURN);
|
|
203
|
+
}
|
|
204
|
+
return session.respondToApproval(approval, decision, signal, VOICE_TURN);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
//# sourceMappingURL=voice-turn.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-turn.js","sourceRoot":"","sources":["../src/voice-turn.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAKH,4EAA4E;AAC5E,MAAM,CAAC,MAAM,eAAe,GAAG,yDAAyD,CAAC;AAEzF,oDAAoD;AACpD,MAAM,CAAC,MAAM,OAAO,GAAG,8BAA8B,CAAC;AAEtD,sEAAsE;AACtE,MAAM,CAAC,MAAM,WAAW,GAAG,iDAAiD,CAAC;AAE7E;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC;AAEvC;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAgB,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC;AAEhH;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnE,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;IAC3B,OAAO,GAAG,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QAAE,GAAG,IAAI,CAAC,CAAC;IACvE,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACjC,CAAC;AA2BD;;;GAGG;AACH,MAAM,OAAO,eAAe;IAaG;IAZ7B,+CAA+C;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9C,iEAAiE;IACzD,IAAI,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IACnD,4CAA4C;IACpC,UAAU,GAA2B,IAAI,CAAC;IAClD,yEAAyE;IACjE,MAAM,GAAkB,IAAI,CAAC;IACrC,yEAAyE;IACxD,OAAO,CAAkB;IAClC,cAAc,CAA4B;IAElD,YAA6B,IAAmB;QAAnB,SAAI,GAAJ,IAAI,CAAe;QAC9C,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACrC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC;QAChC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,qCAAqC;IACrC,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC;IAChC,CAAC;IAED;;;;;;OAMG;IACH,GAAG,CAAC,OAAe;QACjB,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QACpE,uEAAuE;QACvE,4CAA4C;QAC5C,MAAM,KAAK,GAAG,GAAoB,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/F,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAc;QAClB,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI;YAAE,OAAO;QACjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,GAAG,CAAC,OAAe;QAC/B,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,MAAiC,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,EAAE;YAC9C,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,CAAC,WAAW,CAAC,CAAC;QACtB,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,eAAe,CAAC,CAAC;QAC/C,IAAI,CAAC;YACH,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;YAC5B,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QAC9E,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,WAAW,CAAC;QACrB,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,WAAW,CAAC,KAAyB;QAC3C,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IAED,wEAAwE;IAChE,KAAK,CAAC,IAAI,CAAC,OAAe,EAAE,MAAmB;QACrD,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QACnH,SAAS,CAAC;YACR,IAAI,QAAQ,GAA2B,IAAI,CAAC;YAC5C,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,wEAAwE;gBACxE,iEAAiE;gBACjE,IAAI,MAAM,CAAC,OAAO;oBAAE,OAAO,WAAW,CAAC;gBACvC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC1B,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC;gBACxB,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;oBAC9C,QAAQ,GAAG;wBACT,UAAU,EAAE,KAAK,CAAC,UAAU;wBAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;qBAC/B,CAAC;gBACJ,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAClC,IAAI,KAAK,CAAC,SAAS,KAAK,iBAAiB;wBAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACxE,OAAO,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnC,CAAC;YACH,CAAC;YACD,IAAI,QAAQ,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC;YACrD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,MAAM,CAAC,OAAO;gBAAE,OAAO,WAAW,CAAC;YACvC,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC7C,IAAI,MAAM,CAAC,OAAO;gBAAE,OAAO,WAAW,CAAC;QACzC,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,MAAM,CAAC,QAAyB,EAAE,MAAmB;QACjE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;QAC9B,IAAI,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjD,OAAO,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QACrF,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAC3D,QAAQ,CAAC,UAAU,EACnB,QAAQ,CAAC,QAAQ,EACjB,QAAQ,CAAC,SAAS,CACnB,CAAC;QACF,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QACxH,CAAC;QACD,OAAO,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAC3E,CAAC;CACF"}
|
package/dist/voice.d.ts
CHANGED
|
@@ -3,17 +3,35 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Core never imports this module statically: the `T2VVoice` facade
|
|
5
5
|
* (`voice-facade.ts`) loads it with `import()` on the first `start()`, and it
|
|
6
|
-
* in turn loads the
|
|
6
|
+
* in turn loads the call libraries, so chat-only partners ship none of them.
|
|
7
7
|
*
|
|
8
|
-
* The engine mints
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
8
|
+
* The engine mints the call (`POST /v1/voice/sessions`) on one of the voice
|
|
9
|
+
* models this SDK runs, and the mint says which. A Gemini Live call gets a
|
|
10
|
+
* ticket: the Pipecat client libraries, loaded on every `start()` because the
|
|
11
|
+
* microphone opens through them before the mint, carry its WebRTC audio to
|
|
12
|
+
* the voice agent service, and the client tools the agent calls arrive here
|
|
13
|
+
* over the RTVI data channel. An ElevenLabs call gets a conversation token:
|
|
14
|
+
* it runs in `voice-elevenlabs.ts` on the `@elevenlabs/client` library, and
|
|
15
|
+
* the agent's requests become engine turns on the end-user's chat session
|
|
16
|
+
* (`voice-turn.ts`), all three loaded with `import()` only for such a call. An
|
|
17
|
+
* ElevenLabs call that can't connect is tried once more, on Gemini Live.
|
|
18
|
+
* Either way, client tools run through the SAME tool registry and permission
|
|
19
|
+
* check as chat, so partners write no new code.
|
|
13
20
|
*/
|
|
14
21
|
import { VoiceStartError } from './voice-error.js';
|
|
22
|
+
import type { T2VSession } from './sessions.js';
|
|
15
23
|
import type { T2VTools } from './tools.js';
|
|
16
|
-
import type { VoiceEventMap, VoiceStartOptions, VoiceState } from './types.js';
|
|
24
|
+
import type { VoiceEventMap, VoiceModelId, VoiceStartOptions, VoiceState } from './types.js';
|
|
25
|
+
/** The voice models this SDK can run, as it asks the engine for them. */
|
|
26
|
+
export declare const VOICE_MODELS_THIS_SDK_RUNS: readonly VoiceModelId[];
|
|
27
|
+
/** What the end-user is told when no voice model this app allows can take the call. */
|
|
28
|
+
export declare const NO_VOICE_MESSAGE = "Voice isn\u2019t available right now.";
|
|
29
|
+
/** Each voice model's name, as the notes say it. */
|
|
30
|
+
export declare const VOICE_MODEL_NAMES: Record<VoiceModelId, string>;
|
|
31
|
+
/** The note when a voice model couldn't connect and another took the call: the engine's text. */
|
|
32
|
+
export declare const NOTE_COULD_NOT_CONNECT = "Using {fallback} \u2014 {wanted} couldn\u2019t connect.";
|
|
33
|
+
/** {@link NOTE_COULD_NOT_CONNECT} for the voice model used (`fallback`) and the one given up (`wanted`). */
|
|
34
|
+
export declare function couldNotConnectNote(fallback: VoiceModelId, wanted: VoiceModelId): string;
|
|
17
35
|
export interface PipecatCallbacks {
|
|
18
36
|
onConnected?: () => void;
|
|
19
37
|
onDisconnected?: () => void;
|
|
@@ -85,6 +103,18 @@ export interface T2VVoiceDeps {
|
|
|
85
103
|
approvalTimeoutMs?: number;
|
|
86
104
|
/** Whether the browser will prompt for the microphone. Default: the Permissions API. */
|
|
87
105
|
micPermission?: () => Promise<PermissionState | 'unknown'>;
|
|
106
|
+
/**
|
|
107
|
+
* The chat session a voice call's turns run on, from the mint's
|
|
108
|
+
* `session_id`: the end-user's one engine session for this partner.
|
|
109
|
+
*/
|
|
110
|
+
sessionFor: (sessionId: string) => T2VSession;
|
|
111
|
+
/**
|
|
112
|
+
* Loads the ElevenLabs client library, only when an ElevenLabs call starts.
|
|
113
|
+
* Default: `loadElevenLabsLibs` (`voice-elevenlabs.ts`).
|
|
114
|
+
*/
|
|
115
|
+
loadElevenLabs?: import('./voice-elevenlabs.js').ElevenLabsLibLoader;
|
|
116
|
+
/** The browser's language tag, for the call's greeting and language. Default: `navigator.language`. */
|
|
117
|
+
language?: () => string;
|
|
88
118
|
}
|
|
89
119
|
export { VoiceStartError };
|
|
90
120
|
/** The voice controller. Construct the {@link T2VVoice} facade (`t2v.voice`) instead. */
|
|
@@ -93,6 +123,26 @@ export declare class T2VVoiceController {
|
|
|
93
123
|
private readonly emitter;
|
|
94
124
|
private _state;
|
|
95
125
|
private client;
|
|
126
|
+
/** The ElevenLabs call, from its start until the call ends. It reports its own end. */
|
|
127
|
+
private elCall;
|
|
128
|
+
/**
|
|
129
|
+
* The conversation id of an ElevenLabs call the engine minted that no
|
|
130
|
+
* {@link elCall} has taken over yet: an ending reports its end.
|
|
131
|
+
*/
|
|
132
|
+
private mintedCall;
|
|
133
|
+
/**
|
|
134
|
+
* The mint in flight, until the start that sent it reads the answer. It
|
|
135
|
+
* may yet return an ElevenLabs call, so an ending meanwhile reports that
|
|
136
|
+
* call's end once it returns.
|
|
137
|
+
*/
|
|
138
|
+
private mintInFlight;
|
|
139
|
+
/**
|
|
140
|
+
* ElevenLabs end reports the engine has not answered yet, and mints an
|
|
141
|
+
* ending came during. The next start waits for them before it mints, since
|
|
142
|
+
* the engine releases a call's credit when it hears of the end. Never
|
|
143
|
+
* rejects.
|
|
144
|
+
*/
|
|
145
|
+
private pendingEndReports;
|
|
96
146
|
private audio;
|
|
97
147
|
private libs;
|
|
98
148
|
private pendingApproval;
|
|
@@ -104,7 +154,13 @@ export declare class T2VVoiceController {
|
|
|
104
154
|
* epoch belongs to a call that was hung up, and must not touch this one.
|
|
105
155
|
*/
|
|
106
156
|
private epoch;
|
|
107
|
-
/**
|
|
157
|
+
/**
|
|
158
|
+
* Bumped by every start as it asks for the microphone. Daily's call object
|
|
159
|
+
* is one per page, so every Pipecat client's microphone is the same track,
|
|
160
|
+
* and it belongs to the latest start that asked for it.
|
|
161
|
+
*/
|
|
162
|
+
private micOpens;
|
|
163
|
+
/** Settles the in-flight `start()`'s waits when the call is hung up mid-start. */
|
|
108
164
|
private abortStart;
|
|
109
165
|
/**
|
|
110
166
|
* The hang-up in progress; a concurrent `stop()` joins it. Tagged with the
|
|
@@ -135,6 +191,73 @@ export declare class T2VVoiceController {
|
|
|
135
191
|
* call is hung up (`stop()`) before it connects.
|
|
136
192
|
*/
|
|
137
193
|
start(options?: VoiceStartOptions): Promise<void>;
|
|
194
|
+
/**
|
|
195
|
+
* A Pipecat client for a Gemini Live call, its callbacks bound to the
|
|
196
|
+
* start that built it (`stale`). They do nothing once the client is no
|
|
197
|
+
* longer the controller's: an ElevenLabs call took over, or the call
|
|
198
|
+
* ended. A refused first offer (503 at capacity, 502) is retried by the
|
|
199
|
+
* transport for several seconds; the ticket cannot succeed on a retry, so
|
|
200
|
+
* it allows none until connected, then gets its default back for mid-call
|
|
201
|
+
* ICE recovery.
|
|
202
|
+
*/
|
|
203
|
+
private buildPipecat;
|
|
204
|
+
/**
|
|
205
|
+
* Connect a Gemini Live call with the mint's ticket and offer URL, its ICE
|
|
206
|
+
* servers already on the transport. Resolves once the bot is ready.
|
|
207
|
+
*/
|
|
208
|
+
private connectGemini;
|
|
209
|
+
/**
|
|
210
|
+
* Ask the engine for a call (`POST /v1/voice/sessions`) on any voice model
|
|
211
|
+
* this SDK runs but `exclude`, with the page's instructions and the
|
|
212
|
+
* browser's language (left out when longer than {@link MAX_LANGUAGE_LENGTH}).
|
|
213
|
+
* The engine chooses the voice model. When none is left for this app and
|
|
214
|
+
* this SDK (403 `voice_model_unavailable`), the start fails with
|
|
215
|
+
* {@link NO_VOICE_MESSAGE}.
|
|
216
|
+
*/
|
|
217
|
+
private mintVoice;
|
|
218
|
+
/**
|
|
219
|
+
* The ElevenLabs call's module, with the voice turn runner, and the
|
|
220
|
+
* library it runs on, both loaded with `import()`.
|
|
221
|
+
*/
|
|
222
|
+
private loadElevenLabsCall;
|
|
223
|
+
/**
|
|
224
|
+
* Start the ElevenLabs call the engine minted, once its module and library
|
|
225
|
+
* have loaded. Its time limit counts from `mintedAt`, when the mint
|
|
226
|
+
* answered. Its requests run as engine turns on the chat session the
|
|
227
|
+
* mint names, with the page prompt as their system message; the agent gets
|
|
228
|
+
* the page prompt unfenced (`voice-elevenlabs.ts`). Resolves `true` when
|
|
229
|
+
* the call started, or the start was hung up (that ending reported the
|
|
230
|
+
* call's end); `false` when it couldn't connect, and its end is still to
|
|
231
|
+
* be reported.
|
|
232
|
+
*/
|
|
233
|
+
private startElevenLabs;
|
|
234
|
+
/**
|
|
235
|
+
* The ElevenLabs call couldn't connect, and nothing has reported its end.
|
|
236
|
+
* The engine is told (it releases the credit it holds for the call), then
|
|
237
|
+
* asked for the call once more, without ElevenLabs, and its note shown.
|
|
238
|
+
* Resolves that mint, or `null` once the start is hung up.
|
|
239
|
+
*/
|
|
240
|
+
private fallBackToGeminiLive;
|
|
241
|
+
/**
|
|
242
|
+
* Keep an end report until the engine answers it: the next start waits for
|
|
243
|
+
* it. A report that fails counts as answered.
|
|
244
|
+
*/
|
|
245
|
+
private keepEndReport;
|
|
246
|
+
/**
|
|
247
|
+
* The end report for a mint that was in flight as the call ended. Once the
|
|
248
|
+
* mint returns, the end of the ElevenLabs call it made is reported, with
|
|
249
|
+
* the ending's reason. Settles once the engine answers that report, or as
|
|
250
|
+
* soon as the mint fails or returns a Gemini Live call. Never rejects.
|
|
251
|
+
*/
|
|
252
|
+
private reportEndAfterMint;
|
|
253
|
+
/**
|
|
254
|
+
* Tell the engine an ElevenLabs call has ended (`POST
|
|
255
|
+
* /v1/voice/calls/{id}/end`), so it settles the credit it holds for the
|
|
256
|
+
* call. Sent with `keepalive`, so it outlives the page. A failure is
|
|
257
|
+
* swallowed: the engine settles a call nobody reported from ElevenLabs'
|
|
258
|
+
* bill.
|
|
259
|
+
*/
|
|
260
|
+
private reportEnd;
|
|
138
261
|
/** Hang up. Safe to call in any state. */
|
|
139
262
|
stop(): Promise<void>;
|
|
140
263
|
/**
|