@datalayer/agent-runtimes 1.5.28 → 1.5.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-embed/chunks/{ChatView-C0PWJ_wj.js → ChatView-C_NOLqwI.js} +3 -3
- package/dist-embed/datalayer-app-main.js +83 -83
- package/lib/apps/apps/appspec.d.ts +0 -5
- package/lib/apps/apps/appspec.js +1 -6
- package/lib/apps/apps/checks.js +13 -9
- package/lib/apps/embed/element.js +14 -3
- package/lib/apps/embed/embedConfig.d.ts +1 -1
- package/lib/apps/embed/embedConfig.js +1 -1
- package/lib/apps/embed/visitorApp.d.ts +9 -2
- package/lib/apps/embed/visitorApp.js +9 -2
- package/lib/apps/plugins/chat/ChatView.js +4 -1
- package/lib/chat/base/ChatBase.js +12 -3
- package/lib/runtimes/variants.d.ts +42 -37
- package/lib/runtimes/variants.js +109 -9
- package/lib/specs/appspecSchema.js +2 -2
- package/lib/specs/envvars.d.ts +4 -0
- package/lib/specs/envvars.js +44 -0
- package/lib/specs/harnesses.d.ts +22 -0
- package/lib/specs/harnesses.js +372 -0
- package/lib/specs/index.d.ts +1 -0
- package/lib/specs/index.js +1 -0
- package/lib/specs/voices.js +42 -0
- package/lib/types/agentspecs.d.ts +70 -1
- package/lib/voice/ComposerVoice.d.ts +10 -2
- package/lib/voice/ComposerVoice.js +10 -2
- package/lib/voice/HandsFree.d.ts +55 -0
- package/lib/voice/HandsFree.js +211 -0
- package/lib/voice/chatVoice.d.ts +2 -1
- package/lib/voice/chatVoice.js +5 -3
- package/lib/voice/handsFree.d.ts +58 -0
- package/lib/voice/handsFree.js +48 -0
- package/lib/voice/index.d.ts +10 -2
- package/lib/voice/index.js +2 -0
- package/package.json +1 -1
- package/scripts/codegen/generate_agents.py +56 -0
- package/scripts/codegen/generate_envvars.py +3 -2
- package/scripts/codegen/generate_harnesses.py +329 -0
- package/scripts/voice/measure.py +74 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
2
|
import { useMemo, useRef } from 'react';
|
|
3
3
|
import { DeviceHearing } from './hearing';
|
|
4
|
+
import { HandsFree } from './HandsFree';
|
|
4
5
|
import { ServerHearing, hearingPlace } from './serverHearing';
|
|
5
6
|
import { VoiceInput } from './VoiceInput';
|
|
6
7
|
import { VoiceMenu } from './VoiceMenu';
|
|
@@ -14,7 +15,7 @@ export function spokenOf(value) {
|
|
|
14
15
|
? said
|
|
15
16
|
: undefined;
|
|
16
17
|
}
|
|
17
|
-
export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled = false, speaking = false, onStopSpeaking, onInterimChange, answers = false, }) {
|
|
18
|
+
export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled = false, speaking = false, onStopSpeaking, running = false, onStopRun, onInterrupted, onInterimChange, answers = false, }) {
|
|
18
19
|
// In the browser, or on Datalayer's servers where the application, the
|
|
19
20
|
// language or the person's detection asks for it (§5, VO-14, VO-34).
|
|
20
21
|
const place = hearingPlace(voice);
|
|
@@ -45,7 +46,14 @@ export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled
|
|
|
45
46
|
if (!hearing) {
|
|
46
47
|
return null;
|
|
47
48
|
}
|
|
48
|
-
return (_jsxs("span", { "data-composer-voice": "", style: { display: 'inline-flex', alignItems: 'center', gap: 2 }, children: [_jsx(VoiceMenu, { voice: voice, answers: answers, disabled: disabled }), _jsx(
|
|
49
|
+
return (_jsxs("span", { "data-composer-voice": "", style: { display: 'inline-flex', alignItems: 'center', gap: 2 }, children: [_jsx(VoiceMenu, { voice: voice, answers: answers, disabled: disabled }), voice.input === 'hands_free' ? (_jsx(HandsFree, { hearing: hearing, language: voice.language, consentKey: voice.consentKey ?? 'chat', microphoneId: voice.microphoneId, turnSilenceMs: voice.turnSilenceMs, disabled: disabled, running: running, speaking: speaking, onStopSpeaking: onStopSpeaking, onStopRun: onStopRun, onInterrupted: onInterrupted, onInterim: (words) => {
|
|
50
|
+
setInput(words ? afterBase(words) : base.current);
|
|
51
|
+
onInterimChange?.(words !== '');
|
|
52
|
+
}, send: (text, metadata) => {
|
|
53
|
+
onInterimChange?.(false);
|
|
54
|
+
setInput(base.current);
|
|
55
|
+
send(text, metadata);
|
|
56
|
+
} })) : null, _jsx(VoiceInput, { hearing: hearing, language: voice.language, consentKey: voice.consentKey ?? 'chat', silenceMs: voice.silenceMs, microphoneId: voice.microphoneId, disabled: disabled, speaking: speaking, onStopSpeaking: onStopSpeaking, onListen: () => {
|
|
49
57
|
base.current = input.trim();
|
|
50
58
|
onStopSpeaking?.();
|
|
51
59
|
},
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* *Talk with it*: a conversation by voice, hands-free (VOICE.md VO-12,
|
|
3
|
+
* VO-13), beside the composer's microphone where the application's voice
|
|
4
|
+
* says `input: hands_free`.
|
|
5
|
+
*
|
|
6
|
+
* On, the microphone stays open. The words are written in the composer as
|
|
7
|
+
* they are heard (VO-11); once the person has spoken and been silent for the
|
|
8
|
+
* turn's silence (`turnSilenceMs`, 700 ms unless said), what was said is
|
|
9
|
+
* heard whole and **sent** — nothing waits for a click. The answer is said as
|
|
10
|
+
* it is written (hands-free implies `output: always`), and listening goes
|
|
11
|
+
* on through it: speech heard while the agent thinks or speaks is
|
|
12
|
+
* **barge-in** (VO-13) — its voice stops at once, the run is stopped while
|
|
13
|
+
* the answer is still being written, the interrupted answer is marked in
|
|
14
|
+
* the conversation, and what was heard starts the next turn. The state is
|
|
15
|
+
* shown at all times: *Listening…*, *Thinking…*, *Speaking…*, *Paused*.
|
|
16
|
+
* `Esc` or *Pause* pauses it; the toggle again turns it off.
|
|
17
|
+
*
|
|
18
|
+
* The microphone is opened anew when the agent begins answering, so what it
|
|
19
|
+
* holds then is the person's; the browser's echo cancellation keeps the
|
|
20
|
+
* answer's own sound out (`openMicrophone`). The first use asks for the
|
|
21
|
+
* microphone as the click-to-talk does (VO-61), under the same consent.
|
|
22
|
+
*
|
|
23
|
+
* @module voice/HandsFree
|
|
24
|
+
*/
|
|
25
|
+
import type { JSX } from 'react';
|
|
26
|
+
import type { Hearing } from './hearing';
|
|
27
|
+
import type { SpokenMetadata } from './types';
|
|
28
|
+
export interface HandsFreeProps {
|
|
29
|
+
hearing: Hearing;
|
|
30
|
+
/** The language listened to, BCP 47. */
|
|
31
|
+
language: string;
|
|
32
|
+
/** Whose consent to the microphone is remembered: the application's id. */
|
|
33
|
+
consentKey: string;
|
|
34
|
+
/** The microphone the person chose (VO-30); the browser's own when unsaid. */
|
|
35
|
+
microphoneId?: string;
|
|
36
|
+
/** The silence after speech that ends a turn, in ms; 700 unless said (VO-12). */
|
|
37
|
+
turnSilenceMs?: number;
|
|
38
|
+
/** A turn, heard whole: sent at once. */
|
|
39
|
+
send: (text: string, metadata: SpokenMetadata) => void;
|
|
40
|
+
/** The words heard so far, for the composer (VO-11); `''` gives it back. */
|
|
41
|
+
onInterim?: (text: string) => void;
|
|
42
|
+
/** The agent is answering: a run is in progress. */
|
|
43
|
+
running?: boolean;
|
|
44
|
+
/** The agent's answer is being said. */
|
|
45
|
+
speaking?: boolean;
|
|
46
|
+
/** Stops the agent's voice at once (barge-in, VO-13). */
|
|
47
|
+
onStopSpeaking?: () => void;
|
|
48
|
+
/** Stops the run while the answer is still being written (barge-in, VO-13). */
|
|
49
|
+
onStopRun?: () => void;
|
|
50
|
+
/** The answer being written was interrupted: marked in the conversation (VO-13). */
|
|
51
|
+
onInterrupted?: () => void;
|
|
52
|
+
disabled?: boolean;
|
|
53
|
+
}
|
|
54
|
+
export declare function HandsFree({ hearing, language, consentKey, microphoneId, turnSilenceMs, send, onInterim, running, speaking, onStopSpeaking, onStopRun, onInterrupted, disabled, }: HandsFreeProps): JSX.Element;
|
|
55
|
+
export default HandsFree;
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import { jsx as _jsx, Fragment as _Fragment, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
|
+
import { useCallback, useEffect, useRef, useState } from 'react';
|
|
3
|
+
import { Button, IconButton, Text, Tooltip } from '@primer/react';
|
|
4
|
+
import { Box } from '@datalayer/primer-addons';
|
|
5
|
+
import { openMicrophone } from './capture';
|
|
6
|
+
import { CONSENT_SENTENCE, SERVER_CONSENT_SENTENCE, consent, consentKeyFor, consented, } from './consent';
|
|
7
|
+
import { HANDS_FREE_WORDS, IDLE_PAUSE_MS, TURN_SILENCE_MS, bargedIn, handsFreeState, turnEnded, } from './handsFree';
|
|
8
|
+
import { INTERIM_MS } from './VoiceInput';
|
|
9
|
+
/** Waits. */
|
|
10
|
+
const pause = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
11
|
+
export function HandsFree({ hearing, language, consentKey, microphoneId, turnSilenceMs = TURN_SILENCE_MS, send, onInterim, running = false, speaking = false, onStopSpeaking, onStopRun, onInterrupted, disabled = false, }) {
|
|
12
|
+
const [on, setOn] = useState(false);
|
|
13
|
+
const [paused, setPaused] = useState(false);
|
|
14
|
+
const [asking, setAsking] = useState(false);
|
|
15
|
+
const [problem, setProblem] = useState();
|
|
16
|
+
const state = handsFreeState({ on, paused, speaking, running });
|
|
17
|
+
const stateRef = useRef(state);
|
|
18
|
+
stateRef.current = state;
|
|
19
|
+
const runningRef = useRef(running);
|
|
20
|
+
runningRef.current = running;
|
|
21
|
+
const capture = useRef(undefined);
|
|
22
|
+
const latest = useRef({
|
|
23
|
+
send,
|
|
24
|
+
onInterim,
|
|
25
|
+
onStopSpeaking,
|
|
26
|
+
onStopRun,
|
|
27
|
+
onInterrupted,
|
|
28
|
+
});
|
|
29
|
+
latest.current = {
|
|
30
|
+
send,
|
|
31
|
+
onInterim,
|
|
32
|
+
onStopSpeaking,
|
|
33
|
+
onStopRun,
|
|
34
|
+
onInterrupted,
|
|
35
|
+
};
|
|
36
|
+
const key = consentKeyFor(consentKey, hearing.where);
|
|
37
|
+
const turnOn = useCallback(() => {
|
|
38
|
+
if (!consented(key)) {
|
|
39
|
+
setAsking(true);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
setProblem(undefined);
|
|
43
|
+
setPaused(false);
|
|
44
|
+
setOn(true);
|
|
45
|
+
}, [key]);
|
|
46
|
+
const turnOff = useCallback(() => {
|
|
47
|
+
setOn(false);
|
|
48
|
+
setPaused(false);
|
|
49
|
+
capture.current?.cancel();
|
|
50
|
+
capture.current = undefined;
|
|
51
|
+
latest.current.onInterim?.('');
|
|
52
|
+
}, []);
|
|
53
|
+
/*
|
|
54
|
+
* The loop: while on and not paused, the microphone is open and what it
|
|
55
|
+
* heard so far is looked at every INTERIM_MS. Listening, the turn ends on
|
|
56
|
+
* the silence and is sent; thinking or speaking, speech is barge-in. The
|
|
57
|
+
* microphone is opened anew when the agent begins answering, and after
|
|
58
|
+
* each turn, so what it holds is only what came since.
|
|
59
|
+
*/
|
|
60
|
+
const heardWhile = useRef('listening');
|
|
61
|
+
useEffect(() => {
|
|
62
|
+
if (!on || paused) {
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
let alive = true;
|
|
66
|
+
let quietSince = Date.now();
|
|
67
|
+
void (async () => {
|
|
68
|
+
try {
|
|
69
|
+
await hearing.ready(language);
|
|
70
|
+
capture.current = await openMicrophone(microphoneId);
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
if (alive) {
|
|
74
|
+
setProblem(error instanceof Error
|
|
75
|
+
? error.message
|
|
76
|
+
: 'The microphone could not be opened.');
|
|
77
|
+
setOn(false);
|
|
78
|
+
}
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
heardWhile.current =
|
|
82
|
+
stateRef.current === 'listening' ? 'listening' : 'answering';
|
|
83
|
+
while (alive) {
|
|
84
|
+
await pause(INTERIM_MS);
|
|
85
|
+
const open = capture.current;
|
|
86
|
+
if (!alive || !open) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
const now = stateRef.current;
|
|
90
|
+
const answering = now === 'thinking' || now === 'speaking';
|
|
91
|
+
// The agent began answering, or finished: hear anew from here.
|
|
92
|
+
if ((answering && heardWhile.current === 'listening') ||
|
|
93
|
+
(!answering && heardWhile.current === 'answering')) {
|
|
94
|
+
open.cancel();
|
|
95
|
+
try {
|
|
96
|
+
capture.current = await openMicrophone(microphoneId);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
heardWhile.current = answering ? 'answering' : 'listening';
|
|
102
|
+
quietSince = Date.now();
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
let so;
|
|
106
|
+
try {
|
|
107
|
+
so = await hearing.hearSoFar(open.peek(), language);
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
so = undefined;
|
|
111
|
+
}
|
|
112
|
+
if (!alive || capture.current !== open || !so) {
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
if (so.text) {
|
|
116
|
+
latest.current.onInterim?.(so.text);
|
|
117
|
+
}
|
|
118
|
+
if (so.spoke) {
|
|
119
|
+
quietSince = Date.now();
|
|
120
|
+
}
|
|
121
|
+
if (bargedIn(now, so)) {
|
|
122
|
+
// The person speaks over the agent (VO-13): its voice and its run
|
|
123
|
+
// stop; what was heard goes on as the next turn.
|
|
124
|
+
latest.current.onStopSpeaking?.();
|
|
125
|
+
if (now === 'thinking' || runningRef.current) {
|
|
126
|
+
latest.current.onStopRun?.();
|
|
127
|
+
}
|
|
128
|
+
latest.current.onInterrupted?.();
|
|
129
|
+
heardWhile.current = 'listening';
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
if (now === 'listening' && turnEnded(so, turnSilenceMs)) {
|
|
133
|
+
capture.current = undefined;
|
|
134
|
+
const audio = await open.stop();
|
|
135
|
+
let heard;
|
|
136
|
+
try {
|
|
137
|
+
heard = await hearing.hear(audio, language);
|
|
138
|
+
}
|
|
139
|
+
catch (error) {
|
|
140
|
+
heard = undefined;
|
|
141
|
+
if (alive) {
|
|
142
|
+
setProblem(error instanceof Error
|
|
143
|
+
? error.message
|
|
144
|
+
: 'What you said could not be heard.');
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
if (!alive) {
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
latest.current.onInterim?.('');
|
|
151
|
+
if (heard) {
|
|
152
|
+
latest.current.send(heard.text, heard.metadata);
|
|
153
|
+
}
|
|
154
|
+
try {
|
|
155
|
+
capture.current = await openMicrophone(microphoneId);
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
heardWhile.current = 'listening';
|
|
161
|
+
quietSince = Date.now();
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
if (now === 'listening' && Date.now() - quietSince >= IDLE_PAUSE_MS) {
|
|
165
|
+
setPaused(true);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
})();
|
|
170
|
+
return () => {
|
|
171
|
+
alive = false;
|
|
172
|
+
capture.current?.cancel();
|
|
173
|
+
capture.current = undefined;
|
|
174
|
+
};
|
|
175
|
+
// The loop reads the state through a ref: it is not restarted on it.
|
|
176
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
177
|
+
}, [on, paused, hearing, language, microphoneId, turnSilenceMs]);
|
|
178
|
+
// Esc pauses it.
|
|
179
|
+
useEffect(() => {
|
|
180
|
+
if (!on) {
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
const down = (event) => {
|
|
184
|
+
if (event.key === 'Escape' && !paused) {
|
|
185
|
+
setPaused(true);
|
|
186
|
+
capture.current?.cancel();
|
|
187
|
+
capture.current = undefined;
|
|
188
|
+
latest.current.onInterim?.('');
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
window.addEventListener('keydown', down);
|
|
192
|
+
return () => window.removeEventListener('keydown', down);
|
|
193
|
+
}, [on, paused]);
|
|
194
|
+
return (_jsxs(Box, { display: "inline-flex", alignItems: "center", sx: { gap: 1, position: 'relative' }, "data-hands-free": state, children: [_jsx(Tooltip, { text: on
|
|
195
|
+
? 'Stop talking with it'
|
|
196
|
+
: 'Talk with it: speak, it answers, and listens on (hands-free)', direction: "n", children: _jsx(Button, { size: "small", variant: on ? 'primary' : 'invisible', "aria-pressed": on, "aria-label": on ? 'Stop talking with it' : 'Talk with it', disabled: disabled, "data-hands-free-toggle": "", onClick: () => (on ? turnOff() : turnOn()), children: HANDS_FREE_WORDS.off }) }), on ? (_jsxs(_Fragment, { children: [_jsx(Text, { sx: { fontSize: 0, color: 'fg.muted' }, role: "status", "aria-live": "polite", "data-hands-free-state": state, children: HANDS_FREE_WORDS[state] }), paused ? (_jsx(IconButton, { icon: () => _jsx("span", { "aria-hidden": "true", children: "\u25B6" }), size: "small", variant: "invisible", "aria-label": "Resume listening", "data-hands-free-resume": "", onClick: () => setPaused(false) })) : (_jsx(IconButton, { icon: () => _jsx("span", { "aria-hidden": "true", children: "\u275A\u275A" }), size: "small", variant: "invisible", "aria-label": "Pause listening (Esc)", "data-hands-free-pause": "", onClick: () => {
|
|
197
|
+
setPaused(true);
|
|
198
|
+
capture.current?.cancel();
|
|
199
|
+
capture.current = undefined;
|
|
200
|
+
latest.current.onInterim?.('');
|
|
201
|
+
} }))] })) : null, problem ? (_jsx(Text, { sx: { fontSize: 0, color: 'danger.fg' }, "data-hands-free-problem": "", children: problem })) : null, asking ? (_jsxs(Box, { role: "dialog", "aria-label": "The microphone", "data-hands-free-consent": "", position: "absolute", bottom: "100%", left: 0, width: 280, p: 3, bg: "canvas.overlay", border: "1px solid", borderColor: "border.default", borderRadius: 2, boxShadow: "shadow.large", zIndex: 10, children: [_jsx(Text, { as: "p", sx: { fontSize: 1, mb: 2 }, children: hearing.where === 'server'
|
|
202
|
+
? SERVER_CONSENT_SENTENCE
|
|
203
|
+
: CONSENT_SENTENCE }), _jsxs(Box, { display: "flex", gap: 2, justifyContent: "flex-end", children: [_jsx(Button, { size: "small", onClick: () => setAsking(false), children: "Not now" }), _jsx(Button, { size: "small", variant: "primary", "data-hands-free-consent-yes": "", onClick: () => {
|
|
204
|
+
consent(key);
|
|
205
|
+
setAsking(false);
|
|
206
|
+
setProblem(undefined);
|
|
207
|
+
setPaused(false);
|
|
208
|
+
setOn(true);
|
|
209
|
+
}, children: "Use the microphone" })] })] })) : null] }));
|
|
210
|
+
}
|
|
211
|
+
export default HandsFree;
|
package/lib/voice/chatVoice.d.ts
CHANGED
|
@@ -35,7 +35,8 @@ export interface ChatVoicePlace {
|
|
|
35
35
|
* - **Listening**: a microphone in the composer — click it and speak, the
|
|
36
36
|
* words written in the box as they are heard, stopped by a second click
|
|
37
37
|
* or when the person stops speaking; or held, and let go (VO-10, VO-11).
|
|
38
|
-
* `hands_free` (VO-12)
|
|
38
|
+
* `hands_free` (VO-12): the microphone, and *Talk with it* beside it; it
|
|
39
|
+
* implies answers said as they are written.
|
|
39
40
|
* - **Speaking**: its answers said by Datalayer's speech service, on request
|
|
40
41
|
* or always (VO-20, VO-21); refused here when its host says no service.
|
|
41
42
|
*/
|
package/lib/voice/chatVoice.js
CHANGED
|
@@ -35,7 +35,8 @@ export const DATALAYER_VOICE_ENGINES = {
|
|
|
35
35
|
* - **Listening**: a microphone in the composer — click it and speak, the
|
|
36
36
|
* words written in the box as they are heard, stopped by a second click
|
|
37
37
|
* or when the person stops speaking; or held, and let go (VO-10, VO-11).
|
|
38
|
-
* `hands_free` (VO-12)
|
|
38
|
+
* `hands_free` (VO-12): the microphone, and *Talk with it* beside it; it
|
|
39
|
+
* implies answers said as they are written.
|
|
39
40
|
* - **Speaking**: its answers said by Datalayer's speech service, on request
|
|
40
41
|
* or always (VO-20, VO-21); refused here when its host says no service.
|
|
41
42
|
*/
|
|
@@ -49,8 +50,9 @@ export function chatVoiceOf(spec, place) {
|
|
|
49
50
|
const language = spec.language || place.language || 'en-US';
|
|
50
51
|
return {
|
|
51
52
|
language,
|
|
52
|
-
input: spec.input
|
|
53
|
-
|
|
53
|
+
input: spec.input,
|
|
54
|
+
// Hands-free implies the answers heard as they are written (VO-12).
|
|
55
|
+
output: spec.input === 'hands_free' ? 'always' : spec.output,
|
|
54
56
|
voice: spec.voice || voiceFor(language)?.id || '',
|
|
55
57
|
modelsUrl: place.modelsUrl || DATALAYER_SPEECH_MODELS_URL,
|
|
56
58
|
engines: DATALAYER_VOICE_ENGINES,
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A conversation by voice, hands-free (VOICE.md VO-12, VO-13): what the
|
|
3
|
+
* composer is doing at every moment, and when a turn ends or the person
|
|
4
|
+
* speaks over the agent. Pure: the component (`HandsFree`) keeps the
|
|
5
|
+
* microphone and the clock, this decides.
|
|
6
|
+
*
|
|
7
|
+
* - **Listening**: the microphone is open; the turn ends after
|
|
8
|
+
* `turnSilenceMs` of silence once the person has spoken (700 ms unless
|
|
9
|
+
* the application or the person says otherwise), and is sent.
|
|
10
|
+
* - **Thinking**: the turn was sent and the agent is answering, nothing
|
|
11
|
+
* said yet.
|
|
12
|
+
* - **Speaking**: the answer is being said. The microphone stays open:
|
|
13
|
+
* speech heard now is barge-in (VO-13) — the voice stops at once, the run
|
|
14
|
+
* is stopped when the answer is still being written, and what was heard
|
|
15
|
+
* starts the next turn.
|
|
16
|
+
* - **Paused**: the person paused it (`Esc`, *Pause*), or nothing was said
|
|
17
|
+
* for `IDLE_PAUSE_MS`; nothing is heard until resumed.
|
|
18
|
+
*
|
|
19
|
+
* @module voice/handsFree
|
|
20
|
+
*/
|
|
21
|
+
/** What hands-free is doing, shown at all times (VO-12). */
|
|
22
|
+
export type HandsFreeState = 'off' | 'listening' | 'thinking' | 'speaking' | 'paused';
|
|
23
|
+
/** The word for each state, as the composer shows it. */
|
|
24
|
+
export declare const HANDS_FREE_WORDS: Record<HandsFreeState, string>;
|
|
25
|
+
/** The silence after speech that ends a turn, in ms, unless said otherwise (VO-12). */
|
|
26
|
+
export declare const TURN_SILENCE_MS = 700;
|
|
27
|
+
/** Listening with nothing said for this long pauses it: a microphone is not left open on an empty room. */
|
|
28
|
+
export declare const IDLE_PAUSE_MS: number;
|
|
29
|
+
/** What decides the state. */
|
|
30
|
+
export interface HandsFreeInputs {
|
|
31
|
+
/** The toggle is on. */
|
|
32
|
+
on: boolean;
|
|
33
|
+
/** The person paused it, or the room went quiet. */
|
|
34
|
+
paused: boolean;
|
|
35
|
+
/** The agent's answer is being said. */
|
|
36
|
+
speaking: boolean;
|
|
37
|
+
/** A run is in progress: the agent is answering. */
|
|
38
|
+
running: boolean;
|
|
39
|
+
}
|
|
40
|
+
/** The state for what is known: speaking wins over thinking, pausing over both. */
|
|
41
|
+
export declare function handsFreeState(inputs: HandsFreeInputs): HandsFreeState;
|
|
42
|
+
/** What the microphone heard so far, as the hearing says it (`HeardSoFar`). */
|
|
43
|
+
export interface HeardSoFarLike {
|
|
44
|
+
spoke: boolean;
|
|
45
|
+
silentMs: number;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Whether the turn ended: the person spoke, and has been silent for the
|
|
49
|
+
* turn's silence since (VO-12).
|
|
50
|
+
*/
|
|
51
|
+
export declare function turnEnded(heard: HeardSoFarLike, turnSilenceMs?: number): boolean;
|
|
52
|
+
/**
|
|
53
|
+
* Whether the person spoke over the agent (VO-13): speech found since the
|
|
54
|
+
* agent began answering, while it thinks or speaks. The microphone is opened
|
|
55
|
+
* anew when the agent begins, so what it holds is the person's, not the
|
|
56
|
+
* answer's echo — the browser's echo cancellation keeps that out.
|
|
57
|
+
*/
|
|
58
|
+
export declare function bargedIn(state: HandsFreeState, heard: HeardSoFarLike): boolean;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2025-2026 Datalayer, Inc.
|
|
3
|
+
* Distributed under the terms of the Modified BSD License.
|
|
4
|
+
*/
|
|
5
|
+
/** The word for each state, as the composer shows it. */
|
|
6
|
+
export const HANDS_FREE_WORDS = {
|
|
7
|
+
off: 'Talk with it',
|
|
8
|
+
listening: 'Listening…',
|
|
9
|
+
thinking: 'Thinking…',
|
|
10
|
+
speaking: 'Speaking…',
|
|
11
|
+
paused: 'Paused',
|
|
12
|
+
};
|
|
13
|
+
/** The silence after speech that ends a turn, in ms, unless said otherwise (VO-12). */
|
|
14
|
+
export const TURN_SILENCE_MS = 700;
|
|
15
|
+
/** Listening with nothing said for this long pauses it: a microphone is not left open on an empty room. */
|
|
16
|
+
export const IDLE_PAUSE_MS = 5 * 60_000;
|
|
17
|
+
/** The state for what is known: speaking wins over thinking, pausing over both. */
|
|
18
|
+
export function handsFreeState(inputs) {
|
|
19
|
+
if (!inputs.on) {
|
|
20
|
+
return 'off';
|
|
21
|
+
}
|
|
22
|
+
if (inputs.paused) {
|
|
23
|
+
return 'paused';
|
|
24
|
+
}
|
|
25
|
+
if (inputs.speaking) {
|
|
26
|
+
return 'speaking';
|
|
27
|
+
}
|
|
28
|
+
if (inputs.running) {
|
|
29
|
+
return 'thinking';
|
|
30
|
+
}
|
|
31
|
+
return 'listening';
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Whether the turn ended: the person spoke, and has been silent for the
|
|
35
|
+
* turn's silence since (VO-12).
|
|
36
|
+
*/
|
|
37
|
+
export function turnEnded(heard, turnSilenceMs = TURN_SILENCE_MS) {
|
|
38
|
+
return heard.spoke && heard.silentMs >= turnSilenceMs;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Whether the person spoke over the agent (VO-13): speech found since the
|
|
42
|
+
* agent began answering, while it thinks or speaks. The microphone is opened
|
|
43
|
+
* anew when the agent begins, so what it holds is the person's, not the
|
|
44
|
+
* answer's echo — the browser's echo cancellation keeps that out.
|
|
45
|
+
*/
|
|
46
|
+
export function bargedIn(state, heard) {
|
|
47
|
+
return (state === 'speaking' || state === 'thinking') && heard.spoke;
|
|
48
|
+
}
|
package/lib/voice/index.d.ts
CHANGED
|
@@ -14,6 +14,8 @@ export * from './pinned';
|
|
|
14
14
|
export * from './capture';
|
|
15
15
|
export * from './hearing';
|
|
16
16
|
export * from './serverHearing';
|
|
17
|
+
export * from './handsFree';
|
|
18
|
+
export { HandsFree, type HandsFreeProps } from './HandsFree';
|
|
17
19
|
export * from './speaker';
|
|
18
20
|
export * from './consent';
|
|
19
21
|
export * from './settings';
|
|
@@ -47,8 +49,14 @@ export type { VoiceSpeaker } from './useVoiceSpeaker';
|
|
|
47
49
|
export interface ChatVoice {
|
|
48
50
|
/** The language listened to and spoken, BCP 47. */
|
|
49
51
|
language: string;
|
|
50
|
-
/**
|
|
51
|
-
|
|
52
|
+
/**
|
|
53
|
+
* `push_to_talk`: the microphone in the composer; `hands_free`: that, and
|
|
54
|
+
* *Talk with it* beside it — a conversation by voice (VO-12, VO-13);
|
|
55
|
+
* `off`: none.
|
|
56
|
+
*/
|
|
57
|
+
input: 'off' | 'push_to_talk' | 'hands_free';
|
|
58
|
+
/** The silence after speech that ends a hands-free turn, in ms (VO-12): 700 when unsaid. */
|
|
59
|
+
turnSilenceMs?: number;
|
|
52
60
|
/**
|
|
53
61
|
* When answers are heard: `always`, as they are written; `on_request`, a
|
|
54
62
|
* *Read aloud* under each answer (VO-21); or `off`, the default.
|
package/lib/voice/index.js
CHANGED
|
@@ -8,6 +8,8 @@ export * from './pinned';
|
|
|
8
8
|
export * from './capture';
|
|
9
9
|
export * from './hearing';
|
|
10
10
|
export * from './serverHearing';
|
|
11
|
+
export * from './handsFree';
|
|
12
|
+
export { HandsFree } from './HandsFree';
|
|
11
13
|
export * from './speaker';
|
|
12
14
|
export * from './consent';
|
|
13
15
|
export * from './settings';
|
package/package.json
CHANGED
|
@@ -304,6 +304,7 @@ def load_yaml_specs(specs_dir: Path) -> List[tuple[str, Dict[str, Any]]]:
|
|
|
304
304
|
|
|
305
305
|
check_model_additionals(resolved, load_model_ids(specs_dir))
|
|
306
306
|
check_protocols(resolved, load_protocol_ids(specs_dir))
|
|
307
|
+
check_harnesses(resolved, load_harnesses(specs_dir))
|
|
307
308
|
check_voices(resolved)
|
|
308
309
|
return resolved
|
|
309
310
|
|
|
@@ -364,6 +365,60 @@ def check_protocols(
|
|
|
364
365
|
raise SystemExit("\n".join(problems))
|
|
365
366
|
|
|
366
367
|
|
|
368
|
+
BUILTIN_HARNESSES = ("pydantic-ai", "vercel-ai")
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def load_harnesses(specs_dir: Path) -> dict[str, dict[str, Any]]:
|
|
372
|
+
"""Every harness of the catalogue (`agentspecs/harnesses`, SHIELD.md SH-01), by id."""
|
|
373
|
+
harnesses: dict[str, dict[str, Any]] = {}
|
|
374
|
+
for yaml_file in sorted((specs_dir.parent / "harnesses").glob("*.yaml")):
|
|
375
|
+
if yaml_file.name == "register.yaml":
|
|
376
|
+
continue
|
|
377
|
+
harness = yaml.safe_load(yaml_file.read_text()) or {}
|
|
378
|
+
if harness.get("id"):
|
|
379
|
+
harnesses[str(harness["id"])] = harness
|
|
380
|
+
return harnesses
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def check_harnesses(
|
|
384
|
+
specs: List[tuple[str, Dict[str, Any]]], harnesses: dict[str, dict[str, Any]]
|
|
385
|
+
) -> None:
|
|
386
|
+
"""Refuse an agent naming a harness the catalogue does not know, or asking what it cannot honour (SH-01)."""
|
|
387
|
+
problems = []
|
|
388
|
+
for _, spec in specs:
|
|
389
|
+
harness_id = str(spec.get("harness") or BUILTIN_HARNESSES[0])
|
|
390
|
+
if harness_id in BUILTIN_HARNESSES:
|
|
391
|
+
continue
|
|
392
|
+
harness = harnesses.get(harness_id)
|
|
393
|
+
if harness is None:
|
|
394
|
+
problems.append(
|
|
395
|
+
f"{spec.get('id')}: there is no harness named {harness_id!r}: one of "
|
|
396
|
+
f"{', '.join([*BUILTIN_HARNESSES, *sorted(harnesses)])}"
|
|
397
|
+
)
|
|
398
|
+
continue
|
|
399
|
+
if harness_id == "custom":
|
|
400
|
+
problems.append(
|
|
401
|
+
f"{spec.get('id')}: the harness 'custom' names no command: an organization's "
|
|
402
|
+
"own harness spec names it"
|
|
403
|
+
)
|
|
404
|
+
continue
|
|
405
|
+
honours = harness.get("honours") or {}
|
|
406
|
+
if spec.get("model") and not honours.get("model"):
|
|
407
|
+
problems.append(
|
|
408
|
+
f"{spec.get('id')}: the harness {harness_id!r} does not take a model"
|
|
409
|
+
)
|
|
410
|
+
if spec.get("subagents") and not honours.get("subagents"):
|
|
411
|
+
problems.append(
|
|
412
|
+
f"{spec.get('id')}: the harness {harness_id!r} has its own subagents"
|
|
413
|
+
)
|
|
414
|
+
if spec.get("skills") and (harness.get("skills") or {}).get("format") == "none":
|
|
415
|
+
problems.append(
|
|
416
|
+
f"{spec.get('id')}: the harness {harness_id!r} reads no skills"
|
|
417
|
+
)
|
|
418
|
+
if problems:
|
|
419
|
+
raise SystemExit("\n".join(problems))
|
|
420
|
+
|
|
421
|
+
|
|
367
422
|
def load_model_ids(specs_dir: Path) -> set[str]:
|
|
368
423
|
"""Every id the models catalogue answers to: its ids and their aliases."""
|
|
369
424
|
ids: set[str] = set()
|
|
@@ -1726,6 +1781,7 @@ export * from './uiRenderers';
|
|
|
1726
1781
|
export * from './protocols';
|
|
1727
1782
|
export * from './frames';
|
|
1728
1783
|
export * from './cogs';
|
|
1784
|
+
export * from './harnesses';
|
|
1729
1785
|
export * from './guards';
|
|
1730
1786
|
export * from './gates';
|
|
1731
1787
|
export * from './tracks';
|
|
@@ -13,6 +13,7 @@ Usage:
|
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
15
|
import argparse
|
|
16
|
+
import json
|
|
16
17
|
import sys
|
|
17
18
|
from pathlib import Path
|
|
18
19
|
from typing import Any
|
|
@@ -85,7 +86,7 @@ def generate_python_code(specs: list[dict[str, Any]]) -> str:
|
|
|
85
86
|
f' id="{envvar_id}",',
|
|
86
87
|
f' version="{version}",',
|
|
87
88
|
f' name="{spec["name"]}",',
|
|
88
|
-
f
|
|
89
|
+
f" description={json.dumps(spec['description'], ensure_ascii=False)},",
|
|
89
90
|
f" registrationUrl={registration_url_value},",
|
|
90
91
|
f" tags={_fmt_list(spec.get('tags', []))},",
|
|
91
92
|
f" icon={icon},",
|
|
@@ -185,7 +186,7 @@ def generate_typescript_code(specs: list[dict[str, Any]]) -> str:
|
|
|
185
186
|
f" id: '{envvar_id}',",
|
|
186
187
|
f" version: '{version}',",
|
|
187
188
|
f" name: '{spec['name']}',",
|
|
188
|
-
f" description:
|
|
189
|
+
f" description: {json.dumps(spec['description'], ensure_ascii=False)},",
|
|
189
190
|
]
|
|
190
191
|
)
|
|
191
192
|
|