@datalayer/agent-runtimes 1.5.27 → 1.5.30

package/lib/voice/HandsFree.d.ts

@@ -0,0 +1,55 @@
+/**
+ * *Talk with it*: a conversation by voice, hands-free (VOICE.md VO-12,
+ * VO-13), beside the composer's microphone where the application's voice
+ * says `input: hands_free`.
+ *
+ * On, the microphone stays open. The words are written in the composer as
+ * they are heard (VO-11); once the person has spoken and been silent for the
+ * turn's silence (`turnSilenceMs`, 700 ms unless said), what was said is
+ * heard whole and **sent** — nothing waits for a click. The answer is said as
+ * it is written (hands-free implies `output: always`), and listening goes
+ * on through it: speech heard while the agent thinks or speaks is
+ * **barge-in** (VO-13) — its voice stops at once, the run is stopped while
+ * the answer is still being written, the interrupted answer is marked in
+ * the conversation, and what was heard starts the next turn. The state is
+ * shown at all times: *Listening…*, *Thinking…*, *Speaking…*, *Paused*.
+ * `Esc` or *Pause* pauses it; the toggle again turns it off.
+ *
+ * The microphone is opened anew when the agent begins answering, so what it
+ * holds then is the person's; the browser's echo cancellation keeps the
+ * answer's own sound out (`openMicrophone`). The first use asks for the
+ * microphone as the click-to-talk does (VO-61), under the same consent.
+ *
+ * @module voice/HandsFree
+ */
+import type { JSX } from 'react';
+import type { Hearing } from './hearing';
+import type { SpokenMetadata } from './types';
+export interface HandsFreeProps {
+    hearing: Hearing;
+    /** The language listened to, BCP 47. */
+    language: string;
+    /** Whose consent to the microphone is remembered: the application's id. */
+    consentKey: string;
+    /** The microphone the person chose (VO-30); the browser's own when unsaid. */
+    microphoneId?: string;
+    /** The silence after speech that ends a turn, in ms; 700 unless said (VO-12). */
+    turnSilenceMs?: number;
+    /** A turn, heard whole: sent at once. */
+    send: (text: string, metadata: SpokenMetadata) => void;
+    /** The words heard so far, for the composer (VO-11); `''` gives it back. */
+    onInterim?: (text: string) => void;
+    /** The agent is answering: a run is in progress. */
+    running?: boolean;
+    /** The agent's answer is being said. */
+    speaking?: boolean;
+    /** Stops the agent's voice at once (barge-in, VO-13). */
+    onStopSpeaking?: () => void;
+    /** Stops the run while the answer is still being written (barge-in, VO-13). */
+    onStopRun?: () => void;
+    /** The answer being written was interrupted: marked in the conversation (VO-13). */
+    onInterrupted?: () => void;
+    disabled?: boolean;
+}
+export declare function HandsFree({ hearing, language, consentKey, microphoneId, turnSilenceMs, send, onInterim, running, speaking, onStopSpeaking, onStopRun, onInterrupted, disabled, }: HandsFreeProps): JSX.Element;
+export default HandsFree;

package/lib/voice/HandsFree.js

@@ -0,0 +1,211 @@
+import { jsx as _jsx, Fragment as _Fragment, jsxs as _jsxs } from "react/jsx-runtime";
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { Button, IconButton, Text, Tooltip } from '@primer/react';
+import { Box } from '@datalayer/primer-addons';
+import { openMicrophone } from './capture';
+import { CONSENT_SENTENCE, SERVER_CONSENT_SENTENCE, consent, consentKeyFor, consented, } from './consent';
+import { HANDS_FREE_WORDS, IDLE_PAUSE_MS, TURN_SILENCE_MS, bargedIn, handsFreeState, turnEnded, } from './handsFree';
+import { INTERIM_MS } from './VoiceInput';
+/** Waits. */
+const pause = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+export function HandsFree({ hearing, language, consentKey, microphoneId, turnSilenceMs = TURN_SILENCE_MS, send, onInterim, running = false, speaking = false, onStopSpeaking, onStopRun, onInterrupted, disabled = false, }) {
+    const [on, setOn] = useState(false);
+    const [paused, setPaused] = useState(false);
+    const [asking, setAsking] = useState(false);
+    const [problem, setProblem] = useState();
+    const state = handsFreeState({ on, paused, speaking, running });
+    const stateRef = useRef(state);
+    stateRef.current = state;
+    const runningRef = useRef(running);
+    runningRef.current = running;
+    const capture = useRef(undefined);
+    const latest = useRef({
+        send,
+        onInterim,
+        onStopSpeaking,
+        onStopRun,
+        onInterrupted,
+    });
+    latest.current = {
+        send,
+        onInterim,
+        onStopSpeaking,
+        onStopRun,
+        onInterrupted,
+    };
+    const key = consentKeyFor(consentKey, hearing.where);
+    const turnOn = useCallback(() => {
+        if (!consented(key)) {
+            setAsking(true);
+            return;
+        }
+        setProblem(undefined);
+        setPaused(false);
+        setOn(true);
+    }, [key]);
+    const turnOff = useCallback(() => {
+        setOn(false);
+        setPaused(false);
+        capture.current?.cancel();
+        capture.current = undefined;
+        latest.current.onInterim?.('');
+    }, []);
+    /*
+     * The loop: while on and not paused, the microphone is open and what it
+     * heard so far is looked at every INTERIM_MS. Listening, the turn ends on
+     * the silence and is sent; thinking or speaking, speech is barge-in. The
+     * microphone is opened anew when the agent begins answering, and after
+     * each turn, so what it holds is only what came since.
+     */
+    const heardWhile = useRef('listening');
+    useEffect(() => {
+        if (!on || paused) {
+            return;
+        }
+        let alive = true;
+        let quietSince = Date.now();
+        void (async () => {
+            try {
+                await hearing.ready(language);
+                capture.current = await openMicrophone(microphoneId);
+            }
+            catch (error) {
+                if (alive) {
+                    setProblem(error instanceof Error
+                        ? error.message
+                        : 'The microphone could not be opened.');
+                    setOn(false);
+                }
+                return;
+            }
+            heardWhile.current =
+                stateRef.current === 'listening' ? 'listening' : 'answering';
+            while (alive) {
+                await pause(INTERIM_MS);
+                const open = capture.current;
+                if (!alive || !open) {
+                    return;
+                }
+                const now = stateRef.current;
+                const answering = now === 'thinking' || now === 'speaking';
+                // The agent began answering, or finished: hear anew from here.
+                if ((answering && heardWhile.current === 'listening') ||
+                    (!answering && heardWhile.current === 'answering')) {
+                    open.cancel();
+                    try {
+                        capture.current = await openMicrophone(microphoneId);
+                    }
+                    catch {
+                        return;
+                    }
+                    heardWhile.current = answering ? 'answering' : 'listening';
+                    quietSince = Date.now();
+                    continue;
+                }
+                let so;
+                try {
+                    so = await hearing.hearSoFar(open.peek(), language);
+                }
+                catch {
+                    so = undefined;
+                }
+                if (!alive || capture.current !== open || !so) {
+                    continue;
+                }
+                if (so.text) {
+                    latest.current.onInterim?.(so.text);
+                }
+                if (so.spoke) {
+                    quietSince = Date.now();
+                }
+                if (bargedIn(now, so)) {
+                    // The person speaks over the agent (VO-13): its voice and its run
+                    // stop; what was heard goes on as the next turn.
+                    latest.current.onStopSpeaking?.();
+                    if (now === 'thinking' || runningRef.current) {
+                        latest.current.onStopRun?.();
+                    }
+                    latest.current.onInterrupted?.();
+                    heardWhile.current = 'listening';
+                    continue;
+                }
+                if (now === 'listening' && turnEnded(so, turnSilenceMs)) {
+                    capture.current = undefined;
+                    const audio = await open.stop();
+                    let heard;
+                    try {
+                        heard = await hearing.hear(audio, language);
+                    }
+                    catch (error) {
+                        heard = undefined;
+                        if (alive) {
+                            setProblem(error instanceof Error
+                                ? error.message
+                                : 'What you said could not be heard.');
+                        }
+                    }
+                    if (!alive) {
+                        return;
+                    }
+                    latest.current.onInterim?.('');
+                    if (heard) {
+                        latest.current.send(heard.text, heard.metadata);
+                    }
+                    try {
+                        capture.current = await openMicrophone(microphoneId);
+                    }
+                    catch {
+                        return;
+                    }
+                    heardWhile.current = 'listening';
+                    quietSince = Date.now();
+                    continue;
+                }
+                if (now === 'listening' && Date.now() - quietSince >= IDLE_PAUSE_MS) {
+                    setPaused(true);
+                    return;
+                }
+            }
+        })();
+        return () => {
+            alive = false;
+            capture.current?.cancel();
+            capture.current = undefined;
+        };
+        // The loop reads the state through a ref: it is not restarted on it.
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [on, paused, hearing, language, microphoneId, turnSilenceMs]);
+    // Esc pauses it.
+    useEffect(() => {
+        if (!on) {
+            return;
+        }
+        const down = (event) => {
+            if (event.key === 'Escape' && !paused) {
+                setPaused(true);
+                capture.current?.cancel();
+                capture.current = undefined;
+                latest.current.onInterim?.('');
+            }
+        };
+        window.addEventListener('keydown', down);
+        return () => window.removeEventListener('keydown', down);
+    }, [on, paused]);
+    return (_jsxs(Box, { display: "inline-flex", alignItems: "center", sx: { gap: 1, position: 'relative' }, "data-hands-free": state, children: [_jsx(Tooltip, { text: on
+                    ? 'Stop talking with it'
+                    : 'Talk with it: speak, it answers, and listens on (hands-free)', direction: "n", children: _jsx(Button, { size: "small", variant: on ? 'primary' : 'invisible', "aria-pressed": on, "aria-label": on ? 'Stop talking with it' : 'Talk with it', disabled: disabled, "data-hands-free-toggle": "", onClick: () => (on ? turnOff() : turnOn()), children: HANDS_FREE_WORDS.off }) }), on ? (_jsxs(_Fragment, { children: [_jsx(Text, { sx: { fontSize: 0, color: 'fg.muted' }, role: "status", "aria-live": "polite", "data-hands-free-state": state, children: HANDS_FREE_WORDS[state] }), paused ? (_jsx(IconButton, { icon: () => _jsx("span", { "aria-hidden": "true", children: "\u25B6" }), size: "small", variant: "invisible", "aria-label": "Resume listening", "data-hands-free-resume": "", onClick: () => setPaused(false) })) : (_jsx(IconButton, { icon: () => _jsx("span", { "aria-hidden": "true", children: "\u275A\u275A" }), size: "small", variant: "invisible", "aria-label": "Pause listening (Esc)", "data-hands-free-pause": "", onClick: () => {
+                            setPaused(true);
+                            capture.current?.cancel();
+                            capture.current = undefined;
+                            latest.current.onInterim?.('');
+                        } }))] })) : null, problem ? (_jsx(Text, { sx: { fontSize: 0, color: 'danger.fg' }, "data-hands-free-problem": "", children: problem })) : null, asking ? (_jsxs(Box, { role: "dialog", "aria-label": "The microphone", "data-hands-free-consent": "", position: "absolute", bottom: "100%", left: 0, width: 280, p: 3, bg: "canvas.overlay", border: "1px solid", borderColor: "border.default", borderRadius: 2, boxShadow: "shadow.large", zIndex: 10, children: [_jsx(Text, { as: "p", sx: { fontSize: 1, mb: 2 }, children: hearing.where === 'server'
+                            ? SERVER_CONSENT_SENTENCE
+                            : CONSENT_SENTENCE }), _jsxs(Box, { display: "flex", gap: 2, justifyContent: "flex-end", children: [_jsx(Button, { size: "small", onClick: () => setAsking(false), children: "Not now" }), _jsx(Button, { size: "small", variant: "primary", "data-hands-free-consent-yes": "", onClick: () => {
+                                    consent(key);
+                                    setAsking(false);
+                                    setProblem(undefined);
+                                    setPaused(false);
+                                    setOn(true);
+                                }, children: "Use the microphone" })] })] })) : null] }));
+}
+export default HandsFree;

package/lib/voice/VoiceInput.d.ts

@@ -18,7 +18,7 @@
  * @module voice/VoiceInput
  */
 import type { JSX } from 'react';
-import type { DeviceHearing } from './hearing';
+import type { Hearing } from './hearing';
 import type { Transcript } from './types';
 /** What the microphone is doing, said in words (VO-67, VO-68). */
 export type ListeningState = 'idle' | 'asking' | 'loading' | 'listening' | 'hearing';
@@ -27,7 +27,7 @@ export declare const INTERIM_MS = 700;
 /** The longest a microphone listens in one go, in ms: then it stops by itself. */
 export declare const LONGEST_MS = 60000;
 export interface VoiceInputProps {
-    hearing: DeviceHearing;
+    hearing: Hearing;
     /** The language listened to, BCP 47. */
     language: string;
     /** What was said, once heard. */

package/lib/voice/VoiceInput.js

@@ -5,7 +5,7 @@ import { Box } from '@datalayer/primer-addons';
 import { MicrophoneIcon } from '@datalayer/icons-react';
 import { MuteIcon } from '@primer/octicons-react';
 import { MicrophoneRefused, openMicrophone } from './capture';
-import { CONSENT_SENTENCE, consent, consented } from './consent';
+import { CONSENT_SENTENCE, SERVER_CONSENT_SENTENCE, consent, consentKeyFor, consented, } from './consent';
 const WORDS = {
     idle: '',
     asking: '',
@@ -49,7 +49,7 @@ export function VoiceInput({ hearing, language, onTranscript, onInterim, silence
         if (disabled || stateRef.current !== 'idle') {
             return;
         }
-        if (!consented(consentKey)) {
+        if (!consented(consentKeyFor(consentKey, hearing.where))) {
             setState('asking');
             return;
         }
@@ -277,8 +277,10 @@ export function VoiceInput({ hearing, language, onTranscript, onInterim, silence
                                 void start();
                             }
                         }
-                    } }) }), state === 'asking' && (_jsxs(Box, { role: "dialog", "aria-label": "Talk instead of typing", "data-voice-consent": "", position: "absolute", bottom: "100%", right: 0, mb: 2, width: 280, p: 3, bg: "canvas.overlay", border: "1px solid", borderColor: "border.default", borderRadius: 2, boxShadow: "shadow.large", zIndex: 10, children: [_jsx(Text, { as: "p", sx: { fontSize: 1, mb: 2 }, children: CONSENT_SENTENCE }), _jsxs(Box, { display: "flex", gap: 2, justifyContent: "flex-end", children: [_jsx(Button, { size: "small", onClick: () => setState('idle'), children: "Not now" }), _jsx(Button, { size: "small", variant: "primary", "data-voice-consent-yes": "", onClick: () => {
-                                    consent(consentKey);
+                    } }) }), state === 'asking' && (_jsxs(Box, { role: "dialog", "aria-label": "Talk instead of typing", "data-voice-consent": "", position: "absolute", bottom: "100%", right: 0, mb: 2, width: 280, p: 3, bg: "canvas.overlay", border: "1px solid", borderColor: "border.default", borderRadius: 2, boxShadow: "shadow.large", zIndex: 10, children: [_jsx(Text, { as: "p", sx: { fontSize: 1, mb: 2 }, children: hearing.where === 'server'
+                            ? SERVER_CONSENT_SENTENCE
+                            : CONSENT_SENTENCE }), _jsxs(Box, { display: "flex", gap: 2, justifyContent: "flex-end", children: [_jsx(Button, { size: "small", onClick: () => setState('idle'), children: "Not now" }), _jsx(Button, { size: "small", variant: "primary", "data-voice-consent-yes": "", onClick: () => {
+                                    consent(consentKeyFor(consentKey, hearing.where));
                                     // Asked, and said yes: listen now, without another press.
                                     stateRef.current = 'idle';
                                     setState('idle');

package/lib/voice/VoiceMenu.js

@@ -20,6 +20,7 @@ export function VoiceMenu({ voice, answers, disabled = false, }) {
     const [devices, setDevices] = useState([]);
     const choose = (key, value) => keep(withVoiceSetting(settings, key, value, catalogueVoiceSpeaks));
     const onDevice = Boolean(settings.onDeviceOnly);
+    const detect = Boolean(settings.detectLanguage);
     const voices = voicesSpeaking(voice.language);
     const named = devices.filter(device => device.label);
     return (_jsxs(ActionMenu, { onOpenChange: open => {
@@ -27,6 +28,6 @@ export function VoiceMenu({ voice, answers, disabled = false, }) {
                 void microphones().then(setDevices);
             }
         }, children: [_jsx(ActionMenu.Anchor, { children: _jsx(IconButton, { icon: GearIcon, size: "small", variant: "invisible", "aria-label": "Voice settings", "data-voice-menu": "", disabled: disabled }) }), _jsx(ActionMenu.Overlay, { width: "medium", children: _jsxs(ActionList, { children: [_jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Language" }), voiceLanguages().map(language => (_jsx(ActionList.Item, { selected: language.value === voice.language, onSelect: () => choose('language', language.value), "data-voice-language": language.value, children: language.label }, language.value)))] }), answers && !onDevice ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Answers heard" }), OUTPUT_CHOICES.map(output => (_jsx(ActionList.Item, { selected: output.value === voice.output, onSelect: () => choose('output', output.value), "data-voice-output": output.value, children: output.label }, output.value)))] }), voices.length > 0 ? (_jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Voice" }), voices.map(spoken => (_jsx(ActionList.Item, { selected: spoken.id === voice.voice, onSelect: () => choose('voice', spoken.id), "data-voice-voice": spoken.id, children: spoken.name }, spoken.id)))] })) : null, _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Speed" }), RATE_CHOICES.map(rate => (_jsx(ActionList.Item, { selected: rate.value === (voice.rate ?? 1), onSelect: () => choose('rate', rate.value), "data-voice-rate": rate.value, children: rate.label }, rate.value)))] }), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Loudness" }), VOLUME_CHOICES.map(volume => (_jsx(ActionList.Item, { selected: volume.value === (voice.volume ?? 1), onSelect: () => choose('volume', volume.value), "data-voice-volume": volume.value, children: volume.label }, volume.value)))] })] })) : null, named.length > 1 ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Microphone" }), named.map(device => (_jsx(ActionList.Item, { selected: device.deviceId ===
-                                                (voice.microphoneId ?? named[0].deviceId), onSelect: () => choose('microphoneId', device.deviceId), "data-voice-microphone": device.deviceId, children: device.label }, device.deviceId)))] })] })) : null, _jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "multiple", children: [_jsxs(ActionList.Item, { selected: Boolean(voice.sendWhatISay), onSelect: () => choose('sendWhatISay', !voice.sendWhatISay), "data-voice-send-what-i-say": "", children: ["Send what I say", _jsx(ActionList.Description, { variant: "block", children: "Sent as soon as it is heard, not left in the box." })] }), _jsxs(ActionList.Item, { selected: onDevice, onSelect: () => choose('onDeviceOnly', !onDevice), "data-voice-on-device": "", children: ["Keep speech on this device", _jsx(ActionList.Description, { variant: "block", children: "Nothing is sent to Datalayer\u2019s speech service: you are heard in the browser, answers are not spoken." })] })] }), Object.keys(settings).length > 0 ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsx(ActionList.Item, { onSelect: () => keep({}), "data-voice-settings-reset": "", children: "Use the application\u2019s" })] })) : null] }) })] }));
+                                                (voice.microphoneId ?? named[0].deviceId), onSelect: () => choose('microphoneId', device.deviceId), "data-voice-microphone": device.deviceId, children: device.label }, device.deviceId)))] })] })) : null, _jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "multiple", children: [_jsxs(ActionList.Item, { selected: Boolean(voice.sendWhatISay), onSelect: () => choose('sendWhatISay', !voice.sendWhatISay), "data-voice-send-what-i-say": "", children: ["Send what I say", _jsx(ActionList.Description, { variant: "block", children: "Sent as soon as it is heard, not left in the box." })] }), voice.speechUrl && !onDevice ? (_jsxs(ActionList.Item, { selected: detect, onSelect: () => choose('detectLanguage', !detect), "data-voice-detect": "", children: ["Detect the language I speak", _jsx(ActionList.Description, { variant: "block", children: "Heard on Datalayer\u2019s servers by Whisper, which finds the language; nothing is kept there." })] })) : null, _jsxs(ActionList.Item, { selected: onDevice, onSelect: () => choose('onDeviceOnly', !onDevice), "data-voice-on-device": "", children: ["Keep speech on this device", _jsx(ActionList.Description, { variant: "block", children: "Nothing is sent to Datalayer\u2019s speech service: you are heard in the browser, answers are not spoken." })] })] }), Object.keys(settings).length > 0 ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsx(ActionList.Item, { onSelect: () => keep({}), "data-voice-settings-reset": "", children: "Use the application\u2019s" })] })) : null] }) })] }));
 }
 export default VoiceMenu;

package/lib/voice/chatVoice.d.ts

@@ -35,7 +35,8 @@ export interface ChatVoicePlace {
  * - **Listening**: a microphone in the composer — click it and speak, the
  *   words written in the box as they are heard, stopped by a second click
  *   or when the person stops speaking; or held, and let go (VO-10, VO-11).
- *   `hands_free` (VO-12) is not built: `checkAppspec` refuses it.
+ *   `hands_free` (VO-12): the microphone, and *Talk with it* beside it; it
+ *   implies answers said as they are written.
  * - **Speaking**: its answers said by Datalayer's speech service, on request
  *   or always (VO-20, VO-21); refused here when its host says no service.
  */
@@ -44,10 +45,11 @@ export declare function chatVoiceOf(spec: AppVoiceSpec | undefined, place: ChatV
  * A chat's voice asked of the speech service as the chat itself speaks to
  * its runtime (VO-37, VO-19): the token its host gave, else the chat's own —
  * a visitor's token minted from ai-inference, an embed's token, or the
- * person's — and the deployment it runs, which an embed's token is checked
- * against. One token for the conversation and its voice, never a second.
+ * person's. One token for the conversation and its voice, never a second.
+ * An embed's deployment is the one its token names (decision 16): the page
+ * says none.
  */
-export declare function voiceAskedAs(voice: ChatVoice | undefined, token: string | undefined, deployment: string | undefined): ChatVoice | undefined;
+export declare function voiceAskedAs(voice: ChatVoice | undefined, token: string | undefined): ChatVoice | undefined;
 /** A chat's voice with every place said: what the chat reads. */
 export type SettledVoice = Required<Pick<ChatVoice, 'language' | 'input' | 'output' | 'voice' | 'modelsUrl' | 'engines' | 'silenceMs'>> & Omit<ChatVoice, 'language' | 'input' | 'output' | 'voice' | 'modelsUrl' | 'engines' | 'silenceMs'>;
 /** How long a silence after speech stops a microphone clicked on, when unsaid (VO-10). */

package/lib/voice/chatVoice.js

@@ -35,7 +35,8 @@ export const DATALAYER_VOICE_ENGINES = {
  * - **Listening**: a microphone in the composer — click it and speak, the
  *   words written in the box as they are heard, stopped by a second click
  *   or when the person stops speaking; or held, and let go (VO-10, VO-11).
- *   `hands_free` (VO-12) is not built: `checkAppspec` refuses it.
+ *   `hands_free` (VO-12): the microphone, and *Talk with it* beside it; it
+ *   implies answers said as they are written.
  * - **Speaking**: its answers said by Datalayer's speech service, on request
  *   or always (VO-20, VO-21); refused here when its host says no service.
  */
@@ -49,34 +50,32 @@ export function chatVoiceOf(spec, place) {
     const language = spec.language || place.language || 'en-US';
     return {
         language,
-        input: spec.input === 'off' ? 'off' : 'push_to_talk',
-        output: spec.output,
+        input: spec.input,
+        // Hands-free implies the answers heard as they are written (VO-12).
+        output: spec.input === 'hands_free' ? 'always' : spec.output,
         voice: spec.voice || voiceFor(language)?.id || '',
         modelsUrl: place.modelsUrl || DATALAYER_SPEECH_MODELS_URL,
         engines: DATALAYER_VOICE_ENGINES,
         speechUrl: place.speechUrl,
         token: place.token,
         consentKey: place.consentKey,
+        where: spec.where ?? 'auto',
     };
 }
 /**
  * A chat's voice asked of the speech service as the chat itself speaks to
  * its runtime (VO-37, VO-19): the token its host gave, else the chat's own —
  * a visitor's token minted from ai-inference, an embed's token, or the
- * person's — and the deployment it runs, which an embed's token is checked
- * against. One token for the conversation and its voice, never a second.
+ * person's. One token for the conversation and its voice, never a second.
+ * An embed's deployment is the one its token names (decision 16): the page
+ * says none.
  */
-export function voiceAskedAs(voice, token, deployment) {
+export function voiceAskedAs(voice, token) {
     if (!voice) {
         return undefined;
     }
     const asked = voice.token || token;
-    const runs = voice.deployment || deployment;
-    return {
-        ...voice,
-        ...(asked ? { token: asked } : {}),
-        ...(runs ? { deployment: runs } : {}),
-    };
+    return asked ? { ...voice, token: asked } : { ...voice };
 }
 /** How long a silence after speech stops a microphone clicked on, when unsaid (VO-10). */
 export const SILENCE_MS = 1500;

package/lib/voice/consent.d.ts

@@ -4,3 +4,11 @@ export declare function consented(app: string): boolean;
 export declare function consent(app: string): void;
 /** The sentence said before the first use: where the audio goes (VO-61). */
 export declare const CONSENT_SENTENCE = "What you say is transcribed on this device: the sound never leaves it, and is not kept. The words go in the message box, for you to send.";
+/**
+ * The sentence said before the first use when what is said is heard on
+ * Datalayer's servers (VO-34, VO-61): asked apart from the device's, since
+ * a yes to one is not a yes to the other.
+ */
+export declare const SERVER_CONSENT_SENTENCE = "What you say is sent to Datalayer\u2019s servers to be transcribed: the sound and the words are not kept there. The words go in the message box, for you to send.";
+/** Whose consent is remembered for hearing in a place: the application's, apart for the server. */
+export declare function consentKeyFor(app: string, where: 'device' | 'server'): string;

package/lib/voice/consent.js

@@ -32,3 +32,13 @@ export function consent(app) {
 }
 /** The sentence said before the first use: where the audio goes (VO-61). */
 export const CONSENT_SENTENCE = 'What you say is transcribed on this device: the sound never leaves it, and is not kept. The words go in the message box, for you to send.';
+/**
+ * The sentence said before the first use when what is said is heard on
+ * Datalayer's servers (VO-34, VO-61): asked apart from the device's, since
+ * a yes to one is not a yes to the other.
+ */
+export const SERVER_CONSENT_SENTENCE = 'What you say is sent to Datalayer’s servers to be transcribed: the sound and the words are not kept there. The words go in the message box, for you to send.';
+/** Whose consent is remembered for hearing in a place: the application's, apart for the server. */
+export function consentKeyFor(app, where) {
+    return where === 'server' ? `${app}:server` : app;
+}

package/lib/voice/handsFree.d.ts

@@ -0,0 +1,58 @@
+/**
+ * A conversation by voice, hands-free (VOICE.md VO-12, VO-13): what the
+ * composer is doing at every moment, and when a turn ends or the person
+ * speaks over the agent. Pure: the component (`HandsFree`) keeps the
+ * microphone and the clock, this decides.
+ *
+ * - **Listening**: the microphone is open; the turn ends after
+ *   `turnSilenceMs` of silence once the person has spoken (700 ms unless
+ *   the application or the person says otherwise), and is sent.
+ * - **Thinking**: the turn was sent and the agent is answering, nothing
+ *   said yet.
+ * - **Speaking**: the answer is being said. The microphone stays open:
+ *   speech heard now is barge-in (VO-13) — the voice stops at once, the run
+ *   is stopped when the answer is still being written, and what was heard
+ *   starts the next turn.
+ * - **Paused**: the person paused it (`Esc`, *Pause*), or nothing was said
+ *   for `IDLE_PAUSE_MS`; nothing is heard until resumed.
+ *
+ * @module voice/handsFree
+ */
+/** What hands-free is doing, shown at all times (VO-12). */
+export type HandsFreeState = 'off' | 'listening' | 'thinking' | 'speaking' | 'paused';
+/** The word for each state, as the composer shows it. */
+export declare const HANDS_FREE_WORDS: Record<HandsFreeState, string>;
+/** The silence after speech that ends a turn, in ms, unless said otherwise (VO-12). */
+export declare const TURN_SILENCE_MS = 700;
+/** Listening with nothing said for this long pauses it: a microphone is not left open on an empty room. */
+export declare const IDLE_PAUSE_MS: number;
+/** What decides the state. */
+export interface HandsFreeInputs {
+    /** The toggle is on. */
+    on: boolean;
+    /** The person paused it, or the room went quiet. */
+    paused: boolean;
+    /** The agent's answer is being said. */
+    speaking: boolean;
+    /** A run is in progress: the agent is answering. */
+    running: boolean;
+}
+/** The state for what is known: speaking wins over thinking, pausing over both. */
+export declare function handsFreeState(inputs: HandsFreeInputs): HandsFreeState;
+/** What the microphone heard so far, as the hearing says it (`HeardSoFar`). */
+export interface HeardSoFarLike {
+    spoke: boolean;
+    silentMs: number;
+}
+/**
+ * Whether the turn ended: the person spoke, and has been silent for the
+ * turn's silence since (VO-12).
+ */
+export declare function turnEnded(heard: HeardSoFarLike, turnSilenceMs?: number): boolean;
+/**
+ * Whether the person spoke over the agent (VO-13): speech found since the
+ * agent began answering, while it thinks or speaks. The microphone is opened
+ * anew when the agent begins, so what it holds is the person's, not the
+ * answer's echo — the browser's echo cancellation keeps that out.
+ */
+export declare function bargedIn(state: HandsFreeState, heard: HeardSoFarLike): boolean;

package/lib/voice/handsFree.js

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/** The word for each state, as the composer shows it. */
+export const HANDS_FREE_WORDS = {
+    off: 'Talk with it',
+    listening: 'Listening…',
+    thinking: 'Thinking…',
+    speaking: 'Speaking…',
+    paused: 'Paused',
+};
+/** The silence after speech that ends a turn, in ms, unless said otherwise (VO-12). */
+export const TURN_SILENCE_MS = 700;
+/** Listening with nothing said for this long pauses it: a microphone is not left open on an empty room. */
+export const IDLE_PAUSE_MS = 5 * 60_000;
+/** The state for what is known: speaking wins over thinking, pausing over both. */
+export function handsFreeState(inputs) {
+    if (!inputs.on) {
+        return 'off';
+    }
+    if (inputs.paused) {
+        return 'paused';
+    }
+    if (inputs.speaking) {
+        return 'speaking';
+    }
+    if (inputs.running) {
+        return 'thinking';
+    }
+    return 'listening';
+}
+/**
+ * Whether the turn ended: the person spoke, and has been silent for the
+ * turn's silence since (VO-12).
+ */
+export function turnEnded(heard, turnSilenceMs = TURN_SILENCE_MS) {
+    return heard.spoke && heard.silentMs >= turnSilenceMs;
+}
+/**
+ * Whether the person spoke over the agent (VO-13): speech found since the
+ * agent began answering, while it thinks or speaks. The microphone is opened
+ * anew when the agent begins, so what it holds is the person's, not the
+ * answer's echo — the browser's echo cancellation keeps that out.
+ */
+export function bargedIn(state, heard) {
+    return (state === 'speaking' || state === 'thinking') && heard.spoke;
+}

package/lib/voice/hearing.d.ts

@@ -22,15 +22,30 @@ export interface HeardSoFar {
     /** How long it has been silent since the speech last ended, in ms. */
     silentMs: number;
 }
+/**
+ * What hears a press, wherever it runs (§5): in the page (`DeviceHearing`)
+ * or on Datalayer's servers (`ServerHearing`).
+ */
+export interface Hearing {
+    /** Where the sound is heard: what the consent says, and a transcript's `where`. */
+    readonly where: 'device' | 'server';
+    /** Load what hears a language, saying how far it has come. */
+    ready(language: string, progress?: HearingProgress): Promise<void>;
+    /** What was said so far while the microphone stays open (VO-11). */
+    hearSoFar(audio: Float32Array, language: string): Promise<HeardSoFar>;
+    /** What was said in a press, or `undefined` when nothing was. */
+    hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
+}
 /** Joins the speech a VAD found, in order. */
 export declare function joinSegments(segments: Float32Array[]): Float32Array;
 /**
  * The page's hearing: loaded once, on first use, from Datalayer's origin
  * (`modelsUrl`), every file checked against its pin.
  */
-export declare class DeviceHearing {
+export declare class DeviceHearing implements Hearing {
     private readonly modelsUrl;
     private readonly engines;
+    readonly where: "device";
     private recognizers;
     private vad?;
     constructor(modelsUrl: string, engines: VoiceEngines);
@@ -40,6 +55,13 @@ export declare class DeviceHearing {
     sizeFor(language: string): number;
     /** Load what hears a language, saying how far it has come. */
     ready(language: string, progress?: HearingProgress): Promise<void>;
+    /** Load only what finds speech (Silero VAD): what hearing on the server needs here. */
+    readyToFindSpeech(progress?: HearingProgress): Promise<void>;
+    /** The speech in what was heard, and how long it has been silent since it last ended, in ms. */
+    findSpeech(audio: Float32Array): Promise<{
+        speech: Float32Array;
+        silentMs: number;
+    }>;
     private recognizer;
     private activity;
     /** The speech Silero VAD finds in what was heard, and where the last ended, in ms. */

package/lib/voice/hearing.js

@@ -45,6 +45,7 @@ export function joinSegments(segments) {
 export class DeviceHearing {
     modelsUrl;
     engines;
+    where = 'device';
     recognizers = new Map();
     vad;
     constructor(modelsUrl, engines) {
@@ -67,6 +68,22 @@ export class DeviceHearing {
     async ready(language, progress) {
         await Promise.all([this.recognizer(language, progress), this.activity()]);
     }
+    /** Load only what finds speech (Silero VAD): what hearing on the server needs here. */
+    async readyToFindSpeech(progress) {
+        const total = downloadSize('silero-vad');
+        progress?.(0, total);
+        await this.activity();
+        progress?.(total, total);
+    }
+    /** The speech in what was heard, and how long it has been silent since it last ended, in ms. */
+    async findSpeech(audio) {
+        const { speech, lastEnd } = await this.speechIn(audio);
+        const heardMs = (audio.length / HEARING_RATE) * 1000;
+        return {
+            speech,
+            silentMs: lastEnd === undefined ? 0 : Math.max(0, heardMs - lastEnd),
+        };
+    }
     recognizer(language, progress) {
         const model = this.modelFor(language);
         let loading = this.recognizers.get(model.id);
@@ -163,9 +180,7 @@ export class DeviceHearing {
      * since the speech ended — what stops a click to talk (VO-10).
      */
     async hearSoFar(audio, language) {
-        const { speech, lastEnd } = await this.speechIn(audio);
-        const heardMs = (audio.length / HEARING_RATE) * 1000;
-        const silentMs = lastEnd === undefined ? 0 : Math.max(0, heardMs - lastEnd);
+        const { speech, silentMs } = await this.findSpeech(audio);
         if (speech.length < HEARING_RATE * 0.2) {
             return { text: '', spoke: false, silentMs };
         }