@datalayer/agent-runtimes 1.5.28 → 1.5.30

package/lib/apps/apps/appspec.d.ts

@@ -99,11 +99,6 @@ export declare const PAP_PROFILE: PapProfile;
  * (`pap_profile_problem`); `undefined` when it is one Datalayer speaks.
  */
 export declare function papProfileProblem(value: unknown): string | undefined;
-/**
- * Why `interface.voice.input: hands_free` is refused until VOICE.md VO-12 is
- * built (decided 14 October 2026): never taken for click-to-talk.
- */
-export declare const HANDS_FREE_NOT_BUILT = "hands_free is not built yet (VOICE.md VO-12): say push_to_talk \u2014 click the microphone or hold it, speak, and it stops when let go or once the person stops speaking";
 /** An application's voice (VOICE.md VO-41): off unless said. */
 export declare const DEFAULT_VOICE: AppVoiceSpec;
 /** What a page's output may be drawn with (LOOP P-05), and the property of each that shows its value. */

package/lib/apps/apps/appspec.js

@@ -713,11 +713,6 @@ function parseSurface(data) {
         composedAt: text(data.composed_at),
     };
 }
-/**
- * Why `interface.voice.input: hands_free` is refused until VOICE.md VO-12 is
- * built (decided 14 October 2026): never taken for click-to-talk.
- */
-export const HANDS_FREE_NOT_BUILT = 'hands_free is not built yet (VOICE.md VO-12): say push_to_talk — click the microphone or hold it, speak, and it stops when let go or once the person stops speaking';
 /** An application's voice (VOICE.md VO-41): off unless said. */
 export const DEFAULT_VOICE = {
     enabled: false,
@@ -731,7 +726,7 @@ function parseVoice(data) {
     const voice = isData(data) ? data : {};
     return {
         enabled: voice.enabled === true,
-        input: oneOf(voice.input, ['off', 'push_to_talk'], DEFAULT_VOICE.input),
+        input: oneOf(voice.input, ['off', 'push_to_talk', 'hands_free'], DEFAULT_VOICE.input),
         output: oneOf(voice.output, ['off', 'on_request', 'always'], DEFAULT_VOICE.output),
         voice: text(voice.voice),
         language: text(voice.language),

package/lib/apps/apps/checks.js

@@ -39,7 +39,7 @@ import { getTeamSpec } from '../../specs/teams';
 import { getBackendToolSpec } from '../../specs/backendTools';
 import { getTrack } from '../../specs/tracks';
 import { pluginsOffSetupNotes } from '../plugins/canvas-blocks';
-import { APP_THEME_MODES, APP_THEME_VARIANTS, HANDS_FREE_NOT_BUILT, isAssistantCharacterId, PAGE_OUTPUT_SHOWS, parseAppspec, } from './appspec';
+import { APP_THEME_MODES, APP_THEME_VARIANTS, isAssistantCharacterId, PAGE_OUTPUT_SHOWS, parseAppspec, } from './appspec';
 import { classesOf, splitRef, toolBehaviours } from './rules';
 import { COMMAND_INPUT, COMMAND_NAME, MODE_ID } from './composer';
 import { HOST_NAME, HOST_USERS, hostToolsOf } from './hostTools';
@@ -685,7 +685,7 @@ const ENUMS = {
     visibility: ['private', 'invited', 'organization', 'link', 'public'],
     mode: ['inline', 'bubble', 'panel', 'assistant'],
     balloon: ['history', 'current'],
-    voiceInput: ['off', 'push_to_talk'],
+    voiceInput: ['off', 'push_to_talk', 'hands_free'],
     voiceOutput: ['off', 'on_request', 'always'],
     voiceWhere: ['auto', 'device', 'server'],
     themeVariant: APP_THEME_VARIANTS,
@@ -843,14 +843,9 @@ export function documentShapeProblems(document) {
         if (ui.assistant !== undefined && !isAssistantCharacterId(ui.assistant)) {
             at('interface.assistant', 'names a character by the id a plugin contributes it under, lowercase words joined by a hyphen: `paperclip`, `acme-owl`');
         }
-        // Its voice (VOICE.md VO-41); hands-free refused until it is built (VO-12).
+        // Its voice (VOICE.md VO-41, VO-12).
         mapping(ui.voice, 'interface.voice', voice => {
-            if (voice.input === 'hands_free') {
-                at('interface.voice.input', HANDS_FREE_NOT_BUILT);
-            }
-            else {
-                oneOf(voice.input, ENUMS.voiceInput, 'interface.voice.input');
-            }
+            oneOf(voice.input, ENUMS.voiceInput, 'interface.voice.input');
             oneOf(voice.output, ENUMS.voiceOutput, 'interface.voice.output');
             oneOf(voice.where, ENUMS.voiceWhere, 'interface.voice.where');
         });

package/lib/apps/embed/element.js

@@ -383,7 +383,7 @@ export function defineDatalayerAppElement(options = {}) {
                 // A public application, as a visitor without an account (D-07):
                 // an example kept warm or an address open to visitors, run on the
                 // visitors' runtime; a decision framed; a refusal said.
-                const visitor = await readVisitorApp({ api: this.api(), app });
+                const visitor = await readVisitorApp({ api: this.agents(), app });
                 if (visitor.kind === 'decision') {
                     return { kind: 'frame', src: frameSrcOf(this.origin(), visitor.app) };
                 }
@@ -408,11 +408,22 @@ export function defineDatalayerAppElement(options = {}) {
             }
             return this.spoken;
         }
-        /** Datalayer's services: the `api` attribute, else the platform's, else prod1's. */
+        /**
+         * ai-agents, where a visitor's address is read and beside which the
+         * visitors' runtime and ai-inference answer: the `api` attribute, else
+         * the platform's, else r1's. Not Spacer's address (`api()`), which lives
+         * on another plane.
+         */
+        agents() {
+            return (this.getAttribute('api') ||
+                coreStore.getState().configuration?.aiAgentsUrl ||
+                DEFAULT_API).replace(/\/+$/, '');
+        }
+        /** Spacer, where an application is read with its embed token: the `api` attribute, else the platform's, else prod1's. */
         api() {
             return (this.getAttribute('api') ||
                 coreStore.getState().configuration?.spacerUrl ||
-                DEFAULT_API).replace(/\/+$/, '');
+                'https://prod1.datalayer.run').replace(/\/+$/, '');
         }
         async load() {
             const turn = ++this.loading;

package/lib/apps/embed/embedConfig.d.ts

@@ -32,7 +32,7 @@ export declare const EMBED_ATTRIBUTES: {
     readonly theme: "light, dark or auto (the visitor’s system); auto by default.";
     readonly font: "A CSS font family for the application’s text; Inter, then the system’s sans-serif, by default.";
     readonly server: "An agent-runtimes server the application’s agent runs on; a Datalayer runtime by default.";
-    readonly api: "Datalayer’s API, where an application is read with its token; the platform’s by default.";
+    readonly api: "Datalayer’s API: where an application is read with its token (Spacer), and where a visitor’s address is read (ai-agents); the platform’s by default.";
     readonly height: "The height of an inline application, in pixels; 640 by default.";
     readonly resume: "true or false: whether a visit’s conversation is picked up again after the page reloads, kept in the page’s storage; true by default.";
     readonly language: "The language the visitor reads the application in, as BCP 47 tags it (fr, pt-BR): its translation into it, and the chat’s own words; the visitor’s browser’s by default.";

package/lib/apps/embed/embedConfig.js

@@ -23,7 +23,7 @@ export const EMBED_ATTRIBUTES = {
     theme: 'light, dark or auto (the visitor’s system); auto by default.',
     font: 'A CSS font family for the application’s text; Inter, then the system’s sans-serif, by default.',
     server: 'An agent-runtimes server the application’s agent runs on; a Datalayer runtime by default.',
-    api: 'Datalayer’s API, where an application is read with its token; the platform’s by default.',
+    api: 'Datalayer’s API: where an application is read with its token (Spacer), and where a visitor’s address is read (ai-agents); the platform’s by default.',
     height: 'The height of an inline application, in pixels; 640 by default.',
     resume: 'true or false: whether a visit’s conversation is picked up again after the page reloads, kept in the page’s storage; true by default.',
     language: 'The language the visitor reads the application in, as BCP 47 tags it (fr, pt-BR): its translation into it, and the chat’s own words; the visitor’s browser’s by default.',

package/lib/apps/embed/visitorApp.d.ts

@@ -42,8 +42,15 @@ export declare const VISITORS_PATH = "/api/runtime-visitors";
 export declare const VISITOR_EXAMPLES: readonly string[];
 /** The key of an application at its address, as a visitor's token names it. */
 export declare const AT_PREFIX = "at:";
-/** Datalayer's services, where the element names none (`api`). */
-export declare const DEFAULT_API = "https://prod1.datalayer.run";
+/**
+ * Datalayer's services for a visitor, where the element names none (`api`)
+ * and the platform's configuration says none: ai-agents, where an address is
+ * read, and beside it the visitors' runtime and ai-inference, which mints the
+ * visitor's token. Not Spacer's address, which reads an application with its
+ * embed token: the two live on different planes (ai-agents on r1, Spacer on
+ * prod1), found 15 October 2026 on datalayer.ai's embed.
+ */
+export declare const DEFAULT_API = "https://r1.datalayer.run";
 /** An application's id on Datalayer: a ULID. */
 export declare const isAppUid: (value: string) => boolean;
 /** What `app` with no token resolves to. */

package/lib/apps/embed/visitorApp.js

@@ -14,8 +14,15 @@ export const VISITOR_EXAMPLES = [
 ];
 /** The key of an application at its address, as a visitor's token names it. */
 export const AT_PREFIX = 'at:';
-/** Datalayer's services, where the element names none (`api`). */
-export const DEFAULT_API = 'https://prod1.datalayer.run';
+/**
+ * Datalayer's services for a visitor, where the element names none (`api`)
+ * and the platform's configuration says none: ai-agents, where an address is
+ * read, and beside it the visitors' runtime and ai-inference, which mints the
+ * visitor's token. Not Spacer's address, which reads an application with its
+ * embed token: the two live on different planes (ai-agents on r1, Spacer on
+ * prod1), found 15 October 2026 on datalayer.ai's embed.
+ */
+export const DEFAULT_API = 'https://r1.datalayer.run';
 /** An application's id on Datalayer: a ULID. */
 export const isAppUid = (value) => /^[0-9A-HJKMNP-TV-Z]{26}$/.test(value);
 /** An address, as ai-agents spells one. */

package/lib/apps/plugins/chat/ChatView.js

@@ -1700,7 +1700,10 @@ export default function ChatView({ workspace }) {
         footerExtras: promptActions.length > 0 || (voice && voice.input !== 'off') ? (_jsxs(_Fragment, { children: [promptActions.length > 0 ? (_jsx(ReactorSlot, { slot: LoopSlots.promptAction, props: { workspace } })) : null, voice && voice.input !== 'off' ? (_jsx(ComposerVoice, { voice: voice, input: draft, setInput: setDraft, disabled: chatDisabled, 
                     // The chat's own speaker, never a second one: the microphone
                     // opening stops it, and *Stop speaking* while it speaks.
-                    speaking: speaking, onStopSpeaking: () => controlsRef.current?.stopSpeaking?.(), onInterimChange: setVoiceInterim, 
+                    speaking: speaking, onStopSpeaking: () => controlsRef.current?.stopSpeaking?.(), 
+                    // Hands-free (VO-12, VO-13): the run in progress, stopped when
+                    // the person speaks over it.
+                    running: busy, onStopRun: () => controlsRef.current?.stop?.(), onInterimChange: setVoiceInterim, 
                     // The menu offers when answers are heard where the
                     // application lets them be (VO-21, VO-30).
                     answers: Boolean(config?.voice?.output &&

package/lib/chat/base/ChatBase.js

@@ -2968,6 +2968,8 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
     const ownVoice = useVoiceSpeaker(settled, !hostSpeaks);
     useSpokenAnswers(displayItems, isLoading, !hostSpeaks && settled?.output === 'always', ownVoice.speaker);
     const speakingNow = voiceSpeaking || ownVoice.state.speaking;
+    // Answers spoken over, hands-free (VOICE.md VO-13): marked *interrupted*.
+    const [interruptedIds, setInterruptedIds] = useState(() => new Set());
     const stopSpeaking = onStopSpeaking ?? (ownVoice.speaker ? ownVoice.stop : undefined);
     const readAnswerAloud = onReadAloud ?? (ownVoice.speaker ? ownVoice.read : undefined);
     const handleSend = useCallback(async (messageOverride, 
@@ -3724,10 +3726,10 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
             .reverse()
             .find((item) => !isToolCallMessage(item) && item.role === 'assistant')?.id
         : undefined;
-    const readAloudFooter = readAnswerAloud
+    const readAloudFooter = readAnswerAloud || interruptedIds.size
         ? (message) => {
             const text = getMessageText(message);
-            return readsAloud(message, text, writingId) ? (_jsx(ReadAloud, { onRead: () => readAnswerAloud(text) })) : null;
+            return (_jsxs(_Fragment, { children: [interruptedIds.has(message.id) ? (_jsx(Text, { sx: { fontSize: 0, color: 'fg.muted', fontStyle: 'italic' }, "data-interrupted": "", children: "interrupted" })) : null, readAnswerAloud && readsAloud(message, text, writingId) ? (_jsx(ReadAloud, { onRead: () => readAnswerAloud(text) })) : null] }));
         }
         : undefined;
     const messagesContent = children ? (children) : (_jsxs(Box, { display: "flex", flexDirection: "column", minHeight: 0, 
@@ -3745,7 +3747,14 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
                 // so every chat with a headless notebook gets the same transcript.
                 notebookToolSurfacesId: notebookToolSurfacesId, approvalConfig: approvalConfig, messagesEndRef: messagesEndRef, onRespond: handleRespond, showTurnFooters: showTurnFooter, renderMessageFooter: readAloudFooter, agentUsage: agentUsage, onRemoveItems: handleRemoveItems, mcpServers: filteredMcpServers, emptyContent: launching ? null : (_jsx(ChatEmptyState, { emptyState: emptyState, brandIcon: brandIcon, description: resolvedDescription, suggestions: resolvedSuggestions, submitOnSuggestionClick: submitOnSuggestionClick, onSuggestionSubmit: handleSuggestionSubmit, onSuggestionFill: handleSuggestionFill })) }), trailingContent] }));
     // The microphone, in the composer's footer (VO-10, VO-11).
-    const voiceControl = settled && settled.input !== 'off' ? (_jsx(ComposerVoice, { voice: settled, input: input, setInput: setInput, disabled: disabled || launching, speaking: speakingNow, onStopSpeaking: stopSpeaking, onInterimChange: setVoiceInterim, 
+    const voiceControl = settled && settled.input !== 'off' ? (_jsx(ComposerVoice, { voice: settled, input: input, setInput: setInput, disabled: disabled || launching, speaking: speakingNow, onStopSpeaking: stopSpeaking, 
+        // Hands-free (VO-12, VO-13): the run in progress, stopped when the
+        // person speaks over it, and the answer marked interrupted.
+        running: isLoading, onStopRun: handleStop, onInterrupted: () => {
+            if (writingId) {
+                setInterruptedIds(ids => new Set([...ids, writingId]));
+            }
+        }, onInterimChange: setVoiceInterim, 
         // The menu offers when answers are heard where the application
         // lets them be (VO-21, VO-30).
         answers: Boolean(voice?.output && voice.output !== 'off' && voice.speechUrl), onSpoken: metadata => {

package/lib/specs/appspecSchema.js

@@ -2522,8 +2522,8 @@ export const APPSPEC_SCHEMA = {
             type: 'string',
         },
         VoiceInput: {
-            description: 'How a person talks to the application (VOICE.md VO-10).\n\n`hands_free` (VO-12) is not one yet: it is refused in a sentence until it\nis built, never taken for click-to-talk.',
-            enum: ['off', 'push_to_talk'],
+            description: 'How a person talks to the application (VOICE.md VO-10, VO-12).\n\n`push_to_talk`: the microphone in the composer, clicked or held.\n`hands_free`: a conversation by voice — the composer offers *Talk with\nit*; a turn ends after a short silence and is sent, the answer is said,\nand listening goes on; speaking over the agent stops its voice (VO-13).\nHands-free implies answers said as they are written (`output: always`).',
+            enum: ['off', 'push_to_talk', 'hands_free'],
             title: 'VoiceInput',
             type: 'string',
         },

package/lib/specs/voices.js

@@ -375,6 +375,48 @@ export const SPEECH_MODEL_CATALOGUE = {
             },
         ],
     },
+    'whisper-small': {
+        id: 'whisper-small',
+        version: '0.0.1',
+        name: 'Whisper Small',
+        description: "Speech to text on Datalayer's servers for short spoken turns, on CPU: OpenAI's Whisper small (244M parameters, 99 languages), converted to CTranslate2 and run by faster-whisper, its weights quantized to 8 bits when loaded. About four times faster than large-v3-turbo; on the fixtures it hears English as well (3.3% against 2.4%) and French less well (16.1% against 2.6%). Recordings longer than a turn go to large-v3-turbo.",
+        provider: 'openai',
+        task: 'stt',
+        engine: 'faster-whisper',
+        dtype: 'float16',
+        languages: ['en', 'fr', 'es', 'it', 'pt', 'de', 'nl', 'ja', 'zh', 'hi'],
+        where: ['server'],
+        streaming: false,
+        licence: {
+            weights: 'MIT',
+            code: 'MIT',
+        },
+        attribution: '',
+        upstream: 'https://huggingface.co/openai/whisper-small',
+        revision: '536b0662742c02347bc0e980a01041f333bce120',
+        files: [
+            {
+                path: 'config.json',
+                sha256: 'b55496ac7940a7ae47d2c01eab40edfd8701feec1229d9cce3b40014383fb828',
+                size: 2370,
+            },
+            {
+                path: 'tokenizer.json',
+                sha256: 'fb7b63191e9bb045082c79fd742a3106a12c99513ab30df4a0d47fa6cb6fd0ab',
+                size: 2203239,
+            },
+            {
+                path: 'vocabulary.txt',
+                sha256: '34ce3fe1c5041027b3f8d42912270993f986dbc4bb34cf27f951e34a1e453913',
+                size: 459861,
+            },
+            {
+                path: 'model.bin',
+                sha256: '3e305921506d8872816023e4c273e75d2419fb89b24da97b4fe7bce14170d671',
+                size: 483546902,
+            },
+        ],
+    },
 };
 /** Whether a voice speaks a language: `fr` and `fr-FR` are spoken by a `fr-FR` voice. */
 export function voiceSpeaks(voice, language) {

package/lib/types/agentspecs.d.ts

@@ -958,7 +958,7 @@ export interface AppPageSpec {
 export interface AppVoiceSpec {
     enabled: boolean;
     /** `hands_free` (VO-12) is refused until it is built. */
-    input: 'off' | 'push_to_talk';
+    input: 'off' | 'push_to_talk' | 'hands_free';
     output: 'off' | 'on_request' | 'always';
     voice: string;
     language: string;

package/lib/voice/ComposerVoice.d.ts

@@ -7,7 +7,9 @@
  * person to edit and send — or sent at once with *Send what I say*. While
  * the words are still being heard the host is told (`onInterimChange`), to
  * grey them until they are final (VO-11). Beside the microphone, the
- * person's voice menu (`VoiceMenu`, VO-30).
+ * person's voice menu (`VoiceMenu`, VO-30). Where the application's voice
+ * says `hands_free`, *Talk with it* too (`HandsFree`, VO-12, VO-13): a turn
+ * is sent by itself and the agent's answer may be spoken over.
  *
  * @module voice/ComposerVoice
  */
@@ -29,6 +31,12 @@ export interface ComposerVoiceProps {
     speaking?: boolean;
     /** Stops the agent's voice, also when the microphone opens (VO-13). */
     onStopSpeaking?: () => void;
+    /** The agent is answering: a run is in progress (hands-free, VO-12). */
+    running?: boolean;
+    /** Stops the run while the answer is still being written (barge-in, VO-13). */
+    onStopRun?: () => void;
+    /** The answer being written was spoken over: marked in the conversation (VO-13). */
+    onInterrupted?: () => void;
     /**
      * The words in the box are still being heard (`true`), or final
      * (`false`): the host greys them until they are (VO-11).
@@ -42,5 +50,5 @@ export interface ComposerVoiceProps {
 }
 /** Whether something rides on a run as a spoken message's metadata (VO-27). */
 export declare function spokenOf(value: unknown): SpokenMetadata | undefined;
-export declare function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled, speaking, onStopSpeaking, onInterimChange, answers, }: ComposerVoiceProps): JSX.Element | null;
+export declare function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled, speaking, onStopSpeaking, running, onStopRun, onInterrupted, onInterimChange, answers, }: ComposerVoiceProps): JSX.Element | null;
 export default ComposerVoice;

package/lib/voice/ComposerVoice.js

@@ -1,6 +1,7 @@
 import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
 import { useMemo, useRef } from 'react';
 import { DeviceHearing } from './hearing';
+import { HandsFree } from './HandsFree';
 import { ServerHearing, hearingPlace } from './serverHearing';
 import { VoiceInput } from './VoiceInput';
 import { VoiceMenu } from './VoiceMenu';
@@ -14,7 +15,7 @@ export function spokenOf(value) {
         ? said
         : undefined;
 }
-export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled = false, speaking = false, onStopSpeaking, onInterimChange, answers = false, }) {
+export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled = false, speaking = false, onStopSpeaking, running = false, onStopRun, onInterrupted, onInterimChange, answers = false, }) {
     // In the browser, or on Datalayer's servers where the application, the
     // language or the person's detection asks for it (§5, VO-14, VO-34).
     const place = hearingPlace(voice);
@@ -45,7 +46,14 @@ export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled
     if (!hearing) {
         return null;
     }
-    return (_jsxs("span", { "data-composer-voice": "", style: { display: 'inline-flex', alignItems: 'center', gap: 2 }, children: [_jsx(VoiceMenu, { voice: voice, answers: answers, disabled: disabled }), _jsx(VoiceInput, { hearing: hearing, language: voice.language, consentKey: voice.consentKey ?? 'chat', silenceMs: voice.silenceMs, microphoneId: voice.microphoneId, disabled: disabled, speaking: speaking, onStopSpeaking: onStopSpeaking, onListen: () => {
+    return (_jsxs("span", { "data-composer-voice": "", style: { display: 'inline-flex', alignItems: 'center', gap: 2 }, children: [_jsx(VoiceMenu, { voice: voice, answers: answers, disabled: disabled }), voice.input === 'hands_free' ? (_jsx(HandsFree, { hearing: hearing, language: voice.language, consentKey: voice.consentKey ?? 'chat', microphoneId: voice.microphoneId, turnSilenceMs: voice.turnSilenceMs, disabled: disabled, running: running, speaking: speaking, onStopSpeaking: onStopSpeaking, onStopRun: onStopRun, onInterrupted: onInterrupted, onInterim: (words) => {
+                    setInput(words ? afterBase(words) : base.current);
+                    onInterimChange?.(words !== '');
+                }, send: (text, metadata) => {
+                    onInterimChange?.(false);
+                    setInput(base.current);
+                    send(text, metadata);
+                } })) : null, _jsx(VoiceInput, { hearing: hearing, language: voice.language, consentKey: voice.consentKey ?? 'chat', silenceMs: voice.silenceMs, microphoneId: voice.microphoneId, disabled: disabled, speaking: speaking, onStopSpeaking: onStopSpeaking, onListen: () => {
                     base.current = input.trim();
                     onStopSpeaking?.();
                 }, 

package/lib/voice/HandsFree.d.ts

@@ -0,0 +1,55 @@
+/**
+ * *Talk with it*: a conversation by voice, hands-free (VOICE.md VO-12,
+ * VO-13), beside the composer's microphone where the application's voice
+ * says `input: hands_free`.
+ *
+ * On, the microphone stays open. The words are written in the composer as
+ * they are heard (VO-11); once the person has spoken and been silent for the
+ * turn's silence (`turnSilenceMs`, 700 ms unless said), what was said is
+ * heard whole and **sent** — nothing waits for a click. The answer is said as
+ * it is written (hands-free implies `output: always`), and listening goes
+ * on through it: speech heard while the agent thinks or speaks is
+ * **barge-in** (VO-13) — its voice stops at once, the run is stopped while
+ * the answer is still being written, the interrupted answer is marked in
+ * the conversation, and what was heard starts the next turn. The state is
+ * shown at all times: *Listening…*, *Thinking…*, *Speaking…*, *Paused*.
+ * `Esc` or *Pause* pauses it; the toggle again turns it off.
+ *
+ * The microphone is opened anew when the agent begins answering, so what it
+ * holds then is the person's; the browser's echo cancellation keeps the
+ * answer's own sound out (`openMicrophone`). The first use asks for the
+ * microphone as the click-to-talk does (VO-61), under the same consent.
+ *
+ * @module voice/HandsFree
+ */
+import type { JSX } from 'react';
+import type { Hearing } from './hearing';
+import type { SpokenMetadata } from './types';
+export interface HandsFreeProps {
+    hearing: Hearing;
+    /** The language listened to, BCP 47. */
+    language: string;
+    /** Whose consent to the microphone is remembered: the application's id. */
+    consentKey: string;
+    /** The microphone the person chose (VO-30); the browser's own when unsaid. */
+    microphoneId?: string;
+    /** The silence after speech that ends a turn, in ms; 700 unless said (VO-12). */
+    turnSilenceMs?: number;
+    /** A turn, heard whole: sent at once. */
+    send: (text: string, metadata: SpokenMetadata) => void;
+    /** The words heard so far, for the composer (VO-11); `''` gives it back. */
+    onInterim?: (text: string) => void;
+    /** The agent is answering: a run is in progress. */
+    running?: boolean;
+    /** The agent's answer is being said. */
+    speaking?: boolean;
+    /** Stops the agent's voice at once (barge-in, VO-13). */
+    onStopSpeaking?: () => void;
+    /** Stops the run while the answer is still being written (barge-in, VO-13). */
+    onStopRun?: () => void;
+    /** The answer being written was interrupted: marked in the conversation (VO-13). */
+    onInterrupted?: () => void;
+    disabled?: boolean;
+}
+export declare function HandsFree({ hearing, language, consentKey, microphoneId, turnSilenceMs, send, onInterim, running, speaking, onStopSpeaking, onStopRun, onInterrupted, disabled, }: HandsFreeProps): JSX.Element;
+export default HandsFree;

package/lib/voice/HandsFree.js

@@ -0,0 +1,211 @@
+import { jsx as _jsx, Fragment as _Fragment, jsxs as _jsxs } from "react/jsx-runtime";
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { Button, IconButton, Text, Tooltip } from '@primer/react';
+import { Box } from '@datalayer/primer-addons';
+import { openMicrophone } from './capture';
+import { CONSENT_SENTENCE, SERVER_CONSENT_SENTENCE, consent, consentKeyFor, consented, } from './consent';
+import { HANDS_FREE_WORDS, IDLE_PAUSE_MS, TURN_SILENCE_MS, bargedIn, handsFreeState, turnEnded, } from './handsFree';
+import { INTERIM_MS } from './VoiceInput';
+/** Waits. */
+const pause = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+export function HandsFree({ hearing, language, consentKey, microphoneId, turnSilenceMs = TURN_SILENCE_MS, send, onInterim, running = false, speaking = false, onStopSpeaking, onStopRun, onInterrupted, disabled = false, }) {
+    const [on, setOn] = useState(false);
+    const [paused, setPaused] = useState(false);
+    const [asking, setAsking] = useState(false);
+    const [problem, setProblem] = useState();
+    const state = handsFreeState({ on, paused, speaking, running });
+    const stateRef = useRef(state);
+    stateRef.current = state;
+    const runningRef = useRef(running);
+    runningRef.current = running;
+    const capture = useRef(undefined);
+    const latest = useRef({
+        send,
+        onInterim,
+        onStopSpeaking,
+        onStopRun,
+        onInterrupted,
+    });
+    latest.current = {
+        send,
+        onInterim,
+        onStopSpeaking,
+        onStopRun,
+        onInterrupted,
+    };
+    const key = consentKeyFor(consentKey, hearing.where);
+    const turnOn = useCallback(() => {
+        if (!consented(key)) {
+            setAsking(true);
+            return;
+        }
+        setProblem(undefined);
+        setPaused(false);
+        setOn(true);
+    }, [key]);
+    const turnOff = useCallback(() => {
+        setOn(false);
+        setPaused(false);
+        capture.current?.cancel();
+        capture.current = undefined;
+        latest.current.onInterim?.('');
+    }, []);
+    /*
+     * The loop: while on and not paused, the microphone is open and what it
+     * heard so far is looked at every INTERIM_MS. Listening, the turn ends on
+     * the silence and is sent; thinking or speaking, speech is barge-in. The
+     * microphone is opened anew when the agent begins answering, and after
+     * each turn, so what it holds is only what came since.
+     */
+    const heardWhile = useRef('listening');
+    useEffect(() => {
+        if (!on || paused) {
+            return;
+        }
+        let alive = true;
+        let quietSince = Date.now();
+        void (async () => {
+            try {
+                await hearing.ready(language);
+                capture.current = await openMicrophone(microphoneId);
+            }
+            catch (error) {
+                if (alive) {
+                    setProblem(error instanceof Error
+                        ? error.message
+                        : 'The microphone could not be opened.');
+                    setOn(false);
+                }
+                return;
+            }
+            heardWhile.current =
+                stateRef.current === 'listening' ? 'listening' : 'answering';
+            while (alive) {
+                await pause(INTERIM_MS);
+                const open = capture.current;
+                if (!alive || !open) {
+                    return;
+                }
+                const now = stateRef.current;
+                const answering = now === 'thinking' || now === 'speaking';
+                // The agent began answering, or finished: hear anew from here.
+                if ((answering && heardWhile.current === 'listening') ||
+                    (!answering && heardWhile.current === 'answering')) {
+                    open.cancel();
+                    try {
+                        capture.current = await openMicrophone(microphoneId);
+                    }
+                    catch {
+                        return;
+                    }
+                    heardWhile.current = answering ? 'answering' : 'listening';
+                    quietSince = Date.now();
+                    continue;
+                }
+                let so;
+                try {
+                    so = await hearing.hearSoFar(open.peek(), language);
+                }
+                catch {
+                    so = undefined;
+                }
+                if (!alive || capture.current !== open || !so) {
+                    continue;
+                }
+                if (so.text) {
+                    latest.current.onInterim?.(so.text);
+                }
+                if (so.spoke) {
+                    quietSince = Date.now();
+                }
+                if (bargedIn(now, so)) {
+                    // The person speaks over the agent (VO-13): its voice and its run
+                    // stop; what was heard goes on as the next turn.
+                    latest.current.onStopSpeaking?.();
+                    if (now === 'thinking' || runningRef.current) {
+                        latest.current.onStopRun?.();
+                    }
+                    latest.current.onInterrupted?.();
+                    heardWhile.current = 'listening';
+                    continue;
+                }
+                if (now === 'listening' && turnEnded(so, turnSilenceMs)) {
+                    capture.current = undefined;
+                    const audio = await open.stop();
+                    let heard;
+                    try {
+                        heard = await hearing.hear(audio, language);
+                    }
+                    catch (error) {
+                        heard = undefined;
+                        if (alive) {
+                            setProblem(error instanceof Error
+                                ? error.message
+                                : 'What you said could not be heard.');
+                        }
+                    }
+                    if (!alive) {
+                        return;
+                    }
+                    latest.current.onInterim?.('');
+                    if (heard) {
+                        latest.current.send(heard.text, heard.metadata);
+                    }
+                    try {
+                        capture.current = await openMicrophone(microphoneId);
+                    }
+                    catch {
+                        return;
+                    }
+                    heardWhile.current = 'listening';
+                    quietSince = Date.now();
+                    continue;
+                }
+                if (now === 'listening' && Date.now() - quietSince >= IDLE_PAUSE_MS) {
+                    setPaused(true);
+                    return;
+                }
+            }
+        })();
+        return () => {
+            alive = false;
+            capture.current?.cancel();
+            capture.current = undefined;
+        };
+        // The loop reads the state through a ref: it is not restarted on it.
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [on, paused, hearing, language, microphoneId, turnSilenceMs]);
+    // Esc pauses it.
+    useEffect(() => {
+        if (!on) {
+            return;
+        }
+        const down = (event) => {
+            if (event.key === 'Escape' && !paused) {
+                setPaused(true);
+                capture.current?.cancel();
+                capture.current = undefined;
+                latest.current.onInterim?.('');
+            }
+        };
+        window.addEventListener('keydown', down);
+        return () => window.removeEventListener('keydown', down);
+    }, [on, paused]);
+    return (_jsxs(Box, { display: "inline-flex", alignItems: "center", sx: { gap: 1, position: 'relative' }, "data-hands-free": state, children: [_jsx(Tooltip, { text: on
+                    ? 'Stop talking with it'
+                    : 'Talk with it: speak, it answers, and listens on (hands-free)', direction: "n", children: _jsx(Button, { size: "small", variant: on ? 'primary' : 'invisible', "aria-pressed": on, "aria-label": on ? 'Stop talking with it' : 'Talk with it', disabled: disabled, "data-hands-free-toggle": "", onClick: () => (on ? turnOff() : turnOn()), children: HANDS_FREE_WORDS.off }) }), on ? (_jsxs(_Fragment, { children: [_jsx(Text, { sx: { fontSize: 0, color: 'fg.muted' }, role: "status", "aria-live": "polite", "data-hands-free-state": state, children: HANDS_FREE_WORDS[state] }), paused ? (_jsx(IconButton, { icon: () => _jsx("span", { "aria-hidden": "true", children: "\u25B6" }), size: "small", variant: "invisible", "aria-label": "Resume listening", "data-hands-free-resume": "", onClick: () => setPaused(false) })) : (_jsx(IconButton, { icon: () => _jsx("span", { "aria-hidden": "true", children: "\u275A\u275A" }), size: "small", variant: "invisible", "aria-label": "Pause listening (Esc)", "data-hands-free-pause": "", onClick: () => {
+                            setPaused(true);
+                            capture.current?.cancel();
+                            capture.current = undefined;
+                            latest.current.onInterim?.('');
+                        } }))] })) : null, problem ? (_jsx(Text, { sx: { fontSize: 0, color: 'danger.fg' }, "data-hands-free-problem": "", children: problem })) : null, asking ? (_jsxs(Box, { role: "dialog", "aria-label": "The microphone", "data-hands-free-consent": "", position: "absolute", bottom: "100%", left: 0, width: 280, p: 3, bg: "canvas.overlay", border: "1px solid", borderColor: "border.default", borderRadius: 2, boxShadow: "shadow.large", zIndex: 10, children: [_jsx(Text, { as: "p", sx: { fontSize: 1, mb: 2 }, children: hearing.where === 'server'
+                            ? SERVER_CONSENT_SENTENCE
+                            : CONSENT_SENTENCE }), _jsxs(Box, { display: "flex", gap: 2, justifyContent: "flex-end", children: [_jsx(Button, { size: "small", onClick: () => setAsking(false), children: "Not now" }), _jsx(Button, { size: "small", variant: "primary", "data-hands-free-consent-yes": "", onClick: () => {
+                                    consent(key);
+                                    setAsking(false);
+                                    setProblem(undefined);
+                                    setPaused(false);
+                                    setOn(true);
+                                }, children: "Use the microphone" })] })] })) : null] }));
+}
+export default HandsFree;

package/lib/voice/chatVoice.d.ts

@@ -35,7 +35,8 @@ export interface ChatVoicePlace {
  * - **Listening**: a microphone in the composer — click it and speak, the
  *   words written in the box as they are heard, stopped by a second click
  *   or when the person stops speaking; or held, and let go (VO-10, VO-11).
- *   `hands_free` (VO-12) is not built: `checkAppspec` refuses it.
+ *   `hands_free` (VO-12): the microphone, and *Talk with it* beside it; it
+ *   implies answers said as they are written.
  * - **Speaking**: its answers said by Datalayer's speech service, on request
  *   or always (VO-20, VO-21); refused here when its host says no service.
  */