@datalayer/agent-runtimes 1.5.44 → 1.5.45

package/lib/apps/plugins/chat/ChatView.js

@@ -1702,8 +1702,8 @@ export default function ChatView({ workspace }) {
                     // opening stops it, and *Stop speaking* while it speaks.
                     speaking: speaking, onStopSpeaking: () => controlsRef.current?.stopSpeaking?.(), 
                     // Hands-free (VO-12, VO-13): the run in progress, stopped when
-                    // the person speaks over it.
-                    running: busy, onStopRun: () => controlsRef.current?.stop?.(), onInterimChange: setVoiceInterim, 
+                    // the person speaks over it, and the answer marked interrupted.
+                    running: busy, onStopRun: () => controlsRef.current?.stop?.(), onInterrupted: () => controlsRef.current?.markInterrupted(), onInterimChange: setVoiceInterim, 
                     // The menu offers when answers are heard where the
                     // application lets them be (VO-21, VO-30).
                     answers: Boolean(config?.voice?.output &&

package/lib/chat/ChatFloating.js

@@ -1030,7 +1030,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
                     ...poweredByProps,
                 }, description: description, onStateUpdate: onStateUpdate, onNewChat: onNewChat, suggestions: suggestions, submitOnSuggestionClick: submitOnSuggestionClick, hideMessagesAfterToolUI: hideMessagesAfterToolUI, avatarConfig: {
                     showAvatars: true,
-                }, placeholder: "Type a message...", backgroundColor: "canvas.subtle", frontendTools: frontendTools, showModelSelector: showModelSelector, availableModels: availableModels, showToolsMenu: showToolsMenu, showSkillsMenu: showSkillsMenu, showTokenUsage: showTokenUsage, showContextRing: showContextRing, runtimeId: runtimeId, historyEndpoint: historyEndpoint, historyAuthToken: historyAuthToken, pendingPrompt: pendingPrompt ?? suggestionPrompt, inspector: inspector, showInformation: showInformation, onInformationClick: onInformationClick, onToolCallStart: onToolCallStart, onToolCallComplete: onToolCallComplete, showToolApprovalBanner: showToolApprovalBanner, pendingApprovals: pendingApprovals, onApproveApproval: onApproveApproval, onRejectApproval: onRejectApproval, voice: voice, voiceSpeaking: speakerState.speaking, onStopSpeaking: speaker ? stopSpeaking : undefined, onReadAloud: speaker ? readAnswerAloud : undefined, ...panelProps, ...assistantChat, onLoadingChange: handleLoadingChange, onDisplayItemsChange: handleDisplayItemsChange, onSendReady: handleSendReady, children: conversationBalloon && currentOpen ? (
+                }, placeholder: "Type a message...", backgroundColor: "canvas.subtle", frontendTools: frontendTools, showModelSelector: showModelSelector, availableModels: availableModels, showToolsMenu: showToolsMenu, showSkillsMenu: showSkillsMenu, showTokenUsage: showTokenUsage, showContextRing: showContextRing, runtimeId: runtimeId, historyEndpoint: historyEndpoint, historyAuthToken: historyAuthToken, pendingPrompt: pendingPrompt ?? suggestionPrompt, inspector: inspector, showInformation: showInformation, onInformationClick: onInformationClick, onToolCallStart: onToolCallStart, onToolCallComplete: onToolCallComplete, showToolApprovalBanner: showToolApprovalBanner, pendingApprovals: pendingApprovals, onApproveApproval: onApproveApproval, onRejectApproval: onRejectApproval, voice: voice, voiceSpeaking: speakerState.speaking, voiceSentence: speakerState.sentence, onStopSpeaking: speaker ? stopSpeaking : undefined, onReadAloud: speaker ? readAnswerAloud : undefined, ...panelProps, ...assistantChat, onLoadingChange: handleLoadingChange, onDisplayItemsChange: handleDisplayItemsChange, onSendReady: handleSendReady, children: conversationBalloon && currentOpen ? (
                 // Current: the one thing said or done now, over the composer.
                 _jsx(Box, { px: 3, pt: 3, pb: 2, overflow: "hidden", children: _jsx(CurrentBalloonBody, { text: toolSaid ? undefined : (currentWords ?? description), fullText: toolSaid ? undefined : currentWhole, tool: toolSaid, busy: atWork, speaking: answering && atWork, waiting: atWork && !answering }) })) : (children) }))] }));
     /*

package/lib/chat/base/ChatBase.js

@@ -52,11 +52,13 @@ import { EphemeralSurfaceControl } from '../EphemeralSurfaceControl';
 import { ChatMessageList, } from '../messages/ChatMessageList';
 import { InputPrompt } from '../prompt/InputPrompt';
 import { ComposerVoice, spokenOf } from '../../voice/ComposerVoice';
+import { answerSpokenOver } from '../../voice/handsFree';
 import { ReadAloud, readsAloud } from '../../voice/ReadAloud';
 import { settledVoice } from '../../voice/chatVoice';
 import { useVoiceSettings } from '../../voice/useVoiceSettings';
 import { useVoiceSpeaker } from '../../voice/useVoiceSpeaker';
 import { useSpokenAnswers } from '../../voice/useSpokenAnswers';
+import { saidMessageId, useSaidSentence } from '../../voice/captions';
 import { lineToSay } from '../../voice/sayAloud';
 import { isFloatingChatViewMode } from '../viewModes';
 import { ToolApprovalBanner, ToolApprovalDialog, } from '../tools';
@@ -424,7 +426,7 @@ function ChatBaseInner({ title, subtitle, showHeader = false, showTokenUsage = t
 // Mode selection
 useStore: useStoreMode = true, protocol: protocolRaw, onSendMessage, onSendReady, onLoadingChange, onSpeakingChange, onContextSnapshot, enableStreaming = false, 
 // Extended props
-brandIcon, avatarConfig, headerButtons, showPoweredBy = false, poweredByProps, emptyState, renderHeader, showTurnFooter = true, notebookToolSurfacesId, footerContent, welcome, trailingContent, voice, voiceSpeaking = false, onStopSpeaking, onReadAloud, showInformation = false, onInformationClick, headerContent, children, borderRadius, backgroundColor, border, boxShadow, compact = false, placeholder, description = 'Start a conversation with the AI agent.', onStateUpdate, onNewChat, onClear, onMessagesChange, onItemsChange, onDisplayItemsChange, autoFocus = false, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, focusTrigger, frontendTools: frontendToolsProp, enableEphemeralNotebook = false, initialEphemeralNotebookOpen = true, onEphemeralNotebookOpenChange, enableEphemeralDocument = false, initialEphemeralSurfaceMode, onEphemeralSurfaceModeChange, collapsed = false, onExpandFromCollapsed, ephemeralNotebookToolbar, ephemeralNotebookToolbarExtraItems, ephemeralDocumentToolbarExtraItems, ephemeralNotebookCollaborationProvider, ephemeralNotebookCollaborationDocumentId, ephemeralDocumentCollaboration, ephemeralRuntimeOverride, 
+brandIcon, avatarConfig, headerButtons, showPoweredBy = false, poweredByProps, emptyState, renderHeader, showTurnFooter = true, notebookToolSurfacesId, footerContent, welcome, trailingContent, voice, voiceSpeaking = false, voiceSentence, onStopSpeaking, onReadAloud, showInformation = false, onInformationClick, headerContent, children, borderRadius, backgroundColor, border, boxShadow, compact = false, placeholder, description = 'Start a conversation with the AI agent.', onStateUpdate, onNewChat, onClear, onMessagesChange, onItemsChange, onDisplayItemsChange, autoFocus = false, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, focusTrigger, frontendTools: frontendToolsProp, enableEphemeralNotebook = false, initialEphemeralNotebookOpen = true, onEphemeralNotebookOpenChange, enableEphemeralDocument = false, initialEphemeralSurfaceMode, onEphemeralSurfaceModeChange, collapsed = false, onExpandFromCollapsed, ephemeralNotebookToolbar, ephemeralNotebookToolbarExtraItems, ephemeralDocumentToolbarExtraItems, ephemeralNotebookCollaborationProvider, ephemeralNotebookCollaborationDocumentId, ephemeralDocumentCollaboration, ephemeralRuntimeOverride, 
 // Tool invocation hooks
 onToolCallStart, onToolCallComplete, inspector: inspectorGiven, 
 // Identity/Authorization props
@@ -2974,6 +2976,10 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
     const ownVoice = useVoiceSpeaker(settled, !hostSpeaks);
     useSpokenAnswers(displayItems, isLoading, !hostSpeaks && settled?.output === 'always', ownVoice.speaker);
     const speakingNow = voiceSpeaking || ownVoice.state.speaking;
+    // The sentence being said, marked in the conversation (VO-65).
+    const sentenceSaid = hostSpeaks ? voiceSentence : ownVoice.state.sentence;
+    const saidConversationRef = useRef(null);
+    useSaidSentence(saidConversationRef, sentenceSaid, displayItems);
     /*
      * A line the agent wants heard, not written (VO-35): a caption while the
      * run goes on, said by the speaker where answers are said by themselves.
@@ -3001,6 +3007,14 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
     }, [isLoading]);
     // Answers spoken over, hands-free (VOICE.md VO-13): marked *interrupted*.
     const [interruptedIds, setInterruptedIds] = useState(() => new Set());
+    // The conversation as shown, read when the person speaks over an answer.
+    const shownItemsRef = useRef([]);
+    const markInterrupted = useCallback(() => {
+        const spokenOver = answerSpokenOver(shownItemsRef.current.filter((item) => !isToolCallMessage(item)));
+        if (spokenOver) {
+            setInterruptedIds(ids => new Set([...ids, spokenOver]));
+        }
+    }, []);
     const stopSpeaking = onStopSpeaking ?? (ownVoice.speaker ? ownVoice.stop : undefined);
     const readAnswerAloud = onReadAloud ?? (ownVoice.speaker ? ownVoice.read : undefined);
     const handleSend = useCallback(async (messageOverride, 
@@ -3370,6 +3384,8 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
             thread: () => threadIdRef.current,
             // Its speaker, for a host whose composer holds the microphone (VO-10).
             stopSpeaking,
+            // The answer spoken over, marked for a host's own composer (VO-13).
+            markInterrupted,
         });
         return () => {
             onSendReady(null);
@@ -3382,6 +3398,7 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
         handleStop,
         handleNewChat,
         stopSpeaking,
+        markInterrupted,
     ]);
     // Streaming state, for a host that draws the prompt.
     useEffect(() => {
@@ -3748,6 +3765,8 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
             ...displayItems,
         ]
         : displayItems;
+    shownItemsRef.current = welcomedItems;
+    const saidId = saidMessageId(welcomedItems, sentenceSaid);
     /*
      * *Read aloud* under each answer the agent has finished (VOICE.md VO-21),
      * when the host reads answers aloud on request.
@@ -3763,7 +3782,7 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
             return (_jsxs(_Fragment, { children: [interruptedIds.has(message.id) ? (_jsx(Text, { sx: { fontSize: 0, color: 'fg.muted', fontStyle: 'italic' }, "data-interrupted": "", children: "interrupted" })) : null, readAnswerAloud && readsAloud(message, text, writingId) ? (_jsx(ReadAloud, { onRead: () => readAnswerAloud(text) })) : null] }));
         }
         : undefined;
-    const messagesContent = children ? (children) : (_jsxs(Box, { display: "flex", flexDirection: "column", minHeight: 0, 
+    const messagesContent = children ? (children) : (_jsxs(Box, { ref: saidConversationRef, display: "flex", flexDirection: "column", minHeight: 0, 
         /*
             A reading column, not a full-bleed sheet.
   
@@ -3776,16 +3795,12 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
                 // Tool calls are drawn by the page's Reactor plugins; a named
                 // notebook's cell surfaces stand in for the default row of its tools,
                 // so every chat with a headless notebook gets the same transcript.
-                notebookToolSurfacesId: notebookToolSurfacesId, approvalConfig: approvalConfig, messagesEndRef: messagesEndRef, onRespond: handleRespond, showTurnFooters: showTurnFooter, renderMessageFooter: readAloudFooter, agentUsage: agentUsage, onRemoveItems: handleRemoveItems, mcpServers: filteredMcpServers, emptyContent: launching ? null : (_jsx(ChatEmptyState, { emptyState: emptyState, brandIcon: brandIcon, description: resolvedDescription, suggestions: resolvedSuggestions, submitOnSuggestionClick: submitOnSuggestionClick, onSuggestionSubmit: handleSuggestionSubmit, onSuggestionFill: handleSuggestionFill })) }), trailingContent] }));
+                notebookToolSurfacesId: notebookToolSurfacesId, approvalConfig: approvalConfig, messagesEndRef: messagesEndRef, onRespond: handleRespond, showTurnFooters: showTurnFooter, renderMessageFooter: readAloudFooter, saidMessageId: saidId, agentUsage: agentUsage, onRemoveItems: handleRemoveItems, mcpServers: filteredMcpServers, emptyContent: launching ? null : (_jsx(ChatEmptyState, { emptyState: emptyState, brandIcon: brandIcon, description: resolvedDescription, suggestions: resolvedSuggestions, submitOnSuggestionClick: submitOnSuggestionClick, onSuggestionSubmit: handleSuggestionSubmit, onSuggestionFill: handleSuggestionFill })) }), trailingContent] }));
     // The microphone, in the composer's footer (VO-10, VO-11).
     const voiceControl = settled && settled.input !== 'off' ? (_jsx(ComposerVoice, { voice: settled, input: input, setInput: setInput, disabled: disabled || launching, speaking: speakingNow, onStopSpeaking: stopSpeaking, 
         // Hands-free (VO-12, VO-13): the run in progress, stopped when the
         // person speaks over it, and the answer marked interrupted.
-        running: isLoading, onStopRun: handleStop, onInterrupted: () => {
-            if (writingId) {
-                setInterruptedIds(ids => new Set([...ids, writingId]));
-            }
-        }, onInterimChange: setVoiceInterim, 
+        running: isLoading, onStopRun: handleStop, onInterrupted: markInterrupted, onInterimChange: setVoiceInterim, 
         // The menu offers when answers are heard where the application
         // lets them be (VO-21, VO-30).
         answers: Boolean(voice?.output && voice.output !== 'off' && voice.speechUrl), onSpoken: metadata => {

package/lib/chat/messages/ChatMessageList.d.ts

@@ -78,6 +78,12 @@ export interface ChatMessageListProps {
      * `note`, which has no footer of its own.
      */
     renderMessageFooter?: (message: ChatMessage) => ReactNode;
+    /**
+     * The answer a sentence is being said from (VOICE.md VO-65): its row is
+     * marked `aria-current` and `data-voice-saying`, with a bar beside it —
+     * the sentence itself is underlined by `useSaidSentence`.
+     */
+    saidMessageId?: string;
     /**
      * The MCP servers the agent has, with the tools each serves: a tool call is
      * drawn with the marks of the server it belongs to. Skills, frontend tools
@@ -109,5 +115,5 @@ export interface SubagentChatPanelProps {
  * approval decisions are sent over the shared monitoring socket.
  */
 export declare function SubagentChatPanel({ toolCallId, subagentName: subagentNameHint, height, avatarConfig, }: SubagentChatPanelProps): React.ReactElement | null;
-export declare function ChatMessageList({ displayItems, actOn, upload, isLoading, isStreaming, showLoadingIndicator, hideMessagesAfterToolUI, avatarConfig, padding, notebookToolSurfacesId, emptyContent, messagesEndRef, onRespond, approvalConfig, showTurnFooters, agentUsage, onRemoveItems, renderMessageFooter, mcpServers, density, }: ChatMessageListProps): import("react").JSX.Element;
+export declare function ChatMessageList({ displayItems, actOn, upload, isLoading, isStreaming, showLoadingIndicator, hideMessagesAfterToolUI, avatarConfig, padding, notebookToolSurfacesId, emptyContent, messagesEndRef, onRespond, approvalConfig, showTurnFooters, agentUsage, onRemoveItems, renderMessageFooter, saidMessageId, mcpServers, density, }: ChatMessageListProps): import("react").JSX.Element;
 export {};

package/lib/chat/messages/ChatMessageList.js

@@ -366,7 +366,7 @@ function DefaultToolCallRenderer({ item, onRespond, mcpServers, density, }) {
 // ---------------------------------------------------------------------------
 // Component
 // ---------------------------------------------------------------------------
-export function ChatMessageList({ displayItems, actOn, upload, isLoading, isStreaming, showLoadingIndicator, hideMessagesAfterToolUI, avatarConfig, padding, notebookToolSurfacesId, emptyContent, messagesEndRef, onRespond, approvalConfig, showTurnFooters = false, agentUsage, onRemoveItems, renderMessageFooter, mcpServers, density = 'comfortable', }) {
+export function ChatMessageList({ displayItems, actOn, upload, isLoading, isStreaming, showLoadingIndicator, hideMessagesAfterToolUI, avatarConfig, padding, notebookToolSurfacesId, emptyContent, messagesEndRef, onRespond, approvalConfig, showTurnFooters = false, agentUsage, onRemoveItems, renderMessageFooter, saidMessageId, mcpServers, density = 'comfortable', }) {
     /*
      * Who draws a tool call: the Reactor plugins of the page, when it has a
      * platform. Each plugin binds its own tools (`ToolBindings`) and renders its
@@ -541,7 +541,13 @@ export function ChatMessageList({ displayItems, actOn, upload, isLoading, isStre
         }
         // The host's footer, asked once: a row only when it draws something.
         const messageFooter = renderMessageFooter?.(message);
-        return (_jsx(Box, { "data-chat-message": message.role, display: "flex", flexDirection: "column", alignItems: isUser ? 'flex-end' : 'flex-start', px: padding, py: compact ? '2px' : 1, children: _jsxs(Box, { display: "flex", gap: 2, flexDirection: isUser ? 'row-reverse' : 'row', alignItems: "flex-start", 
+        // The answer being said (VO-65): marked by a bar and for assistive tech.
+        const said = saidMessageId !== undefined && message.id === saidMessageId;
+        return (_jsx(Box, { "data-chat-message": message.role, "data-voice-saying": said ? '' : undefined, "aria-current": said ? 'true' : undefined, sx: said
+                ? {
+                    boxShadow: 'inset 3px 0 0 var(--borderColor-accent-emphasis, #0969da)',
+                }
+                : undefined, display: "flex", flexDirection: "column", alignItems: isUser ? 'flex-end' : 'flex-start', px: padding, py: compact ? '2px' : 1, children: _jsxs(Box, { display: "flex", gap: 2, flexDirection: isUser ? 'row-reverse' : 'row', alignItems: "flex-start", 
                 /*
                         Full width, always.
       

package/lib/protocols/VercelAIAdapter.js

@@ -627,6 +627,17 @@ export class VercelAIAdapter extends BaseProtocolAdapter {
                                     });
                                 }
                             }
+                            else if (typeof event.type === 'string' &&
+                                event.type.startsWith('data-')) {
+                                // What the server says besides the conversation's text, as
+                                // AG-UI's custom events say it: `data-voice.say`, a line the
+                                // agent says aloud, and `data-voice.settings` (VOICE.md VO-36).
+                                this.emit({
+                                    type: 'activity',
+                                    activity: { type: event.type, data: event.data },
+                                    timestamp: new Date(),
+                                });
+                            }
                         }
                         catch (parseError) {
                             console.error('[VercelAIAdapter] Failed to parse SSE data:', parseError);

package/lib/specs/voices.js

@@ -100,6 +100,68 @@ export const SPEECH_MODEL_CATALOGUE = {
             },
         ],
     },
+    'moonshine-base-en-server': {
+        id: 'moonshine-base-en-server',
+        version: '0.0.1',
+        name: 'Moonshine Base (English, server)',
+        description: "Speech to text on Datalayer's servers, in English: Useful Sensors' Moonshine Base (61M parameters), the same 8-bit ONNX export the browser runs, run by onnxruntime on the CPU with a greedy decoder. Its encoder is as long as the speech, with no 30-second window: measured beside Whisper small for short spoken turns (decision 18).",
+        provider: 'usefulsensors',
+        task: 'stt',
+        engine: 'onnxruntime',
+        dtype: 'q8',
+        languages: ['en'],
+        where: ['server'],
+        streaming: false,
+        licence: {
+            weights: 'MIT',
+            code: 'MIT',
+        },
+        attribution: '',
+        upstream: 'https://huggingface.co/UsefulSensors/moonshine-base',
+        revision: 'b1e9b6aae3c3c7298f10c3798393fdf38e8fbbad',
+        files: [
+            {
+                path: 'config.json',
+                sha256: 'fab7241d1e9fc6c2370c4c6dfb5da79bb54d67ed9ab6b507ac51d29d2abe01d1',
+                size: 922,
+            },
+            {
+                path: 'generation_config.json',
+                sha256: 'f9b3f711b57be7def2e50a8942f64f36ee0a55fad5b84ff93a687b6c5bcc1d44',
+                size: 147,
+            },
+            {
+                path: 'preprocessor_config.json',
+                sha256: 'fa43a7017ef85cd1d0fba0d9aae77c8adb16990ae6f11115631f41ec5d8aa679',
+                size: 128,
+            },
+            {
+                path: 'special_tokens_map.json',
+                sha256: 'ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356',
+                size: 3,
+            },
+            {
+                path: 'tokenizer.json',
+                sha256: '7b913404bdd039af4756783218af4440bc07fb7d6d8258d677e34f95b3ec416f',
+                size: 3761754,
+            },
+            {
+                path: 'tokenizer_config.json',
+                sha256: 'edaee394565d428ea98a663ae7209cdcfeefc5585c42d7a570ff7c986df2cd15',
+                size: 135735,
+            },
+            {
+                path: 'onnx/encoder_model_quantized.onnx',
+                sha256: '1dd9ab0a7f987113d30affcba5a068d11c8f90fa0223caa3e491ade431ad9751',
+                size: 20513063,
+            },
+            {
+                path: 'onnx/decoder_model_merged_quantized.onnx',
+                sha256: 'cc9f3cd6698a369c6008b41aa60aa3fb3322e7f03c9bdf19d8e6b7200afca4f3',
+                size: 42498870,
+            },
+        ],
+    },
     'moonshine-base-en': {
         id: 'moonshine-base-en',
         version: '0.0.1',

package/lib/types/chat.d.ts

@@ -689,6 +689,11 @@ export interface ChatBaseProps {
          * Absent while its answers are not heard.
          */
         stopSpeaking?: () => void;
+        /**
+         * Marks the answer the person spoke over *interrupted* (VOICE.md
+         * VO-13), for a host whose composer holds the microphone.
+         */
+        markInterrupted: () => void;
     } | null) => void;
     /**
      * Reports whether the agent's voice is speaking, for a host that draws the
@@ -1013,6 +1018,11 @@ export interface ChatBaseProps {
     voice?: ChatVoice;
     /** The agent is speaking: the composer offers *Stop speaking* (`Esc`). */
     voiceSpeaking?: boolean;
+    /**
+     * The sentence the host's speaker is saying, marked in the conversation
+     * (VOICE.md VO-65); a chat that speaks for itself marks its own.
+     */
+    voiceSentence?: string;
     /** Stops the agent's voice: the composer's *Stop speaking*, and the microphone opening (VO-13). */
     onStopSpeaking?: () => void;
     /**

package/lib/voice/ComposerVoice.js

@@ -2,7 +2,7 @@ import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
 import { useMemo, useRef } from 'react';
 import { DeviceHearing } from './hearing';
 import { HandsFree } from './HandsFree';
-import { ServerHearing, hearingPlace } from './serverHearing';
+import { AutoHearing, ServerHearing, hearingPlace } from './serverHearing';
 import { VoiceInput } from './VoiceInput';
 import { VoiceMenu } from './VoiceMenu';
 /** Whether something rides on a run as a spoken message's metadata (VO-27). */
@@ -24,13 +24,20 @@ export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled
             return undefined;
         }
         const device = new DeviceHearing(voice.modelsUrl, voice.engines);
-        return place === 'server' && voice.speechUrl
+        const server = voice.speechUrl
             ? new ServerHearing(device, {
                 speechUrl: voice.speechUrl,
                 token: voice.token,
                 detect: voice.detect,
             })
-            : device;
+            : undefined;
+        // `where: auto` (§5): the device when it hears the language and its model
+        // loads, the server otherwise — and back to the device when the server
+        // refuses (a visitor's day spent).
+        if (server && !voice.detect && (voice.where ?? 'auto') === 'auto') {
+            return new AutoHearing(device, server, voice.language);
+        }
+        return place === 'server' && server ? server : device;
     }, [
         voice.input,
         voice.modelsUrl,
@@ -39,6 +46,8 @@ export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled
         voice.speechUrl,
         voice.token,
         voice.detect,
+        voice.where,
+        voice.language,
     ]);
     // What the box held when the microphone opened: the words heard go after it.
     const base = useRef('');

package/lib/voice/captions.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Captions always (VOICE.md VO-65): the sentence being said is marked in the
+ * open conversation, not by colour alone. The answer that holds it is marked
+ * `aria-current` and `data-voice-saying` with a bar beside it; the sentence
+ * itself is underlined in place through the CSS Custom Highlight API, which
+ * draws over the text without touching the DOM React owns. A browser without
+ * that API keeps the answer's mark.
+ *
+ * @module voice/captions
+ */
+import { type RefObject } from 'react';
+/** The name the sentence's highlight is registered under. */
+export declare const SAID_HIGHLIGHT = "voice-said";
+/**
+ * The id of the answer a sentence is being said from: the newest answer of
+ * the agent whose words hold it, or none.
+ */
+export declare function saidMessageId(items: readonly unknown[], sentence: string | undefined): string | undefined;
+/**
+ * Where a sentence is in what an element shows, as a `Range` over its text
+ * — across the bold, the links and the line breaks it is drawn with — or
+ * none when the element does not show it as said (a code block said as *a
+ * code block*).
+ */
+export declare function sentenceRange(root: Node, sentence: string): Range | undefined;
+/**
+ * Underlines `sentence` in the answer marked `data-voice-saying` under
+ * `root`, for as long as it is said; the mark goes when it ends. Drawn
+ * again whenever `version` changes (the answer still being written).
+ */
+export declare function useSaidSentence(root: RefObject<HTMLElement | null>, sentence: string | undefined, version: unknown): void;

package/lib/voice/captions.js

@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * Captions always (VOICE.md VO-65): the sentence being said is marked in the
+ * open conversation, not by colour alone. The answer that holds it is marked
+ * `aria-current` and `data-voice-saying` with a bar beside it; the sentence
+ * itself is underlined in place through the CSS Custom Highlight API, which
+ * draws over the text without touching the DOM React owns. A browser without
+ * that API keeps the answer's mark.
+ *
+ * @module voice/captions
+ */
+import { useEffect } from 'react';
+import { plainWords } from './sentences';
+import { answerText } from './useSpokenAnswers';
+/** The name the sentence's highlight is registered under. */
+export const SAID_HIGHLIGHT = 'voice-said';
+/** What the sentence being said looks like: underlined, thick, offset. */
+const SAID_STYLE = `::highlight(${SAID_HIGHLIGHT}) {
+  text-decoration: underline;
+  text-decoration-thickness: 2px;
+  text-underline-offset: 3px;
+  background-color: var(--bgColor-accent-muted, rgba(84, 174, 255, 0.2));
+}`;
+/** Words compared as words: spaces collapsed, case kept. */
+function squeezed(text) {
+    return text.replace(/\s+/g, ' ').trim();
+}
+/**
+ * The id of the answer a sentence is being said from: the newest answer of
+ * the agent whose words hold it, or none.
+ */
+export function saidMessageId(items, sentence) {
+    const said = squeezed(sentence ?? '');
+    if (!said) {
+        return undefined;
+    }
+    for (let index = items.length - 1; index >= 0; index -= 1) {
+        const text = answerText(items[index]);
+        if (text !== undefined && squeezed(plainWords(text)).includes(said)) {
+            const id = items[index].id;
+            return typeof id === 'string' ? id : undefined;
+        }
+    }
+    return undefined;
+}
+/**
+ * Where a sentence is in what an element shows, as a `Range` over its text
+ * — across the bold, the links and the line breaks it is drawn with — or
+ * none when the element does not show it as said (a code block said as *a
+ * code block*).
+ */
+export function sentenceRange(root, sentence) {
+    const said = squeezed(sentence);
+    if (!said || !root.ownerDocument) {
+        return undefined;
+    }
+    const document = root.ownerDocument;
+    const walker = document.createTreeWalker(root, 4 /* NodeFilter.SHOW_TEXT */);
+    // The text, spaces collapsed, and where each of its characters came from.
+    let text = '';
+    const from = [];
+    let space = true;
+    for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+        const value = node.nodeValue ?? '';
+        for (let offset = 0; offset < value.length; offset += 1) {
+            const char = value[offset];
+            if (/\s/.test(char)) {
+                if (space) {
+                    continue;
+                }
+                space = true;
+                text += ' ';
+            }
+            else {
+                space = false;
+                text += char;
+            }
+            from.push({ node, offset });
+        }
+    }
+    const start = text.indexOf(said);
+    if (start < 0) {
+        return undefined;
+    }
+    const first = from[start];
+    const last = from[start + said.length - 1];
+    const range = document.createRange();
+    range.setStart(first.node, first.offset);
+    range.setEnd(last.node, last.offset + 1);
+    return range;
+}
+function registry() {
+    const css = globalThis.CSS;
+    return css?.highlights;
+}
+function highlightOf(range) {
+    const Highlight = globalThis.Highlight;
+    return Highlight ? new Highlight(range) : undefined;
+}
+let styled = false;
+function ensureStyle(document) {
+    if (styled || document.getElementById('datalayer-voice-said')) {
+        styled = true;
+        return;
+    }
+    const style = document.createElement('style');
+    style.id = 'datalayer-voice-said';
+    style.textContent = SAID_STYLE;
+    document.head.appendChild(style);
+    styled = true;
+}
+/**
+ * Underlines `sentence` in the answer marked `data-voice-saying` under
+ * `root`, for as long as it is said; the mark goes when it ends. Drawn
+ * again whenever `version` changes (the answer still being written).
+ */
+export function useSaidSentence(root, sentence, version) {
+    useEffect(() => {
+        const highlights = registry();
+        const element = root.current?.querySelector('[data-voice-saying]');
+        if (!highlights || !element || !sentence) {
+            return;
+        }
+        const underline = () => {
+            const range = sentenceRange(element, sentence);
+            const highlight = range ? highlightOf(range) : undefined;
+            if (!highlight) {
+                return false;
+            }
+            ensureStyle(element.ownerDocument);
+            highlights.set(SAID_HIGHLIGHT, highlight);
+            return true;
+        };
+        // The answer's Markdown may be drawn after this effect: wait for it.
+        let watcher;
+        if (!underline() && typeof MutationObserver !== 'undefined') {
+            watcher = new MutationObserver(() => {
+                if (underline()) {
+                    watcher?.disconnect();
+                }
+            });
+            watcher.observe(element, {
+                childList: true,
+                subtree: true,
+                characterData: true,
+            });
+        }
+        return () => {
+            watcher?.disconnect();
+            highlights.delete(SAID_HIGHLIGHT);
+        };
+    }, [root, sentence, version]);
+}

package/lib/voice/handsFree.d.ts

@@ -56,3 +56,15 @@ export declare function turnEnded(heard: HeardSoFarLike, turnSilenceMs?: number)
  * answer's echo — the browser's echo cancellation keeps that out.
  */
 export declare function bargedIn(state: HandsFreeState, heard: HeardSoFarLike): boolean;
+/** A message as the conversation holds it: enough to find the answer spoken over. */
+export interface ConversationItemLike {
+    id: string;
+    role?: string;
+}
+/**
+ * The answer the person spoke over (VO-13): the agent's last message after
+ * the person's last one — still being written, or written and being said.
+ * `undefined` while the agent has not answered the turn yet: there is
+ * nothing to mark.
+ */
+export declare function answerSpokenOver(items: readonly ConversationItemLike[]): string | undefined;

package/lib/voice/handsFree.js

@@ -46,3 +46,21 @@ export function turnEnded(heard, turnSilenceMs = TURN_SILENCE_MS) {
 export function bargedIn(state, heard) {
     return (state === 'speaking' || state === 'thinking') && heard.spoke;
 }
+/**
+ * The answer the person spoke over (VO-13): the agent's last message after
+ * the person's last one — still being written, or written and being said.
+ * `undefined` while the agent has not answered the turn yet: there is
+ * nothing to mark.
+ */
+export function answerSpokenOver(items) {
+    for (let i = items.length - 1; i >= 0; i -= 1) {
+        const role = items[i].role;
+        if (role === 'user') {
+            return undefined;
+        }
+        if (role === 'assistant') {
+            return items[i].id;
+        }
+    }
+    return undefined;
+}

package/lib/voice/serverHearing.d.ts

@@ -12,6 +12,9 @@ export declare const STREAM_FINAL_MS = 60000;
 export declare const SERVER_HEARING_MODEL = "whisper-large-v3-turbo";
 /** Where a chat's voice hears: in the browser, or on Datalayer's servers. */
 export type HearingPlace = 'device' | 'server';
+/** The speech service refused a press, in its own sentence (a limit, a visitor's day). */
+export declare class ServerRefused extends Error {
+}
 /** What decides where a chat's voice hears. */
 export interface HearingAsked {
     language: string;
@@ -88,3 +91,29 @@ export declare class ServerHearing implements Hearing {
     hearSoFar(audio: Float32Array, language?: string): Promise<HeardSoFar>;
     hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
 }
+/** Said when the device's model did not load and the server will hear instead. */
+export declare const MOVED_TO_SERVER = "This device could not load its model: what you say will be heard on Datalayer's servers instead. Press the microphone again to agree.";
+/**
+ * `where: auto` (§5): each press heard on the device when a model there
+ * hears the language and loads, on the speech service otherwise.
+ *
+ * The device's model failing to load moves the hearing to the server, and
+ * the press is refused in a sentence so that the server's own consent is
+ * asked at the next one. The server refusing a press — a limit, a visitor's
+ * day spent — moves it back to the device when a model there hears the
+ * language, and the press is heard there: the sound then stays in the page.
+ */
+export declare class AutoHearing implements Hearing {
+    private readonly device;
+    private readonly server;
+    private readonly hearsHere;
+    private place;
+    /** The device's model did not load: it is not tried again on this page. */
+    private deviceBroken;
+    constructor(device: Hearing, server: Hearing, language: string, hearsHere?: (language: string) => boolean);
+    get where(): HearingPlace;
+    private get current();
+    ready(language: string, progress?: HearingProgress): Promise<void>;
+    hearSoFar(audio: Float32Array, language: string): Promise<HeardSoFar>;
+    hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
+}