@datalayer/agent-runtimes 1.3.63 → 1.3.64

package/lib/chat/ChatFloating.d.ts

@@ -16,6 +16,7 @@
  * @module chat/ChatFloating
  */
 import React from 'react';
+import type { ChatVoice } from '../voice';
 import { type AssistantCharacter } from './assistant/characters';
 import type { AssistantCharacterData } from './assistant/formats/types';
 import { type AssistantSaying } from './assistant/state';
@@ -149,10 +150,17 @@ export interface ChatFloatingProps extends ChatCommonProps {
      * own props (`protocol`, `suggestions`, …) are then not read.
      */
     conversation?: FloatingConversation;
+    /**
+     * Its voice (VOICE.md V1): push-to-talk in the composer, heard on the
+     * device; with `output: 'always'`, its answers said by Datalayer's speech
+     * service as they are written, the assistant's mouth moving with the sound
+     * and its balloon showing the sentence being said.
+     */
+    voice?: ChatVoice;
 }
 /**
  * ChatFloating component
  * A floating chat window built on ChatBase
  */
-export declare function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode, title, description, position, defaultOpen, width, height, showHeader, showButton, showNewChatButton, showClearButton, showSettingsButton, enableKeyboardShortcuts, toggleShortcut, showPoweredBy, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose, escapeToClose, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip, brandColor, buttonGlow, offset, animationDuration, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick, hideMessagesAfterToolUI, defaultViewMode, onViewModeChange, sidebarMountPoint, showPanelBackdrop, availableModels, showModelSelector, showToolsMenu, showSkillsMenu, showTokenUsage, showContextRing, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching, launchingMessage, assistantCharacter, conversation, decisions, }: ChatFloatingProps): React.JSX.Element;
+export declare function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode, title, description, position, defaultOpen, width, height, showHeader, showButton, showNewChatButton, showClearButton, showSettingsButton, enableKeyboardShortcuts, toggleShortcut, showPoweredBy, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose, escapeToClose, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip, brandColor, buttonGlow, offset, animationDuration, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick, hideMessagesAfterToolUI, defaultViewMode, onViewModeChange, sidebarMountPoint, showPanelBackdrop, availableModels, showModelSelector, showToolsMenu, showSkillsMenu, showTokenUsage, showContextRing, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching, launchingMessage, assistantCharacter, conversation, decisions, voice, }: ChatFloatingProps): React.JSX.Element;
 export default ChatFloating;

package/lib/chat/ChatFloating.js

@@ -33,6 +33,8 @@ import { disabledChatViewModes, resolveMountPoint } from './viewModes';
 import { AssistantStage } from './assistant/AssistantStage';
 import { decisionsAskerAt } from './assistant/decisions';
 import { DecisionAsk } from './assistant/DecisionAsk';
+import { ServerSpeaker } from '../voice/speaker';
+import { useSpokenAnswers } from '../voice/useSpokenAnswers';
 import { BalloonApprovalMessage, CONVERSATION_BALLOON_WIDTH, ConversationBalloonClose, balloonTailAt, conversationBalloonHeight, conversationBalloonSx, peekLine, } from './assistant/ConversationBalloon';
 import { SpeechBalloon } from './assistant/SpeechBalloon';
 import { DEFAULT_ASSISTANT_CHARACTER, } from './assistant/characters';
@@ -66,7 +68,7 @@ const clamp = (value, max) => Math.max(8, Math.min(value, Math.max(8, max)));
  * ChatFloating component
  * A floating chat window built on ChatBase
  */
-export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode = true, title = 'Chat', description = 'Start a conversation with the AI agent.', position = 'bottom-right', defaultOpen = false, width = 400, height = 550, showHeader = true, showButton = true, showNewChatButton = true, showClearButton = true, showSettingsButton = false, enableKeyboardShortcuts = true, toggleShortcut = '/', showPoweredBy = true, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose = false, escapeToClose = true, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip = 'Chat with AI', brandColor, buttonGlow = true, offset = 20, animationDuration = 200, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, defaultViewMode = 'floating', onViewModeChange, sidebarMountPoint = null, showPanelBackdrop = false, availableModels, showModelSelector = false, showToolsMenu = false, showSkillsMenu = false, showTokenUsage = true, showContextRing = false, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation = false, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching = false, launchingMessage, assistantCharacter = DEFAULT_ASSISTANT_CHARACTER, conversation, decisions, }) {
+export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode = true, title = 'Chat', description = 'Start a conversation with the AI agent.', position = 'bottom-right', defaultOpen = false, width = 400, height = 550, showHeader = true, showButton = true, showNewChatButton = true, showClearButton = true, showSettingsButton = false, enableKeyboardShortcuts = true, toggleShortcut = '/', showPoweredBy = true, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose = false, escapeToClose = true, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip = 'Chat with AI', brandColor, buttonGlow = true, offset = 20, animationDuration = 200, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, defaultViewMode = 'floating', onViewModeChange, sidebarMountPoint = null, showPanelBackdrop = false, availableModels, showModelSelector = false, showToolsMenu = false, showSkillsMenu = false, showTokenUsage = true, showContextRing = false, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation = false, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching = false, launchingMessage, assistantCharacter = DEFAULT_ASSISTANT_CHARACTER, conversation, decisions, voice, }) {
     // Store-based state
     const storeIsOpen = useChatOpen();
     const storeMessages = useChatMessages();
@@ -129,6 +131,47 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
         pendingApproval: false,
     });
     const [ownAnswering, setAnswering] = useState(false);
+    /*
+     * Its voice (VOICE.md V1): the answers said by the speech service, as
+     * they are written (VO-20), the state following the sound (VO-22).
+     */
+    const [voiceItems, setVoiceItems] = useState([]);
+    const [speakerState, setSpeakerState] = useState({
+        speaking: false,
+    });
+    const speaker = useMemo(() => voice && voice.output === 'always' && voice.speechUrl
+        ? new ServerSpeaker({
+            speechUrl: voice.speechUrl,
+            voice: voice.voice,
+            language: voice.language,
+            token: voice.token,
+            onState: setSpeakerState,
+        })
+        : undefined, [voice?.output, voice?.speechUrl]);
+    useEffect(() => {
+        speaker?.update({
+            voice: voice?.voice,
+            language: voice?.language,
+            token: voice?.token,
+        });
+    }, [speaker, voice?.voice, voice?.language, voice?.token]);
+    useEffect(() => () => speaker?.close(), [speaker]);
+    // A browser plays only what a gesture allowed: the first one does.
+    useEffect(() => {
+        if (!speaker) {
+            return;
+        }
+        const unlock = () => speaker.unlock();
+        window.addEventListener('pointerdown', unlock, { once: true });
+        window.addEventListener('keydown', unlock, { once: true });
+        return () => {
+            window.removeEventListener('pointerdown', unlock);
+            window.removeEventListener('keydown', unlock);
+        };
+    }, [speaker]);
+    useSpokenAnswers(voiceItems, chatBusy, !!speaker, speaker);
+    const stopSpeaking = useCallback(() => speaker?.stop(), [speaker]);
+    const mouthLevel = useMemo(() => (speaker ? () => speaker.level() : undefined), [speaker]);
     const [arriving, setArriving] = useState(false);
     const [leaving, setLeaving] = useState(false);
     const [assistantAway, setAssistantAway] = useState(keptAway);
@@ -173,6 +216,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
         arriving,
         leaving,
         speaking: answering,
+        voicing: speakerState.speaking,
     });
     const unheard = !!saying && saying.id !== heardId;
     /*
@@ -202,21 +246,27 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
         : undefined, [decisions?.serverUrl, decisions?.token]);
     const assistantBalloon = assistantState === 'paused'
         ? { text: ASSISTANT_WORDS.paused }
-        : balloonApproval
-            ? { text: ASSISTANT_WORDS.approval, approval: balloonApproval }
-            : assistantState === 'waiting'
-                ? { text: ASSISTANT_WORDS.waiting }
-                : unheard && saying
-                    ? {
-                        // A peek: the first words; the rest is in the conversation.
-                        ...peekLine(saying.text),
-                        onDismiss: () => {
-                            setHeardId(saying.id);
-                            setFreshSaying(false);
-                        },
-                    }
-                    : { text: description };
-    const balloonInsists = assistantState === 'greeting' ||
+        : speakerState.sentence
+            ? // Heard: the sentence being said is the one in the balloon (VO-26).
+                { text: speakerState.sentence }
+            : speakerState.refused
+                ? { text: speakerState.refused }
+                : balloonApproval
+                    ? { text: ASSISTANT_WORDS.approval, approval: balloonApproval }
+                    : assistantState === 'waiting'
+                        ? { text: ASSISTANT_WORDS.waiting }
+                        : unheard && saying
+                            ? {
+                                // A peek: the first words; the rest is in the conversation.
+                                ...peekLine(saying.text),
+                                onDismiss: () => {
+                                    setHeardId(saying.id);
+                                    setFreshSaying(false);
+                                },
+                            }
+                            : { text: description };
+    const balloonInsists = !!speakerState.sentence ||
+        assistantState === 'greeting' ||
         assistantState === 'waiting' ||
         !!balloonApproval ||
         (unheard && (assistantState === 'speaking' || freshSaying));
@@ -233,6 +283,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
             ? previous
             : tool);
         setAnswering(newestIsAnswer(items));
+        setVoiceItems(items);
         const said = latestSaying(items);
         setSaying(previous => previous?.id === said?.id && previous?.text === said?.text
             ? previous
@@ -906,7 +957,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
                     ...poweredByProps,
                 }, renderToolResult: renderToolResult, description: description, onStateUpdate: onStateUpdate, onNewChat: onNewChat, suggestions: suggestions, submitOnSuggestionClick: submitOnSuggestionClick, hideMessagesAfterToolUI: hideMessagesAfterToolUI, avatarConfig: {
                     showAvatars: true,
-                }, placeholder: "Type a message...", backgroundColor: "canvas.subtle", frontendTools: frontendTools, showModelSelector: showModelSelector, availableModels: availableModels, showToolsMenu: showToolsMenu, showSkillsMenu: showSkillsMenu, showTokenUsage: showTokenUsage, showContextRing: showContextRing, runtimeId: runtimeId, historyEndpoint: historyEndpoint, historyAuthToken: historyAuthToken, pendingPrompt: pendingPrompt, showInformation: showInformation, onInformationClick: onInformationClick, onToolCallStart: onToolCallStart, onToolCallComplete: onToolCallComplete, showToolApprovalBanner: showToolApprovalBanner, pendingApprovals: pendingApprovals, onApproveApproval: onApproveApproval, onRejectApproval: onRejectApproval, ...panelProps, ...assistantChat, onLoadingChange: handleLoadingChange, onDisplayItemsChange: handleDisplayItemsChange, children: children }))] }));
+                }, placeholder: "Type a message...", backgroundColor: "canvas.subtle", frontendTools: frontendTools, showModelSelector: showModelSelector, availableModels: availableModels, showToolsMenu: showToolsMenu, showSkillsMenu: showSkillsMenu, showTokenUsage: showTokenUsage, showContextRing: showContextRing, runtimeId: runtimeId, historyEndpoint: historyEndpoint, historyAuthToken: historyAuthToken, pendingPrompt: pendingPrompt, showInformation: showInformation, onInformationClick: onInformationClick, onToolCallStart: onToolCallStart, onToolCallComplete: onToolCallComplete, showToolApprovalBanner: showToolApprovalBanner, pendingApprovals: pendingApprovals, onApproveApproval: onApproveApproval, onRejectApproval: onRejectApproval, voice: voice, voiceSpeaking: speakerState.speaking, onStopSpeaking: speaker ? stopSpeaking : undefined, ...panelProps, ...assistantChat, onLoadingChange: handleLoadingChange, onDisplayItemsChange: handleDisplayItemsChange, children: children }))] }));
     /*
      * The floating popup's button. Words not yet heard make it blink for the
      * person's attention, until the chat is opened; hovered, they show in a
@@ -954,7 +1005,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
                         left: px(stageDrag.position.left),
                         top: px(stageDrag.position.top),
                     }
-                    : getPositionStyles(), stageRef: stageRef, onDragStart: stageDrag.onHandlePointerDown, open: isOpen, onToggle: handleToggle, balloon: assistantBalloon, insist: balloonInsists, onDismiss: dismissAssistant, ownRef: popupRef })), showButton && !isOpen && !assistantShown && (_jsx(Box, { sx: {
+                    : getPositionStyles(), stageRef: stageRef, onDragStart: stageDrag.onHandlePointerDown, open: isOpen, onToggle: handleToggle, balloon: assistantBalloon, insist: balloonInsists, onDismiss: dismissAssistant, ownRef: popupRef, mouthLevel: mouthLevel })), showButton && !isOpen && !assistantShown && (_jsx(Box, { sx: {
                     ...getPositionStyles(),
                 }, children: _jsxs(Box, { sx: {
                         position: 'relative',

package/lib/chat/assistant/AssistantStage.d.ts

@@ -94,6 +94,19 @@ export interface AssistantStageProps {
      * decision*, and stays while its form or its answer is on screen.
      */
     decide?: DecisionAsker;
+    /**
+     * How loud its voice is now, between 0 and 1, while it is heard (VOICE.md
+     * VO-23): the mouth opens with the sound, in three openings, instead of
+     * the talking loop. Still, for a reader who asks for reduced motion.
+     */
+    mouthLevel?: () => number;
 }
-export declare function AssistantStage({ character, state, size, place, stageRef, onDragStart, open, onToggle, balloon, insist, onDismiss, ownRef, decide, }: AssistantStageProps): JSX.Element;
+/** The openings of the mouth, by level: shut, a little, half, wide. */
+export declare function mouthOpening(level: number): 0 | 1 | 2 | 3;
+/**
+ * Moves the mouth with the voice: the opening is written on the stage as
+ * `data-assistant-mouth`, each frame, without drawing the character again.
+ */
+export declare function useMouth(stageRef: RefObject<HTMLElement | null>, mouthLevel: (() => number) | undefined, speaking: boolean): void;
+export declare function AssistantStage({ character, state, size, place, stageRef, onDragStart, open, onToggle, balloon, insist, onDismiss, ownRef, decide, mouthLevel, }: AssistantStageProps): JSX.Element;
 export default AssistantStage;

package/lib/chat/assistant/AssistantStage.js

@@ -180,7 +180,36 @@ export function useKeepClear(stageRef, own, paused) {
     }, [stageRef, own, paused]);
     return obstructed ? 'obstacle' : pointerBusy ? 'pointer' : undefined;
 }
-export function AssistantStage({ character, state, size = 88, place, stageRef, onDragStart, open, onToggle, balloon, insist = false, onDismiss, ownRef, decide, }) {
+/** The openings of the mouth, by level: shut, a little, half, wide. */
+export function mouthOpening(level) {
+    return level < 0.08 ? 0 : level < 0.3 ? 1 : level < 0.6 ? 2 : 3;
+}
+/**
+ * Moves the mouth with the voice: the opening is written on the stage as
+ * `data-assistant-mouth`, each frame, without drawing the character again.
+ */
+export function useMouth(stageRef, mouthLevel, speaking) {
+    useEffect(() => {
+        const stage = stageRef.current;
+        const still = typeof window !== 'undefined' &&
+            !!window.matchMedia?.('(prefers-reduced-motion: reduce)').matches;
+        if (!stage || !mouthLevel || !speaking || still) {
+            stage?.removeAttribute('data-assistant-mouth');
+            return;
+        }
+        let frame = 0;
+        const move = () => {
+            stage.setAttribute('data-assistant-mouth', String(mouthOpening(mouthLevel())));
+            frame = requestAnimationFrame(move);
+        };
+        frame = requestAnimationFrame(move);
+        return () => {
+            cancelAnimationFrame(frame);
+            stage.removeAttribute('data-assistant-mouth');
+        };
+    }, [stageRef, mouthLevel, speaking]);
+}
+export function AssistantStage({ character, state, size = 88, place, stageRef, onDragStart, open, onToggle, balloon, insist = false, onDismiss, ownRef, decide, mouthLevel, }) {
     // A shipped one by id, a drawing contributed by a plugin (T-24), or a
     // character read from a file (T-26).
     const shipped = typeof character === 'string'
@@ -199,6 +228,7 @@ export function AssistantStage({ character, state, size = 88, place, stageRef, o
     // Where the press began: a press that moves is a drag, not a click.
     const pressedAt = useRef(null);
     const aside = useKeepClear(stageRef, ownRef, menuOpen);
+    useMouth(stageRef, mouthLevel, state === 'speaking');
     // A decision being asked, or its answer, keeps the balloon up.
     const [deciding, setDeciding] = useState(false);
     const showBalloon = !aside && !open && !!balloon && (hovered || insist || deciding);
@@ -261,6 +291,23 @@ export function AssistantStage({ character, state, size = 88, place, stageRef, o
             '&[data-assistant-state="speaking"] .assistant-body': {
                 animation: 'assistantFloat 1.6s ease-in-out infinite',
             },
+            // Heard (VO-23): the mouth opens with the sound, not on a loop.
+            '&[data-assistant-mouth] .assistant-mouth': {
+                animation: 'none',
+                transition: 'transform 60ms linear',
+            },
+            '&[data-assistant-mouth="0"] .assistant-mouth': {
+                transform: 'scaleY(1)',
+            },
+            '&[data-assistant-mouth="1"] .assistant-mouth': {
+                transform: 'scaleY(1.5)',
+            },
+            '&[data-assistant-mouth="2"] .assistant-mouth': {
+                transform: 'scaleY(2.1)',
+            },
+            '&[data-assistant-mouth="3"] .assistant-mouth': {
+                transform: 'scaleY(2.8)',
+            },
             // Paused (R-17): it dozes — its eyes shut, its breath slow, a little
             // greyed — and, still, it is asleep: the shut eyes are not a motion.
             '&[data-assistant-state="paused"] .assistant-body': {

package/lib/chat/assistant/state.d.ts

@@ -17,6 +17,11 @@ export interface AssistantMoment {
     leaving?: boolean;
     /** The answer is arriving: words are being written. */
     speaking?: boolean;
+    /**
+     * Its answer is being heard: sound is playing (VOICE.md VO-22). It speaks
+     * while the audio plays, even after the text has ended.
+     */
+    voicing?: boolean;
 }
 /**
  * The state to act, from the application's presence and the moment.
@@ -24,7 +29,8 @@ export interface AssistantMoment {
  * Leaving wins, being short; then paused — a paused application neither
  * greets nor speaks, it dozes; then arriving; then waiting for the person,
  * which must never be missed; then speaking over thinking, since words
- * arriving are what the person watches; then the presence as it is.
+ * arriving are what the person watches — and while its voice is heard, even
+ * once the words have all arrived; then the presence as it is.
  */
 export declare function assistantStateOf(presence: PresenceState, moment?: AssistantMoment): AssistantState;
 /**

package/lib/chat/assistant/state.js

@@ -8,7 +8,8 @@
  * Leaving wins, being short; then paused — a paused application neither
  * greets nor speaks, it dozes; then arriving; then waiting for the person,
  * which must never be missed; then speaking over thinking, since words
- * arriving are what the person watches; then the presence as it is.
+ * arriving are what the person watches — and while its voice is heard, even
+ * once the words have all arrived; then the presence as it is.
  */
 export function assistantStateOf(presence, moment = {}) {
     if (moment.leaving) {
@@ -23,7 +24,7 @@ export function assistantStateOf(presence, moment = {}) {
     if (presence === 'waiting') {
         return 'waiting';
     }
-    if (moment.speaking && presence !== 'idle') {
+    if ((moment.speaking && presence !== 'idle') || moment.voicing) {
         return 'speaking';
     }
     return presence;

package/lib/chat/base/ChatBase.js

@@ -45,6 +45,8 @@ import { PoweredByTag } from '../display/PoweredByTag';
 import { EphemeralSurfaceControl } from '../EphemeralSurfaceControl';
 import { ChatMessageList, } from '../messages/ChatMessageList';
 import { InputPrompt } from '../prompt/InputPrompt';
+import { DeviceHearing } from '../../voice/hearing';
+import { VoiceInput } from '../../voice/VoiceInput';
 import { isFloatingChatViewMode } from '../viewModes';
 import { ToolApprovalBanner, ToolApprovalDialog, } from '../tools';
 import { EphemeralNotebook } from '../notebook/EphemeralNotebook';
@@ -509,7 +511,7 @@ function ChatBaseInner({ title, subtitle, showHeader = false, showTokenUsage = t
 // Mode selection
 useStore: useStoreMode = true, protocol: protocolRaw, onSendMessage, onSendReady, onLoadingChange, onContextSnapshot, enableStreaming = false, 
 // Extended props
-brandIcon, avatarConfig, headerButtons, showPoweredBy = false, poweredByProps, emptyState, renderToolResult, renderHeader, showTurnFooter = true, notebookToolSurfacesId, footerContent, welcome, trailingContent, showInformation = false, onInformationClick, headerContent, children, borderRadius, backgroundColor, border, boxShadow, compact = false, placeholder, description = 'Start a conversation with the AI agent.', onStateUpdate, onNewChat, onClear, onMessagesChange, onItemsChange, onDisplayItemsChange, autoFocus = false, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, focusTrigger, frontendTools: frontendToolsProp, enableEphemeralNotebook = false, initialEphemeralNotebookOpen = true, onEphemeralNotebookOpenChange, enableEphemeralDocument = false, initialEphemeralSurfaceMode, onEphemeralSurfaceModeChange, collapsed = false, onExpandFromCollapsed, ephemeralNotebookToolbar, ephemeralNotebookToolbarExtraItems, ephemeralDocumentToolbarExtraItems, ephemeralNotebookCollaborationProvider, ephemeralNotebookCollaborationDocumentId, ephemeralDocumentCollaboration, ephemeralRuntimeOverride, 
+brandIcon, avatarConfig, headerButtons, showPoweredBy = false, poweredByProps, emptyState, renderToolResult, renderHeader, showTurnFooter = true, notebookToolSurfacesId, footerContent, welcome, trailingContent, voice, voiceSpeaking = false, onStopSpeaking, showInformation = false, onInformationClick, headerContent, children, borderRadius, backgroundColor, border, boxShadow, compact = false, placeholder, description = 'Start a conversation with the AI agent.', onStateUpdate, onNewChat, onClear, onMessagesChange, onItemsChange, onDisplayItemsChange, autoFocus = false, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, focusTrigger, frontendTools: frontendToolsProp, enableEphemeralNotebook = false, initialEphemeralNotebookOpen = true, onEphemeralNotebookOpenChange, enableEphemeralDocument = false, initialEphemeralSurfaceMode, onEphemeralSurfaceModeChange, collapsed = false, onExpandFromCollapsed, ephemeralNotebookToolbar, ephemeralNotebookToolbarExtraItems, ephemeralDocumentToolbarExtraItems, ephemeralNotebookCollaborationProvider, ephemeralNotebookCollaborationDocumentId, ephemeralDocumentCollaboration, ephemeralRuntimeOverride, 
 // Tool invocation hooks
 onToolCallStart, onToolCallComplete, 
 // Identity/Authorization props
@@ -2819,10 +2821,17 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
     // ========================================================================
     // handleSend
     // ========================================================================
+    // Voice (VOICE.md V1): what was said and put in the composer, until sent.
+    const spokenRef = useRef(undefined);
+    const hearing = useMemo(() => voice && voice.input !== 'off'
+        ? new DeviceHearing(voice.modelsUrl, voice.engines)
+        : undefined, [voice?.input, voice?.modelsUrl, voice?.engines]);
     const handleSend = useCallback(async (messageOverride, 
     // AG-UI's: what goes with the run besides the conversation — an
     // application's page says there what it did (LOOP R-04).
-    forwardedProps) => {
+    forwardedPropsGiven, 
+    // How the message was heard, when it was said (VOICE.md VO-27).
+    spokenGiven) => {
         const messageContent = (messageOverride ?? input).trim();
         if (!messageContent || isLoading)
             return;
@@ -2831,7 +2840,15 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
         stoppedRef.current = false;
         suppressAssistantTextForToolOnlyRef.current =
             isToolCallOnlyPrompt(messageContent);
-        const userMessage = createUserMessage(messageContent);
+        // Said, then sent: by the microphone at once, or from the composer it
+        // was put in — the words may have been edited, it was still said.
+        const spoken = spokenGiven ??
+            (messageOverride === undefined ? spokenRef.current : undefined);
+        spokenRef.current = undefined;
+        const forwardedProps = spoken
+            ? { ...(forwardedPropsGiven ?? {}), voice: spoken }
+            : forwardedPropsGiven;
+        const userMessage = createUserMessage(messageContent, spoken ? { ...spoken } : undefined);
         const currentMessages = displayItems.filter((item) => !isToolCallMessage(item));
         const allMessages = [...currentMessages, userMessage];
         setDisplayItems(prev => [...prev, userMessage]);
@@ -3513,7 +3530,18 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
             mx: 'auto',
             bg: 'canvas.default',
         }, children: [_jsx(ChatMessageList, { displayItems: welcomedItems, isLoading: isLoading, isStreaming: isStreaming, showLoadingIndicator: showLoadingIndicator, hideMessagesAfterToolUI: hideMessagesAfterToolUI, avatarConfig: defaultAvatarConfig, padding: padding, renderToolResult: effectiveRenderToolResult, approvalConfig: approvalConfig, messagesEndRef: messagesEndRef, onRespond: handleRespond, showTurnFooters: showTurnFooter, agentUsage: agentUsage, onRemoveItems: handleRemoveItems, mcpServers: filteredMcpServers, emptyContent: launching ? null : (_jsx(ChatEmptyState, { emptyState: emptyState, brandIcon: brandIcon, description: resolvedDescription, suggestions: resolvedSuggestions, submitOnSuggestionClick: submitOnSuggestionClick, onSuggestionSubmit: handleSuggestionSubmit, onSuggestionFill: handleSuggestionFill })) }), trailingContent] }));
-    const inputToolbar = showInput ? (_jsx(InputPrompt, { input: input, setInput: setInput, isLoading: isLoading, kernelStatus: liveKernelStatus, connectionConfirmed: connectionConfirmed, placeholder: placeholder, 
+    // The microphone, in the composer's footer (VO-10).
+    const voiceControl = hearing && voice ? (_jsx(VoiceInput, { hearing: hearing, language: voice.language, consentKey: voice.consentKey ?? 'chat', disabled: disabled || launching, speaking: voiceSpeaking, onStopSpeaking: onStopSpeaking, onListen: onStopSpeaking, onTranscript: (transcript) => {
+            if (voice.sendWhatISay) {
+                void handleSend(transcript.text, undefined, transcript.metadata);
+                return;
+            }
+            spokenRef.current = transcript.metadata;
+            setInput(input.trim()
+                ? `${input.trim()} ${transcript.text}`
+                : transcript.text);
+        } })) : undefined;
+    const inputToolbar = showInput ? (_jsx(InputPrompt, { footerExtras: voiceControl, input: input, setInput: setInput, isLoading: isLoading, kernelStatus: liveKernelStatus, connectionConfirmed: connectionConfirmed, placeholder: placeholder, 
         /*
           The same openers the empty state offers, typed into the box they would
           be typed into.

package/lib/chat/messages/ChatMessageList.d.ts

@@ -1,9 +1,3 @@
-/**
- * MessageList — Renders the scrollable list of chat messages, tool calls,
- * and the typing indicator.
- *
- * @module chat/messages/MessageList
- */
 import { type ReactNode, type RefObject } from 'react';
 import { type MarkedMcpServer } from '../marks';
 import type { DisplayItem, AvatarConfig, RenderToolResult } from '../../types/chat';

package/lib/chat/messages/ChatMessageList.js

@@ -9,6 +9,7 @@ import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-run
  *
  * @module chat/messages/MessageList
  */
+import { MicrophoneIcon } from '@datalayer/icons-react';
 import { Fragment, useState, useCallback, useEffect, useMemo, useRef, } from 'react';
 import { Text, Spinner, Link, Label, RelativeTime } from '@primer/react';
 import { ArrowRightIcon, BroadcastIcon, ClockIcon, DependabotIcon, PersonIcon, SkipIcon, ToolsIcon, XCircleFillIcon, } from '@primer/octicons-react';
@@ -708,11 +709,16 @@ export function ChatMessageList({ displayItems, isLoading, isStreaming, showLoad
                                         // the agent reads — see `formSubmission`.
                                         const text = getMessageText(message);
                                         const submission = parseFormSubmission(text);
-                                        return submission ? (_jsx(FormSubmissionMessage, { submission: submission })) : (_jsx(Text, { sx: {
+                                        return submission ? (_jsx(FormSubmissionMessage, { submission: submission })) : (_jsxs(Text, { sx: {
                                                 fontSize: 1,
                                                 whiteSpace: 'pre-wrap',
                                                 wordBreak: 'break-word',
-                                            }, children: text }));
+                                            }, children: [message.metadata?.input === 'voice' ? (_jsx(Box, { as: "span", "data-spoken": "", title: "Said aloud", "aria-label": "Said aloud:", sx: {
+                                                        display: 'inline-flex',
+                                                        verticalAlign: 'text-bottom',
+                                                        mr: 1,
+                                                        opacity: 0.85,
+                                                    }, children: _jsx(MicrophoneIcon, { size: 14, "aria-hidden": "true" }) })) : null, text] }));
                                     })()) : message.live && !getMessageText(message) ? (_jsx(TypingDots, { size: 6 })) : (_jsx(Box, { sx: streamdownMarkdownStyles, children: _jsx(Streamdown, { children: normalizeAssistantMarkdown(getMessageText(message) || (isStreaming ? '...' : '')) }) }))] }), renderMessageFooter ? (_jsx(Box, { sx: {
                                     display: 'flex',
                                     gap: 2,

package/lib/examples/VoiceChatExample.d.ts

@@ -0,0 +1,20 @@
+/**
+ * A voice chat with the floating assistant (VOICE.md V1).
+ *
+ * Hold the microphone (or `Ctrl`+`Space`), speak, let go: what you said is
+ * heard in this page — Moonshine for English, Whisper for French, Silero VAD
+ * keeping only the speech — and put in the composer, marked as said. Send
+ * it, and the answer is read aloud as it is written, sentence by sentence,
+ * by Datalayer's speech service (Kokoro, on ai-agents), the assistant's
+ * mouth moving with the sound and its balloon showing the sentence said.
+ *
+ * Three places, each named in the address when it is not this machine's:
+ * - `?agentRuntimesUrl=` — the agent-runtimes server and its `assistant`
+ *   agent (http://127.0.0.1:8765);
+ * - `?voiceModelsUrl=` — the browser's models, as pinned
+ *   (`scripts/voice/serve_store.py`, http://127.0.0.1:8770);
+ * - `?speechUrl=` — ai-agents' speech service (http://127.0.0.1:4401).
+ */
+import React from 'react';
+declare const VoiceChatExample: React.FC;
+export default VoiceChatExample;

package/lib/examples/VoiceChatExample.js

@@ -0,0 +1,64 @@
+import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * A voice chat with the floating assistant (VOICE.md V1).
+ *
+ * Hold the microphone (or `Ctrl`+`Space`), speak, let go: what you said is
+ * heard in this page — Moonshine for English, Whisper for French, Silero VAD
+ * keeping only the speech — and put in the composer, marked as said. Send
+ * it, and the answer is read aloud as it is written, sentence by sentence,
+ * by Datalayer's speech service (Kokoro, on ai-agents), the assistant's
+ * mouth moving with the sound and its balloon showing the sentence said.
+ *
+ * Three places, each named in the address when it is not this machine's:
+ * - `?agentRuntimesUrl=` — the agent-runtimes server and its `assistant`
+ *   agent (http://127.0.0.1:8765);
+ * - `?voiceModelsUrl=` — the browser's models, as pinned
+ *   (`scripts/voice/serve_store.py`, http://127.0.0.1:8770);
+ * - `?speechUrl=` — ai-agents' speech service (http://127.0.0.1:4401).
+ */
+import { useMemo, useState } from 'react';
+import { Heading, Link, SegmentedControl, Text } from '@primer/react';
+import { Box } from '@datalayer/primer-addons';
+import { useSimpleAuthStore } from '@datalayer/core/lib/views/otel';
+import { ThemedProvider } from './utils/themedProvider';
+import { ChatFloating } from '../chat';
+import { VOICE_CATALOGUE, voiceFor } from '../specs/voices';
+function fromAddress(name, otherwise) {
+    return ((typeof window === 'undefined'
+        ? null
+        : new URLSearchParams(window.location.search).get(name)) || otherwise).replace(/\/+$/, '');
+}
+const SERVER = fromAddress('agentRuntimesUrl', 'http://127.0.0.1:8765');
+const MODELS = fromAddress('voiceModelsUrl', 'http://127.0.0.1:8770');
+const SPEECH = fromAddress('speechUrl', 'http://127.0.0.1:4401');
+/** The packages that hear, loaded the first time the microphone is used. */
+const ENGINES = {
+    transformers: () => import('@huggingface/transformers'),
+    vad: () => import('@ricky0123/vad-web'),
+};
+const LANGUAGES = [
+    { tag: 'en-US', label: 'English' },
+    { tag: 'fr-FR', label: 'Français' },
+];
+const VoiceChatExample = () => {
+    const { token } = useSimpleAuthStore();
+    const [language, setLanguage] = useState('en-US');
+    const voiceSpec = voiceFor(language) ?? VOICE_CATALOGUE['kokoro-af-heart'];
+    const voice = useMemo(() => ({
+        language,
+        input: 'push_to_talk',
+        output: 'always',
+        voice: voiceSpec.id,
+        modelsUrl: MODELS,
+        engines: ENGINES,
+        speechUrl: SPEECH,
+        token: token || undefined,
+        consentKey: 'voice-chat-example',
+    }), [language, voiceSpec.id, token]);
+    return (_jsx(ThemedProvider, { children: _jsxs(Box, { sx: { minHeight: '100vh', bg: 'canvas.default', p: 4 }, children: [_jsxs(Box, { sx: { maxWidth: 720, mx: 'auto' }, children: [_jsx(Heading, { as: "h1", sx: { mb: 2 }, children: "Voice chat" }), _jsxs(Text, { as: "p", sx: { color: 'fg.muted', mb: 3 }, children: ["Talk to the assistant instead of typing: hold the microphone in its composer, or ", _jsx("kbd", { children: "Ctrl" }), "+", _jsx("kbd", { children: "Space" }), ", speak, and let go. What you say is heard in this page and never leaves it; the words go in the message box for you to send. Its answers are read aloud as they are written, by Datalayer\u2019s speech service, and not kept.", ' ', _jsx("kbd", { children: "Esc" }), " stops its voice."] }), _jsxs(Box, { sx: { display: 'flex', alignItems: 'center', gap: 3, mb: 3 }, children: [_jsx(SegmentedControl, { "aria-label": "Language", children: LANGUAGES.map(option => (_jsx(SegmentedControl.Button, { selected: language === option.tag, onClick: () => setLanguage(option.tag), "data-voice-language": option.tag, children: option.label }, option.tag))) }), _jsxs(Text, { sx: { color: 'fg.muted', fontSize: 1 }, children: ["Voice: ", voiceSpec.name] })] }), voiceSpec.attribution && (_jsx(Text, { as: "p", sx: { fontSize: 0, color: 'fg.muted', mb: 3 }, children: voiceSpec.attribution })), _jsxs(Text, { as: "p", sx: { fontSize: 0, color: 'fg.muted' }, children: ["Models from ", MODELS, ", speech from ", SPEECH, ", the agent at ", SERVER, ". See", ' ', _jsx(Link, { href: "https://agent-runtimes.datalayer.tech/chat/voice", children: "Voice" }), ' ', "for how to run each."] })] }), _jsx(ChatFloating, { defaultViewMode: "assistant", assistantCharacter: "paperclip", protocol: "vercel-ai", endpoint: `${SERVER}/api/v1/vercel-ai/assistant`, title: "Assistant", description: "Hello! Hold the microphone and ask me anything.", position: "bottom-right", useStore: false, voice: voice })] }) }));
+};
+export default VoiceChatExample;

package/lib/examples/example-selector.js

@@ -88,6 +88,7 @@ export const EXAMPLE_ENTRIES = [
     makeEntry('DecksAgent', () => import('./DecksAgent'), 'A deck beside the chat: the decks plugin in a Loop, and an agent that writes presentations as data.'),
     makeEntry('CellExample', () => import('./CellExample'), 'Simple cell example.'),
     makeEntry('ChatAssistantExample', () => import('./ChatAssistantExample'), 'The floating assistant: the chat as a character on the page.'),
+    makeEntry('VoiceChatExample', () => import('./VoiceChatExample'), 'A voice chat with the floating assistant: push-to-talk heard in the page, answers read aloud by the speech service, the mouth moving with the sound.'),
     makeEntry('ChatAssistantGalleryExample', () => import('./ChatAssistantGalleryExample'), 'Every representation of the floating assistant: each character in each state, its balloon, light and dark, still or moving, one at a time or as a grid.'),
     makeEntry('ChatCustomExample', () => import('./ChatCustomExample'), 'Custom chat experience composition example.'),
     makeEntry('ChatExample', () => import('./ChatExample'), 'Baseline chat integration example.'),

package/lib/examples/main.js

@@ -70,6 +70,9 @@ const ANONYMOUS_EXAMPLES = new Set([
     // An application beside its computer, on the local agent-runtimes server:
     // nothing allocated on Datalayer, and the machine itself needs no token.
     'LoopAppComputerExample',
+    // The Sales and Accounting team: Sales in the page on a visitor's trial
+    // key, Accounting on the local server, which answers the machine itself.
+    'AgentA2ATeamExample',
     // Decide, on the local agent-runtimes server as above: Jev is asked
     // through the server's ai-inference, with the server's own key.
     'AgentDecideExample',
@@ -88,6 +91,9 @@ const ANONYMOUS_EXAMPLES = new Set([
     'ChatAssistantExample',
     // Every character in every state, from static data: no agent at all.
     'ChatAssistantGalleryExample',
+    // Voice: heard in the page, the agent on the local server, the speech
+    // service named in the address.
+    'VoiceChatExample',
 ]);
 const getExampleGroup = (id) => {
     if (id === 'AgentspecsExample' ||
@@ -108,7 +114,7 @@ const getExampleGroup = (id) => {
     // runtime: checkpoints, hooks, memory, guardrails…
     if (id.startsWith('Agent'))
         return 'Capabilities';
-    if (id.startsWith('Chat'))
+    if (id.startsWith('Chat') || id === 'VoiceChatExample')
         return 'Chat';
     // The document examples: the ones on the Lexical editor, and the page
     // with a document on it.

package/lib/loop/apps/appspec.d.ts

@@ -24,7 +24,7 @@
  *
  * @module loop/apps/appspec
  */
-import type { AppAccent, AppAssistantCharacter, AppEmbedMode, AppKind, AppLayout, AppSpec } from '../../types/agentspecs';
+import type { AppAccent, AppAssistantCharacter, AppEmbedMode, AppVoiceSpec, AppKind, AppLayout, AppSpec } from '../../types/agentspecs';
 /** The version of the spec itself. */
 export declare const APP_SCHEMA = "loop.app/v1";
 /** The face of an application that has not chosen one. */
@@ -51,6 +51,8 @@ type Data = Record<string, unknown>;
 export declare function retentionDays(keepFor: string): number;
 /** An application with nothing said: what *Start blank* opens. */
 export declare function emptyAppspec(kind?: AppKind): AppSpec;
+/** An application's voice (VOICE.md VO-41): off unless said. */
+export declare const DEFAULT_VOICE: AppVoiceSpec;
 /** What reading a document gives: the application, and what the document got wrong. */
 export interface ParsedAppspec {
     app: AppSpec;

package/lib/loop/apps/appspec.js

@@ -214,6 +214,26 @@ function parseSurface(data) {
         composedAt: text(data.composed_at),
     };
 }
+/** An application's voice (VOICE.md VO-41): off unless said. */
+export const DEFAULT_VOICE = {
+    enabled: false,
+    input: 'push_to_talk',
+    output: 'on_request',
+    voice: '',
+    language: '',
+    where: 'auto',
+};
+function parseVoice(data) {
+    const voice = isData(data) ? data : {};
+    return {
+        enabled: voice.enabled === true,
+        input: oneOf(voice.input, ['off', 'push_to_talk', 'hands_free'], DEFAULT_VOICE.input),
+        output: oneOf(voice.output, ['off', 'on_request', 'always'], DEFAULT_VOICE.output),
+        voice: text(voice.voice),
+        language: text(voice.language),
+        where: oneOf(voice.where, ['auto', 'device', 'server'], DEFAULT_VOICE.where),
+    };
+}
 function parseInterface(data, kind) {
     const parsed = {
         layout: oneOf(data.layout, ['chat', 'page', 'split'], DEFAULT_LAYOUTS[kind]),
@@ -225,6 +245,7 @@ function parseInterface(data, kind) {
         })),
         settings: records(data.settings).map(parseSetting),
         components: texts(data.components),
+        voice: parseVoice(data.voice),
     };
     if (isData(data.surface)) {
         parsed.surface = parseSurface(data.surface);
@@ -493,6 +514,16 @@ function dumpInterface(spec, kind) {
     if (spec.assistant) {
         writer.data.assistant = spec.assistant;
     }
+    // Its voice, when it says one (VO-41): a spec made before voice has none.
+    if (spec.voice) {
+        writer.part('voice', new Writer()
+            .value('enabled', spec.voice.enabled, DEFAULT_VOICE.enabled)
+            .value('input', spec.voice.input, DEFAULT_VOICE.input)
+            .value('output', spec.voice.output, DEFAULT_VOICE.output)
+            .text('voice', spec.voice.voice)
+            .text('language', spec.voice.language)
+            .value('where', spec.voice.where, DEFAULT_VOICE.where).data);
+    }
     return writer.data;
 }
 function dumpDecision(decision) {