@datalayer/agent-runtimes 1.3.63 → 1.3.65

package/lib/chat/ChatFloating.d.ts

@@ -16,6 +16,7 @@
  * @module chat/ChatFloating
  */
 import React from 'react';
+import type { ChatVoice } from '../voice';
 import { type AssistantCharacter } from './assistant/characters';
 import type { AssistantCharacterData } from './assistant/formats/types';
 import { type AssistantSaying } from './assistant/state';
@@ -149,10 +150,17 @@ export interface ChatFloatingProps extends ChatCommonProps {
      * own props (`protocol`, `suggestions`, …) are then not read.
      */
     conversation?: FloatingConversation;
+    /**
+     * Its voice (VOICE.md V1): push-to-talk in the composer, heard on the
+     * device; with `output: 'always'`, its answers said by Datalayer's speech
+     * service as they are written, the assistant's mouth moving with the sound
+     * and its balloon showing the sentence being said.
+     */
+    voice?: ChatVoice;
 }
 /**
  * ChatFloating component
  * A floating chat window built on ChatBase
  */
-export declare function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode, title, description, position, defaultOpen, width, height, showHeader, showButton, showNewChatButton, showClearButton, showSettingsButton, enableKeyboardShortcuts, toggleShortcut, showPoweredBy, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose, escapeToClose, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip, brandColor, buttonGlow, offset, animationDuration, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick, hideMessagesAfterToolUI, defaultViewMode, onViewModeChange, sidebarMountPoint, showPanelBackdrop, availableModels, showModelSelector, showToolsMenu, showSkillsMenu, showTokenUsage, showContextRing, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching, launchingMessage, assistantCharacter, conversation, decisions, }: ChatFloatingProps): React.JSX.Element;
+export declare function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode, title, description, position, defaultOpen, width, height, showHeader, showButton, showNewChatButton, showClearButton, showSettingsButton, enableKeyboardShortcuts, toggleShortcut, showPoweredBy, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose, escapeToClose, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip, brandColor, buttonGlow, offset, animationDuration, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick, hideMessagesAfterToolUI, defaultViewMode, onViewModeChange, sidebarMountPoint, showPanelBackdrop, availableModels, showModelSelector, showToolsMenu, showSkillsMenu, showTokenUsage, showContextRing, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching, launchingMessage, assistantCharacter, conversation, decisions, voice, }: ChatFloatingProps): React.JSX.Element;
 export default ChatFloating;

package/lib/chat/ChatFloating.js

@@ -33,6 +33,8 @@ import { disabledChatViewModes, resolveMountPoint } from './viewModes';
 import { AssistantStage } from './assistant/AssistantStage';
 import { decisionsAskerAt } from './assistant/decisions';
 import { DecisionAsk } from './assistant/DecisionAsk';
+import { ServerSpeaker } from '../voice/speaker';
+import { useSpokenAnswers } from '../voice/useSpokenAnswers';
 import { BalloonApprovalMessage, CONVERSATION_BALLOON_WIDTH, ConversationBalloonClose, balloonTailAt, conversationBalloonHeight, conversationBalloonSx, peekLine, } from './assistant/ConversationBalloon';
 import { SpeechBalloon } from './assistant/SpeechBalloon';
 import { DEFAULT_ASSISTANT_CHARACTER, } from './assistant/characters';
@@ -66,7 +68,7 @@ const clamp = (value, max) => Math.max(8, Math.min(value, Math.max(8, max)));
  * ChatFloating component
  * A floating chat window built on ChatBase
  */
-export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode = true, title = 'Chat', description = 'Start a conversation with the AI agent.', position = 'bottom-right', defaultOpen = false, width = 400, height = 550, showHeader = true, showButton = true, showNewChatButton = true, showClearButton = true, showSettingsButton = false, enableKeyboardShortcuts = true, toggleShortcut = '/', showPoweredBy = true, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose = false, escapeToClose = true, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip = 'Chat with AI', brandColor, buttonGlow = true, offset = 20, animationDuration = 200, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, defaultViewMode = 'floating', onViewModeChange, sidebarMountPoint = null, showPanelBackdrop = false, availableModels, showModelSelector = false, showToolsMenu = false, showSkillsMenu = false, showTokenUsage = true, showContextRing = false, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation = false, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching = false, launchingMessage, assistantCharacter = DEFAULT_ASSISTANT_CHARACTER, conversation, decisions, }) {
+export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode = true, title = 'Chat', description = 'Start a conversation with the AI agent.', position = 'bottom-right', defaultOpen = false, width = 400, height = 550, showHeader = true, showButton = true, showNewChatButton = true, showClearButton = true, showSettingsButton = false, enableKeyboardShortcuts = true, toggleShortcut = '/', showPoweredBy = true, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose = false, escapeToClose = true, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip = 'Chat with AI', brandColor, buttonGlow = true, offset = 20, animationDuration = 200, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, defaultViewMode = 'floating', onViewModeChange, sidebarMountPoint = null, showPanelBackdrop = false, availableModels, showModelSelector = false, showToolsMenu = false, showSkillsMenu = false, showTokenUsage = true, showContextRing = false, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation = false, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching = false, launchingMessage, assistantCharacter = DEFAULT_ASSISTANT_CHARACTER, conversation, decisions, voice, }) {
     // Store-based state
     const storeIsOpen = useChatOpen();
     const storeMessages = useChatMessages();
@@ -129,6 +131,47 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
         pendingApproval: false,
     });
     const [ownAnswering, setAnswering] = useState(false);
+    /*
+     * Its voice (VOICE.md V1): the answers said by the speech service, as
+     * they are written (VO-20), the state following the sound (VO-22).
+     */
+    const [voiceItems, setVoiceItems] = useState([]);
+    const [speakerState, setSpeakerState] = useState({
+        speaking: false,
+    });
+    const speaker = useMemo(() => voice && voice.output === 'always' && voice.speechUrl
+        ? new ServerSpeaker({
+            speechUrl: voice.speechUrl,
+            voice: voice.voice,
+            language: voice.language,
+            token: voice.token,
+            onState: setSpeakerState,
+        })
+        : undefined, [voice?.output, voice?.speechUrl]);
+    useEffect(() => {
+        speaker?.update({
+            voice: voice?.voice,
+            language: voice?.language,
+            token: voice?.token,
+        });
+    }, [speaker, voice?.voice, voice?.language, voice?.token]);
+    useEffect(() => () => speaker?.close(), [speaker]);
+    // A browser plays only what a gesture allowed: the first one does.
+    useEffect(() => {
+        if (!speaker) {
+            return;
+        }
+        const unlock = () => speaker.unlock();
+        window.addEventListener('pointerdown', unlock, { once: true });
+        window.addEventListener('keydown', unlock, { once: true });
+        return () => {
+            window.removeEventListener('pointerdown', unlock);
+            window.removeEventListener('keydown', unlock);
+        };
+    }, [speaker]);
+    useSpokenAnswers(voiceItems, chatBusy, !!speaker, speaker);
+    const stopSpeaking = useCallback(() => speaker?.stop(), [speaker]);
+    const mouthLevel = useMemo(() => (speaker ? () => speaker.level() : undefined), [speaker]);
     const [arriving, setArriving] = useState(false);
     const [leaving, setLeaving] = useState(false);
     const [assistantAway, setAssistantAway] = useState(keptAway);
@@ -173,6 +216,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
         arriving,
         leaving,
         speaking: answering,
+        voicing: speakerState.speaking,
     });
     const unheard = !!saying && saying.id !== heardId;
     /*
@@ -202,21 +246,27 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
         : undefined, [decisions?.serverUrl, decisions?.token]);
     const assistantBalloon = assistantState === 'paused'
         ? { text: ASSISTANT_WORDS.paused }
-        : balloonApproval
-            ? { text: ASSISTANT_WORDS.approval, approval: balloonApproval }
-            : assistantState === 'waiting'
-                ? { text: ASSISTANT_WORDS.waiting }
-                : unheard && saying
-                    ? {
-                        // A peek: the first words; the rest is in the conversation.
-                        ...peekLine(saying.text),
-                        onDismiss: () => {
-                            setHeardId(saying.id);
-                            setFreshSaying(false);
-                        },
-                    }
-                    : { text: description };
-    const balloonInsists = assistantState === 'greeting' ||
+        : speakerState.sentence
+            ? // Heard: the sentence being said is the one in the balloon (VO-26).
+                { text: speakerState.sentence }
+            : speakerState.refused
+                ? { text: speakerState.refused }
+                : balloonApproval
+                    ? { text: ASSISTANT_WORDS.approval, approval: balloonApproval }
+                    : assistantState === 'waiting'
+                        ? { text: ASSISTANT_WORDS.waiting }
+                        : unheard && saying
+                            ? {
+                                // A peek: the first words; the rest is in the conversation.
+                                ...peekLine(saying.text),
+                                onDismiss: () => {
+                                    setHeardId(saying.id);
+                                    setFreshSaying(false);
+                                },
+                            }
+                            : { text: description };
+    const balloonInsists = !!speakerState.sentence ||
+        assistantState === 'greeting' ||
         assistantState === 'waiting' ||
         !!balloonApproval ||
         (unheard && (assistantState === 'speaking' || freshSaying));
@@ -233,6 +283,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
             ? previous
             : tool);
         setAnswering(newestIsAnswer(items));
+        setVoiceItems(items);
         const said = latestSaying(items);
         setSaying(previous => previous?.id === said?.id && previous?.text === said?.text
             ? previous
@@ -906,7 +957,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
                     ...poweredByProps,
                 }, renderToolResult: renderToolResult, description: description, onStateUpdate: onStateUpdate, onNewChat: onNewChat, suggestions: suggestions, submitOnSuggestionClick: submitOnSuggestionClick, hideMessagesAfterToolUI: hideMessagesAfterToolUI, avatarConfig: {
                     showAvatars: true,
-                }, placeholder: "Type a message...", backgroundColor: "canvas.subtle", frontendTools: frontendTools, showModelSelector: showModelSelector, availableModels: availableModels, showToolsMenu: showToolsMenu, showSkillsMenu: showSkillsMenu, showTokenUsage: showTokenUsage, showContextRing: showContextRing, runtimeId: runtimeId, historyEndpoint: historyEndpoint, historyAuthToken: historyAuthToken, pendingPrompt: pendingPrompt, showInformation: showInformation, onInformationClick: onInformationClick, onToolCallStart: onToolCallStart, onToolCallComplete: onToolCallComplete, showToolApprovalBanner: showToolApprovalBanner, pendingApprovals: pendingApprovals, onApproveApproval: onApproveApproval, onRejectApproval: onRejectApproval, ...panelProps, ...assistantChat, onLoadingChange: handleLoadingChange, onDisplayItemsChange: handleDisplayItemsChange, children: children }))] }));
+                }, placeholder: "Type a message...", backgroundColor: "canvas.subtle", frontendTools: frontendTools, showModelSelector: showModelSelector, availableModels: availableModels, showToolsMenu: showToolsMenu, showSkillsMenu: showSkillsMenu, showTokenUsage: showTokenUsage, showContextRing: showContextRing, runtimeId: runtimeId, historyEndpoint: historyEndpoint, historyAuthToken: historyAuthToken, pendingPrompt: pendingPrompt, showInformation: showInformation, onInformationClick: onInformationClick, onToolCallStart: onToolCallStart, onToolCallComplete: onToolCallComplete, showToolApprovalBanner: showToolApprovalBanner, pendingApprovals: pendingApprovals, onApproveApproval: onApproveApproval, onRejectApproval: onRejectApproval, voice: voice, voiceSpeaking: speakerState.speaking, onStopSpeaking: speaker ? stopSpeaking : undefined, ...panelProps, ...assistantChat, onLoadingChange: handleLoadingChange, onDisplayItemsChange: handleDisplayItemsChange, children: children }))] }));
     /*
      * The floating popup's button. Words not yet heard make it blink for the
      * person's attention, until the chat is opened; hovered, they show in a
@@ -954,7 +1005,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
                         left: px(stageDrag.position.left),
                         top: px(stageDrag.position.top),
                     }
-                    : getPositionStyles(), stageRef: stageRef, onDragStart: stageDrag.onHandlePointerDown, open: isOpen, onToggle: handleToggle, balloon: assistantBalloon, insist: balloonInsists, onDismiss: dismissAssistant, ownRef: popupRef })), showButton && !isOpen && !assistantShown && (_jsx(Box, { sx: {
+                    : getPositionStyles(), stageRef: stageRef, onDragStart: stageDrag.onHandlePointerDown, open: isOpen, onToggle: handleToggle, balloon: assistantBalloon, insist: balloonInsists, onDismiss: dismissAssistant, ownRef: popupRef, mouthLevel: mouthLevel })), showButton && !isOpen && !assistantShown && (_jsx(Box, { sx: {
                     ...getPositionStyles(),
                 }, children: _jsxs(Box, { sx: {
                         position: 'relative',

package/lib/chat/assistant/AssistantStage.d.ts

@@ -94,6 +94,19 @@ export interface AssistantStageProps {
      * decision*, and stays while its form or its answer is on screen.
      */
     decide?: DecisionAsker;
+    /**
+     * How loud its voice is now, between 0 and 1, while it is heard (VOICE.md
+     * VO-23): the mouth opens with the sound, in three openings, instead of
+     * the talking loop. Still, for a reader who asks for reduced motion.
+     */
+    mouthLevel?: () => number;
 }
-export declare function AssistantStage({ character, state, size, place, stageRef, onDragStart, open, onToggle, balloon, insist, onDismiss, ownRef, decide, }: AssistantStageProps): JSX.Element;
+/** The openings of the mouth, by level: shut, a little, half, wide. */
+export declare function mouthOpening(level: number): 0 | 1 | 2 | 3;
+/**
+ * Moves the mouth with the voice: the opening is written on the stage as
+ * `data-assistant-mouth`, each frame, without drawing the character again.
+ */
+export declare function useMouth(stageRef: RefObject<HTMLElement | null>, mouthLevel: (() => number) | undefined, speaking: boolean): void;
+export declare function AssistantStage({ character, state, size, place, stageRef, onDragStart, open, onToggle, balloon, insist, onDismiss, ownRef, decide, mouthLevel, }: AssistantStageProps): JSX.Element;
 export default AssistantStage;

package/lib/chat/assistant/AssistantStage.js

@@ -180,7 +180,36 @@ export function useKeepClear(stageRef, own, paused) {
     }, [stageRef, own, paused]);
     return obstructed ? 'obstacle' : pointerBusy ? 'pointer' : undefined;
 }
-export function AssistantStage({ character, state, size = 88, place, stageRef, onDragStart, open, onToggle, balloon, insist = false, onDismiss, ownRef, decide, }) {
+/** The openings of the mouth, by level: shut, a little, half, wide. */
+export function mouthOpening(level) {
+    return level < 0.08 ? 0 : level < 0.3 ? 1 : level < 0.6 ? 2 : 3;
+}
+/**
+ * Moves the mouth with the voice: the opening is written on the stage as
+ * `data-assistant-mouth`, each frame, without drawing the character again.
+ */
+export function useMouth(stageRef, mouthLevel, speaking) {
+    useEffect(() => {
+        const stage = stageRef.current;
+        const still = typeof window !== 'undefined' &&
+            !!window.matchMedia?.('(prefers-reduced-motion: reduce)').matches;
+        if (!stage || !mouthLevel || !speaking || still) {
+            stage?.removeAttribute('data-assistant-mouth');
+            return;
+        }
+        let frame = 0;
+        const move = () => {
+            stage.setAttribute('data-assistant-mouth', String(mouthOpening(mouthLevel())));
+            frame = requestAnimationFrame(move);
+        };
+        frame = requestAnimationFrame(move);
+        return () => {
+            cancelAnimationFrame(frame);
+            stage.removeAttribute('data-assistant-mouth');
+        };
+    }, [stageRef, mouthLevel, speaking]);
+}
+export function AssistantStage({ character, state, size = 88, place, stageRef, onDragStart, open, onToggle, balloon, insist = false, onDismiss, ownRef, decide, mouthLevel, }) {
     // A shipped one by id, a drawing contributed by a plugin (T-24), or a
     // character read from a file (T-26).
     const shipped = typeof character === 'string'
@@ -199,6 +228,7 @@ export function AssistantStage({ character, state, size = 88, place, stageRef, o
     // Where the press began: a press that moves is a drag, not a click.
     const pressedAt = useRef(null);
     const aside = useKeepClear(stageRef, ownRef, menuOpen);
+    useMouth(stageRef, mouthLevel, state === 'speaking');
     // A decision being asked, or its answer, keeps the balloon up.
     const [deciding, setDeciding] = useState(false);
     const showBalloon = !aside && !open && !!balloon && (hovered || insist || deciding);
@@ -261,6 +291,23 @@ export function AssistantStage({ character, state, size = 88, place, stageRef, o
             '&[data-assistant-state="speaking"] .assistant-body': {
                 animation: 'assistantFloat 1.6s ease-in-out infinite',
             },
+            // Heard (VO-23): the mouth opens with the sound, not on a loop.
+            '&[data-assistant-mouth] .assistant-mouth': {
+                animation: 'none',
+                transition: 'transform 60ms linear',
+            },
+            '&[data-assistant-mouth="0"] .assistant-mouth': {
+                transform: 'scaleY(1)',
+            },
+            '&[data-assistant-mouth="1"] .assistant-mouth': {
+                transform: 'scaleY(1.5)',
+            },
+            '&[data-assistant-mouth="2"] .assistant-mouth': {
+                transform: 'scaleY(2.1)',
+            },
+            '&[data-assistant-mouth="3"] .assistant-mouth': {
+                transform: 'scaleY(2.8)',
+            },
             // Paused (R-17): it dozes — its eyes shut, its breath slow, a little
             // greyed — and, still, it is asleep: the shut eyes are not a motion.
             '&[data-assistant-state="paused"] .assistant-body': {

package/lib/chat/assistant/state.d.ts

@@ -17,6 +17,11 @@ export interface AssistantMoment {
     leaving?: boolean;
     /** The answer is arriving: words are being written. */
     speaking?: boolean;
+    /**
+     * Its answer is being heard: sound is playing (VOICE.md VO-22). It speaks
+     * while the audio plays, even after the text has ended.
+     */
+    voicing?: boolean;
 }
 /**
  * The state to act, from the application's presence and the moment.
@@ -24,7 +29,8 @@ export interface AssistantMoment {
  * Leaving wins, being short; then paused — a paused application neither
  * greets nor speaks, it dozes; then arriving; then waiting for the person,
  * which must never be missed; then speaking over thinking, since words
- * arriving are what the person watches; then the presence as it is.
+ * arriving are what the person watches — and while its voice is heard, even
+ * once the words have all arrived; then the presence as it is.
  */
 export declare function assistantStateOf(presence: PresenceState, moment?: AssistantMoment): AssistantState;
 /**

package/lib/chat/assistant/state.js

@@ -8,7 +8,8 @@
  * Leaving wins, being short; then paused — a paused application neither
  * greets nor speaks, it dozes; then arriving; then waiting for the person,
  * which must never be missed; then speaking over thinking, since words
- * arriving are what the person watches; then the presence as it is.
+ * arriving are what the person watches — and while its voice is heard, even
+ * once the words have all arrived; then the presence as it is.
  */
 export function assistantStateOf(presence, moment = {}) {
     if (moment.leaving) {
@@ -23,7 +24,7 @@ export function assistantStateOf(presence, moment = {}) {
     if (presence === 'waiting') {
         return 'waiting';
     }
-    if (moment.speaking && presence !== 'idle') {
+    if ((moment.speaking && presence !== 'idle') || moment.voicing) {
         return 'speaking';
     }
     return presence;

package/lib/chat/base/ChatBase.js

@@ -45,6 +45,8 @@ import { PoweredByTag } from '../display/PoweredByTag';
 import { EphemeralSurfaceControl } from '../EphemeralSurfaceControl';
 import { ChatMessageList, } from '../messages/ChatMessageList';
 import { InputPrompt } from '../prompt/InputPrompt';
+import { DeviceHearing } from '../../voice/hearing';
+import { VoiceInput } from '../../voice/VoiceInput';
 import { isFloatingChatViewMode } from '../viewModes';
 import { ToolApprovalBanner, ToolApprovalDialog, } from '../tools';
 import { EphemeralNotebook } from '../notebook/EphemeralNotebook';
@@ -509,7 +511,7 @@ function ChatBaseInner({ title, subtitle, showHeader = false, showTokenUsage = t
 // Mode selection
 useStore: useStoreMode = true, protocol: protocolRaw, onSendMessage, onSendReady, onLoadingChange, onContextSnapshot, enableStreaming = false, 
 // Extended props
-brandIcon, avatarConfig, headerButtons, showPoweredBy = false, poweredByProps, emptyState, renderToolResult, renderHeader, showTurnFooter = true, notebookToolSurfacesId, footerContent, welcome, trailingContent, showInformation = false, onInformationClick, headerContent, children, borderRadius, backgroundColor, border, boxShadow, compact = false, placeholder, description = 'Start a conversation with the AI agent.', onStateUpdate, onNewChat, onClear, onMessagesChange, onItemsChange, onDisplayItemsChange, autoFocus = false, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, focusTrigger, frontendTools: frontendToolsProp, enableEphemeralNotebook = false, initialEphemeralNotebookOpen = true, onEphemeralNotebookOpenChange, enableEphemeralDocument = false, initialEphemeralSurfaceMode, onEphemeralSurfaceModeChange, collapsed = false, onExpandFromCollapsed, ephemeralNotebookToolbar, ephemeralNotebookToolbarExtraItems, ephemeralDocumentToolbarExtraItems, ephemeralNotebookCollaborationProvider, ephemeralNotebookCollaborationDocumentId, ephemeralDocumentCollaboration, ephemeralRuntimeOverride, 
+brandIcon, avatarConfig, headerButtons, showPoweredBy = false, poweredByProps, emptyState, renderToolResult, renderHeader, showTurnFooter = true, notebookToolSurfacesId, footerContent, welcome, trailingContent, voice, voiceSpeaking = false, onStopSpeaking, showInformation = false, onInformationClick, headerContent, children, borderRadius, backgroundColor, border, boxShadow, compact = false, placeholder, description = 'Start a conversation with the AI agent.', onStateUpdate, onNewChat, onClear, onMessagesChange, onItemsChange, onDisplayItemsChange, autoFocus = false, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, focusTrigger, frontendTools: frontendToolsProp, enableEphemeralNotebook = false, initialEphemeralNotebookOpen = true, onEphemeralNotebookOpenChange, enableEphemeralDocument = false, initialEphemeralSurfaceMode, onEphemeralSurfaceModeChange, collapsed = false, onExpandFromCollapsed, ephemeralNotebookToolbar, ephemeralNotebookToolbarExtraItems, ephemeralDocumentToolbarExtraItems, ephemeralNotebookCollaborationProvider, ephemeralNotebookCollaborationDocumentId, ephemeralDocumentCollaboration, ephemeralRuntimeOverride, 
 // Tool invocation hooks
 onToolCallStart, onToolCallComplete, 
 // Identity/Authorization props
@@ -2819,10 +2821,17 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
     // ========================================================================
     // handleSend
     // ========================================================================
+    // Voice (VOICE.md V1): what was said and put in the composer, until sent.
+    const spokenRef = useRef(undefined);
+    const hearing = useMemo(() => voice && voice.input !== 'off'
+        ? new DeviceHearing(voice.modelsUrl, voice.engines)
+        : undefined, [voice?.input, voice?.modelsUrl, voice?.engines]);
     const handleSend = useCallback(async (messageOverride, 
     // AG-UI's: what goes with the run besides the conversation — an
     // application's page says there what it did (LOOP R-04).
-    forwardedProps) => {
+    forwardedPropsGiven, 
+    // How the message was heard, when it was said (VOICE.md VO-27).
+    spokenGiven) => {
         const messageContent = (messageOverride ?? input).trim();
         if (!messageContent || isLoading)
             return;
@@ -2831,7 +2840,15 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
         stoppedRef.current = false;
         suppressAssistantTextForToolOnlyRef.current =
             isToolCallOnlyPrompt(messageContent);
-        const userMessage = createUserMessage(messageContent);
+        // Said, then sent: by the microphone at once, or from the composer it
+        // was put in — the words may have been edited, it was still said.
+        const spoken = spokenGiven ??
+            (messageOverride === undefined ? spokenRef.current : undefined);
+        spokenRef.current = undefined;
+        const forwardedProps = spoken
+            ? { ...(forwardedPropsGiven ?? {}), voice: spoken }
+            : forwardedPropsGiven;
+        const userMessage = createUserMessage(messageContent, spoken ? { ...spoken } : undefined);
         const currentMessages = displayItems.filter((item) => !isToolCallMessage(item));
         const allMessages = [...currentMessages, userMessage];
         setDisplayItems(prev => [...prev, userMessage]);
@@ -3513,7 +3530,18 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
             mx: 'auto',
             bg: 'canvas.default',
         }, children: [_jsx(ChatMessageList, { displayItems: welcomedItems, isLoading: isLoading, isStreaming: isStreaming, showLoadingIndicator: showLoadingIndicator, hideMessagesAfterToolUI: hideMessagesAfterToolUI, avatarConfig: defaultAvatarConfig, padding: padding, renderToolResult: effectiveRenderToolResult, approvalConfig: approvalConfig, messagesEndRef: messagesEndRef, onRespond: handleRespond, showTurnFooters: showTurnFooter, agentUsage: agentUsage, onRemoveItems: handleRemoveItems, mcpServers: filteredMcpServers, emptyContent: launching ? null : (_jsx(ChatEmptyState, { emptyState: emptyState, brandIcon: brandIcon, description: resolvedDescription, suggestions: resolvedSuggestions, submitOnSuggestionClick: submitOnSuggestionClick, onSuggestionSubmit: handleSuggestionSubmit, onSuggestionFill: handleSuggestionFill })) }), trailingContent] }));
-    const inputToolbar = showInput ? (_jsx(InputPrompt, { input: input, setInput: setInput, isLoading: isLoading, kernelStatus: liveKernelStatus, connectionConfirmed: connectionConfirmed, placeholder: placeholder, 
+    // The microphone, in the composer's footer (VO-10).
+    const voiceControl = hearing && voice ? (_jsx(VoiceInput, { hearing: hearing, language: voice.language, consentKey: voice.consentKey ?? 'chat', disabled: disabled || launching, speaking: voiceSpeaking, onStopSpeaking: onStopSpeaking, onListen: onStopSpeaking, onTranscript: (transcript) => {
+            if (voice.sendWhatISay) {
+                void handleSend(transcript.text, undefined, transcript.metadata);
+                return;
+            }
+            spokenRef.current = transcript.metadata;
+            setInput(input.trim()
+                ? `${input.trim()} ${transcript.text}`
+                : transcript.text);
+        } })) : undefined;
+    const inputToolbar = showInput ? (_jsx(InputPrompt, { footerExtras: voiceControl, input: input, setInput: setInput, isLoading: isLoading, kernelStatus: liveKernelStatus, connectionConfirmed: connectionConfirmed, placeholder: placeholder, 
         /*
           The same openers the empty state offers, typed into the box they would
           be typed into.

package/lib/chat/messages/ChatMessageList.d.ts

@@ -1,9 +1,3 @@
-/**
- * MessageList — Renders the scrollable list of chat messages, tool calls,
- * and the typing indicator.
- *
- * @module chat/messages/MessageList
- */
 import { type ReactNode, type RefObject } from 'react';
 import { type MarkedMcpServer } from '../marks';
 import type { DisplayItem, AvatarConfig, RenderToolResult } from '../../types/chat';

package/lib/chat/messages/ChatMessageList.js

@@ -9,6 +9,7 @@ import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-run
  *
  * @module chat/messages/MessageList
  */
+import { MicrophoneIcon } from '@datalayer/icons-react';
 import { Fragment, useState, useCallback, useEffect, useMemo, useRef, } from 'react';
 import { Text, Spinner, Link, Label, RelativeTime } from '@primer/react';
 import { ArrowRightIcon, BroadcastIcon, ClockIcon, DependabotIcon, PersonIcon, SkipIcon, ToolsIcon, XCircleFillIcon, } from '@primer/octicons-react';
@@ -708,11 +709,16 @@ export function ChatMessageList({ displayItems, isLoading, isStreaming, showLoad
                                         // the agent reads — see `formSubmission`.
                                         const text = getMessageText(message);
                                         const submission = parseFormSubmission(text);
-                                        return submission ? (_jsx(FormSubmissionMessage, { submission: submission })) : (_jsx(Text, { sx: {
+                                        return submission ? (_jsx(FormSubmissionMessage, { submission: submission })) : (_jsxs(Text, { sx: {
                                                 fontSize: 1,
                                                 whiteSpace: 'pre-wrap',
                                                 wordBreak: 'break-word',
-                                            }, children: text }));
+                                            }, children: [message.metadata?.input === 'voice' ? (_jsx(Box, { as: "span", "data-spoken": "", title: "Said aloud", "aria-label": "Said aloud:", sx: {
+                                                        display: 'inline-flex',
+                                                        verticalAlign: 'text-bottom',
+                                                        mr: 1,
+                                                        opacity: 0.85,
+                                                    }, children: _jsx(MicrophoneIcon, { size: 14, "aria-hidden": "true" }) })) : null, text] }));
                                     })()) : message.live && !getMessageText(message) ? (_jsx(TypingDots, { size: 6 })) : (_jsx(Box, { sx: streamdownMarkdownStyles, children: _jsx(Streamdown, { children: normalizeAssistantMarkdown(getMessageText(message) || (isStreaming ? '...' : '')) }) }))] }), renderMessageFooter ? (_jsx(Box, { sx: {
                                     display: 'flex',
                                     gap: 2,

package/lib/components/teams/A2ATeamGraph.d.ts

@@ -0,0 +1,58 @@
+/**
+ * A team of two over A2A, as a graph (React Flow): each member is its
+ * character (`AssistantStage`), with its name and where it runs under it, and
+ * one edge links the entry to the peer.
+ *
+ * The edge is still until a message travels on it. While the entry asks, it
+ * flows from the entry to the peer; while the peer answers, from the peer to
+ * the entry ({@link A2ATeamFlow}, from the `A2APeerEvent` phases). A reader
+ * who asks for no motion sees the same thing without the movement: the arrow
+ * at the end the message goes to, and the word for it on the edge.
+ *
+ * The graph is a picture, not an editor: nothing is dragged, panned or
+ * zoomed, and the wheel scrolls the page. Its viewport is worked out from its
+ * box, so it fits a phone as it fits a desk.
+ *
+ * @module components/teams/A2ATeamGraph
+ */
+import type { JSX } from 'react';
+import { type Viewport } from '@xyflow/react';
+import '@xyflow/react/dist/base.css';
+import type { A2ATeamFlow } from './a2aTeamFlow';
+import type { A2ATeamPersona } from './useA2ATeam';
+/** One member of the team, as the graph draws it. */
+export type A2ATeamGraphMember = {
+    id: string;
+    name: string;
+    emoji?: string;
+    /** Its character, as its Appspec's `interface.assistant` names it. */
+    character: string;
+    /** Where it runs, in a few words: "in your browser", "on a runtime". */
+    where: string;
+    persona: A2ATeamPersona;
+    /** A click on the character. */
+    onToggle?: () => void;
+    /** It was sent away, or called back. */
+    onAway?: (away: boolean) => void;
+};
+export type A2ATeamGraphProps = {
+    /** The member that asks: drawn on the left. */
+    entry: A2ATeamGraphMember;
+    /** The member asked over A2A: drawn on the right. */
+    peer: A2ATeamGraphMember;
+    /** Which way the link carries a message now. */
+    flow: A2ATeamFlow;
+    /** Whether the peer is reached: the edge is drawn faint until it is. */
+    connected: boolean;
+    /** What the edge says at rest: `A2A · <the peer's skill>`. */
+    label?: string;
+    /** The character's size, in pixels at full scale. */
+    size?: number;
+};
+/** The words on the edge for a flow. */
+export declare function flowWords(flow: A2ATeamFlow, entry: string, peer: string): string;
+/** The viewport that shows the graph's width whole in a box `width` wide, never larger than drawn. */
+export declare function teamViewport(width: number): Viewport;
+/** Draw a team of two over A2A, the link moving with what travels on it. */
+export declare function A2ATeamGraph({ entry, peer, flow, connected, label, size, }: A2ATeamGraphProps): JSX.Element;
+export default A2ATeamGraph;