@datalayer/agent-runtimes 1.3.62 → 1.3.64

package/lib/chat/ChatFloating.d.ts

@@ -16,6 +16,7 @@
  * @module chat/ChatFloating
  */
 import React from 'react';
+import type { ChatVoice } from '../voice';
 import { type AssistantCharacter } from './assistant/characters';
 import type { AssistantCharacterData } from './assistant/formats/types';
 import { type AssistantSaying } from './assistant/state';
@@ -149,10 +150,17 @@ export interface ChatFloatingProps extends ChatCommonProps {
      * own props (`protocol`, `suggestions`, …) are then not read.
      */
     conversation?: FloatingConversation;
+    /**
+     * Its voice (VOICE.md V1): push-to-talk in the composer, heard on the
+     * device; with `output: 'always'`, its answers said by Datalayer's speech
+     * service as they are written, the assistant's mouth moving with the sound
+     * and its balloon showing the sentence being said.
+     */
+    voice?: ChatVoice;
 }
 /**
  * ChatFloating component
  * A floating chat window built on ChatBase
  */
-export declare function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode, title, description, position, defaultOpen, width, height, showHeader, showButton, showNewChatButton, showClearButton, showSettingsButton, enableKeyboardShortcuts, toggleShortcut, showPoweredBy, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose, escapeToClose, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip, brandColor, buttonGlow, offset, animationDuration, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick, hideMessagesAfterToolUI, defaultViewMode, onViewModeChange, sidebarMountPoint, showPanelBackdrop, availableModels, showModelSelector, showToolsMenu, showSkillsMenu, showTokenUsage, showContextRing, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching, launchingMessage, assistantCharacter, conversation, decisions, }: ChatFloatingProps): React.JSX.Element;
+export declare function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode, title, description, position, defaultOpen, width, height, showHeader, showButton, showNewChatButton, showClearButton, showSettingsButton, enableKeyboardShortcuts, toggleShortcut, showPoweredBy, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose, escapeToClose, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip, brandColor, buttonGlow, offset, animationDuration, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick, hideMessagesAfterToolUI, defaultViewMode, onViewModeChange, sidebarMountPoint, showPanelBackdrop, availableModels, showModelSelector, showToolsMenu, showSkillsMenu, showTokenUsage, showContextRing, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching, launchingMessage, assistantCharacter, conversation, decisions, voice, }: ChatFloatingProps): React.JSX.Element;
 export default ChatFloating;

package/lib/chat/ChatFloating.js

@@ -33,6 +33,8 @@ import { disabledChatViewModes, resolveMountPoint } from './viewModes';
 import { AssistantStage } from './assistant/AssistantStage';
 import { decisionsAskerAt } from './assistant/decisions';
 import { DecisionAsk } from './assistant/DecisionAsk';
+import { ServerSpeaker } from '../voice/speaker';
+import { useSpokenAnswers } from '../voice/useSpokenAnswers';
 import { BalloonApprovalMessage, CONVERSATION_BALLOON_WIDTH, ConversationBalloonClose, balloonTailAt, conversationBalloonHeight, conversationBalloonSx, peekLine, } from './assistant/ConversationBalloon';
 import { SpeechBalloon } from './assistant/SpeechBalloon';
 import { DEFAULT_ASSISTANT_CHARACTER, } from './assistant/characters';
@@ -66,7 +68,7 @@ const clamp = (value, max) => Math.max(8, Math.min(value, Math.max(8, max)));
  * ChatFloating component
  * A floating chat window built on ChatBase
  */
-export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode = true, title = 'Chat', description = 'Start a conversation with the AI agent.', position = 'bottom-right', defaultOpen = false, width = 400, height = 550, showHeader = true, showButton = true, showNewChatButton = true, showClearButton = true, showSettingsButton = false, enableKeyboardShortcuts = true, toggleShortcut = '/', showPoweredBy = true, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose = false, escapeToClose = true, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip = 'Chat with AI', brandColor, buttonGlow = true, offset = 20, animationDuration = 200, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, defaultViewMode = 'floating', onViewModeChange, sidebarMountPoint = null, showPanelBackdrop = false, availableModels, showModelSelector = false, showToolsMenu = false, showSkillsMenu = false, showTokenUsage = true, showContextRing = false, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation = false, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching = false, launchingMessage, assistantCharacter = DEFAULT_ASSISTANT_CHARACTER, conversation, decisions, }) {
+export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useStoreMode = true, title = 'Chat', description = 'Start a conversation with the AI agent.', position = 'bottom-right', defaultOpen = false, width = 400, height = 550, showHeader = true, showButton = true, showNewChatButton = true, showClearButton = true, showSettingsButton = false, enableKeyboardShortcuts = true, toggleShortcut = '/', showPoweredBy = true, promptVariant, mentionableAgents, poweredByProps, clickOutsideToClose = false, escapeToClose = true, className, onSettingsClick, onNewChat, onOpen, onClose, onStateUpdate, children, brandIcon, buttonIcon, buttonTooltip = 'Chat with AI', brandColor, buttonGlow = true, offset = 20, animationDuration = 200, renderToolResult, frontendTools, initialState: _initialState, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, defaultViewMode = 'floating', onViewModeChange, sidebarMountPoint = null, showPanelBackdrop = false, availableModels, showModelSelector = false, showToolsMenu = false, showSkillsMenu = false, showTokenUsage = true, showContextRing = false, themeVariant, themeOverrides, colorMode, runtimeId, historyEndpoint, authToken, historyAuthToken, pendingPrompt, showInformation = false, onInformationClick, onToolCallStart, onToolCallComplete, showToolApprovalBanner, pendingApprovals, onApproveApproval, onRejectApproval, panelProps, launching = false, launchingMessage, assistantCharacter = DEFAULT_ASSISTANT_CHARACTER, conversation, decisions, voice, }) {
     // Store-based state
     const storeIsOpen = useChatOpen();
     const storeMessages = useChatMessages();
@@ -129,6 +131,47 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
         pendingApproval: false,
     });
     const [ownAnswering, setAnswering] = useState(false);
+    /*
+     * Its voice (VOICE.md V1): the answers said by the speech service, as
+     * they are written (VO-20), the state following the sound (VO-22).
+     */
+    const [voiceItems, setVoiceItems] = useState([]);
+    const [speakerState, setSpeakerState] = useState({
+        speaking: false,
+    });
+    const speaker = useMemo(() => voice && voice.output === 'always' && voice.speechUrl
+        ? new ServerSpeaker({
+            speechUrl: voice.speechUrl,
+            voice: voice.voice,
+            language: voice.language,
+            token: voice.token,
+            onState: setSpeakerState,
+        })
+        : undefined, [voice?.output, voice?.speechUrl]);
+    useEffect(() => {
+        speaker?.update({
+            voice: voice?.voice,
+            language: voice?.language,
+            token: voice?.token,
+        });
+    }, [speaker, voice?.voice, voice?.language, voice?.token]);
+    useEffect(() => () => speaker?.close(), [speaker]);
+    // A browser plays only what a gesture allowed: the first one does.
+    useEffect(() => {
+        if (!speaker) {
+            return;
+        }
+        const unlock = () => speaker.unlock();
+        window.addEventListener('pointerdown', unlock, { once: true });
+        window.addEventListener('keydown', unlock, { once: true });
+        return () => {
+            window.removeEventListener('pointerdown', unlock);
+            window.removeEventListener('keydown', unlock);
+        };
+    }, [speaker]);
+    useSpokenAnswers(voiceItems, chatBusy, !!speaker, speaker);
+    const stopSpeaking = useCallback(() => speaker?.stop(), [speaker]);
+    const mouthLevel = useMemo(() => (speaker ? () => speaker.level() : undefined), [speaker]);
     const [arriving, setArriving] = useState(false);
     const [leaving, setLeaving] = useState(false);
     const [assistantAway, setAssistantAway] = useState(keptAway);
@@ -173,6 +216,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
         arriving,
         leaving,
         speaking: answering,
+        voicing: speakerState.speaking,
     });
     const unheard = !!saying && saying.id !== heardId;
     /*
@@ -202,21 +246,27 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
         : undefined, [decisions?.serverUrl, decisions?.token]);
     const assistantBalloon = assistantState === 'paused'
         ? { text: ASSISTANT_WORDS.paused }
-        : balloonApproval
-            ? { text: ASSISTANT_WORDS.approval, approval: balloonApproval }
-            : assistantState === 'waiting'
-                ? { text: ASSISTANT_WORDS.waiting }
-                : unheard && saying
-                    ? {
-                        // A peek: the first words; the rest is in the conversation.
-                        ...peekLine(saying.text),
-                        onDismiss: () => {
-                            setHeardId(saying.id);
-                            setFreshSaying(false);
-                        },
-                    }
-                    : { text: description };
-    const balloonInsists = assistantState === 'greeting' ||
+        : speakerState.sentence
+            ? // Heard: the sentence being said is the one in the balloon (VO-26).
+                { text: speakerState.sentence }
+            : speakerState.refused
+                ? { text: speakerState.refused }
+                : balloonApproval
+                    ? { text: ASSISTANT_WORDS.approval, approval: balloonApproval }
+                    : assistantState === 'waiting'
+                        ? { text: ASSISTANT_WORDS.waiting }
+                        : unheard && saying
+                            ? {
+                                // A peek: the first words; the rest is in the conversation.
+                                ...peekLine(saying.text),
+                                onDismiss: () => {
+                                    setHeardId(saying.id);
+                                    setFreshSaying(false);
+                                },
+                            }
+                            : { text: description };
+    const balloonInsists = !!speakerState.sentence ||
+        assistantState === 'greeting' ||
         assistantState === 'waiting' ||
         !!balloonApproval ||
         (unheard && (assistantState === 'speaking' || freshSaying));
@@ -233,6 +283,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
             ? previous
             : tool);
         setAnswering(newestIsAnswer(items));
+        setVoiceItems(items);
         const said = latestSaying(items);
         setSaying(previous => previous?.id === said?.id && previous?.text === said?.text
             ? previous
@@ -906,7 +957,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
                     ...poweredByProps,
                 }, renderToolResult: renderToolResult, description: description, onStateUpdate: onStateUpdate, onNewChat: onNewChat, suggestions: suggestions, submitOnSuggestionClick: submitOnSuggestionClick, hideMessagesAfterToolUI: hideMessagesAfterToolUI, avatarConfig: {
                     showAvatars: true,
-                }, placeholder: "Type a message...", backgroundColor: "canvas.subtle", frontendTools: frontendTools, showModelSelector: showModelSelector, availableModels: availableModels, showToolsMenu: showToolsMenu, showSkillsMenu: showSkillsMenu, showTokenUsage: showTokenUsage, showContextRing: showContextRing, runtimeId: runtimeId, historyEndpoint: historyEndpoint, historyAuthToken: historyAuthToken, pendingPrompt: pendingPrompt, showInformation: showInformation, onInformationClick: onInformationClick, onToolCallStart: onToolCallStart, onToolCallComplete: onToolCallComplete, showToolApprovalBanner: showToolApprovalBanner, pendingApprovals: pendingApprovals, onApproveApproval: onApproveApproval, onRejectApproval: onRejectApproval, ...panelProps, ...assistantChat, onLoadingChange: handleLoadingChange, onDisplayItemsChange: handleDisplayItemsChange, children: children }))] }));
+                }, placeholder: "Type a message...", backgroundColor: "canvas.subtle", frontendTools: frontendTools, showModelSelector: showModelSelector, availableModels: availableModels, showToolsMenu: showToolsMenu, showSkillsMenu: showSkillsMenu, showTokenUsage: showTokenUsage, showContextRing: showContextRing, runtimeId: runtimeId, historyEndpoint: historyEndpoint, historyAuthToken: historyAuthToken, pendingPrompt: pendingPrompt, showInformation: showInformation, onInformationClick: onInformationClick, onToolCallStart: onToolCallStart, onToolCallComplete: onToolCallComplete, showToolApprovalBanner: showToolApprovalBanner, pendingApprovals: pendingApprovals, onApproveApproval: onApproveApproval, onRejectApproval: onRejectApproval, voice: voice, voiceSpeaking: speakerState.speaking, onStopSpeaking: speaker ? stopSpeaking : undefined, ...panelProps, ...assistantChat, onLoadingChange: handleLoadingChange, onDisplayItemsChange: handleDisplayItemsChange, children: children }))] }));
     /*
      * The floating popup's button. Words not yet heard make it blink for the
      * person's attention, until the chat is opened; hovered, they show in a
@@ -954,7 +1005,7 @@ export function ChatFloating({ endpoint, protocol: protocolProp, useStore: useSt
                         left: px(stageDrag.position.left),
                         top: px(stageDrag.position.top),
                     }
-                    : getPositionStyles(), stageRef: stageRef, onDragStart: stageDrag.onHandlePointerDown, open: isOpen, onToggle: handleToggle, balloon: assistantBalloon, insist: balloonInsists, onDismiss: dismissAssistant, ownRef: popupRef })), showButton && !isOpen && !assistantShown && (_jsx(Box, { sx: {
+                    : getPositionStyles(), stageRef: stageRef, onDragStart: stageDrag.onHandlePointerDown, open: isOpen, onToggle: handleToggle, balloon: assistantBalloon, insist: balloonInsists, onDismiss: dismissAssistant, ownRef: popupRef, mouthLevel: mouthLevel })), showButton && !isOpen && !assistantShown && (_jsx(Box, { sx: {
                     ...getPositionStyles(),
                 }, children: _jsxs(Box, { sx: {
                         position: 'relative',

package/lib/chat/assistant/AssistantStage.d.ts

@@ -94,6 +94,19 @@ export interface AssistantStageProps {
      * decision*, and stays while its form or its answer is on screen.
      */
     decide?: DecisionAsker;
+    /**
+     * How loud its voice is now, between 0 and 1, while it is heard (VOICE.md
+     * VO-23): the mouth opens with the sound, in three openings, instead of
+     * the talking loop. Still, for a reader who asks for reduced motion.
+     */
+    mouthLevel?: () => number;
 }
-export declare function AssistantStage({ character, state, size, place, stageRef, onDragStart, open, onToggle, balloon, insist, onDismiss, ownRef, decide, }: AssistantStageProps): JSX.Element;
+/** The openings of the mouth, by level: shut, a little, half, wide. */
+export declare function mouthOpening(level: number): 0 | 1 | 2 | 3;
+/**
+ * Moves the mouth with the voice: the opening is written on the stage as
+ * `data-assistant-mouth`, each frame, without drawing the character again.
+ */
+export declare function useMouth(stageRef: RefObject<HTMLElement | null>, mouthLevel: (() => number) | undefined, speaking: boolean): void;
+export declare function AssistantStage({ character, state, size, place, stageRef, onDragStart, open, onToggle, balloon, insist, onDismiss, ownRef, decide, mouthLevel, }: AssistantStageProps): JSX.Element;
 export default AssistantStage;

package/lib/chat/assistant/AssistantStage.js

@@ -180,7 +180,36 @@ export function useKeepClear(stageRef, own, paused) {
     }, [stageRef, own, paused]);
     return obstructed ? 'obstacle' : pointerBusy ? 'pointer' : undefined;
 }
-export function AssistantStage({ character, state, size = 88, place, stageRef, onDragStart, open, onToggle, balloon, insist = false, onDismiss, ownRef, decide, }) {
+/** The openings of the mouth, by level: shut, a little, half, wide. */
+export function mouthOpening(level) {
+    return level < 0.08 ? 0 : level < 0.3 ? 1 : level < 0.6 ? 2 : 3;
+}
+/**
+ * Moves the mouth with the voice: the opening is written on the stage as
+ * `data-assistant-mouth`, each frame, without drawing the character again.
+ */
+export function useMouth(stageRef, mouthLevel, speaking) {
+    useEffect(() => {
+        const stage = stageRef.current;
+        const still = typeof window !== 'undefined' &&
+            !!window.matchMedia?.('(prefers-reduced-motion: reduce)').matches;
+        if (!stage || !mouthLevel || !speaking || still) {
+            stage?.removeAttribute('data-assistant-mouth');
+            return;
+        }
+        let frame = 0;
+        const move = () => {
+            stage.setAttribute('data-assistant-mouth', String(mouthOpening(mouthLevel())));
+            frame = requestAnimationFrame(move);
+        };
+        frame = requestAnimationFrame(move);
+        return () => {
+            cancelAnimationFrame(frame);
+            stage.removeAttribute('data-assistant-mouth');
+        };
+    }, [stageRef, mouthLevel, speaking]);
+}
+export function AssistantStage({ character, state, size = 88, place, stageRef, onDragStart, open, onToggle, balloon, insist = false, onDismiss, ownRef, decide, mouthLevel, }) {
     // A shipped one by id, a drawing contributed by a plugin (T-24), or a
     // character read from a file (T-26).
     const shipped = typeof character === 'string'
@@ -199,6 +228,7 @@ export function AssistantStage({ character, state, size = 88, place, stageRef, o
     // Where the press began: a press that moves is a drag, not a click.
     const pressedAt = useRef(null);
     const aside = useKeepClear(stageRef, ownRef, menuOpen);
+    useMouth(stageRef, mouthLevel, state === 'speaking');
     // A decision being asked, or its answer, keeps the balloon up.
     const [deciding, setDeciding] = useState(false);
     const showBalloon = !aside && !open && !!balloon && (hovered || insist || deciding);
@@ -261,6 +291,23 @@ export function AssistantStage({ character, state, size = 88, place, stageRef, o
             '&[data-assistant-state="speaking"] .assistant-body': {
                 animation: 'assistantFloat 1.6s ease-in-out infinite',
             },
+            // Heard (VO-23): the mouth opens with the sound, not on a loop.
+            '&[data-assistant-mouth] .assistant-mouth': {
+                animation: 'none',
+                transition: 'transform 60ms linear',
+            },
+            '&[data-assistant-mouth="0"] .assistant-mouth': {
+                transform: 'scaleY(1)',
+            },
+            '&[data-assistant-mouth="1"] .assistant-mouth': {
+                transform: 'scaleY(1.5)',
+            },
+            '&[data-assistant-mouth="2"] .assistant-mouth': {
+                transform: 'scaleY(2.1)',
+            },
+            '&[data-assistant-mouth="3"] .assistant-mouth': {
+                transform: 'scaleY(2.8)',
+            },
             // Paused (R-17): it dozes — its eyes shut, its breath slow, a little
             // greyed — and, still, it is asleep: the shut eyes are not a motion.
             '&[data-assistant-state="paused"] .assistant-body': {
@@ -275,6 +322,11 @@ export function AssistantStage({ character, state, size = 88, place, stageRef, o
             '&[data-assistant-state="goodbye"] .assistant-body': {
                 animation: 'assistantLeave 0.6s ease-in forwards',
             },
+            // A sprite plays its own goodbye, and leaves as the drawn ones do:
+            // a character whose file has no goodbye would otherwise stay.
+            '&[data-assistant-state="goodbye"] .assistant-sprite': {
+                animation: 'assistantLeave 0.6s ease-in forwards',
+            },
             // One still frame per state for a reader who asks for no motion.
             '@media (prefers-reduced-motion: reduce)': {
                 '& *': { animation: 'none !important' },

package/lib/chat/assistant/SpriteCharacter.js

@@ -51,7 +51,7 @@ export function SpriteCharacter({ character, state, size, }) {
     }, [animation, frame, play, character, state, still]);
     const { width, height } = character.frameSize;
     const scale = size / Math.max(width, height);
-    return (_jsx("span", { role: "img", "aria-label": character.name, "data-sprite-animation": play.name, style: {
+    return (_jsx("span", { className: "assistant-sprite", role: "img", "aria-label": character.name, "data-sprite-animation": play.name, style: {
             position: 'relative',
             display: 'block',
             width: size,

package/lib/chat/assistant/characters.js

@@ -40,7 +40,6 @@ const WIZARD = {
         brim: '#4A3C9E',
         robe: '#5B4BB7',
         face: '#F2D3B3',
-        faceEdge: '#A0694A',
         beard: '#F4F4F6',
         beardEdge: '#7A808A',
         star: '#F8D469',
@@ -51,7 +50,6 @@ const WIZARD = {
         brim: '#7B67DB',
         robe: '#8F7CE8',
         face: '#E8C4A0',
-        faceEdge: '#E8C4A0',
         beard: '#E4E4EA',
         beardEdge: loopColors.grayDark,
         star: '#F8D469',
@@ -100,7 +98,7 @@ function PaperClip({ size, mode = 'light', }) {
 /** A wizard: a starred hat, a white beard, kind eyes. */
 function Wizard({ size, mode = 'light', }) {
     const c = WIZARD[mode];
-    return (_jsx("svg", { width: size, height: size, viewBox: "0 0 96 96", role: "img", "aria-label": "Wizard", "data-assistant-mode": mode, children: _jsxs("g", { className: "assistant-body", children: [_jsx("path", { d: "M24 92 Q48 58 72 92 Z", fill: c.robe }), _jsx("circle", { cx: 48, cy: 48, r: 17, fill: c.face, stroke: c.faceEdge, strokeWidth: 1.5 }), _jsx("path", { d: "M31 50 Q48 92 65 50 Q57 60 48 60 Q39 60 31 50 Z", fill: c.beard, stroke: c.beardEdge, strokeWidth: 1.2 }), _jsx("path", { d: "M22 36 L48 2 L74 36 Q48 30 22 36 Z", fill: c.hat }), _jsx("path", { d: "M20 37 Q48 29 76 37 Q48 43 20 37 Z", fill: c.brim }), _jsx("path", { d: "M52 14 l2 4 4 1 -3 3 1 4 -4 -2 -4 2 1 -4 -3 -3 4 -1 Z", fill: c.star }), _jsx("circle", { cx: 40, cy: 26, r: 1.8, fill: c.star }), _jsx(Eyes, { cx: 48, cy: 46, gap: 13, r: 5, colours: c.eye }), _jsx("path", { className: "assistant-mouth", d: "M44 58 Q48 61 52 58", stroke: INK, strokeWidth: 2, fill: "none", strokeLinecap: "round" })] }) }));
+    return (_jsx("svg", { width: size, height: size, viewBox: "0 0 96 96", role: "img", "aria-label": "Wizard", "data-assistant-mode": mode, children: _jsxs("g", { className: "assistant-body", children: [_jsx("path", { d: "M34 58 Q48 52 62 58 Q70 74 76 92 Q48 98 20 92 Q26 74 34 58 Z", fill: c.robe }), _jsx("path", { d: "M21 90 Q48 96 75 90 L76 92 Q48 98 20 92 Z", fill: c.brim }), _jsx("circle", { cx: 30, cy: 82, r: 1.6, fill: c.star }), _jsx("circle", { cx: 67, cy: 78, r: 1.3, fill: c.star }), _jsx("ellipse", { cx: 48, cy: 46, rx: 13, ry: 12, fill: c.face }), _jsx("path", { d: "M33 38 Q31 66 40 77 Q48 89 56 77 Q65 66 63 38 Q61 50 56 53 Q48 58 40 53 Q35 50 33 38 Z", fill: c.beard, stroke: c.beardEdge, strokeWidth: 1, strokeLinejoin: "round" }), _jsx("path", { className: "assistant-mouth", d: "M44 59 Q48 62 52 59", stroke: INK, strokeWidth: 1.8, fill: "none", strokeLinecap: "round" }), _jsx("path", { d: "M39 55 Q44 51 48 54 Q52 51 57 55 Q52 57 48 56 Q44 57 39 55 Z", fill: c.beard, stroke: c.beardEdge, strokeWidth: 0.8 }), _jsx("path", { d: "M25 36 Q35 26 39 14 Q43 2 58 3 Q50 8 53 18 Q58 30 71 36 Q48 30 25 36 Z", fill: c.hat }), _jsx("path", { d: "M18 37 Q48 27 78 37 Q48 45 18 37 Z", fill: c.brim }), _jsx("path", { d: "M47 17 l1.8 3.6 4 .6 -2.9 2.8 .7 4 -3.6 -1.9 -3.6 1.9 .7 -4 -2.9 -2.8 4 -.6 Z", fill: c.star }), _jsx("circle", { cx: 56, cy: 28, r: 1.4, fill: c.star }), _jsx(Eyes, { cx: 48, cy: 46, gap: 11, r: 4, colours: c.eye })] }) }));
 }
 /** A cat: round, two ears, whiskers, a tail. */
 function Cat({ size, mode = 'light', }) {
@@ -146,7 +144,6 @@ export const ASSISTANT_CHARACTERS = [
                     WIZARD.light.hat,
                     WIZARD.light.brim,
                     WIZARD.light.robe,
-                    WIZARD.light.faceEdge,
                     WIZARD.light.beardEdge,
                 ],
                 eye: eyeOf(WIZARD.light.eye, WIZARD.light.face),
@@ -156,7 +153,6 @@ export const ASSISTANT_CHARACTERS = [
                     WIZARD.dark.hat,
                     WIZARD.dark.brim,
                     WIZARD.dark.robe,
-                    WIZARD.dark.faceEdge,
                     WIZARD.dark.beardEdge,
                 ],
                 eye: eyeOf(WIZARD.dark.eye, WIZARD.dark.face),

package/lib/chat/assistant/state.d.ts

@@ -17,6 +17,11 @@ export interface AssistantMoment {
     leaving?: boolean;
     /** The answer is arriving: words are being written. */
     speaking?: boolean;
+    /**
+     * Its answer is being heard: sound is playing (VOICE.md VO-22). It speaks
+     * while the audio plays, even after the text has ended.
+     */
+    voicing?: boolean;
 }
 /**
  * The state to act, from the application's presence and the moment.
@@ -24,7 +29,8 @@ export interface AssistantMoment {
  * Leaving wins, being short; then paused — a paused application neither
  * greets nor speaks, it dozes; then arriving; then waiting for the person,
  * which must never be missed; then speaking over thinking, since words
- * arriving are what the person watches; then the presence as it is.
+ * arriving are what the person watches — and while its voice is heard, even
+ * once the words have all arrived; then the presence as it is.
  */
 export declare function assistantStateOf(presence: PresenceState, moment?: AssistantMoment): AssistantState;
 /**

package/lib/chat/assistant/state.js

@@ -8,7 +8,8 @@
  * Leaving wins, being short; then paused — a paused application neither
  * greets nor speaks, it dozes; then arriving; then waiting for the person,
  * which must never be missed; then speaking over thinking, since words
- * arriving are what the person watches; then the presence as it is.
+ * arriving are what the person watches — and while its voice is heard, even
+ * once the words have all arrived; then the presence as it is.
  */
 export function assistantStateOf(presence, moment = {}) {
     if (moment.leaving) {
@@ -23,7 +24,7 @@ export function assistantStateOf(presence, moment = {}) {
     if (presence === 'waiting') {
         return 'waiting';
     }
-    if (moment.speaking && presence !== 'idle') {
+    if ((moment.speaking && presence !== 'idle') || moment.voicing) {
         return 'speaking';
     }
     return presence;

package/lib/chat/base/ChatBase.js

@@ -45,6 +45,8 @@ import { PoweredByTag } from '../display/PoweredByTag';
 import { EphemeralSurfaceControl } from '../EphemeralSurfaceControl';
 import { ChatMessageList, } from '../messages/ChatMessageList';
 import { InputPrompt } from '../prompt/InputPrompt';
+import { DeviceHearing } from '../../voice/hearing';
+import { VoiceInput } from '../../voice/VoiceInput';
 import { isFloatingChatViewMode } from '../viewModes';
 import { ToolApprovalBanner, ToolApprovalDialog, } from '../tools';
 import { EphemeralNotebook } from '../notebook/EphemeralNotebook';
@@ -509,7 +511,7 @@ function ChatBaseInner({ title, subtitle, showHeader = false, showTokenUsage = t
 // Mode selection
 useStore: useStoreMode = true, protocol: protocolRaw, onSendMessage, onSendReady, onLoadingChange, onContextSnapshot, enableStreaming = false, 
 // Extended props
-brandIcon, avatarConfig, headerButtons, showPoweredBy = false, poweredByProps, emptyState, renderToolResult, renderHeader, showTurnFooter = true, notebookToolSurfacesId, footerContent, welcome, trailingContent, showInformation = false, onInformationClick, headerContent, children, borderRadius, backgroundColor, border, boxShadow, compact = false, placeholder, description = 'Start a conversation with the AI agent.', onStateUpdate, onNewChat, onClear, onMessagesChange, onItemsChange, onDisplayItemsChange, autoFocus = false, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, focusTrigger, frontendTools: frontendToolsProp, enableEphemeralNotebook = false, initialEphemeralNotebookOpen = true, onEphemeralNotebookOpenChange, enableEphemeralDocument = false, initialEphemeralSurfaceMode, onEphemeralSurfaceModeChange, collapsed = false, onExpandFromCollapsed, ephemeralNotebookToolbar, ephemeralNotebookToolbarExtraItems, ephemeralDocumentToolbarExtraItems, ephemeralNotebookCollaborationProvider, ephemeralNotebookCollaborationDocumentId, ephemeralDocumentCollaboration, ephemeralRuntimeOverride, 
+brandIcon, avatarConfig, headerButtons, showPoweredBy = false, poweredByProps, emptyState, renderToolResult, renderHeader, showTurnFooter = true, notebookToolSurfacesId, footerContent, welcome, trailingContent, voice, voiceSpeaking = false, onStopSpeaking, showInformation = false, onInformationClick, headerContent, children, borderRadius, backgroundColor, border, boxShadow, compact = false, placeholder, description = 'Start a conversation with the AI agent.', onStateUpdate, onNewChat, onClear, onMessagesChange, onItemsChange, onDisplayItemsChange, autoFocus = false, suggestions, submitOnSuggestionClick = true, hideMessagesAfterToolUI = false, focusTrigger, frontendTools: frontendToolsProp, enableEphemeralNotebook = false, initialEphemeralNotebookOpen = true, onEphemeralNotebookOpenChange, enableEphemeralDocument = false, initialEphemeralSurfaceMode, onEphemeralSurfaceModeChange, collapsed = false, onExpandFromCollapsed, ephemeralNotebookToolbar, ephemeralNotebookToolbarExtraItems, ephemeralDocumentToolbarExtraItems, ephemeralNotebookCollaborationProvider, ephemeralNotebookCollaborationDocumentId, ephemeralDocumentCollaboration, ephemeralRuntimeOverride, 
 // Tool invocation hooks
 onToolCallStart, onToolCallComplete, 
 // Identity/Authorization props
@@ -2819,10 +2821,17 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
     // ========================================================================
     // handleSend
     // ========================================================================
+    // Voice (VOICE.md V1): what was said and put in the composer, until sent.
+    const spokenRef = useRef(undefined);
+    const hearing = useMemo(() => voice && voice.input !== 'off'
+        ? new DeviceHearing(voice.modelsUrl, voice.engines)
+        : undefined, [voice?.input, voice?.modelsUrl, voice?.engines]);
     const handleSend = useCallback(async (messageOverride, 
     // AG-UI's: what goes with the run besides the conversation — an
     // application's page says there what it did (LOOP R-04).
-    forwardedProps) => {
+    forwardedPropsGiven, 
+    // How the message was heard, when it was said (VOICE.md VO-27).
+    spokenGiven) => {
         const messageContent = (messageOverride ?? input).trim();
         if (!messageContent || isLoading)
             return;
@@ -2831,7 +2840,15 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
         stoppedRef.current = false;
         suppressAssistantTextForToolOnlyRef.current =
             isToolCallOnlyPrompt(messageContent);
-        const userMessage = createUserMessage(messageContent);
+        // Said, then sent: by the microphone at once, or from the composer it
+        // was put in — the words may have been edited, it was still said.
+        const spoken = spokenGiven ??
+            (messageOverride === undefined ? spokenRef.current : undefined);
+        spokenRef.current = undefined;
+        const forwardedProps = spoken
+            ? { ...(forwardedPropsGiven ?? {}), voice: spoken }
+            : forwardedPropsGiven;
+        const userMessage = createUserMessage(messageContent, spoken ? { ...spoken } : undefined);
         const currentMessages = displayItems.filter((item) => !isToolCallMessage(item));
         const allMessages = [...currentMessages, userMessage];
         setDisplayItems(prev => [...prev, userMessage]);
@@ -3513,7 +3530,18 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
             mx: 'auto',
             bg: 'canvas.default',
         }, children: [_jsx(ChatMessageList, { displayItems: welcomedItems, isLoading: isLoading, isStreaming: isStreaming, showLoadingIndicator: showLoadingIndicator, hideMessagesAfterToolUI: hideMessagesAfterToolUI, avatarConfig: defaultAvatarConfig, padding: padding, renderToolResult: effectiveRenderToolResult, approvalConfig: approvalConfig, messagesEndRef: messagesEndRef, onRespond: handleRespond, showTurnFooters: showTurnFooter, agentUsage: agentUsage, onRemoveItems: handleRemoveItems, mcpServers: filteredMcpServers, emptyContent: launching ? null : (_jsx(ChatEmptyState, { emptyState: emptyState, brandIcon: brandIcon, description: resolvedDescription, suggestions: resolvedSuggestions, submitOnSuggestionClick: submitOnSuggestionClick, onSuggestionSubmit: handleSuggestionSubmit, onSuggestionFill: handleSuggestionFill })) }), trailingContent] }));
-    const inputToolbar = showInput ? (_jsx(InputPrompt, { input: input, setInput: setInput, isLoading: isLoading, kernelStatus: liveKernelStatus, connectionConfirmed: connectionConfirmed, placeholder: placeholder, 
+    // The microphone, in the composer's footer (VO-10).
+    const voiceControl = hearing && voice ? (_jsx(VoiceInput, { hearing: hearing, language: voice.language, consentKey: voice.consentKey ?? 'chat', disabled: disabled || launching, speaking: voiceSpeaking, onStopSpeaking: onStopSpeaking, onListen: onStopSpeaking, onTranscript: (transcript) => {
+            if (voice.sendWhatISay) {
+                void handleSend(transcript.text, undefined, transcript.metadata);
+                return;
+            }
+            spokenRef.current = transcript.metadata;
+            setInput(input.trim()
+                ? `${input.trim()} ${transcript.text}`
+                : transcript.text);
+        } })) : undefined;
+    const inputToolbar = showInput ? (_jsx(InputPrompt, { footerExtras: voiceControl, input: input, setInput: setInput, isLoading: isLoading, kernelStatus: liveKernelStatus, connectionConfirmed: connectionConfirmed, placeholder: placeholder, 
         /*
           The same openers the empty state offers, typed into the box they would
           be typed into.

package/lib/chat/messages/ChatMessageList.d.ts

@@ -1,9 +1,3 @@
-/**
- * MessageList — Renders the scrollable list of chat messages, tool calls,
- * and the typing indicator.
- *
- * @module chat/messages/MessageList
- */
 import { type ReactNode, type RefObject } from 'react';
 import { type MarkedMcpServer } from '../marks';
 import type { DisplayItem, AvatarConfig, RenderToolResult } from '../../types/chat';

package/lib/chat/messages/ChatMessageList.js

@@ -9,6 +9,7 @@ import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-run
  *
  * @module chat/messages/MessageList
  */
+import { MicrophoneIcon } from '@datalayer/icons-react';
 import { Fragment, useState, useCallback, useEffect, useMemo, useRef, } from 'react';
 import { Text, Spinner, Link, Label, RelativeTime } from '@primer/react';
 import { ArrowRightIcon, BroadcastIcon, ClockIcon, DependabotIcon, PersonIcon, SkipIcon, ToolsIcon, XCircleFillIcon, } from '@primer/octicons-react';
@@ -708,11 +709,16 @@ export function ChatMessageList({ displayItems, isLoading, isStreaming, showLoad
                                         // the agent reads — see `formSubmission`.
                                         const text = getMessageText(message);
                                         const submission = parseFormSubmission(text);
-                                        return submission ? (_jsx(FormSubmissionMessage, { submission: submission })) : (_jsx(Text, { sx: {
+                                        return submission ? (_jsx(FormSubmissionMessage, { submission: submission })) : (_jsxs(Text, { sx: {
                                                 fontSize: 1,
                                                 whiteSpace: 'pre-wrap',
                                                 wordBreak: 'break-word',
-                                            }, children: text }));
+                                            }, children: [message.metadata?.input === 'voice' ? (_jsx(Box, { as: "span", "data-spoken": "", title: "Said aloud", "aria-label": "Said aloud:", sx: {
+                                                        display: 'inline-flex',
+                                                        verticalAlign: 'text-bottom',
+                                                        mr: 1,
+                                                        opacity: 0.85,
+                                                    }, children: _jsx(MicrophoneIcon, { size: 14, "aria-hidden": "true" }) })) : null, text] }));
                                     })()) : message.live && !getMessageText(message) ? (_jsx(TypingDots, { size: 6 })) : (_jsx(Box, { sx: streamdownMarkdownStyles, children: _jsx(Streamdown, { children: normalizeAssistantMarkdown(getMessageText(message) || (isStreaming ? '...' : '')) }) }))] }), renderMessageFooter ? (_jsx(Box, { sx: {
                                     display: 'flex',
                                     gap: 2,

package/lib/config/AgentConfiguration.js

@@ -193,12 +193,6 @@ const AGENT_LIBRARIES = [
         description: 'LangChain agent implementations and tools',
         disabled: true,
     },
-    {
-        value: 'google-adk',
-        label: 'Google ADK',
-        description: 'Google’s Agent Development Kit',
-        disabled: true,
-    },
 ];
 const TRANSPORTS = [
     {

package/lib/examples/AgentA2ATeamExample.js

@@ -2,6 +2,8 @@ import { jsxs as _jsxs, jsx as _jsx } from "react/jsx-runtime";
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { Button, Flash, Heading, Label, Text, Textarea } from '@primer/react';
 import { Box } from '@datalayer/primer-addons';
+import { AnonymousKeyExpired } from '@datalayer/core/lib/components/anonymous/AnonymousKeyExpired';
+import { AnonymousKeyTimer } from '@datalayer/core/lib/components/anonymous/AnonymousKeyTimer';
 import { stepCountIs, ToolLoopAgent } from 'ai';
 import { Streamdown } from 'streamdown';
 import { ThemedProvider } from './utils/themedProvider';
@@ -9,6 +11,7 @@ import { useExampleAgentRuntimesUrl } from './utils/useExampleAgentRuntimesUrl';
 import { AssistantStage } from '../chat/assistant/AssistantStage';
 import { useBrowserInference } from '../hooks/useBrowserInference';
 import { createBrowserModel } from '../runtimes/browser/model';
+import { readJwtClaims, tokenLifetimeMs, useAnonymousSessionStore, } from '../runtimes/browser/anonymousToken';
 import { a2aPeerTool, connectA2APeer, } from '../runtimes/browser/a2aPeer';
 import { ACCOUNTING_APP_0_0_1, SALES_APP_0_0_1 } from '../specs/apps';
 import { SALES_AND_ACCOUNTING_TEAM_SPEC_0_0_1 } from '../specs/teams/teams';
@@ -60,7 +63,17 @@ function AgentA2ATeam() {
     const accountingUrl = import.meta.env.VITE_A2A_ACCOUNTING_URL ||
         `${runtimeUrl}/api/v1/a2a/agents/${ACCOUNTING.id}`;
     const accountingKey = import.meta.env.VITE_A2A_ACCOUNTING_KEY || undefined;
-    const { inference, needsSignIn } = useBrowserInference(true);
+    const { inference, needsSignIn, anonymous } = useBrowserInference(true);
+    // The temporary key's own clock, read out of it: when it ends and how long
+    // it was signed for, as core's countdown draws them.
+    const accountingKeyClock = useMemo(() => {
+        const exp = accountingKey ? readJwtClaims(accountingKey)?.exp : undefined;
+        return accountingKey && exp
+            ? { expiresAt: exp * 1000, grantedMs: tokenLifetimeMs(accountingKey) }
+            : undefined;
+    }, [accountingKey]);
+    const [accountingKeyEnded, setAccountingKeyEnded] = useState(() => accountingKeyClock !== undefined &&
+        accountingKeyClock.expiresAt <= Date.now());
     const [peer, setPeer] = useState(null);
     const [peerError, setPeerError] = useState(null);
     const [sales, setSales] = useState({
@@ -235,7 +248,11 @@ function AgentA2ATeam() {
     const accountingWhere = accountingKey
         ? `on a runtime, ${new URL(accountingUrl).host}, with a temporary key`
         : `on a runtime, ${new URL(accountingUrl).host}`;
-    return (_jsxs(Box, { sx: { p: 4, maxWidth: 1080, mx: 'auto' }, children: [_jsxs(Heading, { as: "h2", sx: { fontSize: 4, mb: 1 }, children: [TEAM.emoji, " ", TEAM.name] }), _jsx(Text, { as: "p", sx: { color: 'fg.muted', mt: 0 }, children: TEAM.description }), needsSignIn && (_jsx(Flash, { variant: "warning", sx: { mb: 3 }, children: "Sales asks its model through the Datalayer inference service: sign in, or wait for a visitor's trial key." })), peerError && (_jsxs(Flash, { variant: "danger", sx: { mb: 3 }, children: ["Accounting is not reachable at ", accountingUrl, ": ", peerError, " Serve it with", ' ', _jsx("code", { children: "python examples/sales-accounting-a2a/serve_accounting.py" }), ' ', "on the local server, or", ' ', _jsx("code", { children: "loop apps run accounting.yaml --cloud --a2a" }), " and set", ' ', _jsx("code", { children: "VITE_A2A_ACCOUNTING_URL" }), "."] })), _jsxs(Box, { sx: {
+    return (_jsxs(Box, { sx: { p: 4, maxWidth: 1080, mx: 'auto' }, children: [_jsxs(Heading, { as: "h2", sx: { fontSize: 4, mb: 1 }, children: [TEAM.emoji, " ", TEAM.name] }), _jsx(Text, { as: "p", sx: { color: 'fg.muted', mt: 0 }, children: TEAM.description }), ((anonymous.status === 'active' && anonymous.expiresAt) ||
+                (accountingKeyClock && !accountingKeyEnded)) && (_jsxs(Box, { sx: { display: 'flex', gap: 4, flexWrap: 'wrap', mb: 3 }, children: [anonymous.status === 'active' && anonymous.expiresAt && (_jsx(AnonymousKeyTimer, { expiresAt: anonymous.expiresAt, grantedMs: anonymous.grantedMs, label: `${SALES.name}'s trial key`, onExpire: () => useAnonymousSessionStore.getState().expire() })), accountingKeyClock && !accountingKeyEnded && (_jsx(AnonymousKeyTimer, { expiresAt: accountingKeyClock.expiresAt, grantedMs: accountingKeyClock.grantedMs, label: `${ACCOUNTING.name}'s temporary key`, onExpire: () => setAccountingKeyEnded(true) }))] })), anonymous.status === 'expired' && (_jsx(Box, { sx: { mb: 3 }, children: _jsx(AnonymousKeyExpired, { agentName: SALES.name, 
+                    // Sales runs in this page: what ended is its model's key, and
+                    // signing in gives it the person's own.
+                    onSignedIn: () => useAnonymousSessionStore.getState().clear() }) })), accountingKeyEnded && (_jsxs(Flash, { variant: "danger", sx: { mb: 3 }, children: [ACCOUNTING.name, "\u2019s temporary key has ended: it no longer answers. Make a new one with", ' ', _jsx("code", { children: "examples/sales-accounting-a2a/make_temp_key.py" }), " and reload."] })), needsSignIn && (_jsx(Flash, { variant: "warning", sx: { mb: 3 }, children: "Sales asks its model through the Datalayer inference service: sign in, or wait for a visitor's trial key." })), peerError && (_jsxs(Flash, { variant: "danger", sx: { mb: 3 }, children: ["Accounting is not reachable at ", accountingUrl, ": ", peerError, " Serve it with", ' ', _jsx("code", { children: "python examples/sales-accounting-a2a/serve_accounting.py" }), ' ', "on the local server, or", ' ', _jsx("code", { children: "loop apps run accounting.yaml --cloud --a2a" }), " and set", ' ', _jsx("code", { children: "VITE_A2A_ACCOUNTING_URL" }), "."] })), _jsxs(Box, { sx: {
                     position: 'relative',
                     height: STAGE_HEIGHT,
                     border: '1px solid',

package/lib/examples/ChatAssistantExample.d.ts

@@ -5,7 +5,9 @@
  *
  * The characters offered are what the enabled plugins contribute to
  * `loop.assistant.character` (T-24): Datalayer's four, and an owl from a
- * small example plugin (`utils/owlCharacterPlugin`).
+ * small example plugin (`utils/owlCharacterPlugin`). Pixel, the test sprite,
+ * is read through the clippy.js reader, as a character a person brings is,
+ * and so are the characters clippy.js publishes, fetched when picked.
  */
 import React from 'react';
 declare const ChatAssistantExample: React.FC;