@datalayer/agent-runtimes 1.3.62 → 1.3.64

package/lib/examples/ChatAssistantExample.js

@@ -10,9 +10,11 @@ import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
  *
  * The characters offered are what the enabled plugins contribute to
  * `loop.assistant.character` (T-24): Datalayer's four, and an owl from a
- * small example plugin (`utils/owlCharacterPlugin`).
+ * small example plugin (`utils/owlCharacterPlugin`). Pixel, the test sprite,
+ * is read through the clippy.js reader, as a character a person brings is,
+ * and so are the characters clippy.js publishes, fetched when picked.
  */
-import { useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import { buildReactorFromPlugins } from '@datalayer/reactor';
 import { Button, Heading, Text } from '@primer/react';
 import { Box } from '@datalayer/primer-addons';
@@ -20,6 +22,8 @@ import { ThemedProvider } from './utils/themedProvider';
 import { ChatFloating } from '../chat';
 import { AssistantCharactersPlugin, assistantCharacterNamed, assistantCharactersOf, } from '../loop/plugins/assistant-characters';
 import { OwlCharacterPlugin } from './utils/owlCharacterPlugin';
+import { readTestSpriteCharacter } from './utils/testSpriteCharacter';
+import { CLIPPY_JS_CHARACTERS, CLIPPY_JS_VERSION, readClippyJsCharacter, } from './utils/clippyJsCharacters';
 import { readAcsCharacter, readClippyCharacter, } from '../chat/assistant/formats';
 /**
  * The agent-runtimes server the assistant talks to: `?agentRuntimesUrl=`
@@ -69,10 +73,72 @@ const ChatAssistantExample = () => {
         ? assistantCharacterNamed(reactor, character)
         : character;
     const [loadError, setLoadError] = useState();
-    return (_jsx(ThemedProvider, { children: _jsxs(Box, { sx: { minHeight: '100vh', bg: 'canvas.default', p: 4 }, children: [_jsxs(Box, { sx: { maxWidth: 720, mx: 'auto' }, children: [_jsx(Heading, { as: "h1", sx: { mb: 2 }, children: "Chat Assistant" }), _jsx(Text, { as: "p", sx: { color: 'fg.muted', mb: 3 }, children: "The chat as a character on the page. It greets you, acts out what the agent is doing, and says the agent\u2019s words in a balloon while the conversation is closed. Click it to talk, drag it to move it, hover it to send it away." }), _jsx(Box, { sx: { display: 'flex', gap: 2, flexWrap: 'wrap' }, children: CATALOGUE.map(option => (_jsx(Button, { variant: option.id === character ? 'primary' : 'default', onClick: () => {
-                                    setCharacter(option.id);
-                                    setLoadError(undefined);
-                                }, "aria-pressed": option.id === character, "data-assistant-character": option.id, children: option.name }, option.id))) }), _jsxs(Box, { as: "section", sx: { mt: 4 }, children: [_jsx(Heading, { as: "h2", sx: { fontSize: 2, mb: 1 }, children: "A character you bring" }), _jsxs(Text, { as: "p", sx: { color: 'fg.muted', mb: 2 }, children: ["The characters of Microsoft Office \u2014 Clippy, Merlin, Links and the others \u2014 are Microsoft\u2019s. Load only a character file you have the right to use: a Microsoft Agent ", _jsx("code", { children: ".acs" }), " file, or a clippy.js character\u2019s ", _jsx("code", { children: "agent.js" }), " and map image. It is read in this page and sent nowhere."] }), _jsx("input", { type: "file", multiple: true, accept: ".acs,.js,.png,.gif,.webp,.jpg,.jpeg", "aria-label": "Character files", onChange: async (event) => {
+    // Pixel, the test sprite, read once through the clippy.js reader.
+    const [pixel, setPixel] = useState();
+    useEffect(() => {
+        let cancelled = false;
+        readTestSpriteCharacter().then(read => {
+            if (!cancelled) {
+                setPixel(read);
+            }
+        }, error => {
+            if (!cancelled) {
+                setLoadError(error instanceof Error ? error.message : String(error));
+            }
+        });
+        return () => {
+            cancelled = true;
+        };
+    }, []);
+    // The object URL lives as long as the page does.
+    useEffect(() => () => {
+        if (pixel) {
+            URL.revokeObjectURL(pixel.sprite);
+        }
+    }, [pixel]);
+    // The clippy.js characters, each fetched the first time it is picked.
+    const [clippyJs, setClippyJs] = useState({});
+    const [fetching, setFetching] = useState();
+    const fetched = useRef([]);
+    useEffect(() => () => {
+        for (const read of fetched.current) {
+            URL.revokeObjectURL(read.sprite);
+        }
+    }, []);
+    const pickClippyJs = async (name) => {
+        const known = clippyJs[name];
+        if (known) {
+            setCharacter(known);
+            setLoadError(undefined);
+            return;
+        }
+        setFetching(name);
+        try {
+            const read = await readClippyJsCharacter(name);
+            fetched.current.push(read);
+            setClippyJs(previous => ({ ...previous, [name]: read }));
+            setCharacter(read);
+            setLoadError(undefined);
+        }
+        catch (error) {
+            setLoadError(error instanceof Error ? error.message : String(error));
+        }
+        finally {
+            setFetching(undefined);
+        }
+    };
+    const isClippyJs = typeof character !== 'string' &&
+        Object.values(clippyJs).includes(character);
+    return (_jsx(ThemedProvider, { children: _jsxs(Box, { sx: { minHeight: '100vh', bg: 'canvas.default', p: 4 }, children: [_jsxs(Box, { sx: { maxWidth: 720, mx: 'auto' }, children: [_jsx(Heading, { as: "h1", sx: { mb: 2 }, children: "Chat Assistant" }), _jsx(Text, { as: "p", sx: { color: 'fg.muted', mb: 3 }, children: "The chat as a character on the page. It greets you, acts out what the agent is doing, and says the agent\u2019s words in a balloon while the conversation is closed. Click it to talk, drag it to move it, hover it to send it away." }), _jsxs(Box, { sx: { display: 'flex', gap: 2, flexWrap: 'wrap' }, children: [CATALOGUE.map(option => (_jsx(Button, { variant: option.id === character ? 'primary' : 'default', onClick: () => {
+                                        setCharacter(option.id);
+                                        setLoadError(undefined);
+                                    }, "aria-pressed": option.id === character, "data-assistant-character": option.id, children: option.name }, option.id))), pixel && (_jsxs(Button, { variant: character === pixel ? 'primary' : 'default', onClick: () => {
+                                        setCharacter(pixel);
+                                        setLoadError(undefined);
+                                    }, "aria-pressed": character === pixel, "data-assistant-character": "sprite", children: [pixel.name, " (sprite)"] }))] }), _jsxs(Box, { as: "section", sx: { mt: 4 }, children: [_jsx(Heading, { as: "h2", sx: { fontSize: 2, mb: 1 }, children: "The clippy.js characters" }), _jsxs(Text, { as: "p", sx: { color: 'fg.muted', mb: 2 }, children: ["Fetched when picked from the", ' ', _jsxs("code", { children: ["clippyjs@", CLIPPY_JS_VERSION] }), " package on jsDelivr and read as data by the same reader. They are not in this repository: clippy.js\u2019s licence covers its code only, and the characters are Microsoft\u2019s."] }), _jsx(Box, { sx: { display: 'flex', gap: 2, flexWrap: 'wrap' }, children: CLIPPY_JS_CHARACTERS.map(name => {
+                                        const picked = clippyJs[name] !== undefined && character === clippyJs[name];
+                                        return (_jsx(Button, { variant: picked ? 'primary' : 'default', onClick: () => void pickClippyJs(name), disabled: fetching !== undefined, "aria-pressed": picked, "data-assistant-character": `clippy-js-${name.toLowerCase()}`, children: fetching === name ? `${name}…` : name }, name));
+                                    }) })] }), _jsxs(Box, { as: "section", sx: { mt: 4 }, children: [_jsx(Heading, { as: "h2", sx: { fontSize: 2, mb: 1 }, children: "A character you bring" }), _jsxs(Text, { as: "p", sx: { color: 'fg.muted', mb: 2 }, children: ["The characters of Microsoft Office \u2014 Clippy, Merlin, Links and the others \u2014 are Microsoft\u2019s. Load only a character file you have the right to use: a Microsoft Agent ", _jsx("code", { children: ".acs" }), " file, or a clippy.js character\u2019s ", _jsx("code", { children: "agent.js" }), " and map image. It is read in this page and sent nowhere."] }), _jsx("input", { type: "file", multiple: true, accept: ".acs,.js,.png,.gif,.webp,.jpg,.jpeg", "aria-label": "Character files", onChange: async (event) => {
                                         const files = Array.from(event.target.files ?? []);
                                         if (!files.length) {
                                             return;
@@ -84,7 +150,9 @@ const ChatAssistantExample = () => {
                                         catch (error) {
                                             setLoadError(error instanceof Error ? error.message : String(error));
                                         }
-                                    } }), loadError && (_jsx(Text, { as: "p", role: "alert", sx: { color: 'danger.fg', mt: 2 }, children: loadError })), typeof character !== 'string' && (_jsxs(Text, { as: "p", sx: { mt: 2 }, children: ["Playing ", character.name, "."] }))] })] }), _jsx(ChatFloating, { defaultViewMode: "assistant", assistantCharacter: drawn, protocol: "vercel-ai", endpoint: `${SERVER}/api/v1/vercel-ai/assistant`, title: "Assistant", description: "Hello! Ask me anything about this page.", position: "bottom-right", useStore: false, 
+                                    } }), loadError && (_jsx(Text, { as: "p", role: "alert", sx: { color: 'danger.fg', mt: 2 }, children: loadError })), typeof character !== 'string' &&
+                                    character !== pixel &&
+                                    !isClippyJs && (_jsxs(Text, { as: "p", sx: { mt: 2 }, children: ["Playing ", character.name, "."] }))] })] }), _jsx(ChatFloating, { defaultViewMode: "assistant", assistantCharacter: drawn, protocol: "vercel-ai", endpoint: `${SERVER}/api/v1/vercel-ai/assistant`, title: "Assistant", description: "Hello! Ask me anything about this page.", position: "bottom-right", useStore: false, 
                     // Ask a decision beside the composer: Jev, through this server.
                     decisions: { serverUrl: SERVER } }, typeof character === 'string' ? character : character.sprite)] }) }));
 };

package/lib/examples/ChatAssistantGalleryExample.d.ts

@@ -1,8 +1,9 @@
 /**
  * Every representation of the floating assistant, as a gallery (LOOP T-21
  * to T-27): each character — Datalayer's four, the owl an example plugin
- * contributes to `loop.assistant.character`, and a test sprite read through
- * the clippy.js reader — in each state it acts, stepped aside, with its
+ * contributes to `loop.assistant.character`, a test sprite read through
+ * the clippy.js reader, and, when asked, the characters clippy.js publishes,
+ * fetched from jsDelivr — in each state it acts, stepped aside, with its
  * balloon, in light and dark, moving or still.
  *
  * Static data, no agent: it opens without a server or an account.

package/lib/examples/ChatAssistantGalleryExample.js

@@ -6,8 +6,9 @@ import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-run
 /**
  * Every representation of the floating assistant, as a gallery (LOOP T-21
  * to T-27): each character — Datalayer's four, the owl an example plugin
- * contributes to `loop.assistant.character`, and a test sprite read through
- * the clippy.js reader — in each state it acts, stepped aside, with its
+ * contributes to `loop.assistant.character`, a test sprite read through
+ * the clippy.js reader, and, when asked, the characters clippy.js publishes,
+ * fetched from jsDelivr — in each state it acts, stepped aside, with its
  * balloon, in light and dark, moving or still.
  *
  * Static data, no agent: it opens without a server or an account.
@@ -21,6 +22,7 @@ import { AssistantStage } from '../chat/assistant/AssistantStage';
 import { useViewportDrag } from '../chat/useViewportDrag';
 import { AssistantGalleryGrid, GalleryObstacle, Still, useGalleryCharacters, } from './utils/AssistantGalleryGrid';
 import { GALLERY_POSES, GALLERY_POSE_LABELS, balloonForPose, sampleApproval, sampleSaying, stateOfPose, } from './utils/assistantGallery';
+import { CLIPPY_JS_VERSION, useClippyJsCharacters, } from './utils/clippyJsCharacters';
 const STAGE_SIZE = 96;
 const PANEL_HEIGHT = 420;
 const LEAVE_MS = 700;
@@ -98,7 +100,10 @@ function InModes({ modes, stacked = false, children, }) {
         }, children: shown.map(mode => (_jsx(Box, { sx: { flex: '1 1 320px', minWidth: 0 }, children: _jsx(DatalayerThemeProvider, { colorMode: mode, theme: config.primerTheme, themeStyles: config.themeStyles, children: children(mode) }) }, mode))) }));
 }
 const ChatAssistantGalleryExample = () => {
-    const { characters, problem } = useGalleryCharacters();
+    const { characters: ours, problem } = useGalleryCharacters();
+    const [withClippyJs, setWithClippyJs] = useState(false);
+    const clippyJs = useClippyJsCharacters(withClippyJs);
+    const characters = [...ours, ...clippyJs.characters];
     const [characterId, setCharacterId] = useState('paperclip');
     const [pose, setPose] = useState('idle');
     const [view, setView] = useState('stage');
@@ -142,7 +147,7 @@ const ChatAssistantGalleryExample = () => {
                 bg: 'canvas.default',
                 color: 'fg.default',
                 p: 4,
-            }, children: _jsxs(Box, { sx: { maxWidth: 1200, mx: 'auto' }, children: [_jsx(Heading, { as: "h1", sx: { mb: 2 }, children: "Chat Assistant gallery" }), _jsxs(Text, { as: "p", sx: { color: 'fg.muted', mb: 3, maxWidth: 760 }, children: ["Every character in every state the floating assistant acts, with its balloon, in light and dark. Datalayer\u2019s four and the owl are what the enabled plugins contribute to", ' ', _jsx("code", { children: "loop.assistant.character" }), "; Pixel is a test sprite of our own, read through the clippy.js reader. No agent is behind it: the words are a sample conversation."] }), _jsxs(Box, { as: "section", "aria-label": "Gallery controls", sx: {
+            }, children: _jsxs(Box, { sx: { maxWidth: 1200, mx: 'auto' }, children: [_jsx(Heading, { as: "h1", sx: { mb: 2 }, children: "Chat Assistant gallery" }), _jsxs(Text, { as: "p", sx: { color: 'fg.muted', mb: 3, maxWidth: 760 }, children: ["Every character in every state the floating assistant acts, with its balloon, in light and dark. Datalayer\u2019s four and the owl are what the enabled plugins contribute to", ' ', _jsx("code", { children: "loop.assistant.character" }), "; Pixel is a test sprite of our own, read through the clippy.js reader. No agent is behind it: the words are a sample conversation."] }), _jsxs(Text, { as: "p", sx: { color: 'fg.muted', mb: 3, maxWidth: 760 }, children: ["The characters clippy.js publishes \u2014 Clippy, Merlin, Links, Rover and the others \u2014 are loaded on request from the", ' ', _jsxs("code", { children: ["clippyjs@", CLIPPY_JS_VERSION] }), " package on jsDelivr (about 15 MB) and read by the same reader. They are not in this repository: clippy.js\u2019s licence covers its code only, and the characters are Microsoft\u2019s."] }), _jsxs(Box, { as: "section", "aria-label": "Gallery controls", sx: {
                             display: 'grid',
                             gap: 3,
                             mb: 4,
@@ -160,7 +165,7 @@ const ChatAssistantGalleryExample = () => {
                                                 ? 'Side by side'
                                                 : option === 'light'
                                                     ? 'Light'
-                                                    : 'Dark' }, option))) }), _jsxs(FormControl, { children: [_jsx(Checkbox, { checked: still, onChange: event => setStill(event.target.checked) }), _jsx(FormControl.Label, { children: "Reduced motion" })] }), _jsx(Button, { onClick: () => setReplay(replay + 1), children: "Replay the motions" })] }), view === 'stage' && (_jsxs(_Fragment, { children: [_jsx(Box, { role: "group", "aria-label": "Character", sx: { display: 'flex', gap: 2, flexWrap: 'wrap' }, children: characters.map(option => (_jsx(Button, { size: "small", variant: option.id === character.id ? 'primary' : 'default', "aria-pressed": option.id === character.id, "data-gallery-character": option.id, onClick: () => setCharacterId(option.id), children: option.name }, option.id))) }), _jsx(Box, { role: "group", "aria-label": "State", sx: { display: 'flex', gap: 2, flexWrap: 'wrap' }, children: GALLERY_POSES.map(option => (_jsx(Button, { size: "small", variant: option === pose ? 'primary' : 'default', "aria-pressed": option === pose, "data-gallery-pose": option, onClick: () => {
+                                                    : 'Dark' }, option))) }), _jsxs(FormControl, { children: [_jsx(Checkbox, { checked: still, onChange: event => setStill(event.target.checked) }), _jsx(FormControl.Label, { children: "Reduced motion" })] }), _jsxs(FormControl, { children: [_jsx(Checkbox, { checked: withClippyJs, onChange: event => setWithClippyJs(event.target.checked), "data-gallery-clippy-js": "" }), _jsxs(FormControl.Label, { children: ["clippy.js characters", clippyJs.loading ? ' (loading…)' : ''] })] }), _jsx(Button, { onClick: () => setReplay(replay + 1), children: "Replay the motions" })] }), view === 'stage' && (_jsxs(_Fragment, { children: [_jsx(Box, { role: "group", "aria-label": "Character", sx: { display: 'flex', gap: 2, flexWrap: 'wrap' }, children: characters.map(option => (_jsx(Button, { size: "small", variant: option.id === character.id ? 'primary' : 'default', "aria-pressed": option.id === character.id, "data-gallery-character": option.id, onClick: () => setCharacterId(option.id), children: option.name }, option.id))) }), _jsx(Box, { role: "group", "aria-label": "State", sx: { display: 'flex', gap: 2, flexWrap: 'wrap' }, children: GALLERY_POSES.map(option => (_jsx(Button, { size: "small", variant: option === pose ? 'primary' : 'default', "aria-pressed": option === pose, "data-gallery-pose": option, onClick: () => {
                                                 setPose(option);
                                                 setDecision(undefined);
                                                 setReplay(replay + 1);
@@ -169,7 +174,7 @@ const ChatAssistantGalleryExample = () => {
                                             gap: 2,
                                             flexWrap: 'wrap',
                                             alignItems: 'center',
-                                        }, children: [_jsx(Button, { onClick: say, "data-gallery-say": "", children: "Say something" }), saying !== undefined && (_jsx(Button, { variant: "invisible", onClick: () => setSaying(undefined), children: "Hush" })), presence === 'away' ? (_jsx(Button, { onClick: callBack, children: "Call it back" })) : (_jsx(Button, { onClick: () => sendAway('page'), disabled: presence !== 'here', "data-gallery-send-away": "", children: "Send it away" })), _jsx(Text, { sx: { color: 'fg.muted', fontSize: 1 }, children: "Drag the character to move it; hover it for its own send-away menu." }), decision && (_jsx(Text, { role: "status", sx: { fontSize: 1 }, children: decision }))] })] }))] }), problem && (_jsxs(Text, { as: "p", role: "alert", sx: { color: 'danger.fg', mb: 3 }, children: ["The test sprite could not be read: ", problem] })), _jsx(Still, { still: still, children: view === 'stage' ? (_jsx(InModes, { modes: modes, children: mode => (_jsx(ModePanel, { mode: mode, character: character, pose: pose, presence: presence, balloon: balloon, insist: insist, onToggle: () => saying === undefined ? say() : setSaying(undefined), onDismiss: sendAway, onCallBack: callBack, away: away, replay: replay })) })) : (_jsx(InModes, { modes: modes, stacked: true, children: mode => (_jsx(Box, { "data-gallery-mode": mode, sx: {
+                                        }, children: [_jsx(Button, { onClick: say, "data-gallery-say": "", children: "Say something" }), saying !== undefined && (_jsx(Button, { variant: "invisible", onClick: () => setSaying(undefined), children: "Hush" })), presence === 'away' ? (_jsx(Button, { onClick: callBack, children: "Call it back" })) : (_jsx(Button, { onClick: () => sendAway('page'), disabled: presence !== 'here', "data-gallery-send-away": "", children: "Send it away" })), _jsx(Text, { sx: { color: 'fg.muted', fontSize: 1 }, children: "Drag the character to move it; hover it for its own send-away menu." }), decision && (_jsx(Text, { role: "status", sx: { fontSize: 1 }, children: decision }))] })] }))] }), problem && (_jsxs(Text, { as: "p", role: "alert", sx: { color: 'danger.fg', mb: 3 }, children: ["The test sprite could not be read: ", problem] })), clippyJs.problem && (_jsxs(Text, { as: "p", role: "alert", sx: { color: 'danger.fg', mb: 3 }, children: ["A clippy.js character could not be read: ", clippyJs.problem] })), _jsx(Still, { still: still, children: view === 'stage' ? (_jsx(InModes, { modes: modes, children: mode => (_jsx(ModePanel, { mode: mode, character: character, pose: pose, presence: presence, balloon: balloon, insist: insist, onToggle: () => saying === undefined ? say() : setSaying(undefined), onDismiss: sendAway, onCallBack: callBack, away: away, replay: replay })) })) : (_jsx(InModes, { modes: modes, stacked: true, children: mode => (_jsx(Box, { "data-gallery-mode": mode, sx: {
                                     bg: 'canvas.default',
                                     border: '1px solid',
                                     borderColor: 'border.default',

package/lib/examples/DecksAgent.js

@@ -1,13 +1,15 @@
 import { jsx as _jsx } from "react/jsx-runtime";
 import { Box } from '@datalayer/primer-addons';
 import { ThemedProvider } from './utils/themedProvider';
-import { registerDecks } from '@datalayer/decks';
-import { exampleDecks } from '@datalayer/decks/examples';
+import { registerDeckComponents, registerDecks } from '@datalayer/decks';
+import { exampleDeckComponents, exampleDecks } from '@datalayer/decks/examples';
 import { LoopEmbed } from '../loop';
 import { DeckViewPlugin, LoopDecksPlugin } from '../loop/plugins/decks';
 // The package's example decks, so the catalog has something to open and the
 // agent's suggestions point at decks that exist. Once, at module load.
 registerDecks(exampleDecks);
+// And the components their slides name (AgentsHero, LiveAppearance…).
+registerDeckComponents(exampleDeckComponents);
 // Module-level, so the embed builds its reactor once rather than on every
 // render of this component.
 const DECK_PLUGINS = [LoopDecksPlugin, DeckViewPlugin];

package/lib/examples/LoopShellExample.js

@@ -3,8 +3,8 @@ import { useMemo } from 'react';
 import { Box } from '@primer/react';
 import { QueryClientProvider } from '@tanstack/react-query';
 import { useReactor } from '@datalayer/reactor/react';
-import { registerDecks } from '@datalayer/decks';
-import { exampleDecks } from '@datalayer/decks/examples';
+import { registerDeckComponents, registerDecks } from '@datalayer/decks';
+import { exampleDeckComponents, exampleDecks } from '@datalayer/decks/examples';
 import { buildLoopReactor, LoopWorkspace } from '../loop/shell';
 import { DeckViewPlugin, LoopDecksPlugin } from '../loop/plugins/decks';
 import { loopPlugins } from '../loop/presets';
@@ -18,6 +18,8 @@ const FRONT_DOOR = 'jupyter-data-analyst';
 // The package's example decks, so the Decks member has something to open and
 // change before it has made one. Once, at module load.
 registerDecks(exampleDecks);
+// And the components their slides name (AgentsHero, LiveAppearance…).
+registerDeckComponents(exampleDeckComponents);
 export function LoopShellExample({ serverUrl = resolveExampleAgentRuntimesUrl('local'), agentId = FRONT_DOOR, defaultEditor = 'none', inheritTheme = false, }) {
     // Built once: rebuilding would restart every plugin on each render.
     const reactor = useMemo(() => buildLoopReactor([

package/lib/examples/VoiceChatExample.d.ts

@@ -0,0 +1,20 @@
+/**
+ * A voice chat with the floating assistant (VOICE.md V1).
+ *
+ * Hold the microphone (or `Ctrl`+`Space`), speak, let go: what you said is
+ * heard in this page — Moonshine for English, Whisper for French, Silero VAD
+ * keeping only the speech — and put in the composer, marked as said. Send
+ * it, and the answer is read aloud as it is written, sentence by sentence,
+ * by Datalayer's speech service (Kokoro, on ai-agents), the assistant's
+ * mouth moving with the sound and its balloon showing the sentence said.
+ *
+ * Three places, each named in the address when it is not this machine's:
+ * - `?agentRuntimesUrl=` — the agent-runtimes server and its `assistant`
+ *   agent (http://127.0.0.1:8765);
+ * - `?voiceModelsUrl=` — the browser's models, as pinned
+ *   (`scripts/voice/serve_store.py`, http://127.0.0.1:8770);
+ * - `?speechUrl=` — ai-agents' speech service (http://127.0.0.1:4401).
+ */
+import React from 'react';
+declare const VoiceChatExample: React.FC;
+export default VoiceChatExample;

package/lib/examples/VoiceChatExample.js

@@ -0,0 +1,64 @@
+import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * A voice chat with the floating assistant (VOICE.md V1).
+ *
+ * Hold the microphone (or `Ctrl`+`Space`), speak, let go: what you said is
+ * heard in this page — Moonshine for English, Whisper for French, Silero VAD
+ * keeping only the speech — and put in the composer, marked as said. Send
+ * it, and the answer is read aloud as it is written, sentence by sentence,
+ * by Datalayer's speech service (Kokoro, on ai-agents), the assistant's
+ * mouth moving with the sound and its balloon showing the sentence said.
+ *
+ * Three places, each named in the address when it is not this machine's:
+ * - `?agentRuntimesUrl=` — the agent-runtimes server and its `assistant`
+ *   agent (http://127.0.0.1:8765);
+ * - `?voiceModelsUrl=` — the browser's models, as pinned
+ *   (`scripts/voice/serve_store.py`, http://127.0.0.1:8770);
+ * - `?speechUrl=` — ai-agents' speech service (http://127.0.0.1:4401).
+ */
+import { useMemo, useState } from 'react';
+import { Heading, Link, SegmentedControl, Text } from '@primer/react';
+import { Box } from '@datalayer/primer-addons';
+import { useSimpleAuthStore } from '@datalayer/core/lib/views/otel';
+import { ThemedProvider } from './utils/themedProvider';
+import { ChatFloating } from '../chat';
+import { VOICE_CATALOGUE, voiceFor } from '../specs/voices';
+function fromAddress(name, otherwise) {
+    return ((typeof window === 'undefined'
+        ? null
+        : new URLSearchParams(window.location.search).get(name)) || otherwise).replace(/\/+$/, '');
+}
+const SERVER = fromAddress('agentRuntimesUrl', 'http://127.0.0.1:8765');
+const MODELS = fromAddress('voiceModelsUrl', 'http://127.0.0.1:8770');
+const SPEECH = fromAddress('speechUrl', 'http://127.0.0.1:4401');
+/** The packages that hear, loaded the first time the microphone is used. */
+const ENGINES = {
+    transformers: () => import('@huggingface/transformers'),
+    vad: () => import('@ricky0123/vad-web'),
+};
+const LANGUAGES = [
+    { tag: 'en-US', label: 'English' },
+    { tag: 'fr-FR', label: 'Français' },
+];
+const VoiceChatExample = () => {
+    const { token } = useSimpleAuthStore();
+    const [language, setLanguage] = useState('en-US');
+    const voiceSpec = voiceFor(language) ?? VOICE_CATALOGUE['kokoro-af-heart'];
+    const voice = useMemo(() => ({
+        language,
+        input: 'push_to_talk',
+        output: 'always',
+        voice: voiceSpec.id,
+        modelsUrl: MODELS,
+        engines: ENGINES,
+        speechUrl: SPEECH,
+        token: token || undefined,
+        consentKey: 'voice-chat-example',
+    }), [language, voiceSpec.id, token]);
+    return (_jsx(ThemedProvider, { children: _jsxs(Box, { sx: { minHeight: '100vh', bg: 'canvas.default', p: 4 }, children: [_jsxs(Box, { sx: { maxWidth: 720, mx: 'auto' }, children: [_jsx(Heading, { as: "h1", sx: { mb: 2 }, children: "Voice chat" }), _jsxs(Text, { as: "p", sx: { color: 'fg.muted', mb: 3 }, children: ["Talk to the assistant instead of typing: hold the microphone in its composer, or ", _jsx("kbd", { children: "Ctrl" }), "+", _jsx("kbd", { children: "Space" }), ", speak, and let go. What you say is heard in this page and never leaves it; the words go in the message box for you to send. Its answers are read aloud as they are written, by Datalayer\u2019s speech service, and not kept.", ' ', _jsx("kbd", { children: "Esc" }), " stops its voice."] }), _jsxs(Box, { sx: { display: 'flex', alignItems: 'center', gap: 3, mb: 3 }, children: [_jsx(SegmentedControl, { "aria-label": "Language", children: LANGUAGES.map(option => (_jsx(SegmentedControl.Button, { selected: language === option.tag, onClick: () => setLanguage(option.tag), "data-voice-language": option.tag, children: option.label }, option.tag))) }), _jsxs(Text, { sx: { color: 'fg.muted', fontSize: 1 }, children: ["Voice: ", voiceSpec.name] })] }), voiceSpec.attribution && (_jsx(Text, { as: "p", sx: { fontSize: 0, color: 'fg.muted', mb: 3 }, children: voiceSpec.attribution })), _jsxs(Text, { as: "p", sx: { fontSize: 0, color: 'fg.muted' }, children: ["Models from ", MODELS, ", speech from ", SPEECH, ", the agent at ", SERVER, ". See", ' ', _jsx(Link, { href: "https://agent-runtimes.datalayer.tech/chat/voice", children: "Voice" }), ' ', "for how to run each."] })] }), _jsx(ChatFloating, { defaultViewMode: "assistant", assistantCharacter: "paperclip", protocol: "vercel-ai", endpoint: `${SERVER}/api/v1/vercel-ai/assistant`, title: "Assistant", description: "Hello! Hold the microphone and ask me anything.", position: "bottom-right", useStore: false, voice: voice })] }) }));
+};
+export default VoiceChatExample;

package/lib/examples/example-selector.js

@@ -88,6 +88,7 @@ export const EXAMPLE_ENTRIES = [
     makeEntry('DecksAgent', () => import('./DecksAgent'), 'A deck beside the chat: the decks plugin in a Loop, and an agent that writes presentations as data.'),
     makeEntry('CellExample', () => import('./CellExample'), 'Simple cell example.'),
     makeEntry('ChatAssistantExample', () => import('./ChatAssistantExample'), 'The floating assistant: the chat as a character on the page.'),
+    makeEntry('VoiceChatExample', () => import('./VoiceChatExample'), 'A voice chat with the floating assistant: push-to-talk heard in the page, answers read aloud by the speech service, the mouth moving with the sound.'),
     makeEntry('ChatAssistantGalleryExample', () => import('./ChatAssistantGalleryExample'), 'Every representation of the floating assistant: each character in each state, its balloon, light and dark, still or moving, one at a time or as a grid.'),
     makeEntry('ChatCustomExample', () => import('./ChatCustomExample'), 'Custom chat experience composition example.'),
     makeEntry('ChatExample', () => import('./ChatExample'), 'Baseline chat integration example.'),

package/lib/examples/main.js

@@ -47,7 +47,7 @@ const DEFAULT_LOCAL_JUPYTER_SERVER_URL = 'http://localhost:8888/api/jupyter-serv
 const DEFAULT_LOCAL_JUPYTER_SERVER_TOKEN = '60c1661cc408f978c309d04157af55c9588ff9557c9380e4fb50785750703da6';
 const DEFAULT_CLOUD_RUNTIME_ENVIRONMENT = 'ai-agents-env';
 const EXAMPLE_GROUP_ORDER = [
-    'Loop',
+    'Apps',
     'A2UI',
     'A2A',
     'AG-UI',
@@ -70,6 +70,9 @@ const ANONYMOUS_EXAMPLES = new Set([
     // An application beside its computer, on the local agent-runtimes server:
     // nothing allocated on Datalayer, and the machine itself needs no token.
     'LoopAppComputerExample',
+    // The Sales and Accounting team: Sales in the page on a visitor's trial
+    // key, Accounting on the local server, which answers the machine itself.
+    'AgentA2ATeamExample',
     // Decide, on the local agent-runtimes server as above: Jev is asked
     // through the server's ai-inference, with the server's own key.
     'AgentDecideExample',
@@ -88,14 +91,15 @@ const ANONYMOUS_EXAMPLES = new Set([
     'ChatAssistantExample',
     // Every character in every state, from static data: no agent at all.
     'ChatAssistantGalleryExample',
+    // Voice: heard in the page, the agent on the local server, the speech
+    // service named in the address.
+    'VoiceChatExample',
 ]);
 const getExampleGroup = (id) => {
     if (id === 'AgentspecsExample' ||
-        id === 'LoopStrategyExample' ||
-        id === 'LoopWorkspaceExample' ||
-        id === 'LoopShellExample' ||
+        id.startsWith('Loop') ||
         id === 'DecksAgent') {
-        return 'Loop';
+        return 'Apps';
     }
     if (id.startsWith('A2Ui'))
         return 'A2UI';
@@ -110,7 +114,7 @@ const getExampleGroup = (id) => {
     // runtime: checkpoints, hooks, memory, guardrails…
     if (id.startsWith('Agent'))
         return 'Capabilities';
-    if (id.startsWith('Chat'))
+    if (id.startsWith('Chat') || id === 'VoiceChatExample')
         return 'Chat';
     // The document examples: the ones on the Lexical editor, and the page
     // with a document on it.
@@ -1072,13 +1076,14 @@ const ExampleAppThemed = ({ selectedExample, isChangingExample, error, ExampleCo
         }
         for (const [groupName, examples] of groups) {
             examples.sort((left, right) => {
-                if (groupName === 'Loop') {
+                if (groupName === 'Apps') {
                     // The shells first, most naked first; then the strategy that drives a
                     // notebook; then the library of specs behind them all.
                     const LOOP_ORDER = [
                         'LoopShellExample',
                         'LoopWorkspaceExample',
                         'LoopStrategyExample',
+                        'LoopAppComputerExample',
                         'AgentspecsExample',
                     ];
                     const loopOrder = (id) => {

package/lib/examples/utils/clippyJsCharacters.d.ts

@@ -0,0 +1,18 @@
+import type { AssistantCharacterData } from '../../chat/assistant/formats/types';
+import type { GalleryCharacter } from './AssistantGalleryGrid';
+/** The version of `clippyjs` the characters are read from. */
+export declare const CLIPPY_JS_VERSION = "0.1.0";
+/** The characters the package publishes: their names, and their folders. */
+export declare const CLIPPY_JS_CHARACTERS: readonly ["Clippy", "Merlin", "Links", "Rover", "Genie", "Genius", "F1", "Peedy", "Rocky", "Bonzi"];
+export type ClippyJsCharacterName = (typeof CLIPPY_JS_CHARACTERS)[number];
+/** One clippy.js character, fetched and read. */
+export declare function readClippyJsCharacter(name: ClippyJsCharacterName): Promise<AssistantCharacterData>;
+/**
+ * The clippy.js characters as the gallery shows them, fetched once `enabled`
+ * and kept while it stays so; each as soon as it is read.
+ */
+export declare function useClippyJsCharacters(enabled: boolean): {
+    characters: readonly GalleryCharacter[];
+    loading: boolean;
+    problem?: string;
+};

package/lib/examples/utils/clippyJsCharacters.js

@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * The characters clippy.js publishes — Clippy, Merlin, Links, Rover and the
+ * others — loaded into the floating assistant's gallery from the `clippyjs`
+ * package on jsDelivr, at the reader's request, and read through the same
+ * clippy.js reader a person's files go through.
+ *
+ * Nothing of them is in this repository: clippy.js's licence covers its code
+ * only, and the characters, their names and their drawings are Microsoft's
+ * (§6.9). The page fetches them when asked and keeps them in memory.
+ *
+ * The package ships each character as ES modules: `agent.mjs` exports the
+ * character's data, `map.mjs` its sprite sheet as a data URL. Both are read as
+ * text, never run: the data is rewritten as the `clippy.ready(...)` call the
+ * reader expects. Sounds are left out.
+ *
+ * @module examples/utils/clippyJsCharacters
+ */
+import { useEffect, useState } from 'react';
+import { readClippyCharacter } from '../../chat/assistant/formats/clippy';
+/** The version of `clippyjs` the characters are read from. */
+export const CLIPPY_JS_VERSION = '0.1.0';
+const BASE = `https://cdn.jsdelivr.net/npm/clippyjs@${CLIPPY_JS_VERSION}/dist/agents`;
+/** The characters the package publishes: their names, and their folders. */
+export const CLIPPY_JS_CHARACTERS = [
+    'Clippy',
+    'Merlin',
+    'Links',
+    'Rover',
+    'Genie',
+    'Genius',
+    'F1',
+    'Peedy',
+    'Rocky',
+    'Bonzi',
+];
+async function fetchText(url) {
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`${url} answered ${response.status}.`);
+    }
+    return response.text();
+}
+/** The value a module exports by default: `var x = <value>;\nexport …`. */
+function defaultExportText(module, what) {
+    const match = /^var [A-Za-z_$][\w$]* = ([\s\S]*?);\s*export\s*\{/m.exec(module);
+    if (!match) {
+        throw new Error(`${what} does not export its value as clippy.js 0.1 does.`);
+    }
+    return match[1];
+}
+/** One clippy.js character, fetched and read. */
+export async function readClippyJsCharacter(name) {
+    const folder = `${BASE}/${name.toLowerCase()}`;
+    const [agentModule, mapModule] = await Promise.all([
+        fetchText(`${folder}/agent.mjs`),
+        fetchText(`${folder}/map.mjs`),
+    ]);
+    const data = defaultExportText(agentModule, `${name}'s agent.mjs`);
+    const mapUrl = defaultExportText(mapModule, `${name}'s map.mjs`);
+    if (!/^"data:image\/(png|gif|webp|jpeg);base64,[A-Za-z0-9+/=]+"$/.test(mapUrl)) {
+        throw new Error(`${name}'s map.mjs does not hold an image.`);
+    }
+    const mapPng = await (await fetch(mapUrl.slice(1, -1))).blob();
+    return readClippyCharacter({
+        agentJs: `clippy.ready(${JSON.stringify(name)}, ${data});`,
+        mapPng,
+    });
+}
+/**
+ * The clippy.js characters as the gallery shows them, fetched once `enabled`
+ * and kept while it stays so; each as soon as it is read.
+ */
+export function useClippyJsCharacters(enabled) {
+    const [read, setRead] = useState([]);
+    const [pending, setPending] = useState(0);
+    const [problem, setProblem] = useState();
+    useEffect(() => {
+        if (!enabled) {
+            return;
+        }
+        let cancelled = false;
+        const loaded = [];
+        setRead([]);
+        setProblem(undefined);
+        setPending(CLIPPY_JS_CHARACTERS.length);
+        for (const name of CLIPPY_JS_CHARACTERS) {
+            readClippyJsCharacter(name).then(character => {
+                if (cancelled) {
+                    URL.revokeObjectURL(character.sprite);
+                    return;
+                }
+                loaded.push(character);
+                setRead(previous => [
+                    ...previous,
+                    {
+                        id: `clippy-js-${name.toLowerCase()}`,
+                        name: character.name,
+                        character,
+                        from: `clippy.js ${CLIPPY_JS_VERSION}`,
+                    },
+                ].sort((a, b) => CLIPPY_JS_CHARACTERS.indexOf(a.name) -
+                    CLIPPY_JS_CHARACTERS.indexOf(b.name)));
+                setPending(count => count - 1);
+            }, error => {
+                if (cancelled) {
+                    return;
+                }
+                setProblem(`${name}: ${error instanceof Error ? error.message : String(error)}`);
+                setPending(count => count - 1);
+            });
+        }
+        // The object URLs live as long as the characters are shown.
+        return () => {
+            cancelled = true;
+            for (const character of loaded) {
+                URL.revokeObjectURL(character.sprite);
+            }
+        };
+    }, [enabled]);
+    return {
+        characters: enabled ? read : [],
+        loading: enabled && pending > 0,
+        problem: enabled ? problem : undefined,
+    };
+}

package/lib/loop/apps/appspec.d.ts

@@ -24,7 +24,7 @@
  *
  * @module loop/apps/appspec
  */
-import type { AppAccent, AppAssistantCharacter, AppEmbedMode, AppKind, AppLayout, AppSpec } from '../../types/agentspecs';
+import type { AppAccent, AppAssistantCharacter, AppEmbedMode, AppVoiceSpec, AppKind, AppLayout, AppSpec } from '../../types/agentspecs';
 /** The version of the spec itself. */
 export declare const APP_SCHEMA = "loop.app/v1";
 /** The face of an application that has not chosen one. */
@@ -51,6 +51,8 @@ type Data = Record<string, unknown>;
 export declare function retentionDays(keepFor: string): number;
 /** An application with nothing said: what *Start blank* opens. */
 export declare function emptyAppspec(kind?: AppKind): AppSpec;
+/** An application's voice (VOICE.md VO-41): off unless said. */
+export declare const DEFAULT_VOICE: AppVoiceSpec;
 /** What reading a document gives: the application, and what the document got wrong. */
 export interface ParsedAppspec {
     app: AppSpec;

package/lib/loop/apps/appspec.js

@@ -214,6 +214,26 @@ function parseSurface(data) {
         composedAt: text(data.composed_at),
     };
 }
+/** An application's voice (VOICE.md VO-41): off unless said. */
+export const DEFAULT_VOICE = {
+    enabled: false,
+    input: 'push_to_talk',
+    output: 'on_request',
+    voice: '',
+    language: '',
+    where: 'auto',
+};
+function parseVoice(data) {
+    const voice = isData(data) ? data : {};
+    return {
+        enabled: voice.enabled === true,
+        input: oneOf(voice.input, ['off', 'push_to_talk', 'hands_free'], DEFAULT_VOICE.input),
+        output: oneOf(voice.output, ['off', 'on_request', 'always'], DEFAULT_VOICE.output),
+        voice: text(voice.voice),
+        language: text(voice.language),
+        where: oneOf(voice.where, ['auto', 'device', 'server'], DEFAULT_VOICE.where),
+    };
+}
 function parseInterface(data, kind) {
     const parsed = {
         layout: oneOf(data.layout, ['chat', 'page', 'split'], DEFAULT_LAYOUTS[kind]),
@@ -225,6 +245,7 @@ function parseInterface(data, kind) {
         })),
         settings: records(data.settings).map(parseSetting),
         components: texts(data.components),
+        voice: parseVoice(data.voice),
     };
     if (isData(data.surface)) {
         parsed.surface = parseSurface(data.surface);
@@ -493,6 +514,16 @@ function dumpInterface(spec, kind) {
     if (spec.assistant) {
         writer.data.assistant = spec.assistant;
     }
+    // Its voice, when it says one (VO-41): a spec made before voice has none.
+    if (spec.voice) {
+        writer.part('voice', new Writer()
+            .value('enabled', spec.voice.enabled, DEFAULT_VOICE.enabled)
+            .value('input', spec.voice.input, DEFAULT_VOICE.input)
+            .value('output', spec.voice.output, DEFAULT_VOICE.output)
+            .text('voice', spec.voice.voice)
+            .text('language', spec.voice.language)
+            .value('where', spec.voice.where, DEFAULT_VOICE.where).data);
+    }
     return writer.data;
 }
 function dumpDecision(decision) {

package/lib/loop/apps/checks.js

@@ -484,7 +484,7 @@ export function documentShapeProblems(document) {
     }
     for (const key of [
         'skills',
-        'tools',
+        'backend_tools',
         'context',
         'contents',
         'notifications',