@datalayer/agent-runtimes 1.5.61 → 1.5.62

package/lib/chat/assistant/AssistantStage.js

@@ -1,5 +1,5 @@
 import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
-import { Suspense, lazy, useCallback, useEffect, useId, useRef, useState, } from 'react';
+import { Suspense, lazy, useCallback, useEffect, useId, useReducer, useRef, useState, } from 'react';
 import { ActionList, ActionMenu, IconButton, useTheme } from '@primer/react';
 import { Box } from '@datalayer/primer-addons';
 import { CodeIcon, CommentDiscussionIcon, DependabotIcon, InfoIcon, MuteIcon, PencilIcon, PlusIcon, PulseIcon, ScreenNormalIcon, SquareFillIcon, TrashIcon, UnmuteIcon, XIcon, } from '@primer/octicons-react';
@@ -11,6 +11,7 @@ import { ASSISTANT_CLEARANCE_VAR } from './clearance';
 import { AssistantContextMenu, LONG_PRESS_MS, opensContextMenu, } from './AssistantContextMenu';
 import { BalloonExpandContext, ExpandedVisual, focusExpanded, } from './BalloonVisual';
 import { SpriteCharacter } from './SpriteCharacter';
+import { BALLOON_GRACE_MS, BALLOON_HOVER_START, balloonHover, graceRuns, } from './balloonHover';
 import { isAssistantModel3D } from './avatar3d';
 import { conversationCount, } from './toolLine';
 import { ASSISTANT_OBSTACLES, POINTER_CALM_MS, mouthOpening, boxesMeet, pointerNear, } from './state';
@@ -393,7 +394,33 @@ export function AssistantStage({ character, state, size = 88, place, stageRef, o
     // Drawn for the chat's colour mode: the dark drawing on a dark page (T-25).
     const { colorScheme } = useTheme();
     const colorMode = colorScheme?.startsWith('dark') ? 'dark' : 'light';
-    const [hovered, setHovered] = useState(false);
+    // The figure and its balloon are one hover region: the balloon stays
+    // while the pointer crosses to it, and closes once the pointer has been
+    // on neither for a moment, on Escape or on a press outside.
+    const [hover, dispatchHover] = useReducer(balloonHover, BALLOON_HOVER_START);
+    const hovered = hover.shown;
+    const hoverGrace = graceRuns(hover);
+    useEffect(() => {
+        if (!hoverGrace) {
+            return;
+        }
+        const grace = setTimeout(() => dispatchHover('grace'), BALLOON_GRACE_MS);
+        return () => clearTimeout(grace);
+    }, [hoverGrace]);
+    useEffect(() => {
+        if (!hovered) {
+            return;
+        }
+        const onPress = (event) => {
+            const target = event.target;
+            if (target && stageRef.current?.contains(target)) {
+                return;
+            }
+            dispatchHover('outside');
+        };
+        document.addEventListener('pointerdown', onPress, true);
+        return () => document.removeEventListener('pointerdown', onPress, true);
+    }, [hovered, stageRef]);
     const [menuOpen, setMenuOpen] = useState(false);
     // Its own menu, where it was opened; the inspector and *About* it opens.
     const [menuAt, setMenuAt] = useState(null);
@@ -596,7 +623,28 @@ export function AssistantStage({ character, state, size = 88, place, stageRef, o
             '@media (prefers-reduced-motion: reduce)': {
                 '& *': { animation: 'none !important' },
             },
-        }, onMouseEnter: () => setHovered(true), onMouseLeave: () => setHovered(false), children: [shown && (_jsx(ExpandedVisual, { visual: shown, target: expandTarget, onClose: () => setExpanded(null) })), showBalloon && balloon && (_jsx(BalloonExpandContext.Provider, { value: expand, children: _jsx(SpeechBalloon, { text: balloon.text, more: balloon.more, approval: balloon.approval, onDismissPeek: balloon.onDismiss, display: balloonDisplay, tool: balloon.tool, speaking: balloon.speaking, busy: balloon.busy, attachment: balloon.attachment, history: balloon.history, onExpand: expand ?? undefined, expandTitle: visual?.title, onExpandHistory: historyVisual
+        }, onMouseEnter: () => dispatchHover('enter'), onMouseLeave: () => dispatchHover('leave'), onFocus: (event) => {
+            // Keyboard focus keeps the balloon; a click's focus does not.
+            let keyboard = true;
+            try {
+                keyboard = event.target.matches(':focus-visible');
+            }
+            catch {
+                // No :focus-visible here: count it.
+            }
+            if (keyboard) {
+                dispatchHover('focus');
+            }
+        }, onBlur: (event) => {
+            const next = event.relatedTarget;
+            if (!next || !event.currentTarget.contains(next)) {
+                dispatchHover('blur');
+            }
+        }, onKeyDown: (event) => {
+            if (event.key === 'Escape' && hovered) {
+                dispatchHover('escape');
+            }
+        }, children: [shown && (_jsx(ExpandedVisual, { visual: shown, target: expandTarget, onClose: () => setExpanded(null) })), showBalloon && balloon && (_jsx(BalloonExpandContext.Provider, { value: expand, children: _jsx(SpeechBalloon, { text: balloon.text, more: balloon.more, approval: balloon.approval, onDismissPeek: balloon.onDismiss, display: balloonDisplay, tool: balloon.tool, speaking: balloon.speaking, busy: balloon.busy, attachment: balloon.attachment, history: balloon.history, onExpand: expand ?? undefined, expandTitle: visual?.title, onExpandHistory: historyVisual
                         ? () => {
                             setExpanded(historyVisual);
                             if (expandTarget) {

package/lib/chat/assistant/balloonHover.d.ts

@@ -0,0 +1,29 @@
+/**
+ * When the assistant's balloon shows for a hover — the hover-intent rule.
+ *
+ * The figure and its balloon are one region: the balloon opens when the
+ * pointer enters either, and stays while it is over either. Leaving them
+ * does not close it at once — the pointer crosses the gap between the figure
+ * and the balloon, or a 3D figure's canvas redraws under it — it closes once
+ * the pointer has been on neither for `BALLOON_GRACE_MS`. Keyboard focus in
+ * the region keeps it open too. Escape, or a press outside, closes it at
+ * once; it opens again on the next enter or focus.
+ *
+ * Pure: the stage dispatches what happened and runs the grace timer while
+ * `graceRuns` says so.
+ */
+/** How long the balloon stays once the pointer is on neither, in ms. */
+export declare const BALLOON_GRACE_MS = 700;
+export interface BalloonHover {
+    /** The balloon shows for the hover. */
+    shown: boolean;
+    /** The pointer is over the figure or its balloon. */
+    over: boolean;
+    /** Keyboard focus is in the figure or its balloon. */
+    focused: boolean;
+}
+export type BalloonHoverEvent = 'enter' | 'leave' | 'focus' | 'blur' | 'grace' | 'escape' | 'outside';
+export declare const BALLOON_HOVER_START: BalloonHover;
+export declare function balloonHover(state: BalloonHover, event: BalloonHoverEvent): BalloonHover;
+/** Whether the grace timer runs: shown, the pointer on neither, no focus. */
+export declare function graceRuns(state: BalloonHover): boolean;

package/lib/chat/assistant/balloonHover.js

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * When the assistant's balloon shows for a hover — the hover-intent rule.
+ *
+ * The figure and its balloon are one region: the balloon opens when the
+ * pointer enters either, and stays while it is over either. Leaving them
+ * does not close it at once — the pointer crosses the gap between the figure
+ * and the balloon, or a 3D figure's canvas redraws under it — it closes once
+ * the pointer has been on neither for `BALLOON_GRACE_MS`. Keyboard focus in
+ * the region keeps it open too. Escape, or a press outside, closes it at
+ * once; it opens again on the next enter or focus.
+ *
+ * Pure: the stage dispatches what happened and runs the grace timer while
+ * `graceRuns` says so.
+ */
+/** How long the balloon stays once the pointer is on neither, in ms. */
+export const BALLOON_GRACE_MS = 700;
+export const BALLOON_HOVER_START = {
+    shown: false,
+    over: false,
+    focused: false,
+};
+export function balloonHover(state, event) {
+    switch (event) {
+        case 'enter':
+            return { ...state, over: true, shown: true };
+        case 'leave':
+            // Still shown: the grace runs.
+            return { ...state, over: false };
+        case 'focus':
+            return { ...state, focused: true, shown: true };
+        case 'blur':
+            return { ...state, focused: false };
+        case 'grace':
+            return state.over || state.focused ? state : { ...state, shown: false };
+        case 'escape':
+            return { ...state, shown: false };
+        case 'outside':
+            return { ...state, shown: false, over: false, focused: false };
+    }
+}
+/** Whether the grace timer runs: shown, the pointer on neither, no focus. */
+export function graceRuns(state) {
+    return state.shown && !state.over && !state.focused;
+}

package/lib/voice/speaker.d.ts

@@ -57,6 +57,14 @@ export interface SpeakerOptions {
      * caller that follows its sentences one by one (a deck's narrator).
      */
     firstClauseFirst?: boolean;
+    /**
+     * Ask each sentence streamed (VO-52): the speech service says it piece by
+     * piece where it pauses and sends each piece's 16-bit PCM as soon as it is
+     * said, and the speaker plays the pieces as they come — the first sound
+     * after the first piece, not the whole sentence. A service that answers a
+     * whole sound instead is played as before.
+     */
+    streamed?: boolean;
     fetcher?: typeof fetch;
 }
 /**
@@ -84,6 +92,8 @@ export declare class ServerSpeaker {
     private buffer?;
     private queue;
     private playing?;
+    /** The pieces of a streamed sentence scheduled to play, stopped together. */
+    private scheduled;
     private running;
     private generation;
     private state;
@@ -109,6 +119,11 @@ export declare class ServerSpeaker {
     private askAhead;
     private fetch;
     private run;
+    /**
+     * A sentence streamed as 16-bit PCM (VO-52): played in slices as they
+     * come, each scheduled right after the last, until the last has ended.
+     */
+    private playStreamed;
     /** Stop at once: what plays and what waits (barge-in, `Esc`; VO-13). */
     stop(): void;
     /** Stop, and let the audio context go. */

package/lib/voice/speaker.js

@@ -13,6 +13,8 @@
  */
 /** The path of the syntheses on ai-agents. */
 export const SYNTHESES_PATH = '/api/ai-agents/v1/speech/syntheses';
+/** Played as it comes: at least this much at a time, in seconds. */
+const STREAMED_SLICE_SECONDS = 0.25;
 /**
  * How many sentences are asked of the speech service ahead of the one
  * playing: enough that the next is ready when one ends, and under the calls
@@ -58,6 +60,8 @@ export class ServerSpeaker {
     buffer;
     queue = [];
     playing;
+    /** The pieces of a streamed sentence scheduled to play, stopped together. */
+    scheduled = [];
     running = false;
     generation = 0;
     state = { speaking: false };
@@ -129,6 +133,7 @@ export class ServerSpeaker {
         }
         // Nothing playing or waiting: this is the first thing heard (VO-52).
         const first = this.options.firstClauseFirst &&
+            !this.options.streamed &&
             !this.queue.length &&
             !this.playing &&
             !this.running;
@@ -165,7 +170,9 @@ export class ServerSpeaker {
                     text,
                     voice,
                     language,
-                    format: 'opus',
+                    ...(this.options.streamed
+                        ? { format: 'pcm', stream: true }
+                        : { format: 'opus' }),
                 }),
             });
             if (!answered.ok) {
@@ -174,7 +181,18 @@ export class ServerSpeaker {
                     ? body.detail
                     : `The answer could not be spoken (${answered.status}).`;
             }
-            return await answered.arrayBuffer();
+            const kind = answered.headers.get('content-type') ?? '';
+            if (kind.toLowerCase().startsWith('audio/l16')) {
+                const rate = Number(answered.headers.get('x-speech-rate')) ||
+                    Number(/rate=(\d+)/i.exec(kind)?.[1]) ||
+                    24000;
+                return {
+                    kind: 'pcm',
+                    rate,
+                    body: answered.body ?? (await answered.arrayBuffer()),
+                };
+            }
+            return { kind: 'whole', audio: await answered.arrayBuffer() };
         }
         catch {
             return 'Datalayer’s speech service could not be reached: the answer is still written here.';
@@ -200,7 +218,11 @@ export class ServerSpeaker {
                 this.queue.shift();
                 this.askAhead();
                 const context = this.audio();
-                const decoded = await context.decodeAudioData(audio);
+                if (audio.kind === 'pcm') {
+                    await this.playStreamed(generation, next, audio);
+                    continue;
+                }
+                const decoded = await context.decodeAudioData(audio.audio);
                 if (generation !== this.generation) {
                     return;
                 }
@@ -230,17 +252,103 @@ export class ServerSpeaker {
             }
         }
     }
+    /**
+     * A sentence streamed as 16-bit PCM (VO-52): played in slices as they
+     * come, each scheduled right after the last, until the last has ended.
+     */
+    async playStreamed(generation, next, answer) {
+        const context = this.audio();
+        const speed = this.speed();
+        if (this.gain) {
+            this.gain.gain.value = this.loudness();
+        }
+        let at = 0;
+        let first = true;
+        let pending = new Uint8Array(0);
+        let last = Promise.resolve();
+        const slice = Math.round(answer.rate * STREAMED_SLICE_SECONDS) * 2;
+        const play = (bytes) => {
+            const samples = new Float32Array(bytes.length / 2);
+            const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length);
+            for (let k = 0; k < samples.length; k++) {
+                samples[k] = view.getInt16(k * 2, true) / 32768;
+            }
+            const buffer = context.createBuffer(1, samples.length, answer.rate);
+            buffer.getChannelData(0).set(samples);
+            const source = context.createBufferSource();
+            source.buffer = buffer;
+            source.playbackRate.value = speed;
+            source.connect(this.analyser);
+            at = Math.max(at, context.currentTime);
+            last = new Promise(resolve => {
+                source.onended = () => {
+                    this.scheduled = this.scheduled.filter(one => one !== source);
+                    resolve();
+                };
+            });
+            this.scheduled.push(source);
+            this.playing = source;
+            if (first) {
+                first = false;
+                this.set({ speaking: true, sentence: next.text, refused: undefined });
+                this.options.onFirstSound?.(Math.round(performance.now() - next.asked));
+            }
+            source.start(at);
+            at += buffer.duration / speed;
+        };
+        const take = (chunk, done) => {
+            const joined = new Uint8Array(pending.length + chunk.length);
+            joined.set(pending);
+            joined.set(chunk, pending.length);
+            pending = joined;
+            while (pending.length >= slice || (done && pending.length >= 2)) {
+                const size = Math.min(pending.length, Math.max(slice, 2));
+                const even = size - (size % 2);
+                play(pending.subarray(0, even));
+                pending = pending.slice(even);
+                if (done && pending.length < 2) {
+                    break;
+                }
+            }
+        };
+        if (answer.body instanceof ArrayBuffer) {
+            take(new Uint8Array(answer.body), true);
+        }
+        else {
+            const reader = answer.body.getReader();
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (generation !== this.generation) {
+                    void reader.cancel().catch(() => undefined);
+                    return;
+                }
+                if (done) {
+                    take(new Uint8Array(0), true);
+                    break;
+                }
+                take(value, false);
+            }
+        }
+        await last;
+        this.playing = undefined;
+    }
     /** Stop at once: what plays and what waits (barge-in, `Esc`; VO-13). */
     stop() {
         this.generation += 1;
         this.queue = [];
         this.running = false;
-        try {
-            this.playing?.stop();
-        }
-        catch {
-            // Already ended.
+        for (const source of [
+            ...this.scheduled,
+            ...(this.playing ? [this.playing] : []),
+        ]) {
+            try {
+                source.stop();
+            }
+            catch {
+                // Already ended.
+            }
         }
+        this.scheduled = [];
         this.playing = undefined;
         this.set({ speaking: false, sentence: undefined });
     }

package/lib/voice/useVoiceSpeaker.js

@@ -30,7 +30,9 @@ export function useVoiceSpeaker(voice, on = true) {
             rate: voice.rate,
             volume: voice.volume,
             onState: setState,
-            // The chat's answers: a long first sentence's first clause first.
+            // The chat's answers: streamed, played as they are said (VO-52);
+            // from a service that does not stream, the first clause first.
+            streamed: true,
             firstClauseFirst: true,
         })
         : undefined, 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@datalayer/agent-runtimes",
-    "version": "1.5.61",
+    "version": "1.5.62",
     "type": "module",
     "workspaces": [
         ".",