@datalayer/agent-runtimes 1.5.43 → 1.5.45

package/lib/voice/captions.js

@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * Captions always (VOICE.md VO-65): the sentence being said is marked in the
+ * open conversation, not by colour alone. The answer that holds it is marked
+ * `aria-current` and `data-voice-saying` with a bar beside it; the sentence
+ * itself is underlined in place through the CSS Custom Highlight API, which
+ * draws over the text without touching the DOM React owns. A browser without
+ * that API keeps the answer's mark.
+ *
+ * @module voice/captions
+ */
+import { useEffect } from 'react';
+import { plainWords } from './sentences';
+import { answerText } from './useSpokenAnswers';
+/** The name the sentence's highlight is registered under. */
+export const SAID_HIGHLIGHT = 'voice-said';
+/** What the sentence being said looks like: underlined, thick, offset. */
+const SAID_STYLE = `::highlight(${SAID_HIGHLIGHT}) {
+  text-decoration: underline;
+  text-decoration-thickness: 2px;
+  text-underline-offset: 3px;
+  background-color: var(--bgColor-accent-muted, rgba(84, 174, 255, 0.2));
+}`;
+/** Words compared as words: spaces collapsed, case kept. */
+function squeezed(text) {
+    return text.replace(/\s+/g, ' ').trim();
+}
+/**
+ * The id of the answer a sentence is being said from: the newest answer of
+ * the agent whose words hold it, or none.
+ */
+export function saidMessageId(items, sentence) {
+    const said = squeezed(sentence ?? '');
+    if (!said) {
+        return undefined;
+    }
+    for (let index = items.length - 1; index >= 0; index -= 1) {
+        const text = answerText(items[index]);
+        if (text !== undefined && squeezed(plainWords(text)).includes(said)) {
+            const id = items[index].id;
+            return typeof id === 'string' ? id : undefined;
+        }
+    }
+    return undefined;
+}
+/**
+ * Where a sentence is in what an element shows, as a `Range` over its text
+ * — across the bold, the links and the line breaks it is drawn with — or
+ * none when the element does not show it as said (a code block said as *a
+ * code block*).
+ */
+export function sentenceRange(root, sentence) {
+    const said = squeezed(sentence);
+    if (!said || !root.ownerDocument) {
+        return undefined;
+    }
+    const document = root.ownerDocument;
+    const walker = document.createTreeWalker(root, 4 /* NodeFilter.SHOW_TEXT */);
+    // The text, spaces collapsed, and where each of its characters came from.
+    let text = '';
+    const from = [];
+    let space = true;
+    for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+        const value = node.nodeValue ?? '';
+        for (let offset = 0; offset < value.length; offset += 1) {
+            const char = value[offset];
+            if (/\s/.test(char)) {
+                if (space) {
+                    continue;
+                }
+                space = true;
+                text += ' ';
+            }
+            else {
+                space = false;
+                text += char;
+            }
+            from.push({ node, offset });
+        }
+    }
+    const start = text.indexOf(said);
+    if (start < 0) {
+        return undefined;
+    }
+    const first = from[start];
+    const last = from[start + said.length - 1];
+    const range = document.createRange();
+    range.setStart(first.node, first.offset);
+    range.setEnd(last.node, last.offset + 1);
+    return range;
+}
+function registry() {
+    const css = globalThis.CSS;
+    return css?.highlights;
+}
+function highlightOf(range) {
+    const Highlight = globalThis.Highlight;
+    return Highlight ? new Highlight(range) : undefined;
+}
+let styled = false;
+function ensureStyle(document) {
+    if (styled || document.getElementById('datalayer-voice-said')) {
+        styled = true;
+        return;
+    }
+    const style = document.createElement('style');
+    style.id = 'datalayer-voice-said';
+    style.textContent = SAID_STYLE;
+    document.head.appendChild(style);
+    styled = true;
+}
+/**
+ * Underlines `sentence` in the answer marked `data-voice-saying` under
+ * `root`, for as long as it is said; the mark goes when it ends. Drawn
+ * again whenever `version` changes (the answer still being written).
+ */
+export function useSaidSentence(root, sentence, version) {
+    useEffect(() => {
+        const highlights = registry();
+        const element = root.current?.querySelector('[data-voice-saying]');
+        if (!highlights || !element || !sentence) {
+            return;
+        }
+        const underline = () => {
+            const range = sentenceRange(element, sentence);
+            const highlight = range ? highlightOf(range) : undefined;
+            if (!highlight) {
+                return false;
+            }
+            ensureStyle(element.ownerDocument);
+            highlights.set(SAID_HIGHLIGHT, highlight);
+            return true;
+        };
+        // The answer's Markdown may be drawn after this effect: wait for it.
+        let watcher;
+        if (!underline() && typeof MutationObserver !== 'undefined') {
+            watcher = new MutationObserver(() => {
+                if (underline()) {
+                    watcher?.disconnect();
+                }
+            });
+            watcher.observe(element, {
+                childList: true,
+                subtree: true,
+                characterData: true,
+            });
+        }
+        return () => {
+            watcher?.disconnect();
+            highlights.delete(SAID_HIGHLIGHT);
+        };
+    }, [root, sentence, version]);
+}

package/lib/voice/chatVoice.d.ts

@@ -59,7 +59,10 @@ export declare const SILENCE_MS = 1500;
  * in: the person's own settings first where they made some (`voiceForPerson`:
  * their language, voice, rate, volume, microphone; VO-14, VO-25, VO-30),
  * else the host's; answers not heard, the language's first voice,
- * Datalayer's models and packages, a silence of 1.5 s. Answers heard need
- * the speech service's address: refused without.
+ * Datalayer's models and packages, a silence of 1.5 s. A person who pauses
+ * longer before they are done has both silences — the one that stops a
+ * microphone clicked on, and the one that ends a hands-free turn — made
+ * longer by their choice (`pause`, VO-69). Answers heard need the speech
+ * service's address: refused without.
  */
 export declare function settledVoice(given: ChatVoice, person?: VoiceSettings): SettledVoice;

package/lib/voice/chatVoice.js

@@ -11,7 +11,8 @@
  * @module voice/chatVoice
  */
 import { voiceFor } from '../specs/voices';
-import { heldVoiceSettings, voiceForPerson, } from './settings';
+import { TURN_SILENCE_MS } from './handsFree';
+import { heldVoiceSettings, saneVoiceSettings, voiceForPerson, } from './settings';
 /**
  * Datalayer's origin for the browser's speech models, as pinned
  * (`s3://datalayer-models-speech` behind its CDN; VO-48, VO-49): each model
@@ -84,11 +85,15 @@ export const SILENCE_MS = 1500;
  * in: the person's own settings first where they made some (`voiceForPerson`:
  * their language, voice, rate, volume, microphone; VO-14, VO-25, VO-30),
  * else the host's; answers not heard, the language's first voice,
- * Datalayer's models and packages, a silence of 1.5 s. Answers heard need
- * the speech service's address: refused without.
+ * Datalayer's models and packages, a silence of 1.5 s. A person who pauses
+ * longer before they are done has both silences — the one that stops a
+ * microphone clicked on, and the one that ends a hands-free turn — made
+ * longer by their choice (`pause`, VO-69). Answers heard need the speech
+ * service's address: refused without.
  */
 export function settledVoice(given, person = heldVoiceSettings()) {
     const voice = voiceForPerson(given, person);
+    const pause = saneVoiceSettings(person).pause ?? 1;
     const output = voice.output ?? 'off';
     if (output !== 'off' && !voice.speechUrl) {
         throw new Error(`A chat whose answers are heard (${output}) names the speech service: its speechUrl is missing.`);
@@ -99,6 +104,7 @@ export function settledVoice(given, person = heldVoiceSettings()) {
         voice: voice.voice || voiceFor(voice.language)?.id || '',
         modelsUrl: voice.modelsUrl || DATALAYER_SPEECH_MODELS_URL,
         engines: voice.engines ?? DATALAYER_VOICE_ENGINES,
-        silenceMs: voice.silenceMs ?? SILENCE_MS,
+        silenceMs: Math.round((voice.silenceMs ?? SILENCE_MS) * pause),
+        turnSilenceMs: Math.round((voice.turnSilenceMs ?? TURN_SILENCE_MS) * pause),
     };
 }

package/lib/voice/choices.d.ts

@@ -28,6 +28,8 @@ export declare function catalogueVoiceSpeaks(voice: string, language: string): b
 /** The speeds offered, as multipliers of the voice's own. */
 export declare const RATE_CHOICES: VoiceChoice<number>[];
 /** The loudnesses offered, 0 to 1. */
+/** How long the person pauses before they are done (VO-69): the silences, times this. */
+export declare const PAUSE_CHOICES: VoiceChoice<number>[];
 export declare const VOLUME_CHOICES: VoiceChoice<number>[];
 /** When answers are heard (VO-21): what a person may choose where the application speaks. */
 export declare const OUTPUT_CHOICES: VoiceChoice<NonNullable<ChatVoice['output']>>[];

package/lib/voice/choices.js

@@ -56,6 +56,12 @@ export const RATE_CHOICES = [
     { value: 1.5, label: 'Fastest' },
 ];
 /** The loudnesses offered, 0 to 1. */
+/** How long the person pauses before they are done (VO-69): the silences, times this. */
+export const PAUSE_CHOICES = [
+    { value: 1, label: 'Usual' },
+    { value: 2, label: 'Longer' },
+    { value: 3, label: 'Much longer' },
+];
 export const VOLUME_CHOICES = [
     { value: 0.25, label: 'Quiet' },
     { value: 0.5, label: 'Half' },

package/lib/voice/handsFree.d.ts

@@ -56,3 +56,15 @@ export declare function turnEnded(heard: HeardSoFarLike, turnSilenceMs?: number)
  * answer's echo — the browser's echo cancellation keeps that out.
  */
 export declare function bargedIn(state: HandsFreeState, heard: HeardSoFarLike): boolean;
+/** A message as the conversation holds it: enough to find the answer spoken over. */
+export interface ConversationItemLike {
+    id: string;
+    role?: string;
+}
+/**
+ * The answer the person spoke over (VO-13): the agent's last message after
+ * the person's last one — still being written, or written and being said.
+ * `undefined` while the agent has not answered the turn yet: there is
+ * nothing to mark.
+ */
+export declare function answerSpokenOver(items: readonly ConversationItemLike[]): string | undefined;

package/lib/voice/handsFree.js

@@ -46,3 +46,21 @@ export function turnEnded(heard, turnSilenceMs = TURN_SILENCE_MS) {
 export function bargedIn(state, heard) {
     return (state === 'speaking' || state === 'thinking') && heard.spoke;
 }
+/**
+ * The answer the person spoke over (VO-13): the agent's last message after
+ * the person's last one — still being written, or written and being said.
+ * `undefined` while the agent has not answered the turn yet: there is
+ * nothing to mark.
+ */
+export function answerSpokenOver(items) {
+    for (let i = items.length - 1; i >= 0; i -= 1) {
+        const role = items[i].role;
+        if (role === 'user') {
+            return undefined;
+        }
+        if (role === 'assistant') {
+            return items[i].id;
+        }
+    }
+    return undefined;
+}

package/lib/voice/index.d.ts

@@ -14,6 +14,7 @@ export * from './pinned';
 export * from './capture';
 export * from './hearing';
 export * from './serverHearing';
+export * from './sayAloud';
 export * from './handsFree';
 export { HandsFree, type HandsFreeProps } from './HandsFree';
 export * from './speaker';

package/lib/voice/index.js

@@ -8,6 +8,7 @@ export * from './pinned';
 export * from './capture';
 export * from './hearing';
 export * from './serverHearing';
+export * from './sayAloud';
 export * from './handsFree';
 export { HandsFree } from './HandsFree';
 export * from './speaker';

package/lib/voice/sayAloud.d.ts

@@ -0,0 +1,25 @@
+/**
+ * What only the runtime knows of voice, as it says it over AG-UI (VOICE.md
+ * VO-35) and Vercel AI (VO-36): `voice.settings` — the running application's
+ * voice and language, at a run's start — and `voice.say` — a short line the
+ * agent wants heard and not written (*One moment*), shown as a caption while
+ * it is said. The chat reads `voice.say` from a run's custom events
+ * (`CUSTOM`, the chat's `activity`) or data parts (`data-voice.say`); it
+ * already knows its application's voice from `/apps/configure` (VO-45), and
+ * `voice.settings` is for clients that do not ask it.
+ *
+ * @module voice/sayAloud
+ */
+/** A line the agent wants heard, not written. */
+export declare const VOICE_SAY_EVENT = "voice.say";
+/** The running application's voice, at a run's start. */
+export declare const VOICE_SETTINGS_EVENT = "voice.settings";
+/** The longest line said aloud: the runtime refuses longer ones. */
+export declare const SAY_ALOUD_MAX_CHARACTERS = 200;
+/** A custom event or data part, as the chat's adapters give it. */
+export interface VoiceActivity {
+    type: string;
+    data: unknown;
+}
+/** The line a `voice.say` says, or `undefined` for anything else. */
+export declare function lineToSay(activity: VoiceActivity): string | undefined;

package/lib/voice/sayAloud.js

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * What only the runtime knows of voice, as it says it over AG-UI (VOICE.md
+ * VO-35) and Vercel AI (VO-36): `voice.settings` — the running application's
+ * voice and language, at a run's start — and `voice.say` — a short line the
+ * agent wants heard and not written (*One moment*), shown as a caption while
+ * it is said. The chat reads `voice.say` from a run's custom events
+ * (`CUSTOM`, the chat's `activity`) or data parts (`data-voice.say`); it
+ * already knows its application's voice from `/apps/configure` (VO-45), and
+ * `voice.settings` is for clients that do not ask it.
+ *
+ * @module voice/sayAloud
+ */
+/** A line the agent wants heard, not written. */
+export const VOICE_SAY_EVENT = 'voice.say';
+/** The running application's voice, at a run's start. */
+export const VOICE_SETTINGS_EVENT = 'voice.settings';
+/** The longest line said aloud: the runtime refuses longer ones. */
+export const SAY_ALOUD_MAX_CHARACTERS = 200;
+const nameOf = (type) => type.replace(/^data-/, '');
+/** The line a `voice.say` says, or `undefined` for anything else. */
+export function lineToSay(activity) {
+    if (nameOf(activity.type) !== VOICE_SAY_EVENT) {
+        return undefined;
+    }
+    const data = activity.data;
+    const text = data && typeof data.text === 'string'
+        ? data.text.split(/\s+/).join(' ').trim()
+        : '';
+    return text ? text.slice(0, SAY_ALOUD_MAX_CHARACTERS) : undefined;
+}

package/lib/voice/serverHearing.d.ts

@@ -2,10 +2,19 @@ import type { Hearing, HearingProgress, HeardSoFar } from './hearing';
 import type { Transcript } from './types';
 /** Where the speech service hears a recording, on ai-agents' host. */
 export declare const TRANSCRIPTIONS_PATH = "/api/ai-agents/v1/speech/transcriptions";
+/** Where the speech service hears speech as it is said. */
+export declare const TRANSCRIPTION_STREAM_PATH = "/api/ai-agents/v1/speech/transcriptions/stream";
+/** A stream no look or hearing has fed for this long is closed: its press was cancelled. */
+export declare const STREAM_IDLE_MS = 3000;
+/** How long the final words of a stream are waited for. */
+export declare const STREAM_FINAL_MS = 60000;
 /** The model the speech service hears with (agentspecs' catalogue). */
 export declare const SERVER_HEARING_MODEL = "whisper-large-v3-turbo";
 /** Where a chat's voice hears: in the browser, or on Datalayer's servers. */
 export type HearingPlace = 'device' | 'server';
+/** The speech service refused a press, in its own sentence (a limit, a visitor's day). */
+export declare class ServerRefused extends Error {
+}
 /** What decides where a chat's voice hears. */
 export interface HearingAsked {
     language: string;
@@ -40,7 +49,26 @@ export interface ServerHearingOptions {
     /** Let Whisper find the language (VO-14), rather than tell it. */
     detect?: boolean;
     fetcher?: typeof fetch;
+    /** Opens a WebSocket; the browser's own unless a test gives one. */
+    socket?: (url: string) => SpeechSocket;
+}
+/** What `ServerHearing` needs of a WebSocket. */
+export interface SpeechSocket {
+    readonly readyState: number;
+    binaryType: string;
+    onopen: ((event: unknown) => void) | null;
+    onmessage: ((event: {
+        data: unknown;
+    }) => void) | null;
+    onclose: ((event: unknown) => void) | null;
+    onerror: ((event: unknown) => void) | null;
+    send(data: string | ArrayBuffer): void;
+    close(code?: number): void;
 }
+/** 16 kHz samples in [-1, 1] as little-endian 16-bit PCM: what a `pcm16` stream takes. */
+export declare function pcm16Of(samples: Float32Array): ArrayBuffer;
+/** The ws(s) address of the stream on ai-agents' origin. */
+export declare function streamUrlOf(speechUrl: string): string;
 /**
  * Hears on Datalayer's servers: the browser's VAD finds the speech and when
  * it stopped; Whisper, on the speech service, writes it.
@@ -52,7 +80,40 @@ export declare class ServerHearing implements Hearing {
     constructor(finder: SpeechFinder, options: ServerHearingOptions);
     /** Only the VAD loads in the page: the words are written on the server. */
     ready(_language: string, progress?: HearingProgress): Promise<void>;
-    /** Whether speech was found and how long it has been silent; its words come at the end. */
-    hearSoFar(audio: Float32Array): Promise<HeardSoFar>;
+    /** The press being heard as it is said, while one is. */
+    private press?;
+    /** The press's stream, opened anew when a new press began (its sound is shorter). */
+    private streamFor;
+    /**
+     * Whether speech was found and how long it has been silent, and the words
+     * the speech service heard so far: the sound goes to its stream as it comes.
+     */
+    hearSoFar(audio: Float32Array, language?: string): Promise<HeardSoFar>;
+    hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
+}
+/** Said when the device's model did not load and the server will hear instead. */
+export declare const MOVED_TO_SERVER = "This device could not load its model: what you say will be heard on Datalayer's servers instead. Press the microphone again to agree.";
+/**
+ * `where: auto` (§5): each press heard on the device when a model there
+ * hears the language and loads, on the speech service otherwise.
+ *
+ * The device's model failing to load moves the hearing to the server, and
+ * the press is refused in a sentence so that the server's own consent is
+ * asked at the next one. The server refusing a press — a limit, a visitor's
+ * day spent — moves it back to the device when a model there hears the
+ * language, and the press is heard there: the sound then stays in the page.
+ */
+export declare class AutoHearing implements Hearing {
+    private readonly device;
+    private readonly server;
+    private readonly hearsHere;
+    private place;
+    /** The device's model did not load: it is not tried again on this page. */
+    private deviceBroken;
+    constructor(device: Hearing, server: Hearing, language: string, hearsHere?: (language: string) => boolean);
+    get where(): HearingPlace;
+    private get current();
+    ready(language: string, progress?: HearingProgress): Promise<void>;
+    hearSoFar(audio: Float32Array, language: string): Promise<HeardSoFar>;
     hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
 }