@datalayer/agent-runtimes 1.3.63 → 1.3.65

package/lib/voice/speaker.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Answers heard, from Datalayer's speech service (VOICE.md VO-20, VO-33,
+ * decision 1): each sentence is sent to ai-agents as soon as it is written
+ * and played as soon as its audio arrives, one after the other, while the
+ * next ones are being made. The level of what plays moves the assistant's
+ * mouth (VO-23). Nothing is kept: each sentence's audio is dropped once played.
+ *
+ * @module voice/speaker
+ */
+/** The path of the syntheses on ai-agents. */
+export declare const SYNTHESES_PATH = "/api/ai-agents/v1/speech/syntheses";
+/** What the speaker is doing, for the assistant and the controls. */
+export interface SpeakerState {
+    /** Sound is playing. */
+    speaking: boolean;
+    /** The sentence being said, for the balloon (VO-26). */
+    sentence?: string;
+    /** Why the last sentence could not be said, in a sentence. */
+    refused?: string;
+}
+export interface SpeakerOptions {
+    /** ai-agents' origin, `https://r1.datalayer.run` or a local one. */
+    speechUrl: string;
+    /** The voice of the catalogue. */
+    voice: string;
+    language: string;
+    /** The person's token, an application's principal's, or a visitor's. */
+    token?: string;
+    /** Called whenever the state changes. */
+    onState?: (state: SpeakerState) => void;
+    /** Called with how long a sentence took from asked to first sound, in ms. */
+    onFirstSound?: (ms: number) => void;
+    fetcher?: typeof fetch;
+}
+/** Plays an answer's sentences, said by the speech service. */
+export declare class ServerSpeaker {
+    private options;
+    private context?;
+    private analyser?;
+    private buffer?;
+    private queue;
+    private playing?;
+    private running;
+    private generation;
+    private state;
+    constructor(options: SpeakerOptions);
+    /** Change the voice, the language or the token for the sentences to come. */
+    update(options: Partial<SpeakerOptions>): void;
+    /**
+     * Must be called from a gesture once (a press, a click): a browser plays
+     * nothing a page did not start from one.
+     */
+    unlock(): void;
+    private audio;
+    /** How loud what plays is, between 0 and 1: what moves the mouth. */
+    level(): number;
+    private set;
+    /** Say a sentence after those already given. */
+    say(text: string): void;
+    private fetch;
+    private run;
+    /** Stop at once: what plays and what waits (barge-in, `Esc`; VO-13). */
+    stop(): void;
+    /** Stop, and let the audio context go. */
+    close(): void;
+}

package/lib/voice/speaker.js

@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * Answers heard, from Datalayer's speech service (VOICE.md VO-20, VO-33,
+ * decision 1): each sentence is sent to ai-agents as soon as it is written
+ * and played as soon as its audio arrives, one after the other, while the
+ * next ones are being made. The level of what plays moves the assistant's
+ * mouth (VO-23). Nothing is kept: each sentence's audio is dropped once played.
+ *
+ * @module voice/speaker
+ */
+/** The path of the syntheses on ai-agents. */
+export const SYNTHESES_PATH = '/api/ai-agents/v1/speech/syntheses';
+/** Plays an answer's sentences, said by the speech service. */
+export class ServerSpeaker {
+    options;
+    context;
+    analyser;
+    buffer;
+    queue = [];
+    playing;
+    running = false;
+    generation = 0;
+    state = { speaking: false };
+    constructor(options) {
+        this.options = options;
+    }
+    /** Change the voice, the language or the token for the sentences to come. */
+    update(options) {
+        this.options = { ...this.options, ...options };
+    }
+    /**
+     * Must be called from a gesture once (a press, a click): a browser plays
+     * nothing a page did not start from one.
+     */
+    unlock() {
+        this.audio();
+        void this.context?.resume();
+    }
+    audio() {
+        if (!this.context) {
+            this.context = new AudioContext();
+            this.analyser = this.context.createAnalyser();
+            this.analyser.fftSize = 512;
+            this.analyser.connect(this.context.destination);
+            this.buffer = new Float32Array(this.analyser.fftSize);
+        }
+        return this.context;
+    }
+    /** How loud what plays is, between 0 and 1: what moves the mouth. */
+    level() {
+        if (!this.analyser || !this.buffer || !this.state.speaking) {
+            return 0;
+        }
+        this.analyser.getFloatTimeDomainData(this.buffer);
+        let sum = 0;
+        for (const sample of this.buffer) {
+            sum += sample * sample;
+        }
+        return Math.min(1, Math.sqrt(sum / this.buffer.length) * 6);
+    }
+    set(state) {
+        this.state = { ...this.state, ...state };
+        this.options.onState?.(this.state);
+    }
+    /** Say a sentence after those already given. */
+    say(text) {
+        const sentence = text.trim();
+        if (!sentence) {
+            return;
+        }
+        this.queue.push({
+            text: sentence,
+            audio: this.fetch(sentence),
+            asked: performance.now(),
+        });
+        if (!this.running) {
+            void this.run(this.generation);
+        }
+    }
+    async fetch(text) {
+        const { speechUrl, voice, language, token, fetcher = fetch } = this.options;
+        try {
+            const answered = await fetcher(`${speechUrl.replace(/\/+$/, '')}${SYNTHESES_PATH}`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    ...(token ? { Authorization: `Bearer ${token}` } : {}),
+                },
+                body: JSON.stringify({ text, voice, language, format: 'opus' }),
+            });
+            if (!answered.ok) {
+                const body = await answered.json().catch(() => ({}));
+                return typeof body?.detail === 'string'
+                    ? body.detail
+                    : `The answer could not be spoken (${answered.status}).`;
+            }
+            return await answered.arrayBuffer();
+        }
+        catch {
+            return 'Datalayer’s speech service could not be reached: the answer is still written here.';
+        }
+    }
+    async run(generation) {
+        this.running = true;
+        try {
+            while (this.queue.length && generation === this.generation) {
+                const next = this.queue.shift();
+                const audio = await next.audio;
+                if (generation !== this.generation) {
+                    return;
+                }
+                if (typeof audio === 'string') {
+                    // Refused: say why once, and drop what was waiting.
+                    this.queue = [];
+                    this.set({ speaking: false, sentence: undefined, refused: audio });
+                    return;
+                }
+                const context = this.audio();
+                const decoded = await context.decodeAudioData(audio);
+                if (generation !== this.generation) {
+                    return;
+                }
+                await new Promise(resolve => {
+                    const source = context.createBufferSource();
+                    source.buffer = decoded;
+                    source.connect(this.analyser);
+                    source.onended = () => resolve();
+                    this.playing = source;
+                    this.set({ speaking: true, sentence: next.text, refused: undefined });
+                    this.options.onFirstSound?.(Math.round(performance.now() - next.asked));
+                    source.start();
+                });
+                this.playing = undefined;
+            }
+        }
+        finally {
+            if (generation === this.generation) {
+                this.running = false;
+                if (this.state.speaking) {
+                    this.set({ speaking: false, sentence: undefined });
+                }
+            }
+        }
+    }
+    /** Stop at once: what plays and what waits (barge-in, `Esc`; VO-13). */
+    stop() {
+        this.generation += 1;
+        this.queue = [];
+        this.running = false;
+        try {
+            this.playing?.stop();
+        }
+        catch {
+            // Already ended.
+        }
+        this.playing = undefined;
+        this.set({ speaking: false, sentence: undefined });
+    }
+    /** Stop, and let the audio context go. */
+    close() {
+        this.stop();
+        void this.context?.close();
+        this.context = undefined;
+    }
+}

package/lib/voice/types.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Voice in the page (VOICE.md §6): what the chat needs to listen and speak.
+ *
+ * The packages that hear — transformers.js for Moonshine and Whisper, and
+ * vad-web for Silero VAD — are given by the host page (`VoiceEngines`), each
+ * loaded only once the person turns voice on: nothing of them is in a page
+ * that does not ask, and the library names no package it does not need.
+ * Their shapes are written here as far as voice uses them, against the
+ * packages' published types (`@huggingface/transformers` 4.3.0,
+ * `@ricky0123/vad-web` 0.0.31).
+ *
+ * @module voice/types
+ */
+/** What transformers.js's `env` holds, as far as voice sets it. */
+export interface TransformersEnv {
+    allowLocalModels: boolean;
+    allowRemoteModels: boolean;
+    remoteHost: string;
+    remotePathTemplate: string;
+    useBrowserCache: boolean;
+    fetch: (input: string | URL, init?: any) => Promise<any>;
+    backends: {
+        onnx: {
+            versions?: Record<string, string | undefined>;
+            wasm?: {
+                wasmPaths?: unknown;
+                proxy?: boolean;
+            };
+        };
+    };
+}
+/** A speech-recognition pipeline: audio at 16 kHz in, words out. */
+export type SpeechRecognizer = (audio: Float32Array, options?: Record<string, unknown>) => Promise<{
+    text: string;
+} | Array<{
+    text: string;
+}>>;
+/** The precisions a speech model of the catalogue is published in. */
+export type SpeechDtype = 'fp32' | 'fp16' | 'q8' | 'int8' | 'uint8' | 'q4';
+/** The part of `@huggingface/transformers` voice uses. */
+export interface TransformersModule {
+    env: TransformersEnv;
+    pipeline: (task: 'automatic-speech-recognition', model: string, options?: {
+        dtype?: SpeechDtype;
+        device?: 'wasm' | 'webgpu';
+        progress_callback?: (info: any) => void;
+    }) => Promise<unknown>;
+}
+/** A speech segment Silero VAD found, in milliseconds. */
+export interface VadSegment {
+    audio: Float32Array;
+    start: number;
+    end: number;
+}
+/** The part of `@ricky0123/vad-web` voice uses. */
+export interface VadModule {
+    NonRealTimeVAD: {
+        new: (options: {
+            modelURL: string;
+            modelFetcher: (path: string) => Promise<ArrayBuffer>;
+            ortConfig?: (ort: any) => void;
+            positiveSpeechThreshold?: number;
+            negativeSpeechThreshold?: number;
+            minSpeechMs?: number;
+            preSpeechPadMs?: number;
+            redemptionMs?: number;
+        }) => Promise<{
+            run: (audio: Float32Array, sampleRate: number) => AsyncIterable<VadSegment>;
+        }>;
+    };
+}
+/** How the host page loads the packages that hear, once voice is turned on. */
+export interface VoiceEngines {
+    transformers: () => Promise<TransformersModule>;
+    vad: () => Promise<VadModule>;
+}
+/** How a message was heard: what rides on it as `metadata` (VO-27). */
+export interface SpokenMetadata {
+    input: 'voice';
+    language: string;
+    engine: string;
+    where: 'device' | 'server';
+}
+/** What the person said, once heard. */
+export interface Transcript {
+    text: string;
+    metadata: SpokenMetadata;
+    /** Seconds of speech kept by the VAD. */
+    seconds: number;
+    /** How long hearing took, from release to words, in ms. */
+    ms: number;
+}

package/lib/voice/types.js

@@ -0,0 +1,5 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+export {};

package/lib/voice/useSpokenAnswers.d.ts

@@ -0,0 +1,12 @@
+/** What the speaker is given. */
+export interface SentenceSink {
+    say: (sentence: string) => void;
+    stop: () => void;
+}
+/** An item's text, when it is the agent's own answer and not a tool call. */
+export declare function answerText(item: unknown): string | undefined;
+/**
+ * Speaks the newest answer of `items` while `on`; `writing` says whether the
+ * agent is still writing it — once it stops, what is left is said.
+ */
+export declare function useSpokenAnswers(items: readonly unknown[], writing: boolean, on: boolean, sink: SentenceSink | undefined): void;

package/lib/voice/useSpokenAnswers.js

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * The answers of a conversation, heard as they are written (VOICE.md VO-20,
+ * VO-21): the newest answer of the agent is cut into sentences as it grows,
+ * and each is given to the speaker as soon as it is complete. The answers
+ * already there when voice was turned on are not read.
+ *
+ * @module voice/useSpokenAnswers
+ */
+import { useEffect, useRef } from 'react';
+import { SentenceCutter } from './sentences';
+/** An item's text, when it is the agent's own answer and not a tool call. */
+export function answerText(item) {
+    const message = item;
+    if (!message ||
+        message.role !== 'assistant' ||
+        typeof message.toolName === 'string') {
+        return undefined;
+    }
+    if (typeof message.content === 'string') {
+        return message.content;
+    }
+    if (Array.isArray(message.content)) {
+        return message.content
+            .map(part => part &&
+            typeof part === 'object' &&
+            part.type === 'text'
+            ? String(part.text ?? '')
+            : '')
+            .join(' ');
+    }
+    return undefined;
+}
+/**
+ * Speaks the newest answer of `items` while `on`; `writing` says whether the
+ * agent is still writing it — once it stops, what is left is said.
+ */
+export function useSpokenAnswers(items, writing, on, sink) {
+    const heard = useRef(new Set());
+    const current = useRef(undefined);
+    const started = useRef(false);
+    useEffect(() => {
+        if (!on || !sink) {
+            started.current = false;
+            current.current = undefined;
+            return;
+        }
+        const answers = items
+            .map((item, index) => ({
+            id: String(item?.id ?? index),
+            text: answerText(item),
+        }))
+            .filter(answer => answer.text !== undefined);
+        if (!started.current) {
+            // What was there before voice was on is not read.
+            started.current = true;
+            answers.forEach(answer => heard.current.add(answer.id));
+            return;
+        }
+        const newest = answers[answers.length - 1];
+        if (!newest ||
+            (heard.current.has(newest.id) && current.current?.id !== newest.id)) {
+            return;
+        }
+        if (current.current?.id !== newest.id) {
+            heard.current.add(newest.id);
+            current.current = {
+                id: newest.id,
+                cutter: new SentenceCutter(),
+                ended: false,
+            };
+        }
+        const reading = current.current;
+        if (reading.ended) {
+            return;
+        }
+        for (const sentence of reading.cutter.feed(newest.text ?? '')) {
+            sink.say(sentence);
+        }
+        if (!writing) {
+            for (const sentence of reading.cutter.end(newest.text ?? '')) {
+                sink.say(sentence);
+            }
+            reading.ended = true;
+        }
+    }, [items, writing, on, sink]);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@datalayer/agent-runtimes",
-    "version": "1.3.63",
+    "version": "1.3.65",
     "type": "module",
     "workspaces": [
         ".",
@@ -69,13 +69,13 @@
         "clean:dist": "rimraf dist",
         "clean:lib": "rimraf lib tsconfig.tsbuildinfo",
         "create:patches": "bash scripts/create-patches.sh",
-        "examples": "run-p server:start server:start:codemode jupyter:start examples:vite",
+        "examples": "run-p server:start server:start:codemode server:start:accounting server:accounting jupyter:start examples:vite",
         "examples2": "run-p server:start server:start:codemode jupyter:start examples2:vite",
         "example-local": "make example-local",
         "examples:fresh": "npm run clean:cache && npm run examples",
         "examples:jupyter": "run-p jupyter:start server:start server:start:codemode examples:vite",
         "examples:nextjs": "npm run dev --workspace=nextjs-notebook-example",
-        "examples:vite": "VITE_DATALAYER_RUNTIMES_URL=${VITE_DATALAYER_RUNTIMES_URL:-https://r1.datalayer.run} VITE_DATALAYER_AGENT_RUNTIMES_URL=${VITE_DATALAYER_AGENT_RUNTIMES_URL:-https://r1.datalayer.run} VITE_APP_TARGET=examples vite",
+        "examples:vite": "VITE_A2A_ACCOUNTING_URL=${VITE_A2A_ACCOUNTING_URL:-http://localhost:8767/api/v1/a2a/agents/accounting} VITE_DATALAYER_RUNTIMES_URL=${VITE_DATALAYER_RUNTIMES_URL:-https://r1.datalayer.run} VITE_DATALAYER_AGENT_RUNTIMES_URL=${VITE_DATALAYER_AGENT_RUNTIMES_URL:-https://r1.datalayer.run} VITE_APP_TARGET=examples vite",
         "examples2:vite": "VITE_DATALAYER_RUNTIMES_URL=${VITE_DATALAYER_RUNTIMES_URL:-https://r1.datalayer.run} VITE_DATALAYER_AGENT_RUNTIMES_URL=${VITE_DATALAYER_AGENT_RUNTIMES_URL:-https://r1.datalayer.run} VITE_APP_TARGET=examples2 vite",
         "format": "prettier --write \"src/**/*.{js,jsx,ts,tsx,css,json,md}\" \"examples/**/*.{js,jsx,ts,tsx,css,json,md,mjs}\"",
         "format:all": "npm run format && npm run format --workspaces --if-present",
@@ -98,6 +98,8 @@
         "server:start": "PYTHONIOENCODING=utf-8 AGENT_RUNTIMES_DEBUG=false AGENT_RUNTIMES_LOG_LEVEL=info DATALAYER_EVALS_MODE=interactive DATALAYER_EVALS_EMIT_LIVE_EVENTS=true python -m agent_runtimes serve --port 8765 --log-level info",
         "server:start:agent-node": "AGENT_NODE_JUPYTER_URL=${AGENT_NODE_JUPYTER_URL:-http://127.0.0.1:8888/api/jupyter-server} AGENT_NODE_JUPYTER_TOKEN=${AGENT_NODE_JUPYTER_TOKEN:-60c1661cc408f978c309d04157af55c9588ff9557c9380e4fb50785750703da6} PYTHONIOENCODING=utf-8 AGENT_RUNTIMES_DEBUG=false AGENT_RUNTIMES_LOG_LEVEL=info DATALAYER_EVALS_MODE=interactive DATALAYER_EVALS_EMIT_LIVE_EVENTS=true AGENT_RUNTIMES_INFERENCE_PROVIDER_OVERRIDE=${AGENT_RUNTIMES_INFERENCE_PROVIDER_OVERRIDE:-datalayer} python -m agent_runtimes serve --node --port 8765 --log-level info",
         "server:start:codemode": "PYTHONIOENCODING=utf-8 AGENT_RUNTIMES_DEBUG=false AGENT_RUNTIMES_LOG_LEVEL=info DATALAYER_EVALS_MODE=interactive DATALAYER_EVALS_EMIT_LIVE_EVENTS=true python -m agent_runtimes serve --port 8766 --log-level info",
+        "server:start:accounting": "PYTHONIOENCODING=utf-8 AGENT_RUNTIMES_DEBUG=false AGENT_RUNTIMES_LOG_LEVEL=info python -m agent_runtimes serve --port 8767 --log-level info",
+        "server:accounting": "python examples/sales-accounting-a2a/serve_accounting.py --url http://127.0.0.1:8767 --wait 300 || echo 'Accounting was not served on 8767: the Agent A2A Team example will say it is not reachable.'",
         "examples:codemode": "run-p server:start server:start:codemode examples:vite",
         "start": "vite",
         "start:acp": "run-p server:start start:acp:vite",
@@ -159,6 +161,7 @@
         "@datalayer/reactor-manager": "^1.0.4",
         "@datalayer/reactor-shell": "^1.0.4",
         "@excalidraw/excalidraw": "^0.18.0",
+        "@huggingface/transformers": "^4.3.0",
         "@jupyter-widgets/base-manager": "^1.0.12",
         "@jupyter-widgets/schema": "^0.5.6",
         "@jupyterlab/apputils": "^4.6.0",
@@ -213,6 +216,7 @@
         "@radix-ui/react-toggle-group": "^1.1.11",
         "@radix-ui/react-tooltip": "^1.2.8",
         "@radix-ui/react-use-controllable-state": "^1.2.2",
+        "@ricky0123/vad-web": "^0.0.31",
         "@rjsf/validator-ajv8": "^5.24.13",
         "@styled-system/css": "^5.1.5",
         "@tailwindcss/vite": "^4.1.13",