@datalayer/agent-runtimes 1.5.26 → 1.5.28

package/lib/voice/index.d.ts

@@ -13,6 +13,7 @@ export * from './sentences';
 export * from './pinned';
 export * from './capture';
 export * from './hearing';
+export * from './serverHearing';
 export * from './speaker';
 export * from './consent';
 export * from './settings';
@@ -64,15 +65,19 @@ export interface ChatVoice {
      * (VO-10, VO-69): 1,500 when unsaid; longer for a person who pauses.
      */
     silenceMs?: number;
-    /** ai-agents, whose speech service says the answers. */
+    /** ai-agents, whose speech service says the answers, and hears where the browser does not. */
     speechUrl?: string;
-    /** The token the speech service is asked with. */
-    token?: string;
     /**
-     * The deployment an embedded application's page speaks for (VO-19): what
-     * the speech service checks an embed token against, and counts its spend on.
+     * Where what is said is heard (§5): `device`, the browser; `server`,
+     * Datalayer's speech service; `auto`, the default, the browser where a
+     * model there hears the language, else the server. The Appspec's
+     * `interface.voice.where`.
      */
-    deployment?: string;
+    where?: 'auto' | 'device' | 'server';
+    /** The person asked for their language to be found by Whisper on the server (VO-14). */
+    detect?: boolean;
+    /** The token the speech service is asked with. */
+    token?: string;
     /** Send what is said at once, rather than put it in the composer. */
     sendWhatISay?: boolean;
     /**

package/lib/voice/index.js

@@ -7,6 +7,7 @@ export * from './sentences';
 export * from './pinned';
 export * from './capture';
 export * from './hearing';
+export * from './serverHearing';
 export * from './speaker';
 export * from './consent';
 export * from './settings';

package/lib/voice/serverHearing.d.ts

@@ -0,0 +1,58 @@
+import type { Hearing, HearingProgress, HeardSoFar } from './hearing';
+import type { Transcript } from './types';
+/** Where the speech service hears a recording, on ai-agents' host. */
+export declare const TRANSCRIPTIONS_PATH = "/api/ai-agents/v1/speech/transcriptions";
+/** The model the speech service hears with (agentspecs' catalogue). */
+export declare const SERVER_HEARING_MODEL = "whisper-large-v3-turbo";
+/** Where a chat's voice hears: in the browser, or on Datalayer's servers. */
+export type HearingPlace = 'device' | 'server';
+/** What decides where a chat's voice hears. */
+export interface HearingAsked {
+    language: string;
+    /** The application's `interface.voice.where`; `auto` when unsaid. */
+    where?: 'auto' | 'device' | 'server';
+    /** The person asked for their language to be detected (VO-14). */
+    detect?: boolean;
+    /** The speech service; none, and everything is heard in the browser. */
+    speechUrl?: string;
+}
+/**
+ * Where a chat's voice hears (§5): in the browser unless the speech service
+ * is named and the person asked for detection, the application says
+ * `server`, or it says `auto` and no model in the browser hears the language.
+ */
+export declare function hearingPlace(voice: HearingAsked): HearingPlace;
+/** Mono samples at 16 kHz as a 16-bit PCM WAV file: what the service reads. */
+export declare function wavOf(samples: Float32Array, rate?: number): ArrayBuffer;
+/** What finds the speech in what was heard: the browser's VAD (`DeviceHearing`). */
+export interface SpeechFinder {
+    readyToFindSpeech(progress?: HearingProgress): Promise<void>;
+    findSpeech(audio: Float32Array): Promise<{
+        speech: Float32Array;
+        silentMs: number;
+    }>;
+}
+export interface ServerHearingOptions {
+    /** ai-agents' origin, where the speech service is. */
+    speechUrl: string;
+    /** The token the conversation speaks with: the person's, a visitor's, an embed's. */
+    token?: string;
+    /** Let Whisper find the language (VO-14), rather than tell it. */
+    detect?: boolean;
+    fetcher?: typeof fetch;
+}
+/**
+ * Hears on Datalayer's servers: the browser's VAD finds the speech and when
+ * it stopped; Whisper, on the speech service, writes it.
+ */
+export declare class ServerHearing implements Hearing {
+    private readonly finder;
+    private readonly options;
+    readonly where: "server";
+    constructor(finder: SpeechFinder, options: ServerHearingOptions);
+    /** Only the VAD loads in the page: the words are written on the server. */
+    ready(_language: string, progress?: HearingProgress): Promise<void>;
+    /** Whether speech was found and how long it has been silent; its words come at the end. */
+    hearSoFar(audio: Float32Array): Promise<HeardSoFar>;
+    hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
+}

package/lib/voice/serverHearing.js

@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * Hearing on Datalayer's servers (VOICE.md VO-34, VO-14, §5): what was said
+ * in one press, kept to its speech by Silero VAD in the page, then sent as a
+ * WAV recording to the speech service's `POST /speech/transcriptions`, where
+ * Whisper large-v3-turbo hears it — in any language it knows, and finding
+ * the language itself when the person asks for that (detection, VO-14). The
+ * sound is held in the request only: the service keeps neither it nor the
+ * words (VO-60).
+ *
+ * Used where the application's voice says `where: server`, where it says
+ * `auto` and no model in the browser hears the language, and where the
+ * person asked for their language to be detected — never when they chose to
+ * keep speech on the device. A clip model: its words come at the end, not
+ * while the person speaks (VO-11).
+ *
+ * @module voice/serverHearing
+ */
+import { transcriberFor } from '../specs/voices';
+import { HEARING_RATE } from './capture';
+/** Where the speech service hears a recording, on ai-agents' host. */
+export const TRANSCRIPTIONS_PATH = '/api/ai-agents/v1/speech/transcriptions';
+/** The model the speech service hears with (agentspecs' catalogue). */
+export const SERVER_HEARING_MODEL = 'whisper-large-v3-turbo';
+/** The shortest speech worth hearing, in samples: a word said alone. */
+const SHORTEST = HEARING_RATE * 0.2;
+/**
+ * Where a chat's voice hears (§5): in the browser unless the speech service
+ * is named and the person asked for detection, the application says
+ * `server`, or it says `auto` and no model in the browser hears the language.
+ */
+export function hearingPlace(voice) {
+    if (!voice.speechUrl) {
+        return 'device';
+    }
+    if (voice.detect || voice.where === 'server') {
+        return 'server';
+    }
+    if (voice.where === 'device') {
+        return 'device';
+    }
+    return transcriberFor(voice.language) ? 'device' : 'server';
+}
+/** Mono samples at 16 kHz as a 16-bit PCM WAV file: what the service reads. */
+export function wavOf(samples, rate = HEARING_RATE) {
+    const bytes = new ArrayBuffer(44 + samples.length * 2);
+    const view = new DataView(bytes);
+    const text = (at, value) => {
+        for (let i = 0; i < value.length; i++) {
+            view.setUint8(at + i, value.charCodeAt(i));
+        }
+    };
+    text(0, 'RIFF');
+    view.setUint32(4, 36 + samples.length * 2, true);
+    text(8, 'WAVE');
+    text(12, 'fmt ');
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true); // PCM
+    view.setUint16(22, 1, true); // mono
+    view.setUint32(24, rate, true);
+    view.setUint32(28, rate * 2, true);
+    view.setUint16(32, 2, true);
+    view.setUint16(34, 16, true);
+    text(36, 'data');
+    view.setUint32(40, samples.length * 2, true);
+    for (let i = 0; i < samples.length; i++) {
+        const sample = Math.max(-1, Math.min(1, samples[i]));
+        view.setInt16(44 + i * 2, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
+    }
+    return bytes;
+}
+/**
+ * Hears on Datalayer's servers: the browser's VAD finds the speech and when
+ * it stopped; Whisper, on the speech service, writes it.
+ */
+export class ServerHearing {
+    finder;
+    options;
+    where = 'server';
+    constructor(finder, options) {
+        this.finder = finder;
+        this.options = options;
+    }
+    /** Only the VAD loads in the page: the words are written on the server. */
+    ready(_language, progress) {
+        return this.finder.readyToFindSpeech(progress);
+    }
+    /** Whether speech was found and how long it has been silent; its words come at the end. */
+    async hearSoFar(audio) {
+        const { speech, silentMs } = await this.finder.findSpeech(audio);
+        return { text: '', spoke: speech.length >= SHORTEST, silentMs };
+    }
+    async hear(audio, language) {
+        const started = performance.now();
+        const { speech } = await this.finder.findSpeech(audio);
+        if (speech.length < SHORTEST) {
+            return undefined;
+        }
+        const { speechUrl, token, detect, fetcher = fetch } = this.options;
+        const query = detect ? '' : `?language=${encodeURIComponent(language)}`;
+        const answered = await fetcher(`${speechUrl.replace(/\/+$/, '')}${TRANSCRIPTIONS_PATH}${query}`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'audio/wav',
+                ...(token ? { Authorization: `Bearer ${token}` } : {}),
+            },
+            body: wavOf(speech),
+        });
+        const body = (await answered.json().catch(() => ({})));
+        if (!answered.ok) {
+            throw new Error(typeof body.detail === 'string' && body.detail
+                ? body.detail
+                : `What you said could not be heard on Datalayer's servers (${answered.status}).`);
+        }
+        const text = typeof body.text === 'string' ? body.text.trim() : '';
+        if (!text) {
+            return undefined;
+        }
+        return {
+            text,
+            metadata: {
+                input: 'voice',
+                language: typeof body.language === 'string' && body.language
+                    ? body.language
+                    : language,
+                engine: SERVER_HEARING_MODEL,
+                where: 'server',
+            },
+            seconds: speech.length / HEARING_RATE,
+            ms: Math.round(performance.now() - started),
+        };
+    }
+}

package/lib/voice/settings.d.ts

@@ -57,6 +57,11 @@ export interface VoiceSettings {
      * speech service, so answers are spoken only if the browser can do it.
      */
     onDeviceOnly?: boolean;
+    /**
+     * Let Whisper find the language they speak (VO-14): what they say is
+     * heard on Datalayer's servers, which detect it, rather than in the browser.
+     */
+    detectLanguage?: boolean;
 }
 /** The settings as they can be used: numbers in range, nothing else trusted. */
 export declare function saneVoiceSettings(settings: VoiceSettings | null | undefined): VoiceSettings;

package/lib/voice/settings.js

@@ -40,6 +40,9 @@ export function saneVoiceSettings(settings) {
     if (typeof kept.onDeviceOnly === 'boolean') {
         sane.onDeviceOnly = kept.onDeviceOnly;
     }
+    if (typeof kept.detectLanguage === 'boolean') {
+        sane.detectLanguage = kept.detectLanguage;
+    }
     return sane;
 }
 /** What was kept, or nothing. Storage may be refused, or hold nonsense. */
@@ -174,6 +177,9 @@ export function voiceForPerson(voice, person = {}, browser = browserLanguage())
     if (chosen.microphoneId) {
         next.microphoneId = chosen.microphoneId;
     }
+    if (chosen.detectLanguage) {
+        next.detect = true;
+    }
     if (chosen.onDeviceOnly) {
         // Nothing is asked of the speech service: no token, no origin. Answers
         // are said by that service only (decision 1: no voice made in the
@@ -181,6 +187,8 @@ export function voiceForPerson(voice, person = {}, browser = browserLanguage())
         next.speechUrl = undefined;
         next.token = undefined;
         next.output = 'off';
+        next.where = 'device';
+        next.detect = false;
     }
     return next;
 }

package/lib/voice/speaker.d.ts

@@ -24,10 +24,11 @@ export interface SpeakerOptions {
     /** The voice of the catalogue. */
     voice: string;
     language: string;
-    /** The person's token, an application's principal's, a visitor's or an embed's. */
+    /**
+     * The person's token, an application's principal's, a visitor's or an
+     * embed's — whose deployment is the one it names (VO-19, decision 16).
+     */
     token?: string;
-    /** The deployment an embed's page speaks for (VO-19): sent with each sentence. */
-    deployment?: string;
     /**
      * How fast it speaks, as a multiplier of the voice's own pace (VO-25,
      * VO-30). Applied here rather than asked of the service: `POST

package/lib/voice/speaker.js

@@ -114,7 +114,7 @@ export class ServerSpeaker {
         }
     }
     async fetch(text) {
-        const { speechUrl, voice, language, token, deployment, fetcher = fetch, } = this.options;
+        const { speechUrl, voice, language, token, fetcher = fetch } = this.options;
         try {
             const answered = await fetcher(`${speechUrl.replace(/\/+$/, '')}${SYNTHESES_PATH}`, {
                 method: 'POST',
@@ -127,7 +127,6 @@ export class ServerSpeaker {
                     voice,
                     language,
                     format: 'opus',
-                    ...(deployment ? { deployment } : {}),
                 }),
             });
             if (!answered.ok) {

package/lib/voice/useVoiceSpeaker.js

@@ -27,7 +27,6 @@ export function useVoiceSpeaker(voice, on = true) {
             voice: voice.voice,
             language: voice.language,
             token: voice.token,
-            deployment: voice.deployment,
             rate: voice.rate,
             volume: voice.volume,
             onState: setState,
@@ -40,7 +39,6 @@ export function useVoiceSpeaker(voice, on = true) {
             voice: voice?.voice,
             language: voice?.language,
             token: voice?.token,
-            deployment: voice?.deployment,
             rate: voice?.rate,
             volume: voice?.volume,
         });
@@ -49,7 +47,6 @@ export function useVoiceSpeaker(voice, on = true) {
         voice?.voice,
         voice?.language,
         voice?.token,
-        voice?.deployment,
         voice?.rate,
         voice?.volume,
     ]);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@datalayer/agent-runtimes",
-    "version": "1.5.26",
+    "version": "1.5.28",
     "type": "module",
     "workspaces": [
         ".",
@@ -158,7 +158,7 @@
         "@datalayer/jupyter-react": "^2.0.20",
         "@datalayer/jupyter-server-nbmodel": "^0.2.8",
         "@datalayer/lexical-loro": "^1.0.6",
-        "@datalayer/primer-addons": "^1.0.39",
+        "@datalayer/primer-addons": "^1.0.40",
         "@datalayer/primer-rjsf": "^1.0.3",
         "@datalayer/personal-agent-protocol": "^0.24.0",
         "@datalayer/reactor": "^1.3.0",