@datalayer/agent-runtimes 1.5.60 → 1.5.62

package/lib/voice/index.d.ts

@@ -10,6 +10,7 @@
 import type { VoiceEngines } from './types';
 export * from './types';
 export * from './sentences';
+export * from './keptSound';
 export * from './pinned';
 export * from './capture';
 export * from './hearing';
@@ -56,7 +57,7 @@ export interface ChatVoice {
      * `off`: none.
      */
     input: 'off' | 'push_to_talk' | 'hands_free';
-    /** The silence after speech that ends a hands-free turn, in ms (VO-12): 700 when unsaid. */
+    /** The silence after speech that ends a hands-free turn, in ms (VO-12): 500 when unsaid. */
     turnSilenceMs?: number;
     /**
      * When answers are heard: `always`, as they are written; `on_request`, a
@@ -87,6 +88,12 @@ export interface ChatVoice {
     detect?: boolean;
     /** The token the speech service is asked with. */
     token?: string;
+    /**
+     * The days the application keeps what is said as sound (VO-63), when its
+     * record keeps `audio`: asked in the microphone's first sentence, each
+     * turn's recording sent with its words. Never for a visitor.
+     */
+    keepsAudio?: number;
     /** Send what is said at once, rather than put it in the composer. */
     sendWhatISay?: boolean;
     /**

package/lib/voice/index.js

@@ -4,6 +4,7 @@
  */
 export * from './types';
 export * from './sentences';
+export * from './keptSound';
 export * from './pinned';
 export * from './capture';
 export * from './hearing';

package/lib/voice/keptSound.d.ts

@@ -0,0 +1,15 @@
+import type { SpokenSound } from './types';
+/** The longest a sound is kept, whatever the record keeps (decision 10). */
+export declare const AUDIO_KEPT_DAYS_AT_MOST = 30;
+/**
+ * How many days an application keeps what is said as sound: `undefined`
+ * when its record does not keep `audio`, else its record's days, 30 at most.
+ */
+export declare function keptAudioDays(record?: {
+    include?: readonly string[];
+    retentionDays?: number;
+}): number | undefined;
+/** The sentence added to the microphone's first one when the sound is kept (VO-61). */
+export declare const keptAudioSentence: (days: number) => string;
+/** A turn's samples (mono, 16 kHz, -1 to 1) as a WAV, base64, to keep. */
+export declare function soundOf(audio: Float32Array): SpokenSound;

package/lib/voice/keptSound.js

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * What the person said, kept as sound (VO-63): an application whose record
+ * keeps `audio` (`record.include`), and a person who agreed in the
+ * microphone's first sentence, send each turn's recording with its words —
+ * `forwardedProps.voice.sound`, a 16-bit mono WAV at 16 kHz — and its
+ * runtime gives it to ai-agents to keep, 30 days at most, deleted with the
+ * conversation. Never for a visitor without an account.
+ *
+ * @module voice/keptSound
+ */
+import { HEARING_RATE } from './capture';
+/** The longest a sound is kept, whatever the record keeps (decision 10). */
+export const AUDIO_KEPT_DAYS_AT_MOST = 30;
+/** The most of a turn kept: a minute. */
+const KEPT_SECONDS_AT_MOST = 60;
+/**
+ * How many days an application keeps what is said as sound: `undefined`
+ * when its record does not keep `audio`, else its record's days, 30 at most.
+ */
+export function keptAudioDays(record) {
+    if (!record?.include?.includes('audio')) {
+        return undefined;
+    }
+    const days = record.retentionDays || AUDIO_KEPT_DAYS_AT_MOST;
+    return Math.max(1, Math.min(days, AUDIO_KEPT_DAYS_AT_MOST));
+}
+/** The sentence added to the microphone's first one when the sound is kept (VO-61). */
+export const keptAudioSentence = (days) => `This application also keeps what you say as sound, with this conversation, for ${days} ${days === 1 ? 'day' : 'days'} at most: its owner can listen to it, and it is deleted with the conversation. Not now, and you type instead.`;
+/** A turn's samples (mono, 16 kHz, -1 to 1) as a WAV, base64, to keep. */
+export function soundOf(audio) {
+    const samples = audio.subarray(0, HEARING_RATE * KEPT_SECONDS_AT_MOST);
+    const bytes = new Uint8Array(44 + samples.length * 2);
+    const view = new DataView(bytes.buffer);
+    const text = (at, value) => {
+        for (let k = 0; k < value.length; k++) {
+            view.setUint8(at + k, value.charCodeAt(k));
+        }
+    };
+    text(0, 'RIFF');
+    view.setUint32(4, 36 + samples.length * 2, true);
+    text(8, 'WAVE');
+    text(12, 'fmt ');
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true);
+    view.setUint16(22, 1, true);
+    view.setUint32(24, HEARING_RATE, true);
+    view.setUint32(28, HEARING_RATE * 2, true);
+    view.setUint16(32, 2, true);
+    view.setUint16(34, 16, true);
+    text(36, 'data');
+    view.setUint32(40, samples.length * 2, true);
+    for (let k = 0; k < samples.length; k++) {
+        const sample = Math.max(-1, Math.min(1, samples[k]));
+        view.setInt16(44 + k * 2, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
+    }
+    let binary = '';
+    for (let k = 0; k < bytes.length; k += 0x8000) {
+        binary += String.fromCharCode(...bytes.subarray(k, k + 0x8000));
+    }
+    return {
+        data: btoa(binary),
+        type: 'audio/wav',
+        seconds: Math.round((samples.length / HEARING_RATE) * 100) / 100,
+    };
+}

package/lib/voice/serverHearing.d.ts

@@ -94,6 +94,8 @@ export declare class ServerHearing implements Hearing {
      * Whether speech was found and how long it has been silent, and the words
      * the speech service heard so far: the sound goes to its stream as it comes.
      */
+    spokeSoFar(audio: Float32Array, shortestSeconds?: number): Promise<boolean>;
+    quietSoFar(audio: Float32Array): Promise<HeardSoFar>;
     hearSoFar(audio: Float32Array, language?: string): Promise<HeardSoFar>;
     hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
 }
@@ -121,5 +123,7 @@ export declare class AutoHearing implements Hearing {
     private get current();
     ready(language: string, progress?: HearingProgress): Promise<void>;
     hearSoFar(audio: Float32Array, language: string): Promise<HeardSoFar>;
+    spokeSoFar(audio: Float32Array, shortestSeconds?: number): Promise<boolean>;
+    quietSoFar(audio: Float32Array): Promise<HeardSoFar>;
     hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
 }

package/lib/voice/serverHearing.js

@@ -315,6 +315,14 @@ export class ServerHearing {
      * Whether speech was found and how long it has been silent, and the words
      * the speech service heard so far: the sound goes to its stream as it comes.
      */
+    async spokeSoFar(audio, shortestSeconds = 0.2) {
+        const { speech } = await this.finder.findSpeech(audio);
+        return speech.length >= HEARING_RATE * shortestSeconds;
+    }
+    async quietSoFar(audio) {
+        const { speech, silentMs } = await this.finder.findSpeech(audio);
+        return { text: '', spoke: speech.length >= SHORTEST, silentMs };
+    }
     async hearSoFar(audio, language = '') {
         const { speech, silentMs } = await this.finder.findSpeech(audio);
         const spoke = speech.length >= SHORTEST;
@@ -450,6 +458,18 @@ export class AutoHearing {
     hearSoFar(audio, language) {
         return this.current.hearSoFar(audio, language);
     }
+    async spokeSoFar(audio, shortestSeconds) {
+        const current = this.current;
+        return current.spokeSoFar
+            ? current.spokeSoFar(audio, shortestSeconds)
+            : (await current.hearSoFar(audio, '')).spoke;
+    }
+    quietSoFar(audio) {
+        const current = this.current;
+        return current.quietSoFar
+            ? current.quietSoFar(audio)
+            : current.hearSoFar(audio, '');
+    }
     async hear(audio, language) {
         if (this.place === 'device') {
             return this.device.hear(audio, language);

package/lib/voice/speaker.d.ts

@@ -50,6 +50,21 @@ export interface SpeakerOptions {
     onState?: (state: SpeakerState) => void;
     /** Called with how long a sentence took from asked to first sound, in ms. */
     onFirstSound?: (ms: number) => void;
+    /**
+     * Say a long first sentence's first clause on its own (VO-52): the service
+     * makes a sentence whole, in about the time its length takes, so the first
+     * sound comes sooner when the first thing asked is shorter. Off for a
+     * caller that follows its sentences one by one (a deck's narrator).
+     */
+    firstClauseFirst?: boolean;
+    /**
+     * Ask each sentence streamed (VO-52): the speech service says it piece by
+     * piece where it pauses and sends each piece's 16-bit PCM as soon as it is
+     * said, and the speaker plays the pieces as they come — the first sound
+     * after the first piece, not the whole sentence. A service that answers a
+     * whole sound instead is played as before.
+     */
+    streamed?: boolean;
     fetcher?: typeof fetch;
 }
 /**
@@ -60,6 +75,14 @@ export interface SpeakerOptions {
  * 14 October 2026).
  */
 export declare const SENTENCES_AHEAD = 2;
+/** A first sentence longer than this is said clause first (`firstClauseFirst`). */
+export declare const FIRST_CLAUSE_SPLIT_CHARS = 60;
+/**
+ * A sentence cut after its first clause — a comma, semicolon, colon or dash
+ * followed by a space — when it is long and neither part is too short (VO-52);
+ * else the sentence alone. Never cut inside a number (`1,000`).
+ */
+export declare function firstClause(sentence: string): string[];
 /** Plays an answer's sentences, said by the speech service. */
 export declare class ServerSpeaker {
     private options;
@@ -69,6 +92,8 @@ export declare class ServerSpeaker {
     private buffer?;
     private queue;
     private playing?;
+    /** The pieces of a streamed sentence scheduled to play, stopped together. */
+    private scheduled;
     private running;
     private generation;
     private state;
@@ -94,6 +119,11 @@ export declare class ServerSpeaker {
     private askAhead;
     private fetch;
     private run;
+    /**
+     * A sentence streamed as 16-bit PCM (VO-52): played in slices as they
+     * come, each scheduled right after the last, until the last has ended.
+     */
+    private playStreamed;
     /** Stop at once: what plays and what waits (barge-in, `Esc`; VO-13). */
     stop(): void;
     /** Stop, and let the audio context go. */

package/lib/voice/speaker.js

@@ -13,6 +13,8 @@
  */
 /** The path of the syntheses on ai-agents. */
 export const SYNTHESES_PATH = '/api/ai-agents/v1/speech/syntheses';
+/** Played as it comes: at least this much at a time, in seconds. */
+const STREAMED_SLICE_SECONDS = 0.25;
 /**
  * How many sentences are asked of the speech service ahead of the one
  * playing: enough that the next is ready when one ends, and under the calls
@@ -21,6 +23,34 @@ export const SYNTHESES_PATH = '/api/ai-agents/v1/speech/syntheses';
  * 14 October 2026).
  */
 export const SENTENCES_AHEAD = 2;
+/** A first sentence longer than this is said clause first (`firstClauseFirst`). */
+export const FIRST_CLAUSE_SPLIT_CHARS = 60;
+/** Neither part of a sentence cut at its first clause is shorter than this. */
+const SHORTEST_CLAUSE_CHARS = 15;
+/**
+ * A sentence cut after its first clause — a comma, semicolon, colon or dash
+ * followed by a space — when it is long and neither part is too short (VO-52);
+ * else the sentence alone. Never cut inside a number (`1,000`).
+ */
+export function firstClause(sentence) {
+    if (sentence.length <= FIRST_CLAUSE_SPLIT_CHARS) {
+        return [sentence];
+    }
+    const boundary = /[,;:\u2013\u2014]\s+/g;
+    let found;
+    while ((found = boundary.exec(sentence)) !== null) {
+        const end = found.index + 1;
+        if (end < SHORTEST_CLAUSE_CHARS) {
+            continue;
+        }
+        const rest = sentence.slice(found.index + found[0].length);
+        if (rest.length < SHORTEST_CLAUSE_CHARS) {
+            break;
+        }
+        return [sentence.slice(0, end), rest];
+    }
+    return [sentence];
+}
 /** Plays an answer's sentences, said by the speech service. */
 export class ServerSpeaker {
     options;
@@ -30,6 +60,8 @@ export class ServerSpeaker {
     buffer;
     queue = [];
     playing;
+    /** The pieces of a streamed sentence scheduled to play, stopped together. */
+    scheduled = [];
     running = false;
     generation = 0;
     state = { speaking: false };
@@ -99,7 +131,15 @@ export class ServerSpeaker {
         if (!sentence) {
             return;
         }
-        this.queue.push({ text: sentence, asked: performance.now() });
+        // Nothing playing or waiting: this is the first thing heard (VO-52).
+        const first = this.options.firstClauseFirst &&
+            !this.options.streamed &&
+            !this.queue.length &&
+            !this.playing &&
+            !this.running;
+        for (const part of first ? firstClause(sentence) : [sentence]) {
+            this.queue.push({ text: part, asked: performance.now() });
+        }
         if (!this.state.speaking) {
             // Something new to say: the last refusal is no longer the news.
             this.set({ speaking: true, refused: undefined });
@@ -130,7 +170,9 @@ export class ServerSpeaker {
                     text,
                     voice,
                     language,
-                    format: 'opus',
+                    ...(this.options.streamed
+                        ? { format: 'pcm', stream: true }
+                        : { format: 'opus' }),
                 }),
             });
             if (!answered.ok) {
@@ -139,7 +181,18 @@ export class ServerSpeaker {
                     ? body.detail
                     : `The answer could not be spoken (${answered.status}).`;
             }
-            return await answered.arrayBuffer();
+            const kind = answered.headers.get('content-type') ?? '';
+            if (kind.toLowerCase().startsWith('audio/l16')) {
+                const rate = Number(answered.headers.get('x-speech-rate')) ||
+                    Number(/rate=(\d+)/i.exec(kind)?.[1]) ||
+                    24000;
+                return {
+                    kind: 'pcm',
+                    rate,
+                    body: answered.body ?? (await answered.arrayBuffer()),
+                };
+            }
+            return { kind: 'whole', audio: await answered.arrayBuffer() };
         }
         catch {
             return 'Datalayer’s speech service could not be reached: the answer is still written here.';
@@ -165,7 +218,11 @@ export class ServerSpeaker {
                 this.queue.shift();
                 this.askAhead();
                 const context = this.audio();
-                const decoded = await context.decodeAudioData(audio);
+                if (audio.kind === 'pcm') {
+                    await this.playStreamed(generation, next, audio);
+                    continue;
+                }
+                const decoded = await context.decodeAudioData(audio.audio);
                 if (generation !== this.generation) {
                     return;
                 }
@@ -195,17 +252,103 @@ export class ServerSpeaker {
             }
         }
     }
+    /**
+     * A sentence streamed as 16-bit PCM (VO-52): played in slices as they
+     * come, each scheduled right after the last, until the last has ended.
+     */
+    async playStreamed(generation, next, answer) {
+        const context = this.audio();
+        const speed = this.speed();
+        if (this.gain) {
+            this.gain.gain.value = this.loudness();
+        }
+        let at = 0;
+        let first = true;
+        let pending = new Uint8Array(0);
+        let last = Promise.resolve();
+        const slice = Math.round(answer.rate * STREAMED_SLICE_SECONDS) * 2;
+        const play = (bytes) => {
+            const samples = new Float32Array(bytes.length / 2);
+            const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length);
+            for (let k = 0; k < samples.length; k++) {
+                samples[k] = view.getInt16(k * 2, true) / 32768;
+            }
+            const buffer = context.createBuffer(1, samples.length, answer.rate);
+            buffer.getChannelData(0).set(samples);
+            const source = context.createBufferSource();
+            source.buffer = buffer;
+            source.playbackRate.value = speed;
+            source.connect(this.analyser);
+            at = Math.max(at, context.currentTime);
+            last = new Promise(resolve => {
+                source.onended = () => {
+                    this.scheduled = this.scheduled.filter(one => one !== source);
+                    resolve();
+                };
+            });
+            this.scheduled.push(source);
+            this.playing = source;
+            if (first) {
+                first = false;
+                this.set({ speaking: true, sentence: next.text, refused: undefined });
+                this.options.onFirstSound?.(Math.round(performance.now() - next.asked));
+            }
+            source.start(at);
+            at += buffer.duration / speed;
+        };
+        const take = (chunk, done) => {
+            const joined = new Uint8Array(pending.length + chunk.length);
+            joined.set(pending);
+            joined.set(chunk, pending.length);
+            pending = joined;
+            while (pending.length >= slice || (done && pending.length >= 2)) {
+                const size = Math.min(pending.length, Math.max(slice, 2));
+                const even = size - (size % 2);
+                play(pending.subarray(0, even));
+                pending = pending.slice(even);
+                if (done && pending.length < 2) {
+                    break;
+                }
+            }
+        };
+        if (answer.body instanceof ArrayBuffer) {
+            take(new Uint8Array(answer.body), true);
+        }
+        else {
+            const reader = answer.body.getReader();
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (generation !== this.generation) {
+                    void reader.cancel().catch(() => undefined);
+                    return;
+                }
+                if (done) {
+                    take(new Uint8Array(0), true);
+                    break;
+                }
+                take(value, false);
+            }
+        }
+        await last;
+        this.playing = undefined;
+    }
     /** Stop at once: what plays and what waits (barge-in, `Esc`; VO-13). */
     stop() {
         this.generation += 1;
         this.queue = [];
         this.running = false;
-        try {
-            this.playing?.stop();
-        }
-        catch {
-            // Already ended.
+        for (const source of [
+            ...this.scheduled,
+            ...(this.playing ? [this.playing] : []),
+        ]) {
+            try {
+                source.stop();
+            }
+            catch {
+                // Already ended.
+            }
         }
+        this.scheduled = [];
         this.playing = undefined;
         this.set({ speaking: false, sentence: undefined });
     }

package/lib/voice/types.d.ts

@@ -80,6 +80,18 @@ export interface SpokenMetadata {
     language: string;
     engine: string;
     where: 'device' | 'server';
+    /**
+     * The turn's sound, sent to be kept with its words when the application's
+     * record keeps `audio` and the person agreed (VO-63); never on the
+     * conversation's own message.
+     */
+    sound?: SpokenSound;
+}
+/** A turn's recording, to keep (VO-63): a WAV, base64. */
+export interface SpokenSound {
+    data: string;
+    type: 'audio/wav';
+    seconds: number;
 }
 /** What the person said, once heard. */
 export interface Transcript {

package/lib/voice/useVoiceSpeaker.js

@@ -30,6 +30,10 @@ export function useVoiceSpeaker(voice, on = true) {
             rate: voice.rate,
             volume: voice.volume,
             onState: setState,
+            // The chat's answers: streamed, played as they are said (VO-52);
+            // from a service that does not stream, the first clause first.
+            streamed: true,
+            firstClauseFirst: true,
         })
         : undefined, 
     // eslint-disable-next-line react-hooks/exhaustive-deps

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@datalayer/agent-runtimes",
-    "version": "1.5.60",
+    "version": "1.5.62",
     "type": "module",
     "workspaces": [
         ".",