@datalayer/agent-runtimes 1.5.59 → 1.5.61

package/lib/voice/chatVoice.js

@@ -41,7 +41,7 @@ export const DATALAYER_VOICE_ENGINES = {
  * - **Speaking**: its answers said by Datalayer's speech service, on request
  *   or always (VO-20, VO-21); refused here when its host says no service.
  */
-export function chatVoiceOf(spec, place) {
+export function chatVoiceOf(spec, place, keepsAudio) {
     if (!spec?.enabled || (spec.input === 'off' && spec.output === 'off')) {
         return undefined;
     }
@@ -61,6 +61,8 @@ export function chatVoiceOf(spec, place) {
         token: place.token,
         consentKey: place.consentKey,
         where: spec.where ?? 'auto',
+        // Kept as sound only when it listens, as its record says (VO-63).
+        ...(keepsAudio && spec.input !== 'off' ? { keepsAudio } : {}),
     };
 }
 /**

package/lib/voice/consent.d.ts

@@ -10,5 +10,9 @@ export declare const CONSENT_SENTENCE = "What you say is transcribed on this dev
  * a yes to one is not a yes to the other.
  */
 export declare const SERVER_CONSENT_SENTENCE = "What you say is sent to Datalayer\u2019s servers to be transcribed: the sound and the words are not kept there. The words go in the message box, for you to send.";
-/** Whose consent is remembered for hearing in a place: the application's, apart for the server. */
-export declare function consentKeyFor(app: string, where: 'device' | 'server'): string;
+/**
+ * Whose consent is remembered for hearing in a place: the application's,
+ * apart for the server, and apart again when what is said is kept as sound
+ * (VO-63) — a yes to being heard is not a yes to being kept.
+ */
+export declare function consentKeyFor(app: string, where: 'device' | 'server', keepsAudio?: number): string;

package/lib/voice/consent.js

@@ -38,7 +38,12 @@ export const CONSENT_SENTENCE = 'What you say is transcribed on this device: the
  * a yes to one is not a yes to the other.
  */
 export const SERVER_CONSENT_SENTENCE = 'What you say is sent to Datalayer’s servers to be transcribed: the sound and the words are not kept there. The words go in the message box, for you to send.';
-/** Whose consent is remembered for hearing in a place: the application's, apart for the server. */
-export function consentKeyFor(app, where) {
-    return where === 'server' ? `${app}:server` : app;
+/**
+ * Whose consent is remembered for hearing in a place: the application's,
+ * apart for the server, and apart again when what is said is kept as sound
+ * (VO-63) — a yes to being heard is not a yes to being kept.
+ */
+export function consentKeyFor(app, where, keepsAudio) {
+    const heard = where === 'server' ? `${app}:server` : app;
+    return keepsAudio ? `${heard}:kept` : heard;
 }

package/lib/voice/handsFree.d.ts

@@ -5,7 +5,7 @@
  * microphone and the clock, this decides.
  *
  * - **Listening**: the microphone is open; the turn ends after
- *   `turnSilenceMs` of silence once the person has spoken (700 ms unless
+ *   `turnSilenceMs` of silence once the person has spoken (500 ms unless
  *   the application or the person says otherwise), and is sent.
  * - **Thinking**: the turn was sent and the agent is answering, nothing
  *   said yet.
@@ -23,7 +23,7 @@ export type HandsFreeState = 'off' | 'listening' | 'thinking' | 'speaking' | 'pa
 /** The word for each state, as the composer shows it. */
 export declare const HANDS_FREE_WORDS: Record<HandsFreeState, string>;
 /** The silence after speech that ends a turn, in ms, unless said otherwise (VO-12). */
-export declare const TURN_SILENCE_MS = 700;
+export declare const TURN_SILENCE_MS = 500;
 /** Listening with nothing said for this long pauses it: a microphone is not left open on an empty room. */
 export declare const IDLE_PAUSE_MS: number;
 /** What decides the state. */

package/lib/voice/handsFree.js

@@ -11,7 +11,7 @@ export const HANDS_FREE_WORDS = {
     paused: 'Paused',
 };
 /** The silence after speech that ends a turn, in ms, unless said otherwise (VO-12). */
-export const TURN_SILENCE_MS = 700;
+export const TURN_SILENCE_MS = 500;
 /** Listening with nothing said for this long pauses it: a microphone is not left open on an empty room. */
 export const IDLE_PAUSE_MS = 5 * 60_000;
 /** The state for what is known: speaking wins over thinking, pausing over both. */

package/lib/voice/hearing.d.ts

@@ -35,6 +35,17 @@ export interface Hearing {
     hearSoFar(audio: Float32Array, language: string): Promise<HeardSoFar>;
     /** What was said in a press, or `undefined` when nothing was. */
     hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
+    /**
+     * Whether there is speech in some sound, found by the VAD alone — nothing
+     * transcribed, nothing sent (VO-52): hands-free's quick look for barge-in.
+     */
+    spokeSoFar?(audio: Float32Array, shortestSeconds?: number): Promise<boolean>;
+    /**
+     * Whether there is speech in some sound and how long it has been quiet
+     * since, by the VAD alone — no words (VO-52): hands-free's quick look for
+     * the end of a turn once the person has spoken.
+     */
+    quietSoFar?(audio: Float32Array): Promise<HeardSoFar>;
 }
 /** Joins the speech a VAD found, in order. */
 export declare function joinSegments(segments: Float32Array[]): Float32Array;
@@ -73,6 +84,8 @@ export declare class DeviceHearing implements Hearing {
      * words, for the composer as they come, and how long it has been silent
      * since the speech ended — what stops a click to talk (VO-10).
      */
+    spokeSoFar(audio: Float32Array, shortestSeconds?: number): Promise<boolean>;
+    quietSoFar(audio: Float32Array): Promise<HeardSoFar>;
     hearSoFar(audio: Float32Array, language: string): Promise<HeardSoFar>;
     /**
      * What was said in a press, or `undefined` when nothing was: its speech

package/lib/voice/hearing.js

@@ -179,6 +179,14 @@ export class DeviceHearing {
      * words, for the composer as they come, and how long it has been silent
      * since the speech ended — what stops a click to talk (VO-10).
      */
+    async spokeSoFar(audio, shortestSeconds = 0.2) {
+        const { speech } = await this.findSpeech(audio);
+        return speech.length >= HEARING_RATE * shortestSeconds;
+    }
+    async quietSoFar(audio) {
+        const { speech, silentMs } = await this.findSpeech(audio);
+        return { text: '', spoke: speech.length >= HEARING_RATE * 0.2, silentMs };
+    }
     async hearSoFar(audio, language) {
         const { speech, silentMs } = await this.findSpeech(audio);
         if (speech.length < HEARING_RATE * 0.2) {

package/lib/voice/index.d.ts

@@ -10,6 +10,7 @@
 import type { VoiceEngines } from './types';
 export * from './types';
 export * from './sentences';
+export * from './keptSound';
 export * from './pinned';
 export * from './capture';
 export * from './hearing';
@@ -56,7 +57,7 @@ export interface ChatVoice {
      * `off`: none.
      */
     input: 'off' | 'push_to_talk' | 'hands_free';
-    /** The silence after speech that ends a hands-free turn, in ms (VO-12): 700 when unsaid. */
+    /** The silence after speech that ends a hands-free turn, in ms (VO-12): 500 when unsaid. */
     turnSilenceMs?: number;
     /**
      * When answers are heard: `always`, as they are written; `on_request`, a
@@ -87,6 +88,12 @@ export interface ChatVoice {
     detect?: boolean;
     /** The token the speech service is asked with. */
     token?: string;
+    /**
+     * The days the application keeps what is said as sound (VO-63), when its
+     * record keeps `audio`: asked in the microphone's first sentence, each
+     * turn's recording sent with its words. Never for a visitor.
+     */
+    keepsAudio?: number;
     /** Send what is said at once, rather than put it in the composer. */
     sendWhatISay?: boolean;
     /**

package/lib/voice/index.js

@@ -4,6 +4,7 @@
  */
 export * from './types';
 export * from './sentences';
+export * from './keptSound';
 export * from './pinned';
 export * from './capture';
 export * from './hearing';

package/lib/voice/keptSound.d.ts

@@ -0,0 +1,15 @@
+import type { SpokenSound } from './types';
+/** The longest a sound is kept, whatever the record keeps (decision 10). */
+export declare const AUDIO_KEPT_DAYS_AT_MOST = 30;
+/**
+ * How many days an application keeps what is said as sound: `undefined`
+ * when its record does not keep `audio`, else its record's days, 30 at most.
+ */
+export declare function keptAudioDays(record?: {
+    include?: readonly string[];
+    retentionDays?: number;
+}): number | undefined;
+/** The sentence added to the microphone's first one when the sound is kept (VO-61). */
+export declare const keptAudioSentence: (days: number) => string;
+/** A turn's samples (mono, 16 kHz, -1 to 1) as a WAV, base64, to keep. */
+export declare function soundOf(audio: Float32Array): SpokenSound;

package/lib/voice/keptSound.js

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * What the person said, kept as sound (VO-63): an application whose record
+ * keeps `audio` (`record.include`), and a person who agreed in the
+ * microphone's first sentence, send each turn's recording with its words —
+ * `forwardedProps.voice.sound`, a 16-bit mono WAV at 16 kHz — and its
+ * runtime gives it to ai-agents to keep, 30 days at most, deleted with the
+ * conversation. Never for a visitor without an account.
+ *
+ * @module voice/keptSound
+ */
+import { HEARING_RATE } from './capture';
+/** The longest a sound is kept, whatever the record keeps (decision 10). */
+export const AUDIO_KEPT_DAYS_AT_MOST = 30;
+/** The most of a turn kept: a minute. */
+const KEPT_SECONDS_AT_MOST = 60;
+/**
+ * How many days an application keeps what is said as sound: `undefined`
+ * when its record does not keep `audio`, else its record's days, 30 at most.
+ */
+export function keptAudioDays(record) {
+    if (!record?.include?.includes('audio')) {
+        return undefined;
+    }
+    const days = record.retentionDays || AUDIO_KEPT_DAYS_AT_MOST;
+    return Math.max(1, Math.min(days, AUDIO_KEPT_DAYS_AT_MOST));
+}
+/** The sentence added to the microphone's first one when the sound is kept (VO-61). */
+export const keptAudioSentence = (days) => `This application also keeps what you say as sound, with this conversation, for ${days} ${days === 1 ? 'day' : 'days'} at most: its owner can listen to it, and it is deleted with the conversation. Not now, and you type instead.`;
+/** A turn's samples (mono, 16 kHz, -1 to 1) as a WAV, base64, to keep. */
+export function soundOf(audio) {
+    const samples = audio.subarray(0, HEARING_RATE * KEPT_SECONDS_AT_MOST);
+    const bytes = new Uint8Array(44 + samples.length * 2);
+    const view = new DataView(bytes.buffer);
+    const text = (at, value) => {
+        for (let k = 0; k < value.length; k++) {
+            view.setUint8(at + k, value.charCodeAt(k));
+        }
+    };
+    text(0, 'RIFF');
+    view.setUint32(4, 36 + samples.length * 2, true);
+    text(8, 'WAVE');
+    text(12, 'fmt ');
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true);
+    view.setUint16(22, 1, true);
+    view.setUint32(24, HEARING_RATE, true);
+    view.setUint32(28, HEARING_RATE * 2, true);
+    view.setUint16(32, 2, true);
+    view.setUint16(34, 16, true);
+    text(36, 'data');
+    view.setUint32(40, samples.length * 2, true);
+    for (let k = 0; k < samples.length; k++) {
+        const sample = Math.max(-1, Math.min(1, samples[k]));
+        view.setInt16(44 + k * 2, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
+    }
+    let binary = '';
+    for (let k = 0; k < bytes.length; k += 0x8000) {
+        binary += String.fromCharCode(...bytes.subarray(k, k + 0x8000));
+    }
+    return {
+        data: btoa(binary),
+        type: 'audio/wav',
+        seconds: Math.round((samples.length / HEARING_RATE) * 100) / 100,
+    };
+}

package/lib/voice/sayAloud.d.ts

@@ -14,6 +14,11 @@
 export declare const VOICE_SAY_EVENT = "voice.say";
 /** The running application's voice, at a run's start. */
 export declare const VOICE_SETTINGS_EVENT = "voice.settings";
+/**
+ * The person spoke over the agent, the runtime says (VO-46): a client
+ * playing its voice stops it.
+ */
+export declare const VOICE_INTERRUPTED_EVENT = "voice.interrupted";
 /** The longest line said aloud: the runtime refuses longer ones. */
 export declare const SAY_ALOUD_MAX_CHARACTERS = 200;
 /** A custom event or data part, as the chat's adapters give it. */
@@ -21,5 +26,7 @@ export interface VoiceActivity {
     type: string;
     data: unknown;
 }
+/** Whether an activity is the runtime's `voice.interrupted` (VO-46). */
+export declare const isVoiceInterrupted: (activity: VoiceActivity) => boolean;
 /** The line a `voice.say` says, or `undefined` for anything else. */
 export declare function lineToSay(activity: VoiceActivity): string | undefined;

package/lib/voice/sayAloud.js

@@ -18,9 +18,16 @@
 export const VOICE_SAY_EVENT = 'voice.say';
 /** The running application's voice, at a run's start. */
 export const VOICE_SETTINGS_EVENT = 'voice.settings';
+/**
+ * The person spoke over the agent, the runtime says (VO-46): a client
+ * playing its voice stops it.
+ */
+export const VOICE_INTERRUPTED_EVENT = 'voice.interrupted';
 /** The longest line said aloud: the runtime refuses longer ones. */
 export const SAY_ALOUD_MAX_CHARACTERS = 200;
 const nameOf = (type) => type.replace(/^data-/, '');
+/** Whether an activity is the runtime's `voice.interrupted` (VO-46). */
+export const isVoiceInterrupted = (activity) => nameOf(activity.type) === VOICE_INTERRUPTED_EVENT;
 /** The line a `voice.say` says, or `undefined` for anything else. */
 export function lineToSay(activity) {
     if (nameOf(activity.type) !== VOICE_SAY_EVENT) {

package/lib/voice/serverHearing.d.ts

@@ -94,6 +94,8 @@ export declare class ServerHearing implements Hearing {
      * Whether speech was found and how long it has been silent, and the words
      * the speech service heard so far: the sound goes to its stream as it comes.
      */
+    spokeSoFar(audio: Float32Array, shortestSeconds?: number): Promise<boolean>;
+    quietSoFar(audio: Float32Array): Promise<HeardSoFar>;
     hearSoFar(audio: Float32Array, language?: string): Promise<HeardSoFar>;
     hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
 }
@@ -121,5 +123,7 @@ export declare class AutoHearing implements Hearing {
     private get current();
     ready(language: string, progress?: HearingProgress): Promise<void>;
     hearSoFar(audio: Float32Array, language: string): Promise<HeardSoFar>;
+    spokeSoFar(audio: Float32Array, shortestSeconds?: number): Promise<boolean>;
+    quietSoFar(audio: Float32Array): Promise<HeardSoFar>;
     hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
 }

package/lib/voice/serverHearing.js

@@ -315,6 +315,14 @@ export class ServerHearing {
      * Whether speech was found and how long it has been silent, and the words
      * the speech service heard so far: the sound goes to its stream as it comes.
      */
+    async spokeSoFar(audio, shortestSeconds = 0.2) {
+        const { speech } = await this.finder.findSpeech(audio);
+        return speech.length >= HEARING_RATE * shortestSeconds;
+    }
+    async quietSoFar(audio) {
+        const { speech, silentMs } = await this.finder.findSpeech(audio);
+        return { text: '', spoke: speech.length >= SHORTEST, silentMs };
+    }
     async hearSoFar(audio, language = '') {
         const { speech, silentMs } = await this.finder.findSpeech(audio);
         const spoke = speech.length >= SHORTEST;
@@ -450,6 +458,18 @@ export class AutoHearing {
     hearSoFar(audio, language) {
         return this.current.hearSoFar(audio, language);
     }
+    async spokeSoFar(audio, shortestSeconds) {
+        const current = this.current;
+        return current.spokeSoFar
+            ? current.spokeSoFar(audio, shortestSeconds)
+            : (await current.hearSoFar(audio, '')).spoke;
+    }
+    quietSoFar(audio) {
+        const current = this.current;
+        return current.quietSoFar
+            ? current.quietSoFar(audio)
+            : current.hearSoFar(audio, '');
+    }
     async hear(audio, language) {
         if (this.place === 'device') {
             return this.device.hear(audio, language);

package/lib/voice/speaker.d.ts

@@ -50,6 +50,13 @@ export interface SpeakerOptions {
     onState?: (state: SpeakerState) => void;
     /** Called with how long a sentence took from asked to first sound, in ms. */
     onFirstSound?: (ms: number) => void;
+    /**
+     * Say a long first sentence's first clause on its own (VO-52): the service
+     * makes a sentence whole, in about the time its length takes, so the first
+     * sound comes sooner when the first thing asked is shorter. Off for a
+     * caller that follows its sentences one by one (a deck's narrator).
+     */
+    firstClauseFirst?: boolean;
     fetcher?: typeof fetch;
 }
 /**
@@ -60,6 +67,14 @@ export interface SpeakerOptions {
  * 14 October 2026).
  */
 export declare const SENTENCES_AHEAD = 2;
+/** A first sentence longer than this is said clause first (`firstClauseFirst`). */
+export declare const FIRST_CLAUSE_SPLIT_CHARS = 60;
+/**
+ * A sentence cut after its first clause — a comma, semicolon, colon or dash
+ * followed by a space — when it is long and neither part is too short (VO-52);
+ * else the sentence alone. Never cut inside a number (`1,000`).
+ */
+export declare function firstClause(sentence: string): string[];
 /** Plays an answer's sentences, said by the speech service. */
 export declare class ServerSpeaker {
     private options;

package/lib/voice/speaker.js

@@ -21,6 +21,34 @@ export const SYNTHESES_PATH = '/api/ai-agents/v1/speech/syntheses';
  * 14 October 2026).
  */
 export const SENTENCES_AHEAD = 2;
+/** A first sentence longer than this is said clause first (`firstClauseFirst`). */
+export const FIRST_CLAUSE_SPLIT_CHARS = 60;
+/** Neither part of a sentence cut at its first clause is shorter than this. */
+const SHORTEST_CLAUSE_CHARS = 15;
+/**
+ * A sentence cut after its first clause — a comma, semicolon, colon or dash
+ * followed by a space — when it is long and neither part is too short (VO-52);
+ * else the sentence alone. Never cut inside a number (`1,000`).
+ */
+export function firstClause(sentence) {
+    if (sentence.length <= FIRST_CLAUSE_SPLIT_CHARS) {
+        return [sentence];
+    }
+    const boundary = /[,;:\u2013\u2014]\s+/g;
+    let found;
+    while ((found = boundary.exec(sentence)) !== null) {
+        const end = found.index + 1;
+        if (end < SHORTEST_CLAUSE_CHARS) {
+            continue;
+        }
+        const rest = sentence.slice(found.index + found[0].length);
+        if (rest.length < SHORTEST_CLAUSE_CHARS) {
+            break;
+        }
+        return [sentence.slice(0, end), rest];
+    }
+    return [sentence];
+}
 /** Plays an answer's sentences, said by the speech service. */
 export class ServerSpeaker {
     options;
@@ -99,7 +127,14 @@ export class ServerSpeaker {
         if (!sentence) {
             return;
         }
-        this.queue.push({ text: sentence, asked: performance.now() });
+        // Nothing playing or waiting: this is the first thing heard (VO-52).
+        const first = this.options.firstClauseFirst &&
+            !this.queue.length &&
+            !this.playing &&
+            !this.running;
+        for (const part of first ? firstClause(sentence) : [sentence]) {
+            this.queue.push({ text: part, asked: performance.now() });
+        }
         if (!this.state.speaking) {
             // Something new to say: the last refusal is no longer the news.
             this.set({ speaking: true, refused: undefined });

package/lib/voice/types.d.ts

@@ -80,6 +80,18 @@ export interface SpokenMetadata {
     language: string;
     engine: string;
     where: 'device' | 'server';
+    /**
+     * The turn's sound, sent to be kept with its words when the application's
+     * record keeps `audio` and the person agreed (VO-63); never on the
+     * conversation's own message.
+     */
+    sound?: SpokenSound;
+}
+/** A turn's recording, to keep (VO-63): a WAV, base64. */
+export interface SpokenSound {
+    data: string;
+    type: 'audio/wav';
+    seconds: number;
 }
 /** What the person said, once heard. */
 export interface Transcript {

package/lib/voice/useVoiceSpeaker.js

@@ -30,6 +30,8 @@ export function useVoiceSpeaker(voice, on = true) {
             rate: voice.rate,
             volume: voice.volume,
             onState: setState,
+            // The chat's answers: a long first sentence's first clause first.
+            firstClauseFirst: true,
         })
         : undefined, 
     // eslint-disable-next-line react-hooks/exhaustive-deps

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@datalayer/agent-runtimes",
-    "version": "1.5.59",
+    "version": "1.5.61",
     "type": "module",
     "workspaces": [
         ".",