@datalayer/agent-runtimes 1.5.43 → 1.5.44

package/lib/voice/VoiceMenu.d.ts

@@ -2,7 +2,8 @@
  * The composer's voice menu (VOICE.md VO-14, VO-21, VO-25, VO-30): beside
  * the microphone, what a person chooses for themselves — the language they
  * speak and hear, when answers are heard, the voice, its speed and loudness,
- * *Send what I say*, the microphone, and *Keep speech on this device*. Each
+ * how long they pause before they are done (VO-69), *Send what I say*, the
+ * microphone, and *Keep speech on this device*. Each
  * choice is kept in the person's settings and heard at once by every chat
  * on the page; *Use the application's* forgets them all.
  *

package/lib/voice/VoiceMenu.js

@@ -2,7 +2,7 @@ import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-run
 import { useState } from 'react';
 import { ActionList, ActionMenu, IconButton } from '@primer/react';
 import { GearIcon } from '@primer/octicons-react';
-import { OUTPUT_CHOICES, RATE_CHOICES, VOLUME_CHOICES, catalogueVoiceSpeaks, voiceLanguages, voicesSpeaking, } from './choices';
+import { OUTPUT_CHOICES, PAUSE_CHOICES, RATE_CHOICES, VOLUME_CHOICES, catalogueVoiceSpeaks, voiceLanguages, voicesSpeaking, } from './choices';
 import { withVoiceSetting } from './settings';
 import { useVoiceSettings } from './useVoiceSettings';
 /** The microphones the browser lists; named only once one was allowed. */
@@ -28,6 +28,6 @@ export function VoiceMenu({ voice, answers, disabled = false, }) {
                 void microphones().then(setDevices);
             }
         }, children: [_jsx(ActionMenu.Anchor, { children: _jsx(IconButton, { icon: GearIcon, size: "small", variant: "invisible", "aria-label": "Voice settings", "data-voice-menu": "", disabled: disabled }) }), _jsx(ActionMenu.Overlay, { width: "medium", children: _jsxs(ActionList, { children: [_jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Language" }), voiceLanguages().map(language => (_jsx(ActionList.Item, { selected: language.value === voice.language, onSelect: () => choose('language', language.value), "data-voice-language": language.value, children: language.label }, language.value)))] }), answers && !onDevice ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Answers heard" }), OUTPUT_CHOICES.map(output => (_jsx(ActionList.Item, { selected: output.value === voice.output, onSelect: () => choose('output', output.value), "data-voice-output": output.value, children: output.label }, output.value)))] }), voices.length > 0 ? (_jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Voice" }), voices.map(spoken => (_jsx(ActionList.Item, { selected: spoken.id === voice.voice, onSelect: () => choose('voice', spoken.id), "data-voice-voice": spoken.id, children: spoken.name }, spoken.id)))] })) : null, _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Speed" }), RATE_CHOICES.map(rate => (_jsx(ActionList.Item, { selected: rate.value === (voice.rate ?? 1), onSelect: () => choose('rate', rate.value), "data-voice-rate": rate.value, children: rate.label }, rate.value)))] }), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Loudness" }), VOLUME_CHOICES.map(volume => (_jsx(ActionList.Item, { selected: volume.value === (voice.volume ?? 1), onSelect: () => choose('volume', volume.value), "data-voice-volume": volume.value, children: volume.label }, volume.value)))] })] })) : null, named.length > 1 ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Microphone" }), named.map(device => (_jsx(ActionList.Item, { selected: device.deviceId ===
-                                                (voice.microphoneId ?? named[0].deviceId), onSelect: () => choose('microphoneId', device.deviceId), "data-voice-microphone": device.deviceId, children: device.label }, device.deviceId)))] })] })) : null, _jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "multiple", children: [_jsxs(ActionList.Item, { selected: Boolean(voice.sendWhatISay), onSelect: () => choose('sendWhatISay', !voice.sendWhatISay), "data-voice-send-what-i-say": "", children: ["Send what I say", _jsx(ActionList.Description, { variant: "block", children: "Sent as soon as it is heard, not left in the box." })] }), voice.speechUrl && !onDevice ? (_jsxs(ActionList.Item, { selected: detect, onSelect: () => choose('detectLanguage', !detect), "data-voice-detect": "", children: ["Detect the language I speak", _jsx(ActionList.Description, { variant: "block", children: "Heard on Datalayer\u2019s servers by Whisper, which finds the language; nothing is kept there." })] })) : null, _jsxs(ActionList.Item, { selected: onDevice, onSelect: () => choose('onDeviceOnly', !onDevice), "data-voice-on-device": "", children: ["Keep speech on this device", _jsx(ActionList.Description, { variant: "block", children: "Nothing is sent to Datalayer\u2019s speech service: you are heard in the browser, answers are not spoken." })] })] }), Object.keys(settings).length > 0 ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsx(ActionList.Item, { onSelect: () => keep({}), "data-voice-settings-reset": "", children: "Use the application\u2019s" })] })) : null] }) })] }));
+                                                (voice.microphoneId ?? named[0].deviceId), onSelect: () => choose('microphoneId', device.deviceId), "data-voice-microphone": device.deviceId, children: device.label }, device.deviceId)))] })] })) : null, _jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Pauses before I am done" }), PAUSE_CHOICES.map(pause => (_jsx(ActionList.Item, { selected: pause.value === (settings.pause ?? 1), onSelect: () => choose('pause', pause.value === 1 ? undefined : pause.value), "data-voice-pause": pause.value, children: pause.label }, pause.value)))] }), _jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "multiple", children: [_jsxs(ActionList.Item, { selected: Boolean(voice.sendWhatISay), onSelect: () => choose('sendWhatISay', !voice.sendWhatISay), "data-voice-send-what-i-say": "", children: ["Send what I say", _jsx(ActionList.Description, { variant: "block", children: "Sent as soon as it is heard, not left in the box." })] }), voice.speechUrl && !onDevice ? (_jsxs(ActionList.Item, { selected: detect, onSelect: () => choose('detectLanguage', !detect), "data-voice-detect": "", children: ["Detect the language I speak", _jsx(ActionList.Description, { variant: "block", children: "Heard on Datalayer\u2019s servers by Whisper, which finds the language; nothing is kept there." })] })) : null, _jsxs(ActionList.Item, { selected: onDevice, onSelect: () => choose('onDeviceOnly', !onDevice), "data-voice-on-device": "", children: ["Keep speech on this device", _jsx(ActionList.Description, { variant: "block", children: "Nothing is sent to Datalayer\u2019s speech service: you are heard in the browser, answers are not spoken." })] })] }), Object.keys(settings).length > 0 ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsx(ActionList.Item, { onSelect: () => keep({}), "data-voice-settings-reset": "", children: "Use the application\u2019s" })] })) : null] }) })] }));
 }
 export default VoiceMenu;

package/lib/voice/chatVoice.d.ts

@@ -59,7 +59,10 @@ export declare const SILENCE_MS = 1500;
  * in: the person's own settings first where they made some (`voiceForPerson`:
  * their language, voice, rate, volume, microphone; VO-14, VO-25, VO-30),
  * else the host's; answers not heard, the language's first voice,
- * Datalayer's models and packages, a silence of 1.5 s. Answers heard need
- * the speech service's address: refused without.
+ * Datalayer's models and packages, a silence of 1.5 s. A person who pauses
+ * longer before they are done has both silences — the one that stops a
+ * microphone clicked on, and the one that ends a hands-free turn — made
+ * longer by their choice (`pause`, VO-69). Answers heard need the speech
+ * service's address: refused without.
  */
 export declare function settledVoice(given: ChatVoice, person?: VoiceSettings): SettledVoice;

package/lib/voice/chatVoice.js

@@ -11,7 +11,8 @@
  * @module voice/chatVoice
  */
 import { voiceFor } from '../specs/voices';
-import { heldVoiceSettings, voiceForPerson, } from './settings';
+import { TURN_SILENCE_MS } from './handsFree';
+import { heldVoiceSettings, saneVoiceSettings, voiceForPerson, } from './settings';
 /**
  * Datalayer's origin for the browser's speech models, as pinned
  * (`s3://datalayer-models-speech` behind its CDN; VO-48, VO-49): each model
@@ -84,11 +85,15 @@ export const SILENCE_MS = 1500;
  * in: the person's own settings first where they made some (`voiceForPerson`:
  * their language, voice, rate, volume, microphone; VO-14, VO-25, VO-30),
  * else the host's; answers not heard, the language's first voice,
- * Datalayer's models and packages, a silence of 1.5 s. Answers heard need
- * the speech service's address: refused without.
+ * Datalayer's models and packages, a silence of 1.5 s. A person who pauses
+ * longer before they are done has both silences — the one that stops a
+ * microphone clicked on, and the one that ends a hands-free turn — made
+ * longer by their choice (`pause`, VO-69). Answers heard need the speech
+ * service's address: refused without.
  */
 export function settledVoice(given, person = heldVoiceSettings()) {
     const voice = voiceForPerson(given, person);
+    const pause = saneVoiceSettings(person).pause ?? 1;
     const output = voice.output ?? 'off';
     if (output !== 'off' && !voice.speechUrl) {
         throw new Error(`A chat whose answers are heard (${output}) names the speech service: its speechUrl is missing.`);
@@ -99,6 +104,7 @@ export function settledVoice(given, person = heldVoiceSettings()) {
         voice: voice.voice || voiceFor(voice.language)?.id || '',
         modelsUrl: voice.modelsUrl || DATALAYER_SPEECH_MODELS_URL,
         engines: voice.engines ?? DATALAYER_VOICE_ENGINES,
-        silenceMs: voice.silenceMs ?? SILENCE_MS,
+        silenceMs: Math.round((voice.silenceMs ?? SILENCE_MS) * pause),
+        turnSilenceMs: Math.round((voice.turnSilenceMs ?? TURN_SILENCE_MS) * pause),
     };
 }

package/lib/voice/choices.d.ts

@@ -28,6 +28,8 @@ export declare function catalogueVoiceSpeaks(voice: string, language: string): b
 /** The speeds offered, as multipliers of the voice's own. */
 export declare const RATE_CHOICES: VoiceChoice<number>[];
 /** The loudnesses offered, 0 to 1. */
+/** How long the person pauses before they are done (VO-69): the silences, times this. */
+export declare const PAUSE_CHOICES: VoiceChoice<number>[];
 export declare const VOLUME_CHOICES: VoiceChoice<number>[];
 /** When answers are heard (VO-21): what a person may choose where the application speaks. */
 export declare const OUTPUT_CHOICES: VoiceChoice<NonNullable<ChatVoice['output']>>[];

package/lib/voice/choices.js

@@ -56,6 +56,12 @@ export const RATE_CHOICES = [
     { value: 1.5, label: 'Fastest' },
 ];
 /** The loudnesses offered, 0 to 1. */
+/** How long the person pauses before they are done (VO-69): the silences, times this. */
+export const PAUSE_CHOICES = [
+    { value: 1, label: 'Usual' },
+    { value: 2, label: 'Longer' },
+    { value: 3, label: 'Much longer' },
+];
 export const VOLUME_CHOICES = [
     { value: 0.25, label: 'Quiet' },
     { value: 0.5, label: 'Half' },

package/lib/voice/index.d.ts

@@ -14,6 +14,7 @@ export * from './pinned';
 export * from './capture';
 export * from './hearing';
 export * from './serverHearing';
+export * from './sayAloud';
 export * from './handsFree';
 export { HandsFree, type HandsFreeProps } from './HandsFree';
 export * from './speaker';

package/lib/voice/index.js

@@ -8,6 +8,7 @@ export * from './pinned';
 export * from './capture';
 export * from './hearing';
 export * from './serverHearing';
+export * from './sayAloud';
 export * from './handsFree';
 export { HandsFree } from './HandsFree';
 export * from './speaker';

package/lib/voice/sayAloud.d.ts

@@ -0,0 +1,25 @@
+/**
+ * What only the runtime knows of voice, as it says it over AG-UI (VOICE.md
+ * VO-35) and Vercel AI (VO-36): `voice.settings` — the running application's
+ * voice and language, at a run's start — and `voice.say` — a short line the
+ * agent wants heard and not written (*One moment*), shown as a caption while
+ * it is said. The chat reads `voice.say` from a run's custom events
+ * (`CUSTOM`, the chat's `activity`) or data parts (`data-voice.say`); it
+ * already knows its application's voice from `/apps/configure` (VO-45), and
+ * `voice.settings` is for clients that do not ask it.
+ *
+ * @module voice/sayAloud
+ */
+/** A line the agent wants heard, not written. */
+export declare const VOICE_SAY_EVENT = "voice.say";
+/** The running application's voice, at a run's start. */
+export declare const VOICE_SETTINGS_EVENT = "voice.settings";
+/** The longest line said aloud: the runtime refuses longer ones. */
+export declare const SAY_ALOUD_MAX_CHARACTERS = 200;
+/** A custom event or data part, as the chat's adapters give it. */
+export interface VoiceActivity {
+    type: string;
+    data: unknown;
+}
+/** The line a `voice.say` says, or `undefined` for anything else. */
+export declare function lineToSay(activity: VoiceActivity): string | undefined;

package/lib/voice/sayAloud.js

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * What only the runtime knows of voice, as it says it over AG-UI (VOICE.md
+ * VO-35) and Vercel AI (VO-36): `voice.settings` — the running application's
+ * voice and language, at a run's start — and `voice.say` — a short line the
+ * agent wants heard and not written (*One moment*), shown as a caption while
+ * it is said. The chat reads `voice.say` from a run's custom events
+ * (`CUSTOM`, the chat's `activity`) or data parts (`data-voice.say`); it
+ * already knows its application's voice from `/apps/configure` (VO-45), and
+ * `voice.settings` is for clients that do not ask it.
+ *
+ * @module voice/sayAloud
+ */
+/** A line the agent wants heard, not written. */
+export const VOICE_SAY_EVENT = 'voice.say';
+/** The running application's voice, at a run's start. */
+export const VOICE_SETTINGS_EVENT = 'voice.settings';
+/** The longest line said aloud: the runtime refuses longer ones. */
+export const SAY_ALOUD_MAX_CHARACTERS = 200;
+const nameOf = (type) => type.replace(/^data-/, '');
+/** The line a `voice.say` says, or `undefined` for anything else. */
+export function lineToSay(activity) {
+    if (nameOf(activity.type) !== VOICE_SAY_EVENT) {
+        return undefined;
+    }
+    const data = activity.data;
+    const text = data && typeof data.text === 'string'
+        ? data.text.split(/\s+/).join(' ').trim()
+        : '';
+    return text ? text.slice(0, SAY_ALOUD_MAX_CHARACTERS) : undefined;
+}

package/lib/voice/serverHearing.d.ts

@@ -2,6 +2,12 @@ import type { Hearing, HearingProgress, HeardSoFar } from './hearing';
 import type { Transcript } from './types';
 /** Where the speech service hears a recording, on ai-agents' host. */
 export declare const TRANSCRIPTIONS_PATH = "/api/ai-agents/v1/speech/transcriptions";
+/** Where the speech service hears speech as it is said. */
+export declare const TRANSCRIPTION_STREAM_PATH = "/api/ai-agents/v1/speech/transcriptions/stream";
+/** A stream no look or hearing has fed for this long is closed: its press was cancelled. */
+export declare const STREAM_IDLE_MS = 3000;
+/** How long the final words of a stream are waited for. */
+export declare const STREAM_FINAL_MS = 60000;
 /** The model the speech service hears with (agentspecs' catalogue). */
 export declare const SERVER_HEARING_MODEL = "whisper-large-v3-turbo";
 /** Where a chat's voice hears: in the browser, or on Datalayer's servers. */
@@ -40,7 +46,26 @@ export interface ServerHearingOptions {
     /** Let Whisper find the language (VO-14), rather than tell it. */
     detect?: boolean;
     fetcher?: typeof fetch;
+    /** Opens a WebSocket; the browser's own unless a test gives one. */
+    socket?: (url: string) => SpeechSocket;
 }
+/** What `ServerHearing` needs of a WebSocket. */
+export interface SpeechSocket {
+    readonly readyState: number;
+    binaryType: string;
+    onopen: ((event: unknown) => void) | null;
+    onmessage: ((event: {
+        data: unknown;
+    }) => void) | null;
+    onclose: ((event: unknown) => void) | null;
+    onerror: ((event: unknown) => void) | null;
+    send(data: string | ArrayBuffer): void;
+    close(code?: number): void;
+}
+/** 16 kHz samples in [-1, 1] as little-endian 16-bit PCM: what a `pcm16` stream takes. */
+export declare function pcm16Of(samples: Float32Array): ArrayBuffer;
+/** The ws(s) address of the stream on ai-agents' origin. */
+export declare function streamUrlOf(speechUrl: string): string;
 /**
  * Hears on Datalayer's servers: the browser's VAD finds the speech and when
  * it stopped; Whisper, on the speech service, writes it.
@@ -52,7 +77,14 @@ export declare class ServerHearing implements Hearing {
     constructor(finder: SpeechFinder, options: ServerHearingOptions);
     /** Only the VAD loads in the page: the words are written on the server. */
     ready(_language: string, progress?: HearingProgress): Promise<void>;
-    /** Whether speech was found and how long it has been silent; its words come at the end. */
-    hearSoFar(audio: Float32Array): Promise<HeardSoFar>;
+    /** The press being heard as it is said, while one is. */
+    private press?;
+    /** The press's stream, opened anew when a new press began (its sound is shorter). */
+    private streamFor;
+    /**
+     * Whether speech was found and how long it has been silent, and the words
+     * the speech service heard so far: the sound goes to its stream as it comes.
+     */
+    hearSoFar(audio: Float32Array, language?: string): Promise<HeardSoFar>;
     hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
 }

package/lib/voice/serverHearing.js

@@ -6,7 +6,8 @@
  * Hearing on Datalayer's servers (VOICE.md VO-34, VO-14, §5): what was said
  * in one press, kept to its speech by Silero VAD in the page, then sent as a
  * WAV recording to the speech service's `POST /speech/transcriptions`, where
- * Whisper large-v3-turbo hears it — in any language it knows, and finding
+ * Whisper hears it — small for a spoken turn, large-v3-turbo past one
+ * (decision 17) — in any language it knows, and finding
  * the language itself when the person asks for that (detection, VO-14). The
  * sound is held in the request only: the service keeps neither it nor the
  * words (VO-60).
@@ -14,8 +15,15 @@
  * Used where the application's voice says `where: server`, where it says
  * `auto` and no model in the browser hears the language, and where the
  * person asked for their language to be detected — never when they chose to
- * keep speech on the device. A clip model: its words come at the end, not
- * while the person speaks (VO-11).
+ * keep speech on the device.
+ *
+ * While the person speaks, the words come too (VO-11): the first look at
+ * what was heard so far opens `WS /speech/transcriptions/stream` (the token
+ * in its first message, VO-37), the sound is sent as it is captured, 16-bit
+ * PCM at 16 kHz, and the service's partial words are what was heard so far,
+ * greyed in the composer until final. At the end of the press the stream is
+ * told so and its final words are the transcript. A press heard without a
+ * look before it (a short one) is sent whole as a recording instead.
  *
  * @module voice/serverHearing
  */
@@ -23,6 +31,12 @@ import { transcriberFor } from '../specs/voices';
 import { HEARING_RATE } from './capture';
 /** Where the speech service hears a recording, on ai-agents' host. */
 export const TRANSCRIPTIONS_PATH = '/api/ai-agents/v1/speech/transcriptions';
+/** Where the speech service hears speech as it is said. */
+export const TRANSCRIPTION_STREAM_PATH = '/api/ai-agents/v1/speech/transcriptions/stream';
+/** A stream no look or hearing has fed for this long is closed: its press was cancelled. */
+export const STREAM_IDLE_MS = 3000;
+/** How long the final words of a stream are waited for. */
+export const STREAM_FINAL_MS = 60_000;
 /** The model the speech service hears with (agentspecs' catalogue). */
 export const SERVER_HEARING_MODEL = 'whisper-large-v3-turbo';
 /** The shortest speech worth hearing, in samples: a word said alone. */
@@ -72,6 +86,162 @@ export function wavOf(samples, rate = HEARING_RATE) {
     }
     return bytes;
 }
+/** 16 kHz samples in [-1, 1] as little-endian 16-bit PCM: what a `pcm16` stream takes. */
+export function pcm16Of(samples) {
+    const bytes = new ArrayBuffer(samples.length * 2);
+    const view = new DataView(bytes);
+    for (let i = 0; i < samples.length; i++) {
+        const sample = Math.max(-1, Math.min(1, samples[i]));
+        view.setInt16(i * 2, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
+    }
+    return bytes;
+}
+/** The ws(s) address of the stream on ai-agents' origin. */
+export function streamUrlOf(speechUrl) {
+    return `${speechUrl.replace(/\/+$/, '').replace(/^http/, 'ws')}${TRANSCRIPTION_STREAM_PATH}`;
+}
+/**
+ * One press heard as it is said: the sound sent as it comes, the service's
+ * partial words kept, its final words awaited at the end.
+ */
+class PressStream {
+    /** Samples of the press sent so far. */
+    sent = 0;
+    /** The newest partial words. */
+    partial = '';
+    /** Why the service refused, in its sentence. */
+    refused;
+    socket;
+    ready = false;
+    queued = [];
+    final;
+    waiting;
+    closed = false;
+    idle;
+    constructor(open, speechUrl, first) {
+        this.socket = open(streamUrlOf(speechUrl));
+        this.socket.binaryType = 'arraybuffer';
+        this.socket.onopen = () => {
+            this.socket.send(JSON.stringify({
+                token: first.token ?? '',
+                language: first.language ?? '',
+                format: 'pcm16',
+            }));
+        };
+        this.socket.onmessage = event => {
+            let said;
+            try {
+                said = JSON.parse(String(event.data));
+            }
+            catch {
+                return;
+            }
+            if (said.type === 'ready') {
+                this.ready = true;
+                for (const chunk of this.queued.splice(0)) {
+                    this.socket.send(chunk);
+                }
+            }
+            else if (said.type === 'partial' && typeof said.text === 'string') {
+                this.partial = said.text.trim();
+            }
+            else if (said.type === 'final') {
+                this.final = {
+                    text: typeof said.text === 'string' ? said.text.trim() : '',
+                    language: typeof said.language === 'string' ? said.language : '',
+                };
+                this.settle();
+            }
+            else if (said.type === 'refused') {
+                this.refused =
+                    typeof said.detail === 'string' && said.detail
+                        ? said.detail
+                        : "What you said could not be heard on Datalayer's servers.";
+                this.settle();
+            }
+        };
+        this.socket.onclose = () => {
+            this.closed = true;
+            this.settle();
+        };
+        this.socket.onerror = () => undefined;
+        this.touch();
+    }
+    /** Send what the press added since the last call. */
+    feed(audio) {
+        this.touch();
+        if (audio.length <= this.sent || this.closed) {
+            return;
+        }
+        const chunk = pcm16Of(audio.subarray(this.sent));
+        this.sent = audio.length;
+        if (this.ready) {
+            this.socket.send(chunk);
+        }
+        else {
+            this.queued.push(chunk);
+        }
+    }
+    /** The end of the press: the final words, or `undefined` when the stream ended without them. */
+    finish(audio) {
+        this.feed(audio);
+        if (this.final || this.closed || this.refused) {
+            return Promise.resolve(this.final);
+        }
+        return new Promise(resolve => {
+            const timer = setTimeout(() => {
+                this.waiting = undefined;
+                this.close();
+                resolve(undefined);
+            }, STREAM_FINAL_MS);
+            this.waiting = final => {
+                clearTimeout(timer);
+                resolve(final);
+            };
+            const end = JSON.stringify({ type: 'end' });
+            if (this.ready) {
+                this.socket.send(end);
+            }
+            else {
+                // Said once the service is ready, after the sound queued before it.
+                this.queued.push(end);
+            }
+        });
+    }
+    close() {
+        if (this.idle) {
+            clearTimeout(this.idle);
+        }
+        this.closed = true;
+        try {
+            this.socket.close(1000);
+        }
+        catch {
+            // Already closed.
+        }
+        this.settle();
+    }
+    touch() {
+        if (this.idle) {
+            clearTimeout(this.idle);
+        }
+        this.idle = setTimeout(() => {
+            if (!this.waiting) {
+                this.close();
+            }
+        }, STREAM_IDLE_MS);
+    }
+    settle() {
+        const waiting = this.waiting;
+        this.waiting = undefined;
+        waiting?.(this.final);
+        if (this.final || this.refused) {
+            if (this.idle) {
+                clearTimeout(this.idle);
+            }
+        }
+    }
+}
 /**
  * Hears on Datalayer's servers: the browser's VAD finds the speech and when
  * it stopped; Whisper, on the speech service, writes it.
@@ -88,17 +258,75 @@ export class ServerHearing {
     ready(_language, progress) {
         return this.finder.readyToFindSpeech(progress);
     }
-    /** Whether speech was found and how long it has been silent; its words come at the end. */
-    async hearSoFar(audio) {
+    /** The press being heard as it is said, while one is. */
+    press;
+    /** The press's stream, opened anew when a new press began (its sound is shorter). */
+    streamFor(audio, language) {
+        const open = this.options.socket ??
+            (typeof WebSocket === 'undefined'
+                ? undefined
+                : (url) => new WebSocket(url));
+        if (!open) {
+            return undefined;
+        }
+        if (this.press && audio.length < this.press.sent) {
+            this.press.close();
+            this.press = undefined;
+        }
+        if (!this.press) {
+            this.press = new PressStream(open, this.options.speechUrl, {
+                token: this.options.token,
+                language: this.options.detect ? '' : language,
+            });
+        }
+        return this.press;
+    }
+    /**
+     * Whether speech was found and how long it has been silent, and the words
+     * the speech service heard so far: the sound goes to its stream as it comes.
+     */
+    async hearSoFar(audio, language = '') {
         const { speech, silentMs } = await this.finder.findSpeech(audio);
-        return { text: '', spoke: speech.length >= SHORTEST, silentMs };
+        const spoke = speech.length >= SHORTEST;
+        const press = this.streamFor(audio, language);
+        press?.feed(audio);
+        return { text: spoke ? (press?.partial ?? '') : '', spoke, silentMs };
     }
     async hear(audio, language) {
         const started = performance.now();
+        const press = this.press && audio.length >= this.press.sent ? this.press : undefined;
+        if (this.press && !press) {
+            this.press.close();
+        }
+        this.press = undefined;
         const { speech } = await this.finder.findSpeech(audio);
         if (speech.length < SHORTEST) {
+            press?.close();
             return undefined;
         }
+        if (press) {
+            const final = await press.finish(audio);
+            press.close();
+            if (press.refused) {
+                throw new Error(press.refused);
+            }
+            if (!final) {
+                throw new Error("What you said could not be heard on Datalayer's servers: the stream ended before its words.");
+            }
+            return final.text
+                ? {
+                    text: final.text,
+                    metadata: {
+                        input: 'voice',
+                        language: final.language || language,
+                        engine: SERVER_HEARING_MODEL,
+                        where: 'server',
+                    },
+                    seconds: audio.length / HEARING_RATE,
+                    ms: Math.round(performance.now() - started),
+                }
+                : undefined;
+        }
         const { speechUrl, token, detect, fetcher = fetch } = this.options;
         const query = detect ? '' : `?language=${encodeURIComponent(language)}`;
         const answered = await fetcher(`${speechUrl.replace(/\/+$/, '')}${TRANSCRIPTIONS_PATH}${query}`, {

package/lib/voice/settings.d.ts

@@ -33,6 +33,16 @@ export declare const VOLUME_RANGE: {
     readonly max: 1;
     readonly default: 1;
 };
+/**
+ * How much longer than usual the person pauses before they are done
+ * (VOICE.md VO-69): the silence that stops a microphone clicked on, and the
+ * one that ends a hands-free turn, times this.
+ */
+export declare const PAUSE_RANGE: {
+    readonly min: 1;
+    readonly max: 4;
+    readonly default: 1;
+};
 /**
  * What a person chose for themselves. Every field is optional: an absent one
  * means "whatever the application says".
@@ -62,6 +72,8 @@ export interface VoiceSettings {
      * heard on Datalayer's servers, which detect it, rather than in the browser.
      */
     detectLanguage?: boolean;
+    /** How much longer they pause before they are done (VO-69): 1 is usual. */
+    pause?: number;
 }
 /** The settings as they can be used: numbers in range, nothing else trusted. */
 export declare function saneVoiceSettings(settings: VoiceSettings | null | undefined): VoiceSettings;

package/lib/voice/settings.js

@@ -7,6 +7,12 @@ export const VOICE_SETTINGS_KEY = 'datalayer-voice-settings';
 /** How fast and how loud, as multipliers of the voice's own. */
 export const RATE_RANGE = { min: 0.5, max: 2, default: 1 };
 export const VOLUME_RANGE = { min: 0, max: 1, default: 1 };
+/**
+ * How much longer than usual the person pauses before they are done
+ * (VOICE.md VO-69): the silence that stops a microphone clicked on, and the
+ * one that ends a hands-free turn, times this.
+ */
+export const PAUSE_RANGE = { min: 1, max: 4, default: 1 };
 function clamp(value, { min, max }) {
     return Math.min(max, Math.max(min, value));
 }
@@ -43,6 +49,9 @@ export function saneVoiceSettings(settings) {
     if (typeof kept.detectLanguage === 'boolean') {
         sane.detectLanguage = kept.detectLanguage;
     }
+    if (typeof kept.pause === 'number' && Number.isFinite(kept.pause)) {
+        sane.pause = clamp(kept.pause, PAUSE_RANGE);
+    }
     return sane;
 }
 /** What was kept, or nothing. Storage may be refused, or hold nonsense. */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@datalayer/agent-runtimes",
-    "version": "1.5.43",
+    "version": "1.5.44",
     "type": "module",
     "workspaces": [
         ".",