@datalayer/agent-runtimes 1.5.43 → 1.5.45

package/lib/voice/serverHearing.js

@@ -6,7 +6,8 @@
  * Hearing on Datalayer's servers (VOICE.md VO-34, VO-14, §5): what was said
  * in one press, kept to its speech by Silero VAD in the page, then sent as a
  * WAV recording to the speech service's `POST /speech/transcriptions`, where
- * Whisper large-v3-turbo hears it — in any language it knows, and finding
+ * Whisper hears it — small for a spoken turn, large-v3-turbo past one
+ * (decision 17) — in any language it knows, and finding
  * the language itself when the person asks for that (detection, VO-14). The
  * sound is held in the request only: the service keeps neither it nor the
  * words (VO-60).
@@ -14,8 +15,15 @@
  * Used where the application's voice says `where: server`, where it says
  * `auto` and no model in the browser hears the language, and where the
  * person asked for their language to be detected — never when they chose to
- * keep speech on the device. A clip model: its words come at the end, not
- * while the person speaks (VO-11).
+ * keep speech on the device.
+ *
+ * While the person speaks, the words come too (VO-11): the first look at
+ * what was heard so far opens `WS /speech/transcriptions/stream` (the token
+ * in its first message, VO-37), the sound is sent as it is captured, 16-bit
+ * PCM at 16 kHz, and the service's partial words are what was heard so far,
+ * greyed in the composer until final. At the end of the press the stream is
+ * told so and its final words are the transcript. A press heard without a
+ * look before it (a short one) is sent whole as a recording instead.
  *
  * @module voice/serverHearing
  */
@@ -23,10 +31,19 @@ import { transcriberFor } from '../specs/voices';
 import { HEARING_RATE } from './capture';
 /** Where the speech service hears a recording, on ai-agents' host. */
 export const TRANSCRIPTIONS_PATH = '/api/ai-agents/v1/speech/transcriptions';
+/** Where the speech service hears speech as it is said. */
+export const TRANSCRIPTION_STREAM_PATH = '/api/ai-agents/v1/speech/transcriptions/stream';
+/** A stream no look or hearing has fed for this long is closed: its press was cancelled. */
+export const STREAM_IDLE_MS = 3000;
+/** How long the final words of a stream are waited for. */
+export const STREAM_FINAL_MS = 60_000;
 /** The model the speech service hears with (agentspecs' catalogue). */
 export const SERVER_HEARING_MODEL = 'whisper-large-v3-turbo';
 /** The shortest speech worth hearing, in samples: a word said alone. */
 const SHORTEST = HEARING_RATE * 0.2;
+/** The speech service refused a press, in its own sentence (a limit, a visitor's day). */
+export class ServerRefused extends Error {
+}
 /**
  * Where a chat's voice hears (§5): in the browser unless the speech service
  * is named and the person asked for detection, the application says
@@ -72,6 +89,162 @@ export function wavOf(samples, rate = HEARING_RATE) {
     }
     return bytes;
 }
+/** 16 kHz samples in [-1, 1] as little-endian 16-bit PCM: what a `pcm16` stream takes. */
+export function pcm16Of(samples) {
+    const bytes = new ArrayBuffer(samples.length * 2);
+    const view = new DataView(bytes);
+    for (let i = 0; i < samples.length; i++) {
+        const sample = Math.max(-1, Math.min(1, samples[i]));
+        view.setInt16(i * 2, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
+    }
+    return bytes;
+}
+/** The ws(s) address of the stream on ai-agents' origin. */
+export function streamUrlOf(speechUrl) {
+    return `${speechUrl.replace(/\/+$/, '').replace(/^http/, 'ws')}${TRANSCRIPTION_STREAM_PATH}`;
+}
+/**
+ * One press heard as it is said: the sound sent as it comes, the service's
+ * partial words kept, its final words awaited at the end.
+ */
+class PressStream {
+    /** Samples of the press sent so far. */
+    sent = 0;
+    /** The newest partial words. */
+    partial = '';
+    /** Why the service refused, in its sentence. */
+    refused;
+    socket;
+    ready = false;
+    queued = [];
+    final;
+    waiting;
+    closed = false;
+    idle;
+    constructor(open, speechUrl, first) {
+        this.socket = open(streamUrlOf(speechUrl));
+        this.socket.binaryType = 'arraybuffer';
+        this.socket.onopen = () => {
+            this.socket.send(JSON.stringify({
+                token: first.token ?? '',
+                language: first.language ?? '',
+                format: 'pcm16',
+            }));
+        };
+        this.socket.onmessage = event => {
+            let said;
+            try {
+                said = JSON.parse(String(event.data));
+            }
+            catch {
+                return;
+            }
+            if (said.type === 'ready') {
+                this.ready = true;
+                for (const chunk of this.queued.splice(0)) {
+                    this.socket.send(chunk);
+                }
+            }
+            else if (said.type === 'partial' && typeof said.text === 'string') {
+                this.partial = said.text.trim();
+            }
+            else if (said.type === 'final') {
+                this.final = {
+                    text: typeof said.text === 'string' ? said.text.trim() : '',
+                    language: typeof said.language === 'string' ? said.language : '',
+                };
+                this.settle();
+            }
+            else if (said.type === 'refused') {
+                this.refused =
+                    typeof said.detail === 'string' && said.detail
+                        ? said.detail
+                        : "What you said could not be heard on Datalayer's servers.";
+                this.settle();
+            }
+        };
+        this.socket.onclose = () => {
+            this.closed = true;
+            this.settle();
+        };
+        this.socket.onerror = () => undefined;
+        this.touch();
+    }
+    /** Send what the press added since the last call. */
+    feed(audio) {
+        this.touch();
+        if (audio.length <= this.sent || this.closed) {
+            return;
+        }
+        const chunk = pcm16Of(audio.subarray(this.sent));
+        this.sent = audio.length;
+        if (this.ready) {
+            this.socket.send(chunk);
+        }
+        else {
+            this.queued.push(chunk);
+        }
+    }
+    /** The end of the press: the final words, or `undefined` when the stream ended without them. */
+    finish(audio) {
+        this.feed(audio);
+        if (this.final || this.closed || this.refused) {
+            return Promise.resolve(this.final);
+        }
+        return new Promise(resolve => {
+            const timer = setTimeout(() => {
+                this.waiting = undefined;
+                this.close();
+                resolve(undefined);
+            }, STREAM_FINAL_MS);
+            this.waiting = final => {
+                clearTimeout(timer);
+                resolve(final);
+            };
+            const end = JSON.stringify({ type: 'end' });
+            if (this.ready) {
+                this.socket.send(end);
+            }
+            else {
+                // Said once the service is ready, after the sound queued before it.
+                this.queued.push(end);
+            }
+        });
+    }
+    close() {
+        if (this.idle) {
+            clearTimeout(this.idle);
+        }
+        this.closed = true;
+        try {
+            this.socket.close(1000);
+        }
+        catch {
+            // Already closed.
+        }
+        this.settle();
+    }
+    touch() {
+        if (this.idle) {
+            clearTimeout(this.idle);
+        }
+        this.idle = setTimeout(() => {
+            if (!this.waiting) {
+                this.close();
+            }
+        }, STREAM_IDLE_MS);
+    }
+    settle() {
+        const waiting = this.waiting;
+        this.waiting = undefined;
+        waiting?.(this.final);
+        if (this.final || this.refused) {
+            if (this.idle) {
+                clearTimeout(this.idle);
+            }
+        }
+    }
+}
 /**
  * Hears on Datalayer's servers: the browser's VAD finds the speech and when
  * it stopped; Whisper, on the speech service, writes it.
@@ -88,17 +261,75 @@ export class ServerHearing {
     ready(_language, progress) {
         return this.finder.readyToFindSpeech(progress);
     }
-    /** Whether speech was found and how long it has been silent; its words come at the end. */
-    async hearSoFar(audio) {
+    /** The press being heard as it is said, while one is. */
+    press;
+    /** The press's stream, opened anew when a new press began (its sound is shorter). */
+    streamFor(audio, language) {
+        const open = this.options.socket ??
+            (typeof WebSocket === 'undefined'
+                ? undefined
+                : (url) => new WebSocket(url));
+        if (!open) {
+            return undefined;
+        }
+        if (this.press && audio.length < this.press.sent) {
+            this.press.close();
+            this.press = undefined;
+        }
+        if (!this.press) {
+            this.press = new PressStream(open, this.options.speechUrl, {
+                token: this.options.token,
+                language: this.options.detect ? '' : language,
+            });
+        }
+        return this.press;
+    }
+    /**
+     * Whether speech was found and how long it has been silent, and the words
+     * the speech service heard so far: the sound goes to its stream as it comes.
+     */
+    async hearSoFar(audio, language = '') {
         const { speech, silentMs } = await this.finder.findSpeech(audio);
-        return { text: '', spoke: speech.length >= SHORTEST, silentMs };
+        const spoke = speech.length >= SHORTEST;
+        const press = this.streamFor(audio, language);
+        press?.feed(audio);
+        return { text: spoke ? (press?.partial ?? '') : '', spoke, silentMs };
     }
     async hear(audio, language) {
         const started = performance.now();
+        const press = this.press && audio.length >= this.press.sent ? this.press : undefined;
+        if (this.press && !press) {
+            this.press.close();
+        }
+        this.press = undefined;
         const { speech } = await this.finder.findSpeech(audio);
         if (speech.length < SHORTEST) {
+            press?.close();
             return undefined;
         }
+        if (press) {
+            const final = await press.finish(audio);
+            press.close();
+            if (press.refused) {
+                throw new ServerRefused(press.refused);
+            }
+            if (!final) {
+                throw new Error("What you said could not be heard on Datalayer's servers: the stream ended before its words.");
+            }
+            return final.text
+                ? {
+                    text: final.text,
+                    metadata: {
+                        input: 'voice',
+                        language: final.language || language,
+                        engine: SERVER_HEARING_MODEL,
+                        where: 'server',
+                    },
+                    seconds: audio.length / HEARING_RATE,
+                    ms: Math.round(performance.now() - started),
+                }
+                : undefined;
+        }
         const { speechUrl, token, detect, fetcher = fetch } = this.options;
         const query = detect ? '' : `?language=${encodeURIComponent(language)}`;
         const answered = await fetcher(`${speechUrl.replace(/\/+$/, '')}${TRANSCRIPTIONS_PATH}${query}`, {
@@ -111,7 +342,7 @@ export class ServerHearing {
         });
         const body = (await answered.json().catch(() => ({})));
         if (!answered.ok) {
-            throw new Error(typeof body.detail === 'string' && body.detail
+            throw new ServerRefused(typeof body.detail === 'string' && body.detail
                 ? body.detail
                 : `What you said could not be heard on Datalayer's servers (${answered.status}).`);
         }
@@ -134,3 +365,76 @@ export class ServerHearing {
         };
     }
 }
+/** Said when the device's model did not load and the server will hear instead. */
+export const MOVED_TO_SERVER = "This device could not load its model: what you say will be heard on Datalayer's servers instead. Press the microphone again to agree.";
+/**
+ * `where: auto` (§5): each press heard on the device when a model there
+ * hears the language and loads, on the speech service otherwise.
+ *
+ * The device's model failing to load moves the hearing to the server, and
+ * the press is refused in a sentence so that the server's own consent is
+ * asked at the next one. The server refusing a press — a limit, a visitor's
+ * day spent — moves it back to the device when a model there hears the
+ * language, and the press is heard there: the sound then stays in the page.
+ */
+export class AutoHearing {
+    device;
+    server;
+    hearsHere;
+    place;
+    /** The device's model did not load: it is not tried again on this page. */
+    deviceBroken = false;
+    constructor(device, server, language, hearsHere = language => Boolean(transcriberFor(language))) {
+        this.device = device;
+        this.server = server;
+        this.hearsHere = hearsHere;
+        this.place = hearsHere(language) ? 'device' : 'server';
+    }
+    get where() {
+        return this.place;
+    }
+    get current() {
+        return this.place === 'device' ? this.device : this.server;
+    }
+    async ready(language, progress) {
+        if (this.place === 'server' &&
+            !this.deviceBroken &&
+            this.hearsHere(language)) {
+            // A language the device hears, chosen after a server-only one.
+            this.place = 'device';
+        }
+        if (this.place === 'device') {
+            try {
+                await this.device.ready(language, progress);
+                return;
+            }
+            catch {
+                this.place = 'server';
+                this.deviceBroken = true;
+                throw new Error(MOVED_TO_SERVER);
+            }
+        }
+        await this.server.ready(language, progress);
+    }
+    hearSoFar(audio, language) {
+        return this.current.hearSoFar(audio, language);
+    }
+    async hear(audio, language) {
+        if (this.place === 'device') {
+            return this.device.hear(audio, language);
+        }
+        try {
+            return await this.server.hear(audio, language);
+        }
+        catch (error) {
+            if (!(error instanceof ServerRefused) ||
+                this.deviceBroken ||
+                !this.hearsHere(language)) {
+                throw error;
+            }
+            this.place = 'device';
+            await this.device.ready(language);
+            return this.device.hear(audio, language);
+        }
+    }
+}

package/lib/voice/settings.d.ts

@@ -33,6 +33,16 @@ export declare const VOLUME_RANGE: {
     readonly max: 1;
     readonly default: 1;
 };
+/**
+ * How much longer than usual the person pauses before they are done
+ * (VOICE.md VO-69): the silence that stops a microphone clicked on, and the
+ * one that ends a hands-free turn, times this.
+ */
+export declare const PAUSE_RANGE: {
+    readonly min: 1;
+    readonly max: 4;
+    readonly default: 1;
+};
 /**
  * What a person chose for themselves. Every field is optional: an absent one
  * means "whatever the application says".
@@ -62,6 +72,8 @@ export interface VoiceSettings {
      * heard on Datalayer's servers, which detect it, rather than in the browser.
      */
     detectLanguage?: boolean;
+    /** How much longer they pause before they are done (VO-69): 1 is usual. */
+    pause?: number;
 }
 /** The settings as they can be used: numbers in range, nothing else trusted. */
 export declare function saneVoiceSettings(settings: VoiceSettings | null | undefined): VoiceSettings;

package/lib/voice/settings.js

@@ -7,6 +7,12 @@ export const VOICE_SETTINGS_KEY = 'datalayer-voice-settings';
 /** How fast and how loud, as multipliers of the voice's own. */
 export const RATE_RANGE = { min: 0.5, max: 2, default: 1 };
 export const VOLUME_RANGE = { min: 0, max: 1, default: 1 };
+/**
+ * How much longer than usual the person pauses before they are done
+ * (VOICE.md VO-69): the silence that stops a microphone clicked on, and the
+ * one that ends a hands-free turn, times this.
+ */
+export const PAUSE_RANGE = { min: 1, max: 4, default: 1 };
 function clamp(value, { min, max }) {
     return Math.min(max, Math.max(min, value));
 }
@@ -43,6 +49,9 @@ export function saneVoiceSettings(settings) {
     if (typeof kept.detectLanguage === 'boolean') {
         sane.detectLanguage = kept.detectLanguage;
     }
+    if (typeof kept.pause === 'number' && Number.isFinite(kept.pause)) {
+        sane.pause = clamp(kept.pause, PAUSE_RANGE);
+    }
     return sane;
 }
 /** What was kept, or nothing. Storage may be refused, or hold nonsense. */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@datalayer/agent-runtimes",
-    "version": "1.5.43",
+    "version": "1.5.45",
     "type": "module",
     "workspaces": [
         ".",

package/scripts/voice/measure.py

@@ -92,6 +92,29 @@ def transcribe(
     return json.loads(done.stdout.strip().splitlines()[-1])
 
 
+#: Spoken turns timed on the server (VO-47, decision 18): a 3 s fixture and the
+#: e2e suite's 9 s question, each heard ``TURN_RUNS`` times told its language.
+TURNS = {
+    "3s": VOICE_TESTS / "fixtures" / "en-6930-75918-0013.flac",
+    "9s": ROOT / "e2e" / "fixtures" / "weather-in-paris.wav",
+}
+TURN_RUNS = 5
+
+
+def server_transcriber(model: Dict[str, Any], folder: Path, threads: int) -> Any:
+    """The speech service's own engine for a model, as the image runs it (``datalayer_ai_agents``)."""
+    from datalayer_ai_agents.speech.transcription import (  # the speech image's own
+        FasterWhisperTranscriber,
+        MoonshineTranscriber,
+    )
+
+    if model["engine"] == "faster-whisper":
+        return FasterWhisperTranscriber(folder, model["id"], threads=threads)
+    if model["engine"] == "onnxruntime":
+        return MoonshineTranscriber(folder, model["id"], threads=threads)
+    raise SystemExit(f"{model['id']}: no server engine {model['engine']!r}")
+
+
 def measure_server(
     store: Path,
     voices: Any,
@@ -99,37 +122,34 @@ def measure_server(
     wavs: Dict[str, Path],
     measured: Path,
     threads: int,
+    only: List[str],
 ) -> None:
     """Each server speech-to-text model on the fixtures, as the speech service runs it (VO-47).
 
-    faster-whisper, 8-bit on the CPU with ``threads`` threads, told the
-    language, one clip at a time: what it heard and how long it took.
+    The service's own engines (faster-whisper 8-bit, Moonshine through
+    onnxruntime) on the CPU with ``threads`` threads, told the language, one
+    clip at a time: what it heard and how long it took; then a 3 s and a 9 s
+    English turn, each heard ``TURN_RUNS`` times (decision 18).
     """
-    from faster_whisper import WhisperModel  # the speech image's own
-
     for model in voices.SPEECH_MODEL_CATALOGUE.values():
         if model["task"] != "stt" or model["where"] != ["server"]:
             continue
+        if only and model["id"] not in only:
+            continue
         started = time.monotonic()
-        whisper = WhisperModel(
-            str(store / model["provider"] / model["id"]),
-            device="cpu",
-            compute_type="int8",
-            cpu_threads=threads,
+        engine = server_transcriber(
+            model, store / model["provider"] / model["id"], threads
         )
         load_ms = round((time.monotonic() - started) * 1000)
         by_language = {}
-        for language in ("en", "fr"):
+        for language in [code for code in ("en", "fr") if code in model["languages"]]:
             clips = [clip for clip in fixtures if clip["language"] == language]
             print(f"{model['id']} on {len(clips)} {language} clips", flush=True)
             results = []
             for clip in clips:
                 samples, _ = sf.read(wavs[clip["id"]], dtype="float32")
                 started = time.monotonic()
-                parts, _ = whisper.transcribe(
-                    samples, language=language, vad_filter=True
-                )
-                text = " ".join(part.text.strip() for part in parts).strip()
+                text = engine.transcribe(samples, language=language).text
                 results.append(
                     {
                         "id": clip["id"],
@@ -139,12 +159,40 @@ def measure_server(
                     }
                 )
             by_language[language] = {"load_ms": load_ms, "clips": results}
+        turns = {}
+        if "en" in model["languages"]:
+            for name, path in TURNS.items():
+                samples, rate = sf.read(path, dtype="float32")
+                samples = resample(samples, rate)
+                engine.transcribe(samples, language="en")  # warm
+                took = []
+                for _ in range(TURN_RUNS):
+                    started = time.monotonic()
+                    heard = engine.transcribe(samples, language="en").text
+                    took.append(round((time.monotonic() - started) * 1000))
+                turns[name] = {
+                    "file": path.name,
+                    "audio_s": round(len(samples) / 16000, 2),
+                    "ms": took,
+                    "median_ms": sorted(took)[len(took) // 2],
+                    "heard": heard,
+                }
+                print(
+                    f"{model['id']} {name} turn: {turns[name]['median_ms']} ms",
+                    flush=True,
+                )
+        runtime = (
+            f"faster-whisper, int8 on the CPU, {threads} threads"
+            if model["engine"] == "faster-whisper"
+            else f"onnxruntime {model['dtype']} on the CPU, greedy, {threads} threads"
+        )
         (measured / f"stt-{model['id']}.json").write_text(
             json.dumps(
                 {
                     "model": model["id"],
-                    "runtime": f"faster-whisper, int8 on the CPU, {threads} threads",
+                    "runtime": runtime,
                     "languages": by_language,
+                    **({"turns": turns} if turns else {}),
                 },
                 ensure_ascii=False,
                 indent=2,
@@ -165,7 +213,13 @@ def main() -> int:
     parser.add_argument(
         "--server",
         action="store_true",
-        help="Measure the server's speech-to-text models instead, with faster-whisper (in the speech image)",
+        help="Measure the server's speech-to-text models instead, with the speech service's engines (in the speech image)",
+    )
+    parser.add_argument(
+        "--model",
+        action="append",
+        default=[],
+        help="With --server, only this model (repeatable)",
     )
     parser.add_argument(
         "--threads", type=int, default=4, help="CPU threads for --server"
@@ -195,7 +249,13 @@ def main() -> int:
                 wavs[clip["id"]] = wav
             if args.server:
                 measure_server(
-                    args.store, voices, fixtures, wavs, measured, args.threads
+                    args.store,
+                    voices,
+                    fixtures,
+                    wavs,
+                    measured,
+                    args.threads,
+                    args.model,
                 )
                 return 0
             # --- speech to text, on the device's models ---------------------------------