@datalayer/agent-runtimes 1.5.28 → 1.5.30

package/lib/voice/chatVoice.js

@@ -35,7 +35,8 @@ export const DATALAYER_VOICE_ENGINES = {
  * - **Listening**: a microphone in the composer — click it and speak, the
  *   words written in the box as they are heard, stopped by a second click
  *   or when the person stops speaking; or held, and let go (VO-10, VO-11).
- *   `hands_free` (VO-12) is not built: `checkAppspec` refuses it.
+ *   `hands_free` (VO-12): the microphone, and *Talk with it* beside it; it
+ *   implies answers said as they are written.
  * - **Speaking**: its answers said by Datalayer's speech service, on request
  *   or always (VO-20, VO-21); refused here when its host says no service.
  */
@@ -49,8 +50,9 @@ export function chatVoiceOf(spec, place) {
     const language = spec.language || place.language || 'en-US';
     return {
         language,
-        input: spec.input === 'off' ? 'off' : 'push_to_talk',
-        output: spec.output,
+        input: spec.input,
+        // Hands-free implies the answers heard as they are written (VO-12).
+        output: spec.input === 'hands_free' ? 'always' : spec.output,
         voice: spec.voice || voiceFor(language)?.id || '',
         modelsUrl: place.modelsUrl || DATALAYER_SPEECH_MODELS_URL,
         engines: DATALAYER_VOICE_ENGINES,

package/lib/voice/handsFree.d.ts

@@ -0,0 +1,58 @@
+/**
+ * A conversation by voice, hands-free (VOICE.md VO-12, VO-13): what the
+ * composer is doing at every moment, and when a turn ends or the person
+ * speaks over the agent. Pure: the component (`HandsFree`) keeps the
+ * microphone and the clock, this decides.
+ *
+ * - **Listening**: the microphone is open; the turn ends after
+ *   `turnSilenceMs` of silence once the person has spoken (700 ms unless
+ *   the application or the person says otherwise), and is sent.
+ * - **Thinking**: the turn was sent and the agent is answering, nothing
+ *   said yet.
+ * - **Speaking**: the answer is being said. The microphone stays open:
+ *   speech heard now is barge-in (VO-13) — the voice stops at once, the run
+ *   is stopped when the answer is still being written, and what was heard
+ *   starts the next turn.
+ * - **Paused**: the person paused it (`Esc`, *Pause*), or nothing was said
+ *   for `IDLE_PAUSE_MS`; nothing is heard until resumed.
+ *
+ * @module voice/handsFree
+ */
+/** What hands-free is doing, shown at all times (VO-12). */
+export type HandsFreeState = 'off' | 'listening' | 'thinking' | 'speaking' | 'paused';
+/** The word for each state, as the composer shows it. */
+export declare const HANDS_FREE_WORDS: Record<HandsFreeState, string>;
+/** The silence after speech that ends a turn, in ms, unless said otherwise (VO-12). */
+export declare const TURN_SILENCE_MS = 700;
+/** Listening with nothing said for this long pauses it: a microphone is not left open on an empty room. */
+export declare const IDLE_PAUSE_MS: number;
+/** What decides the state. */
+export interface HandsFreeInputs {
+    /** The toggle is on. */
+    on: boolean;
+    /** The person paused it, or the room went quiet. */
+    paused: boolean;
+    /** The agent's answer is being said. */
+    speaking: boolean;
+    /** A run is in progress: the agent is answering. */
+    running: boolean;
+}
+/** The state for what is known: speaking wins over thinking, pausing over both. */
+export declare function handsFreeState(inputs: HandsFreeInputs): HandsFreeState;
+/** What the microphone heard so far, as the hearing says it (`HeardSoFar`). */
+export interface HeardSoFarLike {
+    spoke: boolean;
+    silentMs: number;
+}
+/**
+ * Whether the turn ended: the person spoke, and has been silent for the
+ * turn's silence since (VO-12).
+ */
+export declare function turnEnded(heard: HeardSoFarLike, turnSilenceMs?: number): boolean;
+/**
+ * Whether the person spoke over the agent (VO-13): speech found since the
+ * agent began answering, while it thinks or speaks. The microphone is opened
+ * anew when the agent begins, so what it holds is the person's, not the
+ * answer's echo — the browser's echo cancellation keeps that out.
+ */
+export declare function bargedIn(state: HandsFreeState, heard: HeardSoFarLike): boolean;

package/lib/voice/handsFree.js

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/** The word for each state, as the composer shows it. */
+export const HANDS_FREE_WORDS = {
+    off: 'Talk with it',
+    listening: 'Listening…',
+    thinking: 'Thinking…',
+    speaking: 'Speaking…',
+    paused: 'Paused',
+};
+/** The silence after speech that ends a turn, in ms, unless said otherwise (VO-12). */
+export const TURN_SILENCE_MS = 700;
+/** Listening with nothing said for this long pauses it: a microphone is not left open on an empty room. */
+export const IDLE_PAUSE_MS = 5 * 60_000;
+/** The state for what is known: speaking wins over thinking, pausing over both. */
+export function handsFreeState(inputs) {
+    if (!inputs.on) {
+        return 'off';
+    }
+    if (inputs.paused) {
+        return 'paused';
+    }
+    if (inputs.speaking) {
+        return 'speaking';
+    }
+    if (inputs.running) {
+        return 'thinking';
+    }
+    return 'listening';
+}
+/**
+ * Whether the turn ended: the person spoke, and has been silent for the
+ * turn's silence since (VO-12).
+ */
+export function turnEnded(heard, turnSilenceMs = TURN_SILENCE_MS) {
+    return heard.spoke && heard.silentMs >= turnSilenceMs;
+}
+/**
+ * Whether the person spoke over the agent (VO-13): speech found since the
+ * agent began answering, while it thinks or speaks. The microphone is opened
+ * anew when the agent begins, so what it holds is the person's, not the
+ * answer's echo — the browser's echo cancellation keeps that out.
+ */
+export function bargedIn(state, heard) {
+    return (state === 'speaking' || state === 'thinking') && heard.spoke;
+}

package/lib/voice/index.d.ts

@@ -14,6 +14,8 @@ export * from './pinned';
 export * from './capture';
 export * from './hearing';
 export * from './serverHearing';
+export * from './handsFree';
+export { HandsFree, type HandsFreeProps } from './HandsFree';
 export * from './speaker';
 export * from './consent';
 export * from './settings';
@@ -47,8 +49,14 @@ export type { VoiceSpeaker } from './useVoiceSpeaker';
 export interface ChatVoice {
     /** The language listened to and spoken, BCP 47. */
     language: string;
-    /** `push_to_talk`: the microphone in the composer; `off`: none. Hands-free is Phase V3. */
-    input: 'off' | 'push_to_talk';
+    /**
+     * `push_to_talk`: the microphone in the composer; `hands_free`: that, and
+     * *Talk with it* beside it — a conversation by voice (VO-12, VO-13);
+     * `off`: none.
+     */
+    input: 'off' | 'push_to_talk' | 'hands_free';
+    /** The silence after speech that ends a hands-free turn, in ms (VO-12): 700 when unsaid. */
+    turnSilenceMs?: number;
     /**
      * When answers are heard: `always`, as they are written; `on_request`, a
      * *Read aloud* under each answer (VO-21); or `off`, the default.

package/lib/voice/index.js

@@ -8,6 +8,8 @@ export * from './pinned';
 export * from './capture';
 export * from './hearing';
 export * from './serverHearing';
+export * from './handsFree';
+export { HandsFree } from './HandsFree';
 export * from './speaker';
 export * from './consent';
 export * from './settings';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@datalayer/agent-runtimes",
-    "version": "1.5.28",
+    "version": "1.5.30",
     "type": "module",
     "workspaces": [
         ".",

package/scripts/voice/measure.py

@@ -92,6 +92,67 @@ def transcribe(
     return json.loads(done.stdout.strip().splitlines()[-1])
 
 
+def measure_server(
+    store: Path,
+    voices: Any,
+    fixtures: List[Dict[str, Any]],
+    wavs: Dict[str, Path],
+    measured: Path,
+    threads: int,
+) -> None:
+    """Each server speech-to-text model on the fixtures, as the speech service runs it (VO-47).
+
+    faster-whisper, 8-bit on the CPU with ``threads`` threads, told the
+    language, one clip at a time: what it heard and how long it took.
+    """
+    from faster_whisper import WhisperModel  # the speech image's own
+
+    for model in voices.SPEECH_MODEL_CATALOGUE.values():
+        if model["task"] != "stt" or model["where"] != ["server"]:
+            continue
+        started = time.monotonic()
+        whisper = WhisperModel(
+            str(store / model["provider"] / model["id"]),
+            device="cpu",
+            compute_type="int8",
+            cpu_threads=threads,
+        )
+        load_ms = round((time.monotonic() - started) * 1000)
+        by_language = {}
+        for language in ("en", "fr"):
+            clips = [clip for clip in fixtures if clip["language"] == language]
+            print(f"{model['id']} on {len(clips)} {language} clips", flush=True)
+            results = []
+            for clip in clips:
+                samples, _ = sf.read(wavs[clip["id"]], dtype="float32")
+                started = time.monotonic()
+                parts, _ = whisper.transcribe(
+                    samples, language=language, vad_filter=True
+                )
+                text = " ".join(part.text.strip() for part in parts).strip()
+                results.append(
+                    {
+                        "id": clip["id"],
+                        "heard": text,
+                        "ms": round((time.monotonic() - started) * 1000),
+                        "audio_s": round(len(samples) / 16000, 2),
+                    }
+                )
+            by_language[language] = {"load_ms": load_ms, "clips": results}
+        (measured / f"stt-{model['id']}.json").write_text(
+            json.dumps(
+                {
+                    "model": model["id"],
+                    "runtime": f"faster-whisper, int8 on the CPU, {threads} threads",
+                    "languages": by_language,
+                },
+                ensure_ascii=False,
+                indent=2,
+            )
+            + "\n"
+        )
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
     parser.add_argument("--store", type=Path, required=True)
@@ -101,6 +162,14 @@ def main() -> int:
         default=None,
         help="Where @huggingface/transformers is installed, when not in this checkout",
     )
+    parser.add_argument(
+        "--server",
+        action="store_true",
+        help="Measure the server's speech-to-text models instead, with faster-whisper (in the speech image)",
+    )
+    parser.add_argument(
+        "--threads", type=int, default=4, help="CPU threads for --server"
+    )
     args = parser.parse_args()
     voices = catalogue()
     env = dict(os.environ)
@@ -124,6 +193,11 @@ def main() -> int:
                 wav = Path(scratch) / f"{clip['id']}.wav"
                 sf.write(wav, resample(samples, rate), 16000, subtype="PCM_16")
                 wavs[clip["id"]] = wav
+            if args.server:
+                measure_server(
+                    args.store, voices, fixtures, wavs, measured, args.threads
+                )
+                return 0
             # --- speech to text, on the device's models ---------------------------------
             for model in voices.SPEECH_MODEL_CATALOGUE.values():
                 if model["task"] != "stt" or "device" not in model["where"]: