@datalayer/agent-runtimes 1.5.26 → 1.5.28

package/lib/portable/runMeasures.d.ts

@@ -0,0 +1,46 @@
+/**
+ * How a streamed turn went, measured without its words (MOBILE M-036, §10):
+ * the time to its first streamed token, to its complete answer, how long a
+ * Stop took, the category of a stream that failed, and how often a lost
+ * stream was read back. Never a prompt, an answer, a tool's arguments or an
+ * id — only durations, counts and coarse categories — and an OTLP/JSON body
+ * of them for the platform's metrics ingest.
+ *
+ * Pure: the clock is given.
+ *
+ * @module portable/runMeasures
+ */
+import { type AgUiEvent } from './agui';
+/** Why a stream failed, coarsely. */
+export type StreamFailure = 'refused' | 'cut_short' | 'network' | 'other';
+/** One turn, measured. */
+export type RunMeasure = {
+    surface: 'application' | 'agent';
+    firstTokenMs: number | null;
+    answerMs: number | null;
+    stopLatencyMs: number | null;
+    failure: StreamFailure | null;
+};
+/** The category of a stream's failure; never its message. */
+export declare function failureOf(error: unknown): StreamFailure;
+/** Measures one turn from its start: fed its events, its Stop, its failure; read once it ended. */
+export declare class RunMeter {
+    private readonly surface;
+    private readonly now;
+    private readonly started;
+    private firstToken;
+    private answered;
+    private stopLatency;
+    private failed;
+    constructor(surface: RunMeasure['surface'], now?: () => number);
+    event(event: AgUiEvent): void;
+    stopped(latencyMs: number): void;
+    failure(error: unknown): void;
+    read(): RunMeasure;
+}
+/**
+ * The OTLP/JSON body of turns measured and lost streams read back: gauges of
+ * milliseconds and counters, each point with the surface and, for a failure,
+ * its category — nothing else.
+ */
+export declare function otlpMetricsOf(measures: RunMeasure[], recoveries: number, resource: Record<string, string>, nowMs: number): Record<string, unknown>;

package/lib/portable/runMeasures.js

@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+/**
+ * How a streamed turn went, measured without its words (MOBILE M-036, §10):
+ * the time to its first streamed token, to its complete answer, how long a
+ * Stop took, the category of a stream that failed, and how often a lost
+ * stream was read back. Never a prompt, an answer, a tool's arguments or an
+ * id — only durations, counts and coarse categories — and an OTLP/JSON body
+ * of them for the platform's metrics ingest.
+ *
+ * Pure: the clock is given.
+ *
+ * @module portable/runMeasures
+ */
+import { RUN_CUT_SHORT } from './agui';
+import { RuntimeRefusal } from './sessions';
+/** The category of a stream's failure; never its message. */
+export function failureOf(error) {
+    if (error instanceof RuntimeRefusal)
+        return 'refused';
+    const message = error instanceof Error ? error.message : String(error);
+    if (message === RUN_CUT_SHORT || message.includes(RUN_CUT_SHORT))
+        return 'cut_short';
+    if (error instanceof TypeError ||
+        /network|offline|timed? ?out|fetch/i.test(message))
+        return 'network';
+    return 'other';
+}
+/** Measures one turn from its start: fed its events, its Stop, its failure; read once it ended. */
+export class RunMeter {
+    surface;
+    now;
+    started;
+    firstToken = null;
+    answered = null;
+    stopLatency = null;
+    failed = null;
+    constructor(surface, now = () => Date.now()) {
+        this.surface = surface;
+        this.now = now;
+        this.started = now();
+    }
+    event(event) {
+        if (this.firstToken === null && event.type === 'TEXT_MESSAGE_CONTENT') {
+            this.firstToken = this.now() - this.started;
+        }
+        if (event.type === 'RUN_FINISHED' && this.answered === null) {
+            this.answered = this.now() - this.started;
+        }
+    }
+    stopped(latencyMs) {
+        this.stopLatency = Math.max(0, latencyMs);
+    }
+    failure(error) {
+        this.failed = failureOf(error);
+    }
+    read() {
+        return {
+            surface: this.surface,
+            firstTokenMs: this.firstToken,
+            answerMs: this.failed || this.stopLatency !== null ? null : this.answered,
+            stopLatencyMs: this.stopLatency,
+            failure: this.failed,
+        };
+    }
+}
+const attributes = (values) => Object.entries(values).map(([key, value]) => ({
+    key,
+    value: { stringValue: value },
+}));
+/**
+ * The OTLP/JSON body of turns measured and lost streams read back: gauges of
+ * milliseconds and counters, each point with the surface and, for a failure,
+ * its category — nothing else.
+ */
+export function otlpMetricsOf(measures, recoveries, resource, nowMs) {
+    const time = String(BigInt(Math.trunc(nowMs)) * 1000000n);
+    const gauge = (name, pick) => ({
+        name,
+        unit: 'ms',
+        gauge: {
+            dataPoints: measures.flatMap(measure => {
+                const value = pick(measure);
+                return value === null
+                    ? []
+                    : [
+                        {
+                            asDouble: value,
+                            timeUnixNano: time,
+                            attributes: attributes({ surface: measure.surface }),
+                        },
+                    ];
+            }),
+        },
+    });
+    const failures = new Map();
+    for (const measure of measures) {
+        if (measure.failure) {
+            const key = `${measure.surface}\n${measure.failure}`;
+            failures.set(key, (failures.get(key) ?? 0) + 1);
+        }
+    }
+    const metrics = [
+        gauge('datalayer.client.run.first_token', measure => measure.firstTokenMs),
+        gauge('datalayer.client.run.answer', measure => measure.answerMs),
+        gauge('datalayer.client.run.stop_latency', measure => measure.stopLatencyMs),
+        {
+            name: 'datalayer.client.run.failures',
+            sum: {
+                aggregationTemporality: 1,
+                isMonotonic: true,
+                dataPoints: [...failures].map(([key, count]) => {
+                    const [surface, category] = key.split('\n');
+                    return {
+                        asInt: String(count),
+                        timeUnixNano: time,
+                        attributes: attributes({ surface, category }),
+                    };
+                }),
+            },
+        },
+        {
+            name: 'datalayer.client.run.recoveries',
+            sum: {
+                aggregationTemporality: 1,
+                isMonotonic: true,
+                dataPoints: recoveries > 0
+                    ? [
+                        {
+                            asInt: String(recoveries),
+                            timeUnixNano: time,
+                            attributes: [],
+                        },
+                    ]
+                    : [],
+            },
+        },
+    ].filter(metric => 'gauge' in metric
+        ? metric.gauge.dataPoints.length > 0
+        : metric.sum.dataPoints.length > 0);
+    return {
+        resourceMetrics: [
+            {
+                resource: { attributes: attributes(resource) },
+                scopeMetrics: [
+                    {
+                        scope: { name: '@datalayer/agent-runtimes/portable/runMeasures' },
+                        metrics,
+                    },
+                ],
+            },
+        ],
+    };
+}

package/lib/protocols/ACPAdapter.js

@@ -454,15 +454,20 @@ export class ACPAdapter extends BaseProtocolAdapter {
             });
         }
         if (updateType === 'tool_call_update') {
+            // A call's result, once it says it ended; its progress before then is
+            // not a result. ACP names the call it updates.
             const progress = update;
-            this.emit({
-                type: 'tool-result',
-                toolResult: {
-                    success: progress.status !== 'failed',
-                    result: progress.rawOutput,
-                },
-                timestamp: new Date(),
-            });
+            if (progress.status === 'completed' || progress.status === 'failed') {
+                this.emit({
+                    type: 'tool-result',
+                    toolResult: {
+                        toolCallId: progress.toolCallId ?? '',
+                        success: progress.status === 'completed',
+                        result: progress.rawOutput,
+                    },
+                    timestamp: new Date(),
+                });
+            }
         }
         if (updateType === 'agent_thought_chunk') {
             const thought = textOf(update.content);

package/lib/specs/voices.js

@@ -328,6 +328,53 @@ export const SPEECH_MODEL_CATALOGUE = {
             },
         ],
     },
+    'whisper-large-v3-turbo': {
+        id: 'whisper-large-v3-turbo',
+        version: '0.0.1',
+        name: 'Whisper Large v3 Turbo',
+        description: "Speech to text on Datalayer's servers, on CPU: OpenAI's Whisper large-v3-turbo (809M parameters, 99 languages), converted to CTranslate2 in half precision and run by faster-whisper, its weights quantized to 8 bits when loaded. A recording of up to a minute at a time; the language is detected when the page does not say it.",
+        provider: 'openai',
+        task: 'stt',
+        engine: 'faster-whisper',
+        dtype: 'float16',
+        languages: ['en', 'fr', 'es', 'it', 'pt', 'de', 'nl', 'ja', 'zh', 'hi'],
+        where: ['server'],
+        streaming: false,
+        licence: {
+            weights: 'MIT',
+            code: 'MIT',
+        },
+        attribution: '',
+        upstream: 'https://huggingface.co/openai/whisper-large-v3-turbo',
+        revision: '0a363e9161cbc7ed1431c9597a8ceaf0c4f78fcf',
+        files: [
+            {
+                path: 'config.json',
+                sha256: 'b0253ea6c0d3bea6b1e19e91a02acfd3b53f4467362efcb5a3e6b16c9b3a9b7e',
+                size: 2263,
+            },
+            {
+                path: 'preprocessor_config.json',
+                sha256: '7ccc62c6f2765af1f3b46c00c9b5894426835a05021c8b9c01eecb6dfb542711',
+                size: 340,
+            },
+            {
+                path: 'tokenizer.json',
+                sha256: '297b13372ac43916285644fb9687add3cc62ee2a1adb60da3dc25cc94c1871fd',
+                size: 2710337,
+            },
+            {
+                path: 'vocabulary.json',
+                sha256: 'c69260f2ab26d659b7c398f9a2b2b48ed0df16c3b47d7326782fd9cba71690c1',
+                size: 1068114,
+            },
+            {
+                path: 'model.bin',
+                sha256: 'e76620f83d5f5b69efd3d87e3dc180c1bd21df9fbebacfd4335e5e1efcc018da',
+                size: 1617884929,
+            },
+        ],
+    },
 };
 /** Whether a voice speaks a language: `fr` and `fr-FR` are spoken by a `fr-FR` voice. */
 export function voiceSpeaks(voice, language) {

package/lib/utils/utils.d.ts

@@ -4,6 +4,7 @@
  * @module chat/utils
  */
 import { QueryClient } from '@tanstack/react-query';
+import { sanitizeAssistantContent } from '../portable/adapterEvents';
 import type { ChatMessage } from '../types/messages';
 import type { ProtocolConfig } from '../types/protocol';
 import { type BaseProtocolAdapter } from '../protocols';
@@ -16,14 +17,7 @@ export declare const PRIMER_PORTAL_ROOT_ID = "__primerPortalRoot__";
  * Check if an item is a tool call message
  */
 export declare function isToolCallMessage(item: DisplayItem): item is ToolCallMessage;
-/**
- * Strip low-level protocol/tool-call markup from assistant text content.
- *
- * Some models occasionally leak internal call formats (for example,
- * <calls>...</calls> or <tml:invoke ...>) into natural-language responses.
- * Keep user/system text untouched and sanitize assistant-visible text only.
- */
-export declare function sanitizeAssistantContent(content: string): string;
+export { sanitizeAssistantContent };
 /**
  * Extract text content from a ChatMessage
  */

package/lib/utils/utils.js

@@ -8,6 +8,7 @@
  * @module chat/utils
  */
 import { QueryClient } from '@tanstack/react-query';
+import { sanitizeAssistantContent } from '../portable/adapterEvents';
 import { AGUIAdapter, A2AAdapter, VercelAIAdapter, ACPAdapter, BrowserAgentAdapter, } from '../protocols';
 // ---------------------------------------------------------------------------
 // Constants
@@ -32,32 +33,8 @@ export const PRIMER_PORTAL_ROOT_ID = '__primerPortalRoot__';
 export function isToolCallMessage(item) {
     return 'type' in item && item.type === 'tool-call';
 }
-/**
- * Strip low-level protocol/tool-call markup from assistant text content.
- *
- * Some models occasionally leak internal call formats (for example,
- * <calls>...</calls> or <tml:invoke ...>) into natural-language responses.
- * Keep user/system text untouched and sanitize assistant-visible text only.
- */
-export function sanitizeAssistantContent(content) {
-    if (!content || !content.includes('<')) {
-        return content;
-    }
-    let sanitized = content;
-    // Remove complete blocks first.
-    sanitized = sanitized
-        .replace(/<\s*calls\b[\s\S]*?<\s*\/\s*calls\s*>/gi, '')
-        .replace(/<\s*(?:an:)?tml:invoke\b[\s\S]*?<\s*\/\s*(?:an:)?tml:invoke\s*>/gi, '')
-        .replace(/<\s*(?:an:)?tml:parameter\b[\s\S]*?<\s*\/\s*(?:an:)?tml:parameter\s*>/gi, '');
-    // Remove standalone/open/close tags and malformed tag fragments.
-    sanitized = sanitized
-        .replace(/<\s*\/?\s*(?:an:[a-z_]+|tml:[a-z_]+|calls)\b[^>]*>?/gi, '')
-        .replace(/<\s*\/\s*>/g, '')
-        .replace(/^[ \t]+$/gm, '')
-        .replace(/\n{3,}/g, '\n\n')
-        .trim();
-    return sanitized;
-}
+// What a model leaked of its call markup, taken out: one rule for every client.
+export { sanitizeAssistantContent };
 /**
  * Extract text content from a ChatMessage
  */

package/lib/voice/ComposerVoice.js

@@ -1,6 +1,7 @@
 import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
 import { useMemo, useRef } from 'react';
 import { DeviceHearing } from './hearing';
+import { ServerHearing, hearingPlace } from './serverHearing';
 import { VoiceInput } from './VoiceInput';
 import { VoiceMenu } from './VoiceMenu';
 /** Whether something rides on a run as a spoken message's metadata (VO-27). */
@@ -14,9 +15,30 @@ export function spokenOf(value) {
         : undefined;
 }
 export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled = false, speaking = false, onStopSpeaking, onInterimChange, answers = false, }) {
-    const hearing = useMemo(() => voice.input !== 'off'
-        ? new DeviceHearing(voice.modelsUrl, voice.engines)
-        : undefined, [voice.input, voice.modelsUrl, voice.engines]);
+    // In the browser, or on Datalayer's servers where the application, the
+    // language or the person's detection asks for it (§5, VO-14, VO-34).
+    const place = hearingPlace(voice);
+    const hearing = useMemo(() => {
+        if (voice.input === 'off') {
+            return undefined;
+        }
+        const device = new DeviceHearing(voice.modelsUrl, voice.engines);
+        return place === 'server' && voice.speechUrl
+            ? new ServerHearing(device, {
+                speechUrl: voice.speechUrl,
+                token: voice.token,
+                detect: voice.detect,
+            })
+            : device;
+    }, [
+        voice.input,
+        voice.modelsUrl,
+        voice.engines,
+        place,
+        voice.speechUrl,
+        voice.token,
+        voice.detect,
+    ]);
     // What the box held when the microphone opened: the words heard go after it.
     const base = useRef('');
     const afterBase = (words) => [base.current, words].filter(Boolean).join(' ');

package/lib/voice/VoiceInput.d.ts

@@ -18,7 +18,7 @@
  * @module voice/VoiceInput
  */
 import type { JSX } from 'react';
-import type { DeviceHearing } from './hearing';
+import type { Hearing } from './hearing';
 import type { Transcript } from './types';
 /** What the microphone is doing, said in words (VO-67, VO-68). */
 export type ListeningState = 'idle' | 'asking' | 'loading' | 'listening' | 'hearing';
@@ -27,7 +27,7 @@ export declare const INTERIM_MS = 700;
 /** The longest a microphone listens in one go, in ms: then it stops by itself. */
 export declare const LONGEST_MS = 60000;
 export interface VoiceInputProps {
-    hearing: DeviceHearing;
+    hearing: Hearing;
     /** The language listened to, BCP 47. */
     language: string;
     /** What was said, once heard. */

package/lib/voice/VoiceInput.js

@@ -5,7 +5,7 @@ import { Box } from '@datalayer/primer-addons';
 import { MicrophoneIcon } from '@datalayer/icons-react';
 import { MuteIcon } from '@primer/octicons-react';
 import { MicrophoneRefused, openMicrophone } from './capture';
-import { CONSENT_SENTENCE, consent, consented } from './consent';
+import { CONSENT_SENTENCE, SERVER_CONSENT_SENTENCE, consent, consentKeyFor, consented, } from './consent';
 const WORDS = {
     idle: '',
     asking: '',
@@ -49,7 +49,7 @@ export function VoiceInput({ hearing, language, onTranscript, onInterim, silence
         if (disabled || stateRef.current !== 'idle') {
             return;
         }
-        if (!consented(consentKey)) {
+        if (!consented(consentKeyFor(consentKey, hearing.where))) {
             setState('asking');
             return;
         }
@@ -277,8 +277,10 @@ export function VoiceInput({ hearing, language, onTranscript, onInterim, silence
                                 void start();
                             }
                         }
-                    } }) }), state === 'asking' && (_jsxs(Box, { role: "dialog", "aria-label": "Talk instead of typing", "data-voice-consent": "", position: "absolute", bottom: "100%", right: 0, mb: 2, width: 280, p: 3, bg: "canvas.overlay", border: "1px solid", borderColor: "border.default", borderRadius: 2, boxShadow: "shadow.large", zIndex: 10, children: [_jsx(Text, { as: "p", sx: { fontSize: 1, mb: 2 }, children: CONSENT_SENTENCE }), _jsxs(Box, { display: "flex", gap: 2, justifyContent: "flex-end", children: [_jsx(Button, { size: "small", onClick: () => setState('idle'), children: "Not now" }), _jsx(Button, { size: "small", variant: "primary", "data-voice-consent-yes": "", onClick: () => {
-                                    consent(consentKey);
+                    } }) }), state === 'asking' && (_jsxs(Box, { role: "dialog", "aria-label": "Talk instead of typing", "data-voice-consent": "", position: "absolute", bottom: "100%", right: 0, mb: 2, width: 280, p: 3, bg: "canvas.overlay", border: "1px solid", borderColor: "border.default", borderRadius: 2, boxShadow: "shadow.large", zIndex: 10, children: [_jsx(Text, { as: "p", sx: { fontSize: 1, mb: 2 }, children: hearing.where === 'server'
+                            ? SERVER_CONSENT_SENTENCE
+                            : CONSENT_SENTENCE }), _jsxs(Box, { display: "flex", gap: 2, justifyContent: "flex-end", children: [_jsx(Button, { size: "small", onClick: () => setState('idle'), children: "Not now" }), _jsx(Button, { size: "small", variant: "primary", "data-voice-consent-yes": "", onClick: () => {
+                                    consent(consentKeyFor(consentKey, hearing.where));
                                     // Asked, and said yes: listen now, without another press.
                                     stateRef.current = 'idle';
                                     setState('idle');

package/lib/voice/VoiceMenu.js

@@ -20,6 +20,7 @@ export function VoiceMenu({ voice, answers, disabled = false, }) {
     const [devices, setDevices] = useState([]);
     const choose = (key, value) => keep(withVoiceSetting(settings, key, value, catalogueVoiceSpeaks));
     const onDevice = Boolean(settings.onDeviceOnly);
+    const detect = Boolean(settings.detectLanguage);
     const voices = voicesSpeaking(voice.language);
     const named = devices.filter(device => device.label);
     return (_jsxs(ActionMenu, { onOpenChange: open => {
@@ -27,6 +28,6 @@ export function VoiceMenu({ voice, answers, disabled = false, }) {
                 void microphones().then(setDevices);
             }
         }, children: [_jsx(ActionMenu.Anchor, { children: _jsx(IconButton, { icon: GearIcon, size: "small", variant: "invisible", "aria-label": "Voice settings", "data-voice-menu": "", disabled: disabled }) }), _jsx(ActionMenu.Overlay, { width: "medium", children: _jsxs(ActionList, { children: [_jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Language" }), voiceLanguages().map(language => (_jsx(ActionList.Item, { selected: language.value === voice.language, onSelect: () => choose('language', language.value), "data-voice-language": language.value, children: language.label }, language.value)))] }), answers && !onDevice ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Answers heard" }), OUTPUT_CHOICES.map(output => (_jsx(ActionList.Item, { selected: output.value === voice.output, onSelect: () => choose('output', output.value), "data-voice-output": output.value, children: output.label }, output.value)))] }), voices.length > 0 ? (_jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Voice" }), voices.map(spoken => (_jsx(ActionList.Item, { selected: spoken.id === voice.voice, onSelect: () => choose('voice', spoken.id), "data-voice-voice": spoken.id, children: spoken.name }, spoken.id)))] })) : null, _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Speed" }), RATE_CHOICES.map(rate => (_jsx(ActionList.Item, { selected: rate.value === (voice.rate ?? 1), onSelect: () => choose('rate', rate.value), "data-voice-rate": rate.value, children: rate.label }, rate.value)))] }), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Loudness" }), VOLUME_CHOICES.map(volume => (_jsx(ActionList.Item, { selected: volume.value === (voice.volume ?? 1), onSelect: () => choose('volume', volume.value), "data-voice-volume": volume.value, children: volume.label }, volume.value)))] })] })) : null, named.length > 1 ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "single", children: [_jsx(ActionList.GroupHeading, { children: "Microphone" }), named.map(device => (_jsx(ActionList.Item, { selected: device.deviceId ===
-                                                (voice.microphoneId ?? named[0].deviceId), onSelect: () => choose('microphoneId', device.deviceId), "data-voice-microphone": device.deviceId, children: device.label }, device.deviceId)))] })] })) : null, _jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "multiple", children: [_jsxs(ActionList.Item, { selected: Boolean(voice.sendWhatISay), onSelect: () => choose('sendWhatISay', !voice.sendWhatISay), "data-voice-send-what-i-say": "", children: ["Send what I say", _jsx(ActionList.Description, { variant: "block", children: "Sent as soon as it is heard, not left in the box." })] }), _jsxs(ActionList.Item, { selected: onDevice, onSelect: () => choose('onDeviceOnly', !onDevice), "data-voice-on-device": "", children: ["Keep speech on this device", _jsx(ActionList.Description, { variant: "block", children: "Nothing is sent to Datalayer\u2019s speech service: you are heard in the browser, answers are not spoken." })] })] }), Object.keys(settings).length > 0 ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsx(ActionList.Item, { onSelect: () => keep({}), "data-voice-settings-reset": "", children: "Use the application\u2019s" })] })) : null] }) })] }));
+                                                (voice.microphoneId ?? named[0].deviceId), onSelect: () => choose('microphoneId', device.deviceId), "data-voice-microphone": device.deviceId, children: device.label }, device.deviceId)))] })] })) : null, _jsx(ActionList.Divider, {}), _jsxs(ActionList.Group, { selectionVariant: "multiple", children: [_jsxs(ActionList.Item, { selected: Boolean(voice.sendWhatISay), onSelect: () => choose('sendWhatISay', !voice.sendWhatISay), "data-voice-send-what-i-say": "", children: ["Send what I say", _jsx(ActionList.Description, { variant: "block", children: "Sent as soon as it is heard, not left in the box." })] }), voice.speechUrl && !onDevice ? (_jsxs(ActionList.Item, { selected: detect, onSelect: () => choose('detectLanguage', !detect), "data-voice-detect": "", children: ["Detect the language I speak", _jsx(ActionList.Description, { variant: "block", children: "Heard on Datalayer\u2019s servers by Whisper, which finds the language; nothing is kept there." })] })) : null, _jsxs(ActionList.Item, { selected: onDevice, onSelect: () => choose('onDeviceOnly', !onDevice), "data-voice-on-device": "", children: ["Keep speech on this device", _jsx(ActionList.Description, { variant: "block", children: "Nothing is sent to Datalayer\u2019s speech service: you are heard in the browser, answers are not spoken." })] })] }), Object.keys(settings).length > 0 ? (_jsxs(_Fragment, { children: [_jsx(ActionList.Divider, {}), _jsx(ActionList.Item, { onSelect: () => keep({}), "data-voice-settings-reset": "", children: "Use the application\u2019s" })] })) : null] }) })] }));
 }
 export default VoiceMenu;

package/lib/voice/chatVoice.d.ts

@@ -44,10 +44,11 @@ export declare function chatVoiceOf(spec: AppVoiceSpec | undefined, place: ChatV
  * A chat's voice asked of the speech service as the chat itself speaks to
  * its runtime (VO-37, VO-19): the token its host gave, else the chat's own —
  * a visitor's token minted from ai-inference, an embed's token, or the
- * person's — and the deployment it runs, which an embed's token is checked
- * against. One token for the conversation and its voice, never a second.
+ * person's. One token for the conversation and its voice, never a second.
+ * An embed's deployment is the one its token names (decision 16): the page
+ * says none.
  */
-export declare function voiceAskedAs(voice: ChatVoice | undefined, token: string | undefined, deployment: string | undefined): ChatVoice | undefined;
+export declare function voiceAskedAs(voice: ChatVoice | undefined, token: string | undefined): ChatVoice | undefined;
 /** A chat's voice with every place said: what the chat reads. */
 export type SettledVoice = Required<Pick<ChatVoice, 'language' | 'input' | 'output' | 'voice' | 'modelsUrl' | 'engines' | 'silenceMs'>> & Omit<ChatVoice, 'language' | 'input' | 'output' | 'voice' | 'modelsUrl' | 'engines' | 'silenceMs'>;
 /** How long a silence after speech stops a microphone clicked on, when unsaid (VO-10). */

package/lib/voice/chatVoice.js

@@ -57,26 +57,23 @@ export function chatVoiceOf(spec, place) {
         speechUrl: place.speechUrl,
         token: place.token,
         consentKey: place.consentKey,
+        where: spec.where ?? 'auto',
     };
 }
 /**
  * A chat's voice asked of the speech service as the chat itself speaks to
  * its runtime (VO-37, VO-19): the token its host gave, else the chat's own —
  * a visitor's token minted from ai-inference, an embed's token, or the
- * person's — and the deployment it runs, which an embed's token is checked
- * against. One token for the conversation and its voice, never a second.
+ * person's. One token for the conversation and its voice, never a second.
+ * An embed's deployment is the one its token names (decision 16): the page
+ * says none.
  */
-export function voiceAskedAs(voice, token, deployment) {
+export function voiceAskedAs(voice, token) {
     if (!voice) {
         return undefined;
     }
     const asked = voice.token || token;
-    const runs = voice.deployment || deployment;
-    return {
-        ...voice,
-        ...(asked ? { token: asked } : {}),
-        ...(runs ? { deployment: runs } : {}),
-    };
+    return asked ? { ...voice, token: asked } : { ...voice };
 }
 /** How long a silence after speech stops a microphone clicked on, when unsaid (VO-10). */
 export const SILENCE_MS = 1500;

package/lib/voice/consent.d.ts

@@ -4,3 +4,11 @@ export declare function consented(app: string): boolean;
 export declare function consent(app: string): void;
 /** The sentence said before the first use: where the audio goes (VO-61). */
 export declare const CONSENT_SENTENCE = "What you say is transcribed on this device: the sound never leaves it, and is not kept. The words go in the message box, for you to send.";
+/**
+ * The sentence said before the first use when what is said is heard on
+ * Datalayer's servers (VO-34, VO-61): asked apart from the device's, since
+ * a yes to one is not a yes to the other.
+ */
+export declare const SERVER_CONSENT_SENTENCE = "What you say is sent to Datalayer\u2019s servers to be transcribed: the sound and the words are not kept there. The words go in the message box, for you to send.";
+/** Whose consent is remembered for hearing in a place: the application's, apart for the server. */
+export declare function consentKeyFor(app: string, where: 'device' | 'server'): string;

package/lib/voice/consent.js

@@ -32,3 +32,13 @@ export function consent(app) {
 }
 /** The sentence said before the first use: where the audio goes (VO-61). */
 export const CONSENT_SENTENCE = 'What you say is transcribed on this device: the sound never leaves it, and is not kept. The words go in the message box, for you to send.';
+/**
+ * The sentence said before the first use when what is said is heard on
+ * Datalayer's servers (VO-34, VO-61): asked apart from the device's, since
+ * a yes to one is not a yes to the other.
+ */
+export const SERVER_CONSENT_SENTENCE = 'What you say is sent to Datalayer’s servers to be transcribed: the sound and the words are not kept there. The words go in the message box, for you to send.';
+/** Whose consent is remembered for hearing in a place: the application's, apart for the server. */
+export function consentKeyFor(app, where) {
+    return where === 'server' ? `${app}:server` : app;
+}

package/lib/voice/hearing.d.ts

@@ -22,15 +22,30 @@ export interface HeardSoFar {
     /** How long it has been silent since the speech last ended, in ms. */
     silentMs: number;
 }
+/**
+ * What hears a press, wherever it runs (§5): in the page (`DeviceHearing`)
+ * or on Datalayer's servers (`ServerHearing`).
+ */
+export interface Hearing {
+    /** Where the sound is heard: what the consent says, and a transcript's `where`. */
+    readonly where: 'device' | 'server';
+    /** Load what hears a language, saying how far it has come. */
+    ready(language: string, progress?: HearingProgress): Promise<void>;
+    /** What was said so far while the microphone stays open (VO-11). */
+    hearSoFar(audio: Float32Array, language: string): Promise<HeardSoFar>;
+    /** What was said in a press, or `undefined` when nothing was. */
+    hear(audio: Float32Array, language: string): Promise<Transcript | undefined>;
+}
 /** Joins the speech a VAD found, in order. */
 export declare function joinSegments(segments: Float32Array[]): Float32Array;
 /**
  * The page's hearing: loaded once, on first use, from Datalayer's origin
  * (`modelsUrl`), every file checked against its pin.
  */
-export declare class DeviceHearing {
+export declare class DeviceHearing implements Hearing {
     private readonly modelsUrl;
     private readonly engines;
+    readonly where: "device";
     private recognizers;
     private vad?;
     constructor(modelsUrl: string, engines: VoiceEngines);
@@ -40,6 +55,13 @@ export declare class DeviceHearing {
     sizeFor(language: string): number;
     /** Load what hears a language, saying how far it has come. */
     ready(language: string, progress?: HearingProgress): Promise<void>;
+    /** Load only what finds speech (Silero VAD): what hearing on the server needs here. */
+    readyToFindSpeech(progress?: HearingProgress): Promise<void>;
+    /** The speech in what was heard, and how long it has been silent since it last ended, in ms. */
+    findSpeech(audio: Float32Array): Promise<{
+        speech: Float32Array;
+        silentMs: number;
+    }>;
     private recognizer;
     private activity;
     /** The speech Silero VAD finds in what was heard, and where the last ended, in ms. */

package/lib/voice/hearing.js

@@ -45,6 +45,7 @@ export function joinSegments(segments) {
 export class DeviceHearing {
     modelsUrl;
     engines;
+    where = 'device';
     recognizers = new Map();
     vad;
     constructor(modelsUrl, engines) {
@@ -67,6 +68,22 @@ export class DeviceHearing {
     async ready(language, progress) {
         await Promise.all([this.recognizer(language, progress), this.activity()]);
     }
+    /** Load only what finds speech (Silero VAD): what hearing on the server needs here. */
+    async readyToFindSpeech(progress) {
+        const total = downloadSize('silero-vad');
+        progress?.(0, total);
+        await this.activity();
+        progress?.(total, total);
+    }
+    /** The speech in what was heard, and how long it has been silent since it last ended, in ms. */
+    async findSpeech(audio) {
+        const { speech, lastEnd } = await this.speechIn(audio);
+        const heardMs = (audio.length / HEARING_RATE) * 1000;
+        return {
+            speech,
+            silentMs: lastEnd === undefined ? 0 : Math.max(0, heardMs - lastEnd),
+        };
+    }
     recognizer(language, progress) {
         const model = this.modelFor(language);
         let loading = this.recognizers.get(model.id);
@@ -163,9 +180,7 @@ export class DeviceHearing {
      * since the speech ended — what stops a click to talk (VO-10).
      */
     async hearSoFar(audio, language) {
-        const { speech, lastEnd } = await this.speechIn(audio);
-        const heardMs = (audio.length / HEARING_RATE) * 1000;
-        const silentMs = lastEnd === undefined ? 0 : Math.max(0, heardMs - lastEnd);
+        const { speech, silentMs } = await this.findSpeech(audio);
         if (speech.length < HEARING_RATE * 0.2) {
             return { text: '', spoke: false, silentMs };
         }