@datalayer/agent-runtimes 1.3.63 → 1.3.64

package/lib/protocols/VercelAIAdapter.js

@@ -187,6 +187,11 @@ export class VercelAIAdapter extends BaseProtocolAdapter {
                         id: msg.id,
                         role: msg.role,
                         parts,
+                        // How a message was heard, when it was said (VOICE.md VO-27):
+                        // the runtime keeps the turn marked spoken.
+                        ...(msg.role === 'user' && msg.metadata?.input === 'voice'
+                            ? { metadata: msg.metadata }
+                            : {}),
                     };
                 });
                 // For tool-result messages, add them as separate tool parts

package/lib/specs/apps.js

@@ -67,6 +67,14 @@ export const ACCOUNTING_APP_0_0_1 = {
         settings: [],
         components: [],
         assistant: 'wizard',
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -196,6 +204,14 @@ export const CUSTOMER_INTERVIEW_APP_0_0_1 = {
         ],
         components: [],
         assistant: 'cat',
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -303,6 +319,14 @@ export const DATA_QUALITY_APP_0_0_1 = {
             'TextField',
             'Button',
         ],
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -450,6 +474,14 @@ export const DECIDE_APP_0_0_1 = {
         settings: [],
         components: [],
         assistant: 'wizard',
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -593,6 +625,14 @@ export const INBOX_TRIAGE_APP_0_0_1 = {
         ],
         settings: [],
         components: [],
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.9,
@@ -722,6 +762,14 @@ export const MODEL_CHOICE_APP_0_0_1 = {
             'TextField',
             'Button',
         ],
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -1046,6 +1094,14 @@ export const PIPELINE_REPORT_APP_0_0_1 = {
             composedAt: '',
         },
         assistant: 'eyes',
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.9,
@@ -1332,6 +1388,14 @@ export const QUOTE_CALCULATOR_APP_0_0_1 = {
             composedBy: 'template',
             composedAt: '',
         },
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 1.0,
@@ -1559,6 +1623,14 @@ export const REPORT_FROM_A_FILE_APP_0_0_1 = {
             composedAt: '',
         },
         assistant: 'wizard',
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -1672,6 +1744,14 @@ export const SALES_APP_0_0_1 = {
         settings: [],
         components: [],
         assistant: 'paperclip',
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -1778,6 +1858,14 @@ export const SHIP_OR_FIX_APP_0_0_1 = {
             'TextField',
             'Button',
         ],
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -1945,6 +2033,14 @@ export const SUPPLIER_COMPARISON_APP_0_0_1 = {
             'TextField',
             'Button',
         ],
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -2230,6 +2326,14 @@ export const SUPPORT_DESK_APP_0_0_1 = {
             composedAt: '',
         },
         assistant: 'paperclip',
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,
@@ -2356,6 +2460,14 @@ export const WEB_RESEARCH_APP_0_0_1 = {
             },
         ],
         components: [],
+        voice: {
+            enabled: false,
+            input: 'push_to_talk',
+            output: 'on_request',
+            voice: '',
+            language: '',
+            where: 'auto',
+        },
     },
     tests: {
         readyAt: 0.8,

package/lib/specs/appspecSchema.js

@@ -326,6 +326,10 @@ export const APPSPEC_SCHEMA = {
                     description: "The character its floating assistant shows, by the id a plugin contributes it under (lowercase letters and digits, words joined by a hyphen): Datalayer's are `paperclip`, `wizard`, `cat` and `eyes`. The paper clip when unsaid; an id no enabled plugin contributes is refused where the plugins are known, the runtime and the page. Said here, it wins over a person's own choice in their settings",
                     title: 'Assistant',
                 },
+                voice: {
+                    $ref: '#/$defs/AppVoice',
+                    description: 'Its voice: whether it listens and speaks, with which voice, in which language (off unless said)',
+                },
             },
             title: 'AppInterface',
             type: 'object',
@@ -716,6 +720,48 @@ export const APPSPEC_SCHEMA = {
             title: 'AppVerified',
             type: 'object',
         },
+        AppVoice: {
+            additionalProperties: false,
+            description: 'Its voice (VOICE.md VO-41): whether it listens, whether it speaks, with which voice, in which language.\n\nOff unless said. What is said becomes a message, and what is heard is the\nanswer the conversation shows: the text stays the truth.',
+            properties: {
+                enabled: {
+                    default: false,
+                    description: 'Whether it has a voice at all; off unless said',
+                    title: 'Enabled',
+                    type: 'boolean',
+                },
+                input: {
+                    $ref: '#/$defs/VoiceInput',
+                    default: 'push_to_talk',
+                    description: '`off`, `push_to_talk` (hold a key or the microphone, speak, let go) or `hands_free`',
+                },
+                output: {
+                    $ref: '#/$defs/VoiceOutput',
+                    default: 'on_request',
+                    description: 'When its answers are heard: `off`, `on_request` (a Read aloud on each answer) or `always`',
+                },
+                voice: {
+                    default: '',
+                    description: "The voice it speaks with, an id of the voice catalogue (`kokoro-af-heart`); the language's first when unsaid",
+                    title: 'Voice',
+                    type: 'string',
+                },
+                language: {
+                    default: '',
+                    description: "The language it listens and speaks in, BCP 47 (`en-US`, `fr-FR`); the person's when unsaid",
+                    pattern: '^(?:[a-z]{2,3}(?:-[A-Z]{2})?)?$',
+                    title: 'Language',
+                    type: 'string',
+                },
+                where: {
+                    $ref: '#/$defs/VoiceWhere',
+                    default: 'auto',
+                    description: "Where its speech runs: `auto`, `device` (the person's browser) or `server` (Datalayer's)",
+                },
+            },
+            title: 'AppVoice',
+            type: 'object',
+        },
         Behaviour: {
             description: 'What an application does when it meets an action: the four a person chooses from.',
             enum: ['do_it', 'if_asked', 'ask_first', 'leave_to_me'],
@@ -791,6 +837,7 @@ export const APPSPEC_SCHEMA = {
                 'sources',
                 'outputs',
                 'feedback',
+                'audio',
             ],
             title: 'RecordItem',
             type: 'string',
@@ -813,6 +860,24 @@ export const APPSPEC_SCHEMA = {
             title: 'Visibility',
             type: 'string',
         },
+        VoiceInput: {
+            description: 'How a person talks to the application (VOICE.md VO-10, VO-12).',
+            enum: ['off', 'push_to_talk', 'hands_free'],
+            title: 'VoiceInput',
+            type: 'string',
+        },
+        VoiceOutput: {
+            description: 'When its answers are heard (VO-21).',
+            enum: ['off', 'on_request', 'always'],
+            title: 'VoiceOutput',
+            type: 'string',
+        },
+        VoiceWhere: {
+            description: "Where its speech runs: in the person's browser, on Datalayer's servers, or the better of the two (§5).",
+            enum: ['auto', 'device', 'server'],
+            title: 'VoiceWhere',
+            type: 'string',
+        },
     },
     additionalProperties: false,
     description: 'Specification for an application.',

package/lib/specs/index.d.ts

@@ -14,6 +14,7 @@ export * from './memory';
 export * from './models';
 export * from './modelProviders';
 export * from './notifications';
+export * from './voices';
 export * from './outputs';
 export * from './skills';
 export * from './backendTools';

package/lib/specs/index.js

@@ -18,6 +18,7 @@ export * from './memory';
 export * from './models';
 export * from './modelProviders';
 export * from './notifications';
+export * from './voices';
 export * from './outputs';
 export * from './skills';
 export * from './backendTools';

package/lib/specs/voices.d.ts

@@ -0,0 +1,72 @@
+/**
+ * The voice catalogue (VOICE.md VO-40): voices and speech models.
+ *
+ * This file is AUTO-GENERATED from agentspecs (voices, speech-models).
+ * DO NOT EDIT MANUALLY - run 'make specs' to regenerate.
+ *
+ * @module specs/voices
+ */
+/** Where a step of speech runs: the person's browser, or Datalayer's servers. */
+export type SpeechWhere = 'device' | 'server';
+/** The licences a voice or a model is admitted by. */
+export interface SpeechLicence {
+    weights: string;
+    code?: string;
+    dataset?: string;
+}
+/** A voice an application may speak with. */
+export interface VoiceSpec {
+    id: string;
+    version: string;
+    name: string;
+    description: string;
+    engine: string;
+    /** The speech model it is a voice of. */
+    model: string;
+    /** The engine's own name for it. */
+    voice: string;
+    /** BCP 47. */
+    languages: string[];
+    where: SpeechWhere[];
+    licence: SpeechLicence;
+    /** What a page listing the voices shows, when the licence asks. */
+    attribution: string;
+    watermark: boolean;
+    sample: string;
+}
+/** One file of a model, pinned by its hash and its size. */
+export interface PinnedFile {
+    path: string;
+    sha256: string;
+    size: number;
+}
+/** A model of speech: to text, to speech, or voice activity. */
+export interface SpeechModelSpec {
+    id: string;
+    version: string;
+    name: string;
+    task: 'stt' | 'tts' | 'vad';
+    engine: string;
+    dtype: string;
+    languages: string[];
+    where: SpeechWhere[];
+    streaming: boolean;
+    licence: SpeechLicence;
+    attribution: string;
+    upstream: string;
+    revision: string;
+    files: PinnedFile[];
+}
+/** The licences the register allows anywhere, the browser included. */
+export declare const SPEECH_ALLOWED_LICENCES: readonly string[];
+export declare const VOICE_CATALOGUE: Record<string, VoiceSpec>;
+export declare const SPEECH_MODEL_CATALOGUE: Record<string, SpeechModelSpec>;
+/** Whether a voice speaks a language: `fr` and `fr-FR` are spoken by a `fr-FR` voice. */
+export declare function voiceSpeaks(voice: VoiceSpec, language: string): boolean;
+/** The voice a language is spoken with when none is said: the first that speaks it. */
+export declare function voiceFor(language: string): VoiceSpec | undefined;
+/**
+ * The speech-to-text model for a language, on the device: Moonshine where it
+ * hears it, Whisper otherwise (VOICE.md decision 3).
+ */
+export declare function transcriberFor(language: string): SpeechModelSpec | undefined;