@datalayer/agent-runtimes 1.5.16 → 1.5.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-embed/chunks/ChatView-Bz0eyWo6.js +32 -0
- package/dist-embed/datalayer-app-main.js +102 -102
- package/lib/apps/apps/appspec.d.ts +36 -4
- package/lib/apps/apps/appspec.js +104 -11
- package/lib/apps/apps/threads.d.ts +20 -0
- package/lib/apps/apps/threads.js +31 -0
- package/lib/apps/plugins/canvas-blocks/index.d.ts +7 -9
- package/lib/apps/plugins/canvas-blocks/index.js +8 -10
- package/lib/apps/plugins/canvas-parts/index.d.ts +4 -1
- package/lib/apps/plugins/canvas-parts/index.js +7 -3
- package/lib/apps/plugins/chat/ChatView.js +32 -7
- package/lib/chat/ChatFloating.js +12 -4
- package/lib/chat/base/ChatBase.js +31 -5
- package/lib/chat/messages/ChatMessageList.js +1 -1
- package/lib/config/AgentConfiguration.d.ts +5 -1
- package/lib/examples/AgentA2ATeamExample.js +6 -5
- package/lib/examples/ChatExample.js +2 -2
- package/lib/examples/VoiceChatExample.js +9 -3
- package/lib/hooks/useBrowserAgent.d.ts +1 -1
- package/lib/hooks/useBrowserAgent.js +1 -1
- package/lib/protocols/AGUIAdapter.d.ts +11 -0
- package/lib/protocols/AGUIAdapter.js +27 -2
- package/lib/specs/agents/agents.js +169 -169
- package/lib/specs/apps.d.ts +32 -32
- package/lib/specs/apps.js +624 -143
- package/lib/specs/appspecSchema.js +30 -2
- package/lib/specs/cogs.js +3 -3
- package/lib/specs/index.d.ts +1 -0
- package/lib/specs/index.js +1 -0
- package/lib/specs/protocols.js +2 -2
- package/lib/specs/sceneSchema.js +2 -0
- package/lib/specs/scenes.js +7 -7
- package/lib/specs/teams/teams.js +11 -11
- package/lib/specs/ui.d.ts +32 -6
- package/lib/specs/ui.js +760 -26
- package/lib/specs/uiRenderers.d.ts +17 -0
- package/lib/specs/uiRenderers.js +40 -0
- package/lib/specs/voices.d.ts +2 -0
- package/lib/specs/voices.js +5 -0
- package/lib/types/agentspecs.d.ts +63 -12
- package/lib/types/chat.d.ts +6 -0
- package/lib/types/protocol.d.ts +17 -0
- package/lib/utils/utils.js +8 -1
- package/lib/voice/ReadAloud.d.ts +35 -0
- package/lib/voice/ReadAloud.js +29 -0
- package/lib/voice/index.d.ts +7 -2
- package/lib/voice/index.js +1 -0
- package/package.json +2 -2
- package/scripts/codegen/generate_agents.py +60 -11
- package/scripts/codegen/generate_ui.py +68 -22
- package/scripts/codegen/generate_ui_renderers.py +176 -0
- package/scripts/codegen/generate_voices.py +5 -1
- package/dist-embed/chunks/ChatView-nnI1-MOR.js +0 -32
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* UI Renderer Catalog.
|
|
3
|
+
*
|
|
4
|
+
* How a UI plugin's interface reaches a host that is not Datalayer's own
|
|
5
|
+
* page (AGENTIC_UI R8): A2UI, MCP Apps, MCP UI.
|
|
6
|
+
*
|
|
7
|
+
* This file is AUTO-GENERATED from YAML specifications.
|
|
8
|
+
* DO NOT EDIT MANUALLY - run 'make specs' to regenerate.
|
|
9
|
+
*/
|
|
10
|
+
import type { UiRendererSpec } from '../types/agentspecs';
|
|
11
|
+
export declare const A2UI_UI_RENDERER_0_0_1: UiRendererSpec;
|
|
12
|
+
export declare const MCP_APPS_UI_RENDERER_0_0_1: UiRendererSpec;
|
|
13
|
+
export declare const MCP_UI_UI_RENDERER_0_0_1: UiRendererSpec;
|
|
14
|
+
export declare const UI_RENDERER_CATALOGUE: Record<string, UiRendererSpec>;
|
|
15
|
+
/** The UI renderer a plugin's `renderers` or a `ui` choice names, or undefined. */
|
|
16
|
+
export declare function getUiRenderer(rendererId: string): UiRendererSpec | undefined;
|
|
17
|
+
export declare function listUiRenderers(): UiRendererSpec[];
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2025-2026 Datalayer, Inc.
|
|
3
|
+
* Distributed under the terms of the Modified BSD License.
|
|
4
|
+
*/
|
|
5
|
+
export const A2UI_UI_RENDERER_0_0_1 = {
|
|
6
|
+
id: 'a2ui',
|
|
7
|
+
version: '0.0.1',
|
|
8
|
+
name: 'A2UI',
|
|
9
|
+
description: 'An agent describes an interface — a form, a table, a card — as a tree of components from a catalogue the host allows, and the host renders it; what the user does in it comes back to the agent as an action.',
|
|
10
|
+
docsUrl: 'https://a2ui.org/',
|
|
11
|
+
enabled: true,
|
|
12
|
+
};
|
|
13
|
+
export const MCP_APPS_UI_RENDERER_0_0_1 = {
|
|
14
|
+
id: 'mcp-apps',
|
|
15
|
+
version: '0.0.1',
|
|
16
|
+
name: 'MCP Apps',
|
|
17
|
+
description: "An MCP server ships an interactive app with a tool: the host renders it in a sandboxed frame beside the conversation, and the app calls the server's tools through the host.",
|
|
18
|
+
docsUrl: 'https://modelcontextprotocol.io/docs/extensions/apps',
|
|
19
|
+
enabled: false,
|
|
20
|
+
};
|
|
21
|
+
export const MCP_UI_UI_RENDERER_0_0_1 = {
|
|
22
|
+
id: 'mcp-ui',
|
|
23
|
+
version: '0.0.1',
|
|
24
|
+
name: 'MCP UI',
|
|
25
|
+
description: 'An MCP tool answers with a UI resource — HTML, a remote page or a component — that the host renders in place of plain text.',
|
|
26
|
+
docsUrl: 'https://mcpui.dev/',
|
|
27
|
+
enabled: false,
|
|
28
|
+
};
|
|
29
|
+
export const UI_RENDERER_CATALOGUE = {
|
|
30
|
+
a2ui: A2UI_UI_RENDERER_0_0_1,
|
|
31
|
+
'mcp-apps': MCP_APPS_UI_RENDERER_0_0_1,
|
|
32
|
+
'mcp-ui': MCP_UI_UI_RENDERER_0_0_1,
|
|
33
|
+
};
|
|
34
|
+
/** The UI renderer a plugin's `renderers` or a `ui` choice names, or undefined. */
|
|
35
|
+
export function getUiRenderer(rendererId) {
|
|
36
|
+
return UI_RENDERER_CATALOGUE[rendererId];
|
|
37
|
+
}
|
|
38
|
+
export function listUiRenderers() {
|
|
39
|
+
return Object.values(UI_RENDERER_CATALOGUE);
|
|
40
|
+
}
|
package/lib/specs/voices.d.ts
CHANGED
package/lib/specs/voices.js
CHANGED
|
@@ -72,6 +72,7 @@ export const SPEECH_MODEL_CATALOGUE = {
|
|
|
72
72
|
id: 'kokoro-82m',
|
|
73
73
|
version: '0.0.1',
|
|
74
74
|
name: 'Kokoro 82M',
|
|
75
|
+
description: "Text to speech on Datalayer's servers, on CPU: hexgrad's Kokoro-82M v1.0 exported to ONNX, in full precision, with the voices file that holds its 54 voices, run by kokoro-onnx. Its phonemes come from eSpeak NG (GPL-3.0), which runs on the server and is never sent to a browser. Full precision, not the 8-bit export: on a 16-core CPU the 8-bit one speaks slower than real time (6 s for 3.9 s of speech), this one about four times faster than real time (1 s).",
|
|
75
76
|
task: 'tts',
|
|
76
77
|
engine: 'kokoro-onnx',
|
|
77
78
|
dtype: 'fp32',
|
|
@@ -102,6 +103,7 @@ export const SPEECH_MODEL_CATALOGUE = {
|
|
|
102
103
|
id: 'moonshine-base-en',
|
|
103
104
|
version: '0.0.1',
|
|
104
105
|
name: 'Moonshine Base (English)',
|
|
106
|
+
description: "Speech to text in the browser, in English: Useful Sensors' Moonshine Base (61M parameters), its ONNX export quantized to 8 bits, run by transformers.js. One of the English models whose weights are MIT; the legacy non-English Moonshine models are under a non-commercial licence and are not listed.",
|
|
105
107
|
task: 'stt',
|
|
106
108
|
engine: 'transformers.js',
|
|
107
109
|
dtype: 'q8',
|
|
@@ -162,6 +164,7 @@ export const SPEECH_MODEL_CATALOGUE = {
|
|
|
162
164
|
id: 'moonshine-tiny-en',
|
|
163
165
|
version: '0.0.1',
|
|
164
166
|
name: 'Moonshine Tiny (English)',
|
|
167
|
+
description: "Speech to text in the browser, in English: Useful Sensors' Moonshine Tiny (27M parameters), its ONNX export quantized to 8 bits, run by transformers.js. One of the English models whose weights are MIT; the legacy non-English Moonshine models are under a non-commercial licence and are not listed.",
|
|
165
168
|
task: 'stt',
|
|
166
169
|
engine: 'transformers.js',
|
|
167
170
|
dtype: 'q8',
|
|
@@ -222,6 +225,7 @@ export const SPEECH_MODEL_CATALOGUE = {
|
|
|
222
225
|
id: 'silero-vad',
|
|
223
226
|
version: '0.0.1',
|
|
224
227
|
name: 'Silero VAD',
|
|
228
|
+
description: 'Where speech starts and ends in what the microphone heard, in the browser: Silero VAD (about 2 MB), the build @ricky0123/vad-web 0.0.31 ships and runs on onnxruntime-web. Push-to-talk keeps the speech it finds and drops the rest.',
|
|
225
229
|
task: 'vad',
|
|
226
230
|
engine: 'vad-web',
|
|
227
231
|
dtype: 'fp32',
|
|
@@ -247,6 +251,7 @@ export const SPEECH_MODEL_CATALOGUE = {
|
|
|
247
251
|
id: 'whisper-base',
|
|
248
252
|
version: '0.0.1',
|
|
249
253
|
name: 'Whisper Base',
|
|
254
|
+
description: "Speech to text in the browser for the languages Moonshine's MIT models do not speak, French first: OpenAI's Whisper base (74M parameters, 99 languages), its ONNX export quantized to 8 bits, run by transformers.js. A clip at a time.",
|
|
250
255
|
task: 'stt',
|
|
251
256
|
engine: 'transformers.js',
|
|
252
257
|
dtype: 'q8',
|
|
@@ -106,25 +106,71 @@ export interface ComponentSpec {
|
|
|
106
106
|
example?: Record<string, unknown>;
|
|
107
107
|
}
|
|
108
108
|
/**
|
|
109
|
-
*
|
|
110
|
-
*
|
|
109
|
+
* A UI plugin (`agentspecs/ui`, AGENTIC_UI R8): one Reactor plugin of this
|
|
110
|
+
* package a Maker composes an interface with, read from its registration, and
|
|
111
|
+
* the UI renderers it may be exposed through. An agent's or an application's
|
|
112
|
+
* `ui` names plugins, each with its renderer.
|
|
111
113
|
*/
|
|
112
114
|
export interface UISpec {
|
|
113
|
-
/** What
|
|
115
|
+
/** What a `ui` choice's `plugin` names (e.g. 'chat'). */
|
|
114
116
|
id: string;
|
|
115
117
|
version: string;
|
|
118
|
+
/** What a Maker reads. */
|
|
116
119
|
name: string;
|
|
117
|
-
/** What
|
|
120
|
+
/** What it is, in a Maker's words. */
|
|
118
121
|
description: string;
|
|
119
|
-
/**
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
|
|
122
|
+
/** Its face. */
|
|
123
|
+
emoji: string;
|
|
124
|
+
/** Its Octicon. */
|
|
125
|
+
octicon?: string;
|
|
126
|
+
/** The name the Reactor plugin registers under. */
|
|
127
|
+
plugin: string;
|
|
128
|
+
/** Its folder under `src/apps/plugins`. */
|
|
129
|
+
folder: string;
|
|
130
|
+
/** Registered once per application, source or UI spec, as `<plugin>-<its id>`. */
|
|
131
|
+
per?: 'application' | 'source' | 'ui';
|
|
132
|
+
/** Its Contributions, as the reactor registers them. */
|
|
133
|
+
contributions: {
|
|
134
|
+
/** The Slots it fills. */
|
|
135
|
+
slots: string[];
|
|
136
|
+
/** The Contribution Points it contributes to. */
|
|
137
|
+
points: string[];
|
|
138
|
+
/** The Commands it adds: Reactor Commands by id, the composer's `/` commands by name. */
|
|
139
|
+
commands: string[];
|
|
140
|
+
};
|
|
141
|
+
/** The Contribution Points it offers. */
|
|
142
|
+
contributionPoints: string[];
|
|
143
|
+
/** Its settings, as the JSON Schema of an object; each default the plugin's. */
|
|
144
|
+
settings: Record<string, unknown>;
|
|
145
|
+
/** The UI renderers it may be exposed through, ids of `agentspecs/ui-renderers`. */
|
|
146
|
+
renderers: string[];
|
|
123
147
|
/** The catalog its components are written in. */
|
|
124
148
|
catalog: string;
|
|
125
149
|
/** The visual components it renders (LOOP C-13). */
|
|
126
150
|
components: ComponentSpec[];
|
|
127
151
|
}
|
|
152
|
+
/**
|
|
153
|
+
* A UI renderer (`agentspecs/ui-renderers`, AGENTIC_UI R8): how a UI plugin's
|
|
154
|
+
* interface reaches a host that is not Datalayer's own page.
|
|
155
|
+
*/
|
|
156
|
+
export interface UiRendererSpec {
|
|
157
|
+
/** What a plugin's `renderers` lists (e.g. 'a2ui'). */
|
|
158
|
+
id: string;
|
|
159
|
+
version: string;
|
|
160
|
+
name: string;
|
|
161
|
+
description: string;
|
|
162
|
+
/** Its own documentation. */
|
|
163
|
+
docsUrl: string;
|
|
164
|
+
/** Whether an application may be exposed through it today. */
|
|
165
|
+
enabled: boolean;
|
|
166
|
+
}
|
|
167
|
+
/** A UI plugin an agent or an application names, and the renderer it is exposed through, when it is. */
|
|
168
|
+
export interface UiChoice {
|
|
169
|
+
/** A UI plugin of `agentspecs/ui`, by id. */
|
|
170
|
+
plugin: string;
|
|
171
|
+
/** The UI renderer of `agentspecs/ui-renderers` it is exposed through, by id. */
|
|
172
|
+
renderer?: string;
|
|
173
|
+
}
|
|
128
174
|
/** The layer of an agent's visit a protocol answers (AGENT_PROTOCOLS §1). */
|
|
129
175
|
export type ProtocolLayer = 'transport' | 'consent' | 'identity' | 'commerce' | 'ui';
|
|
130
176
|
/**
|
|
@@ -1257,8 +1303,8 @@ export interface AppSpec {
|
|
|
1257
1303
|
* is listed, and a planned one only while it is not offered.
|
|
1258
1304
|
*/
|
|
1259
1305
|
protocols: string[];
|
|
1260
|
-
/**
|
|
1261
|
-
ui:
|
|
1306
|
+
/** Its UI plugins, each with the renderer it is exposed through, when it is (AGENTIC_UI R8). */
|
|
1307
|
+
ui: UiChoice[];
|
|
1262
1308
|
/** Personal Agent Protocol; absent, the application has none. */
|
|
1263
1309
|
pap?: AppPapSpec;
|
|
1264
1310
|
/** PACT: the businesses it asks for each person (AGENT_PROTOCOLS PI-01). */
|
|
@@ -1396,8 +1442,8 @@ export interface Agentspec {
|
|
|
1396
1442
|
* PS-02). `protocol` says how a page reaches it; this, what it speaks to the world.
|
|
1397
1443
|
*/
|
|
1398
1444
|
protocols?: string[];
|
|
1399
|
-
/**
|
|
1400
|
-
ui?:
|
|
1445
|
+
/** Its UI plugins, each with the renderer it is exposed through, when it is (AGENTIC_UI R8). */
|
|
1446
|
+
ui?: UiChoice[];
|
|
1401
1447
|
/** Trigger configuration (type, cron, event source, prompt) */
|
|
1402
1448
|
trigger?: AgentTriggerConfig;
|
|
1403
1449
|
/** Model configuration (temperature, max_tokens) */
|
|
@@ -1437,6 +1483,11 @@ export interface Agentspec {
|
|
|
1437
1483
|
parameters?: Record<string, any>;
|
|
1438
1484
|
/** Subagent delegation configuration. */
|
|
1439
1485
|
subagents?: SubAgentsConfig;
|
|
1486
|
+
/**
|
|
1487
|
+
* Its voice, for an agent that is not an application (VOICE.md VO-41):
|
|
1488
|
+
* the fields an Appspec says under `interface.voice`; absent when unsaid.
|
|
1489
|
+
*/
|
|
1490
|
+
voice?: AppVoiceSpec;
|
|
1440
1491
|
}
|
|
1441
1492
|
/**
|
|
1442
1493
|
* Where a subagent reached over A2A lives, or how to launch it.
|
package/lib/types/chat.d.ts
CHANGED
|
@@ -1004,6 +1004,12 @@ export interface ChatBaseProps {
|
|
|
1004
1004
|
voiceSpeaking?: boolean;
|
|
1005
1005
|
/** Stops the agent's voice: the composer's *Stop speaking*, and the microphone opening (VO-13). */
|
|
1006
1006
|
onStopSpeaking?: () => void;
|
|
1007
|
+
/**
|
|
1008
|
+
* Reads an answer aloud (VOICE.md VO-21): given, each answer the agent has
|
|
1009
|
+
* finished writing has a *Read aloud* under it, which calls it with the
|
|
1010
|
+
* answer's text.
|
|
1011
|
+
*/
|
|
1012
|
+
onReadAloud?: (answer: string) => void;
|
|
1007
1013
|
/**
|
|
1008
1014
|
* Show the information icon in the header.
|
|
1009
1015
|
* When clicked, fires onInformationClick.
|
package/lib/types/protocol.d.ts
CHANGED
|
@@ -59,6 +59,23 @@ export interface ProtocolConfig {
|
|
|
59
59
|
* never the bytes. Set for an application's agent.
|
|
60
60
|
*/
|
|
61
61
|
uiFiles?: string;
|
|
62
|
+
/**
|
|
63
|
+
* The application's sessions (`…/api/v1/apps/sessions`), each AG-UI thread
|
|
64
|
+
* one of them: Stop is its session's (`{appSessions}/{thread}/stop`). Set
|
|
65
|
+
* for an application's agent. The runtime's generic `/ag-ui/terminate`
|
|
66
|
+
* never knew a session's thread, and a deployment's runtime does not
|
|
67
|
+
* answer it (OwnRoutesOnly).
|
|
68
|
+
*/
|
|
69
|
+
appSessions?: string;
|
|
70
|
+
/**
|
|
71
|
+
* The runtime answers only the application's own routes (OwnRoutesOnly,
|
|
72
|
+
* PAP_DATALAYER's release gates): it serves a deployment — kept always on,
|
|
73
|
+
* launched by a hosted page for it — or is the visitors' runtime. Nothing
|
|
74
|
+
* outside them is opened: not the monitoring stream (an application's
|
|
75
|
+
* session asks its approvals in the conversation, over `uiSocket`), not
|
|
76
|
+
* the sandbox's socket, nor any runtime-wide setting. Needs `appSessions`.
|
|
77
|
+
*/
|
|
78
|
+
ownRoutesOnly?: boolean;
|
|
62
79
|
/** Additional protocol options */
|
|
63
80
|
options?: Record<string, unknown>;
|
|
64
81
|
}
|
package/lib/utils/utils.js
CHANGED
|
@@ -195,7 +195,14 @@ export function createProtocolAdapter(config) {
|
|
|
195
195
|
};
|
|
196
196
|
switch (config.type) {
|
|
197
197
|
case 'ag-ui':
|
|
198
|
-
|
|
198
|
+
if (config.ownRoutesOnly && !config.appSessions) {
|
|
199
|
+
// Its runtime answers no generic Stop: an application's own or none.
|
|
200
|
+
throw new Error('A chat whose runtime answers only the application’s own routes is an application’s: it needs its sessions (appSessions).');
|
|
201
|
+
}
|
|
202
|
+
return new AGUIAdapter({
|
|
203
|
+
...adapterConfig,
|
|
204
|
+
...(config.appSessions ? { appSessions: config.appSessions } : {}),
|
|
205
|
+
});
|
|
199
206
|
case 'a2a':
|
|
200
207
|
return new A2AAdapter(adapterConfig);
|
|
201
208
|
case 'vercel-ai':
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* *Read aloud*, under an answer (VOICE.md VO-21, `on_request`): the answer,
|
|
3
|
+
* read as plain words and cut into sentences, given to the speaker — what it
|
|
4
|
+
* was saying stops first. The press is the gesture a browser plays sound
|
|
5
|
+
* after.
|
|
6
|
+
*
|
|
7
|
+
* @module voice/ReadAloud
|
|
8
|
+
*/
|
|
9
|
+
import type { JSX } from 'react';
|
|
10
|
+
import type { SentenceSink } from './useSpokenAnswers';
|
|
11
|
+
/** What *Read aloud* gives an answer to: a sink a press may unlock. */
|
|
12
|
+
export interface ReadingSink extends SentenceSink {
|
|
13
|
+
unlock?: () => void;
|
|
14
|
+
}
|
|
15
|
+
/** An answer read from its start: what was being said stops, its sentences follow. */
|
|
16
|
+
export declare function readAloud(sink: ReadingSink, answer: string): string[];
|
|
17
|
+
/** A message, as far as *Read aloud* reads it. */
|
|
18
|
+
export interface ReadableMessage {
|
|
19
|
+
id: string;
|
|
20
|
+
role: string;
|
|
21
|
+
live?: boolean;
|
|
22
|
+
note?: unknown;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Whether a message has *Read aloud* under it: an answer of the agent with
|
|
26
|
+
* words, finished — not live, and not the one the agent is writing now.
|
|
27
|
+
*/
|
|
28
|
+
export declare function readsAloud(message: ReadableMessage, text: string, writingId: string | undefined): boolean;
|
|
29
|
+
export interface ReadAloudProps {
|
|
30
|
+
/** Reads the answer aloud. */
|
|
31
|
+
onRead: () => void;
|
|
32
|
+
}
|
|
33
|
+
/** The button under an answer. */
|
|
34
|
+
export declare function ReadAloud({ onRead }: ReadAloudProps): JSX.Element;
|
|
35
|
+
export default ReadAloud;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { jsx as _jsx } from "react/jsx-runtime";
|
|
2
|
+
import { IconButton, Tooltip } from '@primer/react';
|
|
3
|
+
import { UnmuteIcon } from '@primer/octicons-react';
|
|
4
|
+
import { SentenceCutter } from './sentences';
|
|
5
|
+
/** An answer read from its start: what was being said stops, its sentences follow. */
|
|
6
|
+
export function readAloud(sink, answer) {
|
|
7
|
+
sink.unlock?.();
|
|
8
|
+
sink.stop();
|
|
9
|
+
const cutter = new SentenceCutter();
|
|
10
|
+
const sentences = [...cutter.feed(answer), ...cutter.end(answer)];
|
|
11
|
+
sentences.forEach(sentence => sink.say(sentence));
|
|
12
|
+
return sentences;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Whether a message has *Read aloud* under it: an answer of the agent with
|
|
16
|
+
* words, finished — not live, and not the one the agent is writing now.
|
|
17
|
+
*/
|
|
18
|
+
export function readsAloud(message, text, writingId) {
|
|
19
|
+
return (message.role === 'assistant' &&
|
|
20
|
+
!message.live &&
|
|
21
|
+
!message.note &&
|
|
22
|
+
message.id !== writingId &&
|
|
23
|
+
text.trim().length > 0);
|
|
24
|
+
}
|
|
25
|
+
/** The button under an answer. */
|
|
26
|
+
export function ReadAloud({ onRead }) {
|
|
27
|
+
return (_jsx(Tooltip, { text: "Read aloud", direction: "n", children: _jsx(IconButton, { "data-read-aloud": "", icon: UnmuteIcon, "aria-label": "Read aloud", size: "small", variant: "invisible", onClick: onRead }) }));
|
|
28
|
+
}
|
|
29
|
+
export default ReadAloud;
|
package/lib/voice/index.d.ts
CHANGED
|
@@ -18,6 +18,8 @@ export * from './consent';
|
|
|
18
18
|
export * from './useSpokenAnswers';
|
|
19
19
|
export { VoiceInput } from './VoiceInput';
|
|
20
20
|
export type { VoiceInputProps, ListeningState } from './VoiceInput';
|
|
21
|
+
export { ReadAloud, readAloud, readsAloud } from './ReadAloud';
|
|
22
|
+
export type { ReadAloudProps, ReadableMessage, ReadingSink } from './ReadAloud';
|
|
21
23
|
/**
|
|
22
24
|
* A chat's voice: what a host gives `ChatBase` and `ChatFloating` to let a
|
|
23
25
|
* person talk to the agent and hear it — the Appspec's `interface.voice`,
|
|
@@ -28,8 +30,11 @@ export interface ChatVoice {
|
|
|
28
30
|
language: string;
|
|
29
31
|
/** `push_to_talk`, or `off`. Hands-free is Phase V3. */
|
|
30
32
|
input: 'off' | 'push_to_talk';
|
|
31
|
-
/**
|
|
32
|
-
|
|
33
|
+
/**
|
|
34
|
+
* When answers are heard: `always`, as they are written; `on_request`, a
|
|
35
|
+
* *Read aloud* under each answer (VO-21); or `off`.
|
|
36
|
+
*/
|
|
37
|
+
output: 'off' | 'on_request' | 'always';
|
|
33
38
|
/** The voice of the catalogue answers are spoken with. */
|
|
34
39
|
voice: string;
|
|
35
40
|
/** Datalayer's origin for the browser's models: the pinned store. */
|
package/lib/voice/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@datalayer/agent-runtimes",
|
|
3
|
-
"version": "1.5.
|
|
3
|
+
"version": "1.5.18",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"workspaces": [
|
|
6
6
|
".",
|
|
@@ -160,7 +160,7 @@
|
|
|
160
160
|
"@datalayer/lexical-loro": "^1.0.6",
|
|
161
161
|
"@datalayer/primer-addons": "^1.0.39",
|
|
162
162
|
"@datalayer/primer-rjsf": "^1.0.3",
|
|
163
|
-
"@datalayer/personal-agent-protocol": "^0.
|
|
163
|
+
"@datalayer/personal-agent-protocol": "^0.24.0",
|
|
164
164
|
"@datalayer/reactor": "^1.3.0",
|
|
165
165
|
"@datalayer/reactor-commands": "^1.0.4",
|
|
166
166
|
"@datalayer/reactor-manager": "^1.0.4",
|
|
@@ -236,9 +236,21 @@ def load_fragments(specs_dir: Path) -> List[Dict[str, Any]]:
|
|
|
236
236
|
|
|
237
237
|
def refuse_tools_field(spec: Dict[str, Any], path: Path) -> None:
|
|
238
238
|
"""Refuse an agent spec that still says `tools` (it is `backend_tools` now) or
|
|
239
|
-
`ui_plugin` (it is `ui` now, AGENT_PROTOCOLS PS-03).
|
|
239
|
+
`ui_plugin` (it is `ui` now, AGENT_PROTOCOLS PS-03).
|
|
240
|
+
"""
|
|
240
241
|
if "ui_plugin" in spec:
|
|
241
|
-
raise SystemExit(
|
|
242
|
+
raise SystemExit(
|
|
243
|
+
f"Error: {path} says `ui_plugin`: the field is `ui`, one of agentspecs/ui"
|
|
244
|
+
)
|
|
245
|
+
ui = spec.get("ui")
|
|
246
|
+
if ui is not None and (
|
|
247
|
+
not isinstance(ui, list)
|
|
248
|
+
or not all(isinstance(choice, dict) and "plugin" in choice for choice in ui)
|
|
249
|
+
):
|
|
250
|
+
raise SystemExit(
|
|
251
|
+
f"Error: {path} says `ui: {ui!r}`: `ui` lists UI plugins, each "
|
|
252
|
+
"`{plugin, renderer}` (AGENTIC_UI R8); a renderer is said as a plugin's `renderer`"
|
|
253
|
+
)
|
|
242
254
|
if "tools" in spec:
|
|
243
255
|
raise SystemExit(
|
|
244
256
|
f"Error: {path} says `tools`: the field is `backend_tools` — the tools "
|
|
@@ -292,9 +304,33 @@ def load_yaml_specs(specs_dir: Path) -> List[tuple[str, Dict[str, Any]]]:
|
|
|
292
304
|
|
|
293
305
|
check_model_additionals(resolved, load_model_ids(specs_dir))
|
|
294
306
|
check_protocols(resolved, load_protocol_ids(specs_dir))
|
|
307
|
+
check_voices(resolved)
|
|
295
308
|
return resolved
|
|
296
309
|
|
|
297
310
|
|
|
311
|
+
def check_voices(specs: List[tuple[str, Dict[str, Any]]]) -> None:
|
|
312
|
+
"""Refuse an agent whose `voice` is not an `interface.voice` block of the voice catalogue (VOICE.md VO-41)."""
|
|
313
|
+
from agentspecs.apps import agent_voice_problems
|
|
314
|
+
|
|
315
|
+
problems = [
|
|
316
|
+
problem
|
|
317
|
+
for _, spec in specs
|
|
318
|
+
if "voice" in spec
|
|
319
|
+
for problem in agent_voice_problems(str(spec.get("id")), spec["voice"])
|
|
320
|
+
]
|
|
321
|
+
if problems:
|
|
322
|
+
raise SystemExit("\n".join(problems))
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def voice_of(spec: Dict[str, Any]) -> Dict[str, Any] | None:
|
|
326
|
+
"""An agent's voice with every field said, as the Appspec's block reads it; None when unsaid."""
|
|
327
|
+
if "voice" not in spec:
|
|
328
|
+
return None
|
|
329
|
+
from agentspecs.apps import AppVoice
|
|
330
|
+
|
|
331
|
+
return AppVoice(**spec["voice"]).model_dump(mode="json")
|
|
332
|
+
|
|
333
|
+
|
|
298
334
|
def load_protocol_ids(specs_dir: Path) -> set[str]:
|
|
299
335
|
"""Every id of the protocols catalogue (`agentspecs/protocols`, AGENT_PROTOCOLS PS-01)."""
|
|
300
336
|
ids: set[str] = set()
|
|
@@ -305,7 +341,9 @@ def load_protocol_ids(specs_dir: Path) -> set[str]:
|
|
|
305
341
|
return ids
|
|
306
342
|
|
|
307
343
|
|
|
308
|
-
def check_protocols(
|
|
344
|
+
def check_protocols(
|
|
345
|
+
specs: List[tuple[str, Dict[str, Any]]], protocol_ids: set[str]
|
|
346
|
+
) -> None:
|
|
309
347
|
"""Refuse an agent whose `protocols` is not a list of the catalogue's ids, each once (PS-02)."""
|
|
310
348
|
problems = []
|
|
311
349
|
for _, spec in specs:
|
|
@@ -313,11 +351,15 @@ def check_protocols(specs: List[tuple[str, Dict[str, Any]]], protocol_ids: set[s
|
|
|
313
351
|
if said is None:
|
|
314
352
|
continue
|
|
315
353
|
if not isinstance(said, list) or len(set(map(str, said))) != len(said):
|
|
316
|
-
problems.append(
|
|
354
|
+
problems.append(
|
|
355
|
+
f"{spec.get('id')}: protocols is a list of protocol ids, each once"
|
|
356
|
+
)
|
|
317
357
|
continue
|
|
318
358
|
unknown = [str(p) for p in said if str(p) not in protocol_ids]
|
|
319
359
|
if unknown:
|
|
320
|
-
problems.append(
|
|
360
|
+
problems.append(
|
|
361
|
+
f"{spec.get('id')}: protocols names what the catalogue does not know: {', '.join(unknown)}"
|
|
362
|
+
)
|
|
321
363
|
if problems:
|
|
322
364
|
raise SystemExit("\n".join(problems))
|
|
323
365
|
|
|
@@ -507,6 +549,10 @@ from agent_runtimes.types import (
|
|
|
507
549
|
if model_additionals
|
|
508
550
|
else ""
|
|
509
551
|
)
|
|
552
|
+
voice = voice_of(spec)
|
|
553
|
+
voice_py_line = (
|
|
554
|
+
f" voice={_fmt_py_literal(voice)},\n" if voice is not None else ""
|
|
555
|
+
)
|
|
510
556
|
inference_provider = spec.get("inference_provider")
|
|
511
557
|
inference_provider_str = (
|
|
512
558
|
f'"{inference_provider}"' if inference_provider else "None"
|
|
@@ -539,8 +585,7 @@ from agent_runtimes.types import (
|
|
|
539
585
|
protocol_val = spec.get("protocol")
|
|
540
586
|
protocol_str = f'"{protocol_val}"' if protocol_val else "None"
|
|
541
587
|
delegable_str = _fmt_py_delegable(spec.get("delegable", []))
|
|
542
|
-
|
|
543
|
-
ui_str = f'"{ui_id}"' if ui_id else "None"
|
|
588
|
+
ui_str = _fmt_py_literal(list(spec.get("ui") or []))
|
|
544
589
|
trigger_val = spec.get("trigger")
|
|
545
590
|
model_cfg = spec.get("model_config")
|
|
546
591
|
mcp_srv_tools = spec.get("mcp_server_tools")
|
|
@@ -670,7 +715,7 @@ from agent_runtimes.types import (
|
|
|
670
715
|
tool_hooks={_fmt_py_literal(tool_hooks_val)},
|
|
671
716
|
parameters={_fmt_py_literal(parameters_val)},
|
|
672
717
|
subagents={subagents_str},
|
|
673
|
-
)
|
|
718
|
+
{voice_py_line})
|
|
674
719
|
|
|
675
720
|
'''
|
|
676
721
|
|
|
@@ -1137,6 +1182,10 @@ const FRONTEND_TOOL_MAP: Record<string, any> = {
|
|
|
1137
1182
|
if model_additionals
|
|
1138
1183
|
else ""
|
|
1139
1184
|
)
|
|
1185
|
+
voice = voice_of(spec)
|
|
1186
|
+
voice_ts_line = (
|
|
1187
|
+
f" voice: {_fmt_ts_literal(voice)},\n" if voice is not None else ""
|
|
1188
|
+
)
|
|
1140
1189
|
inference_provider = spec.get("inference_provider")
|
|
1141
1190
|
inference_provider_ts = (
|
|
1142
1191
|
f"'{inference_provider}'" if inference_provider else "undefined"
|
|
@@ -1167,8 +1216,7 @@ const FRONTEND_TOOL_MAP: Record<string, any> = {
|
|
|
1167
1216
|
protocol_val = spec.get("protocol")
|
|
1168
1217
|
protocol_ts = f"'{protocol_val}'" if protocol_val else "undefined"
|
|
1169
1218
|
delegable_ts = _fmt_ts_delegable(spec.get("delegable", []))
|
|
1170
|
-
|
|
1171
|
-
ui_ts = f"'{ui_id}'" if ui_id else "undefined"
|
|
1219
|
+
ui_ts = _fmt_ts_literal(list(spec.get("ui") or []))
|
|
1172
1220
|
trigger_val = spec.get("trigger")
|
|
1173
1221
|
model_cfg = spec.get("model_config")
|
|
1174
1222
|
mcp_srv_tools = spec.get("mcp_server_tools")
|
|
@@ -1246,7 +1294,7 @@ const FRONTEND_TOOL_MAP: Record<string, any> = {
|
|
|
1246
1294
|
toolHooks: {_fmt_ts_literal(tool_hooks_val)},
|
|
1247
1295
|
parameters: {_fmt_ts_literal(parameters_val)},
|
|
1248
1296
|
subagents: {subagents_ts},
|
|
1249
|
-
}};
|
|
1297
|
+
{voice_ts_line}}};
|
|
1250
1298
|
|
|
1251
1299
|
"""
|
|
1252
1300
|
|
|
@@ -1674,6 +1722,7 @@ export * from './skills';
|
|
|
1674
1722
|
export * from './backendTools';
|
|
1675
1723
|
export * from './triggers';
|
|
1676
1724
|
export * from './ui';
|
|
1725
|
+
export * from './uiRenderers';
|
|
1677
1726
|
export * from './protocols';
|
|
1678
1727
|
export * from './frames';
|
|
1679
1728
|
export * from './cogs';
|