@datalayer/agent-runtimes 1.5.44 → 1.5.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-embed/chunks/{ChatView-kCmBbnDa.js → ChatView-Dy-IgGFL.js} +1 -1
- package/dist-embed/datalayer-app-main.js +99 -94
- package/lib/apps/plugins/chat/ChatView.js +2 -2
- package/lib/chat/ChatFloating.js +1 -1
- package/lib/chat/base/ChatBase.js +23 -8
- package/lib/chat/messages/ChatMessageList.d.ts +7 -1
- package/lib/chat/messages/ChatMessageList.js +8 -2
- package/lib/portable/hosts/__tests__/fixtures/lock-with-jupyter.json +40 -0
- package/lib/portable/hosts/__tests__/fixtures/npm-ls-with-jupyter.json +21 -0
- package/lib/portable/hosts/agentClient.d.ts +167 -0
- package/lib/portable/hosts/agentClient.js +346 -0
- package/lib/portable/hosts/approvalRenderers.d.ts +43 -0
- package/lib/portable/hosts/approvalRenderers.js +180 -0
- package/lib/portable/hosts/client.d.ts +95 -0
- package/lib/portable/hosts/client.js +174 -0
- package/lib/portable/hosts/contracts.d.ts +199 -0
- package/lib/portable/hosts/contracts.js +235 -0
- package/lib/portable/hosts/contracts.json +381 -0
- package/lib/portable/hosts/deadLetters.d.ts +41 -0
- package/lib/portable/hosts/deadLetters.js +124 -0
- package/lib/portable/hosts/deliveries.d.ts +76 -0
- package/lib/portable/hosts/deliveries.js +147 -0
- package/lib/portable/hosts/forbiddenGraph.d.ts +77 -0
- package/lib/portable/hosts/forbiddenGraph.js +174 -0
- package/lib/portable/hosts/health.d.ts +35 -0
- package/lib/portable/hosts/health.js +92 -0
- package/lib/portable/hosts/index.d.ts +40 -0
- package/lib/portable/hosts/index.js +45 -0
- package/lib/portable/hosts/observability.d.ts +67 -0
- package/lib/portable/hosts/observability.js +171 -0
- package/lib/portable/hosts/postgresQueue.d.ts +55 -0
- package/lib/portable/hosts/postgresQueue.js +217 -0
- package/lib/portable/hosts/queue.d.ts +47 -0
- package/lib/portable/hosts/queue.js +63 -0
- package/lib/portable/hosts/quotas.d.ts +81 -0
- package/lib/portable/hosts/quotas.js +147 -0
- package/lib/portable/hosts/rateLimit.d.ts +13 -0
- package/lib/portable/hosts/rateLimit.js +8 -0
- package/lib/portable/hosts/resolution.d.ts +54 -0
- package/lib/portable/hosts/resolution.js +46 -0
- package/lib/portable/hosts/schema.d.ts +19 -0
- package/lib/portable/hosts/schema.js +81 -0
- package/lib/portable/hosts/secretCipher.d.ts +23 -0
- package/lib/portable/hosts/secretCipher.js +45 -0
- package/lib/portable/hosts/sessionUid.d.ts +7 -0
- package/lib/portable/hosts/sessionUid.js +28 -0
- package/lib/portable/hosts/turnState.d.ts +45 -0
- package/lib/portable/hosts/turnState.js +113 -0
- package/lib/portable/index.d.ts +2 -0
- package/lib/portable/index.js +3 -0
- package/lib/portable/semantic-events.json +480 -0
- package/lib/portable/semantic.d.ts +78 -0
- package/lib/portable/semantic.js +100 -0
- package/lib/portable/semanticOfAgUi.d.ts +41 -0
- package/lib/portable/semanticOfAgUi.js +108 -0
- package/lib/protocols/A2AAdapter.js +19 -0
- package/lib/protocols/VercelAIAdapter.js +11 -0
- package/lib/specs/voices.js +62 -0
- package/lib/test-setup.js +17 -14
- package/lib/types/chat.d.ts +10 -0
- package/lib/voice/ComposerVoice.js +12 -3
- package/lib/voice/captions.d.ts +31 -0
- package/lib/voice/captions.js +156 -0
- package/lib/voice/handsFree.d.ts +12 -0
- package/lib/voice/handsFree.js +18 -0
- package/lib/voice/serverHearing.d.ts +29 -0
- package/lib/voice/serverHearing.js +78 -2
- package/package.json +1 -1
- package/scripts/voice/measure.py +77 -17
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2025-2026 Datalayer, Inc.
|
|
3
|
+
* Distributed under the terms of the Modified BSD License.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* The semantic agent events a chat host renders (Slack, Discord), version 1:
|
|
7
|
+
* a session's AG-UI stream reduced to what a host shows (`semanticOfAgUi.ts`),
|
|
8
|
+
* never a notebook widget or a web component. The contract and its fixture
|
|
9
|
+
* cases are `semantic-events.json`, beside this file, tested here
|
|
10
|
+
* (`__tests__/semantic.unit.test.ts`) and by the runtime that emits the
|
|
11
|
+
* events (`agent_runtimes/tests/test_semantic_events_contract.py`).
|
|
12
|
+
*
|
|
13
|
+
* Moved from the Slack service (plans/DISCORD.md DC-005, SLACK S-004, S-205).
|
|
14
|
+
*
|
|
15
|
+
* @module portable/semantic
|
|
16
|
+
*/
|
|
17
|
+
export const SEMANTIC_EVENTS_VERSION = 1;
|
|
18
|
+
export const SEMANTIC_KINDS = [
|
|
19
|
+
'text',
|
|
20
|
+
'progress',
|
|
21
|
+
'tool',
|
|
22
|
+
'approval',
|
|
23
|
+
'table',
|
|
24
|
+
'file',
|
|
25
|
+
'citation',
|
|
26
|
+
'ask',
|
|
27
|
+
'completion',
|
|
28
|
+
'stop',
|
|
29
|
+
'error',
|
|
30
|
+
];
|
|
31
|
+
/** Whether an event ends the turn's presentation. */
|
|
32
|
+
export function isTerminal(event) {
|
|
33
|
+
return (event.kind === 'completion' ||
|
|
34
|
+
event.kind === 'stop' ||
|
|
35
|
+
event.kind === 'error' ||
|
|
36
|
+
event.kind === 'ask');
|
|
37
|
+
}
|
|
38
|
+
const TABLE_ROW = /^\s*\|.*\|\s*$/;
|
|
39
|
+
const TABLE_RULE = /^\s*\|(\s*:?-{3,}:?\s*\|)+\s*$/;
|
|
40
|
+
const LINK = /\[([^\]\n]{1,300})\]\((https:\/\/[^\s)]{1,2000})\)/g;
|
|
41
|
+
function cells(line) {
|
|
42
|
+
return line
|
|
43
|
+
.trim()
|
|
44
|
+
.replace(/^\|/, '')
|
|
45
|
+
.replace(/\|$/, '')
|
|
46
|
+
.split('|')
|
|
47
|
+
.map(cell => cell.trim());
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The parts of an answer as a host shows them: its Markdown tables as tables,
|
|
51
|
+
* its HTTPS links as citations (once each, in order), the rest as words. Only
|
|
52
|
+
* `https:` links are citations: any other scheme stays words.
|
|
53
|
+
*/
|
|
54
|
+
export function answerParts(answer) {
|
|
55
|
+
const parts = [];
|
|
56
|
+
const lines = answer.split('\n');
|
|
57
|
+
let words = [];
|
|
58
|
+
const flush = () => {
|
|
59
|
+
const text = words.join('\n').trim();
|
|
60
|
+
if (text) {
|
|
61
|
+
parts.push({ kind: 'text', text });
|
|
62
|
+
}
|
|
63
|
+
words = [];
|
|
64
|
+
};
|
|
65
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
66
|
+
const line = lines[index] ?? '';
|
|
67
|
+
const rule = lines[index + 1] ?? '';
|
|
68
|
+
if (TABLE_ROW.test(line) && TABLE_RULE.test(rule)) {
|
|
69
|
+
flush();
|
|
70
|
+
const columns = cells(line);
|
|
71
|
+
const rows = [];
|
|
72
|
+
index += 2;
|
|
73
|
+
while (index < lines.length && TABLE_ROW.test(lines[index] ?? '')) {
|
|
74
|
+
const row = cells(lines[index] ?? '');
|
|
75
|
+
rows.push(columns.map((_, column) => row[column] ?? ''));
|
|
76
|
+
index += 1;
|
|
77
|
+
}
|
|
78
|
+
index -= 1;
|
|
79
|
+
parts.push({ kind: 'table', columns, rows });
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
words.push(line);
|
|
83
|
+
}
|
|
84
|
+
flush();
|
|
85
|
+
const seen = new Set();
|
|
86
|
+
for (const part of [...parts]) {
|
|
87
|
+
if (part.kind !== 'text') {
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
for (const match of part.text.matchAll(LINK)) {
|
|
91
|
+
const [, title, url] = match;
|
|
92
|
+
if (!title || !url || seen.has(url)) {
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
seen.add(url);
|
|
96
|
+
parts.push({ kind: 'citation', title, url });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return parts;
|
|
100
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A session's AG-UI events reduced to the semantic events a chat host
|
|
3
|
+
* renders (`semantic.ts`, version 1). A stream is read with `AgUiFrames` or
|
|
4
|
+
* `readRunStream` (`agui.ts`); this says what each event means to a host.
|
|
5
|
+
*
|
|
6
|
+
* The runtime's `loop.turn` event says which turn a stream is and where it
|
|
7
|
+
* stands (`running`, `finished`, `failed`: agent_runtimes/loop/apps/
|
|
8
|
+
* external_turns.py); a stopped session's stream ends without a run end,
|
|
9
|
+
* so `stop` is never read from an event: the host that pressed Stop says it.
|
|
10
|
+
* A question is read from `RUN_FINISHED.result.asking` only: the run stream
|
|
11
|
+
* carries no `loop.ask` event (that is the record's word, `record.py`).
|
|
12
|
+
*
|
|
13
|
+
* Moved from the Slack service (plans/DISCORD.md DC-005, SLACK S-205).
|
|
14
|
+
*
|
|
15
|
+
* @module portable/semanticOfAgUi
|
|
16
|
+
*/
|
|
17
|
+
import type { AgUiEvent } from './agui';
|
|
18
|
+
import type { AskQuestion, SemanticEvent } from './semantic';
|
|
19
|
+
/** What one turn says of itself: its id, and whether it is a replay. */
|
|
20
|
+
export type TurnInfo = Readonly<{
|
|
21
|
+
replayed: boolean;
|
|
22
|
+
session: string;
|
|
23
|
+
state: string;
|
|
24
|
+
turn: string;
|
|
25
|
+
}>;
|
|
26
|
+
/** The turn a `loop.turn` event names, or `undefined` for any other event. */
|
|
27
|
+
export declare function turnOf(event: AgUiEvent): TurnInfo | undefined;
|
|
28
|
+
/**
|
|
29
|
+
* The question a turn ends on (`RUN_FINISHED.result.asking`), as
|
|
30
|
+
* a host asks it, or `undefined` when the value is not a question. A kind a
|
|
31
|
+
* host does not know is answered in Datalayer, as a form (SLACK S-307):
|
|
32
|
+
* never guessed to be text. (`conversation.ts`'s `questionOf`, for a native
|
|
33
|
+
* client, throws on such a kind instead.)
|
|
34
|
+
*/
|
|
35
|
+
export declare function askQuestionOf(value: unknown): AskQuestion | undefined;
|
|
36
|
+
/**
|
|
37
|
+
* The semantic events (version 1) one AG-UI event means: none for an event a
|
|
38
|
+
* host does not show. `tools` keeps each running call's name by its id, for
|
|
39
|
+
* its result.
|
|
40
|
+
*/
|
|
41
|
+
export declare function semanticOf(event: AgUiEvent, tools: Map<string, string>): SemanticEvent[];
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2025-2026 Datalayer, Inc.
|
|
3
|
+
* Distributed under the terms of the Modified BSD License.
|
|
4
|
+
*/
|
|
5
|
+
/** A string field of an event, or the fallback: never an object stringified. */
|
|
6
|
+
const stringOf = (value, fallback = '') => typeof value === 'string' ? value : fallback;
|
|
7
|
+
const isRecord = (value) => Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
8
|
+
/** The turn a `loop.turn` event names, or `undefined` for any other event. */
|
|
9
|
+
export function turnOf(event) {
|
|
10
|
+
if (event.type !== 'CUSTOM' || event.name !== 'loop.turn') {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
const value = isRecord(event.value) ? event.value : {};
|
|
14
|
+
return {
|
|
15
|
+
replayed: value.replayed === true,
|
|
16
|
+
session: stringOf(value.session),
|
|
17
|
+
state: stringOf(value.state),
|
|
18
|
+
turn: stringOf(value.turn),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
/** What a tool's result says of itself: JSON when it is JSON, else nothing. */
|
|
22
|
+
function resultRecord(content) {
|
|
23
|
+
if (isRecord(content)) {
|
|
24
|
+
return content;
|
|
25
|
+
}
|
|
26
|
+
if (typeof content !== 'string') {
|
|
27
|
+
return undefined;
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
const parsed = JSON.parse(content);
|
|
31
|
+
return isRecord(parsed) ? parsed : undefined;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The question a turn ends on (`RUN_FINISHED.result.asking`), as
|
|
39
|
+
* a host asks it, or `undefined` when the value is not a question. A kind a
|
|
40
|
+
* host does not know is answered in Datalayer, as a form (SLACK S-307):
|
|
41
|
+
* never guessed to be text. (`conversation.ts`'s `questionOf`, for a native
|
|
42
|
+
* client, throws on such a kind instead.)
|
|
43
|
+
*/
|
|
44
|
+
export function askQuestionOf(value) {
|
|
45
|
+
if (!isRecord(value) || typeof value.prompt !== 'string') {
|
|
46
|
+
return undefined;
|
|
47
|
+
}
|
|
48
|
+
const kind = value.kind === 'text' ||
|
|
49
|
+
value.kind === 'choice' ||
|
|
50
|
+
value.kind === 'file' ||
|
|
51
|
+
value.kind === 'form'
|
|
52
|
+
? value.kind
|
|
53
|
+
: 'form';
|
|
54
|
+
const options = Array.isArray(value.options)
|
|
55
|
+
? value.options.filter((one) => typeof one === 'string')
|
|
56
|
+
: undefined;
|
|
57
|
+
return kind === 'choice'
|
|
58
|
+
? { kind, prompt: value.prompt, options: options ?? [] }
|
|
59
|
+
: { kind, prompt: value.prompt };
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The semantic events (version 1) one AG-UI event means: none for an event a
|
|
63
|
+
* host does not show. `tools` keeps each running call's name by its id, for
|
|
64
|
+
* its result.
|
|
65
|
+
*/
|
|
66
|
+
export function semanticOf(event, tools) {
|
|
67
|
+
switch (event.type) {
|
|
68
|
+
case 'TEXT_MESSAGE_CONTENT':
|
|
69
|
+
case 'TEXT_MESSAGE_CHUNK':
|
|
70
|
+
return typeof event.delta === 'string' && event.delta
|
|
71
|
+
? [{ kind: 'text', text: event.delta }]
|
|
72
|
+
: [];
|
|
73
|
+
case 'TOOL_CALL_START': {
|
|
74
|
+
const id = stringOf(event.toolCallId);
|
|
75
|
+
const name = stringOf(event.toolCallName, 'a tool');
|
|
76
|
+
tools.set(id, name);
|
|
77
|
+
return [{ kind: 'tool', id, name, status: 'running' }];
|
|
78
|
+
}
|
|
79
|
+
case 'TOOL_CALL_RESULT': {
|
|
80
|
+
const id = stringOf(event.toolCallId);
|
|
81
|
+
const name = tools.get(id) ?? 'a tool';
|
|
82
|
+
const result = resultRecord(event.content);
|
|
83
|
+
if (result?.pending_approval === true) {
|
|
84
|
+
return [{ kind: 'approval', toolCallId: id, toolName: name }];
|
|
85
|
+
}
|
|
86
|
+
const failed = result?.success === false || result?.isError === true;
|
|
87
|
+
return [
|
|
88
|
+
{ kind: 'tool', id, name, status: failed ? 'failed' : 'complete' },
|
|
89
|
+
];
|
|
90
|
+
}
|
|
91
|
+
case 'STEP_STARTED':
|
|
92
|
+
return typeof event.stepName === 'string' && event.stepName
|
|
93
|
+
? [{ kind: 'progress', label: event.stepName }]
|
|
94
|
+
: [];
|
|
95
|
+
case 'RUN_FINISHED': {
|
|
96
|
+
const asking = askQuestionOf(isRecord(event.result) ? event.result.asking : undefined);
|
|
97
|
+
return asking
|
|
98
|
+
? [{ kind: 'ask', question: asking }]
|
|
99
|
+
: [{ kind: 'completion' }];
|
|
100
|
+
}
|
|
101
|
+
case 'RUN_ERROR':
|
|
102
|
+
return [
|
|
103
|
+
{ kind: 'error', message: stringOf(event.message, 'The run failed.') },
|
|
104
|
+
];
|
|
105
|
+
default:
|
|
106
|
+
return [];
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import { ORCHESTRATION_EXTENSION_URI, ENVELOPE_KEY as DATALAYER_META_KEY, readUsageMeta, } from '@datalayer/agent-teams';
|
|
11
11
|
import { generateMessageId, createAssistantMessage } from '../types/messages';
|
|
12
12
|
import { BaseProtocolAdapter } from './BaseProtocolAdapter';
|
|
13
|
+
import { VOICE_SAY_EVENT, VOICE_SETTINGS_EVENT } from '../voice/sayAloud';
|
|
13
14
|
/**
|
|
14
15
|
* The header fasta2a-based workers (`agent-runtimes`'s own, and any other
|
|
15
16
|
* built on it) actually read to activate an extension for one request —
|
|
@@ -432,6 +433,24 @@ export class A2AAdapter extends BaseProtocolAdapter {
|
|
|
432
433
|
if (eventKind === 'status-update') {
|
|
433
434
|
const state = task.status?.state;
|
|
434
435
|
console.log('[A2AAdapter] Status update:', state, 'final:', task.final);
|
|
436
|
+
// The agent's voice and the lines it says aloud, in a working status
|
|
437
|
+
// message's metadata (VOICE.md VO-36): activities the chat says and
|
|
438
|
+
// captions, as AG-UI's `voice.settings` and `voice.say` custom events.
|
|
439
|
+
const voice = task.status?.message?.metadata?.voice;
|
|
440
|
+
if (voice?.settings) {
|
|
441
|
+
this.emit({
|
|
442
|
+
type: 'activity',
|
|
443
|
+
activity: { type: VOICE_SETTINGS_EVENT, data: voice.settings },
|
|
444
|
+
timestamp: new Date(),
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
if (voice?.say) {
|
|
448
|
+
this.emit({
|
|
449
|
+
type: 'activity',
|
|
450
|
+
activity: { type: VOICE_SAY_EVENT, data: voice.say },
|
|
451
|
+
timestamp: new Date(),
|
|
452
|
+
});
|
|
453
|
+
}
|
|
435
454
|
if (task.final &&
|
|
436
455
|
(state === 'completed' || state === 'failed' || state === 'canceled')) {
|
|
437
456
|
// Only read the envelope once the worker actually named the
|
|
@@ -627,6 +627,17 @@ export class VercelAIAdapter extends BaseProtocolAdapter {
|
|
|
627
627
|
});
|
|
628
628
|
}
|
|
629
629
|
}
|
|
630
|
+
else if (typeof event.type === 'string' &&
|
|
631
|
+
event.type.startsWith('data-')) {
|
|
632
|
+
// What the server says besides the conversation's text, as
|
|
633
|
+
// AG-UI's custom events say it: `data-voice.say`, a line the
|
|
634
|
+
// agent says aloud, and `data-voice.settings` (VOICE.md VO-36).
|
|
635
|
+
this.emit({
|
|
636
|
+
type: 'activity',
|
|
637
|
+
activity: { type: event.type, data: event.data },
|
|
638
|
+
timestamp: new Date(),
|
|
639
|
+
});
|
|
640
|
+
}
|
|
630
641
|
}
|
|
631
642
|
catch (parseError) {
|
|
632
643
|
console.error('[VercelAIAdapter] Failed to parse SSE data:', parseError);
|
package/lib/specs/voices.js
CHANGED
|
@@ -100,6 +100,68 @@ export const SPEECH_MODEL_CATALOGUE = {
|
|
|
100
100
|
},
|
|
101
101
|
],
|
|
102
102
|
},
|
|
103
|
+
'moonshine-base-en-server': {
|
|
104
|
+
id: 'moonshine-base-en-server',
|
|
105
|
+
version: '0.0.1',
|
|
106
|
+
name: 'Moonshine Base (English, server)',
|
|
107
|
+
description: "Speech to text on Datalayer's servers, in English: Useful Sensors' Moonshine Base (61M parameters), the same 8-bit ONNX export the browser runs, run by onnxruntime on the CPU with a greedy decoder. Its encoder is as long as the speech, with no 30-second window: measured beside Whisper small for short spoken turns (decision 18).",
|
|
108
|
+
provider: 'usefulsensors',
|
|
109
|
+
task: 'stt',
|
|
110
|
+
engine: 'onnxruntime',
|
|
111
|
+
dtype: 'q8',
|
|
112
|
+
languages: ['en'],
|
|
113
|
+
where: ['server'],
|
|
114
|
+
streaming: false,
|
|
115
|
+
licence: {
|
|
116
|
+
weights: 'MIT',
|
|
117
|
+
code: 'MIT',
|
|
118
|
+
},
|
|
119
|
+
attribution: '',
|
|
120
|
+
upstream: 'https://huggingface.co/UsefulSensors/moonshine-base',
|
|
121
|
+
revision: 'b1e9b6aae3c3c7298f10c3798393fdf38e8fbbad',
|
|
122
|
+
files: [
|
|
123
|
+
{
|
|
124
|
+
path: 'config.json',
|
|
125
|
+
sha256: 'fab7241d1e9fc6c2370c4c6dfb5da79bb54d67ed9ab6b507ac51d29d2abe01d1',
|
|
126
|
+
size: 922,
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
path: 'generation_config.json',
|
|
130
|
+
sha256: 'f9b3f711b57be7def2e50a8942f64f36ee0a55fad5b84ff93a687b6c5bcc1d44',
|
|
131
|
+
size: 147,
|
|
132
|
+
},
|
|
133
|
+
{
|
|
134
|
+
path: 'preprocessor_config.json',
|
|
135
|
+
sha256: 'fa43a7017ef85cd1d0fba0d9aae77c8adb16990ae6f11115631f41ec5d8aa679',
|
|
136
|
+
size: 128,
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
path: 'special_tokens_map.json',
|
|
140
|
+
sha256: 'ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356',
|
|
141
|
+
size: 3,
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
path: 'tokenizer.json',
|
|
145
|
+
sha256: '7b913404bdd039af4756783218af4440bc07fb7d6d8258d677e34f95b3ec416f',
|
|
146
|
+
size: 3761754,
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
path: 'tokenizer_config.json',
|
|
150
|
+
sha256: 'edaee394565d428ea98a663ae7209cdcfeefc5585c42d7a570ff7c986df2cd15',
|
|
151
|
+
size: 135735,
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
path: 'onnx/encoder_model_quantized.onnx',
|
|
155
|
+
sha256: '1dd9ab0a7f987113d30affcba5a068d11c8f90fa0223caa3e491ade431ad9751',
|
|
156
|
+
size: 20513063,
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
path: 'onnx/decoder_model_merged_quantized.onnx',
|
|
160
|
+
sha256: 'cc9f3cd6698a369c6008b41aa60aa3fb3322e7f03c9bdf19d8e6b7200afca4f3',
|
|
161
|
+
size: 42498870,
|
|
162
|
+
},
|
|
163
|
+
],
|
|
164
|
+
},
|
|
103
165
|
'moonshine-base-en': {
|
|
104
166
|
id: 'moonshine-base-en',
|
|
105
167
|
version: '0.0.1',
|
package/lib/test-setup.js
CHANGED
|
@@ -68,20 +68,23 @@ class MockDataTransfer {
|
|
|
68
68
|
// Add missing DOM APIs to global scope
|
|
69
69
|
global.DragEvent = MockDragEvent;
|
|
70
70
|
global.DataTransfer = MockDataTransfer;
|
|
71
|
-
// Mock window.matchMedia
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
}
|
|
71
|
+
// Mock window.matchMedia. A test that runs in node (`@vitest-environment
|
|
72
|
+
// node`, the portable host code: plans/DISCORD.md §4.2) has no window.
|
|
73
|
+
if (typeof window !== 'undefined') {
|
|
74
|
+
Object.defineProperty(window, 'matchMedia', {
|
|
75
|
+
writable: true,
|
|
76
|
+
value: (query) => ({
|
|
77
|
+
matches: false,
|
|
78
|
+
media: query,
|
|
79
|
+
onchange: null,
|
|
80
|
+
addListener: () => { }, // deprecated
|
|
81
|
+
removeListener: () => { }, // deprecated
|
|
82
|
+
addEventListener: () => { },
|
|
83
|
+
removeEventListener: () => { },
|
|
84
|
+
dispatchEvent: () => { },
|
|
85
|
+
}),
|
|
86
|
+
});
|
|
87
|
+
}
|
|
85
88
|
// Mock Simple-related modules to prevent import errors
|
|
86
89
|
import { vi } from 'vitest';
|
|
87
90
|
vi.mock('@datalayer/jupyter-react', () => ({
|
package/lib/types/chat.d.ts
CHANGED
|
@@ -689,6 +689,11 @@ export interface ChatBaseProps {
|
|
|
689
689
|
* Absent while its answers are not heard.
|
|
690
690
|
*/
|
|
691
691
|
stopSpeaking?: () => void;
|
|
692
|
+
/**
|
|
693
|
+
* Marks the answer the person spoke over *interrupted* (VOICE.md
|
|
694
|
+
* VO-13), for a host whose composer holds the microphone.
|
|
695
|
+
*/
|
|
696
|
+
markInterrupted: () => void;
|
|
692
697
|
} | null) => void;
|
|
693
698
|
/**
|
|
694
699
|
* Reports whether the agent's voice is speaking, for a host that draws the
|
|
@@ -1013,6 +1018,11 @@ export interface ChatBaseProps {
|
|
|
1013
1018
|
voice?: ChatVoice;
|
|
1014
1019
|
/** The agent is speaking: the composer offers *Stop speaking* (`Esc`). */
|
|
1015
1020
|
voiceSpeaking?: boolean;
|
|
1021
|
+
/**
|
|
1022
|
+
* The sentence the host's speaker is saying, marked in the conversation
|
|
1023
|
+
* (VOICE.md VO-65); a chat that speaks for itself marks its own.
|
|
1024
|
+
*/
|
|
1025
|
+
voiceSentence?: string;
|
|
1016
1026
|
/** Stops the agent's voice: the composer's *Stop speaking*, and the microphone opening (VO-13). */
|
|
1017
1027
|
onStopSpeaking?: () => void;
|
|
1018
1028
|
/**
|
|
@@ -2,7 +2,7 @@ import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
|
2
2
|
import { useMemo, useRef } from 'react';
|
|
3
3
|
import { DeviceHearing } from './hearing';
|
|
4
4
|
import { HandsFree } from './HandsFree';
|
|
5
|
-
import { ServerHearing, hearingPlace } from './serverHearing';
|
|
5
|
+
import { AutoHearing, ServerHearing, hearingPlace } from './serverHearing';
|
|
6
6
|
import { VoiceInput } from './VoiceInput';
|
|
7
7
|
import { VoiceMenu } from './VoiceMenu';
|
|
8
8
|
/** Whether something rides on a run as a spoken message's metadata (VO-27). */
|
|
@@ -24,13 +24,20 @@ export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled
|
|
|
24
24
|
return undefined;
|
|
25
25
|
}
|
|
26
26
|
const device = new DeviceHearing(voice.modelsUrl, voice.engines);
|
|
27
|
-
|
|
27
|
+
const server = voice.speechUrl
|
|
28
28
|
? new ServerHearing(device, {
|
|
29
29
|
speechUrl: voice.speechUrl,
|
|
30
30
|
token: voice.token,
|
|
31
31
|
detect: voice.detect,
|
|
32
32
|
})
|
|
33
|
-
:
|
|
33
|
+
: undefined;
|
|
34
|
+
// `where: auto` (§5): the device when it hears the language and its model
|
|
35
|
+
// loads, the server otherwise — and back to the device when the server
|
|
36
|
+
// refuses (a visitor's day spent).
|
|
37
|
+
if (server && !voice.detect && (voice.where ?? 'auto') === 'auto') {
|
|
38
|
+
return new AutoHearing(device, server, voice.language);
|
|
39
|
+
}
|
|
40
|
+
return place === 'server' && server ? server : device;
|
|
34
41
|
}, [
|
|
35
42
|
voice.input,
|
|
36
43
|
voice.modelsUrl,
|
|
@@ -39,6 +46,8 @@ export function ComposerVoice({ voice, input, setInput, onSpoken, send, disabled
|
|
|
39
46
|
voice.speechUrl,
|
|
40
47
|
voice.token,
|
|
41
48
|
voice.detect,
|
|
49
|
+
voice.where,
|
|
50
|
+
voice.language,
|
|
42
51
|
]);
|
|
43
52
|
// What the box held when the microphone opened: the words heard go after it.
|
|
44
53
|
const base = useRef('');
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Captions always (VOICE.md VO-65): the sentence being said is marked in the
|
|
3
|
+
* open conversation, not by colour alone. The answer that holds it is marked
|
|
4
|
+
* `aria-current` and `data-voice-saying` with a bar beside it; the sentence
|
|
5
|
+
* itself is underlined in place through the CSS Custom Highlight API, which
|
|
6
|
+
* draws over the text without touching the DOM React owns. A browser without
|
|
7
|
+
* that API keeps the answer's mark.
|
|
8
|
+
*
|
|
9
|
+
* @module voice/captions
|
|
10
|
+
*/
|
|
11
|
+
import { type RefObject } from 'react';
|
|
12
|
+
/** The name the sentence's highlight is registered under. */
|
|
13
|
+
export declare const SAID_HIGHLIGHT = "voice-said";
|
|
14
|
+
/**
|
|
15
|
+
* The id of the answer a sentence is being said from: the newest answer of
|
|
16
|
+
* the agent whose words hold it, or none.
|
|
17
|
+
*/
|
|
18
|
+
export declare function saidMessageId(items: readonly unknown[], sentence: string | undefined): string | undefined;
|
|
19
|
+
/**
|
|
20
|
+
* Where a sentence is in what an element shows, as a `Range` over its text
|
|
21
|
+
* — across the bold, the links and the line breaks it is drawn with — or
|
|
22
|
+
* none when the element does not show it as said (a code block said as *a
|
|
23
|
+
* code block*).
|
|
24
|
+
*/
|
|
25
|
+
export declare function sentenceRange(root: Node, sentence: string): Range | undefined;
|
|
26
|
+
/**
|
|
27
|
+
* Underlines `sentence` in the answer marked `data-voice-saying` under
|
|
28
|
+
* `root`, for as long as it is said; the mark goes when it ends. Drawn
|
|
29
|
+
* again whenever `version` changes (the answer still being written).
|
|
30
|
+
*/
|
|
31
|
+
export declare function useSaidSentence(root: RefObject<HTMLElement | null>, sentence: string | undefined, version: unknown): void;
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2025-2026 Datalayer, Inc.
|
|
3
|
+
* Distributed under the terms of the Modified BSD License.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Captions always (VOICE.md VO-65): the sentence being said is marked in the
|
|
7
|
+
* open conversation, not by colour alone. The answer that holds it is marked
|
|
8
|
+
* `aria-current` and `data-voice-saying` with a bar beside it; the sentence
|
|
9
|
+
* itself is underlined in place through the CSS Custom Highlight API, which
|
|
10
|
+
* draws over the text without touching the DOM React owns. A browser without
|
|
11
|
+
* that API keeps the answer's mark.
|
|
12
|
+
*
|
|
13
|
+
* @module voice/captions
|
|
14
|
+
*/
|
|
15
|
+
import { useEffect } from 'react';
|
|
16
|
+
import { plainWords } from './sentences';
|
|
17
|
+
import { answerText } from './useSpokenAnswers';
|
|
18
|
+
/** The name the sentence's highlight is registered under. */
|
|
19
|
+
export const SAID_HIGHLIGHT = 'voice-said';
|
|
20
|
+
/** What the sentence being said looks like: underlined, thick, offset. */
|
|
21
|
+
const SAID_STYLE = `::highlight(${SAID_HIGHLIGHT}) {
|
|
22
|
+
text-decoration: underline;
|
|
23
|
+
text-decoration-thickness: 2px;
|
|
24
|
+
text-underline-offset: 3px;
|
|
25
|
+
background-color: var(--bgColor-accent-muted, rgba(84, 174, 255, 0.2));
|
|
26
|
+
}`;
|
|
27
|
+
/** Words compared as words: spaces collapsed, case kept. */
|
|
28
|
+
function squeezed(text) {
|
|
29
|
+
return text.replace(/\s+/g, ' ').trim();
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* The id of the answer a sentence is being said from: the newest answer of
|
|
33
|
+
* the agent whose words hold it, or none.
|
|
34
|
+
*/
|
|
35
|
+
export function saidMessageId(items, sentence) {
|
|
36
|
+
const said = squeezed(sentence ?? '');
|
|
37
|
+
if (!said) {
|
|
38
|
+
return undefined;
|
|
39
|
+
}
|
|
40
|
+
for (let index = items.length - 1; index >= 0; index -= 1) {
|
|
41
|
+
const text = answerText(items[index]);
|
|
42
|
+
if (text !== undefined && squeezed(plainWords(text)).includes(said)) {
|
|
43
|
+
const id = items[index].id;
|
|
44
|
+
return typeof id === 'string' ? id : undefined;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Where a sentence is in what an element shows, as a `Range` over its text
|
|
51
|
+
* — across the bold, the links and the line breaks it is drawn with — or
|
|
52
|
+
* none when the element does not show it as said (a code block said as *a
|
|
53
|
+
* code block*).
|
|
54
|
+
*/
|
|
55
|
+
export function sentenceRange(root, sentence) {
|
|
56
|
+
const said = squeezed(sentence);
|
|
57
|
+
if (!said || !root.ownerDocument) {
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
const document = root.ownerDocument;
|
|
61
|
+
const walker = document.createTreeWalker(root, 4 /* NodeFilter.SHOW_TEXT */);
|
|
62
|
+
// The text, spaces collapsed, and where each of its characters came from.
|
|
63
|
+
let text = '';
|
|
64
|
+
const from = [];
|
|
65
|
+
let space = true;
|
|
66
|
+
for (let node = walker.nextNode(); node; node = walker.nextNode()) {
|
|
67
|
+
const value = node.nodeValue ?? '';
|
|
68
|
+
for (let offset = 0; offset < value.length; offset += 1) {
|
|
69
|
+
const char = value[offset];
|
|
70
|
+
if (/\s/.test(char)) {
|
|
71
|
+
if (space) {
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
space = true;
|
|
75
|
+
text += ' ';
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
space = false;
|
|
79
|
+
text += char;
|
|
80
|
+
}
|
|
81
|
+
from.push({ node, offset });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const start = text.indexOf(said);
|
|
85
|
+
if (start < 0) {
|
|
86
|
+
return undefined;
|
|
87
|
+
}
|
|
88
|
+
const first = from[start];
|
|
89
|
+
const last = from[start + said.length - 1];
|
|
90
|
+
const range = document.createRange();
|
|
91
|
+
range.setStart(first.node, first.offset);
|
|
92
|
+
range.setEnd(last.node, last.offset + 1);
|
|
93
|
+
return range;
|
|
94
|
+
}
|
|
95
|
+
function registry() {
|
|
96
|
+
const css = globalThis.CSS;
|
|
97
|
+
return css?.highlights;
|
|
98
|
+
}
|
|
99
|
+
function highlightOf(range) {
|
|
100
|
+
const Highlight = globalThis.Highlight;
|
|
101
|
+
return Highlight ? new Highlight(range) : undefined;
|
|
102
|
+
}
|
|
103
|
+
let styled = false;
|
|
104
|
+
function ensureStyle(document) {
|
|
105
|
+
if (styled || document.getElementById('datalayer-voice-said')) {
|
|
106
|
+
styled = true;
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
const style = document.createElement('style');
|
|
110
|
+
style.id = 'datalayer-voice-said';
|
|
111
|
+
style.textContent = SAID_STYLE;
|
|
112
|
+
document.head.appendChild(style);
|
|
113
|
+
styled = true;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Underlines `sentence` in the answer marked `data-voice-saying` under
|
|
117
|
+
* `root`, for as long as it is said; the mark goes when it ends. Drawn
|
|
118
|
+
* again whenever `version` changes (the answer still being written).
|
|
119
|
+
*/
|
|
120
|
+
export function useSaidSentence(root, sentence, version) {
|
|
121
|
+
useEffect(() => {
|
|
122
|
+
const highlights = registry();
|
|
123
|
+
const element = root.current?.querySelector('[data-voice-saying]');
|
|
124
|
+
if (!highlights || !element || !sentence) {
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
const underline = () => {
|
|
128
|
+
const range = sentenceRange(element, sentence);
|
|
129
|
+
const highlight = range ? highlightOf(range) : undefined;
|
|
130
|
+
if (!highlight) {
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
ensureStyle(element.ownerDocument);
|
|
134
|
+
highlights.set(SAID_HIGHLIGHT, highlight);
|
|
135
|
+
return true;
|
|
136
|
+
};
|
|
137
|
+
// The answer's Markdown may be drawn after this effect: wait for it.
|
|
138
|
+
let watcher;
|
|
139
|
+
if (!underline() && typeof MutationObserver !== 'undefined') {
|
|
140
|
+
watcher = new MutationObserver(() => {
|
|
141
|
+
if (underline()) {
|
|
142
|
+
watcher?.disconnect();
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
watcher.observe(element, {
|
|
146
|
+
childList: true,
|
|
147
|
+
subtree: true,
|
|
148
|
+
characterData: true,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
return () => {
|
|
152
|
+
watcher?.disconnect();
|
|
153
|
+
highlights.delete(SAID_HIGHLIGHT);
|
|
154
|
+
};
|
|
155
|
+
}, [root, sentence, version]);
|
|
156
|
+
}
|