@olares/lares-core 0.15.18 → 0.15.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/brand/identity.js +2 -25
- package/i18n/voice.js +2 -0
- package/package.json +1 -1
- package/router/tts.js +6 -2
- package/voice/client.js +9 -2
- package/voice/conversation.js +14 -2
- package/voice/duplex.js +9 -4
- package/voice/reply-stream.js +15 -7
- package/voice/service.js +3 -2
- package/voice/speech.js +10 -11
- package/voice/stream-client.js +1 -13
- package/voice/stream-gateway.js +62 -44
- package/voice/synthesis-run.js +95 -0
package/brand/identity.js
CHANGED
|
@@ -10,11 +10,6 @@ export const UPSTREAM_TITLE = "DeepSeek Harness";
|
|
|
10
10
|
|
|
11
11
|
export const THEME_COLOR = "#6366F1";
|
|
12
12
|
|
|
13
|
-
/** Letter-by-letter, so a model cannot drop the "a" and emit "olores". */
|
|
14
|
-
function platformSpelling() {
|
|
15
|
-
return `Spell the platform name exactly ${PLATFORM_NAME} (${[...PLATFORM_NAME].join(", ")}).`;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
13
|
export function replaceProductTitle(title) {
|
|
19
14
|
return String(title ?? "").split(UPSTREAM_TITLE).join(PRODUCT_NAME);
|
|
20
15
|
}
|
|
@@ -23,11 +18,10 @@ export function replaceProductTitle(title) {
|
|
|
23
18
|
export function identityPrompt() {
|
|
24
19
|
return [
|
|
25
20
|
`You are ${PRODUCT_NAME}, a helpful assistant running on ${PLATFORM_NAME}.`,
|
|
26
|
-
platformSpelling(),
|
|
27
21
|
`Prefer olares-cli for ${PLATFORM_NAME} platform tasks when skills apply.`,
|
|
28
22
|
`olares-cli is on PATH and already signed in as the user: ${PLATFORM_NAME} mounts this application's credential into the container. Never run profile login or profile import — a platform-issued profile refuses both.`,
|
|
29
23
|
"For image, video, audio, or 3D generation, creation, or image editing — including FlowStudio scenes and workflows (T2V, I2V, R2V) — follow the lares-media-create skill: list that family on Router, then call media_generate, which waits for the result and publishes it. Do not curl FlowStudio. Do not diagnose Router or sync catalogs unless that list is empty.",
|
|
30
|
-
"
|
|
24
|
+
"Use read/write/edit for files; use background jobs for long shell work.",
|
|
31
25
|
"User-visible files belong in the session workspace (cwd) as relative paths. Do not write them under /app: that is the application overlay, and the chat preview cannot open it.",
|
|
32
26
|
"For the current date or time, run `date`; the process TZ is the user's configured Olares time zone.",
|
|
33
27
|
"Images the user attaches are already in context; read_image is only for image files that exist on disk, never for an attachment.",
|
|
@@ -46,7 +40,6 @@ export function surfacePrompt(webUrl) {
|
|
|
46
40
|
`You are interacting with the user through ${PRODUCT_NAME} at ${webUrl}.`,
|
|
47
41
|
`When the user refers to "this page", "this GUI", or "this app" without naming another target, they mean ${PRODUCT_NAME}.`,
|
|
48
42
|
"The browser provides no implicit DOM, route, or screenshot context.",
|
|
49
|
-
platformSpelling(),
|
|
50
43
|
`If asked who you are, answer as ${PRODUCT_NAME} on ${PLATFORM_NAME}.`,
|
|
51
44
|
].join(" ");
|
|
52
45
|
}
|
|
@@ -61,9 +54,7 @@ You are helping inside an ${PLATFORM_NAME} workspace via ${PRODUCT_NAME}.
|
|
|
61
54
|
- For image / video / audio / 3D generation, including FlowStudio scenes and workflows, follow the lares-media-create skill: list that family, POST the matching Router shim route, land the file. Do not curl FlowStudio.
|
|
62
55
|
- Stay inside the workspace for file edits unless the user explicitly asks otherwise.
|
|
63
56
|
- Write files the user should open as workspace-relative paths (session cwd). \`/app\` is the running application and is not previewable.
|
|
64
|
-
-
|
|
65
|
-
- Use write or edit only when the user asked you to change or produce that file. Prefer those tools over shell for that file work.
|
|
66
|
-
- ${platformSpelling()}
|
|
57
|
+
- Prefer structured fs tools (read / write / edit) over shell for file work.
|
|
67
58
|
- Long-running shell work can use background jobs; check results with job_output.
|
|
68
59
|
- \`@path\` in a user message is workspace-relative, not absolute; \`/id\` names a skill.
|
|
69
60
|
- If asked who you are, you are ${PRODUCT_NAME} on ${PLATFORM_NAME}.
|
|
@@ -76,20 +67,6 @@ export const LEGACY_AGENTS_SEEDS = [
|
|
|
76
67
|
|
|
77
68
|
You are helping inside an Olares workspace via Lares.
|
|
78
69
|
|
|
79
|
-
- Prefer olares-cli skills for platform tasks (market, cluster, files, router, …).
|
|
80
|
-
- olares-cli is already signed in as the user from the credential Olares mounts into this container. Never run \`profile login\` or \`profile import\`; a platform-issued profile refuses both.
|
|
81
|
-
- For image / video / audio / 3D generation, including FlowStudio scenes and workflows, follow the lares-media-create skill: list that family, POST the matching Router shim route, land the file. Do not curl FlowStudio.
|
|
82
|
-
- Stay inside the workspace for file edits unless the user explicitly asks otherwise.
|
|
83
|
-
- Write files the user should open as workspace-relative paths (session cwd). \`/app\` is the running application and is not previewable.
|
|
84
|
-
- Prefer structured fs tools (read / write / edit) over shell for file work.
|
|
85
|
-
- Long-running shell work can use background jobs; check results with job_output.
|
|
86
|
-
- \`@path\` in a user message is workspace-relative, not absolute; \`/id\` names a skill.
|
|
87
|
-
- If asked who you are, you are Lares on Olares.
|
|
88
|
-
`,
|
|
89
|
-
`# AGENTS.md
|
|
90
|
-
|
|
91
|
-
You are helping inside an Olares workspace via Lares.
|
|
92
|
-
|
|
93
70
|
- Prefer olares-cli skills for platform tasks (market, cluster, files, router, …).
|
|
94
71
|
- For image / video / audio / 3D generation, including FlowStudio scenes and workflows, follow the lares-media-create skill: list that family, POST the matching Router shim route, land the file. Do not curl FlowStudio.
|
|
95
72
|
- Stay inside the workspace for file edits unless the user explicitly asks otherwise.
|
package/i18n/voice.js
CHANGED
|
@@ -140,6 +140,7 @@ export const ZH = {
|
|
|
140
140
|
"error.voice_tts_unavailable": "没有可用的语音播报模型,请先在语音设置里安装",
|
|
141
141
|
"error.voice_speak_disabled": "语音播报是关着的",
|
|
142
142
|
"error.voice_speak_failed": "语音播报失败,请重试",
|
|
143
|
+
"error.voice_playback_failed": "语音播报失败,请重试",
|
|
143
144
|
"error.voice_speak_timeout": "语音播报超时,请重试",
|
|
144
145
|
"error.voice_speak_unready": "当前不能保存语音播报开关",
|
|
145
146
|
"error.voice_gpu_failed": "切换显卡模式失败({detail})。可以点重试;仍然失败,请到 Olares「设置 → GPU」查看显卡状态",
|
|
@@ -423,6 +424,7 @@ export const EN = {
|
|
|
423
424
|
"error.voice_tts_unavailable": "No speech model is available. Install one in voice settings first",
|
|
424
425
|
"error.voice_speak_disabled": "Speech playback is off",
|
|
425
426
|
"error.voice_speak_failed": "Couldn't speak the answer, please retry",
|
|
427
|
+
"error.voice_playback_failed": "Couldn't speak the answer, please retry",
|
|
426
428
|
"error.voice_speak_timeout": "Speech timed out, please retry",
|
|
427
429
|
"error.voice_speak_unready": "Speech playback can't be saved right now",
|
|
428
430
|
"error.voice_gpu_failed":
|
package/package.json
CHANGED
package/router/tts.js
CHANGED
|
@@ -89,6 +89,9 @@ export async function synthesizeStream(request, onChunk) {
|
|
|
89
89
|
const signal = request.signal
|
|
90
90
|
? AbortSignal.any([request.signal, timeout])
|
|
91
91
|
: timeout;
|
|
92
|
+
// A complete PCM body. The Qwen speech engine's streaming generator does not
|
|
93
|
+
// yield, so `stream: true` holds the model slot until the client times out
|
|
94
|
+
// and the readout never leaves "preparing".
|
|
92
95
|
res = await fetch(`${routerShimBaseUrl()}/audio/speech`, {
|
|
93
96
|
method: "POST",
|
|
94
97
|
headers: { "content-type": "application/json", accept: "audio/l16,audio/wav" },
|
|
@@ -96,7 +99,6 @@ export async function synthesizeStream(request, onChunk) {
|
|
|
96
99
|
model: request.model,
|
|
97
100
|
input: text,
|
|
98
101
|
response_format: "pcm",
|
|
99
|
-
stream: true,
|
|
100
102
|
...(request.voice ? { voice: request.voice } : {}),
|
|
101
103
|
}),
|
|
102
104
|
signal,
|
|
@@ -121,15 +123,17 @@ export async function synthesizeStream(request, onChunk) {
|
|
|
121
123
|
if (!Number.isInteger(sampleRate) || sampleRate < 8000 || sampleRate > 192000) throw new VoiceError("voice_audio_format", 502, "missing or invalid PCM sample rate");
|
|
122
124
|
await request.onFormat?.({ sampleRate, channels: 1, encoding: "pcm_s16le" });
|
|
123
125
|
const reader = res.body.getReader();
|
|
126
|
+
let bytes = 0;
|
|
124
127
|
try {
|
|
125
128
|
while (true) {
|
|
126
129
|
const { done, value } = await reader.read();
|
|
127
130
|
if (done) break;
|
|
128
|
-
if (value?.byteLength) await onChunk(Buffer.from(value));
|
|
131
|
+
if (value?.byteLength) { bytes += value.byteLength; await onChunk(Buffer.from(value)); }
|
|
129
132
|
}
|
|
130
133
|
} finally {
|
|
131
134
|
await reader.cancel().catch(() => {});
|
|
132
135
|
reader.releaseLock();
|
|
133
136
|
}
|
|
137
|
+
if (!bytes) throw new VoiceError("voice_audio_format", 502, "speech returned no audio");
|
|
134
138
|
return { streamed: true, sampleRate };
|
|
135
139
|
}
|
package/voice/client.js
CHANGED
|
@@ -106,8 +106,15 @@ export async function postSpeak({ key, text, done, part }) {
|
|
|
106
106
|
};
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
-
export async function postPlaybackStop(signal) {
|
|
110
|
-
const
|
|
109
|
+
export async function postPlaybackStop(source, signal) {
|
|
110
|
+
const body = source && typeof source === "object" && typeof source.addEventListener !== "function" ? source : undefined;
|
|
111
|
+
if (!body || typeof body.sessionId !== "string" || !Number.isSafeInteger(body.turn)) return { stopped: false };
|
|
112
|
+
const res = await fetch(`${API}/playback/stop`, {
|
|
113
|
+
method: "POST",
|
|
114
|
+
signal: body ? signal : source,
|
|
115
|
+
headers: { ...voiceHeaders(), ...(body ? { "content-type": "application/json" } : {}) },
|
|
116
|
+
body: body ? JSON.stringify({ sessionId: body.sessionId, turn: body.turn, step: body.step, attemptId: body.attemptId }) : undefined,
|
|
117
|
+
});
|
|
111
118
|
if (!res.ok) throw new Error(String(res.status));
|
|
112
119
|
return res.json();
|
|
113
120
|
}
|
package/voice/conversation.js
CHANGED
|
@@ -34,7 +34,12 @@ export function createVoiceConversation({ request, transcribe, playback, onReply
|
|
|
34
34
|
const controller = createVoiceController({ voiceId, input, idleTimeoutMs, recoveryTimeoutMs, onExpired: () => stop(), onTranscript: onTranscript ? async (value) => { await onTranscript(value); if (!continuous) await stop(); } : null, metric: (event) => {
|
|
35
35
|
metric(event);
|
|
36
36
|
void call("/session/metric", { stage: event.stage, durationMs: event.durationMs }).catch(() => {});
|
|
37
|
-
}, output: { get canPause() { return playback.canPause === true; }, key: () => playback.key, pause: () => playback.pause?.(), resume: () => playback.resume?.(), async stop() {
|
|
37
|
+
}, output: { get canPause() { return playback.canPause === true; }, key: () => playback.key, pause: () => playback.pause?.(), resume: () => playback.resume?.(), async stop() {
|
|
38
|
+
const reply = lastUtterance;
|
|
39
|
+
playback.stop();
|
|
40
|
+
if (!reply || !Number.isFinite(reply.turn) || typeof lastSessionId !== "string") return;
|
|
41
|
+
await call("/playback/stop", { sessionId: lastSessionId, turn: reply.turn, step: reply.step, attemptId: reply.attemptId });
|
|
42
|
+
} }, agent: {
|
|
38
43
|
lazy: true,
|
|
39
44
|
create: async () => null,
|
|
40
45
|
async submit({ text, requestId, signal }) {
|
|
@@ -67,7 +72,7 @@ export function createVoiceConversation({ request, transcribe, playback, onReply
|
|
|
67
72
|
if (reserve) spokenKey = key;
|
|
68
73
|
controller.observeAgent(Boolean(state.running), Boolean(reserve));
|
|
69
74
|
if (!readAloud || reply?.silent) controller.observePlayback(false);
|
|
70
|
-
if (!turnMuted && readAloud && key !== mutedKey && reply) onReply({ ...reply, sessionId: state.sessionId,
|
|
75
|
+
if (!turnMuted && readAloud && key !== mutedKey && reply) onReply({ ...reply, sessionId: state.sessionId, source: { sessionId: state.sessionId, turn: reply.turn, step: reply.step, attemptId: reply.attemptId }, key: `${voiceId}:${state.sessionId}:${reply.key}` }, headers);
|
|
71
76
|
}
|
|
72
77
|
if (device === "headset" && !state.running && !reading) {
|
|
73
78
|
const audio = await call("/playback/status");
|
|
@@ -106,6 +111,13 @@ export function createVoiceConversation({ request, transcribe, playback, onReply
|
|
|
106
111
|
clearTimeout(timer);
|
|
107
112
|
const initial = onTranscript ? {} : await call("/session", { sessionId, readAloud, presentationSession, ...(preset ? { preset } : {}), ...(backend ? { backend } : {}) });
|
|
108
113
|
if (!shouldStart()) { await stop(); return; }
|
|
114
|
+
// The answer already on screen belongs to the chat reader. Re-subscribing
|
|
115
|
+
// here attaches to a run that reader is about to cancel.
|
|
116
|
+
if (initial?.utterance && typeof initial.sessionId === "string") {
|
|
117
|
+
mute(initial.utterance, initial.sessionId);
|
|
118
|
+
mutedKey = `${initial.sessionId}:${initial.utterance.key}`;
|
|
119
|
+
spokenKey = mutedKey;
|
|
120
|
+
}
|
|
109
121
|
await controller.start({ sessionId, continuous, manual, running: initial.running, shouldStart });
|
|
110
122
|
controller.setOutputMuted(!readAloud);
|
|
111
123
|
if (!onTranscript) void watch();
|
package/voice/duplex.js
CHANGED
|
@@ -5,13 +5,16 @@
|
|
|
5
5
|
export function createDuplexCoordinator(stopPlayback = async () => {}) {
|
|
6
6
|
let speech = null;
|
|
7
7
|
|
|
8
|
-
function beginSpeech(externalSignal) {
|
|
9
|
-
if (speech) {
|
|
8
|
+
function beginSpeech(externalSignal, id = "") {
|
|
9
|
+
if (speech && (!id || speech.id !== id)) {
|
|
10
10
|
speech.controller.abort();
|
|
11
11
|
void Promise.resolve(stopPlayback()).catch(() => {});
|
|
12
|
+
speech = null;
|
|
12
13
|
}
|
|
14
|
+
if (speech && id && speech.id === id) return speech.pub;
|
|
13
15
|
const controller = new AbortController();
|
|
14
16
|
const lease = {
|
|
17
|
+
id,
|
|
15
18
|
controller,
|
|
16
19
|
active() {
|
|
17
20
|
return speech === lease && !controller.signal.aborted;
|
|
@@ -20,6 +23,7 @@ export function createDuplexCoordinator(stopPlayback = async () => {}) {
|
|
|
20
23
|
if (speech === lease) speech = null;
|
|
21
24
|
},
|
|
22
25
|
};
|
|
26
|
+
lease.pub = { signal: controller.signal, active: lease.active, finish: lease.finish };
|
|
23
27
|
speech = lease;
|
|
24
28
|
if (externalSignal) {
|
|
25
29
|
const abort = () => {
|
|
@@ -30,10 +34,11 @@ export function createDuplexCoordinator(stopPlayback = async () => {}) {
|
|
|
30
34
|
if (externalSignal.aborted) abort();
|
|
31
35
|
else externalSignal.addEventListener("abort", abort, { once: true });
|
|
32
36
|
}
|
|
33
|
-
return
|
|
37
|
+
return lease.pub;
|
|
34
38
|
}
|
|
35
39
|
|
|
36
|
-
async function interrupt() {
|
|
40
|
+
async function interrupt(id = "") {
|
|
41
|
+
if (id && speech?.id !== id) return;
|
|
37
42
|
const current = speech;
|
|
38
43
|
speech = null;
|
|
39
44
|
current?.controller.abort();
|
package/voice/reply-stream.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { VoiceError } from "../router/stt.js";
|
|
1
2
|
import { reduceUtterance } from "./speech.js";
|
|
2
3
|
import { createSpeechSegmenter } from "./script.js";
|
|
3
4
|
|
|
4
|
-
const
|
|
5
|
+
const STATUS = { voice_reply_invalid: 400, voice_reply_forbidden: 403 };
|
|
6
|
+
const fault = code => new VoiceError(code, STATUS[code] ?? 409, code);
|
|
5
7
|
const identity = source => JSON.stringify([source.sessionId, source.turn, source.step]);
|
|
6
8
|
|
|
7
9
|
/** Only trusted Agent events may authorize live text; client input supplies an address, never text. */
|
|
@@ -10,7 +12,7 @@ export function createReplyStreamRegistry({ inspect, owner, now = Date.now, segm
|
|
|
10
12
|
const notify = row => { row.version = (row.version || 0) + 1; for (const wake of row.listeners) wake(); };
|
|
11
13
|
const fail = (row, code) => { row.segmenter.reset(); row.ready = []; row.error = code; row.done = true; row.at = now(); notify(row); };
|
|
12
14
|
const advance = row => {
|
|
13
|
-
if (!row.
|
|
15
|
+
if (!row.subscribers || row.error || !row.eligible && !row.done) return;
|
|
14
16
|
try { row.ready.push(...row.segmenter.update(row.utterance?.text || "", { done: row.done })); }
|
|
15
17
|
catch (error) { fail(row, error.message === "voice_reply_rewritten" ? error.message : "voice_reply_invalid"); }
|
|
16
18
|
};
|
|
@@ -26,7 +28,7 @@ export function createReplyStreamRegistry({ inspect, owner, now = Date.now, segm
|
|
|
26
28
|
}
|
|
27
29
|
if (rows.size >= 128) return;
|
|
28
30
|
rows.set(key, { sessionId, turn: frame.turn, step: frame.step, attemptId: frame.attemptId,
|
|
29
|
-
segmenter: segmenterFactory(), ready: [], eligible: frame.speechEligible === true, utterance: null, next: 0, done: false, error: null, at: now(), listeners: new Set() });
|
|
31
|
+
segmenter: segmenterFactory(), ready: [], eligible: frame.speechEligible === true, utterance: null, next: 0, done: false, error: null, at: now(), listeners: new Set(), subscribers: 0 });
|
|
30
32
|
return;
|
|
31
33
|
}
|
|
32
34
|
for (const row of rows.values()) {
|
|
@@ -59,10 +61,10 @@ export function createReplyStreamRegistry({ inspect, owner, now = Date.now, segm
|
|
|
59
61
|
await inspect(source.sessionId);
|
|
60
62
|
const row = rows.get(identity(source));
|
|
61
63
|
if (!row || source.attemptId && source.attemptId !== row.attemptId) throw fault("voice_reply_unavailable");
|
|
62
|
-
if (row.claimed) throw fault("voice_output_owned");
|
|
63
|
-
row.claimed = true; advance(row);
|
|
64
64
|
return async function* segments(signal) {
|
|
65
65
|
let index = 0;
|
|
66
|
+
row.subscribers += 1;
|
|
67
|
+
advance(row);
|
|
66
68
|
try {
|
|
67
69
|
while (!signal.aborted) {
|
|
68
70
|
if (row.error) throw fault(row.error);
|
|
@@ -86,11 +88,17 @@ export function createReplyStreamRegistry({ inspect, owner, now = Date.now, segm
|
|
|
86
88
|
}
|
|
87
89
|
throw fault("voice_cancelled");
|
|
88
90
|
} finally {
|
|
89
|
-
|
|
90
|
-
row.segmenter.reset();
|
|
91
|
+
row.subscribers -= 1;
|
|
91
92
|
}
|
|
92
93
|
};
|
|
93
94
|
},
|
|
95
|
+
/** Ends this answer for every subscriber. A later turn is a different row. */
|
|
96
|
+
cancel(source) {
|
|
97
|
+
if (!source || typeof source.sessionId !== "string") return;
|
|
98
|
+
const row = rows.get(identity(source));
|
|
99
|
+
if (!row || row.done) return;
|
|
100
|
+
fail(row, "voice_cancelled");
|
|
101
|
+
},
|
|
94
102
|
remove(sessionId) { for (const [key, row] of rows) if (row.sessionId === sessionId) { fail(row, "voice_session_closed"); rows.delete(key); } },
|
|
95
103
|
close() { for (const row of rows.values()) fail(row, "voice_session_closed"); rows.clear(); },
|
|
96
104
|
};
|
package/voice/service.js
CHANGED
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
} from "./session.js";
|
|
23
23
|
import { createCaptureSession } from "./capture.js";
|
|
24
24
|
import { createDuplexCoordinator } from "./duplex.js";
|
|
25
|
+
import { utteranceKey } from "./synthesis-run.js";
|
|
25
26
|
import { createDeviceLease } from "./lease.js";
|
|
26
27
|
import { normalizeSttPcm } from "./pcm.js";
|
|
27
28
|
import { gapAfter, createSpeechSegmenter } from "./script.js";
|
|
@@ -486,8 +487,8 @@ async function readAloud(readout, model, voice, lease, options) {
|
|
|
486
487
|
|
|
487
488
|
export function playbackStatus(scope) { return { playing: (scope?.duplex ?? duplex).isSpeaking() }; }
|
|
488
489
|
|
|
489
|
-
export function interruptSpeech(scope) {
|
|
490
|
-
return (scope?.duplex ?? duplex).interrupt();
|
|
490
|
+
export function interruptSpeech(scope, source) {
|
|
491
|
+
return (scope?.duplex ?? duplex).interrupt(utteranceKey(source));
|
|
491
492
|
}
|
|
492
493
|
|
|
493
494
|
export function languageForTranscribe(requestedLanguage, configLanguage) {
|
package/voice/speech.js
CHANGED
|
@@ -192,9 +192,6 @@ export function createSentencePlayer({ send, play, halt, silence, fail = () => {
|
|
|
192
192
|
let lastKey = null;
|
|
193
193
|
let silencedKey = null;
|
|
194
194
|
const consumed = new Set();
|
|
195
|
-
const mutedTurns = new Set();
|
|
196
|
-
let lastTurn = null;
|
|
197
|
-
const turnKey = (value) => Number.isFinite(value.turn) ? `${value.sessionId ?? ""}:${value.turn}` : null;
|
|
198
195
|
const mark = (set, key) => { if (key != null) set.add(key); };
|
|
199
196
|
|
|
200
197
|
const emitOutput = value => {
|
|
@@ -273,20 +270,20 @@ export function createSentencePlayer({ send, play, halt, silence, fail = () => {
|
|
|
273
270
|
if (utterance.silent) {
|
|
274
271
|
if (utterance.key !== lastKey || silencedKey === lastKey) return;
|
|
275
272
|
silencedKey = lastKey;
|
|
273
|
+
const active = Boolean(current);
|
|
274
|
+
const source = current?.source;
|
|
276
275
|
stop();
|
|
277
|
-
silence();
|
|
276
|
+
if (active) silence(source);
|
|
278
277
|
return;
|
|
279
278
|
}
|
|
280
279
|
if (!utterance.text) return;
|
|
281
|
-
const identity = turnKey(utterance);
|
|
282
|
-
if (identity && mutedTurns.has(identity)) return;
|
|
283
280
|
// A later tool call can invalidate uncommitted assistant text.
|
|
284
281
|
if (stream && !utterance.done && !utterance.source) return;
|
|
285
282
|
if (current?.key !== utterance.key) {
|
|
286
283
|
if (consumed.has(utterance.key)) return;
|
|
287
|
-
if (current) {
|
|
284
|
+
if (current) { const source = current.source; stop(); silence(source); }
|
|
288
285
|
gen += 1;
|
|
289
|
-
lastKey = utterance.key;
|
|
286
|
+
lastKey = utterance.key; mark(consumed, lastKey);
|
|
290
287
|
current = {
|
|
291
288
|
gen,
|
|
292
289
|
abort: new AbortController(),
|
|
@@ -302,7 +299,7 @@ export function createSentencePlayer({ send, play, halt, silence, fail = () => {
|
|
|
302
299
|
};
|
|
303
300
|
current.identity = utterance;
|
|
304
301
|
const run = current;
|
|
305
|
-
onState(true); emitOutput({ ...utterance, status: "preparing", stop: () => { mark(
|
|
302
|
+
onState(true); emitOutput({ ...utterance, status: "preparing", stop: () => { mark(consumed, run.key); const source = run.source; stop(); silence(source); } });
|
|
306
303
|
handOver(run);
|
|
307
304
|
read(run).catch((err) => failed(run, err));
|
|
308
305
|
return;
|
|
@@ -316,9 +313,11 @@ export function createSentencePlayer({ send, play, halt, silence, fail = () => {
|
|
|
316
313
|
pause() { if (!current?.controls?.canPause) return false; current.controls.pause(); emitOutput({ key: current.key, status: "confirming" }); return true; },
|
|
317
314
|
resume() { if (!current?.controls?.canPause) return false; current.controls.resume(); emitOutput({ key: current.key, status: "playing" }); return true; },
|
|
318
315
|
stop() {
|
|
319
|
-
|
|
316
|
+
if (!current) return;
|
|
317
|
+
mark(consumed, current.key);
|
|
318
|
+
const source = current.source;
|
|
320
319
|
stop();
|
|
321
|
-
silence();
|
|
320
|
+
silence(source);
|
|
322
321
|
},
|
|
323
322
|
};
|
|
324
323
|
}
|
package/voice/stream-client.js
CHANGED
|
@@ -33,19 +33,7 @@ function streamMetric(options, event) {
|
|
|
33
33
|
export async function playAudioStream(options) {
|
|
34
34
|
const started = Date.now();
|
|
35
35
|
const connection = await openAudioStream({ ...options, operation: "tts" });
|
|
36
|
-
if (!connection)
|
|
37
|
-
const response = await options.request("/api/lares/voice/speak", { method: "POST", headers: options.headers, signal: options.signal, body: { text: options.text } });
|
|
38
|
-
if (!response?.ok) throw new Error(response?.body?.error?.code || "voice_stream_failed");
|
|
39
|
-
if (response.body.sentences === 0) return true;
|
|
40
|
-
if (response.body.bluetooth === "played") return true;
|
|
41
|
-
if (options.source) throw new Error("voice_reply_unavailable");
|
|
42
|
-
const fallback = createBrowserPlayer();
|
|
43
|
-
options.controls?.({ canPause: true, pause: () => fallback.pause(), resume: () => fallback.resume() });
|
|
44
|
-
options.onStatus?.("playing");
|
|
45
|
-
const cancel = () => fallback.stop(); options.signal?.addEventListener("abort", cancel, { once: true });
|
|
46
|
-
try { if (!options.signal?.aborted) await fallback.play(wavBlob(response.body.audio)); } finally { options.signal?.removeEventListener("abort", cancel); fallback.stop(); }
|
|
47
|
-
return true;
|
|
48
|
-
}
|
|
36
|
+
if (!connection) throw new Error("voice_stream_failed");
|
|
49
37
|
const file = createBrowserPlayer();
|
|
50
38
|
const player = createPcmStreamPlayer({ onProgress: (frames) => { if (connection.readyState === 1) connection.send(JSON.stringify({ type: "progress", frames })); },
|
|
51
39
|
onMetric: (event) => { if (event.stage === "playback_started") options.onStatus?.("playing"); streamMetric(options, { ...event, durationMs: Date.now() - started }); } });
|
package/voice/stream-gateway.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ttsVoiceForModel } from "./tts-voice.js";
|
|
2
2
|
import { speechSegments } from "./script.js";
|
|
3
|
+
import { createSynthesisRuns, utteranceKey } from "./synthesis-run.js";
|
|
3
4
|
import { randomUUID } from "node:crypto";
|
|
4
5
|
import { WebSocket, WebSocketServer } from "ws";
|
|
5
6
|
import { synthesizeStream } from "../router/tts.js";
|
|
@@ -11,13 +12,14 @@ const fault = (code, status = 409) => new VoiceError(code, status, code);
|
|
|
11
12
|
/** Single-use HTTP grants bind raw audio sockets to the authenticated Host scope. */
|
|
12
13
|
export function createAudioStreamGateway({ config, models, replies, synthesize = synthesizeStream, env = process.env, metric = () => {} }) {
|
|
13
14
|
const tickets = new Map(), clients = new Set();
|
|
15
|
+
const synthesis = createSynthesisRuns({ synthesize, metric });
|
|
14
16
|
const server = new WebSocketServer({ noServer: true, maxPayload: 96 * 1024, perMessageDeflate: false,
|
|
15
17
|
handleProtocols: (values) => values.has("lares-voice") ? "lares-voice" : false });
|
|
16
18
|
return {
|
|
17
19
|
async ticket(scope, body) {
|
|
18
|
-
if (scope.device !== "device") return { supported: false };
|
|
19
20
|
if (!["tts", "stt"].includes(body.operation)) throw fault("voice_config_invalid", 400);
|
|
20
21
|
const tts = body.operation === "tts";
|
|
22
|
+
if (!tts && scope.device !== "device") return { supported: false };
|
|
21
23
|
const snapshot = structuredClone(scope.voiceConfig || config());
|
|
22
24
|
if (tts && (scope.presentationSession || !scope.active)) {
|
|
23
25
|
const current = config();
|
|
@@ -35,7 +37,7 @@ export function createAudioStreamGateway({ config, models, replies, synthesize =
|
|
|
35
37
|
const reply = tts && body.source ? await replies?.bind(scope, body.source) : null;
|
|
36
38
|
if (body.source && !reply) throw fault("voice_reply_unavailable");
|
|
37
39
|
const ticket = randomUUID();
|
|
38
|
-
tickets.set(ticket, { scope, model: selected, operation: body.operation, text: body.text, reply,
|
|
40
|
+
tickets.set(ticket, { scope, model: selected, operation: body.operation, text: body.text, reply, replySource: body.source || null,
|
|
39
41
|
config: snapshot, language: body.language || snapshot.language, expires: Date.now() + 30000 });
|
|
40
42
|
return { supported: true, ticket };
|
|
41
43
|
},
|
|
@@ -95,52 +97,68 @@ export function createAudioStreamGateway({ config, models, replies, synthesize =
|
|
|
95
97
|
upstream.on("error", error);
|
|
96
98
|
upstream.on("close", () => { send({ type: "eof" }); close(); });
|
|
97
99
|
} else {
|
|
98
|
-
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
const {
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
if (!lease.active() || abort.signal.aborted) throw fault("voice_cancelled");
|
|
100
|
+
const key = grant.replySource ? utteranceKey(grant.replySource) : `text:${grant.text}`;
|
|
101
|
+
const run = synthesis.attach(key, (signal) => grant.reply ? grant.reply(signal) : (async function* text() {
|
|
102
|
+
const parts = speechSegments(grant.text, { done: true });
|
|
103
|
+
for (let index = 0; index < parts.length; index += 1) yield { text: parts[index], index };
|
|
104
|
+
})(), { model: grant.model, voice: ttsVoiceForModel(grant.config, grant.model), voiceId: grant.scope.id });
|
|
105
|
+
let cursor = 0;
|
|
106
|
+
const unsub = run.subscribe(() => { waiter?.(); waiter = null; });
|
|
107
|
+
const waitFor = () => new Promise((resolve, reject) => {
|
|
108
|
+
if (abort.signal.aborted) { reject(fault("voice_cancelled")); return; }
|
|
109
|
+
const wake = () => { abort.signal.removeEventListener("abort", wake); waiter = null; if (abort.signal.aborted) reject(fault("voice_cancelled")); else resolve(); };
|
|
110
|
+
waiter = wake;
|
|
111
|
+
abort.signal.addEventListener("abort", wake, { once: true });
|
|
112
|
+
});
|
|
113
|
+
const pump = async () => {
|
|
114
|
+
while (!abort.signal.aborted) {
|
|
115
|
+
if (cursor >= run.events.length) {
|
|
116
|
+
if (run.settled) return;
|
|
117
|
+
await waitFor();
|
|
118
|
+
continue;
|
|
118
119
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
120
|
+
const event = run.events[cursor++];
|
|
121
|
+
if (event.type === "segment") send({ type: "segment", index: event.index, attemptId: event.attemptId, startFrame: sentFrames });
|
|
122
|
+
else if (event.type === "format") {
|
|
123
|
+
const format = event.format;
|
|
124
|
+
if (outputFormat && (format.channels !== outputFormat.channels || format.encoding !== outputFormat.encoding)) throw fault("voice_audio_format");
|
|
125
|
+
outputFormat = format;
|
|
126
|
+
if (!rate) { rate = format.sampleRate; send({ type: "format", ...format }); }
|
|
127
|
+
else if (rate !== format.sampleRate) throw fault("voice_audio_format");
|
|
128
|
+
} else if (event.type === "audio") {
|
|
129
|
+
if (!rate) throw fault("voice_audio_format", 502);
|
|
130
|
+
for (let offset = 0; offset < event.chunk.length; offset += 16384) {
|
|
131
|
+
while (pausedAt || sentFrames - playedFrames > rate * 2 || client.bufferedAmount > 128 * 1024) {
|
|
132
|
+
await new Promise((resolve, reject) => { const timer = setTimeout(() => { waiter = null; reject(fault(pausedAt ? "voice_interruption_timeout" : "voice_audio_backpressure")); }, pausedAt ? Math.max(1, 10000 - (Date.now() - pausedAt)) : 10000); waiter = () => { clearTimeout(timer); resolve(); }; });
|
|
133
|
+
if (abort.signal.aborted || client.readyState !== 1) throw fault("voice_cancelled");
|
|
134
|
+
}
|
|
135
|
+
if (client.readyState !== 1) throw fault("voice_cancelled");
|
|
136
|
+
const part = event.chunk.subarray(offset, offset + 16384);
|
|
137
|
+
sentFrames += part.length / 2;
|
|
138
|
+
client.send(part);
|
|
139
|
+
if (!outputStarted) { outputStarted = true; metric({ stage: "synthesis_first_audio", voiceId: grant.scope.id, at: Date.now(), durationMs: Date.now() - started }); }
|
|
140
|
+
}
|
|
141
|
+
} else if (event.type === "segment.end") send({ type: "segment.end", index: event.index, endFrame: sentFrames });
|
|
142
|
+
else if (event.type === "file") {
|
|
143
|
+
if (rate) throw fault("voice_audio_format");
|
|
144
|
+
await new Promise((resolve, reject) => {
|
|
145
|
+
const timer = setTimeout(() => reject(fault("voice_timeout")), 120000);
|
|
146
|
+
fileDone = () => { clearTimeout(timer); resolve(); };
|
|
147
|
+
send({ type: "file", audio: Buffer.from(event.audio).toString("base64") });
|
|
148
|
+
});
|
|
149
|
+
} else if (event.type === "done") { send({ type: "done" }); return; }
|
|
150
|
+
else if (event.type === "cancelled") throw fault("voice_cancelled");
|
|
151
|
+
else if (event.type === "error") throw fault(event.code || "voice_stream_failed");
|
|
124
152
|
}
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
if (carry !== null) throw fault("voice_audio_format");
|
|
128
|
-
if (!result.streamed) {
|
|
129
|
-
if (rate) throw fault("voice_audio_format");
|
|
130
|
-
await new Promise((resolve, reject) => {
|
|
131
|
-
const timer = setTimeout(() => reject(fault("voice_timeout")), 120000);
|
|
132
|
-
fileDone = () => { clearTimeout(timer); resolve(); };
|
|
133
|
-
send({ type: "file", audio: result.audio.toString("base64") });
|
|
134
|
-
});
|
|
135
|
-
}
|
|
136
|
-
if (!lease.active()) throw fault("voice_cancelled");
|
|
137
|
-
} };
|
|
138
|
-
const wake = () => { waiter?.(); waiter = null; fileDone?.(); fileDone = null; close(); };
|
|
139
|
-
lease.signal.addEventListener("abort", wake, { once: true });
|
|
140
|
-
void run().then(() => { if (!abort.signal.aborted) send({ type: "done" }); }).catch(error).finally(() => { lease.signal.removeEventListener("abort", wake); lease.finish(); });
|
|
153
|
+
};
|
|
154
|
+
void pump().catch((cause) => { if (!abort.signal.aborted) error(cause); }).finally(() => { unsub(); close(); });
|
|
141
155
|
}
|
|
142
156
|
});
|
|
143
157
|
},
|
|
144
|
-
|
|
158
|
+
cancel(source) {
|
|
159
|
+
synthesis.cancel(utteranceKey(source));
|
|
160
|
+
replies?.cancel?.(source);
|
|
161
|
+
},
|
|
162
|
+
close() { tickets.clear(); synthesis.close(); for (const client of clients) client.terminate(); server.close(); },
|
|
145
163
|
};
|
|
146
164
|
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/** One answer is `sessionId + turn + step + attemptId`. Listeners share that id. */
|
|
2
|
+
export function utteranceKey(source) {
|
|
3
|
+
if (!source || typeof source.sessionId !== "string" || !Number.isSafeInteger(source.turn) || !Number.isSafeInteger(source.step)) return "";
|
|
4
|
+
return JSON.stringify([source.sessionId, source.turn, source.step, source.attemptId || ""]);
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Synthesizes each answer once and fans the PCM out.
|
|
9
|
+
* Cancelling stops delivery immediately. The current response body is still
|
|
10
|
+
* read to the end before the next request is allowed to start, so a cut-off
|
|
11
|
+
* transfer cannot occupy the speech path for the following answer.
|
|
12
|
+
*/
|
|
13
|
+
export function createSynthesisRuns({ synthesize, metric = () => {} }) {
|
|
14
|
+
const runs = new Map();
|
|
15
|
+
let chain = Promise.resolve();
|
|
16
|
+
const enqueue = (task) => {
|
|
17
|
+
const next = chain.then(task, task);
|
|
18
|
+
chain = next.then(() => {}, () => {});
|
|
19
|
+
return next;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
function start(key, open, request) {
|
|
23
|
+
const subscribers = new Set();
|
|
24
|
+
const events = [];
|
|
25
|
+
const abort = new AbortController();
|
|
26
|
+
let cancelled = false;
|
|
27
|
+
let settled = false;
|
|
28
|
+
let carry = null;
|
|
29
|
+
const notify = () => { for (const listener of subscribers) listener(); };
|
|
30
|
+
const push = (event) => { events.push(event); notify(); };
|
|
31
|
+
const finished = (async () => {
|
|
32
|
+
const segments = open(abort.signal);
|
|
33
|
+
try {
|
|
34
|
+
for await (const segment of segments) {
|
|
35
|
+
if (cancelled) break;
|
|
36
|
+
metric({ stage: "synthesis_segment", voiceId: request.voiceId, index: segment.index, chars: segment.text?.length || 0, at: Date.now() });
|
|
37
|
+
push({ type: "segment", index: segment.index, attemptId: segment.attemptId });
|
|
38
|
+
const result = await enqueue(async () => {
|
|
39
|
+
if (cancelled) return { skipped: true };
|
|
40
|
+
return synthesize({
|
|
41
|
+
text: segment.text,
|
|
42
|
+
model: request.model,
|
|
43
|
+
voice: request.voice,
|
|
44
|
+
onFormat(format) { push({ type: "format", format }); },
|
|
45
|
+
}, async (chunk) => {
|
|
46
|
+
if (cancelled || !chunk?.byteLength) return;
|
|
47
|
+
let buf = Buffer.from(chunk);
|
|
48
|
+
if (carry !== null) { buf = Buffer.concat([Buffer.from([carry]), buf]); carry = null; }
|
|
49
|
+
if (buf.length % 2) { carry = buf[buf.length - 1]; buf = buf.subarray(0, -1); }
|
|
50
|
+
if (!buf.length) return;
|
|
51
|
+
push({ type: "audio", chunk: buf });
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
if (cancelled || result?.skipped) break;
|
|
55
|
+
if (carry !== null) throw Object.assign(new Error("voice_audio_format"), { code: "voice_audio_format" });
|
|
56
|
+
push({ type: "segment.end", index: segment.index });
|
|
57
|
+
if (!result.streamed) {
|
|
58
|
+
push({ type: "file", audio: result.audio });
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
push(cancelled ? { type: "cancelled" } : { type: "done" });
|
|
63
|
+
} catch (err) {
|
|
64
|
+
const code = err?.code || err?.message || "voice_failed";
|
|
65
|
+
push(cancelled || code === "voice_cancelled" ? { type: "cancelled" } : { type: "error", code });
|
|
66
|
+
} finally {
|
|
67
|
+
settled = true;
|
|
68
|
+
notify();
|
|
69
|
+
await segments?.return?.().catch(() => {});
|
|
70
|
+
}
|
|
71
|
+
})();
|
|
72
|
+
return {
|
|
73
|
+
events,
|
|
74
|
+
get settled() { return settled; },
|
|
75
|
+
subscribe(listener) { subscribers.add(listener); listener(); return () => subscribers.delete(listener); },
|
|
76
|
+
cancel() { if (cancelled || settled) return; cancelled = true; abort.abort(); notify(); },
|
|
77
|
+
finished,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
/** `open` is used only for the first subscriber of `key`. */
|
|
83
|
+
attach(key, open, request) {
|
|
84
|
+
let run = runs.get(key);
|
|
85
|
+
if (!run) {
|
|
86
|
+
run = start(key, open, request);
|
|
87
|
+
runs.set(key, run);
|
|
88
|
+
run.finished.finally(() => { if (runs.get(key) === run) runs.delete(key); });
|
|
89
|
+
}
|
|
90
|
+
return run;
|
|
91
|
+
},
|
|
92
|
+
cancel(key) { if (key) runs.get(key)?.cancel(); },
|
|
93
|
+
close() { for (const run of runs.values()) run.cancel(); runs.clear(); },
|
|
94
|
+
};
|
|
95
|
+
}
|