@olares/lares-core 0.15.11 → 0.15.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/i18n/voice.js +180 -10
- package/larepass/runtime.js +2 -0
- package/larepass/settings.js +3 -0
- package/larepass/transcript.js +7 -0
- package/package.json +28 -2
- package/router/tts.js +1 -1
- package/tools/http.js +3 -2
- package/voice/agent-session.js +134 -0
- package/voice/agent.js +9 -1
- package/voice/browser-capture.js +274 -0
- package/voice/browser-playback.js +30 -0
- package/voice/capture.js +12 -2
- package/voice/client.js +29 -14
- package/voice/composite-gateway.js +156 -0
- package/voice/config.js +28 -18
- package/voice/controller.js +154 -0
- package/voice/conversation.js +135 -0
- package/voice/device-input.js +79 -0
- package/voice/dialogue.js +99 -0
- package/voice/duplex.js +1 -1
- package/voice/external.js +26 -0
- package/voice/gpu-mode.js +147 -59
- package/voice/gpu-problem.js +47 -0
- package/voice/host/bluetooth-ready.py +2 -0
- package/voice/host-input.js +38 -0
- package/voice/host-service.js +216 -0
- package/voice/input.js +2 -1
- package/voice/install.js +9 -5
- package/voice/languages.js +1 -1
- package/voice/lease.js +16 -0
- package/voice/options.js +23 -0
- package/voice/panel.js +88 -0
- package/voice/projection.js +67 -0
- package/voice/protocol-client.js +24 -0
- package/voice/realtime-audio.js +107 -0
- package/voice/realtime-conversation.js +126 -0
- package/voice/realtime-gateway.js +168 -0
- package/voice/realtime.js +63 -0
- package/voice/records.js +96 -0
- package/voice/router-audio.js +26 -0
- package/voice/script.js +33 -12
- package/voice/service.js +185 -38
- package/voice/session.js +11 -1
- package/voice/speech.js +9 -3
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { createVoiceAgentSession } from "./agent-session.js";
|
|
2
|
+
import { createVoiceRecords } from "./records.js";
|
|
3
|
+
import { createVoiceDialogue, cleanDictation } from "./dialogue.js";
|
|
4
|
+
import { readConfig } from "./config.js";
|
|
5
|
+
import { VOICE_PRESETS } from "./options.js";
|
|
6
|
+
|
|
7
|
+
const failure = (code, status) => Object.assign(new Error(code), { code, status });
|
|
8
|
+
|
|
9
|
+
/** Presets and task policy stay in core; the Host supplies an authenticated scope. */
|
|
10
|
+
export function createVoiceHostService(api, { env = process.env, metric = () => {}, records = createVoiceRecords(), config = readConfig, chat } = {}) {
|
|
11
|
+
const ownerOf = (scope) => scope.owner || "@local";
|
|
12
|
+
const keyOf = (scope) => scope.session?.snapshot().sessionId || `voice:${scope.contextId || scope.id}`;
|
|
13
|
+
const append = (scope, role, text, turnId, taskId = null, fields = {}) => records.append(ownerOf(scope), keyOf(scope), {
|
|
14
|
+
id: `${scope.id}:${turnId}:${role}`, voiceId: scope.id, clientId: scope.id, turnId, role, text: text.slice(0, 16000), taskId, anchorSeq: scope.session?.snapshot().cursor ?? -1, ...fields,
|
|
15
|
+
});
|
|
16
|
+
const ensurePresentation = async (scope) => {
|
|
17
|
+
if (!scope.presentationSession || scope.session.snapshot().sessionId) return;
|
|
18
|
+
const before = keyOf(scope);
|
|
19
|
+
await scope.session.ensureSession();
|
|
20
|
+
if (!scope.active) throw failure("voice_session_closed", 409);
|
|
21
|
+
for (const row of records.read(ownerOf(scope), before).records) records.append(ownerOf(scope), keyOf(scope), row);
|
|
22
|
+
records.clear(ownerOf(scope), before);
|
|
23
|
+
records.bind(ownerOf(scope), scope.contextId, keyOf(scope));
|
|
24
|
+
};
|
|
25
|
+
const scopes = new Set();
|
|
26
|
+
const service = {
|
|
27
|
+
async clean(scope, text) {
|
|
28
|
+
if (typeof text !== "string" || text.length > 16000) throw failure("voice_input_invalid", 400);
|
|
29
|
+
return cleanDictation(text, { config: config(), owner: scope.owner, chat });
|
|
30
|
+
},
|
|
31
|
+
async open(scope, body) {
|
|
32
|
+
if (scope.active || scope.opening || scope.closing) throw failure("voice_session_active", 409);
|
|
33
|
+
const epoch = scope.epoch = (scope.epoch || 0) + 1;
|
|
34
|
+
scope.opening = true;
|
|
35
|
+
try {
|
|
36
|
+
if (body.sessionId != null && (typeof body.sessionId !== "string" || !body.sessionId || body.sessionId.length > 256)) throw failure("voice_session_invalid", 400);
|
|
37
|
+
if (body.contextId != null && (typeof body.contextId !== "string" || !/^[a-zA-Z0-9_-]{8,128}$/.test(body.contextId))) throw failure("voice_context_invalid", 400);
|
|
38
|
+
scope.presentationSession = body.presentationSession === true;
|
|
39
|
+
scope.contextId = body.contextId || scope.id;
|
|
40
|
+
const binding = records.read(ownerOf(scope), `voice:${scope.contextId}`).binding;
|
|
41
|
+
const boundSessionId = body.sessionId || binding || null;
|
|
42
|
+
if (boundSessionId) await api.inspect(boundSessionId);
|
|
43
|
+
if (scope.epoch !== epoch) throw failure("voice_session_closed", 409);
|
|
44
|
+
await scope.session?.close();
|
|
45
|
+
const selected = config();
|
|
46
|
+
const preset = body.preset || selected.preset;
|
|
47
|
+
if (!VOICE_PRESETS.includes(preset)) throw failure("voice_preset_invalid", 400);
|
|
48
|
+
scopes.add(scope);
|
|
49
|
+
scope.voiceAbort = new AbortController();
|
|
50
|
+
scope.voiceConfig = { ...selected, preset };
|
|
51
|
+
const history = records.read(ownerOf(scope), boundSessionId || `voice:${scope.contextId || scope.id}`);
|
|
52
|
+
scope.recoveryContext = history.summary || null;
|
|
53
|
+
const restoredRequests = [...new Set(history.records.map((row) => row.taskId).filter(Boolean))].map((requestId) => ({ requestId }));
|
|
54
|
+
scope.session = createVoiceAgentSession(api, { sessionId: boundSessionId,
|
|
55
|
+
preset: scope.authorizedBinding?.agentPreset || env.LARES_VOICE_AGENT_PRESET || undefined, workspaceId: scope.authorizedBinding?.workspaceId || env.LARES_VOICE_WORKSPACE_ID || undefined, restoredRequests,
|
|
56
|
+
onChange: () => { if (scope.active) service.state(scope); },
|
|
57
|
+
metric: (fields) => metric({ voiceId: scope.id, ...fields }),
|
|
58
|
+
});
|
|
59
|
+
await scope.session.open();
|
|
60
|
+
if (scope.epoch !== epoch) throw failure("voice_session_closed", 409);
|
|
61
|
+
scope.recordKey = keyOf(scope);
|
|
62
|
+
scope.voiceResults = []; scope.summaryError = null;
|
|
63
|
+
scope.active = true; scope.readAloud = body.readAloud === true;
|
|
64
|
+
scope.voiceState = { version: 0, running: false, utterance: null, error: null };
|
|
65
|
+
scope.voiceRequests = new Map(); scope.deliveredTasks = new Set(restoredRequests.map((item) => item.requestId));
|
|
66
|
+
scope.dialogue = createVoiceDialogue({ config: scope.voiceConfig, owner: scope.owner, context: history.summary?.recent || [], summary: history.summary?.text || "", chat,
|
|
67
|
+
submit: async (text, requestId, operation) => {
|
|
68
|
+
if (operation === "status") return scope.session.snapshot();
|
|
69
|
+
const id = `${requestId}:delegate`;
|
|
70
|
+
// The source transcript is persisted before any side effect, even for lazy business sessions.
|
|
71
|
+
const before = keyOf(scope);
|
|
72
|
+
const source = records.read(ownerOf(scope), keyOf(scope)).records.find((row) => row.id === `${scope.id}:${requestId}:user`);
|
|
73
|
+
if (source) append(scope, "user", source.text, requestId, id);
|
|
74
|
+
append(scope, "task", text, requestId, id);
|
|
75
|
+
let state;
|
|
76
|
+
try { state = await scope.session.submit(text, id, scope.submitMode || "queue"); }
|
|
77
|
+
finally { if (before !== keyOf(scope)) {
|
|
78
|
+
for (const row of records.read(ownerOf(scope), before).records) records.append(ownerOf(scope), keyOf(scope), row);
|
|
79
|
+
records.clear(ownerOf(scope), before);
|
|
80
|
+
records.bind(ownerOf(scope), scope.contextId, keyOf(scope));
|
|
81
|
+
} }
|
|
82
|
+
return { status: "pending", sessionId: state.sessionId, requestId: id };
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
return service.state(scope);
|
|
86
|
+
} finally { if (scope.epoch === epoch) scope.opening = false; }
|
|
87
|
+
},
|
|
88
|
+
async prompt(scope, body) {
|
|
89
|
+
if (!scope.active || !scope.session) throw failure("voice_session_closed", 409);
|
|
90
|
+
if (typeof body.text !== "string" || !body.text.trim() || body.text.length > 16000 || typeof body.requestId !== "string" || !body.requestId || body.requestId.length > 128) throw failure("voice_session_invalid", 400);
|
|
91
|
+
if (scope.voiceConfig.preset === "dictation") return { ...service.state(scope), text: body.text };
|
|
92
|
+
await ensurePresentation(scope);
|
|
93
|
+
if (scope.voiceConfig.preset === "direct") {
|
|
94
|
+
const before = keyOf(scope);
|
|
95
|
+
append(scope, "user", body.text, body.requestId, body.requestId);
|
|
96
|
+
try { return await scope.session.submit(body.text, body.requestId, body.mode); }
|
|
97
|
+
finally {
|
|
98
|
+
if (before !== keyOf(scope)) {
|
|
99
|
+
for (const row of records.read(ownerOf(scope), before).records) records.append(ownerOf(scope), keyOf(scope), row);
|
|
100
|
+
records.clear(ownerOf(scope), before);
|
|
101
|
+
records.bind(ownerOf(scope), scope.contextId, keyOf(scope));
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (scope.voiceRequests.has(body.requestId)) return scope.voiceRequests.get(body.requestId);
|
|
106
|
+
if (scope.voiceRequests.size >= 256) throw failure("voice_session_limit", 429);
|
|
107
|
+
const operation = (scope.voiceSerial || Promise.resolve()).catch(() => {}).then(async () => {
|
|
108
|
+
if (!scope.active || scope.voiceAbort.signal.aborted) throw failure("voice_session_closed", 409);
|
|
109
|
+
scope.submitMode = body.mode === "steer" ? "steer" : "queue";
|
|
110
|
+
scope.voiceState.running = true; scope.voiceState.error = null; scope.voiceState.version++;
|
|
111
|
+
append(scope, "user", body.text, body.requestId);
|
|
112
|
+
try {
|
|
113
|
+
const text = await scope.dialogue.reply(body.text, body.requestId, scope.voiceAbort.signal, (partial) => {
|
|
114
|
+
const source = records.read(ownerOf(scope), keyOf(scope)).records.find((row) => row.id === `${scope.id}:${body.requestId}:user`);
|
|
115
|
+
append(scope, "assistant", partial, body.requestId, source?.taskId || null, { status: "streaming" });
|
|
116
|
+
});
|
|
117
|
+
const source = records.read(ownerOf(scope), keyOf(scope)).records.find((row) => row.id === `${scope.id}:${body.requestId}:user`);
|
|
118
|
+
append(scope, "assistant", text, body.requestId, source?.taskId || null);
|
|
119
|
+
const key = keyOf(scope);
|
|
120
|
+
const checkpointVersion = records.read(ownerOf(scope), key).version;
|
|
121
|
+
scope.summaryTask = scope.dialogue.summarize().then((summary) => records.checkpoint(ownerOf(scope), key, summary, checkpointVersion)).catch((cause) => {
|
|
122
|
+
scope.summaryError = cause.code || cause.message || "voice_summary_failed";
|
|
123
|
+
});
|
|
124
|
+
scope.voiceState.utterance = { key: body.requestId, turn: scope.voiceState.version, step: 0, text, done: true, silent: false };
|
|
125
|
+
} catch (cause) {
|
|
126
|
+
scope.voiceState.error = cause.code || cause.message || "voice_failed";
|
|
127
|
+
const partial = records.read(ownerOf(scope), keyOf(scope)).records.find((row) => row.id === `${scope.id}:${body.requestId}:assistant`);
|
|
128
|
+
if (partial) append(scope, "assistant", partial.text, body.requestId, partial.taskId, { status: "failed" });
|
|
129
|
+
throw cause;
|
|
130
|
+
}
|
|
131
|
+
finally { scope.voiceState.running = false; scope.voiceState.version++; }
|
|
132
|
+
return service.state(scope);
|
|
133
|
+
});
|
|
134
|
+
scope.voiceSerial = operation;
|
|
135
|
+
scope.voiceRequests.set(body.requestId, operation);
|
|
136
|
+
return operation;
|
|
137
|
+
},
|
|
138
|
+
state(scope) {
|
|
139
|
+
const state = scope.session?.snapshot() || {};
|
|
140
|
+
if (!scope.voiceState) return { ...state, contextId: scope.contextId, active: scope.active === true };
|
|
141
|
+
// Completion is keyed by durable request/turn, never by the session's aggregate idle flag.
|
|
142
|
+
for (const task of state.tasks || []) {
|
|
143
|
+
if (!["completed", "failed", "cancelled"].includes(task.status) || scope.deliveredTasks.has(task.requestId)) continue;
|
|
144
|
+
append(scope, "task", JSON.stringify(task), `${task.requestId}:result`, task.requestId);
|
|
145
|
+
scope.deliveredTasks.add(task.requestId);
|
|
146
|
+
scope.voiceResults ??= [];
|
|
147
|
+
scope.voiceResults.push(task);
|
|
148
|
+
}
|
|
149
|
+
if (scope.voiceConfig.preset === "direct") return { ...state, contextId: scope.contextId, active: scope.active === true };
|
|
150
|
+
// Pending business work does not block the next speech turn.
|
|
151
|
+
if (scope.active && !scope.voiceState.running && scope.voiceResults?.length && !scope.voiceInputActive) {
|
|
152
|
+
const task = scope.voiceResults.shift();
|
|
153
|
+
scope.dialogue.observeTask(task);
|
|
154
|
+
const text = task.status === "completed" ? task.text : `Task ${task.status}: ${task.error || ""}`;
|
|
155
|
+
scope.voiceState.utterance = { key: `task:${task.requestId}`, turn: ++scope.voiceState.version, step: 0, text, done: true, silent: false };
|
|
156
|
+
append(scope, "assistant", text, `result:${task.requestId}`, task.requestId, { taskTurn: task.turn });
|
|
157
|
+
}
|
|
158
|
+
return { ...state, ...scope.voiceState, contextId: scope.contextId, active: scope.active === true, backgroundRunning: state.running, summaryError: scope.summaryError || null, preset: scope.voiceConfig.preset };
|
|
159
|
+
},
|
|
160
|
+
async remove(sessionId) {
|
|
161
|
+
for (const scope of scopes) if (scope.session?.snapshot().sessionId === sessionId) { await service.close(scope); scopes.delete(scope); }
|
|
162
|
+
records.clearConversation(sessionId);
|
|
163
|
+
},
|
|
164
|
+
async record(scope, role, text, turnId, fields = {}) {
|
|
165
|
+
if (!text?.trim()) return;
|
|
166
|
+
if (role === "user") await ensurePresentation(scope);
|
|
167
|
+
const key = keyOf(scope);
|
|
168
|
+
if (scope.recordKey && scope.recordKey !== key) {
|
|
169
|
+
for (const row of records.read(ownerOf(scope), scope.recordKey).records) records.append(ownerOf(scope), key, row);
|
|
170
|
+
records.clear(ownerOf(scope), scope.recordKey);
|
|
171
|
+
records.bind(ownerOf(scope), scope.contextId, key);
|
|
172
|
+
}
|
|
173
|
+
scope.recordKey = key;
|
|
174
|
+
return append(scope, role, text.slice(0, 16000), String(turnId || scope.voiceState.version), role === "task" ? String(turnId) : null, fields);
|
|
175
|
+
},
|
|
176
|
+
async history(scope, body, signal) {
|
|
177
|
+
const { sessionId, version, waitMs = 0 } = body;
|
|
178
|
+
if (typeof sessionId !== "string" || !sessionId || sessionId.length > 256 || !Number.isInteger(waitMs) || waitMs < 0 || waitMs > 20000) throw failure("voice_records_invalid", 400);
|
|
179
|
+
await api.inspect(sessionId);
|
|
180
|
+
const owner = ownerOf(scope);
|
|
181
|
+
const page = () => records.page(owner, sessionId, body);
|
|
182
|
+
let result = page();
|
|
183
|
+
if (waitMs && result.version === version && !signal?.aborted) {
|
|
184
|
+
await new Promise((resolve) => {
|
|
185
|
+
let timer;
|
|
186
|
+
const done = () => { clearTimeout(timer); unsubscribe(); signal?.removeEventListener("abort", done); resolve(); };
|
|
187
|
+
const unsubscribe = records.subscribe(owner, sessionId, done);
|
|
188
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
189
|
+
timer = setTimeout(done, waitMs);
|
|
190
|
+
if (signal?.aborted) done();
|
|
191
|
+
});
|
|
192
|
+
result = page();
|
|
193
|
+
}
|
|
194
|
+
return result;
|
|
195
|
+
},
|
|
196
|
+
records(scope) { return records.read(ownerOf(scope), keyOf(scope)); },
|
|
197
|
+
async clearRecords(scope) { const key = keyOf(scope); await service.close(scope); records.clear(ownerOf(scope), key); },
|
|
198
|
+
output(scope, value) { if (typeof value !== "boolean") throw failure("voice_output_invalid", 400); scope.readAloud = value; },
|
|
199
|
+
resume: (scope) => scope.session?.resume(),
|
|
200
|
+
cancel: (scope) => scope.session?.cancel(),
|
|
201
|
+
async close(scope) {
|
|
202
|
+
scope.epoch = (scope.epoch || 0) + 1; scope.opening = false; scope.closing = true;
|
|
203
|
+
scope.active = false; scope.voiceAbort?.abort(); scope.realtimeSocket?.close();
|
|
204
|
+
try {
|
|
205
|
+
scope.capture?.stop(true);
|
|
206
|
+
const cleanup = await Promise.allSettled([scope.duplex?.interrupt(), scope.realtimeAudio?.stop(), scope.session?.close()]);
|
|
207
|
+
await scope.voiceSerial?.catch(() => {});
|
|
208
|
+
await scope.summaryTask;
|
|
209
|
+
scopes.delete(scope);
|
|
210
|
+
const failed = cleanup.find((result) => result.status === "rejected");
|
|
211
|
+
if (failed) throw failed.reason;
|
|
212
|
+
} finally { scope.closing = false; }
|
|
213
|
+
},
|
|
214
|
+
};
|
|
215
|
+
return service;
|
|
216
|
+
}
|
package/voice/input.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
/** Where a take is recorded. The desktop page uses the headset; the phone page uses its own mic. */
|
|
2
|
-
export function captureSource(surface) {
|
|
2
|
+
export function captureSource(surface, selected = "auto") {
|
|
3
|
+
if (selected === "device" || selected === "headset") return selected;
|
|
3
4
|
return surface === "mobile" ? "device" : "headset";
|
|
4
5
|
}
|
package/voice/install.js
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { VoiceError } from "../router/stt.js";
|
|
3
3
|
|
|
4
|
-
/**
|
|
4
|
+
/**
|
|
5
|
+
* Charts the voice panel installs, stops, and resumes through olares-cli market.
|
|
6
|
+
* No version is pinned: install takes whatever the market holds, and status
|
|
7
|
+
* and model matching go by app name, so chart updates need no Lares release.
|
|
8
|
+
*/
|
|
5
9
|
export const VOICE_CHARTS = {
|
|
6
|
-
stt: { app: "audioqwen3asr06v3",
|
|
7
|
-
tts: { app: "audioqwen3tts06v3",
|
|
10
|
+
stt: { app: "audioqwen3asr06v3", model: "Qwen3-ASR-0.6B" },
|
|
11
|
+
tts: { app: "audioqwen3tts06v3", model: "Qwen3-TTS-12Hz-0.6B-CustomVoice" },
|
|
8
12
|
};
|
|
9
13
|
|
|
10
14
|
const OUTPUT_LIMIT = 8000;
|
|
@@ -101,13 +105,13 @@ export async function installVoiceChart(kind, options = {}) {
|
|
|
101
105
|
const chart = chartFor(kind);
|
|
102
106
|
const spawnFn = options.spawnFn ?? spawn;
|
|
103
107
|
const result = await runOlaresCli(
|
|
104
|
-
["market", "install", chart.app, "-s", "market.test", "--
|
|
108
|
+
["market", "install", chart.app, "-s", "market.test", "--compute-mode", "nvidia", "-o", "json"],
|
|
105
109
|
spawnFn,
|
|
106
110
|
);
|
|
107
111
|
if (!accepted(result)) {
|
|
108
112
|
throw new VoiceError("voice_install_failed", 502, result.message);
|
|
109
113
|
}
|
|
110
|
-
return { app: chart.app
|
|
114
|
+
return { app: chart.app };
|
|
111
115
|
}
|
|
112
116
|
|
|
113
117
|
/**
|
package/voice/languages.js
CHANGED
|
@@ -14,7 +14,7 @@ export function voiceValueFromMenu(id) {
|
|
|
14
14
|
return id === VOICE_AUTO ? "" : id;
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
export
|
|
17
|
+
export { VOICE_DEFAULTS as EMPTY_VOICE_CONFIG } from "./options.js";
|
|
18
18
|
|
|
19
19
|
export function voiceChoiceItems(ids) {
|
|
20
20
|
return ids.map((id) => ({ id, label: id }));
|
package/voice/lease.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** Physical audio is shared; a new page must not silently evict its owner. */
|
|
2
|
+
export function createDeviceLease({ now = Date.now, ttlMs = 30_000 } = {}) {
|
|
3
|
+
let lease = null;
|
|
4
|
+
const busy = () => Object.assign(new Error("voice_device_busy"), { code: "voice_device_busy", status: 409 });
|
|
5
|
+
return {
|
|
6
|
+
acquire(owner) {
|
|
7
|
+
if (lease && lease.owner !== owner && lease.until > now()) throw busy();
|
|
8
|
+
lease = { owner, until: now() + ttlMs };
|
|
9
|
+
},
|
|
10
|
+
touch(owner) {
|
|
11
|
+
if (lease?.owner === owner) lease.until = now() + ttlMs;
|
|
12
|
+
},
|
|
13
|
+
release(owner) { if (lease?.owner === owner) lease = null; },
|
|
14
|
+
owns: (owner) => lease?.owner === owner && lease.until > now(),
|
|
15
|
+
};
|
|
16
|
+
}
|
package/voice/options.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export const VOICE_DEFAULTS = {
|
|
2
|
+
preset: "assistant",
|
|
3
|
+
dictationCleanup: false,
|
|
4
|
+
chatModel: "default-chat",
|
|
5
|
+
instructions: "",
|
|
6
|
+
voice: "",
|
|
7
|
+
mode: "single",
|
|
8
|
+
mobileMode: "continuous",
|
|
9
|
+
endpointSilenceMs: 500,
|
|
10
|
+
idleTimeoutMs: 300_000,
|
|
11
|
+
recoveryTimeoutMs: 120_000,
|
|
12
|
+
device: "auto",
|
|
13
|
+
backend: "pipeline",
|
|
14
|
+
realtimeModel: "",
|
|
15
|
+
model: "",
|
|
16
|
+
ttsModel: "",
|
|
17
|
+
language: "",
|
|
18
|
+
speak: false,
|
|
19
|
+
// Read answers while they are written. Needs the GPU in MemorySlice, see gpu-mode.js.
|
|
20
|
+
streaming: false,
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export const VOICE_PRESETS = Object.freeze(["dictation", "chat", "assistant", "direct"]);
|
package/voice/panel.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/** Framework-independent panel lifetime; factories only supply audio/platform ports. */
|
|
2
|
+
export function createVoicePanel({ create, complete, onSession = () => {} }) {
|
|
3
|
+
let state = { visible: false, mode: "single", sessionId: null, draft: "", snapshot: null, busy: false, error: null, device: "", levels: [], startedAt: null, elapsedMs: 0 };
|
|
4
|
+
let call = null;
|
|
5
|
+
let unsubscribe = () => {};
|
|
6
|
+
let serial = Promise.resolve();
|
|
7
|
+
let generation = 0;
|
|
8
|
+
const drafts = new Map();
|
|
9
|
+
const listeners = new Set();
|
|
10
|
+
const emit = (patch) => { state = { ...state, ...patch }; for (const listener of listeners) listener(); };
|
|
11
|
+
const enqueue = (operation) => {
|
|
12
|
+
const result = serial.catch(() => {}).then(async () => { emit({ busy: true, error: null }); try { return await operation(); } catch (error) { emit({ error: error.code || error.message }); throw error; } finally { emit({ busy: false }); } });
|
|
13
|
+
serial = result; return result;
|
|
14
|
+
};
|
|
15
|
+
const finishCapture = async () => {
|
|
16
|
+
if (!call || state.mode !== "single") return;
|
|
17
|
+
const controller = call.controller;
|
|
18
|
+
if (!["recording", "transcribing"].includes(controller.getSnapshot().capture)) return;
|
|
19
|
+
await new Promise((resolve, reject) => {
|
|
20
|
+
const timer = setTimeout(() => { off(); reject(new Error("voice_recognition_timeout")); }, 45000);
|
|
21
|
+
const check = () => { if (controller.getSnapshot().capture === "idle") { clearTimeout(timer); off(); resolve(); } };
|
|
22
|
+
const off = controller.subscribe(check);
|
|
23
|
+
controller.stopRecording(); check();
|
|
24
|
+
});
|
|
25
|
+
};
|
|
26
|
+
const pause = async () => { try { await finishCapture(); } finally { if (call?.controller.getSnapshot().connection === "open") await call.pause(); } };
|
|
27
|
+
const end = async () => { generation++; unsubscribe(); unsubscribe = () => {}; const previous = call; call = null; try { await previous?.stop(); } finally { emit({ snapshot: null, startedAt: null, elapsedMs: 0, levels: [] }); } };
|
|
28
|
+
const reset = async (patch) => {
|
|
29
|
+
try { await pause(); } finally { try { await end(); } finally { emit(patch); } }
|
|
30
|
+
};
|
|
31
|
+
return {
|
|
32
|
+
getSnapshot: () => state,
|
|
33
|
+
subscribe(listener) { listeners.add(listener); return () => listeners.delete(listener); },
|
|
34
|
+
get conversation() { return call; },
|
|
35
|
+
open({ mode = state.mode, sessionId = state.sessionId } = {}) {
|
|
36
|
+
emit({ visible: true });
|
|
37
|
+
return enqueue(async () => {
|
|
38
|
+
if (sessionId !== state.sessionId) { await reset({ sessionId, draft: drafts.get(sessionId) || "" }); }
|
|
39
|
+
if (mode !== state.mode) { await reset({ mode }); }
|
|
40
|
+
});
|
|
41
|
+
},
|
|
42
|
+
bind(sessionId) {
|
|
43
|
+
if (sessionId === state.sessionId) return Promise.resolve();
|
|
44
|
+
return enqueue(async () => { await reset({ sessionId, draft: drafts.get(sessionId) || "" }); });
|
|
45
|
+
},
|
|
46
|
+
close() { emit({ visible: false }); return enqueue(pause); },
|
|
47
|
+
pause: () => enqueue(pause),
|
|
48
|
+
start() { return enqueue(async () => {
|
|
49
|
+
if (call?.controller.getSnapshot().connection === "paused") { await call.resume(); return; }
|
|
50
|
+
if (call?.controller.getSnapshot().connection === "open") return;
|
|
51
|
+
await end();
|
|
52
|
+
const target = state.sessionId; const token = ++generation;
|
|
53
|
+
const conversation = await create({ mode: state.mode, sessionId: target,
|
|
54
|
+
onTranscript({ text }) {
|
|
55
|
+
if (token !== generation) return;
|
|
56
|
+
const next = [drafts.get(target), text].filter(Boolean).join(" ");
|
|
57
|
+
drafts.set(target, next);
|
|
58
|
+
if (state.sessionId === target) emit({ draft: next });
|
|
59
|
+
},
|
|
60
|
+
onDevice(device) { if (token === generation) emit({ device }); },
|
|
61
|
+
});
|
|
62
|
+
call = conversation;
|
|
63
|
+
unsubscribe = call.controller.subscribe(() => {
|
|
64
|
+
if (token !== generation) return;
|
|
65
|
+
const snapshot = call.controller.getSnapshot();
|
|
66
|
+
if (!state.sessionId && snapshot.sessionId) { emit({ sessionId: snapshot.sessionId }); onSession(snapshot.sessionId); }
|
|
67
|
+
emit({ snapshot, ...(snapshot.inputLevel == null ? {} : { levels: [...state.levels.slice(-19), snapshot.inputLevel] }),
|
|
68
|
+
...(snapshot.capture === "recording" && state.snapshot?.capture !== "recording" ? { startedAt: Date.now(), elapsedMs: 0 } : snapshot.capture !== "recording" && state.startedAt ? { startedAt: null, elapsedMs: Date.now() - state.startedAt } : {}) });
|
|
69
|
+
});
|
|
70
|
+
await call.start({ sessionId: target, continuous: state.mode === "continuous" });
|
|
71
|
+
}); },
|
|
72
|
+
complete() { return enqueue(async () => {
|
|
73
|
+
await finishCapture();
|
|
74
|
+
const text = state.draft;
|
|
75
|
+
if (!text) return;
|
|
76
|
+
await complete({ sessionId: state.sessionId, text });
|
|
77
|
+
drafts.delete(state.sessionId); emit({ draft: "", visible: false }); await end();
|
|
78
|
+
}); },
|
|
79
|
+
discard() { return enqueue(async () => { await end(); drafts.delete(state.sessionId); emit({ draft: "" }); }); },
|
|
80
|
+
end: () => enqueue(end),
|
|
81
|
+
action(action) { return enqueue(async () => {
|
|
82
|
+
if (action === "mute") await call?.setReadAloud(state.snapshot.outputMuted);
|
|
83
|
+
if (action === "playback") await call?.controller.stopPlayback();
|
|
84
|
+
if (action === "input") { if (state.snapshot.inputPaused) call?.controller.resumeInput(); else await call?.controller.pauseInput(); }
|
|
85
|
+
}); },
|
|
86
|
+
dispose: () => enqueue(async () => { try { await end(); } finally { listeners.clear(); drafts.clear(); } }),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/** Read-only records: native dsh events remain the authority for task execution. */
|
|
2
|
+
export function projectVoiceRecords(records, native = []) {
|
|
3
|
+
const tasks = new Map();
|
|
4
|
+
for (const row of records) {
|
|
5
|
+
if (row.role !== "task" || !row.taskId) continue;
|
|
6
|
+
try { const task = JSON.parse(row.text); if (Number.isInteger(task.turn)) tasks.set(row.taskId, task.turn); } catch { /* A delegation request is text, not a result. */ }
|
|
7
|
+
}
|
|
8
|
+
const request = (row) => row.requestId || row.data?.source?.rpcId;
|
|
9
|
+
const turn = (row) => row.turn ?? row.data?.turn ?? row.location?.turn?.turn;
|
|
10
|
+
const role = (row) => row.type || row.kind;
|
|
11
|
+
const rows = [];
|
|
12
|
+
for (const record of records) {
|
|
13
|
+
if (!["user", "assistant"].includes(record.role) || !record.text) continue;
|
|
14
|
+
const taskTurn = record.taskTurn ?? tasks.get(record.taskId) ?? turn(native.find((row) => request(row) === record.taskId) || {});
|
|
15
|
+
const match = record.taskId && [...native].reverse().find((row) => record.role === "user"
|
|
16
|
+
? role(row) === "user" && request(row) === record.taskId
|
|
17
|
+
: ["assistant", "closing", "turn-tail"].includes(role(row)) && taskTurn != null && turn(row) === taskTurn);
|
|
18
|
+
if (match) {
|
|
19
|
+
if (record.role === "assistant" && tasks.has(record.taskId) && !record.turnId?.startsWith("result:")) continue;
|
|
20
|
+
if (record.role === "assistant" && record.text !== (match.text || match.data?.text || (match.data?.closing?.blocks || match.data?.blocks || []).filter((block) => block.type === "text").map((block) => block.text).join(""))) rows.push({ ...record, attachedTo: match.key || match.id, summary: true });
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
rows.push({ ...record, type: record.role, pending: record.status === "streaming" });
|
|
24
|
+
}
|
|
25
|
+
return rows.sort((a, b) => a.anchorSeq - b.anchorSeq || a.sequence - b.sequence || a.id.localeCompare(b.id));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function mergeVoiceTranscript(native, records) {
|
|
29
|
+
const projected = projectVoiceRecords(records, native);
|
|
30
|
+
const rows = [...native];
|
|
31
|
+
for (const row of projected.filter((item) => !item.summary)) {
|
|
32
|
+
const next = rows.findIndex((item) => !item.voiceId && (item.anchorSeq ?? item.seq ?? Infinity) > row.anchorSeq);
|
|
33
|
+
rows.splice(next < 0 ? rows.length : next, 0, row);
|
|
34
|
+
}
|
|
35
|
+
for (const row of projected.filter((item) => item.summary)) {
|
|
36
|
+
const index = rows.findIndex((item) => (item.key || item.id) === row.attachedTo);
|
|
37
|
+
if (index >= 0) rows[index] = { ...rows[index], voiceSummary: row.text };
|
|
38
|
+
}
|
|
39
|
+
return rows;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Paginated long polling is independent of capture, panel mounts and playback. */
|
|
43
|
+
export function subscribeVoiceHistory({ request, sessionId, onRecords, onError = () => {} }) {
|
|
44
|
+
const abort = new AbortController();
|
|
45
|
+
let version = -1;
|
|
46
|
+
const task = (async () => {
|
|
47
|
+
while (!abort.signal.aborted) {
|
|
48
|
+
try {
|
|
49
|
+
let after = 0; let next; let snapshotVersion; const rows = [];
|
|
50
|
+
do {
|
|
51
|
+
const response = await request("/api/lares/voice/history", { method: "POST", signal: abort.signal,
|
|
52
|
+
body: { sessionId, after, limit: 200, version: after ? undefined : version, waitMs: after || version < 0 ? 0 : 20000 } });
|
|
53
|
+
if (!response.ok) throw new Error(response.body?.error?.code || "voice_records_failed");
|
|
54
|
+
if (snapshotVersion != null && response.body.version !== snapshotVersion) { version = -1; break; }
|
|
55
|
+
snapshotVersion = response.body.version;
|
|
56
|
+
rows.push(...response.body.records); next = response.body.next; after = next;
|
|
57
|
+
if (!next) { version = response.body.version; if (!abort.signal.aborted) onRecords(rows); }
|
|
58
|
+
} while (next && !abort.signal.aborted);
|
|
59
|
+
} catch (error) {
|
|
60
|
+
if (abort.signal.aborted) return;
|
|
61
|
+
onError(error);
|
|
62
|
+
await new Promise((resolve) => { const finish = () => { clearTimeout(timer); abort.signal.removeEventListener("abort", finish); resolve(); }; const timer = setTimeout(finish, 2000); abort.signal.addEventListener("abort", finish, { once: true }); });
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
})();
|
|
66
|
+
return Object.assign(() => abort.abort(), { done: task });
|
|
67
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Transport-neutral client: the caller obtains an authenticated Router socket. */
|
|
2
|
+
export function createVoiceProtocolClient(socket, { onEvent = () => {}, onClose = () => {} } = {}) {
|
|
3
|
+
const send = (event) => {
|
|
4
|
+
if (socket.readyState !== 1) throw new Error("voice_disconnected");
|
|
5
|
+
if (socket.bufferedAmount > 1024 * 1024) throw new Error("voice_audio_backpressure");
|
|
6
|
+
socket.send(JSON.stringify(event));
|
|
7
|
+
};
|
|
8
|
+
const message = (event) => {
|
|
9
|
+
let value;
|
|
10
|
+
try { value = JSON.parse(String(event.data)); } catch { onEvent({ type: "error", error: { code: "voice_protocol" } }); return; }
|
|
11
|
+
if (value && typeof value.type === "string") onEvent(value);
|
|
12
|
+
};
|
|
13
|
+
const close = () => onClose();
|
|
14
|
+
socket.addEventListener("message", message); socket.addEventListener("close", close);
|
|
15
|
+
return {
|
|
16
|
+
configure({ automatic = false, contextId, audio = true } = {}) { send({ type: "session.update", session: { type: "realtime", output_modalities: audio ? ["audio"] : ["text"], ...(contextId ? { voice_context_id: contextId } : {}), audio: { input: { format: { type: "audio/pcm", rate: 24000 }, turn_detection: automatic ? { type: "server_vad" } : null }, output: { format: { type: "audio/pcm", rate: 24000 } } } }); },
|
|
17
|
+
append(audio) { if (typeof audio !== "string" || audio.length > 64000) throw new Error("voice_audio_limit"); send({ type: "input_audio_buffer.append", audio }); },
|
|
18
|
+
commit() { send({ type: "input_audio_buffer.commit" }); },
|
|
19
|
+
discard() { send({ type: "input_audio_buffer.clear" }); },
|
|
20
|
+
playbackCompleted() { send({ type: "voice.playback.completed" }); },
|
|
21
|
+
stopOutput() { send({ type: "response.cancel" }); },
|
|
22
|
+
close() { socket.removeEventListener("message", message); socket.removeEventListener("close", close); socket.close(); },
|
|
23
|
+
};
|
|
24
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/** Streaming rate conversion keeps fractional phase across device frames. */
|
|
2
|
+
export function createPcmResampler(inputRate, outputRate = 24000) {
|
|
3
|
+
if (!(inputRate > 0 && outputRate > 0)) throw new Error("voice_audio_format");
|
|
4
|
+
let position = 0;
|
|
5
|
+
return (samples) => {
|
|
6
|
+
const values = [];
|
|
7
|
+
const step = inputRate / outputRate;
|
|
8
|
+
for (; position < samples.length; position += step) {
|
|
9
|
+
const i = Math.floor(position);
|
|
10
|
+
const fraction = position - i;
|
|
11
|
+
const value = samples[i] * (1 - fraction) + samples[Math.min(i + 1, samples.length - 1)] * fraction;
|
|
12
|
+
values.push(Math.max(-1, Math.min(1, value)));
|
|
13
|
+
}
|
|
14
|
+
position -= samples.length;
|
|
15
|
+
const bytes = new Uint8Array(values.length * 2);
|
|
16
|
+
const view = new DataView(bytes.buffer);
|
|
17
|
+
for (let i = 0; i < values.length; i++) view.setInt16(i * 2, Math.round(values[i] * 32767), true);
|
|
18
|
+
return bytes;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export function pcmSamples(bytes) {
|
|
22
|
+
if (bytes.length % 2) throw new Error("voice_audio_format");
|
|
23
|
+
const values = new Float32Array(bytes.length / 2);
|
|
24
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
25
|
+
for (let i = 0; i < values.length; i++) values[i] = view.getInt16(i * 2, true) / 32768;
|
|
26
|
+
return values;
|
|
27
|
+
}
|
|
28
|
+
function base64(bytes) { return btoa(String.fromCharCode(...bytes)); }
|
|
29
|
+
function unbase64(value) { return Uint8Array.from(atob(value), (c) => c.charCodeAt(0)); }
|
|
30
|
+
|
|
31
|
+
export function createPcmBrowserAudio({ send, onPlayback = () => {}, onError = () => {} }) {
|
|
32
|
+
let context;
|
|
33
|
+
let stream;
|
|
34
|
+
let source;
|
|
35
|
+
let processor;
|
|
36
|
+
let mute;
|
|
37
|
+
let next = 0;
|
|
38
|
+
let first = 0;
|
|
39
|
+
let closed = true;
|
|
40
|
+
let generation = 0;
|
|
41
|
+
const players = new Set();
|
|
42
|
+
const drains = new Set();
|
|
43
|
+
const stopPlayback = () => {
|
|
44
|
+
generation++;
|
|
45
|
+
for (const player of players) { player.onended = null; player.stop(); player.disconnect(); }
|
|
46
|
+
players.clear();
|
|
47
|
+
next = context?.currentTime || 0;
|
|
48
|
+
for (const resolve of drains) resolve();
|
|
49
|
+
drains.clear();
|
|
50
|
+
onPlayback(false);
|
|
51
|
+
};
|
|
52
|
+
return {
|
|
53
|
+
async start() {
|
|
54
|
+
const epoch = ++generation;
|
|
55
|
+
closed = false;
|
|
56
|
+
const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext;
|
|
57
|
+
if (!AudioCtx || !globalThis.navigator?.mediaDevices?.getUserMedia) throw new Error("voice_unsupported");
|
|
58
|
+
const opened = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
|
|
59
|
+
if (closed || epoch !== generation) { opened.getTracks().forEach((track) => track.stop()); return; }
|
|
60
|
+
stream = opened;
|
|
61
|
+
for (const track of stream.getTracks()) track.addEventListener("ended", () => { if (!closed) onError(new Error("voice_device_lost")); }, { once: true });
|
|
62
|
+
context ??= new AudioCtx(); await context.resume();
|
|
63
|
+
source = context.createMediaStreamSource(stream);
|
|
64
|
+
processor = context.createScriptProcessor(2048, 1, 1);
|
|
65
|
+
mute = context.createGain(); mute.gain.value = 0;
|
|
66
|
+
const convert = createPcmResampler(context.sampleRate);
|
|
67
|
+
processor.onaudioprocess = (event) => {
|
|
68
|
+
if (closed || players.size) return;
|
|
69
|
+
send({ type: "input_audio_buffer.append", audio: base64(convert(event.inputBuffer.getChannelData(0))) });
|
|
70
|
+
};
|
|
71
|
+
source.connect(processor); processor.connect(mute); mute.connect(context.destination);
|
|
72
|
+
},
|
|
73
|
+
pauseInput() {
|
|
74
|
+
generation++;
|
|
75
|
+
if (processor) processor.onaudioprocess = null;
|
|
76
|
+
source?.disconnect(); processor?.disconnect(); mute?.disconnect();
|
|
77
|
+
source = null; processor = null; mute = null;
|
|
78
|
+
stream?.getTracks().forEach((track) => track.stop()); stream = null;
|
|
79
|
+
},
|
|
80
|
+
write(value) {
|
|
81
|
+
if (!context || closed) return;
|
|
82
|
+
const samples = pcmSamples(unbase64(value));
|
|
83
|
+
if (next - context.currentTime > 15) throw new Error("voice_audio_backpressure");
|
|
84
|
+
const buffer = context.createBuffer(1, samples.length, 24000); buffer.copyToChannel(samples, 0);
|
|
85
|
+
const player = context.createBufferSource(); player.buffer = buffer; player.connect(context.destination);
|
|
86
|
+
if (!players.size) { first = context.currentTime; onPlayback(true); }
|
|
87
|
+
players.add(player);
|
|
88
|
+
player.onended = () => {
|
|
89
|
+
players.delete(player); player.disconnect();
|
|
90
|
+
if (!players.size) {
|
|
91
|
+
for (const resolve of drains) resolve(); drains.clear(); onPlayback(false);
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
next = Math.max(context.currentTime + 0.02, next); player.start(next); next += buffer.duration;
|
|
95
|
+
},
|
|
96
|
+
drain() { return players.size ? new Promise((resolve) => drains.add(resolve)) : Promise.resolve(); },
|
|
97
|
+
playedMs() { return context ? Math.max(0, Math.floor((Math.min(context.currentTime, next) - first) * 1000)) : 0; },
|
|
98
|
+
stopPlayback,
|
|
99
|
+
async stop() {
|
|
100
|
+
closed = true; stopPlayback();
|
|
101
|
+
if (processor) processor.onaudioprocess = null;
|
|
102
|
+
source?.disconnect(); processor?.disconnect(); mute?.disconnect();
|
|
103
|
+
stream?.getTracks().forEach((track) => track.stop()); stream = null;
|
|
104
|
+
await context?.close(); context = null;
|
|
105
|
+
},
|
|
106
|
+
};
|
|
107
|
+
}
|