@olares/lares-core 0.15.12 → 0.15.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/i18n/voice.js +104 -0
- package/larepass/runtime.js +2 -2
- package/larepass/settings.js +5 -1
- package/larepass/transcript.js +7 -0
- package/package.json +8 -2
- package/voice/agent-session.js +4 -3
- package/voice/capture.js +6 -3
- package/voice/client.js +7 -2
- package/voice/config.js +14 -2
- package/voice/controller.js +23 -11
- package/voice/conversation.js +17 -9
- package/voice/device-input.js +3 -2
- package/voice/dialogue.js +44 -11
- package/voice/dictation.js +18 -0
- package/voice/hold.js +51 -0
- package/voice/host-input.js +2 -1
- package/voice/host-service.js +57 -11
- package/voice/options.js +5 -1
- package/voice/panel.js +140 -0
- package/voice/presentation.js +22 -0
- package/voice/projection.js +85 -0
- package/voice/realtime-conversation.js +6 -2
- package/voice/realtime-gateway.js +11 -5
- package/voice/records.js +28 -4
- package/voice/service.js +8 -6
- package/voice/shortcuts.js +23 -0
package/i18n/voice.js
CHANGED
|
@@ -1,4 +1,56 @@
|
|
|
1
1
|
export const ZH = {
|
|
2
|
+
"input.keyboard": "键盘",
|
|
3
|
+
"input.stopSpeaking": "停止朗读",
|
|
4
|
+
"gesture.cancelRelease": "松开取消",
|
|
5
|
+
"gesture.slideCancel": "上滑取消",
|
|
6
|
+
"gesture.release": "松开发送",
|
|
7
|
+
"gesture.waiting": "按住说话,松开发送",
|
|
8
|
+
"gesture.keyboard": "按住空格说话 · Esc 取消 · 回车开始/结束",
|
|
9
|
+
|
|
10
|
+
"error.voice_capture_limit": "录音已达到一分钟上限,请重新录制较短内容;本次未自动提交。",
|
|
11
|
+
"dictation.settings": "听写处理",
|
|
12
|
+
"dictation.preset": "文字处理方式",
|
|
13
|
+
"dictation.configure": "配置预设",
|
|
14
|
+
"dictation.raw": "原文转写",
|
|
15
|
+
"dictation.clean": "简单清洗",
|
|
16
|
+
"dictation.outline": "整理提纲",
|
|
17
|
+
"dictation.summary": "简洁总结",
|
|
18
|
+
"dictation.model": "处理模型",
|
|
19
|
+
"dictation.instructions": "处理指令",
|
|
20
|
+
"dictation.reset": "恢复默认指令",
|
|
21
|
+
"dictation.noLlm": "直接使用识别结果,不调用语言模型。",
|
|
22
|
+
"dictation.finish": "完成",
|
|
23
|
+
"dictation.insert": "放入输入框",
|
|
24
|
+
"dictation.discard": "放弃",
|
|
25
|
+
"dictation.original": "查看原文",
|
|
26
|
+
"dictation.processing": "正在整理文字…",
|
|
27
|
+
"dictation.preview": "预览并编辑",
|
|
28
|
+
"dictation.retry": "重试",
|
|
29
|
+
"dictation.useOriginal": "使用原文",
|
|
30
|
+
"dictation.replaceEdited": "重新整理将替换你的编辑,是否继续?",
|
|
31
|
+
"dictation.hold": "按住说话",
|
|
32
|
+
"dictation.release": "松开发送 · 上滑取消",
|
|
33
|
+
"dictation.holdHint": "按住说话,松开发送;也可按住空格,或按回车开始/结束。",
|
|
34
|
+
"dictation.speaking": "正在回复 · 按住说话可打断",
|
|
35
|
+
"dictation.shortcut": "默认听写快捷键",
|
|
36
|
+
"dictation.shortcutHint": "按组合键设置,退格清除;仅当前页面生效。",
|
|
37
|
+
"dictation.shortcutConflict": "快捷键重复、被占用或格式不支持,请换一个组合键。",
|
|
38
|
+
"error.voice_dictation_model_missing": "请先在设置中选择此预设的处理模型,或使用原文转写。",
|
|
39
|
+
"error.voice_manual_realtime_unavailable": "Realtime 暂不支持按住说话,请在设置中使用普通语音后端。",
|
|
40
|
+
|
|
41
|
+
"panel.title": "语音",
|
|
42
|
+
"panel.close": "关闭",
|
|
43
|
+
"panel.paused": "语音已暂停",
|
|
44
|
+
"panel.start": "开始",
|
|
45
|
+
"panel.continue": "继续",
|
|
46
|
+
"panel.pause": "暂停",
|
|
47
|
+
"panel.complete": "完成听写",
|
|
48
|
+
"panel.discard": "丢弃听写",
|
|
49
|
+
"panel.draft": "听写草稿",
|
|
50
|
+
"panel.draftHint": "识别结果会保留在这里,完成后追加到输入框。",
|
|
51
|
+
"panel.messages": "完整回复显示在当前会话中。关闭此面板会暂停音频,任务继续执行。",
|
|
52
|
+
"panel.elapsed": "录音时长",
|
|
53
|
+
|
|
2
54
|
"settings.backend": "语音后端",
|
|
3
55
|
"settings.backend.pipeline": "STT → Lares → TTS",
|
|
4
56
|
"settings.realtime.model": "Realtime 模型",
|
|
@@ -181,6 +233,58 @@ export const ZH = {
|
|
|
181
233
|
};
|
|
182
234
|
|
|
183
235
|
export const EN = {
|
|
236
|
+
"input.keyboard": "Keyboard",
|
|
237
|
+
"input.stopSpeaking": "Stop speaking",
|
|
238
|
+
"gesture.cancelRelease": "Release to cancel",
|
|
239
|
+
"gesture.slideCancel": "Slide up to cancel",
|
|
240
|
+
"gesture.release": "Release to send",
|
|
241
|
+
"gesture.waiting": "Hold to speak, release to send",
|
|
242
|
+
"gesture.keyboard": "Hold Space · Esc to cancel · Enter to start/finish",
|
|
243
|
+
|
|
244
|
+
"error.voice_capture_limit": "Recording reached the one-minute limit. Record a shorter take; nothing was submitted.",
|
|
245
|
+
"dictation.settings": "Dictation processing",
|
|
246
|
+
"dictation.preset": "Text processing",
|
|
247
|
+
"dictation.configure": "Configure preset",
|
|
248
|
+
"dictation.raw": "Original transcript",
|
|
249
|
+
"dictation.clean": "Light cleanup",
|
|
250
|
+
"dictation.outline": "Outline",
|
|
251
|
+
"dictation.summary": "Concise summary",
|
|
252
|
+
"dictation.model": "Processing model",
|
|
253
|
+
"dictation.instructions": "Processing instructions",
|
|
254
|
+
"dictation.reset": "Restore default instructions",
|
|
255
|
+
"dictation.noLlm": "Use the ASR result directly without a language model.",
|
|
256
|
+
"dictation.finish": "Done",
|
|
257
|
+
"dictation.insert": "Insert into composer",
|
|
258
|
+
"dictation.discard": "Discard",
|
|
259
|
+
"dictation.original": "View original",
|
|
260
|
+
"dictation.processing": "Refining text…",
|
|
261
|
+
"dictation.preview": "Preview and edit",
|
|
262
|
+
"dictation.retry": "Retry",
|
|
263
|
+
"dictation.useOriginal": "Use original",
|
|
264
|
+
"dictation.replaceEdited": "Processing again will replace your edits. Continue?",
|
|
265
|
+
"dictation.hold": "Hold to speak",
|
|
266
|
+
"dictation.release": "Release to send · Slide up to cancel",
|
|
267
|
+
"dictation.holdHint": "Hold to speak, release to send. Hold Space, or press Enter to start/finish.",
|
|
268
|
+
"dictation.speaking": "Speaking · Hold to interrupt",
|
|
269
|
+
"dictation.shortcut": "Default dictation shortcut",
|
|
270
|
+
"dictation.shortcutHint": "Press a key combination; Backspace clears it. Active page only.",
|
|
271
|
+
"dictation.shortcutConflict": "Duplicate, reserved or unsupported shortcut. Choose another combination.",
|
|
272
|
+
"error.voice_dictation_model_missing": "Select a processing model in Settings or use the original transcript.",
|
|
273
|
+
"error.voice_manual_realtime_unavailable": "Realtime does not yet support push-to-talk. Select the pipeline backend in Settings.",
|
|
274
|
+
|
|
275
|
+
"panel.title": "Voice",
|
|
276
|
+
"panel.close": "Close",
|
|
277
|
+
"panel.paused": "Voice paused",
|
|
278
|
+
"panel.start": "Start",
|
|
279
|
+
"panel.continue": "Continue",
|
|
280
|
+
"panel.pause": "Pause",
|
|
281
|
+
"panel.complete": "Complete dictation",
|
|
282
|
+
"panel.discard": "Discard dictation",
|
|
283
|
+
"panel.draft": "Dictation draft",
|
|
284
|
+
"panel.draftHint": "Recognized text stays here until you complete dictation.",
|
|
285
|
+
"panel.messages": "Replies appear in this conversation. Closing pauses audio; tasks keep running.",
|
|
286
|
+
"panel.elapsed": "Recording time",
|
|
287
|
+
|
|
184
288
|
"settings.backend": "Voice backend",
|
|
185
289
|
"settings.backend.pipeline": "STT → Lares → TTS",
|
|
186
290
|
"settings.realtime.model": "Realtime model",
|
package/larepass/runtime.js
CHANGED
|
@@ -864,11 +864,11 @@ export function createChatRuntime(client) {
|
|
|
864
864
|
},
|
|
865
865
|
voiceSocket(ticket) { return client.voiceSocket(ticket); },
|
|
866
866
|
voiceRequest(path, options) { return client.voiceRequest(path, options); },
|
|
867
|
-
transcribe(blob, language, signal) {
|
|
867
|
+
transcribe(blob, language, signal, model) {
|
|
868
868
|
if (typeof client.transcribe !== "function") {
|
|
869
869
|
return Promise.reject(new Error("voice_failed"));
|
|
870
870
|
}
|
|
871
|
-
return client.transcribe(blob, language, signal);
|
|
871
|
+
return client.transcribe(blob, language, signal, model);
|
|
872
872
|
},
|
|
873
873
|
reportLog(scope, event, fields) {
|
|
874
874
|
if (typeof client.reportLog !== "function") return Promise.resolve(false);
|
package/larepass/settings.js
CHANGED
|
@@ -8,6 +8,8 @@ import {
|
|
|
8
8
|
|
|
9
9
|
const modelsStore = createSnapshotStore();
|
|
10
10
|
const voiceStore = createSnapshotStore();
|
|
11
|
+
const voiceListeners = new Set();
|
|
12
|
+
export function subscribeVoiceSettings(listener) { voiceListeners.add(listener); const current = voiceStore.peek(); if (current) listener(current); return () => voiceListeners.delete(listener); }
|
|
11
13
|
const searchStore = createSnapshotStore();
|
|
12
14
|
const conversationStore = createSnapshotStore();
|
|
13
15
|
const conversationListeners = new Set();
|
|
@@ -112,7 +114,9 @@ export function createHostSettings(request) {
|
|
|
112
114
|
await request("/api/lares/voice/config", { method: "POST", body: patch }),
|
|
113
115
|
"/api/lares/voice/config",
|
|
114
116
|
);
|
|
115
|
-
|
|
117
|
+
const value = await this.voice(true);
|
|
118
|
+
for (const listener of voiceListeners) listener(value);
|
|
119
|
+
return value;
|
|
116
120
|
},
|
|
117
121
|
async search(options = {}) {
|
|
118
122
|
const query = options.force ? "?refresh=1" : "";
|
package/larepass/transcript.js
CHANGED
|
@@ -114,10 +114,15 @@ export function foldTranscript(entries) {
|
|
|
114
114
|
let utterance = null;
|
|
115
115
|
let streamCursor = { loc: "", kind: "", index: 0 };
|
|
116
116
|
let activeTurn = 0;
|
|
117
|
+
let activeSeq = -1;
|
|
118
|
+
let activeRequest = null;
|
|
117
119
|
const ended = new Map();
|
|
118
120
|
|
|
119
121
|
const tag = (row) => {
|
|
120
122
|
row.turn = activeTurn;
|
|
123
|
+
row.anchorSeq = activeSeq;
|
|
124
|
+
row.key ||= `dsh:${activeSeq}:${row.type}`;
|
|
125
|
+
if (row.type === "user") row.requestId = activeRequest;
|
|
121
126
|
return row;
|
|
122
127
|
};
|
|
123
128
|
|
|
@@ -234,6 +239,8 @@ export function foldTranscript(entries) {
|
|
|
234
239
|
|
|
235
240
|
for (const entry of Array.isArray(entries) ? entries : []) {
|
|
236
241
|
const event = eventOf(entry);
|
|
242
|
+
activeSeq = event?.seq ?? activeSeq;
|
|
243
|
+
if (event?.type === "user/message") activeRequest = event.data?.requestId ?? event.data?.source?.rpcId ?? null;
|
|
237
244
|
if (!event) continue;
|
|
238
245
|
const data = event.data ?? {};
|
|
239
246
|
const view = entryView(entry, event);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@olares/lares-core",
|
|
3
|
-
"version": "0.15.
|
|
3
|
+
"version": "0.15.14",
|
|
4
4
|
"description": "Lares 共享业务核心:Router、workspace、文件/媒体、Olares 身份(无 UI、无平台壳)",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"repository": {
|
|
@@ -189,7 +189,13 @@
|
|
|
189
189
|
"./voice/external": "./voice/external.js",
|
|
190
190
|
"./voice/composite-gateway": "./voice/composite-gateway.js",
|
|
191
191
|
"./voice/router-audio": "./voice/router-audio.js",
|
|
192
|
-
"./voice/protocol-client": "./voice/protocol-client.js"
|
|
192
|
+
"./voice/protocol-client": "./voice/protocol-client.js",
|
|
193
|
+
"./voice/presentation": "./voice/presentation.js",
|
|
194
|
+
"./voice/hold": "./voice/hold.js",
|
|
195
|
+
"./voice/dictation": "./voice/dictation.js",
|
|
196
|
+
"./voice/shortcuts": "./voice/shortcuts.js",
|
|
197
|
+
"./voice/panel": "./voice/panel.js",
|
|
198
|
+
"./voice/projection": "./voice/projection.js"
|
|
193
199
|
},
|
|
194
200
|
"dependencies": {
|
|
195
201
|
"ws": "^8.22.0"
|
package/voice/agent-session.js
CHANGED
|
@@ -5,7 +5,7 @@ import { reduceUtterance } from "./speech.js";
|
|
|
5
5
|
export const SPOKEN_PROMPT = "This is a voice conversation. Keep user-facing answers concise and natural for speech. Do not read reasoning or tool payloads aloud. Preserve the task's factual results and use the user's language.";
|
|
6
6
|
|
|
7
7
|
/** dsh opens a turn before claiming input; rpcId associates input with that open turn. */
|
|
8
|
-
export function createVoiceAgentSession(api, { sessionId = null, preset, workspaceId, restoredRequests = [], metric = () => {} } = {}) {
|
|
8
|
+
export function createVoiceAgentSession(api, { sessionId = null, preset, workspaceId, restoredRequests = [], onChange = () => {}, metric = () => {} } = {}) {
|
|
9
9
|
let creating;
|
|
10
10
|
let following;
|
|
11
11
|
let closed = false;
|
|
@@ -26,7 +26,7 @@ export function createVoiceAgentSession(api, { sessionId = null, preset, workspa
|
|
|
26
26
|
const decoder = createSessionFollow();
|
|
27
27
|
const unfinished = (task) => ["accepted", "running", "waiting", "unknown"].includes(task.status);
|
|
28
28
|
const snapshot = () => ({ sessionId, running: observedRunning || [...requests.values()].some(unfinished), utterance, error, version,
|
|
29
|
-
requestId: latestRequest,
|
|
29
|
+
requestId: latestRequest, cursor,
|
|
30
30
|
tasks: [...requests.values()].map(({ requestId, turn, status, utterance: reply, error: taskError }) => ({ requestId, turn, status, text: reply?.silent ? "" : reply?.text || "", error: taskError })),
|
|
31
31
|
});
|
|
32
32
|
const ensure = async () => {
|
|
@@ -68,7 +68,7 @@ export function createVoiceAgentSession(api, { sessionId = null, preset, workspa
|
|
|
68
68
|
if (!replay && task.requestId === latestRequest) { utterance = task.utterance; error = task.error; }
|
|
69
69
|
}
|
|
70
70
|
if (event.type === "turn/end" && openTurn === event.data?.turn) { openTurn = null; observedRunning = false; }
|
|
71
|
-
version++;
|
|
71
|
+
version++; onChange();
|
|
72
72
|
};
|
|
73
73
|
const follow = () => {
|
|
74
74
|
if (following && !following.signal.aborted) return ready;
|
|
@@ -100,6 +100,7 @@ export function createVoiceAgentSession(api, { sessionId = null, preset, workspa
|
|
|
100
100
|
};
|
|
101
101
|
return {
|
|
102
102
|
snapshot,
|
|
103
|
+
async ensureSession() { await ensure(); await follow(); return snapshot(); },
|
|
103
104
|
async open() { if (sessionId) await follow(); },
|
|
104
105
|
async resume() {
|
|
105
106
|
following?.abort(); await followTask;
|
package/voice/capture.js
CHANGED
|
@@ -106,7 +106,7 @@ export function createCaptureSession(io) {
|
|
|
106
106
|
else job.stop = true;
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
-
async function start() {
|
|
109
|
+
async function start({ manual = false } = {}) {
|
|
110
110
|
if (job && (job.phase === "preparing" || job.phase === "recording" || job.phase === "transcribing")) {
|
|
111
111
|
const previous = job;
|
|
112
112
|
previous.cancel = true;
|
|
@@ -171,7 +171,10 @@ export function createCaptureSession(io) {
|
|
|
171
171
|
if (run.phase === "recording" && now() - run.seen > CLIENT_LOST_MS) run.cancel = true;
|
|
172
172
|
const available = await io.read(offset);
|
|
173
173
|
const fresh = available.subarray(0, Math.max(0, MAX_CAPTURE_BYTES - pcm.length));
|
|
174
|
-
if (fresh.length < available.length || now() - started >= MAX_CAPTURE_MS)
|
|
174
|
+
if (fresh.length < available.length || now() - started >= MAX_CAPTURE_MS) {
|
|
175
|
+
if (manual) { await io.close?.(); run.phase = "error"; run.error = "voice_capture_limit"; return; }
|
|
176
|
+
run.stop = true;
|
|
177
|
+
}
|
|
175
178
|
offset += fresh.length;
|
|
176
179
|
if (fresh.length) {
|
|
177
180
|
pcm = Buffer.concat([pcm, fresh]);
|
|
@@ -192,7 +195,7 @@ export function createCaptureSession(io) {
|
|
|
192
195
|
}
|
|
193
196
|
if (level.audible) live = true;
|
|
194
197
|
if (level.audible && level.rms >= ENDPOINT.floor) spoke = true;
|
|
195
|
-
if (live && endpoint.push(1, level.rms, FRAME_MS)) run.stop = true;
|
|
198
|
+
if (!manual && live && endpoint.push(1, level.rms, FRAME_MS)) run.stop = true;
|
|
196
199
|
}
|
|
197
200
|
if (run.cancel) {
|
|
198
201
|
await io.close?.();
|
package/voice/client.js
CHANGED
|
@@ -109,7 +109,9 @@ export async function saveVoiceSettings(patch) {
|
|
|
109
109
|
body: JSON.stringify(patch),
|
|
110
110
|
});
|
|
111
111
|
if (!res.ok) throw new Error(String(res.status));
|
|
112
|
-
|
|
112
|
+
const next = await loadVoiceSettings({ force: true });
|
|
113
|
+
globalThis.window?.dispatchEvent?.(new Event("lares-voice-settings"));
|
|
114
|
+
return next;
|
|
113
115
|
}
|
|
114
116
|
|
|
115
117
|
/** Reading mode and the GPU slicing it needs; see core/voice/gpu-mode.js. */
|
|
@@ -239,7 +241,10 @@ export async function postBluetoothReady(signal) {
|
|
|
239
241
|
* do ride the port — all succeed.
|
|
240
242
|
*/
|
|
241
243
|
export async function postTranscribe(blob, language, signal, options = {}) {
|
|
242
|
-
const
|
|
244
|
+
const params = new URLSearchParams();
|
|
245
|
+
if (language !== undefined) params.set("language", language || "");
|
|
246
|
+
if (options.model !== undefined) params.set("model", options.model || "");
|
|
247
|
+
const query = params.size ? `?${params}` : "";
|
|
243
248
|
const base = String(options.baseUrl ?? API).replace(/\/$/, "");
|
|
244
249
|
const url = `${base}/transcribe${query}`;
|
|
245
250
|
const headers = { "content-type": blob.type || "audio/webm" };
|
package/voice/config.js
CHANGED
|
@@ -6,6 +6,9 @@ export { STT_LANGUAGE_CHOICES, VOICE_AUTO, voiceMenuValue, voiceValueFromMenu }
|
|
|
6
6
|
|
|
7
7
|
import { VOICE_DEFAULTS as DEFAULTS, VOICE_PRESETS } from "./options.js";
|
|
8
8
|
|
|
9
|
+
import { DICTATION_PRESETS, validateDictationProfiles } from "./dictation.js";
|
|
10
|
+
import { validateVoiceShortcuts } from "./shortcuts.js";
|
|
11
|
+
|
|
9
12
|
const WRITABLE = new Set(Object.keys(DEFAULTS));
|
|
10
13
|
const SUPPORTED_LANGUAGES = new Set(["", ...STT_LANGUAGE_CHOICES.map((item) => item.id)]);
|
|
11
14
|
const SAFE_MODEL = /^[^\u0000-\u001f\u007f]{0,512}$/;
|
|
@@ -20,6 +23,14 @@ export function validateConfigPatch(raw) {
|
|
|
20
23
|
const patch = {};
|
|
21
24
|
for (const [key, value] of Object.entries(raw)) {
|
|
22
25
|
if (!WRITABLE.has(key)) invalid(`unknown voice setting: ${key}`);
|
|
26
|
+
if (key === "dictationPreset") {
|
|
27
|
+
if (!DICTATION_PRESETS.includes(value)) invalid("invalid dictation preset");
|
|
28
|
+
patch[key] = value; continue;
|
|
29
|
+
}
|
|
30
|
+
if (key === "dictationProfiles" || key === "dictationShortcuts") {
|
|
31
|
+
try { patch[key] = key === "dictationProfiles" ? validateDictationProfiles(value) : validateVoiceShortcuts(value); } catch { invalid("invalid dictation configuration"); }
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
23
34
|
if (key === "preset") {
|
|
24
35
|
if (!VOICE_PRESETS.includes(value)) invalid("invalid voice preset");
|
|
25
36
|
patch[key] = value; continue;
|
|
@@ -33,7 +44,7 @@ export function validateConfigPatch(raw) {
|
|
|
33
44
|
patch[key] = value;
|
|
34
45
|
continue;
|
|
35
46
|
}
|
|
36
|
-
if (key === "speak" || key === "streaming"
|
|
47
|
+
if (key === "speak" || key === "streaming") {
|
|
37
48
|
if (typeof value !== "boolean") invalid(`${key} must be a boolean`);
|
|
38
49
|
patch[key] = value;
|
|
39
50
|
continue;
|
|
@@ -61,12 +72,13 @@ function configPath() {
|
|
|
61
72
|
/** @returns {typeof DEFAULTS} */
|
|
62
73
|
export function readConfig() {
|
|
63
74
|
const stored = readJsonFile(configPath());
|
|
64
|
-
const merged =
|
|
75
|
+
const merged = structuredClone(DEFAULTS);
|
|
65
76
|
for (const key of Object.keys(DEFAULTS)) {
|
|
66
77
|
if (stored?.[key] !== undefined) {
|
|
67
78
|
try { Object.assign(merged, validateConfigPatch({ [key]: stored[key] })); } catch {}
|
|
68
79
|
}
|
|
69
80
|
}
|
|
81
|
+
if (!stored?.dictationPreset && typeof stored?.dictationCleanup === "boolean") merged.dictationPreset = stored.dictationCleanup ? "clean" : "raw";
|
|
70
82
|
return merged;
|
|
71
83
|
}
|
|
72
84
|
|
package/voice/controller.js
CHANGED
|
@@ -3,6 +3,7 @@ import { mintRpcId } from "../larepass/rpc.js";
|
|
|
3
3
|
/** Device, agent and playback ports are injected; UI frameworks own no turn state. */
|
|
4
4
|
export function createVoiceController({ input, agent, output, voiceId = mintRpcId(), metric = () => {}, onTranscript = null, idleTimeoutMs = 300_000, recoveryTimeoutMs = 120_000, onExpired = () => {}, now = Date.now, schedule = setTimeout, unschedule = clearTimeout }) {
|
|
5
5
|
let epoch = 0;
|
|
6
|
+
let manualTurns = false;
|
|
6
7
|
let take = null;
|
|
7
8
|
let idleTimer = null;
|
|
8
9
|
let recoveryTimer = null;
|
|
@@ -19,7 +20,7 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
19
20
|
idleTimer.unref?.();
|
|
20
21
|
};
|
|
21
22
|
let creating = null;
|
|
22
|
-
let state = Object.freeze({ voiceId, sessionId: null, connection: "closed", capture: "idle", agent: "idle", playback: "idle", continuous: false, inputPaused: false, userSpeaking: false, inputLevel: null, outputMuted: false, error: null });
|
|
23
|
+
let state = Object.freeze({ voiceId, sessionId: null, connection: "closed", capture: "idle", agent: "idle", playback: "idle", continuous: false, inputPaused: false, userSpeaking: false, inputLevel: null, inputReady: false, outputMuted: false, error: null });
|
|
23
24
|
const listeners = new Set();
|
|
24
25
|
const change = (patch) => {
|
|
25
26
|
state = Object.freeze({ ...state, ...patch });
|
|
@@ -51,15 +52,15 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
51
52
|
};
|
|
52
53
|
const listen = async () => {
|
|
53
54
|
if (state.connection !== "open" || state.inputPaused || take || state.capture !== "idle") return;
|
|
54
|
-
if (state.continuous && (
|
|
55
|
+
if (!manualTurns && state.continuous && (
|
|
55
56
|
(state.agent !== "idle" && !input.capabilities?.listenWhileWorking)
|
|
56
57
|
|| (state.playback !== "idle" && !input.capabilities?.bargeIn)
|
|
57
58
|
)) return;
|
|
58
59
|
const run = { epoch, abort: new AbortController(), id: mintRpcId(), started: now() };
|
|
59
60
|
take = run;
|
|
60
|
-
change({ capture: "recording", error: null });
|
|
61
|
+
change({ capture: "recording", inputReady: false, error: null });
|
|
61
62
|
try {
|
|
62
|
-
const text = String(await input.capture({ signal: run.abort.signal, continuous: state.continuous, onLevel: (level) => { if (live(run)) change({ inputLevel: Math.max(0, Math.min(1, level)) }); }, onSpeech: () => {
|
|
63
|
+
const text = String(await input.capture({ signal: run.abort.signal, continuous: state.continuous && !manualTurns, onReady: () => { if (live(run)) change({ inputReady: true }); }, onLevel: (level) => { if (live(run)) change({ inputLevel: Math.max(0, Math.min(1, level)) }); }, onSpeech: () => {
|
|
63
64
|
if (live(run)) { touch(); change({ userSpeaking: true }); if (input.capabilities?.bargeIn) void stopPlayback(); }
|
|
64
65
|
}, onTranscribing: () => { if (live(run)) change({ capture: "transcribing", userSpeaking: false }); } }) ?? "").trim();
|
|
65
66
|
if (!live(run)) return;
|
|
@@ -83,7 +84,7 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
83
84
|
if (take === run) {
|
|
84
85
|
take = null;
|
|
85
86
|
change({ capture: "idle", userSpeaking: false, inputLevel: null });
|
|
86
|
-
if (state.continuous && state.connection === "open" && (state.agent === "idle" || input.capabilities?.listenWhileWorking) && state.playback === "idle") {
|
|
87
|
+
if (!manualTurns && state.continuous && state.connection === "open" && (state.agent === "idle" || input.capabilities?.listenWhileWorking) && state.playback === "idle") {
|
|
87
88
|
queueMicrotask(() => { void listen(); });
|
|
88
89
|
}
|
|
89
90
|
}
|
|
@@ -95,16 +96,18 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
95
96
|
setOutputMuted(outputMuted) { change({ outputMuted }); },
|
|
96
97
|
bindSession(sessionId) { if (sessionId) change({ sessionId }); },
|
|
97
98
|
subscribe(listener) { listeners.add(listener); return () => listeners.delete(listener); },
|
|
98
|
-
async start({ sessionId = null, continuous = false, running = false } = {}) {
|
|
99
|
+
async start({ sessionId = null, continuous = false, running = false, manual = false, shouldStart = () => true } = {}) {
|
|
100
|
+
manualTurns = manual;
|
|
99
101
|
const startedEpoch = ++epoch;
|
|
100
102
|
creating = null;
|
|
101
103
|
await cancelInput();
|
|
102
104
|
if (startedEpoch !== epoch) return;
|
|
103
105
|
await stopPlayback();
|
|
104
106
|
if (startedEpoch !== epoch) return;
|
|
107
|
+
if (!shouldStart()) return;
|
|
105
108
|
change({ voiceId, sessionId, continuous, inputPaused: false, connection: "open", agent: running ? "running" : "idle", error: null });
|
|
106
109
|
clearTimers(); touch();
|
|
107
|
-
void listen();
|
|
110
|
+
if (!manualTurns) void listen();
|
|
108
111
|
},
|
|
109
112
|
listen,
|
|
110
113
|
stopRecording: () => input.stop?.(),
|
|
@@ -118,12 +121,19 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
118
121
|
observeAgent(running, playbackExpected = false) {
|
|
119
122
|
if (playbackExpected && take && !input.capabilities?.bargeIn) void cancelInput();
|
|
120
123
|
change({ agent: running ? "running" : "idle", ...(playbackExpected ? { playback: "pending" } : {}) });
|
|
121
|
-
if (!running && state.continuous) void listen();
|
|
124
|
+
if (!manualTurns && !running && state.continuous) void listen();
|
|
122
125
|
},
|
|
123
126
|
observePlayback(playing) {
|
|
124
127
|
if (playing !== (state.playback === "playing")) touch();
|
|
125
128
|
change({ playback: playing ? "playing" : "idle" });
|
|
126
|
-
if (!playing && state.continuous) void listen();
|
|
129
|
+
if (!manualTurns && !playing && state.continuous) void listen();
|
|
130
|
+
},
|
|
131
|
+
async pause() {
|
|
132
|
+
clearTimers(); epoch += 1;
|
|
133
|
+
change({ connection: "paused", error: null });
|
|
134
|
+
recoveryTimer = schedule(() => { void expire("voice_recovery_expired"); }, recoveryTimeoutMs);
|
|
135
|
+
recoveryTimer.unref?.();
|
|
136
|
+
await Promise.allSettled([cancelInput(), stopPlayback()]);
|
|
127
137
|
},
|
|
128
138
|
async disconnect(code = null) {
|
|
129
139
|
clearTimers();
|
|
@@ -135,12 +145,14 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
135
145
|
change({ connection: code ? "paused" : "closed", error: code });
|
|
136
146
|
await Promise.allSettled([cancelInput(), stopPlayback()]);
|
|
137
147
|
},
|
|
138
|
-
async resume() {
|
|
148
|
+
async resume({ shouldResume = () => true } = {}) {
|
|
149
|
+
const resumedEpoch = epoch;
|
|
139
150
|
if (state.connection !== "paused") return;
|
|
140
151
|
const running = state.sessionId ? await agent.running(state.sessionId) : false;
|
|
152
|
+
if (resumedEpoch !== epoch || !shouldResume()) return;
|
|
141
153
|
clearTimers();
|
|
142
154
|
change({ connection: "open", agent: running ? "running" : "idle", error: null });
|
|
143
|
-
touch(); void listen();
|
|
155
|
+
touch(); if (!manualTurns) void listen();
|
|
144
156
|
},
|
|
145
157
|
};
|
|
146
158
|
return controller;
|
package/voice/conversation.js
CHANGED
|
@@ -3,7 +3,7 @@ import { createDeviceInput } from "./device-input.js";
|
|
|
3
3
|
import { createHostInput } from "./host-input.js";
|
|
4
4
|
import { mintRpcId } from "../larepass/rpc.js";
|
|
5
5
|
|
|
6
|
-
export function createVoiceConversation({ request, transcribe, playback, onReply = () => {}, onPhase, mode = () => "queue", device = "device", metric = () => {}, readAloud = true, preset, endpointSilenceMs, idleTimeoutMs, recoveryTimeoutMs, onText = () => {}, onTranscript = null, input: inputPort }) {
|
|
6
|
+
export function createVoiceConversation({ request, transcribe, playback, onReply = () => {}, onPhase, mode = () => "queue", device = "device", metric = () => {}, readAloud = true, preset, presentationSession = false, endpointSilenceMs, idleTimeoutMs, recoveryTimeoutMs, onText = () => {}, onTranscript = null, input: inputPort, captureConfig }) {
|
|
7
7
|
const voiceId = mintRpcId();
|
|
8
8
|
const headers = { "x-lares-voice-id": voiceId, "x-lares-voice-device": device };
|
|
9
9
|
let timer = null;
|
|
@@ -16,6 +16,7 @@ export function createVoiceConversation({ request, transcribe, playback, onReply
|
|
|
16
16
|
let submitted = false;
|
|
17
17
|
let finalSeen = false;
|
|
18
18
|
let continuous = false;
|
|
19
|
+
let manualCapture = false;
|
|
19
20
|
let reading = false;
|
|
20
21
|
const call = async (path, body, signal) => {
|
|
21
22
|
const res = await request(`/api/lares/voice${path}`, { method: body === undefined ? "GET" : "POST", headers: { ...headers, "x-lares-voice-speaking": controller.getSnapshot().userSpeaking ? "true" : "false" }, ...(body === undefined ? {} : { body }), signal });
|
|
@@ -23,10 +24,10 @@ export function createVoiceConversation({ request, transcribe, playback, onReply
|
|
|
23
24
|
return res.body;
|
|
24
25
|
};
|
|
25
26
|
const input = inputPort ?? (device === "headset" ? createHostInput({
|
|
26
|
-
start: (signal) => call("/capture", {}, signal), status: (signal) => call("/capture", undefined, signal),
|
|
27
|
+
start: (signal) => call("/capture", { config: captureConfig, manual: manualCapture || Boolean(onTranscript) }, signal), status: (signal) => call("/capture", undefined, signal),
|
|
27
28
|
stop: () => call("/capture/stop", {}), cancel: () => call("/capture/cancel", {}),
|
|
28
29
|
}) : createDeviceInput({ transcribe, onPhase, endpointSilenceMs }));
|
|
29
|
-
const controller = createVoiceController({ voiceId, input, idleTimeoutMs, recoveryTimeoutMs, onExpired: () => stop(), onTranscript: onTranscript ? async (value) => {
|
|
30
|
+
const controller = createVoiceController({ voiceId, input, idleTimeoutMs, recoveryTimeoutMs, onExpired: () => stop(), onTranscript: onTranscript ? async (value) => { await onTranscript(value); if (!continuous) await stop(); } : null, metric: (event) => {
|
|
30
31
|
metric(event);
|
|
31
32
|
void call("/session/metric", { stage: event.stage, durationMs: event.durationMs }).catch(() => {});
|
|
32
33
|
}, output: { async stop() { playback.stop(); await call("/playback/stop", {}); } }, agent: {
|
|
@@ -55,6 +56,7 @@ export function createVoiceConversation({ request, transcribe, playback, onReply
|
|
|
55
56
|
if (reply?.text && !reply.silent) onText(reply.text);
|
|
56
57
|
const key = reply ? `${state.sessionId}:${reply.key}` : null;
|
|
57
58
|
lastReplyKey = key;
|
|
59
|
+
if (["recording", "transcribing"].includes(controller.getSnapshot().capture)) mutedKey = key;
|
|
58
60
|
const reserve = readAloud && key !== mutedKey && reply?.text && !reply.silent && key !== spokenKey;
|
|
59
61
|
if (reserve) spokenKey = key;
|
|
60
62
|
controller.observeAgent(Boolean(state.running), Boolean(reserve));
|
|
@@ -91,23 +93,29 @@ export function createVoiceConversation({ request, transcribe, playback, onReply
|
|
|
91
93
|
else controller.observePlayback(playing);
|
|
92
94
|
},
|
|
93
95
|
get sessionId() { return lastSessionId; },
|
|
94
|
-
async start({ sessionId = null, continuous: nextContinuous = false } = {}) {
|
|
96
|
+
async start({ sessionId = null, continuous: nextContinuous = false, manual = false, shouldStart = () => true } = {}) {
|
|
97
|
+
manualCapture = manual;
|
|
95
98
|
continuous = nextContinuous; submitted = false; finalSeen = false; spokenKey = null;
|
|
96
99
|
disposed = false; version = -1;
|
|
97
100
|
clearTimeout(timer);
|
|
98
|
-
const initial = onTranscript ? {} : await call("/session", { sessionId, readAloud, ...(preset ? { preset } : {}) });
|
|
99
|
-
|
|
101
|
+
const initial = onTranscript ? {} : await call("/session", { sessionId, readAloud, presentationSession, ...(preset ? { preset } : {}) });
|
|
102
|
+
if (!shouldStart()) { await stop(); return; }
|
|
103
|
+
await controller.start({ sessionId, continuous, manual, running: initial.running, shouldStart });
|
|
100
104
|
controller.setOutputMuted(!readAloud);
|
|
101
105
|
if (!onTranscript) void watch();
|
|
102
106
|
},
|
|
103
107
|
stop,
|
|
104
|
-
async
|
|
105
|
-
|
|
108
|
+
async pause() { clearTimeout(timer); await controller.pause(); },
|
|
109
|
+
async resume(options = {}) {
|
|
110
|
+
if (onTranscript) { disposed = false; await controller.resume(options); return; }
|
|
106
111
|
const current = await call("/session");
|
|
112
|
+
version = current.version;
|
|
113
|
+
mutedKey = current.utterance ? `${current.sessionId}:${current.utterance.key}` : null;
|
|
114
|
+
spokenKey = mutedKey;
|
|
107
115
|
if (!current.active) await call("/session", { sessionId: lastSessionId, readAloud });
|
|
108
116
|
else await call("/session/resume", {});
|
|
109
117
|
disposed = false;
|
|
110
|
-
await controller.resume();
|
|
118
|
+
await controller.resume(options);
|
|
111
119
|
void watch();
|
|
112
120
|
},
|
|
113
121
|
async setReadAloud(value) {
|
package/voice/device-input.js
CHANGED
|
@@ -48,7 +48,7 @@ export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = c
|
|
|
48
48
|
};
|
|
49
49
|
return {
|
|
50
50
|
capabilities: { duplex: true, bargeIn: false, listenWhileWorking: true, audio: "encoded" },
|
|
51
|
-
capture({ signal, onTranscribing, onSpeech, onLevel = () => {}, continuous = false } = {}) {
|
|
51
|
+
capture({ signal, onTranscribing, onReady, onSpeech, onLevel = () => {}, continuous = false } = {}) {
|
|
52
52
|
cancel();
|
|
53
53
|
return new Promise((resolve, reject) => {
|
|
54
54
|
settle = resolve;
|
|
@@ -62,6 +62,7 @@ export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = c
|
|
|
62
62
|
} }, onSpeech, onLevel, { silenceMs: endpointSilenceMs });
|
|
63
63
|
}, onPhase(phase, extra) {
|
|
64
64
|
onPhase(phase, extra);
|
|
65
|
+
if (phase === "recording") onReady?.();
|
|
65
66
|
if (phase === "transcribing") onTranscribing?.();
|
|
66
67
|
if (phase === "error") {
|
|
67
68
|
if (continuous && ["voice_no_input", "voice_too_short"].includes(extra.error)) { done(""); return; }
|
|
@@ -69,7 +70,7 @@ export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = c
|
|
|
69
70
|
} });
|
|
70
71
|
signal?.addEventListener("abort", cancel, { once: true });
|
|
71
72
|
if (signal?.aborted) { cancel(); return; }
|
|
72
|
-
timer = setTimeout(() => recorder?.stop(), 60_000);
|
|
73
|
+
timer = setTimeout(() => { if (continuous) recorder?.stop(); else { settle = null; cancel(); reject(new Error("voice_capture_limit")); } }, 60_000);
|
|
73
74
|
void recorder.start();
|
|
74
75
|
});
|
|
75
76
|
},
|