@olares/lares-core 0.15.27 → 0.15.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/i18n/voice.js +14 -0
- package/package.json +4 -2
- package/voice/browser-capture.js +29 -2
- package/voice/controller.js +35 -4
- package/voice/device-input.js +17 -43
- package/voice/endpoint.js +58 -2
- package/voice/errors.js +40 -0
- package/voice/host-service.js +27 -11
- package/voice/lares-realtime-gateway.js +99 -19
- package/voice/panel.js +13 -1
- package/voice/pcm-player.js +37 -8
- package/voice/presentation.js +14 -0
- package/voice/protocol-client.js +12 -4
- package/voice/router-audio.js +15 -1
- package/voice/service.js +19 -3
- package/voice/speech.js +7 -1
- package/voice/stream-client.js +25 -3
- package/voice/stream-gateway.js +10 -2
- package/voice/synthesis-run.js +11 -1
- package/voice/unified-conversation.js +122 -29
- package/voice/vad.js +169 -0
package/i18n/voice.js
CHANGED
|
@@ -132,6 +132,13 @@ export const ZH = {
|
|
|
132
132
|
"error.voice_too_short": "录音太短",
|
|
133
133
|
"error.voice_no_input": "没有检测到声音",
|
|
134
134
|
"error.voice_failed": "转写失败,请重试",
|
|
135
|
+
"error.voice_asr_timeout": "转写等待超时,请重试",
|
|
136
|
+
"error.voice_audio_backpressure": "播放缓冲已满,请重试",
|
|
137
|
+
"error.voice_playback_failed": "播放失败,请重试",
|
|
138
|
+
"error.voice_stream_state": "语音连接状态异常,请重试",
|
|
139
|
+
"error.voice_interruption_timeout": "打断等待超时,已停止朗读",
|
|
140
|
+
"error.turn_busy": "上一轮还在处理,请再说一次",
|
|
141
|
+
"error.waiting_for_playback": "上一段回复还在播放,请稍后再说",
|
|
135
142
|
"error.voice_unsupported": "当前浏览器不支持录音",
|
|
136
143
|
"error.voice_bluetooth_not_ready": "蓝牙麦克风还没连上,请重试",
|
|
137
144
|
"error.voice_bluetooth_unavailable": "这台机器的蓝牙还没就绪。请确认机器有蓝牙并已打开,然后在 Olares 市场重启 Lares,重启时会重新安装蓝牙组件",
|
|
@@ -436,6 +443,13 @@ export const EN = {
|
|
|
436
443
|
"error.voice_too_short": "Recording too short",
|
|
437
444
|
"error.voice_no_input": "No voice detected",
|
|
438
445
|
"error.voice_failed": "Transcription failed, please retry",
|
|
446
|
+
"error.voice_asr_timeout": "Timed out waiting for the transcript, please retry",
|
|
447
|
+
"error.voice_audio_backpressure": "Playback buffer is full, please retry",
|
|
448
|
+
"error.voice_playback_failed": "Playback failed, please retry",
|
|
449
|
+
"error.voice_stream_state": "The voice connection lost its place, please retry",
|
|
450
|
+
"error.voice_interruption_timeout": "Interruption timed out; playback stopped",
|
|
451
|
+
"error.turn_busy": "The previous turn is still running, please speak again",
|
|
452
|
+
"error.waiting_for_playback": "The previous answer is still playing, please wait",
|
|
439
453
|
"error.voice_unsupported": "This browser does not support recording",
|
|
440
454
|
"error.voice_bluetooth_not_ready": "Bluetooth microphone is not ready, please retry",
|
|
441
455
|
"error.voice_bluetooth_unavailable":
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@olares/lares-core",
|
|
3
|
-
"version": "0.15.
|
|
3
|
+
"version": "0.15.29",
|
|
4
4
|
"description": "Lares 共享业务核心:Router、workspace、文件/媒体、Olares 身份(无 UI、无平台壳)",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"repository": {
|
|
@@ -224,7 +224,9 @@
|
|
|
224
224
|
"./router/default-selection": {
|
|
225
225
|
"types": "./router/default-selection.d.ts",
|
|
226
226
|
"default": "./router/default-selection.js"
|
|
227
|
-
}
|
|
227
|
+
},
|
|
228
|
+
"./voice/vad": "./voice/vad.js",
|
|
229
|
+
"./voice/errors": "./voice/errors.js"
|
|
228
230
|
},
|
|
229
231
|
"dependencies": {
|
|
230
232
|
"emoji-regex": "11.0.0",
|
package/voice/browser-capture.js
CHANGED
|
@@ -161,8 +161,26 @@ export function createVoiceCapture({
|
|
|
161
161
|
emit("transcribing", { elapsed: took, error: "" });
|
|
162
162
|
abort = new AbortController();
|
|
163
163
|
try {
|
|
164
|
-
|
|
165
|
-
|
|
164
|
+
let text = "";
|
|
165
|
+
let committed = false, replayable = true;
|
|
166
|
+
if (streaming) {
|
|
167
|
+
const port = streaming;
|
|
168
|
+
streaming = null;
|
|
169
|
+
committed = port.committed === true;
|
|
170
|
+
replayable = port.replayable !== false;
|
|
171
|
+
try {
|
|
172
|
+
text = String(await port.finish() ?? "").trim();
|
|
173
|
+
} catch (err) {
|
|
174
|
+
if (dead || abort.signal.aborted || cancel) return;
|
|
175
|
+
text = "";
|
|
176
|
+
}
|
|
177
|
+
// finish() is the commit; the Host transcribed this audio and returned
|
|
178
|
+
// nothing. Uploading the blob again would submit the same take twice.
|
|
179
|
+
committed ||= port.committed === true;
|
|
180
|
+
replayable = port.replayable !== false;
|
|
181
|
+
}
|
|
182
|
+
// The socket can finish with nothing while the recording still has the words.
|
|
183
|
+
if (!text && (!committed || replayable)) text = String(await transcribe(blob, abort.signal) ?? "").trim();
|
|
166
184
|
if (dead || cancel) return;
|
|
167
185
|
if (!text) {
|
|
168
186
|
trace("no-input", {});
|
|
@@ -204,6 +222,15 @@ export function createVoiceCapture({
|
|
|
204
222
|
recorder.stop();
|
|
205
223
|
} catch {
|
|
206
224
|
scheduleFinish();
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
// Some WebViews never fire onstop. The button already switched the UI to
|
|
228
|
+
// transcribing, so finish with the chunks collected so far instead of waiting.
|
|
229
|
+
if (!finishTimer) {
|
|
230
|
+
finishTimer = setTimeout(() => {
|
|
231
|
+
finishTimer = 0;
|
|
232
|
+
void finish();
|
|
233
|
+
}, 1000);
|
|
207
234
|
}
|
|
208
235
|
};
|
|
209
236
|
|
package/voice/controller.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createInterruptionGuard } from "./interruption.js";
|
|
2
|
+
import { isTransientVoiceError, voiceErrorCode } from "./errors.js";
|
|
2
3
|
import { mintRpcId } from "../larepass/rpc.js";
|
|
3
4
|
|
|
4
5
|
/** Device, agent and playback ports are injected; UI frameworks own no turn state. */
|
|
@@ -6,6 +7,7 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
6
7
|
let epoch = 0;
|
|
7
8
|
let manualTurns = false;
|
|
8
9
|
let take = null;
|
|
10
|
+
let playbackError = null;
|
|
9
11
|
let idleTimer = null;
|
|
10
12
|
let recoveryTimer = null;
|
|
11
13
|
const clearTimers = () => { unschedule(idleTimer); unschedule(recoveryTimer); };
|
|
@@ -64,9 +66,13 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
64
66
|
)) return;
|
|
65
67
|
const run = { epoch, abort: new AbortController(), id: mintRpcId(), started: now() };
|
|
66
68
|
take = run;
|
|
67
|
-
|
|
69
|
+
// A playback failure is not cleared by merely listening again; the next
|
|
70
|
+
// reply that actually renders clears it.
|
|
71
|
+
change({ capture: "recording", inputReady: false, error: playbackError });
|
|
68
72
|
try {
|
|
69
|
-
|
|
73
|
+
// A call outlives each take. `manual` only means the finger ends the
|
|
74
|
+
// recording; it must not turn that take into a one-shot dictation.
|
|
75
|
+
const text = String(await input.capture({ signal: run.abort.signal, continuous: state.continuous, manual: manualTurns, onReady: () => { if (live(run)) change({ inputReady: true }); }, onLevel: (level) => { if (live(run)) change({ inputLevel: Math.max(0, Math.min(1, level)) }); }, onSpeech: () => {
|
|
70
76
|
if (live(run)) { touch(); change({ userSpeaking: true }); if (interruptible() && state.playback !== "idle") interruption.suspect(); }
|
|
71
77
|
}, onTranscribing: () => { if (live(run)) { interruption.speechEnded(); change({ capture: "transcribing", userSpeaking: false }); } } }) ?? "").trim();
|
|
72
78
|
if (!live(run)) return;
|
|
@@ -87,8 +93,18 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
87
93
|
// Acceptance is not completion; observeAgent receives the durable turn lifecycle.
|
|
88
94
|
} catch (error) {
|
|
89
95
|
if (live(run)) {
|
|
90
|
-
|
|
91
|
-
|
|
96
|
+
const code = voiceErrorCode(error) || "voice_failed";
|
|
97
|
+
if (isTransientVoiceError(code)) {
|
|
98
|
+
// The take lost a race with the previous turn or with a reply that
|
|
99
|
+
// still owns the shared audio link. Drop only this take: the call
|
|
100
|
+
// stays open and the composer returns to listening for the next one.
|
|
101
|
+
interruption.cancel();
|
|
102
|
+
report("take-dropped", run.started, { turnId: run.id, code });
|
|
103
|
+
change({ capture: "idle", userSpeaking: false, inputLevel: null });
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
interruption.fail(code);
|
|
107
|
+
await controller.disconnect(code);
|
|
92
108
|
}
|
|
93
109
|
} finally {
|
|
94
110
|
if (take === run) {
|
|
@@ -104,11 +120,19 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
104
120
|
getSnapshot: () => state,
|
|
105
121
|
noteActivity: touch,
|
|
106
122
|
setOutputMuted(outputMuted) { change({ outputMuted }); },
|
|
123
|
+
/** Audio that never reached the speaker is visible, not silently swallowed. */
|
|
124
|
+
notePlaybackError(code) {
|
|
125
|
+
const value = voiceErrorCode(code) || "voice_playback_failed";
|
|
126
|
+
playbackError = value;
|
|
127
|
+
report("playback-error", now(), { code: value });
|
|
128
|
+
change({ error: value });
|
|
129
|
+
},
|
|
107
130
|
bindSession(sessionId) { if (sessionId) change({ sessionId }); },
|
|
108
131
|
subscribe(listener) { listeners.add(listener); return () => listeners.delete(listener); },
|
|
109
132
|
async start({ sessionId = null, continuous = false, running = false, manual = false, shouldStart = () => true } = {}) {
|
|
110
133
|
output.arm?.();
|
|
111
134
|
manualTurns = manual;
|
|
135
|
+
playbackError = null;
|
|
112
136
|
const startedEpoch = ++epoch;
|
|
113
137
|
creating = null;
|
|
114
138
|
await cancelInput();
|
|
@@ -137,7 +161,14 @@ export function createVoiceController({ input, agent, output, voiceId = mintRpcI
|
|
|
137
161
|
observePlayback(playing) {
|
|
138
162
|
if (!playing) interruption.outputEnded();
|
|
139
163
|
if (playing !== (state.playback === "playing")) touch();
|
|
164
|
+
// The next reply that actually renders clears the previous audio failure.
|
|
165
|
+
if (playing && playbackError) { playbackError = null; change({ error: null }); }
|
|
140
166
|
change({ playback: playing ? "playing" : "idle" });
|
|
167
|
+
// Output and microphone share one link. As soon as audio is rendered, a
|
|
168
|
+
// take that is still recording would only send the speaker back to ASR,
|
|
169
|
+
// and the Host rejects those frames. Stop it here instead of waiting for
|
|
170
|
+
// the next agent poll.
|
|
171
|
+
if (playing && take && !interruptible()) void cancelInput();
|
|
141
172
|
if (!manualTurns && state.continuous && (!playing || interruptible())) void listen();
|
|
142
173
|
},
|
|
143
174
|
async pause() {
|
package/voice/device-input.js
CHANGED
|
@@ -1,44 +1,16 @@
|
|
|
1
1
|
import { createVoiceCapture } from "./browser-capture.js";
|
|
2
|
-
import {
|
|
2
|
+
import { createEnergyEndpoint, createInputMeter } from "./endpoint.js";
|
|
3
|
+
import { createVoiceVadEndpoint, preloadVoiceVad } from "./vad.js";
|
|
3
4
|
|
|
4
|
-
export
|
|
5
|
-
const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext;
|
|
6
|
-
if (!AudioCtx) throw new Error("voice_unsupported");
|
|
7
|
-
const context = new AudioCtx();
|
|
8
|
-
const source = context.createMediaStreamSource(stream);
|
|
9
|
-
const processor = context.createScriptProcessor(2048, 1, 1);
|
|
10
|
-
const gain = context.createGain();
|
|
11
|
-
gain.gain.value = 0;
|
|
12
|
-
const endpoint = createSpeechEndpoint(options);
|
|
13
|
-
let heard = false, consecutiveMs = 0;
|
|
14
|
-
const silenceTimer = setTimeout(() => { if (!heard) end(false); }, 10_000);
|
|
15
|
-
processor.onaudioprocess = (event) => {
|
|
16
|
-
const samples = event.inputBuffer.getChannelData(0);
|
|
17
|
-
const rms = frameRms(samples);
|
|
18
|
-
onLevel(rms);
|
|
19
|
-
consecutiveMs = rms >= 0.004 ? consecutiveMs + samples.length * 1000 / context.sampleRate : 0;
|
|
20
|
-
if (consecutiveMs >= (options.interruptionMs || 0) && rms >= 0.004 && !heard) { heard = true; onSpeech(); }
|
|
21
|
-
if (endpoint.push(1, rms, samples.length * 1000 / context.sampleRate)) end(true);
|
|
22
|
-
};
|
|
23
|
-
source.connect(processor); processor.connect(gain); gain.connect(context.destination);
|
|
24
|
-
void context.resume();
|
|
25
|
-
return { stop() { clearTimeout(silenceTimer); processor.onaudioprocess = null; source.disconnect(); processor.disconnect(); gain.disconnect(); void context.close(); } };
|
|
26
|
-
}
|
|
5
|
+
export { createEnergyEndpoint };
|
|
27
6
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
const source = context.createMediaStreamSource(stream);
|
|
33
|
-
const analyser = context.createAnalyser(); analyser.fftSize = 512;
|
|
34
|
-
source.connect(analyser);
|
|
35
|
-
const samples = new Float32Array(analyser.fftSize);
|
|
36
|
-
const timer = setInterval(() => { analyser.getFloatTimeDomainData(samples); onLevel(frameRms(samples)); }, 80);
|
|
37
|
-
void context.resume();
|
|
38
|
-
return { stop() { clearInterval(timer); source.disconnect(); analyser.disconnect(); void context.close(); } };
|
|
7
|
+
/** Silero once it is loaded; the energy endpoint otherwise or on any VAD failure. */
|
|
8
|
+
export function defaultEndpoint(stream, end, onSpeech, onLevel, options) {
|
|
9
|
+
return createVoiceVadEndpoint(stream, end, onSpeech, onLevel, options)
|
|
10
|
+
?? createEnergyEndpoint(stream, end, onSpeech, onLevel, options);
|
|
39
11
|
}
|
|
40
12
|
|
|
41
|
-
export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint =
|
|
13
|
+
export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = defaultEndpoint, endpointSilenceMs = 500, media, preparation, streamInput, echoTested = false } = {}) {
|
|
42
14
|
let recorder = null;
|
|
43
15
|
let settle = null;
|
|
44
16
|
let timer = null;
|
|
@@ -49,8 +21,13 @@ export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = c
|
|
|
49
21
|
};
|
|
50
22
|
return {
|
|
51
23
|
capabilities: { duplex: true, bargeIn: echoTested === true, echoTested: echoTested === true, listenWhileWorking: true, audio: "encoded" },
|
|
52
|
-
capture({ signal, onTranscribing, onReady, onSpeech, onLevel = () => {}, continuous = false } = {}) {
|
|
24
|
+
capture({ signal, onTranscribing, onReady, onSpeech, onLevel = () => {}, continuous = false, manual = false } = {}) {
|
|
53
25
|
cancel();
|
|
26
|
+
// Hands-free calls end themselves. A manual hold is still one take of the same call.
|
|
27
|
+
const automatic = continuous && !manual;
|
|
28
|
+
// Only automatic turns need a speech model. The first take may use energy
|
|
29
|
+
// endpointing; the model is warmed for the next one instead of delaying it.
|
|
30
|
+
if (automatic) void preloadVoiceVad();
|
|
54
31
|
return new Promise((resolve, reject) => {
|
|
55
32
|
let ready = false;
|
|
56
33
|
settle = resolve;
|
|
@@ -58,11 +35,8 @@ export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = c
|
|
|
58
35
|
const prepared = preparation; preparation = null;
|
|
59
36
|
recorder = createVoiceCapture({ media, streamInput, preparation: prepared, transcribe, submit: done, onStream: (stream) => {
|
|
60
37
|
for (const track of stream.getTracks()) track.addEventListener?.("ended", () => { settle = null; cancel(); reject(new Error("voice_device_lost")); }, { once: true });
|
|
61
|
-
if (!
|
|
62
|
-
return endpoint(stream, (heard = true) => { if (heard) recorder?.stop(); else {
|
|
63
|
-
if (continuous) { cancel(); done(""); }
|
|
64
|
-
else { settle = null; cancel(); reject(new Error("voice_no_input")); }
|
|
65
|
-
} }, onSpeech, onLevel, { silenceMs: endpointSilenceMs, interruptionMs: echoTested ? 500 : 0 });
|
|
38
|
+
if (!automatic) return createInputMeter(stream, onLevel);
|
|
39
|
+
return endpoint(stream, (heard = true) => { if (heard) recorder?.stop(); else { cancel(); done(""); } }, onSpeech, onLevel, { silenceMs: endpointSilenceMs, interruptionMs: echoTested ? 500 : 0 });
|
|
66
40
|
}, onPhase(phase, extra) {
|
|
67
41
|
onPhase(phase, extra);
|
|
68
42
|
if (phase === "recording" && !ready) {
|
|
@@ -71,7 +45,7 @@ export function createDeviceInput({ transcribe, onPhase = () => {}, endpoint = c
|
|
|
71
45
|
}
|
|
72
46
|
if (phase === "transcribing") onTranscribing?.();
|
|
73
47
|
if (phase === "error") {
|
|
74
|
-
if (continuous && (extra.recognizedEmpty ||
|
|
48
|
+
if (continuous && (extra.recognizedEmpty || ["voice_no_input", "voice_too_short"].includes(extra.error))) { done(""); return; }
|
|
75
49
|
clearTimeout(timer); signal?.removeEventListener("abort", cancel); settle = null; reject(new Error(extra.error)); }
|
|
76
50
|
} });
|
|
77
51
|
signal?.addEventListener("abort", cancel, { once: true });
|
package/voice/endpoint.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Decide when a take is over from Silero's per-frame speech probability and the
|
|
3
3
|
* frame's loudness. A conference mic hears the whole room, so a frame only
|
|
4
4
|
* counts as the speaker when it is human speech and within `relative` of the
|
|
5
|
-
* loudest speech
|
|
6
|
-
* the recording open.
|
|
5
|
+
* loudest speech heard recently in this take. Farther voices then do not hold
|
|
6
|
+
* the recording open, and a single loud cough does not raise the bar forever.
|
|
7
7
|
*/
|
|
8
8
|
export const ENDPOINT = {
|
|
9
9
|
speechProb: 0.5,
|
|
@@ -14,6 +14,9 @@ export const ENDPOINT = {
|
|
|
14
14
|
relative: 0.3,
|
|
15
15
|
floor: 0.004,
|
|
16
16
|
smoothFrames: 5,
|
|
17
|
+
// The reference is a leaky maximum: it halves roughly every five seconds of
|
|
18
|
+
// speech-level audio, so a door slam cannot close the take for the rest of it.
|
|
19
|
+
referenceDecay: 0.995,
|
|
17
20
|
};
|
|
18
21
|
|
|
19
22
|
export function frameRms(frame) {
|
|
@@ -39,6 +42,7 @@ export function createSpeechEndpoint(options = {}) {
|
|
|
39
42
|
if (recent.length > o.smoothFrames) recent.shift();
|
|
40
43
|
const level = recent.reduce((sum, value) => sum + value, 0) / recent.length;
|
|
41
44
|
const human = prob >= o.speechProb;
|
|
45
|
+
reference *= o.referenceDecay;
|
|
42
46
|
if (human && level > reference) reference = level;
|
|
43
47
|
const near = human && level >= Math.max(o.floor, reference * o.relative);
|
|
44
48
|
if (near) {
|
|
@@ -55,3 +59,55 @@ export function createSpeechEndpoint(options = {}) {
|
|
|
55
59
|
},
|
|
56
60
|
};
|
|
57
61
|
}
|
|
62
|
+
|
|
63
|
+
/** Widest room-noise floor a quiet speaker is still expected to clear. */
|
|
64
|
+
const MAX_NOISE_FLOOR = 0.02;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Endpointing without a speech model. The room's floor is estimated from the
|
|
68
|
+
* frames before the speaker starts and only needs to be cleared by that much,
|
|
69
|
+
* so a loud room does not keep every take open and a quiet speaker still ends
|
|
70
|
+
* it. Used by webviews that cannot load Silero, and as its fallback.
|
|
71
|
+
*/
|
|
72
|
+
export function createEnergyEndpoint(stream, end, onSpeech = () => {}, onLevel = () => {}, options = {}) {
|
|
73
|
+
const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext;
|
|
74
|
+
if (!AudioCtx) throw new Error("voice_unsupported");
|
|
75
|
+
const context = new AudioCtx();
|
|
76
|
+
const source = context.createMediaStreamSource(stream);
|
|
77
|
+
const processor = context.createScriptProcessor(2048, 1, 1);
|
|
78
|
+
const gain = context.createGain();
|
|
79
|
+
gain.gain.value = 0;
|
|
80
|
+
const endpoint = createSpeechEndpoint(options);
|
|
81
|
+
let heard = false, consecutiveMs = 0, noise = null, spoke = false;
|
|
82
|
+
const silenceTimer = setTimeout(() => { if (!heard) end(false); }, 10_000);
|
|
83
|
+
processor.onaudioprocess = (event) => {
|
|
84
|
+
const samples = event.inputBuffer.getChannelData(0);
|
|
85
|
+
const rms = frameRms(samples);
|
|
86
|
+
const frameMs = samples.length * 1000 / context.sampleRate;
|
|
87
|
+
onLevel(rms);
|
|
88
|
+
if (!spoke) noise = noise == null ? rms : noise * 0.9 + rms * 0.1;
|
|
89
|
+
const floor = Math.max(ENDPOINT.floor, Math.min(MAX_NOISE_FLOOR, (noise ?? ENDPOINT.floor / 2.5) * 2.5));
|
|
90
|
+
const human = rms >= floor;
|
|
91
|
+
if (human) spoke = true;
|
|
92
|
+
consecutiveMs = human ? consecutiveMs + frameMs : 0;
|
|
93
|
+
if (consecutiveMs >= (options.interruptionMs || 0) && human && !heard) { heard = true; onSpeech(); }
|
|
94
|
+
if (endpoint.push(human ? 1 : 0, rms, frameMs)) end(true);
|
|
95
|
+
};
|
|
96
|
+
source.connect(processor); processor.connect(gain); gain.connect(context.destination);
|
|
97
|
+
void context.resume();
|
|
98
|
+
return { stop() { clearTimeout(silenceTimer); processor.onaudioprocess = null; source.disconnect(); processor.disconnect(); gain.disconnect(); void context.close(); } };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Visible microphone level for a take that ends by the user's finger. */
|
|
102
|
+
export function createInputMeter(stream, onLevel) {
|
|
103
|
+
const AudioCtx = globalThis.AudioContext || globalThis.webkitAudioContext;
|
|
104
|
+
if (!AudioCtx) return { stop() {} };
|
|
105
|
+
const context = new AudioCtx();
|
|
106
|
+
const source = context.createMediaStreamSource(stream);
|
|
107
|
+
const analyser = context.createAnalyser(); analyser.fftSize = 512;
|
|
108
|
+
source.connect(analyser);
|
|
109
|
+
const samples = new Float32Array(analyser.fftSize);
|
|
110
|
+
const timer = setInterval(() => { analyser.getFloatTimeDomainData(samples); onLevel(frameRms(samples)); }, 80);
|
|
111
|
+
void context.resume();
|
|
112
|
+
return { stop() { clearInterval(timer); source.disconnect(); analyser.disconnect(); void context.close(); } };
|
|
113
|
+
}
|
package/voice/errors.js
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A take can lose a race with the turn that is still being submitted or with a
|
|
3
|
+
* reply that is still playing. That is contention on one shared audio link, not
|
|
4
|
+
* a broken voice session: the take is dropped and the call keeps listening.
|
|
5
|
+
*
|
|
6
|
+
* Everything else stays a real failure so it is never hidden from the user.
|
|
7
|
+
*/
|
|
8
|
+
export const TRANSIENT_VOICE_ERRORS = Object.freeze([
|
|
9
|
+
// The previous turn is still being transcribed/submitted (or the Host queued it).
|
|
10
|
+
"turn_busy",
|
|
11
|
+
// Half duplex: the reply owns the link, or its audio is still in flight.
|
|
12
|
+
"waiting_for_playback",
|
|
13
|
+
// One take exceeded the bounded transport buffers.
|
|
14
|
+
"voice_audio_backpressure",
|
|
15
|
+
"voice_stream_state",
|
|
16
|
+
// The take hit the transport's own size/limit for a single utterance.
|
|
17
|
+
"audio_limit_exceeded",
|
|
18
|
+
// The Host never returned a transcript for a committed take.
|
|
19
|
+
"voice_asr_timeout",
|
|
20
|
+
]);
|
|
21
|
+
|
|
22
|
+
const TRANSIENT = new Set(TRANSIENT_VOICE_ERRORS);
|
|
23
|
+
|
|
24
|
+
export function voiceErrorCode(value) {
|
|
25
|
+
if (typeof value === "string") return value;
|
|
26
|
+
return value?.code || value?.message || "";
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** @returns {boolean} true when only this take failed, not the connection. */
|
|
30
|
+
export function isTransientVoiceError(value) {
|
|
31
|
+
return TRANSIENT.has(voiceErrorCode(value));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Marks a rejection so the controller drops one take instead of the call. */
|
|
35
|
+
export function transientVoiceError(code) {
|
|
36
|
+
const error = new Error(code);
|
|
37
|
+
error.code = code;
|
|
38
|
+
error.transient = true;
|
|
39
|
+
return error;
|
|
40
|
+
}
|
package/voice/host-service.js
CHANGED
|
@@ -68,7 +68,7 @@ export function createVoiceHostService(api, { env = process.env, metric = () =>
|
|
|
68
68
|
const restoredRequests = [...new Set(history.records.map((row) => row.taskId).filter(Boolean))].map((requestId) => ({ requestId }));
|
|
69
69
|
scope.session = createVoiceAgentSession(api, { sessionId: boundSessionId,
|
|
70
70
|
preset: scope.authorizedBinding?.agentPreset || env.LARES_VOICE_AGENT_PRESET || undefined, workspaceId: scope.authorizedBinding?.workspaceId || env.LARES_VOICE_WORKSPACE_ID || undefined, restoredRequests,
|
|
71
|
-
onChange: () =>
|
|
71
|
+
onChange: () => service.advance(scope),
|
|
72
72
|
metric: (fields) => metric({ voiceId: scope.id, ...fields }),
|
|
73
73
|
});
|
|
74
74
|
await scope.session.open();
|
|
@@ -146,9 +146,14 @@ export function createVoiceHostService(api, { env = process.env, metric = () =>
|
|
|
146
146
|
scope.voiceRequests.set(body.requestId, operation);
|
|
147
147
|
return operation;
|
|
148
148
|
},
|
|
149
|
-
|
|
149
|
+
/**
|
|
150
|
+
* Turns a durable task's completion into one pending spoken utterance.
|
|
151
|
+
* Called from session events, never from a read: a GET that loses its
|
|
152
|
+
* response must not consume the result it was about to return.
|
|
153
|
+
*/
|
|
154
|
+
advance(scope) {
|
|
155
|
+
if (!scope.active || !scope.voiceState) return;
|
|
150
156
|
const state = scope.session?.snapshot() || {};
|
|
151
|
-
if (!scope.voiceState) return { ...state, contextId: scope.contextId, active: scope.active === true };
|
|
152
157
|
// Completion is keyed by durable request/turn, never by the session's aggregate idle flag.
|
|
153
158
|
for (const task of state.tasks || []) {
|
|
154
159
|
if (!["completed", "failed", "cancelled"].includes(task.status) || scope.deliveredTasks.has(task.requestId)) continue;
|
|
@@ -157,15 +162,26 @@ export function createVoiceHostService(api, { env = process.env, metric = () =>
|
|
|
157
162
|
scope.voiceResults ??= [];
|
|
158
163
|
scope.voiceResults.push(task);
|
|
159
164
|
}
|
|
160
|
-
if (scope.voiceConfig.preset === "direct"
|
|
165
|
+
if (scope.voiceConfig.preset === "direct" || scope.voiceState.running) return;
|
|
161
166
|
// Pending business work does not block the next speech turn.
|
|
162
|
-
if (
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
167
|
+
if (!scope.voiceResults?.length || scope.voiceInputActive) return;
|
|
168
|
+
const task = scope.voiceResults.shift();
|
|
169
|
+
scope.dialogue.observeTask(task);
|
|
170
|
+
const text = task.status === "completed" ? task.text : `Task ${task.status}: ${task.error || ""}`;
|
|
171
|
+
scope.voiceState.utterance = { key: `task:${task.requestId}`, turn: ++scope.voiceState.version, step: 0, text, done: true, silent: false };
|
|
172
|
+
append(scope, "assistant", text, `result:${task.requestId}`, task.requestId, { taskTurn: task.turn });
|
|
173
|
+
},
|
|
174
|
+
/**
|
|
175
|
+
* A queued task result moves into the sticky `utterance` field before the
|
|
176
|
+
* snapshot is returned. The move is idempotent and the utterance survives
|
|
177
|
+
* the read, so a lost response cannot consume the result; HTTP polls and
|
|
178
|
+
* the socket observe the same snapshot.
|
|
179
|
+
*/
|
|
180
|
+
state(scope) {
|
|
181
|
+
service.advance(scope);
|
|
182
|
+
const state = scope.session?.snapshot() || {};
|
|
183
|
+
if (!scope.voiceState) return { ...state, contextId: scope.contextId, active: scope.active === true };
|
|
184
|
+
if (scope.voiceConfig.preset === "direct") return { ...state, contextId: scope.contextId, active: scope.active === true };
|
|
169
185
|
return { ...state, ...scope.voiceState, contextId: scope.contextId, active: scope.active === true, backgroundRunning: state.running, summaryError: scope.summaryError || null, preset: scope.voiceConfig.preset };
|
|
170
186
|
},
|
|
171
187
|
async remove(sessionId) {
|