@jskit-ai/assistant-voice 0.1.40 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/examples/conversation/README.md +1 -1
- package/examples/conversation/package.json +8 -8
- package/package.json +2 -2
- package/src/client/voiceConversation.js +5 -20
- package/src/client/voicePlayback.js +4 -2
- package/src/client/voiceTransport.js +22 -17
- package/src/server/index.js +0 -1
- package/src/server/voiceDaemon.js +12 -8
- package/src/server/voiceProxy.js +31 -3
- package/src/shared/protocol.js +62 -45
- package/src/server/voiceSocketClose.js +0 -31
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Optional duplex voice for Vue 3/Vuetify applications. One shell controller owns
|
|
4
4
|
microphone, playback, active destination and modal. Applications retain their
|
|
5
|
-
conversations, permissions, message admission and history. No
|
|
5
|
+
conversations, permissions, message admission and history. No Genesis
|
|
6
6
|
runtime is required.
|
|
7
7
|
|
|
8
8
|
Install `@jskit-ai/assistant-voice` with Vue and Vuetify. Run your own speech
|
|
@@ -70,8 +70,8 @@ outside its screen. `available = false` releases the active voice session. Dispo
|
|
|
70
70
|
the controller on app teardown or identity replacement. IDs must include the
|
|
71
71
|
identity scope when different people can use the same conversation ID.
|
|
72
72
|
|
|
73
|
-
Optional defaults: `mode` (`push-to-talk` or `hands-free`), `
|
|
74
|
-
|
|
73
|
+
Optional defaults: `mode` (`push-to-talk` or `hands-free`), `reviewBeforeSend`,
|
|
74
|
+
and an installed `voiceId`.
|
|
75
75
|
One-off recordings started through `startHeldRecording()` require review; the
|
|
76
76
|
modal's push-to-talk gesture uses `reviewBeforeSend`. `avatar` and `onVisual`/`onTranscript`
|
|
77
77
|
allow presentation in the surrounding app without adding another audio owner.
|
|
@@ -186,6 +186,8 @@ The engine supplies a recognizer, asynchronous synthesis and optionally a public
|
|
|
186
186
|
voice catalogue. A host may supply synchronous `authorize(token)` and
|
|
187
187
|
`handleRequest(request, response)` for its own grant policy. An authorized
|
|
188
188
|
principal has `tenant` and optional `parentTenant`; both connection quotas apply.
|
|
189
|
+
The catalogue and the WebSocket apply the same rule: any other result, a
|
|
190
|
+
Promise or a thrown error denies.
|
|
189
191
|
`revokeTenant(id)` ends current child or parent sessions as well as pending work.
|
|
190
192
|
The host must reject future requests for revoked grants in its authorizer.
|
|
191
193
|
|
|
@@ -195,7 +197,7 @@ The host must reject future requests for revoked grants in its authorizer.
|
|
|
195
197
|
client binding and one shared realtime connection for five file-backed chats.
|
|
196
198
|
Text and voice retain the same conversation; an optional headless task uses the
|
|
197
199
|
same runtime directly. Its README covers packed-package setup, launch and the
|
|
198
|
-
local origin policy. No
|
|
200
|
+
local origin policy. No Genesis or database runtime is required.
|
|
199
201
|
|
|
200
202
|
Focused package checks run one file at a time, for example:
|
|
201
203
|
`npm test --workspace packages/assistant-voice -- test/voiceController.test.js`.
|
|
@@ -4,7 +4,7 @@ This Vue/Vuetify and Fastify application uses the supplied JSKIT conversation
|
|
|
4
4
|
server integration, client binding and shared realtime connection. Planning,
|
|
5
5
|
Notes, Research, Writing and Review have separate file-backed histories. One
|
|
6
6
|
root voice modal retains its target independently of mounted text cards.
|
|
7
|
-
|
|
7
|
+
Genesis and a database are not required.
|
|
8
8
|
|
|
9
9
|
## Install and start
|
|
10
10
|
|
|
@@ -9,14 +9,14 @@
|
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"@fastify/websocket": "^11.2.0",
|
|
12
|
-
"@jskit-ai/assistant-core": "0.1.
|
|
13
|
-
"@jskit-ai/assistant-runtime": "0.1.
|
|
14
|
-
"@jskit-ai/assistant-voice": "0.1.
|
|
15
|
-
"@jskit-ai/connectors-catalog": "0.1.
|
|
16
|
-
"@jskit-ai/http-runtime": "0.1.
|
|
17
|
-
"@jskit-ai/kernel": "0.1.
|
|
18
|
-
"@jskit-ai/realtime": "0.1.
|
|
19
|
-
"@jskit-ai/shell-web": "0.1.
|
|
12
|
+
"@jskit-ai/assistant-core": "0.1.241",
|
|
13
|
+
"@jskit-ai/assistant-runtime": "0.1.235",
|
|
14
|
+
"@jskit-ai/assistant-voice": "0.1.42",
|
|
15
|
+
"@jskit-ai/connectors-catalog": "0.1.65",
|
|
16
|
+
"@jskit-ai/http-runtime": "0.1.244",
|
|
17
|
+
"@jskit-ai/kernel": "0.1.246",
|
|
18
|
+
"@jskit-ai/realtime": "0.1.245",
|
|
19
|
+
"@jskit-ai/shell-web": "0.1.252",
|
|
20
20
|
"@mdi/js": "^7.4.47",
|
|
21
21
|
"@tanstack/vue-query": "^5.90.5",
|
|
22
22
|
"fastify": "^5.8.0",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jskit-ai/assistant-voice",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.42",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Duplex voice conversations with application-owned targets, authorization and speech services.",
|
|
6
6
|
"scripts": {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"examples"
|
|
19
19
|
],
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@jskit-ai/assistant-core": "0.1.
|
|
21
|
+
"@jskit-ai/assistant-core": "0.1.241",
|
|
22
22
|
"@mdi/js": "^7.4.47",
|
|
23
23
|
"ws": "^8.20.1"
|
|
24
24
|
},
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { computed, onScopeDispose, ref, watch } from "vue";
|
|
2
|
-
import { useVoiceTransport } from "./voiceTransport.js";
|
|
2
|
+
import { createVoiceTurnId, useVoiceTransport } from "./voiceTransport.js";
|
|
3
3
|
import { speechTextFromAssistant, takeStreamingSpeech, VOICE_MAX_SPEECH_TEXT_CHARACTERS } from "../shared/protocol.js";
|
|
4
4
|
import { createConversationNarrationTracker } from "./conversationNarration.js";
|
|
5
5
|
|
|
6
6
|
/** Created inside the controller's effect scope, independent of a mounted screen. */
|
|
7
7
|
export function useVoiceConversation(binding, { socketUrl, createTransport = useVoiceTransport } = {}) {
|
|
8
|
-
const oneOffTalkMode = computed(() => binding.defaults?.talkMode === "hold" ? "hold" : "tap");
|
|
9
8
|
const targetLabel = computed(() => binding.label || "Assistant");
|
|
10
9
|
const readAloud = ref(binding.defaults?.readAloud === true);
|
|
11
10
|
const readAloudChangePending = computed(() => binding.readAloudChangePending === true);
|
|
@@ -43,9 +42,6 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
43
42
|
const microphoneMuted = voice.microphoneMuted;
|
|
44
43
|
const capturing = computed(() => voice.captureState.value !== "idle");
|
|
45
44
|
const speechActive = computed(() => Boolean(voice.activeSpeechTurnId.value));
|
|
46
|
-
const talkLabel = computed(() => voice.listening.value ? "Send recording"
|
|
47
|
-
: capturing.value ? "Cancel recording" : `Talk to ${targetLabel.value}`);
|
|
48
|
-
const liveLabel = computed(() => starting.value ? "Cancel voice startup" : live.value ? "End voice" : "Start voice");
|
|
49
45
|
const callConnection = computed(() => {
|
|
50
46
|
if (voice.reconnecting.value) return "Voice interrupted";
|
|
51
47
|
if (error.value || voice.error.value) return "Voice needs attention";
|
|
@@ -71,19 +67,8 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
71
67
|
if (voice.listening.value) return "Listening to you";
|
|
72
68
|
return callMode.value === "push-to-talk" ? "Hold to speak" : "Preparing microphone…";
|
|
73
69
|
});
|
|
74
|
-
const callDetail = computed(() => {
|
|
75
|
-
if (voice.reconnecting.value) return "Waiting for the voice connection. Your unfinished words remain available for review.";
|
|
76
|
-
if (error.value || voice.error.value) return "Review your message below or try voice again.";
|
|
77
|
-
if (starting.value || live.value && ["connecting", "opening"].includes(voice.captureState.value)) return "Allow microphone access if your browser asks.";
|
|
78
|
-
if (!live.value) return "";
|
|
79
|
-
if (microphoneMuted.value) return "Unmute when you want to speak. You can still listen.";
|
|
80
|
-
if (!readAloud.value) return "Sound is off. Answers still appear in chat.";
|
|
81
|
-
if (callMode.value === "push-to-talk") return "Hold to speak. Release to send. The conversation stays open.";
|
|
82
|
-
return "You can speak naturally, including while I reply.";
|
|
83
|
-
});
|
|
84
70
|
const callModeBusy = computed(() => starting.value || changingCallMode.value || pushHolding.value || sending.value || Boolean(pendingTranscript.value)
|
|
85
71
|
|| capturing.value && Boolean(voice.partialTranscript.value.trim()) || capturing.value && !voice.listening.value);
|
|
86
|
-
const microphoneStatus = computed(() => microphoneMuted.value ? "Microphone muted" : voice.listening.value ? "Microphone on" : "Microphone off");
|
|
87
72
|
const avatarVisual = computed(() => ({
|
|
88
73
|
avatar: binding.avatar,
|
|
89
74
|
state: voice.speaking.value ? "speaking" : pushHolding.value && voice.listening.value ? "listening" : binding.state.status === "working" ? "thinking"
|
|
@@ -274,7 +259,7 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
274
259
|
if (!text) continue;
|
|
275
260
|
if (!reply.started) {
|
|
276
261
|
reply.started = true;
|
|
277
|
-
reply.turnId =
|
|
262
|
+
reply.turnId = createVoiceTurnId("speak");
|
|
278
263
|
const accepted = await voice.speak(text, reply.turnId, { stream: true });
|
|
279
264
|
if (disposed) return;
|
|
280
265
|
if (reply.cancelled) {
|
|
@@ -872,10 +857,10 @@ export function useVoiceConversation(binding, { socketUrl, createTransport = use
|
|
|
872
857
|
(sending.value || Boolean(pendingTranscript.value) || capturing.value));
|
|
873
858
|
const hasUnsentSpeech = computed(() => capturing.value || starting.value || sending.value || Boolean(pendingTranscript.value));
|
|
874
859
|
return {
|
|
875
|
-
|
|
860
|
+
targetLabel, voice, readAloud, readAloudChangePending, error, pendingTranscript, heldReview, holding, sending,
|
|
876
861
|
live, starting, callMode, changingCallMode, pushHolding, microphoneMuted, capturing,
|
|
877
|
-
speechActive,
|
|
878
|
-
|
|
862
|
+
speechActive, callConnection, callStatus, callModeBusy,
|
|
863
|
+
avatarVisual, callAudioLevel, status, heldTranscript, voiceWords, voiceAnswer,
|
|
879
864
|
stopSpeech, inviteSpeech, toggleReadAloud, enableSound, soundPreparing, deliverTranscript, cancelRecording,
|
|
880
865
|
discardRecording, dismissTranscript, talk, toggleMicrophoneMuted, changeCallMode,
|
|
881
866
|
startPushToTalk, finishPushToTalk, cancelPushToTalk, toggleLive, toggleHandsFree, startHeldRecording,
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { VOICE_OUTPUT_SAMPLE_RATE } from "../shared/protocol.js";
|
|
2
|
+
|
|
1
3
|
const MOUTH_POSES = new Set(["closed", "open", "round", "small", "wide"]);
|
|
2
4
|
|
|
3
5
|
function pcm16ArrayBufferToFloat32(buffer) {
|
|
@@ -9,10 +11,10 @@ function pcm16ArrayBufferToFloat32(buffer) {
|
|
|
9
11
|
return samples;
|
|
10
12
|
}
|
|
11
13
|
|
|
12
|
-
function normalizeSpeechSegment(message = {}, sampleRate =
|
|
14
|
+
function normalizeSpeechSegment(message = {}, sampleRate = VOICE_OUTPUT_SAMPLE_RATE) {
|
|
13
15
|
const sampleCount = Math.max(0, Math.floor(Number(message.sampleCount) || 0));
|
|
14
16
|
const durationMs = sampleCount
|
|
15
|
-
? (sampleCount / Math.max(1, Number(sampleRate) ||
|
|
17
|
+
? (sampleCount / Math.max(1, Number(sampleRate) || VOICE_OUTPUT_SAMPLE_RATE)) * 1_000
|
|
16
18
|
: Math.max(0, Number(message.durationMs) || 0);
|
|
17
19
|
const cues = (Array.isArray(message.cues) ? message.cues : [])
|
|
18
20
|
.map((cue) => ({
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { computed, onScopeDispose, ref, watch } from "vue";
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS,
|
|
5
|
+
VOICE_INPUT_SAMPLE_RATE,
|
|
6
|
+
VOICE_MAX_SYNTHESIS_TIMEOUT_MS,
|
|
7
|
+
VOICE_OUTPUT_SAMPLE_RATE,
|
|
8
|
+
speechTextFromAssistant
|
|
9
|
+
} from "../shared/protocol.js";
|
|
4
10
|
import { createMicrophoneCapture } from "./audioCapture.js";
|
|
5
11
|
import {
|
|
6
12
|
normalizeSpeechSegment,
|
|
@@ -8,12 +14,11 @@ import {
|
|
|
8
14
|
visemeCueAtTime
|
|
9
15
|
} from "./voicePlayback.js";
|
|
10
16
|
|
|
11
|
-
const INPUT_SAMPLE_RATE = 16_000;
|
|
12
17
|
const WEBSOCKET_OPEN = 1;
|
|
13
18
|
let microphoneOwner = null;
|
|
14
19
|
const speechOwner = ref(null);
|
|
15
20
|
|
|
16
|
-
function
|
|
21
|
+
function createVoiceTurnId(prefix = "turn") {
|
|
17
22
|
const id = globalThis.crypto?.randomUUID?.() ||
|
|
18
23
|
`${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
19
24
|
return `${prefix}-${id}`;
|
|
@@ -63,7 +68,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
63
68
|
let heartbeatTimer;
|
|
64
69
|
let lastPongAt = 0;
|
|
65
70
|
let lastSpeechProgressAt = 0;
|
|
66
|
-
let synthesisTimeoutMs =
|
|
71
|
+
let synthesisTimeoutMs = VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS;
|
|
67
72
|
let lastStartedSegmentIndex = -1;
|
|
68
73
|
let microphone = null;
|
|
69
74
|
let preserveBufferedMicrophone = false;
|
|
@@ -83,7 +88,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
83
88
|
let playbackQueue = Promise.resolve();
|
|
84
89
|
let queuedPlaybackFrames = 0;
|
|
85
90
|
let speechStreamEnded = false;
|
|
86
|
-
let speechSampleRate =
|
|
91
|
+
let speechSampleRate = VOICE_OUTPUT_SAMPLE_RATE;
|
|
87
92
|
let receivingTurnId = "";
|
|
88
93
|
let receivingSegment = null;
|
|
89
94
|
const speechInputEnded = ref(true);
|
|
@@ -407,7 +412,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
407
412
|
stopAfterSegment = Infinity;
|
|
408
413
|
bufferedSeconds.value = 0;
|
|
409
414
|
synthesisPending.value = 0;
|
|
410
|
-
synthesisTimeoutMs =
|
|
415
|
+
synthesisTimeoutMs = VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS;
|
|
411
416
|
lastStartedSegmentIndex = -1;
|
|
412
417
|
receivingTurnId = "";
|
|
413
418
|
receivingSegment = null;
|
|
@@ -474,7 +479,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
474
479
|
if (!message.turnId || message.turnId !== activeSpeechTurnId.value) {
|
|
475
480
|
return;
|
|
476
481
|
}
|
|
477
|
-
speechSampleRate = Number(message.sampleRate) ||
|
|
482
|
+
speechSampleRate = Number(message.sampleRate) || VOICE_OUTPUT_SAMPLE_RATE;
|
|
478
483
|
speechStreamEnded = false;
|
|
479
484
|
speechState.value = "thinking";
|
|
480
485
|
nextPlaybackTime = 0;
|
|
@@ -484,8 +489,8 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
484
489
|
if (!speechStreamEnded && Number.isSafeInteger(message.segmentIndex) &&
|
|
485
490
|
message.segmentIndex > lastStartedSegmentIndex) {
|
|
486
491
|
lastStartedSegmentIndex = message.segmentIndex;
|
|
487
|
-
synthesisTimeoutMs =
|
|
488
|
-
if (Number.isSafeInteger(message.synthesisTimeoutMs) && message.synthesisTimeoutMs >
|
|
492
|
+
synthesisTimeoutMs = VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS;
|
|
493
|
+
if (Number.isSafeInteger(message.synthesisTimeoutMs) && message.synthesisTimeoutMs > VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS &&
|
|
489
494
|
message.synthesisTimeoutMs <= VOICE_MAX_SYNTHESIS_TIMEOUT_MS) {
|
|
490
495
|
// The engine acquired its queue slot. Allow its bounded cold job until
|
|
491
496
|
// first PCM; repeated metadata and pongs are not synthesis progress.
|
|
@@ -498,7 +503,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
498
503
|
return;
|
|
499
504
|
}
|
|
500
505
|
if (type === "speech.chunk.end" && message.turnId === activeSpeechTurnId.value) {
|
|
501
|
-
synthesisTimeoutMs =
|
|
506
|
+
synthesisTimeoutMs = VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS;
|
|
502
507
|
lastSpeechProgressAt = Date.now();
|
|
503
508
|
synthesisPending.value = Math.max(0, synthesisPending.value - 1);
|
|
504
509
|
return;
|
|
@@ -532,7 +537,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
532
537
|
function handleSocketMessage(event) {
|
|
533
538
|
if (event.data instanceof ArrayBuffer) {
|
|
534
539
|
if (speechOutputEnabled.value && activeSpeechTurnId.value && receivingTurnId === activeSpeechTurnId.value && !speechStreamEnded) {
|
|
535
|
-
synthesisTimeoutMs =
|
|
540
|
+
synthesisTimeoutMs = VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS;
|
|
536
541
|
lastSpeechProgressAt = Date.now();
|
|
537
542
|
enqueuePcm(event.data);
|
|
538
543
|
}
|
|
@@ -652,7 +657,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
652
657
|
microphoneClaim = Symbol("voice microphone");
|
|
653
658
|
microphoneOwner = microphoneClaim;
|
|
654
659
|
preserveBufferedMicrophone = false;
|
|
655
|
-
const turnId =
|
|
660
|
+
const turnId = createVoiceTurnId("listen");
|
|
656
661
|
pendingListenTurnId = turnId;
|
|
657
662
|
if (interruptSpeechOnListen) stopSpeaking();
|
|
658
663
|
captureState.value = "connecting";
|
|
@@ -678,7 +683,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
678
683
|
socket.send(frame);
|
|
679
684
|
}
|
|
680
685
|
},
|
|
681
|
-
targetSampleRate:
|
|
686
|
+
targetSampleRate: VOICE_INPUT_SAMPLE_RATE
|
|
682
687
|
});
|
|
683
688
|
openingMicrophones.add(opening);
|
|
684
689
|
try { capture = await opening; }
|
|
@@ -703,7 +708,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
703
708
|
try {
|
|
704
709
|
endpoint.value = null;
|
|
705
710
|
completedUtterance.value = null;
|
|
706
|
-
sendControl({ sampleRate:
|
|
711
|
+
sendControl({ sampleRate: VOICE_INPUT_SAMPLE_RATE, turnId, type: "listen.start", ...(continuous ? { continuous: true } : {}) });
|
|
707
712
|
captureState.value = "listening";
|
|
708
713
|
} catch (controlError) {
|
|
709
714
|
activeListenTurnId.value = "";
|
|
@@ -738,7 +743,7 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
738
743
|
sendControl({ turnId, type: "listen.stop" });
|
|
739
744
|
}
|
|
740
745
|
|
|
741
|
-
async function speak(text, turnId =
|
|
746
|
+
async function speak(text, turnId = createVoiceTurnId("speak"), { stream = false } = {}) {
|
|
742
747
|
const speechText = speechTextFromAssistant(text);
|
|
743
748
|
if (unmounted || !speechOutputEnabled.value || !speechText) {
|
|
744
749
|
return false;
|
|
@@ -958,6 +963,6 @@ function useVoiceTransport({ socketUrl, speechEnabled = true, interruptSpeechOnL
|
|
|
958
963
|
}
|
|
959
964
|
|
|
960
965
|
export {
|
|
961
|
-
|
|
962
|
-
|
|
966
|
+
createVoiceTurnId,
|
|
967
|
+
useVoiceTransport
|
|
963
968
|
};
|
package/src/server/index.js
CHANGED
|
@@ -2,4 +2,3 @@ export { createVoiceDaemon } from "./voiceDaemon.js";
|
|
|
2
2
|
export { registerVoiceProxyRoute, readVoiceCatalogue } from "./voiceProxy.js";
|
|
3
3
|
export { resolveVoiceProxyConfig, readVoiceAccessToken, voiceEndpoint } from "./voiceProxyConfig.js";
|
|
4
4
|
export { createVoiceAccessToken, verifyVoiceAccessToken, bearerVoiceAccessToken } from "./voiceAccessToken.js";
|
|
5
|
-
export { normalizeVoiceCloseMetadata } from "./voiceSocketClose.js";
|
|
@@ -4,6 +4,7 @@ import { WebSocketServer } from "ws";
|
|
|
4
4
|
|
|
5
5
|
import {
|
|
6
6
|
VOICE_DAEMON_WEBSOCKET_PATH,
|
|
7
|
+
VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS,
|
|
7
8
|
VOICE_INPUT_SAMPLE_RATE,
|
|
8
9
|
VOICE_MAX_AUDIO_FRAME_BYTES,
|
|
9
10
|
VOICE_MAX_CONTROL_BYTES,
|
|
@@ -147,6 +148,13 @@ function createVoiceDaemon({
|
|
|
147
148
|
const sockets = new Set();
|
|
148
149
|
const principals = new WeakMap();
|
|
149
150
|
const speechQueue = createBoundedSerialQueue({ maximumQueued: maximumQueuedSpeech });
|
|
151
|
+
// One acceptance rule for every request: the host authorizer is synchronous
|
|
152
|
+
// and must name a tenant; anything else, including a thrown error, denies.
|
|
153
|
+
function principalFor(request) {
|
|
154
|
+
let principal;
|
|
155
|
+
try { principal = authorize(bearerVoiceAccessToken(request.headers), request); } catch { return null; }
|
|
156
|
+
return principal && typeof principal.tenant === "string" && principal.tenant ? principal : null;
|
|
157
|
+
}
|
|
150
158
|
let ready = false;
|
|
151
159
|
let startedAt = "";
|
|
152
160
|
let stopping = false;
|
|
@@ -167,9 +175,7 @@ function createVoiceDaemon({
|
|
|
167
175
|
}
|
|
168
176
|
try {
|
|
169
177
|
if (request.method === "GET" && requestPathname(request) === "/voices") {
|
|
170
|
-
|
|
171
|
-
try { principal = await authorize(bearerVoiceAccessToken(request.headers), request); } catch { /* Deny invalid credentials. */ }
|
|
172
|
-
if (!principal) { writeJsonResponse(response, 401, { ok: false }); return; }
|
|
178
|
+
if (!principalFor(request)) { writeJsonResponse(response, 401, { ok: false }); return; }
|
|
173
179
|
if (!ready || stopping) { writeJsonResponse(response, 503, { ok: false }); return; }
|
|
174
180
|
writeJsonResponse(response, 200, voiceCatalogue());
|
|
175
181
|
return;
|
|
@@ -359,7 +365,7 @@ function createVoiceDaemon({
|
|
|
359
365
|
const segmentIndex = active.segmentIndex++;
|
|
360
366
|
const synthesisTimeoutMs = engine.synthesisTimeoutMs;
|
|
361
367
|
sendJson(socket, "speech.segment.start", { segmentIndex, turnId,
|
|
362
|
-
...(Number.isSafeInteger(synthesisTimeoutMs) && synthesisTimeoutMs >
|
|
368
|
+
...(Number.isSafeInteger(synthesisTimeoutMs) && synthesisTimeoutMs > VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS &&
|
|
363
369
|
synthesisTimeoutMs <= VOICE_MAX_SYNTHESIS_TIMEOUT_MS ? { synthesisTimeoutMs } : {}) });
|
|
364
370
|
let sampleCount = 0;
|
|
365
371
|
const result = text.trim() ? await engine.synthesize(text.trim(), {
|
|
@@ -511,10 +517,8 @@ function createVoiceDaemon({
|
|
|
511
517
|
socket.destroy();
|
|
512
518
|
return;
|
|
513
519
|
}
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
catch { verified = null; }
|
|
517
|
-
if (!verified || typeof verified.tenant !== "string" || !verified.tenant) {
|
|
520
|
+
const verified = principalFor(request);
|
|
521
|
+
if (!verified) {
|
|
518
522
|
socket.write("HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n");
|
|
519
523
|
socket.destroy();
|
|
520
524
|
return;
|
package/src/server/voiceProxy.js
CHANGED
|
@@ -4,13 +4,41 @@ import {
|
|
|
4
4
|
VOICE_MAX_AUDIO_FRAME_BYTES,
|
|
5
5
|
VOICE_MAX_CONTROL_BYTES,
|
|
6
6
|
assertVoiceAudioFrame,
|
|
7
|
-
parseVoiceClientControl
|
|
7
|
+
parseVoiceClientControl,
|
|
8
|
+
voiceServerMessage
|
|
8
9
|
} from "../shared/protocol.js";
|
|
9
|
-
import { normalizeVoiceCloseMetadata } from "./voiceSocketClose.js";
|
|
10
10
|
|
|
11
11
|
const WEBSOCKET_CONNECTING = 0;
|
|
12
12
|
const WEBSOCKET_OPEN = 1;
|
|
13
13
|
|
|
14
|
+
const RESERVED_CLOSE_CODES = new Set([1004, 1005, 1006, 1015]);
|
|
15
|
+
|
|
16
|
+
function proxyCloseCode(code) {
|
|
17
|
+
const numericCode = Number(code);
|
|
18
|
+
const isStandardCode = Number.isInteger(numericCode)
|
|
19
|
+
&& numericCode >= 1000
|
|
20
|
+
&& numericCode <= 1014
|
|
21
|
+
&& !RESERVED_CLOSE_CODES.has(numericCode);
|
|
22
|
+
const isApplicationCode = Number.isInteger(numericCode)
|
|
23
|
+
&& numericCode >= 3000
|
|
24
|
+
&& numericCode <= 4999;
|
|
25
|
+
return isStandardCode || isApplicationCode ? numericCode : 1000;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function proxyCloseReason(reason) {
|
|
29
|
+
const encoded = Buffer.from(String(reason || ""), "utf8");
|
|
30
|
+
return encoded.byteLength <= 123
|
|
31
|
+
? encoded.toString("utf8")
|
|
32
|
+
: encoded.subarray(0, 120).toString("utf8");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function normalizeVoiceCloseMetadata(code, reason) {
|
|
36
|
+
return Object.freeze({
|
|
37
|
+
code: proxyCloseCode(code),
|
|
38
|
+
reason: proxyCloseReason(reason)
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
14
42
|
async function readVoiceCatalogue(config, { fetchImpl = fetch } = {}) {
|
|
15
43
|
try {
|
|
16
44
|
if (!config?.available) throw new Error("Voice is not configured.");
|
|
@@ -39,7 +67,7 @@ function closeSocket(socket, code, reason) {
|
|
|
39
67
|
|
|
40
68
|
function sendSocketError(socket, code, message) {
|
|
41
69
|
if (socket?.readyState === WEBSOCKET_OPEN) {
|
|
42
|
-
socket.send(
|
|
70
|
+
socket.send(voiceServerMessage("error", { code, message }));
|
|
43
71
|
}
|
|
44
72
|
}
|
|
45
73
|
|
package/src/shared/protocol.js
CHANGED
|
@@ -5,6 +5,7 @@ const VOICE_OUTPUT_SAMPLE_RATE = 22_050;
|
|
|
5
5
|
const VOICE_MAX_CONTROL_BYTES = 16 * 1024;
|
|
6
6
|
const VOICE_MAX_AUDIO_FRAME_BYTES = 64 * 1024;
|
|
7
7
|
const VOICE_MAX_UTTERANCE_SECONDS = 60;
|
|
8
|
+
const VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS = 20_000;
|
|
8
9
|
const VOICE_MAX_SYNTHESIS_TIMEOUT_MS = 120_000;
|
|
9
10
|
const VOICE_MAX_SPEECH_TEXT_CHARACTERS = 4_000;
|
|
10
11
|
const VOICE_MAX_SPEECH_CHUNK_CHARACTERS = 280;
|
|
@@ -12,6 +13,7 @@ const VOICE_SPEECH_EM_DASH_PAUSE_MS = 140;
|
|
|
12
13
|
const VOICE_SPEECH_SENTENCE_PAUSE_MS = 220;
|
|
13
14
|
const VOICE_SPEECH_LINE_PAUSE_MS = 350;
|
|
14
15
|
const VOICE_SPEECH_PARAGRAPH_PAUSE_MS = 650;
|
|
16
|
+
const VOICE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/u;
|
|
15
17
|
const VOICE_TURN_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/u;
|
|
16
18
|
const VOICE_CLIENT_CONTROL_TYPES = new Set([
|
|
17
19
|
"cancel",
|
|
@@ -95,7 +97,7 @@ function parseVoiceClientControl(raw = "") {
|
|
|
95
97
|
return Object.freeze({ type, turnId, revision: input.revision });
|
|
96
98
|
}
|
|
97
99
|
if (type === "speak.start" || type === "speak.append") {
|
|
98
|
-
if (input.voiceId !== undefined && (type !== "speak.start" || typeof input.voiceId !== "string" ||
|
|
100
|
+
if (input.voiceId !== undefined && (type !== "speak.start" || typeof input.voiceId !== "string" || !VOICE_ID_PATTERN.test(input.voiceId))) {
|
|
99
101
|
throw voiceProtocolError("voice_selection_invalid", "Choose an available voice before starting speech.");
|
|
100
102
|
}
|
|
101
103
|
const text = String(input.text || "");
|
|
@@ -150,6 +152,56 @@ function float32ToPcm16Le(samples = new Float32Array()) {
|
|
|
150
152
|
return bytes;
|
|
151
153
|
}
|
|
152
154
|
|
|
155
|
+
const BUILT_IN_RECOGNITION_PHRASES = Object.freeze([
|
|
156
|
+
[/\bc plus plus\b/giu, "C++"],
|
|
157
|
+
[/\b(?:j|jay)\s+(?:s|ess)\s+kit\b/giu, "JSKIT"],
|
|
158
|
+
[/\bnode\s+(?:(?:j|jay)\s+(?:s|ess)|js)\b/giu, "Node.js"],
|
|
159
|
+
[/\bpostgres\s+(?:s\s+)?q\s+l\b/giu, "PostgreSQL"],
|
|
160
|
+
[/\bweb socket\b/giu, "WebSocket"],
|
|
161
|
+
[/\bsherpa onnx\b/giu, "sherpa-onnx"]
|
|
162
|
+
]);
|
|
163
|
+
const BUILT_IN_RECOGNITION_SPELLINGS = Object.freeze({
|
|
164
|
+
acl: "ACL",
|
|
165
|
+
ai: "AI",
|
|
166
|
+
api: "API",
|
|
167
|
+
cmake: "CMake",
|
|
168
|
+
codex: "Codex",
|
|
169
|
+
env: "ENV",
|
|
170
|
+
erd: "ERD",
|
|
171
|
+
fastify: "Fastify",
|
|
172
|
+
git: "Git",
|
|
173
|
+
github: "GitHub",
|
|
174
|
+
http: "HTTP",
|
|
175
|
+
https: "HTTPS",
|
|
176
|
+
hmac: "HMAC",
|
|
177
|
+
id: "ID",
|
|
178
|
+
javascript: "JavaScript",
|
|
179
|
+
jskit: "JSKIT",
|
|
180
|
+
json: "JSON",
|
|
181
|
+
mariadb: "MariaDB",
|
|
182
|
+
mysql: "MySQL",
|
|
183
|
+
npm: "npm",
|
|
184
|
+
onnx: "ONNX",
|
|
185
|
+
opencode: "OpenCode",
|
|
186
|
+
openai: "OpenAI",
|
|
187
|
+
playwright: "Playwright",
|
|
188
|
+
postgresql: "PostgreSQL",
|
|
189
|
+
pty: "PTY",
|
|
190
|
+
sql: "SQL",
|
|
191
|
+
systemd: "systemd",
|
|
192
|
+
typescript: "TypeScript",
|
|
193
|
+
uuid: "UUID",
|
|
194
|
+
vitest: "Vitest",
|
|
195
|
+
vue: "Vue",
|
|
196
|
+
vuetify: "Vuetify",
|
|
197
|
+
websocket: "WebSocket",
|
|
198
|
+
zipformer: "Zipformer"
|
|
199
|
+
});
|
|
200
|
+
const BUILT_IN_RECOGNITION_WORD_PATTERN = new RegExp(
|
|
201
|
+
`\\b(?:${Object.keys(BUILT_IN_RECOGNITION_SPELLINGS).join("|")})\\b`,
|
|
202
|
+
"giu"
|
|
203
|
+
);
|
|
204
|
+
|
|
153
205
|
function normalizeRecognizedText(value = "", vocabulary = {}) {
|
|
154
206
|
const text = String(value || "").replace(/\s+/gu, " ").trim();
|
|
155
207
|
if (!text || text !== text.toUpperCase() || !/[A-Z]/u.test(text)) {
|
|
@@ -158,50 +210,13 @@ function normalizeRecognizedText(value = "", vocabulary = {}) {
|
|
|
158
210
|
const normalized = text.toLowerCase().replace(/(^|[.!?]\s+)([a-z])/gu, (_match, prefix, letter) => (
|
|
159
211
|
`${prefix}${letter.toUpperCase()}`
|
|
160
212
|
));
|
|
161
|
-
let result = normalized
|
|
162
|
-
|
|
163
|
-
.replace(
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
.
|
|
167
|
-
|
|
168
|
-
.replace(/\b(?:ai|sql|api|json|http|https|mysql|mariadb|postgresql|erd|id|uuid|env|pty|acl|hmac|onnx|npm|jskit|opencode|openai|javascript|typescript|vue|vuetify|fastify|playwright|vitest|github|git|systemd|cmake|codex|zipformer|websocket)\b/giu, (word) => ({
|
|
169
|
-
acl: "ACL",
|
|
170
|
-
ai: "AI",
|
|
171
|
-
api: "API",
|
|
172
|
-
cmake: "CMake",
|
|
173
|
-
codex: "Codex",
|
|
174
|
-
env: "ENV",
|
|
175
|
-
erd: "ERD",
|
|
176
|
-
fastify: "Fastify",
|
|
177
|
-
git: "Git",
|
|
178
|
-
github: "GitHub",
|
|
179
|
-
http: "HTTP",
|
|
180
|
-
https: "HTTPS",
|
|
181
|
-
hmac: "HMAC",
|
|
182
|
-
id: "ID",
|
|
183
|
-
javascript: "JavaScript",
|
|
184
|
-
jskit: "JSKIT",
|
|
185
|
-
json: "JSON",
|
|
186
|
-
mariadb: "MariaDB",
|
|
187
|
-
mysql: "MySQL",
|
|
188
|
-
npm: "npm",
|
|
189
|
-
onnx: "ONNX",
|
|
190
|
-
opencode: "OpenCode",
|
|
191
|
-
openai: "OpenAI",
|
|
192
|
-
playwright: "Playwright",
|
|
193
|
-
postgresql: "PostgreSQL",
|
|
194
|
-
pty: "PTY",
|
|
195
|
-
sql: "SQL",
|
|
196
|
-
systemd: "systemd",
|
|
197
|
-
typescript: "TypeScript",
|
|
198
|
-
uuid: "UUID",
|
|
199
|
-
vitest: "Vitest",
|
|
200
|
-
vue: "Vue",
|
|
201
|
-
vuetify: "Vuetify",
|
|
202
|
-
websocket: "WebSocket",
|
|
203
|
-
zipformer: "Zipformer"
|
|
204
|
-
})[word.toLowerCase()] || word);
|
|
213
|
+
let result = normalized;
|
|
214
|
+
for (const [pattern, spelling] of BUILT_IN_RECOGNITION_PHRASES) {
|
|
215
|
+
result = result.replace(pattern, spelling);
|
|
216
|
+
}
|
|
217
|
+
result = result.replace(BUILT_IN_RECOGNITION_WORD_PATTERN, (word) => (
|
|
218
|
+
BUILT_IN_RECOGNITION_SPELLINGS[word.toLowerCase()] || word
|
|
219
|
+
));
|
|
205
220
|
for (const [phrase, spelling] of Object.entries(vocabulary)) {
|
|
206
221
|
const literal = phrase.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
207
222
|
result = result.replace(new RegExp(`\\b${literal}\\b`, "giu"), () => spelling);
|
|
@@ -416,6 +431,8 @@ function voiceServerMessage(type = "", fields = {}) {
|
|
|
416
431
|
|
|
417
432
|
export {
|
|
418
433
|
VOICE_DAEMON_WEBSOCKET_PATH,
|
|
434
|
+
VOICE_DEFAULT_SYNTHESIS_TIMEOUT_MS,
|
|
435
|
+
VOICE_ID_PATTERN,
|
|
419
436
|
VOICE_INPUT_SAMPLE_RATE,
|
|
420
437
|
VOICE_MAX_AUDIO_FRAME_BYTES,
|
|
421
438
|
VOICE_MAX_CONTROL_BYTES,
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
const RESERVED_CLOSE_CODES = new Set([1004, 1005, 1006, 1015]);
|
|
2
|
-
|
|
3
|
-
function proxyCloseCode(code) {
|
|
4
|
-
const numericCode = Number(code);
|
|
5
|
-
const isStandardCode = Number.isInteger(numericCode)
|
|
6
|
-
&& numericCode >= 1000
|
|
7
|
-
&& numericCode <= 1014
|
|
8
|
-
&& !RESERVED_CLOSE_CODES.has(numericCode);
|
|
9
|
-
const isApplicationCode = Number.isInteger(numericCode)
|
|
10
|
-
&& numericCode >= 3000
|
|
11
|
-
&& numericCode <= 4999;
|
|
12
|
-
return isStandardCode || isApplicationCode ? numericCode : 1000;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
function proxyCloseReason(reason) {
|
|
16
|
-
const encoded = Buffer.from(String(reason || ""), "utf8");
|
|
17
|
-
return encoded.byteLength <= 123
|
|
18
|
-
? encoded.toString("utf8")
|
|
19
|
-
: encoded.subarray(0, 120).toString("utf8");
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
function normalizeVoiceCloseMetadata(code, reason) {
|
|
23
|
-
return Object.freeze({
|
|
24
|
-
code: proxyCloseCode(code),
|
|
25
|
-
reason: proxyCloseReason(reason)
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export {
|
|
30
|
-
normalizeVoiceCloseMetadata
|
|
31
|
-
};
|