@acorex-platform/extension-ai-voice-shell-client 0.0.37 → 0.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/fesm2022/acorex-platform-extension-ai-voice-shell-client-contracts.mjs +30 -1
- package/fesm2022/acorex-platform-extension-ai-voice-shell-client-contracts.mjs.map +1 -1
- package/fesm2022/acorex-platform-extension-ai-voice-shell-client.mjs +1226 -593
- package/fesm2022/acorex-platform-extension-ai-voice-shell-client.mjs.map +1 -1
- package/package.json +1 -1
- package/types/acorex-platform-extension-ai-voice-shell-client-contracts.d.ts +84 -2
- package/types/acorex-platform-extension-ai-voice-shell-client.d.ts +32 -63
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import * as i0 from '@angular/core';
|
|
2
|
-
import { inject, DestroyRef, ViewEncapsulation, ChangeDetectionStrategy, Component,
|
|
2
|
+
import { Injectable, inject, DestroyRef, ViewEncapsulation, ChangeDetectionStrategy, Component, signal, NgModule, InjectionToken, computed } from '@angular/core';
|
|
3
3
|
import { AXPSearchCommandProvider, AXP_SEARCH_PROVIDER, AXPFileStorageService } from '@acorex-platform/framework-client/common';
|
|
4
4
|
import * as i1$1 from '@acorex-platform/framework-client/core';
|
|
5
|
-
import { AXPKeyboardShortcutRegistry, AXPComponentSlotModule } from '@acorex-platform/framework-client/core';
|
|
6
|
-
import { AXPCommandExecutor,
|
|
5
|
+
import { AXPKeyboardShortcutRegistry, AXPComponentSlotModule, provideAppStartUpTask, AXP_APP_STARTUP_PRIORITY } from '@acorex-platform/framework-client/core';
|
|
6
|
+
import { AXPCommandExecutor, AXPCommandService, provideCommandSetups } from '@acorex-platform/framework-client/runtime';
|
|
7
7
|
import { AXMAiManagementModule, AXPAiChatToolRunContextService, AXPAiPlatformRuntimeContextBuilder, AXPAiManagerService, persistAiChatAttachmentImage, AIMANAGEMENT_CHAT_TRANSCRIBE_SPEECH_COMMAND_KEY, AIMANAGEMENT_CHAT_SYNTHESIZE_SPEECH_COMMAND_KEY, createBrowserPlayableTtsBlobUrl } from '@acorex-platform/module-ai-management-client';
|
|
8
|
-
import { AXE_AI_VOICE_SHELL_OPEN_COMMAND,
|
|
9
|
-
import {
|
|
8
|
+
import { AXE_AI_VOICE_SHELL_OPEN_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_STOP_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_INTERRUPT_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_START_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_COMMIT_TURN_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_PUSH_AUDIO_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_EXECUTE_TOOL_COMMAND, AXE_AI_VOICE_OPENROUTER_AUDIO_TOPIC } from '@acorex-platform/extension-ai-voice-shell-client/contracts';
|
|
9
|
+
import { AXPRealtimeDispatcherService, AXP_REALTIME_TOPIC_AI_VOICE_SESSION } from '@acorex-platform/framework-client/realtime';
|
|
10
|
+
import { AXPKeyboardShortcutPriority, runtimeCommandPayloadData, createRandomUuid } from '@acorex-platform/framework-shared/core';
|
|
10
11
|
import { AXPLayoutThemeService } from '@acorex-platform/framework-client/themes/shared';
|
|
11
12
|
import * as i1 from '@acorex/components/button';
|
|
12
13
|
import { AXButtonModule } from '@acorex/components/button';
|
|
@@ -15,13 +16,851 @@ import { AXDecoratorModule } from '@acorex/components/decorators';
|
|
|
15
16
|
import * as i3 from '@acorex/core/translation';
|
|
16
17
|
import { AXTranslationModule, AXTranslationService } from '@acorex/core/translation';
|
|
17
18
|
import * as i2$1 from '@angular/common';
|
|
18
|
-
import { AsyncPipe,
|
|
19
|
+
import { AsyncPipe, CommonModule } from '@angular/common';
|
|
19
20
|
import { AXPopupService } from '@acorex/components/popup';
|
|
20
21
|
import { RootConfig } from '@acorex-platform/module-ai-management-client/contracts';
|
|
21
22
|
import { AXPEntityService } from '@acorex-platform/framework-client/layout/entity';
|
|
22
23
|
import { first } from 'lodash';
|
|
23
24
|
import { AXBasePageComponent } from '@acorex/components/page';
|
|
24
25
|
|
|
26
|
+
//#region ----- Types -----
|
|
27
|
+
//#endregion
|
|
28
|
+
//#region ----- Loader -----
|
|
29
|
+
let pending = null;
|
|
30
|
+
/** Loads the LiveKit browser SDK once, when a voice session connects over RTC. */
|
|
31
|
+
function loadLiveKitClient() {
|
|
32
|
+
pending ??= import('livekit-client');
|
|
33
|
+
return pending;
|
|
34
|
+
}
|
|
35
|
+
//#endregion
|
|
36
|
+
|
|
37
|
+
//#region ----- LiveKit connect options -----
|
|
38
|
+
/** Extra time for ICE on local / Docker Desktop. Matches chat-call joins. */
|
|
39
|
+
const AXE_VOICE_LIVE_PEER_CONNECTION_TIMEOUT_MS = 45_000;
|
|
40
|
+
const DEFAULT_DEV_LIVEKIT_WS_PROXY_PATH = '/livekit-ws';
|
|
41
|
+
function axeIsLocalDevHostname(hostname) {
|
|
42
|
+
const normalized = hostname.trim().toLowerCase();
|
|
43
|
+
return normalized === 'localhost' || normalized === '127.0.0.1' || normalized === '[::1]';
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Route LiveKit signaling the same way chat calls do:
|
|
47
|
+
* proxy only when the UI host and LiveKit host disagree (one local, one not).
|
|
48
|
+
* Local UI + local LiveKit (`ws://127.0.0.1:7880`) connects directly.
|
|
49
|
+
*/
|
|
50
|
+
function axeResolveVoiceLiveKitServerUrl(serverUrl) {
|
|
51
|
+
const trimmed = serverUrl?.trim();
|
|
52
|
+
if (!trimmed || typeof window === 'undefined') {
|
|
53
|
+
return trimmed ?? '';
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
const resolved = new URL(trimmed);
|
|
57
|
+
const localUi = axeIsLocalDevHostname(window.location.hostname);
|
|
58
|
+
const localLiveKit = axeIsLocalDevHostname(resolved.hostname);
|
|
59
|
+
const useDevProxy = (localUi && !localLiveKit) || (!localUi && localLiveKit);
|
|
60
|
+
if (!useDevProxy) {
|
|
61
|
+
return trimmed;
|
|
62
|
+
}
|
|
63
|
+
const proxy = new URL(window.location.origin);
|
|
64
|
+
proxy.pathname = DEFAULT_DEV_LIVEKIT_WS_PROXY_PATH;
|
|
65
|
+
return proxy.toString().replace(/\/$/, '');
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return trimmed;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/** Preferred URL first, then the token URL and same-origin `/livekit-ws` as fallbacks. */
|
|
72
|
+
function axeVoiceLiveKitConnectUrls(serverUrl) {
|
|
73
|
+
const trimmed = serverUrl.trim();
|
|
74
|
+
const urls = [];
|
|
75
|
+
const add = (url) => {
|
|
76
|
+
const next = url.trim();
|
|
77
|
+
if (next && !urls.includes(next)) {
|
|
78
|
+
urls.push(next);
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
add(axeResolveVoiceLiveKitServerUrl(trimmed));
|
|
82
|
+
add(trimmed);
|
|
83
|
+
if (typeof window !== 'undefined' && axeIsLocalDevHostname(window.location.hostname)) {
|
|
84
|
+
add(`${window.location.origin.replace(/^http/i, 'ws')}/livekit-ws`);
|
|
85
|
+
}
|
|
86
|
+
return urls.length ? urls : [trimmed];
|
|
87
|
+
}
|
|
88
|
+
function axeVoiceLiveRoomConnectOptions(serverUrl, iceServers, forceRelay) {
|
|
89
|
+
const options = {
|
|
90
|
+
peerConnectionTimeout: AXE_VOICE_LIVE_PEER_CONNECTION_TIMEOUT_MS,
|
|
91
|
+
};
|
|
92
|
+
if (forceRelay) {
|
|
93
|
+
options.rtcConfig = { iceTransportPolicy: 'relay' };
|
|
94
|
+
return options;
|
|
95
|
+
}
|
|
96
|
+
if (iceServers?.length) {
|
|
97
|
+
options.rtcConfig = { iceServers };
|
|
98
|
+
return options;
|
|
99
|
+
}
|
|
100
|
+
if (isLocalLiveKitUrl(serverUrl)) {
|
|
101
|
+
options.rtcConfig = { iceTransportPolicy: 'all' };
|
|
102
|
+
}
|
|
103
|
+
return options;
|
|
104
|
+
}
|
|
105
|
+
function isLocalLiveKitUrl(serverUrl) {
|
|
106
|
+
if (!serverUrl?.trim()) {
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
try {
|
|
110
|
+
const url = new URL(serverUrl.trim());
|
|
111
|
+
return axeIsLocalDevHostname(url.hostname);
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
//#endregion
|
|
118
|
+
|
|
119
|
+
//#region ----- Constants -----
|
|
120
|
+
const VAD = {
|
|
121
|
+
rmsThreshold: 500,
|
|
122
|
+
hangInMs: 80,
|
|
123
|
+
bargeInHangInMs: 280,
|
|
124
|
+
hangInGraceMs: 160,
|
|
125
|
+
silenceHangoverMs: 700,
|
|
126
|
+
minUtteranceMs: 280,
|
|
127
|
+
minUtteranceRms: 180,
|
|
128
|
+
maxUtteranceMs: 8_000,
|
|
129
|
+
prerollMs: 200,
|
|
130
|
+
};
|
|
131
|
+
//#endregion
|
|
132
|
+
//#region ----- PCM helpers -----
|
|
133
|
+
function float32ToPcm16(input) {
|
|
134
|
+
const out = new Int16Array(input.length);
|
|
135
|
+
for (let i = 0; i < input.length; i++) {
|
|
136
|
+
const clamped = Math.max(-1, Math.min(1, input[i] ?? 0));
|
|
137
|
+
out[i] = clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
|
|
138
|
+
}
|
|
139
|
+
return out;
|
|
140
|
+
}
|
|
141
|
+
function pcm16Rms(pcm) {
|
|
142
|
+
if (pcm.length <= 0) {
|
|
143
|
+
return 0;
|
|
144
|
+
}
|
|
145
|
+
let sumSquares = 0;
|
|
146
|
+
for (let i = 0; i < pcm.length; i++) {
|
|
147
|
+
const sample = pcm[i] ?? 0;
|
|
148
|
+
sumSquares += sample * sample;
|
|
149
|
+
}
|
|
150
|
+
return Math.sqrt(sumSquares / pcm.length);
|
|
151
|
+
}
|
|
152
|
+
function pcm16ToBase64(pcm) {
|
|
153
|
+
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
|
154
|
+
const chunk = 0x2000;
|
|
155
|
+
let binary = '';
|
|
156
|
+
for (let i = 0; i < bytes.length; i += chunk) {
|
|
157
|
+
const slice = bytes.subarray(i, Math.min(i + chunk, bytes.length));
|
|
158
|
+
binary += String.fromCharCode(...slice);
|
|
159
|
+
}
|
|
160
|
+
return btoa(binary);
|
|
161
|
+
}
|
|
162
|
+
function base64ToPcm16(raw) {
|
|
163
|
+
const trimmed = raw.trim();
|
|
164
|
+
if (!trimmed) {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
try {
|
|
168
|
+
const binary = atob(trimmed);
|
|
169
|
+
const bytes = new Uint8Array(binary.length);
|
|
170
|
+
for (let i = 0; i < binary.length; i++) {
|
|
171
|
+
bytes[i] = binary.charCodeAt(i);
|
|
172
|
+
}
|
|
173
|
+
const even = bytes.byteLength - (bytes.byteLength % 2);
|
|
174
|
+
if (even < 2) {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
return new Int16Array(bytes.buffer, bytes.byteOffset, even / 2);
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
//#endregion
|
|
184
|
+
//#region ----- Energy VAD -----
|
|
185
|
+
class AXPVoiceOpenRouterEnergyVad {
|
|
186
|
+
constructor() {
|
|
187
|
+
this.preroll = new Int16Array(0);
|
|
188
|
+
this.utterance = new Int16Array(0);
|
|
189
|
+
this.speaking = false;
|
|
190
|
+
this.hangInMs = 0;
|
|
191
|
+
this.hangInSilenceMs = 0;
|
|
192
|
+
this.silenceMs = 0;
|
|
193
|
+
this.utteranceMs = 0;
|
|
194
|
+
this.sampleRate = 48_000;
|
|
195
|
+
}
|
|
196
|
+
push(pcm16, sampleRate, bargeIn = false) {
|
|
197
|
+
if (pcm16.length < 2 || sampleRate <= 0) {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
this.sampleRate = sampleRate;
|
|
201
|
+
const frameMs = (pcm16.length / sampleRate) * 1000;
|
|
202
|
+
const voiced = pcm16Rms(pcm16) >= VAD.rmsThreshold;
|
|
203
|
+
const hangInNeedMs = bargeIn ? VAD.bargeInHangInMs : VAD.hangInMs;
|
|
204
|
+
if (!this.speaking) {
|
|
205
|
+
this.appendPreroll(pcm16, sampleRate);
|
|
206
|
+
if (!voiced) {
|
|
207
|
+
if (this.hangInMs <= 0) {
|
|
208
|
+
return null;
|
|
209
|
+
}
|
|
210
|
+
this.hangInSilenceMs += frameMs;
|
|
211
|
+
this.utterance = concatPcm(this.utterance, pcm16);
|
|
212
|
+
this.utteranceMs += frameMs;
|
|
213
|
+
if (this.hangInSilenceMs >= VAD.hangInGraceMs) {
|
|
214
|
+
this.hangInMs = 0;
|
|
215
|
+
this.hangInSilenceMs = 0;
|
|
216
|
+
this.utterance = new Int16Array(0);
|
|
217
|
+
this.utteranceMs = 0;
|
|
218
|
+
}
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
this.hangInSilenceMs = 0;
|
|
222
|
+
if (this.hangInMs <= 0) {
|
|
223
|
+
this.utterance = concatPcm(this.preroll, pcm16);
|
|
224
|
+
this.preroll = new Int16Array(0);
|
|
225
|
+
this.utteranceMs = frameMs;
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
this.utterance = concatPcm(this.utterance, pcm16);
|
|
229
|
+
this.utteranceMs += frameMs;
|
|
230
|
+
}
|
|
231
|
+
this.hangInMs += frameMs;
|
|
232
|
+
if (this.hangInMs < hangInNeedMs) {
|
|
233
|
+
return null;
|
|
234
|
+
}
|
|
235
|
+
this.speaking = true;
|
|
236
|
+
this.silenceMs = 0;
|
|
237
|
+
return { event: 'start' };
|
|
238
|
+
}
|
|
239
|
+
this.utterance = concatPcm(this.utterance, pcm16);
|
|
240
|
+
this.utteranceMs += frameMs;
|
|
241
|
+
this.silenceMs = voiced ? 0 : this.silenceMs + frameMs;
|
|
242
|
+
const ended = this.silenceMs >= VAD.silenceHangoverMs || this.utteranceMs >= VAD.maxUtteranceMs;
|
|
243
|
+
if (!ended) {
|
|
244
|
+
return null;
|
|
245
|
+
}
|
|
246
|
+
const captured = this.utterance;
|
|
247
|
+
const durationMs = this.utteranceMs;
|
|
248
|
+
const trailingSilenceMs = this.silenceMs;
|
|
249
|
+
const rate = this.sampleRate;
|
|
250
|
+
this.reset();
|
|
251
|
+
const hangoverSamples = Math.max(0, Math.floor((rate * trailingSilenceMs) / 1000));
|
|
252
|
+
const spoken = hangoverSamples > 0 && captured.length > hangoverSamples + 1
|
|
253
|
+
? captured.subarray(0, captured.length - hangoverSamples)
|
|
254
|
+
: captured;
|
|
255
|
+
const spokenMs = Math.max(0, durationMs - trailingSilenceMs);
|
|
256
|
+
if (spokenMs < VAD.minUtteranceMs ||
|
|
257
|
+
spoken.length < 2 ||
|
|
258
|
+
pcm16Rms(spoken) < VAD.minUtteranceRms) {
|
|
259
|
+
return { event: 'cancel' };
|
|
260
|
+
}
|
|
261
|
+
return { event: 'end', pcm: spoken, sampleRate: rate };
|
|
262
|
+
}
|
|
263
|
+
reset() {
|
|
264
|
+
this.preroll = new Int16Array(0);
|
|
265
|
+
this.utterance = new Int16Array(0);
|
|
266
|
+
this.speaking = false;
|
|
267
|
+
this.hangInMs = 0;
|
|
268
|
+
this.hangInSilenceMs = 0;
|
|
269
|
+
this.silenceMs = 0;
|
|
270
|
+
this.utteranceMs = 0;
|
|
271
|
+
}
|
|
272
|
+
appendPreroll(pcm16, sampleRate) {
|
|
273
|
+
const maxSamples = Math.max(1, Math.floor((sampleRate * VAD.prerollMs) / 1000));
|
|
274
|
+
this.preroll = concatPcm(this.preroll, pcm16);
|
|
275
|
+
if (this.preroll.length > maxSamples) {
|
|
276
|
+
this.preroll = this.preroll.subarray(this.preroll.length - maxSamples);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
function concatPcm(left, right) {
|
|
281
|
+
const out = new Int16Array(left.length + right.length);
|
|
282
|
+
out.set(left, 0);
|
|
283
|
+
out.set(right, left.length);
|
|
284
|
+
return out;
|
|
285
|
+
}
|
|
286
|
+
//#endregion
|
|
287
|
+
//#region ----- Mic capture -----
|
|
288
|
+
function startOpenRouterMicCapture(options) {
|
|
289
|
+
const source = options.context.createMediaStreamSource(options.stream);
|
|
290
|
+
const processor = options.context.createScriptProcessor(2048, 1, 1);
|
|
291
|
+
const silent = options.context.createGain();
|
|
292
|
+
silent.gain.value = 0;
|
|
293
|
+
const vad = new AXPVoiceOpenRouterEnergyVad();
|
|
294
|
+
processor.onaudioprocess = (event) => {
|
|
295
|
+
const input = event.inputBuffer.getChannelData(0);
|
|
296
|
+
const pcm = float32ToPcm16(input);
|
|
297
|
+
const result = vad.push(pcm, options.context.sampleRate, options.bargeIn?.() === true);
|
|
298
|
+
if (!result) {
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
if (result.event === 'start') {
|
|
302
|
+
options.onSpeechStart?.();
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
if (result.event === 'end') {
|
|
306
|
+
options.onUtterance(result.pcm, result.sampleRate);
|
|
307
|
+
}
|
|
308
|
+
};
|
|
309
|
+
source.connect(processor);
|
|
310
|
+
processor.connect(silent);
|
|
311
|
+
silent.connect(options.context.destination);
|
|
312
|
+
return {
|
|
313
|
+
stop: () => {
|
|
314
|
+
processor.onaudioprocess = null;
|
|
315
|
+
try {
|
|
316
|
+
processor.disconnect();
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
// ignore
|
|
320
|
+
}
|
|
321
|
+
try {
|
|
322
|
+
source.disconnect();
|
|
323
|
+
}
|
|
324
|
+
catch {
|
|
325
|
+
// ignore
|
|
326
|
+
}
|
|
327
|
+
try {
|
|
328
|
+
silent.disconnect();
|
|
329
|
+
}
|
|
330
|
+
catch {
|
|
331
|
+
// ignore
|
|
332
|
+
}
|
|
333
|
+
vad.reset();
|
|
334
|
+
},
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
//#endregion
|
|
338
|
+
|
|
339
|
+
//#region ----- Imports -----
|
|
340
|
+
//#endregion
|
|
341
|
+
//#region ----- Service -----
|
|
342
|
+
class AXPVoiceLiveMediaService {
|
|
343
|
+
constructor() {
|
|
344
|
+
this.rtcRoom = null;
|
|
345
|
+
this.livekit = null;
|
|
346
|
+
this.peer = null;
|
|
347
|
+
this.localStream = null;
|
|
348
|
+
this.playbackAudio = null;
|
|
349
|
+
this.audioContext = null;
|
|
350
|
+
this.dataChannel = null;
|
|
351
|
+
this.audioSink = null;
|
|
352
|
+
this.remoteAudioElements = new Map();
|
|
353
|
+
this.openRouterMic = null;
|
|
354
|
+
this.openRouterPlayAt = 0;
|
|
355
|
+
this.openRouterGeneration = 0;
|
|
356
|
+
this.openRouterSources = new Set();
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Must run in the same user-gesture turn as the mic click (before any await).
|
|
360
|
+
* Do not call getUserMedia here — a second Chrome profile will prompt twice and
|
|
361
|
+
* the extra stream can lock the device before LiveKit publishes the mic.
|
|
362
|
+
*/
|
|
363
|
+
unlockPlayback() {
|
|
364
|
+
this.ensureAudioContext();
|
|
365
|
+
this.playAttachedAudio();
|
|
366
|
+
void this.rtcRoom?.startAudio().catch(() => undefined);
|
|
367
|
+
}
|
|
368
|
+
async connectRtc(join, callbacks) {
|
|
369
|
+
await this.teardownRtc();
|
|
370
|
+
this.stopOpenRouterMic();
|
|
371
|
+
this.stopOpenRouterPlayback();
|
|
372
|
+
const urls = axeVoiceLiveKitConnectUrls(join.serverUrl);
|
|
373
|
+
let lastErr;
|
|
374
|
+
const livekit = await this.ensureLiveKit();
|
|
375
|
+
for (const url of urls) {
|
|
376
|
+
const room = this.createRtcRoom(livekit, callbacks);
|
|
377
|
+
this.rtcRoom = room;
|
|
378
|
+
try {
|
|
379
|
+
await room.connect(url, join.token, axeVoiceLiveRoomConnectOptions(url, join.iceServers, join.forceRelay));
|
|
380
|
+
this.attachExistingRemoteAudio(room);
|
|
381
|
+
await room.startAudio().catch((err) => {
|
|
382
|
+
console.warn('LiveKit startAudio failed', err);
|
|
383
|
+
});
|
|
384
|
+
try {
|
|
385
|
+
await room.localParticipant.setMicrophoneEnabled(true);
|
|
386
|
+
}
|
|
387
|
+
catch (err) {
|
|
388
|
+
throw new Error(voiceMediaErrorKey(err));
|
|
389
|
+
}
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
catch (err) {
|
|
393
|
+
lastErr = err;
|
|
394
|
+
console.error(`LiveKit connectRtc failed (${url})`, err);
|
|
395
|
+
await this.teardownRtc();
|
|
396
|
+
const key = err instanceof Error && err.message.startsWith('@') ? err.message : voiceMediaErrorKey(err);
|
|
397
|
+
if (key !== '@extensions-ai-voice-shell:errors.livekit-inactive') {
|
|
398
|
+
throw new Error(key);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
throw lastErr instanceof Error && lastErr.message.startsWith('@')
|
|
403
|
+
? lastErr
|
|
404
|
+
: new Error(voiceMediaErrorKey(lastErr));
|
|
405
|
+
}
|
|
406
|
+
async connectOpenRouter(callbacks) {
|
|
407
|
+
await this.teardownRtc();
|
|
408
|
+
this.stopOpenRouterMic();
|
|
409
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true }).catch((err) => {
|
|
410
|
+
throw new Error(voiceMediaErrorKey(err));
|
|
411
|
+
});
|
|
412
|
+
this.localStream = stream;
|
|
413
|
+
const context = this.ensureAudioContext();
|
|
414
|
+
if (!context) {
|
|
415
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
416
|
+
this.localStream = null;
|
|
417
|
+
throw new Error('@extensions-ai-voice-shell:errors.mic-unavailable');
|
|
418
|
+
}
|
|
419
|
+
await context.resume().catch(() => undefined);
|
|
420
|
+
this.openRouterMic = startOpenRouterMicCapture({
|
|
421
|
+
context,
|
|
422
|
+
stream,
|
|
423
|
+
onUtterance: callbacks.onUtterance,
|
|
424
|
+
onSpeechStart: callbacks.onSpeechStart,
|
|
425
|
+
bargeIn: callbacks.bargeIn,
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
playOpenRouterPcm(pcmBase64, sampleRate) {
|
|
429
|
+
const pcm = base64ToPcm16(pcmBase64);
|
|
430
|
+
const context = this.ensureAudioContext();
|
|
431
|
+
if (!pcm || !context || sampleRate <= 0) {
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
void context.resume().catch(() => undefined);
|
|
435
|
+
const buffer = context.createBuffer(1, pcm.length, sampleRate);
|
|
436
|
+
const channel = buffer.getChannelData(0);
|
|
437
|
+
for (let i = 0; i < pcm.length; i++) {
|
|
438
|
+
channel[i] = (pcm[i] ?? 0) / 32768;
|
|
439
|
+
}
|
|
440
|
+
const source = context.createBufferSource();
|
|
441
|
+
source.buffer = buffer;
|
|
442
|
+
source.connect(context.destination);
|
|
443
|
+
const generation = this.openRouterGeneration;
|
|
444
|
+
source.onended = () => {
|
|
445
|
+
this.openRouterSources.delete(source);
|
|
446
|
+
};
|
|
447
|
+
const startAt = Math.max(context.currentTime, this.openRouterPlayAt);
|
|
448
|
+
this.openRouterPlayAt = startAt + buffer.duration;
|
|
449
|
+
this.openRouterSources.add(source);
|
|
450
|
+
try {
|
|
451
|
+
source.start(startAt);
|
|
452
|
+
}
|
|
453
|
+
catch {
|
|
454
|
+
this.openRouterSources.delete(source);
|
|
455
|
+
}
|
|
456
|
+
if (generation !== this.openRouterGeneration) {
|
|
457
|
+
try {
|
|
458
|
+
source.stop();
|
|
459
|
+
}
|
|
460
|
+
catch {
|
|
461
|
+
// ignore
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
async connectVendor(join, callbacks) {
|
|
466
|
+
await this.teardownRtc();
|
|
467
|
+
this.stopOpenRouterMic();
|
|
468
|
+
this.stopOpenRouterPlayback();
|
|
469
|
+
if (join.transport !== 'openai') {
|
|
470
|
+
throw new Error('@extensions-ai-voice-shell:errors.livekit-inactive');
|
|
471
|
+
}
|
|
472
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true }).catch((err) => {
|
|
473
|
+
throw new Error(voiceMediaErrorKey(err));
|
|
474
|
+
});
|
|
475
|
+
this.localStream = stream;
|
|
476
|
+
const peer = new RTCPeerConnection();
|
|
477
|
+
this.peer = peer;
|
|
478
|
+
stream.getTracks().forEach((track) => peer.addTrack(track, stream));
|
|
479
|
+
peer.ontrack = (event) => {
|
|
480
|
+
const [remote] = event.streams;
|
|
481
|
+
if (remote) {
|
|
482
|
+
this.attachMediaStream(remote);
|
|
483
|
+
}
|
|
484
|
+
};
|
|
485
|
+
const channel = peer.createDataChannel('oai-events');
|
|
486
|
+
this.dataChannel = channel;
|
|
487
|
+
channel.addEventListener('message', (event) => {
|
|
488
|
+
this.handleVendorEvent(event.data, callbacks);
|
|
489
|
+
});
|
|
490
|
+
const offer = await peer.createOffer();
|
|
491
|
+
await peer.setLocalDescription(offer);
|
|
492
|
+
const sdpUrl = join.realtimeUrl.trim();
|
|
493
|
+
const response = await fetch(sdpUrl, {
|
|
494
|
+
method: 'POST',
|
|
495
|
+
body: offer.sdp ?? '',
|
|
496
|
+
headers: {
|
|
497
|
+
Authorization: `Bearer ${join.clientSecret}`,
|
|
498
|
+
'Content-Type': 'application/sdp',
|
|
499
|
+
},
|
|
500
|
+
});
|
|
501
|
+
if (!response.ok) {
|
|
502
|
+
throw new Error('@extensions-ai-voice-shell:errors.livekit-inactive');
|
|
503
|
+
}
|
|
504
|
+
const answer = await response.text();
|
|
505
|
+
await peer.setRemoteDescription({ type: 'answer', sdp: answer });
|
|
506
|
+
}
|
|
507
|
+
async interruptVendor() {
|
|
508
|
+
if (this.dataChannel?.readyState === 'open') {
|
|
509
|
+
this.dataChannel.send(JSON.stringify({ type: 'response.cancel' }));
|
|
510
|
+
}
|
|
511
|
+
this.stopPlayback();
|
|
512
|
+
}
|
|
513
|
+
sendVendorFunctionOutput(callId, output) {
|
|
514
|
+
if (this.dataChannel?.readyState !== 'open') {
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
this.dataChannel.send(JSON.stringify({
|
|
518
|
+
type: 'conversation.item.create',
|
|
519
|
+
item: {
|
|
520
|
+
type: 'function_call_output',
|
|
521
|
+
call_id: callId,
|
|
522
|
+
output,
|
|
523
|
+
},
|
|
524
|
+
}));
|
|
525
|
+
this.dataChannel.send(JSON.stringify({ type: 'response.create' }));
|
|
526
|
+
}
|
|
527
|
+
stopPlayback() {
|
|
528
|
+
this.stopOpenRouterPlayback();
|
|
529
|
+
if (this.playbackAudio) {
|
|
530
|
+
this.playbackAudio.pause();
|
|
531
|
+
this.playbackAudio.muted = true;
|
|
532
|
+
}
|
|
533
|
+
for (const { track, element } of this.remoteAudioElements.values()) {
|
|
534
|
+
element.muted = true;
|
|
535
|
+
if (element instanceof HTMLAudioElement) {
|
|
536
|
+
element.volume = 0;
|
|
537
|
+
}
|
|
538
|
+
track.setVolume(0);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
restorePlayback() {
|
|
542
|
+
void this.audioContext?.resume().catch(() => undefined);
|
|
543
|
+
if (this.playbackAudio) {
|
|
544
|
+
this.playbackAudio.muted = false;
|
|
545
|
+
this.playbackAudio.volume = 1;
|
|
546
|
+
void this.playbackAudio.play().catch(() => undefined);
|
|
547
|
+
}
|
|
548
|
+
for (const { track, element } of this.remoteAudioElements.values()) {
|
|
549
|
+
element.muted = false;
|
|
550
|
+
if (element instanceof HTMLAudioElement) {
|
|
551
|
+
element.volume = 1;
|
|
552
|
+
}
|
|
553
|
+
track.setVolume(1);
|
|
554
|
+
}
|
|
555
|
+
this.playAttachedAudio();
|
|
556
|
+
if (!this.rtcRoom) {
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
void this.rtcRoom.startAudio().catch(() => undefined);
|
|
560
|
+
this.attachExistingRemoteAudio(this.rtcRoom);
|
|
561
|
+
}
|
|
562
|
+
async disconnect() {
|
|
563
|
+
this.stopOpenRouterMic();
|
|
564
|
+
this.stopOpenRouterPlayback();
|
|
565
|
+
await this.teardownRtc();
|
|
566
|
+
if (this.playbackAudio) {
|
|
567
|
+
this.playbackAudio.pause();
|
|
568
|
+
this.playbackAudio.srcObject = null;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
async ensureLiveKit() {
|
|
572
|
+
this.livekit ??= await loadLiveKitClient();
|
|
573
|
+
return this.livekit;
|
|
574
|
+
}
|
|
575
|
+
stopOpenRouterMic() {
|
|
576
|
+
this.openRouterMic?.stop();
|
|
577
|
+
this.openRouterMic = null;
|
|
578
|
+
this.localStream?.getTracks().forEach((track) => track.stop());
|
|
579
|
+
this.localStream = null;
|
|
580
|
+
}
|
|
581
|
+
stopOpenRouterPlayback() {
|
|
582
|
+
this.openRouterGeneration += 1;
|
|
583
|
+
this.openRouterPlayAt = 0;
|
|
584
|
+
for (const source of this.openRouterSources) {
|
|
585
|
+
try {
|
|
586
|
+
source.stop();
|
|
587
|
+
}
|
|
588
|
+
catch {
|
|
589
|
+
// ignore
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
this.openRouterSources.clear();
|
|
593
|
+
}
|
|
594
|
+
createRtcRoom(livekit, callbacks) {
|
|
595
|
+
const { Room, RoomEvent } = livekit;
|
|
596
|
+
const room = new Room({
|
|
597
|
+
adaptiveStream: true,
|
|
598
|
+
dynacast: true,
|
|
599
|
+
audioCaptureDefaults: {
|
|
600
|
+
echoCancellation: true,
|
|
601
|
+
noiseSuppression: true,
|
|
602
|
+
autoGainControl: true,
|
|
603
|
+
},
|
|
604
|
+
});
|
|
605
|
+
room.on(RoomEvent.TrackSubscribed, (track) => {
|
|
606
|
+
if (track.kind === 'audio') {
|
|
607
|
+
this.attachRemoteTrack(track);
|
|
608
|
+
}
|
|
609
|
+
});
|
|
610
|
+
room.on(RoomEvent.TrackUnsubscribed, (track) => {
|
|
611
|
+
this.detachRemoteTrack(track);
|
|
612
|
+
});
|
|
613
|
+
room.on(RoomEvent.TrackPublished, (publication) => {
|
|
614
|
+
if (publication.kind === 'audio' && !publication.isSubscribed) {
|
|
615
|
+
void publication.setSubscribed(true);
|
|
616
|
+
}
|
|
617
|
+
});
|
|
618
|
+
room.on(RoomEvent.ParticipantConnected, () => {
|
|
619
|
+
this.attachExistingRemoteAudio(room);
|
|
620
|
+
});
|
|
621
|
+
room.on(RoomEvent.AudioPlaybackStatusChanged, () => {
|
|
622
|
+
if (!room.canPlaybackAudio) {
|
|
623
|
+
void room.startAudio().catch(() => undefined);
|
|
624
|
+
}
|
|
625
|
+
});
|
|
626
|
+
room.on(RoomEvent.ActiveSpeakersChanged, (speakers) => {
|
|
627
|
+
const localId = room.localParticipant.identity;
|
|
628
|
+
if (speakers.some((speaker) => speaker.identity === localId)) {
|
|
629
|
+
callbacks?.onLocalSpeechStart?.();
|
|
630
|
+
}
|
|
631
|
+
});
|
|
632
|
+
return room;
|
|
633
|
+
}
|
|
634
|
+
async teardownRtc() {
|
|
635
|
+
this.dataChannel = null;
|
|
636
|
+
if (this.peer) {
|
|
637
|
+
this.peer.close();
|
|
638
|
+
this.peer = null;
|
|
639
|
+
}
|
|
640
|
+
this.localStream?.getTracks().forEach((track) => track.stop());
|
|
641
|
+
this.localStream = null;
|
|
642
|
+
this.clearRemoteAudio();
|
|
643
|
+
if (this.rtcRoom) {
|
|
644
|
+
this.rtcRoom.removeAllListeners();
|
|
645
|
+
await this.rtcRoom.disconnect();
|
|
646
|
+
this.rtcRoom = null;
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
attachExistingRemoteAudio(room) {
|
|
650
|
+
room.remoteParticipants.forEach((participant) => {
|
|
651
|
+
participant.audioTrackPublications.forEach((publication) => {
|
|
652
|
+
const track = publication.track;
|
|
653
|
+
if (track && track.kind === 'audio') {
|
|
654
|
+
this.attachRemoteTrack(track);
|
|
655
|
+
}
|
|
656
|
+
});
|
|
657
|
+
});
|
|
658
|
+
}
|
|
659
|
+
attachRemoteTrack(track) {
|
|
660
|
+
const RemoteAudioTrack = this.livekit?.RemoteAudioTrack;
|
|
661
|
+
if (!RemoteAudioTrack || !(track instanceof RemoteAudioTrack)) {
|
|
662
|
+
return;
|
|
663
|
+
}
|
|
664
|
+
const key = track.sid ?? track.mediaStreamTrack?.id;
|
|
665
|
+
if (!key || this.remoteAudioElements.has(key)) {
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
668
|
+
const element = track.attach();
|
|
669
|
+
if (element instanceof HTMLAudioElement) {
|
|
670
|
+
element.autoplay = true;
|
|
671
|
+
element.muted = false;
|
|
672
|
+
element.volume = 1;
|
|
673
|
+
}
|
|
674
|
+
this.audioSinkElement().appendChild(element);
|
|
675
|
+
this.remoteAudioElements.set(key, { track, element });
|
|
676
|
+
void element.play().catch((err) => {
|
|
677
|
+
console.warn('LiveKit remote audio play() failed', err);
|
|
678
|
+
});
|
|
679
|
+
void this.rtcRoom?.startAudio().catch(() => undefined);
|
|
680
|
+
}
|
|
681
|
+
detachRemoteTrack(track) {
|
|
682
|
+
const RemoteAudioTrack = this.livekit?.RemoteAudioTrack;
|
|
683
|
+
const key = RemoteAudioTrack && track instanceof RemoteAudioTrack
|
|
684
|
+
? (track.sid ?? track.mediaStreamTrack?.id)
|
|
685
|
+
: track.mediaStreamTrack?.id;
|
|
686
|
+
if (!key) {
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
const attached = this.remoteAudioElements.get(key);
|
|
690
|
+
if (!attached) {
|
|
691
|
+
return;
|
|
692
|
+
}
|
|
693
|
+
attached.track.detach(attached.element);
|
|
694
|
+
attached.element.remove();
|
|
695
|
+
this.remoteAudioElements.delete(key);
|
|
696
|
+
}
|
|
697
|
+
clearRemoteAudio() {
|
|
698
|
+
for (const { track, element } of this.remoteAudioElements.values()) {
|
|
699
|
+
try {
|
|
700
|
+
track.detach(element);
|
|
701
|
+
}
|
|
702
|
+
catch {
|
|
703
|
+
// Detach is best-effort during teardown.
|
|
704
|
+
}
|
|
705
|
+
element.remove();
|
|
706
|
+
}
|
|
707
|
+
this.remoteAudioElements.clear();
|
|
708
|
+
this.audioSink?.remove();
|
|
709
|
+
this.audioSink = null;
|
|
710
|
+
}
|
|
711
|
+
audioSinkElement() {
|
|
712
|
+
if (!this.audioSink && typeof document !== 'undefined') {
|
|
713
|
+
const sink = document.createElement('div');
|
|
714
|
+
sink.setAttribute('data-axe-voice-live-audio-sink', 'true');
|
|
715
|
+
// display:none blocks playback in Chrome — keep the sink in layout but invisible.
|
|
716
|
+
sink.style.cssText =
|
|
717
|
+
'position:fixed;width:1px;height:1px;opacity:0.01;pointer-events:none;left:0;bottom:0;';
|
|
718
|
+
document.body.appendChild(sink);
|
|
719
|
+
this.audioSink = sink;
|
|
720
|
+
}
|
|
721
|
+
return this.audioSink ?? document.body;
|
|
722
|
+
}
|
|
723
|
+
playAttachedAudio() {
|
|
724
|
+
for (const { element } of this.remoteAudioElements.values()) {
|
|
725
|
+
element.muted = false;
|
|
726
|
+
if (element instanceof HTMLAudioElement) {
|
|
727
|
+
element.volume = 1;
|
|
728
|
+
}
|
|
729
|
+
void element.play().catch(() => undefined);
|
|
730
|
+
}
|
|
731
|
+
if (this.playbackAudio) {
|
|
732
|
+
this.playbackAudio.muted = false;
|
|
733
|
+
this.playbackAudio.volume = 1;
|
|
734
|
+
void this.playbackAudio.play().catch(() => undefined);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
attachMediaStream(stream) {
|
|
738
|
+
const audio = this.ensurePlaybackElement();
|
|
739
|
+
if (!audio) {
|
|
740
|
+
return;
|
|
741
|
+
}
|
|
742
|
+
audio.muted = false;
|
|
743
|
+
audio.volume = 1;
|
|
744
|
+
if (audio.srcObject !== stream) {
|
|
745
|
+
audio.srcObject = stream;
|
|
746
|
+
}
|
|
747
|
+
void audio.play().catch((err) => {
|
|
748
|
+
console.warn('LiveKit remote audio play() failed', err);
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
ensureAudioContext() {
|
|
752
|
+
if (typeof AudioContext === 'undefined') {
|
|
753
|
+
return null;
|
|
754
|
+
}
|
|
755
|
+
if (!this.audioContext || this.audioContext.state === 'closed') {
|
|
756
|
+
this.audioContext = new AudioContext();
|
|
757
|
+
}
|
|
758
|
+
void this.audioContext.resume().catch(() => undefined);
|
|
759
|
+
try {
|
|
760
|
+
const buffer = this.audioContext.createBuffer(1, 1, 22050);
|
|
761
|
+
const source = this.audioContext.createBufferSource();
|
|
762
|
+
source.buffer = buffer;
|
|
763
|
+
source.connect(this.audioContext.destination);
|
|
764
|
+
source.start(0);
|
|
765
|
+
}
|
|
766
|
+
catch {
|
|
767
|
+
// Silent tick is best-effort.
|
|
768
|
+
}
|
|
769
|
+
return this.audioContext;
|
|
770
|
+
}
|
|
771
|
+
ensurePlaybackElement() {
|
|
772
|
+
if (typeof Audio === 'undefined' || typeof document === 'undefined') {
|
|
773
|
+
return null;
|
|
774
|
+
}
|
|
775
|
+
const audio = this.playbackAudio ?? new Audio();
|
|
776
|
+
this.playbackAudio = audio;
|
|
777
|
+
audio.autoplay = true;
|
|
778
|
+
audio.muted = false;
|
|
779
|
+
audio.volume = 1;
|
|
780
|
+
audio.setAttribute('playsinline', 'true');
|
|
781
|
+
if (!audio.isConnected) {
|
|
782
|
+
this.audioSinkElement().appendChild(audio);
|
|
783
|
+
}
|
|
784
|
+
return audio;
|
|
785
|
+
}
|
|
786
|
+
handleVendorEvent(raw, callbacks) {
|
|
787
|
+
const text = typeof raw === 'string' ? raw : '';
|
|
788
|
+
if (!text) {
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
let event;
|
|
792
|
+
try {
|
|
793
|
+
event = JSON.parse(text);
|
|
794
|
+
}
|
|
795
|
+
catch {
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
const type = typeof event['type'] === 'string' ? event['type'] : '';
|
|
799
|
+
if (type === 'input_audio_buffer.speech_started') {
|
|
800
|
+
callbacks.onLocalSpeechStart?.();
|
|
801
|
+
return;
|
|
802
|
+
}
|
|
803
|
+
if (type === 'response.function_call_arguments.done') {
|
|
804
|
+
const callId = (typeof event['call_id'] === 'string' ? event['call_id'] : '') ||
|
|
805
|
+
(typeof event['id'] === 'string' ? event['id'] : '');
|
|
806
|
+
const name = typeof event['name'] === 'string' ? event['name'] : '';
|
|
807
|
+
const argumentsJson = typeof event['arguments'] === 'string' ? event['arguments'] : '{}';
|
|
808
|
+
if (callId && name) {
|
|
809
|
+
callbacks.onVendorFunctionCall?.({ callId, name, argumentsJson });
|
|
810
|
+
}
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
if (type === 'response.audio.delta' ||
|
|
814
|
+
type === 'response.output_audio.delta' ||
|
|
815
|
+
type === 'output_audio_buffer.started') {
|
|
816
|
+
this.restorePlayback();
|
|
817
|
+
callbacks.onSpeaking?.(true);
|
|
818
|
+
return;
|
|
819
|
+
}
|
|
820
|
+
if (type === 'response.audio.done' ||
|
|
821
|
+
type === 'response.output_audio.done' ||
|
|
822
|
+
type === 'response.done') {
|
|
823
|
+
callbacks.onSpeaking?.(false);
|
|
824
|
+
}
|
|
825
|
+
if (type === 'conversation.item.input_audio_transcription.completed') {
|
|
826
|
+
const transcript = typeof event['transcript'] === 'string' ? event['transcript'].trim() : '';
|
|
827
|
+
if (transcript) {
|
|
828
|
+
callbacks.onVendorTranscript?.('user', transcript, true);
|
|
829
|
+
}
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
if (type === 'response.audio_transcript.done' || type === 'response.output_audio_transcript.done') {
|
|
833
|
+
const transcript = typeof event['transcript'] === 'string' ? event['transcript'].trim() : '';
|
|
834
|
+
if (transcript) {
|
|
835
|
+
callbacks.onVendorTranscript?.('assistant', transcript, true);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXPVoiceLiveMediaService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); }
|
|
840
|
+
static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXPVoiceLiveMediaService, providedIn: 'root' }); }
|
|
841
|
+
}
|
|
842
|
+
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXPVoiceLiveMediaService, decorators: [{
|
|
843
|
+
type: Injectable,
|
|
844
|
+
args: [{ providedIn: 'root' }]
|
|
845
|
+
}] });
|
|
846
|
+
function voiceMediaErrorKey(err) {
|
|
847
|
+
const name = err instanceof DOMException ? err.name : '';
|
|
848
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
849
|
+
if (name === 'NotAllowedError' ||
|
|
850
|
+
name === 'PermissionDeniedError' ||
|
|
851
|
+
/permission|denied|notallowed/i.test(message)) {
|
|
852
|
+
return '@extensions-ai-voice-shell:errors.mic-denied';
|
|
853
|
+
}
|
|
854
|
+
if (name === 'NotFoundError' ||
|
|
855
|
+
name === 'NotReadableError' ||
|
|
856
|
+
name === 'OverconstrainedError' ||
|
|
857
|
+
name === 'AbortError' ||
|
|
858
|
+
/getUserMedia|Requested device not found|Could not start audio source/i.test(message)) {
|
|
859
|
+
return '@extensions-ai-voice-shell:errors.mic-unavailable';
|
|
860
|
+
}
|
|
861
|
+
return '@extensions-ai-voice-shell:errors.livekit-inactive';
|
|
862
|
+
}
|
|
863
|
+
|
|
25
864
|
//#region ----- Imports -----
|
|
26
865
|
//#endregion
|
|
27
866
|
//#region ----- Constants -----
|
|
@@ -40,6 +879,7 @@ class AXEAiVoiceShellHeaderSlotComponent {
|
|
|
40
879
|
this.commandExecutor = inject(AXPCommandExecutor);
|
|
41
880
|
this.shortcutRegistry = inject(AXPKeyboardShortcutRegistry);
|
|
42
881
|
this.destroyRef = inject(DestroyRef);
|
|
882
|
+
this.media = inject(AXPVoiceLiveMediaService);
|
|
43
883
|
this.themeStore = inject(AXPLayoutThemeService);
|
|
44
884
|
//#endregion
|
|
45
885
|
//#region ----- Computed Properties -----
|
|
@@ -55,6 +895,7 @@ class AXEAiVoiceShellHeaderSlotComponent {
|
|
|
55
895
|
title: '@general:keyboard-shortcuts.actions.open-voice-command',
|
|
56
896
|
allowInEditableFields: true,
|
|
57
897
|
handler: () => {
|
|
898
|
+
this.media.unlockPlayback();
|
|
58
899
|
void this.openVoiceCommand();
|
|
59
900
|
},
|
|
60
901
|
},
|
|
@@ -64,6 +905,7 @@ class AXEAiVoiceShellHeaderSlotComponent {
|
|
|
64
905
|
//#endregion
|
|
65
906
|
//#region ----- Event Handlers -----
|
|
66
907
|
async openVoiceCommand() {
|
|
908
|
+
this.media.unlockPlayback();
|
|
67
909
|
await this.commandExecutor.execute(AXE_AI_VOICE_SHELL_OPEN_COMMAND, {});
|
|
68
910
|
}
|
|
69
911
|
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXEAiVoiceShellHeaderSlotComponent, deps: [], target: i0.ɵɵFactoryTarget.Component }); }
|
|
@@ -100,12 +942,328 @@ i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "22.0.8", ngImpor
|
|
|
100
942
|
type: Injectable
|
|
101
943
|
}] });
|
|
102
944
|
|
|
945
|
+
//#region ----- Imports -----
|
|
946
|
+
//#endregion
|
|
947
|
+
//#region ----- Service -----
|
|
948
|
+
class AXPVoiceCommandShellService {
|
|
949
|
+
constructor() {
|
|
950
|
+
//#region ----- Services & Dependencies -----
|
|
951
|
+
this.popupService = inject(AXPopupService);
|
|
952
|
+
this.translation = inject(AXTranslationService);
|
|
953
|
+
this.commandService = inject(AXPCommandService);
|
|
954
|
+
this.media = inject(AXPVoiceLiveMediaService);
|
|
955
|
+
//#endregion
|
|
956
|
+
//#region ----- State Signals -----
|
|
957
|
+
this.uiState = signal('idle', /* @ts-ignore */
|
|
958
|
+
...(ngDevMode ? [{ debugName: "uiState" }] : /* istanbul ignore next */ []));
|
|
959
|
+
this.statusText = signal('', /* @ts-ignore */
|
|
960
|
+
...(ngDevMode ? [{ debugName: "statusText" }] : /* istanbul ignore next */ []));
|
|
961
|
+
this.lastTranscript = signal('', /* @ts-ignore */
|
|
962
|
+
...(ngDevMode ? [{ debugName: "lastTranscript" }] : /* istanbul ignore next */ []));
|
|
963
|
+
this.lastSpoken = signal('', /* @ts-ignore */
|
|
964
|
+
...(ngDevMode ? [{ debugName: "lastSpoken" }] : /* istanbul ignore next */ []));
|
|
965
|
+
this.errorMessage = signal('', /* @ts-ignore */
|
|
966
|
+
...(ngDevMode ? [{ debugName: "errorMessage" }] : /* istanbul ignore next */ []));
|
|
967
|
+
this.conversationId = signal('', /* @ts-ignore */
|
|
968
|
+
...(ngDevMode ? [{ debugName: "conversationId" }] : /* istanbul ignore next */ []));
|
|
969
|
+
this.sessionId = signal('', /* @ts-ignore */
|
|
970
|
+
...(ngDevMode ? [{ debugName: "sessionId" }] : /* istanbul ignore next */ []));
|
|
971
|
+
//#endregion
|
|
972
|
+
//#region ----- Session -----
|
|
973
|
+
this.openPromise = null;
|
|
974
|
+
this.mediaMode = null;
|
|
975
|
+
this.lastBargeInAt = 0;
|
|
976
|
+
this.speakingStartedAt = 0;
|
|
977
|
+
}
|
|
978
|
+
//#endregion
|
|
979
|
+
//#region ----- Public API -----
|
|
980
|
+
async open(_options) {
|
|
981
|
+
if (this.openPromise) {
|
|
982
|
+
return this.openPromise;
|
|
983
|
+
}
|
|
984
|
+
this.media.unlockPlayback();
|
|
985
|
+
this.resetUi();
|
|
986
|
+
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.connecting'));
|
|
987
|
+
this.uiState.set('thinking');
|
|
988
|
+
const title = await this.translation.translateAsync('@extensions-ai-voice-shell:dialog.title');
|
|
989
|
+
const { AXEAiVoiceVoidDialogComponent } = await Promise.resolve().then(function () { return voiceVoidDialog_component; });
|
|
990
|
+
const popupClosed = this.popupService
|
|
991
|
+
.open(AXEAiVoiceVoidDialogComponent, {
|
|
992
|
+
title,
|
|
993
|
+
size: 'sm',
|
|
994
|
+
header: true,
|
|
995
|
+
closeButton: true,
|
|
996
|
+
maximizable: false,
|
|
997
|
+
draggable: true,
|
|
998
|
+
hasBackdrop: true,
|
|
999
|
+
closeOnBackdropClick: true,
|
|
1000
|
+
closeOnEscape: true,
|
|
1001
|
+
panelClass: ['axe-ai-voice-shell-popup'],
|
|
1002
|
+
})
|
|
1003
|
+
.then(() => undefined);
|
|
1004
|
+
this.openPromise = popupClosed.finally(() => {
|
|
1005
|
+
this.openPromise = null;
|
|
1006
|
+
void this.hangUp(false);
|
|
1007
|
+
});
|
|
1008
|
+
void this.startLiveSession();
|
|
1009
|
+
return this.openPromise;
|
|
1010
|
+
}
|
|
1011
|
+
ensurePlayback() {
|
|
1012
|
+
this.media.unlockPlayback();
|
|
1013
|
+
this.media.restorePlayback();
|
|
1014
|
+
}
|
|
1015
|
+
async hangUp(_closePopup = true) {
|
|
1016
|
+
const sessionId = this.sessionId();
|
|
1017
|
+
this.mediaMode = null;
|
|
1018
|
+
await this.media.disconnect();
|
|
1019
|
+
if (sessionId) {
|
|
1020
|
+
try {
|
|
1021
|
+
await this.commandService.executeRemote(AXE_AI_VOICE_SHELL_VOICE_SESSION_STOP_COMMAND, { sessionId });
|
|
1022
|
+
}
|
|
1023
|
+
catch {
|
|
1024
|
+
// Hang up is best-effort.
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
this.resetUi();
|
|
1028
|
+
}
|
|
1029
|
+
async interrupt() {
|
|
1030
|
+
this.media.stopPlayback();
|
|
1031
|
+
const sessionId = this.sessionId();
|
|
1032
|
+
if (this.mediaMode === 'vendor') {
|
|
1033
|
+
await this.media.interruptVendor();
|
|
1034
|
+
}
|
|
1035
|
+
if (sessionId) {
|
|
1036
|
+
await this.commandService.executeRemote(AXE_AI_VOICE_SHELL_VOICE_SESSION_INTERRUPT_COMMAND, { sessionId });
|
|
1037
|
+
}
|
|
1038
|
+
this.uiState.set('listening');
|
|
1039
|
+
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.listening'));
|
|
1040
|
+
}
|
|
1041
|
+
applyRealtime(payload) {
|
|
1042
|
+
if (!payload?.sessionId || !this.sessionId() || payload.sessionId !== this.sessionId()) {
|
|
1043
|
+
return;
|
|
1044
|
+
}
|
|
1045
|
+
if (payload.conversationId) {
|
|
1046
|
+
this.conversationId.set(payload.conversationId);
|
|
1047
|
+
}
|
|
1048
|
+
if (payload.partialTranscript) {
|
|
1049
|
+
this.lastTranscript.set(payload.partialTranscript);
|
|
1050
|
+
}
|
|
1051
|
+
if (payload.finalTranscript) {
|
|
1052
|
+
this.lastTranscript.set(payload.finalTranscript);
|
|
1053
|
+
}
|
|
1054
|
+
if (payload.spokenDelta) {
|
|
1055
|
+
this.lastSpoken.set(payload.spokenDelta);
|
|
1056
|
+
}
|
|
1057
|
+
if (payload.state === 'ended') {
|
|
1058
|
+
void this.releaseLocalSession();
|
|
1059
|
+
return;
|
|
1060
|
+
}
|
|
1061
|
+
if (payload.errorKey) {
|
|
1062
|
+
void this.setError(payload.errorKey);
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
void this.applyState(payload.state);
|
|
1066
|
+
}
|
|
1067
|
+
applyOpenRouterAudio(payload) {
|
|
1068
|
+
if (!payload?.sessionId || !this.sessionId() || payload.sessionId !== this.sessionId()) {
|
|
1069
|
+
return;
|
|
1070
|
+
}
|
|
1071
|
+
if (!payload.pcmBase64 || !payload.sampleRate) {
|
|
1072
|
+
return;
|
|
1073
|
+
}
|
|
1074
|
+
this.media.playOpenRouterPcm(payload.pcmBase64, payload.sampleRate);
|
|
1075
|
+
}
|
|
1076
|
+
//#endregion
|
|
1077
|
+
//#region ----- Private -----
|
|
1078
|
+
async startLiveSession() {
|
|
1079
|
+
try {
|
|
1080
|
+
const result = await this.commandService.executeRemote(AXE_AI_VOICE_SHELL_VOICE_SESSION_START_COMMAND, {});
|
|
1081
|
+
const data = runtimeCommandPayloadData(result);
|
|
1082
|
+
if (!result?.success || !data?.sessionId) {
|
|
1083
|
+
const key = typeof result?.message?.text === 'string' && result.message.text.startsWith('@')
|
|
1084
|
+
? result.message.text
|
|
1085
|
+
: '@extensions-ai-voice-shell:errors.cascade-unavailable';
|
|
1086
|
+
await this.setError(key);
|
|
1087
|
+
return;
|
|
1088
|
+
}
|
|
1089
|
+
this.sessionId.set(data.sessionId);
|
|
1090
|
+
this.conversationId.set(data.conversationId);
|
|
1091
|
+
this.mediaMode = data.mode;
|
|
1092
|
+
this.errorMessage.set('');
|
|
1093
|
+
if (data.mode === 'openrouter') {
|
|
1094
|
+
await this.media.connectOpenRouter({
|
|
1095
|
+
onUtterance: (pcm, sampleRate) => {
|
|
1096
|
+
void this.pushOpenRouterUtterance(data.sessionId, pcm, sampleRate);
|
|
1097
|
+
},
|
|
1098
|
+
onSpeechStart: () => this.maybeBargeIn(),
|
|
1099
|
+
bargeIn: () => this.uiState() === 'speaking' || this.uiState() === 'thinking',
|
|
1100
|
+
});
|
|
1101
|
+
}
|
|
1102
|
+
else if (data.mode === 'rtc' && data.rtc) {
|
|
1103
|
+
await this.media.connectRtc(data.rtc);
|
|
1104
|
+
}
|
|
1105
|
+
else if (data.mode === 'vendor' && data.vendor) {
|
|
1106
|
+
await this.media.connectVendor(data.vendor, {
|
|
1107
|
+
onSpeaking: (speaking) => {
|
|
1108
|
+
this.uiState.set(speaking ? 'speaking' : 'listening');
|
|
1109
|
+
},
|
|
1110
|
+
onLocalSpeechStart: () => this.maybeBargeIn(),
|
|
1111
|
+
onVendorTranscript: (role, text, isFinal) => {
|
|
1112
|
+
if (role === 'user') {
|
|
1113
|
+
this.lastTranscript.set(text);
|
|
1114
|
+
}
|
|
1115
|
+
else {
|
|
1116
|
+
this.lastSpoken.set(text);
|
|
1117
|
+
}
|
|
1118
|
+
if (isFinal) {
|
|
1119
|
+
void this.commandService.executeRemote(AXE_AI_VOICE_SHELL_VOICE_SESSION_COMMIT_TURN_COMMAND, {
|
|
1120
|
+
sessionId: data.sessionId,
|
|
1121
|
+
role,
|
|
1122
|
+
text,
|
|
1123
|
+
});
|
|
1124
|
+
}
|
|
1125
|
+
},
|
|
1126
|
+
onVendorFunctionCall: (call) => {
|
|
1127
|
+
void this.runVendorTool(data.sessionId, call);
|
|
1128
|
+
},
|
|
1129
|
+
});
|
|
1130
|
+
}
|
|
1131
|
+
else {
|
|
1132
|
+
await this.setError('@extensions-ai-voice-shell:errors.livekit-inactive');
|
|
1133
|
+
return;
|
|
1134
|
+
}
|
|
1135
|
+
this.uiState.set('listening');
|
|
1136
|
+
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.listening'));
|
|
1137
|
+
}
|
|
1138
|
+
catch (err) {
|
|
1139
|
+
const key = err instanceof Error && err.message.startsWith('@')
|
|
1140
|
+
? err.message
|
|
1141
|
+
: '@extensions-ai-voice-shell:errors.cascade-unavailable';
|
|
1142
|
+
await this.hangUp(false);
|
|
1143
|
+
await this.setError(key);
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
maybeBargeIn() {
|
|
1147
|
+
if (this.mediaMode === 'rtc') {
|
|
1148
|
+
return;
|
|
1149
|
+
}
|
|
1150
|
+
const state = this.uiState();
|
|
1151
|
+
if (state !== 'speaking' && state !== 'thinking') {
|
|
1152
|
+
return;
|
|
1153
|
+
}
|
|
1154
|
+
if (state === 'speaking' && Date.now() - this.speakingStartedAt < 250) {
|
|
1155
|
+
return;
|
|
1156
|
+
}
|
|
1157
|
+
const now = Date.now();
|
|
1158
|
+
if (now - this.lastBargeInAt < 400) {
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
this.lastBargeInAt = now;
|
|
1162
|
+
this.media.stopPlayback();
|
|
1163
|
+
void this.interrupt();
|
|
1164
|
+
}
|
|
1165
|
+
async pushOpenRouterUtterance(sessionId, pcm, sampleRate) {
|
|
1166
|
+
if (!sessionId || pcm.length < 2) {
|
|
1167
|
+
return;
|
|
1168
|
+
}
|
|
1169
|
+
try {
|
|
1170
|
+
await this.commandService.executeRemote(AXE_AI_VOICE_SHELL_VOICE_SESSION_PUSH_AUDIO_COMMAND, {
|
|
1171
|
+
sessionId,
|
|
1172
|
+
pcmBase64: pcm16ToBase64(pcm),
|
|
1173
|
+
sampleRate,
|
|
1174
|
+
});
|
|
1175
|
+
}
|
|
1176
|
+
catch (err) {
|
|
1177
|
+
const key = err instanceof Error && err.message.startsWith('@')
|
|
1178
|
+
? err.message
|
|
1179
|
+
: '@extensions-ai-voice-shell:errors.transcribe-failed';
|
|
1180
|
+
await this.setError(key);
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
async runVendorTool(sessionId, call) {
|
|
1184
|
+
try {
|
|
1185
|
+
const result = await this.commandService.executeRemote(AXE_AI_VOICE_SHELL_VOICE_SESSION_EXECUTE_TOOL_COMMAND, {
|
|
1186
|
+
sessionId,
|
|
1187
|
+
name: call.name,
|
|
1188
|
+
argumentsJson: call.argumentsJson,
|
|
1189
|
+
});
|
|
1190
|
+
const data = runtimeCommandPayloadData(result);
|
|
1191
|
+
this.media.sendVendorFunctionOutput(call.callId, data?.output ?? '{"ok":false}');
|
|
1192
|
+
}
|
|
1193
|
+
catch {
|
|
1194
|
+
this.media.sendVendorFunctionOutput(call.callId, '{"ok":false}');
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
async applyState(state) {
|
|
1198
|
+
const map = {
|
|
1199
|
+
connecting: 'thinking',
|
|
1200
|
+
listening: 'listening',
|
|
1201
|
+
thinking: 'thinking',
|
|
1202
|
+
speaking: 'speaking',
|
|
1203
|
+
interrupted: 'listening',
|
|
1204
|
+
ended: 'idle',
|
|
1205
|
+
error: 'error',
|
|
1206
|
+
};
|
|
1207
|
+
this.uiState.set(map[state] ?? 'listening');
|
|
1208
|
+
if (state !== 'error' && state !== 'ended') {
|
|
1209
|
+
this.errorMessage.set('');
|
|
1210
|
+
}
|
|
1211
|
+
if (state === 'interrupted') {
|
|
1212
|
+
this.media.stopPlayback();
|
|
1213
|
+
}
|
|
1214
|
+
if (state === 'speaking') {
|
|
1215
|
+
this.speakingStartedAt = Date.now();
|
|
1216
|
+
this.media.restorePlayback();
|
|
1217
|
+
}
|
|
1218
|
+
const key = state === 'connecting'
|
|
1219
|
+
? '@extensions-ai-voice-shell:states.connecting'
|
|
1220
|
+
: state === 'listening'
|
|
1221
|
+
? '@extensions-ai-voice-shell:states.listening'
|
|
1222
|
+
: state === 'thinking'
|
|
1223
|
+
? '@extensions-ai-voice-shell:states.thinking'
|
|
1224
|
+
: state === 'speaking'
|
|
1225
|
+
? '@extensions-ai-voice-shell:states.speaking'
|
|
1226
|
+
: state === 'interrupted'
|
|
1227
|
+
? '@extensions-ai-voice-shell:states.interrupted'
|
|
1228
|
+
: state === 'error'
|
|
1229
|
+
? '@extensions-ai-voice-shell:states.error'
|
|
1230
|
+
: '@extensions-ai-voice-shell:states.listening';
|
|
1231
|
+
this.statusText.set(await this.translation.translateAsync(key));
|
|
1232
|
+
}
|
|
1233
|
+
async setError(key) {
|
|
1234
|
+
this.errorMessage.set(await this.translation.translateAsync(key));
|
|
1235
|
+
this.uiState.set('error');
|
|
1236
|
+
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.error'));
|
|
1237
|
+
}
|
|
1238
|
+
/** Another tab/profile started a session for this user, or the server stopped us. */
|
|
1239
|
+
async releaseLocalSession() {
|
|
1240
|
+
this.mediaMode = null;
|
|
1241
|
+
this.sessionId.set('');
|
|
1242
|
+
await this.media.disconnect();
|
|
1243
|
+
this.resetUi();
|
|
1244
|
+
}
|
|
1245
|
+
resetUi() {
|
|
1246
|
+
this.sessionId.set('');
|
|
1247
|
+
this.conversationId.set('');
|
|
1248
|
+
this.uiState.set('idle');
|
|
1249
|
+
this.statusText.set('');
|
|
1250
|
+
this.lastTranscript.set('');
|
|
1251
|
+
this.lastSpoken.set('');
|
|
1252
|
+
this.errorMessage.set('');
|
|
1253
|
+
}
|
|
1254
|
+
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXPVoiceCommandShellService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); }
|
|
1255
|
+
static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXPVoiceCommandShellService, providedIn: 'root' }); }
|
|
1256
|
+
}
|
|
1257
|
+
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXPVoiceCommandShellService, decorators: [{
|
|
1258
|
+
type: Injectable,
|
|
1259
|
+
args: [{ providedIn: 'root' }]
|
|
1260
|
+
}] });
|
|
1261
|
+
|
|
103
1262
|
//#region ----- Imports -----
|
|
104
1263
|
//#endregion
|
|
105
1264
|
//#region ----- Module -----
|
|
106
1265
|
/**
|
|
107
|
-
* Central voice AI
|
|
108
|
-
* Register next to {@link AXMAiManagementModule}. Domain modules register {@link AXP_VOICE_CAPABILITY_PROVIDER}.
|
|
1266
|
+
* Central live voice AI shell (void dialog + Conversation room + OpenRouter cascade).
|
|
109
1267
|
*/
|
|
110
1268
|
class AXEAiVoiceShellExtensionModule {
|
|
111
1269
|
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXEAiVoiceShellExtensionModule, deps: [], target: i0.ɵɵFactoryTarget.NgModule }); }
|
|
@@ -122,6 +1280,28 @@ class AXEAiVoiceShellExtensionModule {
|
|
|
122
1280
|
command: () => Promise.resolve().then(function () { return openVoiceCommandShell_command; }).then((c) => c.AXEAiVoiceShellOpenCommand),
|
|
123
1281
|
},
|
|
124
1282
|
]),
|
|
1283
|
+
provideAppStartUpTask(() => {
|
|
1284
|
+
const dispatcher = inject(AXPRealtimeDispatcherService, { optional: true });
|
|
1285
|
+
const shell = inject(AXPVoiceCommandShellService);
|
|
1286
|
+
return {
|
|
1287
|
+
name: 'ai-voice-shell-realtime',
|
|
1288
|
+
priority: AXP_APP_STARTUP_PRIORITY.realtimeHandlers,
|
|
1289
|
+
run: async () => {
|
|
1290
|
+
dispatcher?.register({
|
|
1291
|
+
topic: AXP_REALTIME_TOPIC_AI_VOICE_SESSION,
|
|
1292
|
+
handle: (envelope) => {
|
|
1293
|
+
shell.applyRealtime(envelope.payload);
|
|
1294
|
+
},
|
|
1295
|
+
});
|
|
1296
|
+
dispatcher?.register({
|
|
1297
|
+
topic: AXE_AI_VOICE_OPENROUTER_AUDIO_TOPIC,
|
|
1298
|
+
handle: (envelope) => {
|
|
1299
|
+
shell.applyOpenRouterAudio(envelope.payload);
|
|
1300
|
+
},
|
|
1301
|
+
});
|
|
1302
|
+
},
|
|
1303
|
+
};
|
|
1304
|
+
}),
|
|
125
1305
|
], imports: [AXMAiManagementModule,
|
|
126
1306
|
AXPComponentSlotModule.forChild({
|
|
127
1307
|
'root-header-end': [
|
|
@@ -162,6 +1342,28 @@ i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "22.0.8", ngImpor
|
|
|
162
1342
|
command: () => Promise.resolve().then(function () { return openVoiceCommandShell_command; }).then((c) => c.AXEAiVoiceShellOpenCommand),
|
|
163
1343
|
},
|
|
164
1344
|
]),
|
|
1345
|
+
provideAppStartUpTask(() => {
|
|
1346
|
+
const dispatcher = inject(AXPRealtimeDispatcherService, { optional: true });
|
|
1347
|
+
const shell = inject(AXPVoiceCommandShellService);
|
|
1348
|
+
return {
|
|
1349
|
+
name: 'ai-voice-shell-realtime',
|
|
1350
|
+
priority: AXP_APP_STARTUP_PRIORITY.realtimeHandlers,
|
|
1351
|
+
run: async () => {
|
|
1352
|
+
dispatcher?.register({
|
|
1353
|
+
topic: AXP_REALTIME_TOPIC_AI_VOICE_SESSION,
|
|
1354
|
+
handle: (envelope) => {
|
|
1355
|
+
shell.applyRealtime(envelope.payload);
|
|
1356
|
+
},
|
|
1357
|
+
});
|
|
1358
|
+
dispatcher?.register({
|
|
1359
|
+
topic: AXE_AI_VOICE_OPENROUTER_AUDIO_TOPIC,
|
|
1360
|
+
handle: (envelope) => {
|
|
1361
|
+
shell.applyOpenRouterAudio(envelope.payload);
|
|
1362
|
+
},
|
|
1363
|
+
});
|
|
1364
|
+
},
|
|
1365
|
+
};
|
|
1366
|
+
}),
|
|
165
1367
|
],
|
|
166
1368
|
}]
|
|
167
1369
|
}] });
|
|
@@ -367,479 +1569,6 @@ i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "22.0.8", ngImpor
|
|
|
367
1569
|
args: [{ providedIn: 'root' }]
|
|
368
1570
|
}] });
|
|
369
1571
|
|
|
370
|
-
//#region ----- Browser speech -----
|
|
371
|
-
function canUseBrowserSpeechSynthesis() {
|
|
372
|
-
return typeof window !== 'undefined' && typeof window.speechSynthesis?.speak === 'function';
|
|
373
|
-
}
|
|
374
|
-
/** Unlock speech synthesis during a user gesture so later replies can be spoken. */
|
|
375
|
-
function primeBrowserSpeechSynthesis() {
|
|
376
|
-
if (!canUseBrowserSpeechSynthesis()) {
|
|
377
|
-
return;
|
|
378
|
-
}
|
|
379
|
-
try {
|
|
380
|
-
window.speechSynthesis.cancel();
|
|
381
|
-
const silent = new SpeechSynthesisUtterance(' ');
|
|
382
|
-
silent.volume = 0;
|
|
383
|
-
silent.rate = 10;
|
|
384
|
-
window.speechSynthesis.speak(silent);
|
|
385
|
-
window.speechSynthesis.cancel();
|
|
386
|
-
}
|
|
387
|
-
catch {
|
|
388
|
-
// Ignore browsers that reject the silent prime utterance.
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
function cancelBrowserSpeechSynthesis() {
|
|
392
|
-
if (!canUseBrowserSpeechSynthesis()) {
|
|
393
|
-
return;
|
|
394
|
-
}
|
|
395
|
-
window.speechSynthesis.cancel();
|
|
396
|
-
}
|
|
397
|
-
function speakWithBrowserSpeechSynthesis(text) {
|
|
398
|
-
const trimmed = text.trim();
|
|
399
|
-
if (!trimmed || !canUseBrowserSpeechSynthesis()) {
|
|
400
|
-
return Promise.resolve(false);
|
|
401
|
-
}
|
|
402
|
-
return new Promise((resolve) => {
|
|
403
|
-
try {
|
|
404
|
-
window.speechSynthesis.cancel();
|
|
405
|
-
const utterance = new SpeechSynthesisUtterance(trimmed);
|
|
406
|
-
const done = (ok) => {
|
|
407
|
-
utterance.onend = null;
|
|
408
|
-
utterance.onerror = null;
|
|
409
|
-
resolve(ok);
|
|
410
|
-
};
|
|
411
|
-
utterance.onend = () => done(true);
|
|
412
|
-
utterance.onerror = () => done(false);
|
|
413
|
-
window.speechSynthesis.speak(utterance);
|
|
414
|
-
}
|
|
415
|
-
catch {
|
|
416
|
-
resolve(false);
|
|
417
|
-
}
|
|
418
|
-
});
|
|
419
|
-
}
|
|
420
|
-
//#endregion
|
|
421
|
-
|
|
422
|
-
//#region ----- Imports -----
|
|
423
|
-
//#endregion
|
|
424
|
-
//#region ----- Constants -----
|
|
425
|
-
const INTERVIEW_INTENT_RE = /\b(fill|inspect|inspection|questionnaire|assessment|survey|checklist|فرم|پرسشنامه|بازرسی)\b/i;
|
|
426
|
-
//#endregion
|
|
427
|
-
//#region ----- Service -----
|
|
428
|
-
/**
|
|
429
|
-
* Session coordinator for the central voice void dialog.
|
|
430
|
-
* Does not import assessment internals — only the capability registry and transport.
|
|
431
|
-
*/
|
|
432
|
-
class AXPVoiceCommandShellService {
|
|
433
|
-
constructor() {
|
|
434
|
-
//#region ----- Services & Dependencies -----
|
|
435
|
-
this.popupService = inject(AXPopupService);
|
|
436
|
-
this.translation = inject(AXTranslationService);
|
|
437
|
-
this.registry = inject(AXPVoiceCapabilityRegistry);
|
|
438
|
-
this.transport = inject(AXPHttpSseVoiceTransport);
|
|
439
|
-
//#endregion
|
|
440
|
-
//#region ----- State Signals -----
|
|
441
|
-
this.uiState = signal('idle', /* @ts-ignore */
|
|
442
|
-
...(ngDevMode ? [{ debugName: "uiState" }] : /* istanbul ignore next */ []));
|
|
443
|
-
this.mode = signal('command', /* @ts-ignore */
|
|
444
|
-
...(ngDevMode ? [{ debugName: "mode" }] : /* istanbul ignore next */ []));
|
|
445
|
-
this.statusText = signal('', /* @ts-ignore */
|
|
446
|
-
...(ngDevMode ? [{ debugName: "statusText" }] : /* istanbul ignore next */ []));
|
|
447
|
-
this.lastTranscript = signal('', /* @ts-ignore */
|
|
448
|
-
...(ngDevMode ? [{ debugName: "lastTranscript" }] : /* istanbul ignore next */ []));
|
|
449
|
-
this.lastSpoken = signal('', /* @ts-ignore */
|
|
450
|
-
...(ngDevMode ? [{ debugName: "lastSpoken" }] : /* istanbul ignore next */ []));
|
|
451
|
-
this.errorMessage = signal('', /* @ts-ignore */
|
|
452
|
-
...(ngDevMode ? [{ debugName: "errorMessage" }] : /* istanbul ignore next */ []));
|
|
453
|
-
this.confirmRequest = signal(null, /* @ts-ignore */
|
|
454
|
-
...(ngDevMode ? [{ debugName: "confirmRequest" }] : /* istanbul ignore next */ []));
|
|
455
|
-
this.turns = signal([], /* @ts-ignore */
|
|
456
|
-
...(ngDevMode ? [{ debugName: "turns" }] : /* istanbul ignore next */ []));
|
|
457
|
-
this.interviewProgress = signal(null, /* @ts-ignore */
|
|
458
|
-
...(ngDevMode ? [{ debugName: "interviewProgress" }] : /* istanbul ignore next */ []));
|
|
459
|
-
//#endregion
|
|
460
|
-
//#region ----- Session -----
|
|
461
|
-
this.activeCapability = null;
|
|
462
|
-
this.activeSessionId = null;
|
|
463
|
-
this.pendingSpokenConfirm = null;
|
|
464
|
-
this.currentAudio = null;
|
|
465
|
-
this.currentObjectUrl = null;
|
|
466
|
-
this.openPromise = null;
|
|
467
|
-
}
|
|
468
|
-
//#endregion
|
|
469
|
-
//#region ----- Public API -----
|
|
470
|
-
async open(options) {
|
|
471
|
-
if (this.openPromise) {
|
|
472
|
-
return this.openPromise;
|
|
473
|
-
}
|
|
474
|
-
this.resetSession();
|
|
475
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.idle'));
|
|
476
|
-
const title = await this.translation.translateAsync('@extensions-ai-voice-shell:dialog.title');
|
|
477
|
-
const { AXEAiVoiceVoidDialogComponent } = await Promise.resolve().then(function () { return voiceVoidDialog_component; });
|
|
478
|
-
const popupClosed = this.popupService
|
|
479
|
-
.open(AXEAiVoiceVoidDialogComponent, {
|
|
480
|
-
title,
|
|
481
|
-
size: 'sm',
|
|
482
|
-
header: true,
|
|
483
|
-
closeButton: true,
|
|
484
|
-
maximizable: false,
|
|
485
|
-
draggable: true,
|
|
486
|
-
hasBackdrop: true,
|
|
487
|
-
closeOnBackdropClick: true,
|
|
488
|
-
closeOnEscape: true,
|
|
489
|
-
panelClass: ['axe-ai-voice-shell-popup'],
|
|
490
|
-
})
|
|
491
|
-
.then(() => undefined);
|
|
492
|
-
this.openPromise = popupClosed.finally(() => {
|
|
493
|
-
this.openPromise = null;
|
|
494
|
-
this.stopPlayback();
|
|
495
|
-
this.resetSession();
|
|
496
|
-
});
|
|
497
|
-
if (options?.capabilityId || options?.startInput) {
|
|
498
|
-
void this.beginCapabilitySession(options);
|
|
499
|
-
}
|
|
500
|
-
else if (options?.initialText?.trim()) {
|
|
501
|
-
void this.handleUserText(options.initialText.trim());
|
|
502
|
-
}
|
|
503
|
-
return this.openPromise;
|
|
504
|
-
}
|
|
505
|
-
async enterListening() {
|
|
506
|
-
this.stopPlayback();
|
|
507
|
-
this.uiState.set('listening');
|
|
508
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.listening'));
|
|
509
|
-
this.errorMessage.set('');
|
|
510
|
-
}
|
|
511
|
-
async cancelListening() {
|
|
512
|
-
this.uiState.set('idle');
|
|
513
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.idle'));
|
|
514
|
-
}
|
|
515
|
-
async reportError(key, detail) {
|
|
516
|
-
await this.setError(key, detail);
|
|
517
|
-
}
|
|
518
|
-
async submitRecording(audio) {
|
|
519
|
-
this.uiState.set('thinking');
|
|
520
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.thinking'));
|
|
521
|
-
try {
|
|
522
|
-
const { text } = await this.transport.transcribe(audio);
|
|
523
|
-
if (!text.trim()) {
|
|
524
|
-
await this.setError('@extensions-ai-voice-shell:errors.empty-transcript');
|
|
525
|
-
return;
|
|
526
|
-
}
|
|
527
|
-
await this.handleUserText(text.trim());
|
|
528
|
-
}
|
|
529
|
-
catch (err) {
|
|
530
|
-
await this.setError('@extensions-ai-voice-shell:errors.transcribe-failed', err instanceof Error ? err.message : undefined);
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
async handleUserText(text) {
|
|
534
|
-
const trimmed = text.trim();
|
|
535
|
-
if (!trimmed) {
|
|
536
|
-
return;
|
|
537
|
-
}
|
|
538
|
-
this.lastTranscript.set(trimmed);
|
|
539
|
-
this.appendTurn('user', trimmed);
|
|
540
|
-
this.uiState.set('thinking');
|
|
541
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.thinking'));
|
|
542
|
-
this.errorMessage.set('');
|
|
543
|
-
if (this.mode() === 'confirm' && this.pendingSpokenConfirm) {
|
|
544
|
-
await this.handleConfirmUtterance(trimmed);
|
|
545
|
-
return;
|
|
546
|
-
}
|
|
547
|
-
if (this.mode() === 'interview' && this.activeCapability && this.activeSessionId) {
|
|
548
|
-
await this.applyInterviewUtterance(trimmed);
|
|
549
|
-
return;
|
|
550
|
-
}
|
|
551
|
-
const intent = { text: trimmed };
|
|
552
|
-
const capability = this.registry.getById(AXE_ASSESSMENT_QUESTIONNAIRE_VOICE_CAPABILITY_ID) && INTERVIEW_INTENT_RE.test(trimmed)
|
|
553
|
-
? await this.registry.findForIntent(intent)
|
|
554
|
-
: await this.registry.findForIntent(intent);
|
|
555
|
-
if (capability) {
|
|
556
|
-
await this.beginCapabilitySession({ capabilityId: capability.id, startInput: { text: trimmed } });
|
|
557
|
-
return;
|
|
558
|
-
}
|
|
559
|
-
await this.runCommandAssist(trimmed);
|
|
560
|
-
}
|
|
561
|
-
async confirmYes() {
|
|
562
|
-
if (this.pendingSpokenConfirm?.kind === 'submit') {
|
|
563
|
-
await this.submitInterview();
|
|
564
|
-
return;
|
|
565
|
-
}
|
|
566
|
-
this.confirmRequest.set(null);
|
|
567
|
-
this.mode.set(this.activeSessionId ? 'interview' : 'command');
|
|
568
|
-
this.uiState.set('idle');
|
|
569
|
-
}
|
|
570
|
-
async confirmNo() {
|
|
571
|
-
this.pendingSpokenConfirm = null;
|
|
572
|
-
this.confirmRequest.set(null);
|
|
573
|
-
if (this.activeSessionId) {
|
|
574
|
-
this.mode.set('interview');
|
|
575
|
-
const stay = await this.translation.translateAsync('@extensions-ai-voice-shell:interview.submit-cancelled');
|
|
576
|
-
await this.speak(stay);
|
|
577
|
-
return;
|
|
578
|
-
}
|
|
579
|
-
this.mode.set('command');
|
|
580
|
-
this.uiState.set('idle');
|
|
581
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.idle'));
|
|
582
|
-
}
|
|
583
|
-
async saveInterviewDraft() {
|
|
584
|
-
if (!this.activeCapability?.saveDraft || !this.activeSessionId) {
|
|
585
|
-
return;
|
|
586
|
-
}
|
|
587
|
-
this.uiState.set('thinking');
|
|
588
|
-
try {
|
|
589
|
-
const result = await this.activeCapability.saveDraft(this.activeSessionId);
|
|
590
|
-
const spoken = result && typeof result === 'object' && 'spokenMessage' in result && result.spokenMessage
|
|
591
|
-
? result.spokenMessage
|
|
592
|
-
: await this.translation.translateAsync('@extensions-ai-voice-shell:interview.draft-saved');
|
|
593
|
-
await this.speak(spoken);
|
|
594
|
-
}
|
|
595
|
-
catch (err) {
|
|
596
|
-
await this.setError('@extensions-ai-voice-shell:errors.save-failed', err instanceof Error ? err.message : undefined);
|
|
597
|
-
}
|
|
598
|
-
}
|
|
599
|
-
async cancelInterview() {
|
|
600
|
-
if (this.activeCapability?.cancel && this.activeSessionId) {
|
|
601
|
-
await this.activeCapability.cancel(this.activeSessionId);
|
|
602
|
-
}
|
|
603
|
-
this.resetSession();
|
|
604
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.idle'));
|
|
605
|
-
}
|
|
606
|
-
stopPlayback() {
|
|
607
|
-
if (this.currentAudio) {
|
|
608
|
-
this.currentAudio.pause();
|
|
609
|
-
this.currentAudio = null;
|
|
610
|
-
}
|
|
611
|
-
if (this.currentObjectUrl) {
|
|
612
|
-
URL.revokeObjectURL(this.currentObjectUrl);
|
|
613
|
-
this.currentObjectUrl = null;
|
|
614
|
-
}
|
|
615
|
-
cancelBrowserSpeechSynthesis();
|
|
616
|
-
}
|
|
617
|
-
//#endregion
|
|
618
|
-
//#region ----- Interview / command -----
|
|
619
|
-
async beginCapabilitySession(options) {
|
|
620
|
-
const startInput = options.startInput;
|
|
621
|
-
const startText = startInput != null && typeof startInput === 'object' && 'text' in startInput
|
|
622
|
-
? String(startInput.text ?? '')
|
|
623
|
-
: '';
|
|
624
|
-
const capability = options.capabilityId
|
|
625
|
-
? this.registry.getById(options.capabilityId)
|
|
626
|
-
: await this.registry.findForIntent({
|
|
627
|
-
text: startText || undefined,
|
|
628
|
-
context: startInput != null && typeof startInput === 'object'
|
|
629
|
-
? startInput
|
|
630
|
-
: undefined,
|
|
631
|
-
});
|
|
632
|
-
if (!capability) {
|
|
633
|
-
await this.setError('@extensions-ai-voice-shell:errors.no-capability');
|
|
634
|
-
return;
|
|
635
|
-
}
|
|
636
|
-
this.uiState.set('thinking');
|
|
637
|
-
try {
|
|
638
|
-
const started = await capability.startSession(options.startInput ?? {});
|
|
639
|
-
this.activeCapability = capability;
|
|
640
|
-
this.activeSessionId = started.sessionId;
|
|
641
|
-
this.mode.set('interview');
|
|
642
|
-
if (started.spokenPrompt) {
|
|
643
|
-
await this.speak(started.spokenPrompt);
|
|
644
|
-
}
|
|
645
|
-
if (capability.nextPrompt) {
|
|
646
|
-
await this.speakNextPrompt();
|
|
647
|
-
}
|
|
648
|
-
}
|
|
649
|
-
catch (err) {
|
|
650
|
-
await this.setError('@extensions-ai-voice-shell:errors.start-failed', err instanceof Error ? err.message : undefined);
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
async speakNextPrompt() {
|
|
654
|
-
if (!this.activeCapability?.nextPrompt || !this.activeSessionId) {
|
|
655
|
-
return;
|
|
656
|
-
}
|
|
657
|
-
const next = await this.activeCapability.nextPrompt(this.activeSessionId);
|
|
658
|
-
await this.handleNextPrompt(next);
|
|
659
|
-
}
|
|
660
|
-
async handleNextPrompt(next) {
|
|
661
|
-
if ('done' in next && next.done) {
|
|
662
|
-
this.interviewProgress.set(null);
|
|
663
|
-
await this.enterSubmitConfirm(next.spokenPrompt);
|
|
664
|
-
return;
|
|
665
|
-
}
|
|
666
|
-
if ('spokenPrompt' in next) {
|
|
667
|
-
if ('progress' in next && next.progress) {
|
|
668
|
-
this.interviewProgress.set(next.progress);
|
|
669
|
-
}
|
|
670
|
-
await this.speak(next.spokenPrompt);
|
|
671
|
-
}
|
|
672
|
-
}
|
|
673
|
-
async applyInterviewUtterance(text) {
|
|
674
|
-
if (!this.activeCapability?.applyInput || !this.activeSessionId) {
|
|
675
|
-
return;
|
|
676
|
-
}
|
|
677
|
-
try {
|
|
678
|
-
const result = await this.activeCapability.applyInput(this.activeSessionId, {
|
|
679
|
-
text,
|
|
680
|
-
});
|
|
681
|
-
if (!result.ok) {
|
|
682
|
-
await this.speak(result.spokenMessage ??
|
|
683
|
-
(await this.translation.translateAsync('@extensions-ai-voice-shell:interview.apply-failed')));
|
|
684
|
-
return;
|
|
685
|
-
}
|
|
686
|
-
if (result.next) {
|
|
687
|
-
await this.handleNextPrompt(result.next);
|
|
688
|
-
return;
|
|
689
|
-
}
|
|
690
|
-
await this.speakNextPrompt();
|
|
691
|
-
}
|
|
692
|
-
catch (err) {
|
|
693
|
-
await this.setError('@extensions-ai-voice-shell:errors.apply-failed', err instanceof Error ? err.message : undefined);
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
async enterSubmitConfirm(spoken) {
|
|
697
|
-
this.mode.set('confirm');
|
|
698
|
-
this.pendingSpokenConfirm = { kind: 'submit' };
|
|
699
|
-
const prompt = spoken?.trim() ||
|
|
700
|
-
(await this.translation.translateAsync('@extensions-ai-voice-shell:interview.submit-confirm'));
|
|
701
|
-
this.confirmRequest.set({
|
|
702
|
-
prompt,
|
|
703
|
-
confirmLabelKey: '@extensions-ai-voice-shell:actions.submit',
|
|
704
|
-
cancelLabelKey: '@extensions-ai-voice-shell:actions.keep-editing',
|
|
705
|
-
});
|
|
706
|
-
await this.speak(prompt);
|
|
707
|
-
}
|
|
708
|
-
async handleConfirmUtterance(text) {
|
|
709
|
-
const normalized = text.trim().toLowerCase();
|
|
710
|
-
const yes = /^(yes|y|ok|okay|sure|submit|confirm|بله|آره|باشه|تایید)$/i.test(normalized);
|
|
711
|
-
const no = /^(no|n|cancel|stop|نه|خیر|لغو)$/i.test(normalized);
|
|
712
|
-
if (yes) {
|
|
713
|
-
await this.confirmYes();
|
|
714
|
-
return;
|
|
715
|
-
}
|
|
716
|
-
if (no) {
|
|
717
|
-
await this.confirmNo();
|
|
718
|
-
return;
|
|
719
|
-
}
|
|
720
|
-
await this.speak(await this.translation.translateAsync('@extensions-ai-voice-shell:interview.confirm-repeat'));
|
|
721
|
-
}
|
|
722
|
-
async submitInterview() {
|
|
723
|
-
if (!this.activeCapability?.submit || !this.activeSessionId) {
|
|
724
|
-
return;
|
|
725
|
-
}
|
|
726
|
-
this.uiState.set('thinking');
|
|
727
|
-
try {
|
|
728
|
-
const result = await this.activeCapability.submit(this.activeSessionId);
|
|
729
|
-
this.pendingSpokenConfirm = null;
|
|
730
|
-
this.confirmRequest.set(null);
|
|
731
|
-
if (!result.ok) {
|
|
732
|
-
const spoken = result.spokenMessage ||
|
|
733
|
-
(await this.translation.translateAsync('@extensions-ai-voice-shell:interview.submit-failed'));
|
|
734
|
-
await this.speak(spoken);
|
|
735
|
-
this.mode.set('interview');
|
|
736
|
-
return;
|
|
737
|
-
}
|
|
738
|
-
const spoken = result.spokenMessage ||
|
|
739
|
-
(await this.translation.translateAsync('@extensions-ai-voice-shell:interview.submitted'));
|
|
740
|
-
await this.speak(spoken);
|
|
741
|
-
this.resetSession();
|
|
742
|
-
}
|
|
743
|
-
catch (err) {
|
|
744
|
-
await this.setError('@extensions-ai-voice-shell:errors.submit-failed', err instanceof Error ? err.message : undefined);
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
|
-
async runCommandAssist(text) {
|
|
748
|
-
try {
|
|
749
|
-
const { text: reply } = await this.transport.runAssist(text, {
|
|
750
|
-
voiceShell: true,
|
|
751
|
-
mode: 'command',
|
|
752
|
-
});
|
|
753
|
-
const spoken = reply.trim() || (await this.translation.translateAsync('@extensions-ai-voice-shell:command.empty-reply'));
|
|
754
|
-
await this.speak(spoken);
|
|
755
|
-
}
|
|
756
|
-
catch (err) {
|
|
757
|
-
await this.setError('@extensions-ai-voice-shell:errors.assist-failed', err instanceof Error ? err.message : undefined);
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
async speak(text) {
|
|
761
|
-
const trimmed = text.trim();
|
|
762
|
-
if (!trimmed) {
|
|
763
|
-
this.uiState.set('idle');
|
|
764
|
-
return;
|
|
765
|
-
}
|
|
766
|
-
this.lastSpoken.set(trimmed);
|
|
767
|
-
this.appendTurn('assistant', trimmed);
|
|
768
|
-
this.uiState.set('speaking');
|
|
769
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.speaking'));
|
|
770
|
-
this.stopPlayback();
|
|
771
|
-
try {
|
|
772
|
-
const tts = await this.transport.synthesize(trimmed);
|
|
773
|
-
if (tts.objectUrl) {
|
|
774
|
-
this.currentObjectUrl = tts.objectUrl;
|
|
775
|
-
const played = await this.playObjectUrl(tts.objectUrl);
|
|
776
|
-
if (played) {
|
|
777
|
-
await this.finishSpeaking();
|
|
778
|
-
return;
|
|
779
|
-
}
|
|
780
|
-
}
|
|
781
|
-
}
|
|
782
|
-
catch {
|
|
783
|
-
// Fall through to browser speech when server TTS is unavailable.
|
|
784
|
-
}
|
|
785
|
-
await speakWithBrowserSpeechSynthesis(trimmed);
|
|
786
|
-
await this.finishSpeaking();
|
|
787
|
-
}
|
|
788
|
-
async finishSpeaking() {
|
|
789
|
-
if (this.mode() === 'confirm') {
|
|
790
|
-
this.uiState.set('confirm');
|
|
791
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.confirm'));
|
|
792
|
-
return;
|
|
793
|
-
}
|
|
794
|
-
this.uiState.set('idle');
|
|
795
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.idle'));
|
|
796
|
-
}
|
|
797
|
-
playObjectUrl(url) {
|
|
798
|
-
return new Promise((resolve) => {
|
|
799
|
-
const audio = new Audio(url);
|
|
800
|
-
this.currentAudio = audio;
|
|
801
|
-
const finish = (ok) => {
|
|
802
|
-
audio.removeEventListener('ended', onEnded);
|
|
803
|
-
audio.removeEventListener('error', onError);
|
|
804
|
-
resolve(ok);
|
|
805
|
-
};
|
|
806
|
-
const onEnded = () => finish(true);
|
|
807
|
-
const onError = () => finish(false);
|
|
808
|
-
audio.addEventListener('ended', onEnded);
|
|
809
|
-
audio.addEventListener('error', onError);
|
|
810
|
-
void audio.play().catch(() => finish(false));
|
|
811
|
-
});
|
|
812
|
-
}
|
|
813
|
-
async setError(key, detail) {
|
|
814
|
-
const messageKey = detail?.startsWith('@') ? detail : key;
|
|
815
|
-
this.errorMessage.set(await this.translation.translateAsync(messageKey));
|
|
816
|
-
this.uiState.set('error');
|
|
817
|
-
this.statusText.set(await this.translation.translateAsync('@extensions-ai-voice-shell:states.error'));
|
|
818
|
-
}
|
|
819
|
-
appendTurn(role, text) {
|
|
820
|
-
this.turns.update((current) => [...current, { role, text }]);
|
|
821
|
-
}
|
|
822
|
-
resetSession() {
|
|
823
|
-
this.activeCapability = null;
|
|
824
|
-
this.activeSessionId = null;
|
|
825
|
-
this.pendingSpokenConfirm = null;
|
|
826
|
-
this.confirmRequest.set(null);
|
|
827
|
-
this.interviewProgress.set(null);
|
|
828
|
-
this.mode.set('command');
|
|
829
|
-
this.uiState.set('idle');
|
|
830
|
-
this.statusText.set('');
|
|
831
|
-
this.lastTranscript.set('');
|
|
832
|
-
this.lastSpoken.set('');
|
|
833
|
-
this.errorMessage.set('');
|
|
834
|
-
}
|
|
835
|
-
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXPVoiceCommandShellService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); }
|
|
836
|
-
static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXPVoiceCommandShellService, providedIn: 'root' }); }
|
|
837
|
-
}
|
|
838
|
-
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXPVoiceCommandShellService, decorators: [{
|
|
839
|
-
type: Injectable,
|
|
840
|
-
args: [{ providedIn: 'root' }]
|
|
841
|
-
}] });
|
|
842
|
-
|
|
843
1572
|
//#endregion
|
|
844
1573
|
//#region ----- Command -----
|
|
845
1574
|
class AXEAiVoiceShellOpenCommand {
|
|
@@ -879,141 +1608,45 @@ var openVoiceCommandShell_command = /*#__PURE__*/Object.freeze({
|
|
|
879
1608
|
//#endregion
|
|
880
1609
|
//#region ----- Component -----
|
|
881
1610
|
class AXEAiVoiceVoidDialogComponent extends AXBasePageComponent {
|
|
882
|
-
//#endregion
|
|
883
|
-
//#region ----- Lifecycle -----
|
|
884
1611
|
constructor() {
|
|
885
|
-
super();
|
|
1612
|
+
super(...arguments);
|
|
886
1613
|
//#region ----- Services & Dependencies -----
|
|
887
1614
|
this.shell = inject(AXPVoiceCommandShellService);
|
|
888
|
-
this.platformId = inject(PLATFORM_ID);
|
|
889
|
-
this.destroyRef = inject(DestroyRef);
|
|
890
|
-
this.audioChunks = [];
|
|
891
1615
|
//#endregion
|
|
892
1616
|
//#region ----- Computed Properties -----
|
|
893
1617
|
this.orbClass = computed(() => `voice-void-orb voice-void-orb--${this.shell.uiState()}`, /* @ts-ignore */
|
|
894
1618
|
...(ngDevMode ? [{ debugName: "orbClass" }] : /* istanbul ignore next */ []));
|
|
895
1619
|
this.orbWrapClass = computed(() => `voice-void-orb-wrap voice-void-orb-wrap--${this.shell.uiState()}`, /* @ts-ignore */
|
|
896
1620
|
...(ngDevMode ? [{ debugName: "orbWrapClass" }] : /* istanbul ignore next */ []));
|
|
897
|
-
this.canListen = computed(() => {
|
|
898
|
-
const state = this.shell.uiState();
|
|
899
|
-
return state === 'idle' || state === 'error' || state === 'confirm';
|
|
900
|
-
}, /* @ts-ignore */
|
|
901
|
-
...(ngDevMode ? [{ debugName: "canListen" }] : /* istanbul ignore next */ []));
|
|
902
1621
|
this.isListening = computed(() => this.shell.uiState() === 'listening', /* @ts-ignore */
|
|
903
1622
|
...(ngDevMode ? [{ debugName: "isListening" }] : /* istanbul ignore next */ []));
|
|
904
1623
|
this.isSpeaking = computed(() => this.shell.uiState() === 'speaking', /* @ts-ignore */
|
|
905
1624
|
...(ngDevMode ? [{ debugName: "isSpeaking" }] : /* istanbul ignore next */ []));
|
|
906
|
-
this.
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
const progress = this.shell.interviewProgress();
|
|
910
|
-
if (!progress || progress.totalVisible <= 0) {
|
|
911
|
-
return '';
|
|
912
|
-
}
|
|
913
|
-
return `${progress.answered}/${progress.totalVisible}`;
|
|
1625
|
+
this.isBusy = computed(() => {
|
|
1626
|
+
const state = this.shell.uiState();
|
|
1627
|
+
return state === 'thinking' || state === 'listening' || state === 'speaking';
|
|
914
1628
|
}, /* @ts-ignore */
|
|
915
|
-
...(ngDevMode ? [{ debugName: "
|
|
916
|
-
this.destroyRef.onDestroy(() => {
|
|
917
|
-
this.stopTracks();
|
|
918
|
-
cancelBrowserSpeechSynthesis();
|
|
919
|
-
});
|
|
1629
|
+
...(ngDevMode ? [{ debugName: "isBusy" }] : /* istanbul ignore next */ []));
|
|
920
1630
|
}
|
|
921
1631
|
//#endregion
|
|
922
1632
|
//#region ----- Event Handlers -----
|
|
923
|
-
async
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
}
|
|
927
|
-
if (this.isListening()) {
|
|
928
|
-
await this.stopAndSubmit();
|
|
929
|
-
return;
|
|
930
|
-
}
|
|
931
|
-
if (!this.canListen()) {
|
|
932
|
-
return;
|
|
933
|
-
}
|
|
934
|
-
await this.startRecording();
|
|
935
|
-
}
|
|
936
|
-
async onCancelListen() {
|
|
937
|
-
this.stopTracks();
|
|
938
|
-
this.audioChunks = [];
|
|
939
|
-
await this.shell.cancelListening();
|
|
940
|
-
}
|
|
941
|
-
async onConfirmYes() {
|
|
942
|
-
await this.shell.confirmYes();
|
|
943
|
-
}
|
|
944
|
-
async onConfirmNo() {
|
|
945
|
-
await this.shell.confirmNo();
|
|
946
|
-
}
|
|
947
|
-
async onSaveDraft() {
|
|
948
|
-
await this.shell.saveInterviewDraft();
|
|
949
|
-
}
|
|
950
|
-
async onCancelInterview() {
|
|
951
|
-
await this.shell.cancelInterview();
|
|
952
|
-
}
|
|
953
|
-
//#endregion
|
|
954
|
-
//#region ----- Recording -----
|
|
955
|
-
async startRecording() {
|
|
956
|
-
if (!isPlatformBrowser(this.platformId) || !navigator.mediaDevices?.getUserMedia) {
|
|
957
|
-
await this.shell.reportError('@extensions-ai-voice-shell:errors.mic-unavailable');
|
|
958
|
-
return;
|
|
959
|
-
}
|
|
960
|
-
try {
|
|
961
|
-
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
962
|
-
this.activeStream = stream;
|
|
963
|
-
this.audioChunks = [];
|
|
964
|
-
const recorder = new MediaRecorder(stream);
|
|
965
|
-
this.mediaRecorder = recorder;
|
|
966
|
-
recorder.ondataavailable = (event) => {
|
|
967
|
-
if (event.data.size > 0) {
|
|
968
|
-
this.audioChunks.push(event.data);
|
|
969
|
-
}
|
|
970
|
-
};
|
|
971
|
-
recorder.start(250);
|
|
972
|
-
await this.shell.enterListening();
|
|
973
|
-
}
|
|
974
|
-
catch {
|
|
975
|
-
this.stopTracks();
|
|
976
|
-
await this.shell.reportError('@extensions-ai-voice-shell:errors.mic-denied');
|
|
977
|
-
}
|
|
1633
|
+
async onHangUp() {
|
|
1634
|
+
await this.shell.hangUp(false);
|
|
1635
|
+
this.close();
|
|
978
1636
|
}
|
|
979
|
-
async
|
|
980
|
-
|
|
981
|
-
if (
|
|
982
|
-
this.
|
|
983
|
-
await this.shell.cancelListening();
|
|
984
|
-
return;
|
|
985
|
-
}
|
|
986
|
-
const blob = await new Promise((resolve) => {
|
|
987
|
-
recorder.onstop = () => {
|
|
988
|
-
const mime = recorder.mimeType?.trim() || this.audioChunks[0]?.type?.trim() || 'audio/webm';
|
|
989
|
-
const audioBlob = new Blob(this.audioChunks, { type: mime });
|
|
990
|
-
resolve(audioBlob.size > 0 ? audioBlob : null);
|
|
991
|
-
};
|
|
992
|
-
recorder.requestData();
|
|
993
|
-
recorder.stop();
|
|
994
|
-
});
|
|
995
|
-
this.stopTracks();
|
|
996
|
-
if (!blob) {
|
|
997
|
-
await this.shell.cancelListening();
|
|
998
|
-
return;
|
|
1637
|
+
async onOrbClick() {
|
|
1638
|
+
this.shell.ensurePlayback();
|
|
1639
|
+
if (this.isSpeaking()) {
|
|
1640
|
+
await this.shell.interrupt();
|
|
999
1641
|
}
|
|
1000
|
-
await this.shell.submitRecording(blob);
|
|
1001
1642
|
}
|
|
1002
|
-
|
|
1003
|
-
if (
|
|
1004
|
-
this.mediaRecorder.stop();
|
|
1005
|
-
}
|
|
1006
|
-
this.mediaRecorder = undefined;
|
|
1007
|
-
this.activeStream?.getTracks().forEach((track) => track.stop());
|
|
1008
|
-
this.activeStream = undefined;
|
|
1009
|
-
}
|
|
1010
|
-
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXEAiVoiceVoidDialogComponent, deps: [], target: i0.ɵɵFactoryTarget.Component }); }
|
|
1011
|
-
static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "22.0.8", type: AXEAiVoiceVoidDialogComponent, isStandalone: true, selector: "axe-ai-voice-void-dialog", usesInheritance: true, ngImport: i0, template: "<div class=\"voice-void\">\n <div class=\"voice-void-stage\">\n <!-- ----- Listening / thinking / speaking orb ----- -->\n <div [class]=\"orbWrapClass()\">\n <span class=\"voice-void-ring voice-void-ring--1\" aria-hidden=\"true\"></span>\n <span class=\"voice-void-ring voice-void-ring--2\" aria-hidden=\"true\"></span>\n <span class=\"voice-void-ring voice-void-ring--3\" aria-hidden=\"true\"></span>\n <button\n type=\"button\"\n [class]=\"orbClass()\"\n (click)=\"onMicClick()\"\n [disabled]=\"!canListen() && !isListening()\"\n [attr.aria-label]=\"'@extensions-ai-voice-shell:actions.tap-to-talk' | translate | async\"\n >\n @if (isSpeaking()) {\n <!-- ----- AI is speaking ----- -->\n <i class=\"fa-regular fa-volume-high\"></i>\n }\n <!-- ----- Tap-to-talk / user speaking ----- -->\n @else {\n <i class=\"fa-regular fa-microphone\"></i>\n }\n </button>\n @if (isListening() || isSpeaking()) {\n <!-- ----- Waveform bars: user listen vs AI speak ----- -->\n <span class=\"voice-void-eq\" aria-hidden=\"true\">\n <span></span>\n <span></span>\n <span></span>\n <span></span>\n <span></span>\n </span>\n }\n </div>\n\n <p class=\"voice-void-status\">{{ shell.statusText() }}</p>\n\n @if (progressLabel()) {\n <!-- ----- Interview progress counter ----- -->\n <p class=\"voice-void-progress\">{{ progressLabel() }}</p>\n }\n\n @if (shell.lastSpoken()) {\n <!-- ----- Last spoken assistant text ----- -->\n <p class=\"voice-void-spoken\">{{ shell.lastSpoken() }}</p>\n }\n\n @if (shell.lastTranscript()) {\n <!-- ----- Last user transcript ----- -->\n <p class=\"voice-void-transcript\">{{ shell.lastTranscript() }}</p>\n }\n\n @if (shell.errorMessage()) {\n <!-- ----- Spoken / visible error ----- -->\n <p class=\"voice-void-error\">{{ shell.errorMessage() }}</p>\n }\n </div>\n\n @if (showConfirm()) {\n <!-- ----- Confirm chips for submit / destructive actions ----- -->\n <div class=\"voice-void-confirm\">\n <p class=\"voice-void-confirm-prompt\">{{ shell.confirmRequest()?.prompt }}</p>\n <div class=\"voice-void-confirm-actions\">\n <ax-button\n [text]=\"'@extensions-ai-voice-shell:actions.submit' | translate | async\"\n color=\"primary\"\n (onClick)=\"onConfirmYes()\"\n ></ax-button>\n <ax-button\n [text]=\"'@extensions-ai-voice-shell:actions.keep-editing' | translate | async\"\n [look]=\"'ghost'\"\n (onClick)=\"onConfirmNo()\"\n ></ax-button>\n </div>\n </div>\n }\n\n <!-- ----- Secondary interview / listen actions ----- -->\n <div class=\"voice-void-toolbar\">\n @if (isListening()) {\n <!-- ----- Stop listening without sending ----- -->\n <ax-button\n [text]=\"'@extensions-ai-voice-shell:actions.cancel-listen' | translate | async\"\n [look]=\"'ghost'\"\n (onClick)=\"onCancelListen()\"\n ></ax-button>\n }\n <!-- ----- Interview session actions ----- -->\n @if (shell.mode() === 'interview') {\n <!-- ----- Draft save and cancel while filling by voice ----- -->\n <ax-button\n [text]=\"'@extensions-ai-voice-shell:actions.save-draft' | translate | async\"\n [look]=\"'outline'\"\n (onClick)=\"onSaveDraft()\"\n ></ax-button>\n <ax-button\n [text]=\"'@extensions-ai-voice-shell:actions.cancel-interview' | translate | async\"\n [look]=\"'ghost'\"\n (onClick)=\"onCancelInterview()\"\n ></ax-button>\n }\n </div>\n\n <p class=\"voice-void-hint\">{{ '@extensions-ai-voice-shell:dialog.hint' | translate | async }}</p>\n</div>\n", styles: [":host,axe-ai-voice-void-dialog{display:block;height:auto}.axe-ai-voice-shell-popup .ax-popup-content,.axe-ai-voice-shell-popup .ax-popup-body{overflow:visible}.voice-void{display:flex;flex-direction:column;align-items:center;justify-content:center;min-height:16rem;padding:1.25rem 1rem .75rem;gap:1rem}.voice-void-stage{display:flex;flex-direction:column;align-items:center;gap:.75rem;max-width:22rem;text-align:center}.voice-void-orb-wrap{position:relative;display:flex;align-items:center;justify-content:center;width:9.5rem;height:9.5rem}.voice-void-ring{position:absolute;inset:1.1rem;border-radius:50%;pointer-events:none;opacity:0}.voice-void-orb{position:relative;z-index:1;display:inline-flex;align-items:center;justify-content:center;width:6.25rem;height:6.25rem;border:none;border-radius:50%;background:rgb(var(--ax-sys-color-primary-500, 59 130 246));color:rgb(var(--ax-sys-color-on-primary, 255 255 255));font-size:1.75rem;cursor:pointer;transition:transform .2s ease,box-shadow .2s ease,opacity .2s ease,background-color .2s ease}.voice-void-orb:disabled{cursor:default;opacity:.92}.voice-void-orb--idle{box-shadow:0 .5rem 1.25rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .28)}.voice-void-orb--listening{animation:voice-void-user-pulse 1.05s ease-in-out infinite}.voice-void-orb-wrap--listening .voice-void-ring{border:2px solid rgb(var(--ax-sys-color-primary-400, 96 165 250))}.voice-void-orb-wrap--listening .voice-void-ring--1{animation:voice-void-user-ripple 1.5s ease-out infinite}.voice-void-orb-wrap--listening .voice-void-ring--2{animation:voice-void-user-ripple 1.5s ease-out .28s infinite}.voice-void-orb-wrap--listening .voice-void-ring--3{animation:voice-void-user-ripple 1.5s ease-out .56s infinite}.voice-void-orb--thinking{opacity:.8;animation:voice-void-breathe 1.8s ease-in-out infinite}.voice-void-orb--speaking{background:rgb(var(--ax-sys-color-primary-700, 29 78 216));animation:voice-void-ai-glow 1.2s ease-in-out infinite}.voice-void-orb-wrap--speaking .voice-void-ring{border:2px solid rgb(var(--ax-sys-color-primary-300, 147 197 253) / .7)}.voice-void-orb-wrap--speaking .voice-void-ring--1{animation:voice-void-ai-orbit 2.4s linear infinite}.voice-void-orb-wrap--speaking .voice-void-ring--2{inset:.55rem;animation:voice-void-ai-orbit 3.2s linear infinite reverse}.voice-void-orb-wrap--speaking .voice-void-ring--3{display:none}.voice-void-orb--confirm,.voice-void-orb--error{background:rgb(var(--ax-sys-color-warning-500, 217 119 6))}.voice-void-eq{position:absolute;z-index:2;bottom:.85rem;display:none;align-items:flex-end;gap:.18rem;height:1.1rem;pointer-events:none}.voice-void-orb-wrap--listening .voice-void-eq,.voice-void-orb-wrap--speaking .voice-void-eq{display:flex}.voice-void-eq span{display:block;width:.18rem;border-radius:999px;background:rgb(var(--ax-sys-color-on-primary, 255 255 255));transform-origin:bottom center}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(1),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(1){animation:voice-void-eq-bar .7s ease-in-out infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(2),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(2){animation:voice-void-eq-bar .7s ease-in-out .1s infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(3),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(3){animation:voice-void-eq-bar .7s ease-in-out .18s infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(4),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(4){animation:voice-void-eq-bar .7s ease-in-out .08s infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(5),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(5){animation:voice-void-eq-bar .7s ease-in-out .22s infinite}.voice-void-orb-wrap--speaking .voice-void-eq span{animation-duration:.95s}.voice-void-status{margin:0;font-weight:600}.voice-void-progress,.voice-void-spoken,.voice-void-transcript,.voice-void-error,.voice-void-hint,.voice-void-confirm-prompt{margin:0}.voice-void-spoken,.voice-void-transcript{max-height:4.5rem;overflow:auto;font-size:.875rem}.voice-void-spoken,.voice-void-transcript,.voice-void-hint{color:rgb(var(--ax-sys-color-on-surface) / .8)}.voice-void-error{color:rgb(var(--ax-sys-color-danger, 185 28 28))}.voice-void-confirm,.voice-void-toolbar{display:flex;flex-wrap:wrap;gap:.75rem;justify-content:center;align-items:center}.voice-void-confirm{flex-direction:column}.voice-void-hint{font-size:.8125rem}@keyframes voice-void-user-pulse{0%,to{transform:scale(1);box-shadow:0 0 rgb(var(--ax-sys-color-primary-500, 59 130 246) / .4)}50%{transform:scale(1.06);box-shadow:0 0 0 .65rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .12)}}@keyframes voice-void-user-ripple{0%{transform:scale(.92);opacity:.55}to{transform:scale(1.55);opacity:0}}@keyframes voice-void-ai-glow{0%,to{transform:scale(1);box-shadow:0 0 0 .35rem rgb(var(--ax-sys-color-primary-300, 147 197 253) / .35),0 0 1.75rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .55)}50%{transform:scale(1.04);box-shadow:0 0 0 .7rem rgb(var(--ax-sys-color-primary-300, 147 197 253) / .18),0 0 2.25rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .75)}}@keyframes voice-void-ai-orbit{0%{transform:scale(1) rotate(0);opacity:.45;border-style:dashed}50%{opacity:.85}to{transform:scale(1.12) rotate(360deg);opacity:.45;border-style:dashed}}@keyframes voice-void-breathe{0%,to{transform:scale(1);opacity:.8}50%{transform:scale(1.06);opacity:1}}@keyframes voice-void-eq-bar{0%,to{height:.28rem}40%{height:.95rem}70%{height:.45rem}}\n"], dependencies: [{ kind: "ngmodule", type: CommonModule }, { kind: "ngmodule", type: AXButtonModule }, { kind: "component", type: i1.AXButtonComponent, selector: "ax-button", inputs: ["disabled", "tabIndex", "color", "look", "text", "toggleable", "selected", "iconOnly", "type", "loadingText"], outputs: ["onBlur", "onFocus", "onClick", "selectedChange", "toggleableChange", "lookChange", "colorChange", "disabledChange", "loadingTextChange"] }, { kind: "ngmodule", type: AXDecoratorModule }, { kind: "ngmodule", type: AXTranslationModule }, { kind: "pipe", type: i2$1.AsyncPipe, name: "async" }, { kind: "pipe", type: i3.AXTranslatorPipe, name: "translate" }], changeDetection: i0.ChangeDetectionStrategy.OnPush, encapsulation: i0.ViewEncapsulation.None }); }
|
|
1643
|
+
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXEAiVoiceVoidDialogComponent, deps: null, target: i0.ɵɵFactoryTarget.Component }); }
|
|
1644
|
+
static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "22.0.8", type: AXEAiVoiceVoidDialogComponent, isStandalone: true, selector: "axe-ai-voice-void-dialog", usesInheritance: true, ngImport: i0, template: "<div class=\"voice-void\">\n <div class=\"voice-void-stage\">\n <!-- ----- Listening / thinking / speaking orb ----- -->\n <div [class]=\"orbWrapClass()\">\n <span class=\"voice-void-ring voice-void-ring--1\" aria-hidden=\"true\"></span>\n <span class=\"voice-void-ring voice-void-ring--2\" aria-hidden=\"true\"></span>\n <span class=\"voice-void-ring voice-void-ring--3\" aria-hidden=\"true\"></span>\n <button\n type=\"button\"\n [class]=\"orbClass()\"\n (click)=\"onOrbClick()\"\n [attr.aria-label]=\"'@extensions-ai-voice-shell:actions.interrupt' | translate | async\"\n >\n @if (isSpeaking()) {\n <!-- ----- AI is speaking; click interrupts ----- -->\n <i class=\"fa-regular fa-volume-high\"></i>\n }\n <!-- ----- Live microphone ----- -->\n @else {\n <i class=\"fa-regular fa-microphone\"></i>\n }\n </button>\n @if (isBusy()) {\n <!-- ----- Waveform bars: user listen vs AI speak ----- -->\n <span class=\"voice-void-eq\" aria-hidden=\"true\">\n <span></span>\n <span></span>\n <span></span>\n <span></span>\n <span></span>\n </span>\n }\n </div>\n\n <p class=\"voice-void-status\">{{ shell.statusText() }}</p>\n\n @if (shell.lastSpoken()) {\n <!-- ----- Last spoken assistant text ----- -->\n <p class=\"voice-void-spoken\">{{ shell.lastSpoken() }}</p>\n }\n\n @if (shell.lastTranscript()) {\n <!-- ----- Last user transcript ----- -->\n <p class=\"voice-void-transcript\">{{ shell.lastTranscript() }}</p>\n }\n\n @if (shell.errorMessage()) {\n <!-- ----- Spoken / visible error ----- -->\n <p class=\"voice-void-error\">{{ shell.errorMessage() }}</p>\n }\n </div>\n\n <!-- ----- Hang up ends the live session ----- -->\n <div class=\"voice-void-toolbar\">\n <ax-button\n [text]=\"'@extensions-ai-voice-shell:actions.hang-up' | translate | async\"\n color=\"danger\"\n (onClick)=\"onHangUp()\"\n ></ax-button>\n </div>\n\n <p class=\"voice-void-hint\">{{ '@extensions-ai-voice-shell:dialog.hint' | translate | async }}</p>\n</div>\n", styles: [":host,axe-ai-voice-void-dialog{display:block;height:auto}.axe-ai-voice-shell-popup .ax-popup-content,.axe-ai-voice-shell-popup .ax-popup-body{overflow:visible}.voice-void{display:flex;flex-direction:column;align-items:center;justify-content:center;min-height:16rem;padding:1.25rem 1rem .75rem;gap:1rem}.voice-void-stage{display:flex;flex-direction:column;align-items:center;gap:.75rem;max-width:22rem;text-align:center}.voice-void-orb-wrap{position:relative;display:flex;align-items:center;justify-content:center;width:9.5rem;height:9.5rem}.voice-void-ring{position:absolute;inset:1.1rem;border-radius:50%;pointer-events:none;opacity:0}.voice-void-orb{position:relative;z-index:1;display:inline-flex;align-items:center;justify-content:center;width:6.25rem;height:6.25rem;border:none;border-radius:50%;background:rgb(var(--ax-sys-color-primary-500, 59 130 246));color:rgb(var(--ax-sys-color-on-primary, 255 255 255));font-size:1.75rem;cursor:pointer;transition:transform .2s ease,box-shadow .2s ease,opacity .2s ease,background-color .2s ease}.voice-void-orb:disabled{cursor:default;opacity:.92}.voice-void-orb--idle{box-shadow:0 .5rem 1.25rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .28)}.voice-void-orb--listening{animation:voice-void-user-pulse 1.05s ease-in-out infinite}.voice-void-orb-wrap--listening .voice-void-ring{border:2px solid rgb(var(--ax-sys-color-primary-400, 96 165 250))}.voice-void-orb-wrap--listening .voice-void-ring--1{animation:voice-void-user-ripple 1.5s ease-out infinite}.voice-void-orb-wrap--listening .voice-void-ring--2{animation:voice-void-user-ripple 1.5s ease-out .28s infinite}.voice-void-orb-wrap--listening .voice-void-ring--3{animation:voice-void-user-ripple 1.5s ease-out .56s infinite}.voice-void-orb--thinking{opacity:.8;animation:voice-void-breathe 1.8s ease-in-out infinite}.voice-void-orb--speaking{background:rgb(var(--ax-sys-color-primary-700, 29 78 216));animation:voice-void-ai-glow 1.2s ease-in-out infinite}.voice-void-orb-wrap--speaking .voice-void-ring{border:2px solid rgb(var(--ax-sys-color-primary-300, 147 197 253) / .7)}.voice-void-orb-wrap--speaking .voice-void-ring--1{animation:voice-void-ai-orbit 2.4s linear infinite}.voice-void-orb-wrap--speaking .voice-void-ring--2{inset:.55rem;animation:voice-void-ai-orbit 3.2s linear infinite reverse}.voice-void-orb-wrap--speaking .voice-void-ring--3{display:none}.voice-void-orb--confirm,.voice-void-orb--error{background:rgb(var(--ax-sys-color-warning-500, 217 119 6))}.voice-void-eq{position:absolute;z-index:2;bottom:.85rem;display:none;align-items:flex-end;gap:.18rem;height:1.1rem;pointer-events:none}.voice-void-orb-wrap--listening .voice-void-eq,.voice-void-orb-wrap--speaking .voice-void-eq{display:flex}.voice-void-eq span{display:block;width:.18rem;border-radius:999px;background:rgb(var(--ax-sys-color-on-primary, 255 255 255));transform-origin:bottom center}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(1),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(1){animation:voice-void-eq-bar .7s ease-in-out infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(2),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(2){animation:voice-void-eq-bar .7s ease-in-out .1s infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(3),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(3){animation:voice-void-eq-bar .7s ease-in-out .18s infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(4),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(4){animation:voice-void-eq-bar .7s ease-in-out .08s infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(5),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(5){animation:voice-void-eq-bar .7s ease-in-out .22s infinite}.voice-void-orb-wrap--speaking .voice-void-eq span{animation-duration:.95s}.voice-void-status{margin:0;font-weight:600}.voice-void-progress,.voice-void-spoken,.voice-void-transcript,.voice-void-error,.voice-void-hint,.voice-void-confirm-prompt{margin:0}.voice-void-spoken,.voice-void-transcript{max-height:4.5rem;overflow:auto;font-size:.875rem}.voice-void-spoken,.voice-void-transcript,.voice-void-hint{color:rgb(var(--ax-sys-color-on-surface) / .8)}.voice-void-error{color:rgb(var(--ax-sys-color-danger, 185 28 28))}.voice-void-confirm,.voice-void-toolbar{display:flex;flex-wrap:wrap;gap:.75rem;justify-content:center;align-items:center}.voice-void-confirm{flex-direction:column}.voice-void-hint{font-size:.8125rem}@keyframes voice-void-user-pulse{0%,to{transform:scale(1);box-shadow:0 0 rgb(var(--ax-sys-color-primary-500, 59 130 246) / .4)}50%{transform:scale(1.06);box-shadow:0 0 0 .65rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .12)}}@keyframes voice-void-user-ripple{0%{transform:scale(.92);opacity:.55}to{transform:scale(1.55);opacity:0}}@keyframes voice-void-ai-glow{0%,to{transform:scale(1);box-shadow:0 0 0 .35rem rgb(var(--ax-sys-color-primary-300, 147 197 253) / .35),0 0 1.75rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .55)}50%{transform:scale(1.04);box-shadow:0 0 0 .7rem rgb(var(--ax-sys-color-primary-300, 147 197 253) / .18),0 0 2.25rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .75)}}@keyframes voice-void-ai-orbit{0%{transform:scale(1) rotate(0);opacity:.45;border-style:dashed}50%{opacity:.85}to{transform:scale(1.12) rotate(360deg);opacity:.45;border-style:dashed}}@keyframes voice-void-breathe{0%,to{transform:scale(1);opacity:.8}50%{transform:scale(1.06);opacity:1}}@keyframes voice-void-eq-bar{0%,to{height:.28rem}40%{height:.95rem}70%{height:.45rem}}\n"], dependencies: [{ kind: "ngmodule", type: CommonModule }, { kind: "ngmodule", type: AXButtonModule }, { kind: "component", type: i1.AXButtonComponent, selector: "ax-button", inputs: ["disabled", "tabIndex", "color", "look", "text", "toggleable", "selected", "iconOnly", "type", "loadingText"], outputs: ["onBlur", "onFocus", "onClick", "selectedChange", "toggleableChange", "lookChange", "colorChange", "disabledChange", "loadingTextChange"] }, { kind: "ngmodule", type: AXDecoratorModule }, { kind: "ngmodule", type: AXTranslationModule }, { kind: "pipe", type: i2$1.AsyncPipe, name: "async" }, { kind: "pipe", type: i3.AXTranslatorPipe, name: "translate" }], changeDetection: i0.ChangeDetectionStrategy.OnPush, encapsulation: i0.ViewEncapsulation.None }); }
|
|
1012
1645
|
}
|
|
1013
1646
|
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "22.0.8", ngImport: i0, type: AXEAiVoiceVoidDialogComponent, decorators: [{
|
|
1014
1647
|
type: Component,
|
|
1015
|
-
args: [{ selector: 'axe-ai-voice-void-dialog', changeDetection: ChangeDetectionStrategy.OnPush, imports: [CommonModule, AXButtonModule, AXDecoratorModule, AXTranslationModule], encapsulation: ViewEncapsulation.None, template: "<div class=\"voice-void\">\n <div class=\"voice-void-stage\">\n <!-- ----- Listening / thinking / speaking orb ----- -->\n <div [class]=\"orbWrapClass()\">\n <span class=\"voice-void-ring voice-void-ring--1\" aria-hidden=\"true\"></span>\n <span class=\"voice-void-ring voice-void-ring--2\" aria-hidden=\"true\"></span>\n <span class=\"voice-void-ring voice-void-ring--3\" aria-hidden=\"true\"></span>\n <button\n type=\"button\"\n [class]=\"orbClass()\"\n (click)=\"
|
|
1016
|
-
}]
|
|
1648
|
+
args: [{ selector: 'axe-ai-voice-void-dialog', changeDetection: ChangeDetectionStrategy.OnPush, imports: [CommonModule, AXButtonModule, AXDecoratorModule, AXTranslationModule], encapsulation: ViewEncapsulation.None, template: "<div class=\"voice-void\">\n <div class=\"voice-void-stage\">\n <!-- ----- Listening / thinking / speaking orb ----- -->\n <div [class]=\"orbWrapClass()\">\n <span class=\"voice-void-ring voice-void-ring--1\" aria-hidden=\"true\"></span>\n <span class=\"voice-void-ring voice-void-ring--2\" aria-hidden=\"true\"></span>\n <span class=\"voice-void-ring voice-void-ring--3\" aria-hidden=\"true\"></span>\n <button\n type=\"button\"\n [class]=\"orbClass()\"\n (click)=\"onOrbClick()\"\n [attr.aria-label]=\"'@extensions-ai-voice-shell:actions.interrupt' | translate | async\"\n >\n @if (isSpeaking()) {\n <!-- ----- AI is speaking; click interrupts ----- -->\n <i class=\"fa-regular fa-volume-high\"></i>\n }\n <!-- ----- Live microphone ----- -->\n @else {\n <i class=\"fa-regular fa-microphone\"></i>\n }\n </button>\n @if (isBusy()) {\n <!-- ----- Waveform bars: user listen vs AI speak ----- -->\n <span class=\"voice-void-eq\" aria-hidden=\"true\">\n <span></span>\n <span></span>\n <span></span>\n <span></span>\n <span></span>\n </span>\n }\n </div>\n\n <p class=\"voice-void-status\">{{ shell.statusText() }}</p>\n\n @if (shell.lastSpoken()) {\n <!-- ----- Last spoken assistant text ----- -->\n <p class=\"voice-void-spoken\">{{ shell.lastSpoken() }}</p>\n }\n\n @if (shell.lastTranscript()) {\n <!-- ----- Last user transcript ----- -->\n <p class=\"voice-void-transcript\">{{ shell.lastTranscript() }}</p>\n }\n\n @if (shell.errorMessage()) {\n <!-- ----- Spoken / visible error ----- -->\n <p class=\"voice-void-error\">{{ shell.errorMessage() }}</p>\n }\n </div>\n\n <!-- ----- Hang up ends the live session ----- -->\n <div class=\"voice-void-toolbar\">\n <ax-button\n [text]=\"'@extensions-ai-voice-shell:actions.hang-up' | translate | async\"\n color=\"danger\"\n (onClick)=\"onHangUp()\"\n ></ax-button>\n </div>\n\n <p class=\"voice-void-hint\">{{ '@extensions-ai-voice-shell:dialog.hint' | translate | async }}</p>\n</div>\n", styles: [":host,axe-ai-voice-void-dialog{display:block;height:auto}.axe-ai-voice-shell-popup .ax-popup-content,.axe-ai-voice-shell-popup .ax-popup-body{overflow:visible}.voice-void{display:flex;flex-direction:column;align-items:center;justify-content:center;min-height:16rem;padding:1.25rem 1rem .75rem;gap:1rem}.voice-void-stage{display:flex;flex-direction:column;align-items:center;gap:.75rem;max-width:22rem;text-align:center}.voice-void-orb-wrap{position:relative;display:flex;align-items:center;justify-content:center;width:9.5rem;height:9.5rem}.voice-void-ring{position:absolute;inset:1.1rem;border-radius:50%;pointer-events:none;opacity:0}.voice-void-orb{position:relative;z-index:1;display:inline-flex;align-items:center;justify-content:center;width:6.25rem;height:6.25rem;border:none;border-radius:50%;background:rgb(var(--ax-sys-color-primary-500, 59 130 246));color:rgb(var(--ax-sys-color-on-primary, 255 255 255));font-size:1.75rem;cursor:pointer;transition:transform .2s ease,box-shadow .2s ease,opacity .2s ease,background-color .2s ease}.voice-void-orb:disabled{cursor:default;opacity:.92}.voice-void-orb--idle{box-shadow:0 .5rem 1.25rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .28)}.voice-void-orb--listening{animation:voice-void-user-pulse 1.05s ease-in-out infinite}.voice-void-orb-wrap--listening .voice-void-ring{border:2px solid rgb(var(--ax-sys-color-primary-400, 96 165 250))}.voice-void-orb-wrap--listening .voice-void-ring--1{animation:voice-void-user-ripple 1.5s ease-out infinite}.voice-void-orb-wrap--listening .voice-void-ring--2{animation:voice-void-user-ripple 1.5s ease-out .28s infinite}.voice-void-orb-wrap--listening .voice-void-ring--3{animation:voice-void-user-ripple 1.5s ease-out .56s infinite}.voice-void-orb--thinking{opacity:.8;animation:voice-void-breathe 1.8s ease-in-out infinite}.voice-void-orb--speaking{background:rgb(var(--ax-sys-color-primary-700, 29 78 216));animation:voice-void-ai-glow 1.2s ease-in-out infinite}.voice-void-orb-wrap--speaking .voice-void-ring{border:2px solid rgb(var(--ax-sys-color-primary-300, 147 197 253) / .7)}.voice-void-orb-wrap--speaking .voice-void-ring--1{animation:voice-void-ai-orbit 2.4s linear infinite}.voice-void-orb-wrap--speaking .voice-void-ring--2{inset:.55rem;animation:voice-void-ai-orbit 3.2s linear infinite reverse}.voice-void-orb-wrap--speaking .voice-void-ring--3{display:none}.voice-void-orb--confirm,.voice-void-orb--error{background:rgb(var(--ax-sys-color-warning-500, 217 119 6))}.voice-void-eq{position:absolute;z-index:2;bottom:.85rem;display:none;align-items:flex-end;gap:.18rem;height:1.1rem;pointer-events:none}.voice-void-orb-wrap--listening .voice-void-eq,.voice-void-orb-wrap--speaking .voice-void-eq{display:flex}.voice-void-eq span{display:block;width:.18rem;border-radius:999px;background:rgb(var(--ax-sys-color-on-primary, 255 255 255));transform-origin:bottom center}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(1),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(1){animation:voice-void-eq-bar .7s ease-in-out infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(2),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(2){animation:voice-void-eq-bar .7s ease-in-out .1s infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(3),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(3){animation:voice-void-eq-bar .7s ease-in-out .18s infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(4),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(4){animation:voice-void-eq-bar .7s ease-in-out .08s infinite}.voice-void-orb-wrap--listening .voice-void-eq span:nth-child(5),.voice-void-orb-wrap--speaking .voice-void-eq span:nth-child(5){animation:voice-void-eq-bar .7s ease-in-out .22s infinite}.voice-void-orb-wrap--speaking .voice-void-eq span{animation-duration:.95s}.voice-void-status{margin:0;font-weight:600}.voice-void-progress,.voice-void-spoken,.voice-void-transcript,.voice-void-error,.voice-void-hint,.voice-void-confirm-prompt{margin:0}.voice-void-spoken,.voice-void-transcript{max-height:4.5rem;overflow:auto;font-size:.875rem}.voice-void-spoken,.voice-void-transcript,.voice-void-hint{color:rgb(var(--ax-sys-color-on-surface) / .8)}.voice-void-error{color:rgb(var(--ax-sys-color-danger, 185 28 28))}.voice-void-confirm,.voice-void-toolbar{display:flex;flex-wrap:wrap;gap:.75rem;justify-content:center;align-items:center}.voice-void-confirm{flex-direction:column}.voice-void-hint{font-size:.8125rem}@keyframes voice-void-user-pulse{0%,to{transform:scale(1);box-shadow:0 0 rgb(var(--ax-sys-color-primary-500, 59 130 246) / .4)}50%{transform:scale(1.06);box-shadow:0 0 0 .65rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .12)}}@keyframes voice-void-user-ripple{0%{transform:scale(.92);opacity:.55}to{transform:scale(1.55);opacity:0}}@keyframes voice-void-ai-glow{0%,to{transform:scale(1);box-shadow:0 0 0 .35rem rgb(var(--ax-sys-color-primary-300, 147 197 253) / .35),0 0 1.75rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .55)}50%{transform:scale(1.04);box-shadow:0 0 0 .7rem rgb(var(--ax-sys-color-primary-300, 147 197 253) / .18),0 0 2.25rem rgb(var(--ax-sys-color-primary-500, 59 130 246) / .75)}}@keyframes voice-void-ai-orbit{0%{transform:scale(1) rotate(0);opacity:.45;border-style:dashed}50%{opacity:.85}to{transform:scale(1.12) rotate(360deg);opacity:.45;border-style:dashed}}@keyframes voice-void-breathe{0%,to{transform:scale(1);opacity:.8}50%{transform:scale(1.06);opacity:1}}@keyframes voice-void-eq-bar{0%,to{height:.28rem}40%{height:.95rem}70%{height:.45rem}}\n"] }]
|
|
1649
|
+
}] });
|
|
1017
1650
|
|
|
1018
1651
|
var voiceVoidDialog_component = /*#__PURE__*/Object.freeze({
|
|
1019
1652
|
__proto__: null,
|