arcane-os 0.5.13 → 0.5.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/browser-runtime/ai/browser-wasm-llm-provider.mjs +11 -10
- package/browser-runtime/ai/browser-wllama-runtime.mjs +2 -1
- package/browser-runtime/ai/model-controller.mjs +6 -5
- package/browser-runtime/ai/speech-worker-client.mjs +45 -3
- package/browser-runtime/event-manager.mjs +2 -1
- package/browser-runtime/logging.mjs +58 -0
- package/docs/reference/README.md +8 -7
- package/docs/reference/ai/browser-speech.md +46 -1
- package/docs/reference/inventory/package-api.json +30 -2
- package/docs/reference/runtime-modules.md +18 -0
- package/docs/reference/sdk-api.md +94 -8
- package/package.json +3 -2
- package/runtime/arcane/components/app-bar.html +3 -1
- package/runtime/arcane/components/chat.html +22 -20
- package/runtime/arcane/components/dashboard-config.html +3 -1
- package/runtime/arcane/components/data-maintenance.html +4 -2
- package/runtime/arcane/components/data-view.html +3 -1
- package/runtime/arcane/components/directory-picker.html +3 -1
- package/runtime/arcane/components/file-manager.html +11 -9
- package/runtime/arcane/components/header.html +5 -3
- package/runtime/arcane/components/markdown-document.html +3 -1
- package/runtime/arcane/components/markdown-editor.html +4 -2
- package/runtime/arcane/components/modal.html +3 -1
- package/runtime/arcane/components/screen-capture.html +4 -2
- package/runtime/arcane/components/speech.html +10 -8
- package/runtime/arcane/components/table.html +3 -1
- package/runtime/arcane/components/voice-transcription.html +8 -6
- package/runtime/arcane/entities/Chat.js +5 -4
- package/runtime/arcane/entities/User.js +2 -1
- package/runtime/arcane/modules/AI.js +442 -292
- package/runtime/arcane/modules/AIProviderRuntime.js +359 -82
- package/runtime/arcane/modules/CommunicationAppController.js +2 -1
- package/runtime/arcane/modules/ComponentContracts.js +3 -2
- package/runtime/arcane/modules/ConversationTimebox.js +2 -1
- package/runtime/arcane/modules/DBOPFS.js +5 -4
- package/runtime/arcane/modules/Errors.js +3 -14
- package/runtime/arcane/modules/HTMLImport.js +4 -3
- package/runtime/arcane/modules/LocalAIReadinessController.js +2 -1
- package/runtime/arcane/modules/MD.js +3 -2
- package/runtime/arcane/modules/MailOutbox.mjs +2 -1
- package/runtime/arcane/modules/PersistentAIChatSession.js +2 -1
- package/runtime/arcane/modules/ScreenCapture.js +2 -1
- package/runtime/arcane/modules/ThemeBootstrap.js +3 -2
- package/runtime/arcane/modules/ToolCallRouter.js +2 -1
- package/src/import-map.mjs +65 -10
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
getAIProviderRuntime
|
|
13
13
|
} from './AIProviderRuntime.js';
|
|
14
14
|
import {normalizeOllamaModelIdentifier} from './OllamaModelIdentifier.js';
|
|
15
|
+
import {arcaneLogging} from 'arcane-os/logging';
|
|
15
16
|
|
|
16
17
|
const completeValue=(value)=>value;
|
|
17
18
|
|
|
@@ -546,6 +547,7 @@ function createBuiltInAIStreamBridge(execute,sourceSignal){
|
|
|
546
547
|
const queue=[];
|
|
547
548
|
const waiters=[];
|
|
548
549
|
let complete=false;
|
|
550
|
+
let failed=false;
|
|
549
551
|
let failure=null;
|
|
550
552
|
let detached=false;
|
|
551
553
|
|
|
@@ -594,11 +596,12 @@ function createBuiltInAIStreamBridge(execute,sourceSignal){
|
|
|
594
596
|
return;
|
|
595
597
|
}
|
|
596
598
|
complete=true;
|
|
597
|
-
|
|
599
|
+
failed=arguments.length>0;
|
|
600
|
+
failure=error;
|
|
598
601
|
detachBuiltInAIStreamAbort();
|
|
599
602
|
while(waiters.length){
|
|
600
603
|
const waiter=waiters.shift();
|
|
601
|
-
if(
|
|
604
|
+
if(failed){
|
|
602
605
|
waiter.reject(failure);
|
|
603
606
|
}else{
|
|
604
607
|
waiter.resolve({value:undefined,done:true});
|
|
@@ -618,7 +621,7 @@ function createBuiltInAIStreamBridge(execute,sourceSignal){
|
|
|
618
621
|
}
|
|
619
622
|
).then(
|
|
620
623
|
function acceptBuiltInAIStreamResult(value){
|
|
621
|
-
finishBuiltInAIStream(
|
|
624
|
+
finishBuiltInAIStream();
|
|
622
625
|
return value;
|
|
623
626
|
},
|
|
624
627
|
function rejectBuiltInAIStreamResult(error){
|
|
@@ -647,7 +650,7 @@ function createBuiltInAIStreamBridge(execute,sourceSignal){
|
|
|
647
650
|
return Promise.resolve({value:queue.shift(),done:false});
|
|
648
651
|
}
|
|
649
652
|
if(complete){
|
|
650
|
-
return
|
|
653
|
+
return failed
|
|
651
654
|
?Promise.reject(failure)
|
|
652
655
|
:Promise.resolve({value:undefined,done:true});
|
|
653
656
|
}
|
|
@@ -1177,8 +1180,49 @@ class AI {
|
|
|
1177
1180
|
#builtInSpeechReadiness=Promise.resolve(null);
|
|
1178
1181
|
#speechControlGeneration=0;
|
|
1179
1182
|
#speechFailureSequence=0;
|
|
1183
|
+
#speechDiagnosticSequence=0;
|
|
1184
|
+
#speechJobSequence=0;
|
|
1180
1185
|
#stopOllamaReady=null;
|
|
1181
1186
|
#ttsSegmentation={...DEFAULT_TTS_SEGMENTATION};
|
|
1187
|
+
|
|
1188
|
+
#traceSpeech(phase,detail={}){
|
|
1189
|
+
if(!arcaneLogging.enabled)return null;
|
|
1190
|
+
const diagnosticId=++this.#speechDiagnosticSequence;
|
|
1191
|
+
arcaneLogging.debug('[Arcane speech]',{
|
|
1192
|
+
diagnosticId,
|
|
1193
|
+
phase,
|
|
1194
|
+
timestamp:new Date().toISOString(),
|
|
1195
|
+
timeMs:globalThis.performance?.now?.()??Date.now(),
|
|
1196
|
+
instanceId:this.#events?.instanceId,
|
|
1197
|
+
generation:this.speechGeneration,
|
|
1198
|
+
muted:this.muted,
|
|
1199
|
+
...detail
|
|
1200
|
+
});
|
|
1201
|
+
return diagnosticId;
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
#traceSpeechJob(phase,job,detail={}){
|
|
1205
|
+
if(!arcaneLogging.enabled)return;
|
|
1206
|
+
this.#traceSpeech(phase,{
|
|
1207
|
+
jobId:job.diagnosticId,
|
|
1208
|
+
callId:job.diagnosticCallId,
|
|
1209
|
+
jobGeneration:job.generation,
|
|
1210
|
+
state:job.state,
|
|
1211
|
+
text:job.text,
|
|
1212
|
+
voice:job.voice,
|
|
1213
|
+
speed:job.speed,
|
|
1214
|
+
pauseAfterMs:job.pauseAfterMs??0,
|
|
1215
|
+
scheduledStart:job.scheduledStart,
|
|
1216
|
+
scheduledEnd:job.scheduledEnd,
|
|
1217
|
+
audioTime:job.audioContext?.currentTime,
|
|
1218
|
+
audioContextState:job.audioContext?.state,
|
|
1219
|
+
duration:job.audioBuffer?.duration,
|
|
1220
|
+
sampleRate:job.audioBuffer?.sampleRate,
|
|
1221
|
+
channels:job.audioBuffer?.numberOfChannels,
|
|
1222
|
+
playbackRate:job.sourceNode?.playbackRate?.value,
|
|
1223
|
+
...detail
|
|
1224
|
+
});
|
|
1225
|
+
}
|
|
1182
1226
|
#preferenceTuple=completeValue([
|
|
1183
1227
|
'TWIN',
|
|
1184
1228
|
'LOCAL_SPEACH',
|
|
@@ -2309,6 +2353,7 @@ class AI {
|
|
|
2309
2353
|
}
|
|
2310
2354
|
|
|
2311
2355
|
configureSpeechProviders(selections){
|
|
2356
|
+
this.#traceSpeech('configureSpeechProviders.call',{selections});
|
|
2312
2357
|
const prepared=this.#providerRuntime.validateSpeechConfiguration(selections);
|
|
2313
2358
|
this.#assertDeviceSpeechConfiguration(prepared);
|
|
2314
2359
|
this.#assertSynchronousBrowserSpeechSupersession(
|
|
@@ -2337,6 +2382,7 @@ class AI {
|
|
|
2337
2382
|
);
|
|
2338
2383
|
this.muted=true;
|
|
2339
2384
|
this.stopAudio();
|
|
2385
|
+
this.#traceSpeech('configureSpeechProviders.result',{configured});
|
|
2340
2386
|
return configured;
|
|
2341
2387
|
}
|
|
2342
2388
|
|
|
@@ -2408,6 +2454,7 @@ class AI {
|
|
|
2408
2454
|
}
|
|
2409
2455
|
|
|
2410
2456
|
async transitionSpeechProviders(selections){
|
|
2457
|
+
this.#traceSpeech('transitionSpeechProviders.call',{selections});
|
|
2411
2458
|
const prepared=this.#providerRuntime.validateSpeechConfiguration(selections);
|
|
2412
2459
|
this.#assertDeviceSpeechConfiguration(prepared);
|
|
2413
2460
|
await this.#supersedeBrowserSpeechForRouteChange();
|
|
@@ -2432,6 +2479,7 @@ class AI {
|
|
|
2432
2479
|
});
|
|
2433
2480
|
await this.#reconcileBuiltInSpeechReadiness();
|
|
2434
2481
|
this.muted=true;
|
|
2482
|
+
this.#traceSpeech('transitionSpeechProviders.result',{configured});
|
|
2435
2483
|
return configured;
|
|
2436
2484
|
}
|
|
2437
2485
|
|
|
@@ -2650,6 +2698,7 @@ class AI {
|
|
|
2650
2698
|
...(error?{error}:{}),
|
|
2651
2699
|
reason
|
|
2652
2700
|
});
|
|
2701
|
+
this.#traceSpeech(type,{operationId,...compatibilityDetail});
|
|
2653
2702
|
return this.#events.dispatch(
|
|
2654
2703
|
type,
|
|
2655
2704
|
compatibilityDetail,
|
|
@@ -3367,6 +3416,7 @@ class AI {
|
|
|
3367
3416
|
}
|
|
3368
3417
|
|
|
3369
3418
|
configureBrowserSpeech(configuration,options={}){
|
|
3419
|
+
this.#traceSpeech('configureBrowserSpeech.call',{configuration,options});
|
|
3370
3420
|
const normalized=normalizeBrowserSpeechConfiguration(configuration);
|
|
3371
3421
|
const operation=normalizeBrowserSpeechOperationOptions(
|
|
3372
3422
|
options,
|
|
@@ -3391,6 +3441,10 @@ class AI {
|
|
|
3391
3441
|
)
|
|
3392
3442
|
&&this.#browserSpeechRetiredRecords.size===0
|
|
3393
3443
|
&&!this.#browserSpeechController){
|
|
3444
|
+
this.#traceSpeech('configureBrowserSpeech.result',{
|
|
3445
|
+
descriptor:this.#browserSpeechConfigurationRecord.descriptor,
|
|
3446
|
+
reused:true
|
|
3447
|
+
});
|
|
3394
3448
|
return Promise.resolve(this.#browserSpeechConfigurationRecord.descriptor);
|
|
3395
3449
|
}
|
|
3396
3450
|
const operationId=this.#browserSpeechOperationId('configure-browser-speech');
|
|
@@ -3457,6 +3511,7 @@ class AI {
|
|
|
3457
3511
|
}
|
|
3458
3512
|
|
|
3459
3513
|
disposeBrowserSpeech(options={}){
|
|
3514
|
+
this.#traceSpeech('disposeBrowserSpeech.call',{options});
|
|
3460
3515
|
const operation=normalizeBrowserSpeechOperationOptions(
|
|
3461
3516
|
options,
|
|
3462
3517
|
'AI.disposeBrowserSpeech'
|
|
@@ -3687,6 +3742,7 @@ class AI {
|
|
|
3687
3742
|
}
|
|
3688
3743
|
|
|
3689
3744
|
async setSpeechMuted(muted){
|
|
3745
|
+
const callId=this.#traceSpeech('setSpeechMuted.call',{requestedMuted:muted});
|
|
3690
3746
|
if(typeof muted!=='boolean'){
|
|
3691
3747
|
throw new TypeError('AI speech muted state must be a boolean.');
|
|
3692
3748
|
}
|
|
@@ -3697,6 +3753,7 @@ class AI {
|
|
|
3697
3753
|
}
|
|
3698
3754
|
if(!this.#usesProviderRuntime('tts',this.ttsService)){
|
|
3699
3755
|
if(generation===this.#speechControlGeneration)this.muted=muted;
|
|
3756
|
+
this.#traceSpeech('setSpeechMuted.result',{callId,result:true});
|
|
3700
3757
|
return true;
|
|
3701
3758
|
}
|
|
3702
3759
|
await this.#providerRuntime.setSpeechMuted(muted);
|
|
@@ -3706,44 +3763,71 @@ class AI {
|
|
|
3706
3763
|
||status.state!=='ready'
|
|
3707
3764
|
||status.loaded!==true;
|
|
3708
3765
|
}
|
|
3766
|
+
this.#traceSpeech('setSpeechMuted.result',{callId,result:true});
|
|
3709
3767
|
return true;
|
|
3710
3768
|
}
|
|
3711
3769
|
|
|
3712
|
-
async #
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
}
|
|
3716
|
-
|
|
3717
|
-
let detail='';
|
|
3718
|
-
|
|
3770
|
+
async #fetchHTTPResponse(url,options){
|
|
3771
|
+
const {signal}=options;
|
|
3772
|
+
const retryDelayMs=3000;
|
|
3719
3773
|
try{
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3774
|
+
while(true){
|
|
3775
|
+
if(signal?.aborted){
|
|
3776
|
+
throw normalizeAIRequestAbort(signal.reason);
|
|
3777
|
+
}
|
|
3778
|
+
const response=await fetch(url,options);
|
|
3779
|
+
if(signal?.aborted){
|
|
3780
|
+
throw normalizeAIRequestAbort(signal.reason);
|
|
3781
|
+
}
|
|
3782
|
+
if(response.ok){
|
|
3783
|
+
return response;
|
|
3784
|
+
}
|
|
3729
3785
|
|
|
3730
|
-
|
|
3731
|
-
|
|
3786
|
+
const contentType=response.headers.get('content-type')||'';
|
|
3787
|
+
const error=contentType.includes('application/json')
|
|
3788
|
+
?await response.json()
|
|
3789
|
+
:await response.text();
|
|
3790
|
+
if(signal?.aborted){
|
|
3791
|
+
throw normalizeAIRequestAbort(signal.reason);
|
|
3732
3792
|
}
|
|
3793
|
+
const message=typeof error==='string'
|
|
3794
|
+
?error
|
|
3795
|
+
:error?.error?.message??error?.message;
|
|
3796
|
+
if(
|
|
3797
|
+
response.status!==429
|
|
3798
|
+
||typeof message!=='string'
|
|
3799
|
+
||!message.toLowerCase().includes('overload')
|
|
3800
|
+
){
|
|
3801
|
+
throw error;
|
|
3802
|
+
}
|
|
3803
|
+
|
|
3804
|
+
arcaneLogging.warn(
|
|
3805
|
+
`${message}\nRetrying in ${retryDelayMs / 1000} seconds`,
|
|
3806
|
+
error
|
|
3807
|
+
);
|
|
3808
|
+
await new Promise(function waitForOverloadRetry(resolve,reject){
|
|
3809
|
+
function finishRetryDelay(){
|
|
3810
|
+
signal?.removeEventListener('abort',cancelRetryDelay);
|
|
3811
|
+
resolve();
|
|
3812
|
+
}
|
|
3813
|
+
function cancelRetryDelay(){
|
|
3814
|
+
clearTimeout(timer);
|
|
3815
|
+
signal.removeEventListener('abort',cancelRetryDelay);
|
|
3816
|
+
reject(normalizeAIRequestAbort(signal.reason));
|
|
3817
|
+
}
|
|
3818
|
+
const timer=setTimeout(finishRetryDelay,retryDelayMs);
|
|
3819
|
+
signal?.addEventListener('abort',cancelRetryDelay,{once:true});
|
|
3820
|
+
if(signal?.aborted){
|
|
3821
|
+
cancelRetryDelay();
|
|
3822
|
+
}
|
|
3823
|
+
});
|
|
3733
3824
|
}
|
|
3734
|
-
}catch{
|
|
3735
|
-
|
|
3825
|
+
}catch(error){
|
|
3826
|
+
if(isAIRequestAbort(error,signal)){
|
|
3827
|
+
throw normalizeAIRequestAbort(error);
|
|
3828
|
+
}
|
|
3829
|
+
throw error;
|
|
3736
3830
|
}
|
|
3737
|
-
|
|
3738
|
-
const status=[response.status,response.statusText]
|
|
3739
|
-
.filter(Boolean)
|
|
3740
|
-
.join(' ');
|
|
3741
|
-
const message=`AI request failed${status ? ` (${status})`:''}`;
|
|
3742
|
-
const error=new Error(message);
|
|
3743
|
-
error.code='AI_REQUEST_FAILED';
|
|
3744
|
-
error.status=response.status;
|
|
3745
|
-
error.providerMessage=detail;
|
|
3746
|
-
throw error;
|
|
3747
3831
|
}
|
|
3748
3832
|
|
|
3749
3833
|
#nativeOllama(){
|
|
@@ -4850,35 +4934,16 @@ class AI {
|
|
|
4850
4934
|
destination:this.url
|
|
4851
4935
|
});
|
|
4852
4936
|
const body = JSON.stringify(request);
|
|
4853
|
-
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
|
|
4857
|
-
|
|
4858
|
-
|
|
4859
|
-
|
|
4860
|
-
|
|
4861
|
-
headers:this.#serviceHeaders[this.llmService],
|
|
4862
|
-
body,
|
|
4863
|
-
...(signal?{signal}:{})
|
|
4864
|
-
}
|
|
4865
|
-
);
|
|
4866
|
-
}catch(err){
|
|
4867
|
-
if(signal?.aborted||err?.name==='AbortError'){
|
|
4868
|
-
const error=new Error('The AI request was cancelled.',{cause:err});
|
|
4869
|
-
error.name='AbortError';
|
|
4870
|
-
error.code='ARCANE_AI_REQUEST_ABORTED';
|
|
4871
|
-
throw error;
|
|
4937
|
+
const response=await this.#fetchHTTPResponse(
|
|
4938
|
+
this.url,
|
|
4939
|
+
{
|
|
4940
|
+
method:'POST',
|
|
4941
|
+
credentials,
|
|
4942
|
+
headers:this.#serviceHeaders[this.llmService],
|
|
4943
|
+
body,
|
|
4944
|
+
...(signal?{signal}:{})
|
|
4872
4945
|
}
|
|
4873
|
-
|
|
4874
|
-
'Unable to reach the AI service.',
|
|
4875
|
-
{cause:err}
|
|
4876
|
-
);
|
|
4877
|
-
error.code='AI_SERVICE_UNREACHABLE';
|
|
4878
|
-
throw error;
|
|
4879
|
-
}
|
|
4880
|
-
|
|
4881
|
-
await this.#assertResponseOK(response);
|
|
4946
|
+
);
|
|
4882
4947
|
|
|
4883
4948
|
let sseBuffer='';
|
|
4884
4949
|
const completeToolCallsByChoice=new Map();
|
|
@@ -5174,7 +5239,7 @@ class AI {
|
|
|
5174
5239
|
}
|
|
5175
5240
|
if(receivedSseDone){
|
|
5176
5241
|
await reader.cancel('[DONE]').catch(
|
|
5177
|
-
error=>
|
|
5242
|
+
error=>arcaneLogging.error('Arcane SSE reader cleanup failed.',error)
|
|
5178
5243
|
);
|
|
5179
5244
|
}else{
|
|
5180
5245
|
sseBuffer+=decoder.decode();
|
|
@@ -5598,33 +5663,16 @@ class AI {
|
|
|
5598
5663
|
destination:this.url
|
|
5599
5664
|
});
|
|
5600
5665
|
const body = JSON.stringify(request);
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5606
|
-
this.
|
|
5607
|
-
|
|
5608
|
-
|
|
5609
|
-
credentials: credentials,
|
|
5610
|
-
headers: this.#serviceHeaders[this.llmService],
|
|
5611
|
-
body: body,
|
|
5612
|
-
...(signal?{signal}:{})
|
|
5613
|
-
}
|
|
5614
|
-
);
|
|
5615
|
-
}catch(err){
|
|
5616
|
-
if(isAIRequestAbort(err,signal)){
|
|
5617
|
-
throw normalizeAIRequestAbort(err);
|
|
5666
|
+
const response=await this.#fetchHTTPResponse(
|
|
5667
|
+
this.url,
|
|
5668
|
+
{
|
|
5669
|
+
method:'POST',
|
|
5670
|
+
credentials,
|
|
5671
|
+
headers:this.#serviceHeaders[this.llmService],
|
|
5672
|
+
body,
|
|
5673
|
+
...(signal?{signal}:{})
|
|
5618
5674
|
}
|
|
5619
|
-
|
|
5620
|
-
'Unable to reach the AI service.',
|
|
5621
|
-
{cause:err}
|
|
5622
|
-
);
|
|
5623
|
-
error.code='AI_SERVICE_UNREACHABLE';
|
|
5624
|
-
throw error;
|
|
5625
|
-
}
|
|
5626
|
-
|
|
5627
|
-
await this.#assertResponseOK(response);
|
|
5675
|
+
);
|
|
5628
5676
|
|
|
5629
5677
|
const contentType=response.headers.get('content-type')||'';
|
|
5630
5678
|
|
|
@@ -5664,11 +5712,14 @@ class AI {
|
|
|
5664
5712
|
}
|
|
5665
5713
|
|
|
5666
5714
|
configureTTSSegmentation(options={}){
|
|
5715
|
+
this.#traceSpeech('configureTTSSegmentation.call',{options});
|
|
5667
5716
|
this.#ttsSegmentation=normalizeTTSSegmentation(
|
|
5668
5717
|
options,
|
|
5669
5718
|
this.#ttsSegmentation
|
|
5670
5719
|
);
|
|
5671
|
-
|
|
5720
|
+
const result=this.ttsSegmentation;
|
|
5721
|
+
this.#traceSpeech('configureTTSSegmentation.result',{result});
|
|
5722
|
+
return result;
|
|
5672
5723
|
}
|
|
5673
5724
|
|
|
5674
5725
|
streamTTS(
|
|
@@ -5676,10 +5727,12 @@ class AI {
|
|
|
5676
5727
|
end=false,
|
|
5677
5728
|
options={}
|
|
5678
5729
|
){
|
|
5730
|
+
const callId=this.#traceSpeech('streamTTS.call',{text,end,options});
|
|
5679
5731
|
if(this.muted){
|
|
5680
5732
|
if(end){
|
|
5681
5733
|
this.audioMessageChunks='';
|
|
5682
5734
|
}
|
|
5735
|
+
this.#traceSpeech('streamTTS.result',{callId,result:false,reason:'muted'});
|
|
5683
5736
|
return Promise.resolve(false);
|
|
5684
5737
|
}
|
|
5685
5738
|
|
|
@@ -5691,14 +5744,20 @@ class AI {
|
|
|
5691
5744
|
|
|
5692
5745
|
this.audioMessageChunks+=String(text||'');
|
|
5693
5746
|
const outputs=this.#extractSpeechSegments(end);
|
|
5747
|
+
this.#traceSpeech('streamTTS.segments',{
|
|
5748
|
+
callId,segments:outputs,remainder:this.audioMessageChunks,
|
|
5749
|
+
segmentation:this.ttsSegmentation
|
|
5750
|
+
});
|
|
5694
5751
|
|
|
5695
5752
|
if(!outputs.length){
|
|
5753
|
+
this.#traceSpeech('streamTTS.result',{callId,result:true,reason:'buffered'});
|
|
5696
5754
|
return Promise.resolve(true);
|
|
5697
5755
|
}
|
|
5698
5756
|
|
|
5699
5757
|
try{
|
|
5700
5758
|
this.#assertServiceConfigured(this.ttsService,'tts');
|
|
5701
5759
|
}catch(error){
|
|
5760
|
+
this.#traceSpeech('streamTTS.error',{callId,error});
|
|
5702
5761
|
this.#publishTTSFailure(error,{
|
|
5703
5762
|
boundary:'synthesis',
|
|
5704
5763
|
generation:this.speechGeneration
|
|
@@ -5708,9 +5767,11 @@ class AI {
|
|
|
5708
5767
|
|
|
5709
5768
|
const generation=this.speechGeneration;
|
|
5710
5769
|
const jobs=[];
|
|
5770
|
+
const runtime=this;
|
|
5711
5771
|
|
|
5712
5772
|
for(const [index,output] of outputs.entries()){
|
|
5713
5773
|
jobs.push(this.#queueSpeechJob(output,generation,{
|
|
5774
|
+
diagnosticCallId:callId,
|
|
5714
5775
|
voice,
|
|
5715
5776
|
speed,
|
|
5716
5777
|
waitForPlayback,
|
|
@@ -5720,12 +5781,17 @@ class AI {
|
|
|
5720
5781
|
|
|
5721
5782
|
return Promise.all(jobs).then(
|
|
5722
5783
|
function reportQueuedSpeechResult(results){
|
|
5723
|
-
|
|
5784
|
+
const result=results.every(Boolean);
|
|
5785
|
+
runtime.#traceSpeech('streamTTS.result',{
|
|
5786
|
+
callId,result,results,waitForPlayback
|
|
5787
|
+
});
|
|
5788
|
+
return result;
|
|
5724
5789
|
}
|
|
5725
5790
|
);
|
|
5726
5791
|
}
|
|
5727
5792
|
|
|
5728
5793
|
finishTTS(){
|
|
5794
|
+
this.#traceSpeech('finishTTS.call');
|
|
5729
5795
|
return this.streamTTS('',true);
|
|
5730
5796
|
}
|
|
5731
5797
|
|
|
@@ -5871,6 +5937,8 @@ class AI {
|
|
|
5871
5937
|
|
|
5872
5938
|
#queueSpeechJob(text,generation,options={}){
|
|
5873
5939
|
const job={
|
|
5940
|
+
diagnosticId:++this.#speechJobSequence,
|
|
5941
|
+
diagnosticCallId:options.diagnosticCallId,
|
|
5874
5942
|
abortController:null,
|
|
5875
5943
|
audioBuffer:null,
|
|
5876
5944
|
audioContext:null,
|
|
@@ -5893,6 +5961,7 @@ class AI {
|
|
|
5893
5961
|
:null;
|
|
5894
5962
|
|
|
5895
5963
|
this.speechJobs.push(job);
|
|
5964
|
+
this.#traceSpeechJob('queue.add',job,{queuePosition:this.speechJobs.length-1});
|
|
5896
5965
|
|
|
5897
5966
|
const preparation=Promise.resolve().then(
|
|
5898
5967
|
function synthesizeAvailableSpeech(){
|
|
@@ -5916,7 +5985,9 @@ class AI {
|
|
|
5916
5985
|
}
|
|
5917
5986
|
|
|
5918
5987
|
job.state='synthesizing';
|
|
5988
|
+
this.#traceSpeechJob('generation.start',job);
|
|
5919
5989
|
const audio=await this.#requestSpeechAudio(job);
|
|
5990
|
+
this.#traceSpeechJob('generation.result',job,{audio});
|
|
5920
5991
|
|
|
5921
5992
|
if(job.generation!==this.speechGeneration||this.muted){
|
|
5922
5993
|
return this.#cancelSpeechJob(job);
|
|
@@ -5939,16 +6010,18 @@ class AI {
|
|
|
5939
6010
|
const voice=job.voice===undefined
|
|
5940
6011
|
?(selection?this.#providerSpeechVoice():null)
|
|
5941
6012
|
:job.voice;
|
|
6013
|
+
const payload={
|
|
6014
|
+
model:selection?.modelId||this.modelTTS,
|
|
6015
|
+
input:job.text,
|
|
6016
|
+
...(job.voice!==undefined||voice?{voice}:{}),
|
|
6017
|
+
responseFormat:selection
|
|
6018
|
+
?this.#providerSpeechResponseFormat()
|
|
6019
|
+
:this.audioFormat,
|
|
6020
|
+
speed:job.speed===undefined?this.voiceSpeed:job.speed
|
|
6021
|
+
};
|
|
6022
|
+
this.#traceSpeechJob('generation.request',job,{payload,selection});
|
|
5942
6023
|
const response=await this.fetchTTS(
|
|
5943
|
-
|
|
5944
|
-
model:selection?.modelId||this.modelTTS,
|
|
5945
|
-
input:job.text,
|
|
5946
|
-
...(job.voice!==undefined||voice?{voice}:{}),
|
|
5947
|
-
responseFormat:selection
|
|
5948
|
-
?this.#providerSpeechResponseFormat()
|
|
5949
|
-
:this.audioFormat,
|
|
5950
|
-
speed:job.speed===undefined?this.voiceSpeed:job.speed
|
|
5951
|
-
},
|
|
6024
|
+
payload,
|
|
5952
6025
|
job.abortController.signal
|
|
5953
6026
|
);
|
|
5954
6027
|
return this.#normalizeProviderSpeechAudio(response);
|
|
@@ -6099,213 +6172,241 @@ class AI {
|
|
|
6099
6172
|
}
|
|
6100
6173
|
|
|
6101
6174
|
async fetchTTS(payload={},signal=null){
|
|
6102
|
-
this.#
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
|
|
6107
|
-
|
|
6108
|
-
|
|
6109
|
-
|
|
6110
|
-
|
|
6111
|
-
|
|
6112
|
-
|
|
6113
|
-
|
|
6114
|
-
|
|
6115
|
-
|
|
6116
|
-
|
|
6117
|
-
|
|
6118
|
-
|
|
6119
|
-
|
|
6120
|
-
|
|
6121
|
-
|
|
6122
|
-
|
|
6175
|
+
const callId=this.#traceSpeech('fetchTTS.call',{payload,aborted:signal?.aborted});
|
|
6176
|
+
try{
|
|
6177
|
+
this.#assertServiceConfigured(this.ttsService,'tts');
|
|
6178
|
+
if(!payload
|
|
6179
|
+
||typeof payload!=='object'
|
|
6180
|
+
||Array.isArray(payload)
|
|
6181
|
+
||![Object.prototype,null].includes(Object.getPrototypeOf(payload))){
|
|
6182
|
+
const error=new TypeError('AI.fetchTTS requires a speech request object.');
|
|
6183
|
+
error.code='ARCANE_AI_TTS_REQUEST_INVALID';
|
|
6184
|
+
throw error;
|
|
6185
|
+
}
|
|
6186
|
+
const descriptors=Object.getOwnPropertyDescriptors(payload);
|
|
6187
|
+
const acceptedKeys=new Set([
|
|
6188
|
+
'model',
|
|
6189
|
+
'voice',
|
|
6190
|
+
'input',
|
|
6191
|
+
'responseFormat',
|
|
6192
|
+
'speed'
|
|
6193
|
+
]);
|
|
6194
|
+
for(const key of Reflect.ownKeys(descriptors)){
|
|
6195
|
+
if(typeof key==='symbol'
|
|
6196
|
+
||!acceptedKeys.has(key)
|
|
6197
|
+
||!Object.hasOwn(descriptors[key],'value')){
|
|
6198
|
+
const error=new TypeError(
|
|
6199
|
+
'AI.fetchTTS accepts only model, voice, input, responseFormat, and speed data properties.'
|
|
6200
|
+
);
|
|
6201
|
+
error.code='ARCANE_AI_TTS_REQUEST_INVALID';
|
|
6202
|
+
throw error;
|
|
6203
|
+
}
|
|
6204
|
+
}
|
|
6205
|
+
if(signal&&(
|
|
6206
|
+
typeof signal.aborted!=='boolean'
|
|
6207
|
+
||typeof signal.addEventListener!=='function'
|
|
6208
|
+
||typeof signal.removeEventListener!=='function'
|
|
6209
|
+
)){
|
|
6210
|
+
const error=new TypeError('AI.fetchTTS signal must be an AbortSignal.');
|
|
6211
|
+
error.code='ARCANE_AI_TTS_SIGNAL_INVALID';
|
|
6212
|
+
throw error;
|
|
6213
|
+
}
|
|
6214
|
+
if(signal?.aborted){
|
|
6215
|
+
throw normalizeAIRequestAbort(signal.reason);
|
|
6216
|
+
}
|
|
6217
|
+
|
|
6218
|
+
const input=descriptors.input?.value;
|
|
6219
|
+
if(typeof input!=='string'||!input.trim()){
|
|
6220
|
+
const error=new TypeError('AI.fetchTTS input must be nonempty text.');
|
|
6221
|
+
error.code='ARCANE_AI_TTS_INPUT_INVALID';
|
|
6222
|
+
throw error;
|
|
6223
|
+
}
|
|
6224
|
+
const selection=this.#providerRuntime.selection('tts');
|
|
6225
|
+
const requestedModel=descriptors.model?.value;
|
|
6226
|
+
if(requestedModel!==undefined
|
|
6227
|
+
&&(typeof requestedModel!=='string'
|
|
6228
|
+
||requestedModel.trim()!==requestedModel
|
|
6229
|
+
||!requestedModel)){
|
|
6123
6230
|
const error=new TypeError(
|
|
6124
|
-
'AI.fetchTTS
|
|
6231
|
+
'AI.fetchTTS model must be a nonempty trimmed string when provided.'
|
|
6125
6232
|
);
|
|
6126
|
-
error.code='
|
|
6233
|
+
error.code='ARCANE_AI_TTS_MODEL_INVALID';
|
|
6234
|
+
throw error;
|
|
6235
|
+
}
|
|
6236
|
+
const model=requestedModel
|
|
6237
|
+
||selection?.modelId
|
|
6238
|
+
||this.modelTTS
|
|
6239
|
+
||'';
|
|
6240
|
+
if(!model){
|
|
6241
|
+
const error=new TypeError('AI.fetchTTS model must be selected explicitly.');
|
|
6242
|
+
error.code='ARCANE_AI_TTS_MODEL_REQUIRED';
|
|
6243
|
+
throw error;
|
|
6244
|
+
}
|
|
6245
|
+
if(selection&&model!==selection.modelId){
|
|
6246
|
+
const error=new TypeError(
|
|
6247
|
+
'AI.fetchTTS model must match the admitted TTS route.'
|
|
6248
|
+
);
|
|
6249
|
+
error.code='ARCANE_AI_TTS_MODEL_SELECTION_MISMATCH';
|
|
6250
|
+
throw error;
|
|
6251
|
+
}
|
|
6252
|
+
const requestedVoice=descriptors.voice?.value;
|
|
6253
|
+
if(requestedVoice!==undefined
|
|
6254
|
+
&&(typeof requestedVoice!=='string'
|
|
6255
|
+
||requestedVoice.trim()!==requestedVoice
|
|
6256
|
+
||!requestedVoice)){
|
|
6257
|
+
const error=new TypeError(
|
|
6258
|
+
'AI.fetchTTS voice must be a nonempty trimmed string when provided.'
|
|
6259
|
+
);
|
|
6260
|
+
error.code='ARCANE_AI_TTS_VOICE_INVALID';
|
|
6261
|
+
throw error;
|
|
6262
|
+
}
|
|
6263
|
+
const voice=requestedVoice
|
|
6264
|
+
?requestedVoice
|
|
6265
|
+
:selection
|
|
6266
|
+
?this.#providerSpeechVoice()
|
|
6267
|
+
:this.#builtInSpeechDefaultVoice('tts',this.ttsService);
|
|
6268
|
+
if(!voice){
|
|
6269
|
+
const error=new TypeError(
|
|
6270
|
+
'AI.fetchTTS requires a caller- or model-catalog-admitted voice.'
|
|
6271
|
+
);
|
|
6272
|
+
error.code='ARCANE_AI_TTS_VOICE_REQUIRED';
|
|
6273
|
+
throw error;
|
|
6274
|
+
}
|
|
6275
|
+
const requestedResponseFormat=descriptors.responseFormat?.value;
|
|
6276
|
+
if(requestedResponseFormat!==undefined
|
|
6277
|
+
&&(typeof requestedResponseFormat!=='string'
|
|
6278
|
+
||requestedResponseFormat.trim()!==requestedResponseFormat
|
|
6279
|
+
||!requestedResponseFormat)){
|
|
6280
|
+
const error=new TypeError(
|
|
6281
|
+
'AI.fetchTTS responseFormat must be a nonempty trimmed string when provided.'
|
|
6282
|
+
);
|
|
6283
|
+
error.code='ARCANE_AI_TTS_RESPONSE_FORMAT_INVALID';
|
|
6284
|
+
throw error;
|
|
6285
|
+
}
|
|
6286
|
+
const responseFormat=requestedResponseFormat
|
|
6287
|
+
||(selection?this.#providerSpeechResponseFormat():this.audioFormat)
|
|
6288
|
+
||'';
|
|
6289
|
+
if(!responseFormat){
|
|
6290
|
+
const error=new TypeError('AI.fetchTTS responseFormat must be nonempty.');
|
|
6291
|
+
error.code='ARCANE_AI_TTS_RESPONSE_FORMAT_INVALID';
|
|
6292
|
+
throw error;
|
|
6293
|
+
}
|
|
6294
|
+
const speed=descriptors.speed
|
|
6295
|
+
?Number(descriptors.speed.value)
|
|
6296
|
+
:this.voiceSpeed;
|
|
6297
|
+
if(!Number.isFinite(speed)||speed<=0){
|
|
6298
|
+
const error=new RangeError('AI.fetchTTS speed must be a positive number.');
|
|
6299
|
+
error.code='ARCANE_AI_TTS_SPEED_INVALID';
|
|
6127
6300
|
throw error;
|
|
6128
6301
|
}
|
|
6129
|
-
}
|
|
6130
|
-
if(signal&&(
|
|
6131
|
-
typeof signal.aborted!=='boolean'
|
|
6132
|
-
||typeof signal.addEventListener!=='function'
|
|
6133
|
-
||typeof signal.removeEventListener!=='function'
|
|
6134
|
-
)){
|
|
6135
|
-
const error=new TypeError('AI.fetchTTS signal must be an AbortSignal.');
|
|
6136
|
-
error.code='ARCANE_AI_TTS_SIGNAL_INVALID';
|
|
6137
|
-
throw error;
|
|
6138
|
-
}
|
|
6139
|
-
if(signal?.aborted){
|
|
6140
|
-
throw normalizeAIRequestAbort(signal.reason);
|
|
6141
|
-
}
|
|
6142
6302
|
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
|
|
6148
|
-
|
|
6149
|
-
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
6156
|
-
'
|
|
6157
|
-
|
|
6158
|
-
|
|
6159
|
-
|
|
6160
|
-
|
|
6161
|
-
|
|
6162
|
-
|
|
6163
|
-
||this.modelTTS
|
|
6164
|
-
||'';
|
|
6165
|
-
if(!model){
|
|
6166
|
-
const error=new TypeError('AI.fetchTTS model must be selected explicitly.');
|
|
6167
|
-
error.code='ARCANE_AI_TTS_MODEL_REQUIRED';
|
|
6168
|
-
throw error;
|
|
6169
|
-
}
|
|
6170
|
-
if(selection&&model!==selection.modelId){
|
|
6171
|
-
const error=new TypeError(
|
|
6172
|
-
'AI.fetchTTS model must match the admitted TTS route.'
|
|
6173
|
-
);
|
|
6174
|
-
error.code='ARCANE_AI_TTS_MODEL_SELECTION_MISMATCH';
|
|
6175
|
-
throw error;
|
|
6176
|
-
}
|
|
6177
|
-
const requestedVoice=descriptors.voice?.value;
|
|
6178
|
-
if(requestedVoice!==undefined
|
|
6179
|
-
&&(typeof requestedVoice!=='string'
|
|
6180
|
-
||requestedVoice.trim()!==requestedVoice
|
|
6181
|
-
||!requestedVoice)){
|
|
6182
|
-
const error=new TypeError(
|
|
6183
|
-
'AI.fetchTTS voice must be a nonempty trimmed string when provided.'
|
|
6184
|
-
);
|
|
6185
|
-
error.code='ARCANE_AI_TTS_VOICE_INVALID';
|
|
6186
|
-
throw error;
|
|
6187
|
-
}
|
|
6188
|
-
const voice=requestedVoice
|
|
6189
|
-
?requestedVoice
|
|
6190
|
-
:selection
|
|
6191
|
-
?this.#providerSpeechVoice()
|
|
6192
|
-
:this.#builtInSpeechDefaultVoice('tts',this.ttsService);
|
|
6193
|
-
if(!voice){
|
|
6194
|
-
const error=new TypeError(
|
|
6195
|
-
'AI.fetchTTS requires a caller- or model-catalog-admitted voice.'
|
|
6196
|
-
);
|
|
6197
|
-
error.code='ARCANE_AI_TTS_VOICE_REQUIRED';
|
|
6198
|
-
throw error;
|
|
6199
|
-
}
|
|
6200
|
-
const requestedResponseFormat=descriptors.responseFormat?.value;
|
|
6201
|
-
if(requestedResponseFormat!==undefined
|
|
6202
|
-
&&(typeof requestedResponseFormat!=='string'
|
|
6203
|
-
||requestedResponseFormat.trim()!==requestedResponseFormat
|
|
6204
|
-
||!requestedResponseFormat)){
|
|
6205
|
-
const error=new TypeError(
|
|
6206
|
-
'AI.fetchTTS responseFormat must be a nonempty trimmed string when provided.'
|
|
6207
|
-
);
|
|
6208
|
-
error.code='ARCANE_AI_TTS_RESPONSE_FORMAT_INVALID';
|
|
6209
|
-
throw error;
|
|
6210
|
-
}
|
|
6211
|
-
const responseFormat=requestedResponseFormat
|
|
6212
|
-
||(selection?this.#providerSpeechResponseFormat():this.audioFormat)
|
|
6213
|
-
||'';
|
|
6214
|
-
if(!responseFormat){
|
|
6215
|
-
const error=new TypeError('AI.fetchTTS responseFormat must be nonempty.');
|
|
6216
|
-
error.code='ARCANE_AI_TTS_RESPONSE_FORMAT_INVALID';
|
|
6217
|
-
throw error;
|
|
6218
|
-
}
|
|
6219
|
-
const speed=descriptors.speed
|
|
6220
|
-
?Number(descriptors.speed.value)
|
|
6221
|
-
:this.voiceSpeed;
|
|
6222
|
-
if(!Number.isFinite(speed)||speed<=0){
|
|
6223
|
-
const error=new RangeError('AI.fetchTTS speed must be a positive number.');
|
|
6224
|
-
error.code='ARCANE_AI_TTS_SPEED_INVALID';
|
|
6225
|
-
throw error;
|
|
6226
|
-
}
|
|
6303
|
+
if(selection){
|
|
6304
|
+
this.#traceSpeech('fetchTTS.dispatch',{
|
|
6305
|
+
callId,selection,payload:{model,voice,input,responseFormat,speed}
|
|
6306
|
+
});
|
|
6307
|
+
const response=await this.#providerRuntime.request(
|
|
6308
|
+
'tts',
|
|
6309
|
+
{
|
|
6310
|
+
operation:'synthesize',
|
|
6311
|
+
payload:{model,voice,input,responseFormat,speed},
|
|
6312
|
+
localOnly:false,
|
|
6313
|
+
signal
|
|
6314
|
+
}
|
|
6315
|
+
);
|
|
6316
|
+
this.#traceSpeech('fetchTTS.providerResult',{callId,response});
|
|
6317
|
+
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
6318
|
+
const audio=await this.#normalizeProviderSpeechBlob(response);
|
|
6319
|
+
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
6320
|
+
this.#traceSpeech('fetchTTS.result',{callId,audio});
|
|
6321
|
+
return audio;
|
|
6322
|
+
}
|
|
6227
6323
|
|
|
6228
|
-
|
|
6229
|
-
|
|
6230
|
-
|
|
6231
|
-
|
|
6232
|
-
|
|
6233
|
-
|
|
6234
|
-
localOnly:false,
|
|
6235
|
-
signal
|
|
6236
|
-
}
|
|
6324
|
+
this.#traceSpeech('fetchTTS.dispatch',{
|
|
6325
|
+
callId,service:this.ttsService,payload:{model,voice,input,responseFormat,speed}
|
|
6326
|
+
});
|
|
6327
|
+
const audio=await this.#requestBuiltInSpeechSynthesis(
|
|
6328
|
+
{model,voice,input,responseFormat,speed},
|
|
6329
|
+
signal
|
|
6237
6330
|
);
|
|
6238
|
-
|
|
6239
|
-
const audio=await this.#normalizeProviderSpeechBlob(response);
|
|
6240
|
-
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
6331
|
+
this.#traceSpeech('fetchTTS.result',{callId,audio});
|
|
6241
6332
|
return audio;
|
|
6333
|
+
}catch(error){
|
|
6334
|
+
this.#traceSpeech('fetchTTS.error',{callId,error});
|
|
6335
|
+
throw error;
|
|
6242
6336
|
}
|
|
6243
|
-
|
|
6244
|
-
return this.#requestBuiltInSpeechSynthesis(
|
|
6245
|
-
{model,voice,input,responseFormat,speed},
|
|
6246
|
-
signal
|
|
6247
|
-
);
|
|
6248
6337
|
}
|
|
6249
6338
|
|
|
6250
6339
|
async fetchSTT(audioFile,signal=null){
|
|
6251
|
-
this.#
|
|
6252
|
-
|
|
6253
|
-
|
|
6254
|
-
|
|
6255
|
-
|
|
6256
|
-
|
|
6257
|
-
|
|
6258
|
-
|
|
6259
|
-
|
|
6260
|
-
|
|
6261
|
-
|
|
6262
|
-
|
|
6263
|
-
|
|
6340
|
+
const callId=this.#traceSpeech('fetchSTT.call',{audioFile,aborted:signal?.aborted});
|
|
6341
|
+
try{
|
|
6342
|
+
this.#assertServiceConfigured(this.sttService,'stt');
|
|
6343
|
+
if(signal&&(
|
|
6344
|
+
typeof signal.aborted!=='boolean'
|
|
6345
|
+
||typeof signal.addEventListener!=='function'
|
|
6346
|
+
||typeof signal.removeEventListener!=='function'
|
|
6347
|
+
)){
|
|
6348
|
+
const error=new TypeError('AI.fetchSTT signal must be an AbortSignal.');
|
|
6349
|
+
error.code='ARCANE_AI_STT_SIGNAL_INVALID';
|
|
6350
|
+
throw error;
|
|
6351
|
+
}
|
|
6352
|
+
if(signal?.aborted){
|
|
6353
|
+
throw normalizeAIRequestAbort(signal.reason);
|
|
6354
|
+
}
|
|
6264
6355
|
|
|
6265
|
-
|
|
6266
|
-
|
|
6267
|
-
|
|
6268
|
-
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6276
|
-
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
);
|
|
6282
|
-
const text=typeof response==='string'
|
|
6283
|
-
?response
|
|
6284
|
-
:response?.text;
|
|
6285
|
-
if(typeof text!=='string'){
|
|
6286
|
-
const error=new TypeError(
|
|
6287
|
-
'Arcane returned an invalid provider speech transcription.'
|
|
6356
|
+
if(this.#usesProviderRuntime('stt',this.sttService)){
|
|
6357
|
+
const response=await this.#providerRuntime.request(
|
|
6358
|
+
'stt',
|
|
6359
|
+
{
|
|
6360
|
+
operation:'transcribe',
|
|
6361
|
+
payload:{
|
|
6362
|
+
audio:audioFile,
|
|
6363
|
+
mimeType:typeof Blob==='function'
|
|
6364
|
+
&&audioFile instanceof Blob
|
|
6365
|
+
?String(audioFile.type||'audio/webm')
|
|
6366
|
+
:'audio/webm',
|
|
6367
|
+
model:this.#providerRuntime.selection('stt')?.modelId
|
|
6368
|
+
},
|
|
6369
|
+
localOnly:false,
|
|
6370
|
+
signal
|
|
6371
|
+
}
|
|
6288
6372
|
);
|
|
6289
|
-
|
|
6290
|
-
|
|
6373
|
+
this.#traceSpeech('fetchSTT.providerResult',{callId,response});
|
|
6374
|
+
const text=typeof response==='string'
|
|
6375
|
+
?response
|
|
6376
|
+
:response?.text;
|
|
6377
|
+
if(typeof text!=='string'){
|
|
6378
|
+
const error=new TypeError(
|
|
6379
|
+
'Arcane returned an invalid provider speech transcription.'
|
|
6380
|
+
);
|
|
6381
|
+
error.code='ARCANE_AI_STT_PROVIDER_TRANSCRIPT_INVALID';
|
|
6382
|
+
throw error;
|
|
6383
|
+
}
|
|
6384
|
+
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
6385
|
+
this.#traceSpeech('fetchSTT.result',{callId,text});
|
|
6386
|
+
return text;
|
|
6291
6387
|
}
|
|
6388
|
+
|
|
6389
|
+
const text=await this.#requestBuiltInSpeechTranscription(
|
|
6390
|
+
{
|
|
6391
|
+
audio:audioFile,
|
|
6392
|
+
mimeType:String(audioFile?.type||'audio/webm'),
|
|
6393
|
+
model:this.modelSTT
|
|
6394
|
+
},
|
|
6395
|
+
signal
|
|
6396
|
+
);
|
|
6292
6397
|
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
6398
|
+
this.#traceSpeech('fetchSTT.result',{callId,text});
|
|
6293
6399
|
return text;
|
|
6400
|
+
}catch(error){
|
|
6401
|
+
this.#traceSpeech('fetchSTT.error',{callId,error});
|
|
6402
|
+
throw error;
|
|
6294
6403
|
}
|
|
6295
|
-
|
|
6296
|
-
const text=await this.#requestBuiltInSpeechTranscription(
|
|
6297
|
-
{
|
|
6298
|
-
audio:audioFile,
|
|
6299
|
-
mimeType:String(audioFile?.type||'audio/webm'),
|
|
6300
|
-
model:this.modelSTT
|
|
6301
|
-
},
|
|
6302
|
-
signal
|
|
6303
|
-
);
|
|
6304
|
-
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
6305
|
-
return text;
|
|
6306
6404
|
}
|
|
6307
6405
|
|
|
6308
6406
|
stopAudio(){
|
|
6407
|
+
const callId=this.#traceSpeech('stopAudio.call',{
|
|
6408
|
+
remainder:this.audioMessageChunks,queuedJobs:this.speechJobs.length
|
|
6409
|
+
});
|
|
6309
6410
|
this.speechGeneration+=1;
|
|
6310
6411
|
this.speechResumeAttempt+=1;
|
|
6311
6412
|
this.speechResumePending=false;
|
|
@@ -6318,6 +6419,7 @@ class AI {
|
|
|
6318
6419
|
for(const job of this.speechJobs){
|
|
6319
6420
|
job.abortController?.abort();
|
|
6320
6421
|
job.state='cancelled';
|
|
6422
|
+
this.#traceSpeechJob('queue.cancelled',job,{reason:'stopAudio',callId});
|
|
6321
6423
|
job.resolvePlayback?.(false);
|
|
6322
6424
|
job.resolvePlayback=null;
|
|
6323
6425
|
|
|
@@ -6333,7 +6435,7 @@ class AI {
|
|
|
6333
6435
|
try{
|
|
6334
6436
|
sourceNode.stop();
|
|
6335
6437
|
}catch(error){
|
|
6336
|
-
|
|
6438
|
+
arcaneLogging.warn('AI audio could not be stopped cleanly.',error);
|
|
6337
6439
|
}
|
|
6338
6440
|
}
|
|
6339
6441
|
|
|
@@ -6344,11 +6446,16 @@ class AI {
|
|
|
6344
6446
|
this.sourceNodes.splice(0);
|
|
6345
6447
|
this.currentSpeechJob=null;
|
|
6346
6448
|
this.isSpeaking=false;
|
|
6449
|
+
this.#traceSpeech('stopAudio.result',{callId,result:true});
|
|
6347
6450
|
return true;
|
|
6348
6451
|
}
|
|
6349
6452
|
|
|
6350
6453
|
async resumeAudio(audioContext=null,fromUserGesture=true){
|
|
6454
|
+
const callId=this.#traceSpeech('resumeAudio.call',{
|
|
6455
|
+
fromUserGesture,audioContextState:audioContext?.state
|
|
6456
|
+
});
|
|
6351
6457
|
if(this.muted){
|
|
6458
|
+
this.#traceSpeech('resumeAudio.result',{callId,result:false,reason:'muted'});
|
|
6352
6459
|
return false;
|
|
6353
6460
|
}
|
|
6354
6461
|
|
|
@@ -6361,23 +6468,37 @@ class AI {
|
|
|
6361
6468
|
|
|
6362
6469
|
try{
|
|
6363
6470
|
context=audioContext||this.#getSpeechAudioContext();
|
|
6471
|
+
this.#traceSpeech('resumeAudio.context',{
|
|
6472
|
+
callId,state:context.state,audioTime:context.currentTime,
|
|
6473
|
+
sampleRate:context.sampleRate
|
|
6474
|
+
});
|
|
6364
6475
|
|
|
6365
6476
|
if(context.state==='running'){
|
|
6366
6477
|
this.#clearSpeechUnlock();
|
|
6367
6478
|
this.#requestSpeechPlayback();
|
|
6479
|
+
this.#traceSpeech('resumeAudio.result',{callId,result:true});
|
|
6368
6480
|
return true;
|
|
6369
6481
|
}
|
|
6370
6482
|
|
|
6371
6483
|
if(typeof context.resume!=='function'){
|
|
6372
6484
|
this.#waitForSpeechGesture(null,context);
|
|
6485
|
+
this.#traceSpeech('resumeAudio.result',{
|
|
6486
|
+
callId,result:false,reason:'resume-unavailable'
|
|
6487
|
+
});
|
|
6373
6488
|
return false;
|
|
6374
6489
|
}
|
|
6375
6490
|
|
|
6376
6491
|
attempt=++this.speechResumeAttempt;
|
|
6377
6492
|
this.speechResumePending=true;
|
|
6378
6493
|
await context.resume();
|
|
6494
|
+
this.#traceSpeech('resumeAudio.resumed',{
|
|
6495
|
+
callId,state:context.state,audioTime:context.currentTime
|
|
6496
|
+
});
|
|
6379
6497
|
|
|
6380
6498
|
if(attempt!==this.speechResumeAttempt){
|
|
6499
|
+
this.#traceSpeech('resumeAudio.result',{
|
|
6500
|
+
callId,result:context.state==='running',reason:'superseded'
|
|
6501
|
+
});
|
|
6381
6502
|
return context.state==='running';
|
|
6382
6503
|
}
|
|
6383
6504
|
|
|
@@ -6386,9 +6507,11 @@ class AI {
|
|
|
6386
6507
|
if(context.state==='running'){
|
|
6387
6508
|
this.#clearSpeechUnlock();
|
|
6388
6509
|
this.#requestSpeechPlayback();
|
|
6510
|
+
this.#traceSpeech('resumeAudio.result',{callId,result:true});
|
|
6389
6511
|
return true;
|
|
6390
6512
|
}
|
|
6391
6513
|
}catch(error){
|
|
6514
|
+
this.#traceSpeech('resumeAudio.error',{callId,error,state:context?.state});
|
|
6392
6515
|
if(attempt&&attempt!==this.speechResumeAttempt){
|
|
6393
6516
|
return context?.state==='running';
|
|
6394
6517
|
}
|
|
@@ -6423,6 +6546,9 @@ class AI {
|
|
|
6423
6546
|
}
|
|
6424
6547
|
|
|
6425
6548
|
this.#waitForSpeechGesture(null,context);
|
|
6549
|
+
this.#traceSpeech('resumeAudio.result',{
|
|
6550
|
+
callId,result:false,reason:'waiting-for-gesture',state:context?.state
|
|
6551
|
+
});
|
|
6426
6552
|
return false;
|
|
6427
6553
|
}
|
|
6428
6554
|
|
|
@@ -6433,7 +6559,11 @@ class AI {
|
|
|
6433
6559
|
audioType=this.audioType,
|
|
6434
6560
|
speechJob=null
|
|
6435
6561
|
){
|
|
6562
|
+
this.#traceSpeech('playAudio.call',{
|
|
6563
|
+
audioChunks,audioType,jobId:speechJob?.diagnosticId
|
|
6564
|
+
});
|
|
6436
6565
|
const job=speechJob||{
|
|
6566
|
+
diagnosticId:++this.#speechJobSequence,
|
|
6437
6567
|
abortController:null,
|
|
6438
6568
|
audioBuffer:null,
|
|
6439
6569
|
audioContext:null,
|
|
@@ -6461,6 +6591,10 @@ class AI {
|
|
|
6461
6591
|
const audioBlob=new Blob(audioChunks,{type:audioType});
|
|
6462
6592
|
const arrayBuffer=await audioBlob.arrayBuffer();
|
|
6463
6593
|
const audioBuffer=await playbackContext.decodeAudioData(arrayBuffer);
|
|
6594
|
+
this.#traceSpeechJob('decode.result',job,{
|
|
6595
|
+
duration:audioBuffer.duration,sampleRate:audioBuffer.sampleRate,
|
|
6596
|
+
channels:audioBuffer.numberOfChannels,audioBuffer
|
|
6597
|
+
});
|
|
6464
6598
|
|
|
6465
6599
|
if(this.muted||job.generation!==this.speechGeneration){
|
|
6466
6600
|
return this.#cancelSpeechJob(job);
|
|
@@ -6473,12 +6607,14 @@ class AI {
|
|
|
6473
6607
|
preparedSource.connect(playbackContext.destination);
|
|
6474
6608
|
preparedSource.__arcaneStarted=false;
|
|
6475
6609
|
preparedSource.onended=function finishQueuedSpeechSource(){
|
|
6610
|
+
runtime.#traceSpeechJob('playback.ended',job);
|
|
6476
6611
|
runtime.nextSentance(job);
|
|
6477
6612
|
};
|
|
6478
6613
|
job.audioBuffer=audioBuffer;
|
|
6479
6614
|
job.audioContext=preparedSource.context||playbackContext;
|
|
6480
6615
|
job.sourceNode=preparedSource;
|
|
6481
6616
|
job.state='ready';
|
|
6617
|
+
this.#traceSpeechJob('queue.ready',job);
|
|
6482
6618
|
this.sourceNodes.push(preparedSource);
|
|
6483
6619
|
this.#requestSpeechPlayback();
|
|
6484
6620
|
return true;
|
|
@@ -6496,7 +6632,7 @@ class AI {
|
|
|
6496
6632
|
runtime.#failSpeechJob(job,error,'playback-start');
|
|
6497
6633
|
return;
|
|
6498
6634
|
}
|
|
6499
|
-
|
|
6635
|
+
arcaneLogging.error('AI audio playback failed without an active speech job.',error);
|
|
6500
6636
|
}
|
|
6501
6637
|
);
|
|
6502
6638
|
}
|
|
@@ -6593,6 +6729,13 @@ class AI {
|
|
|
6593
6729
|
}
|
|
6594
6730
|
this.isSpeaking=true;
|
|
6595
6731
|
job.sourceNode.start(scheduledStart);
|
|
6732
|
+
this.#traceSpeechJob('playback.scheduled',job,{
|
|
6733
|
+
audioEnd:hasKnownDuration?scheduledStart+duration:null,
|
|
6734
|
+
previousScheduledEnd:this.speechScheduleTime,
|
|
6735
|
+
sameAudioContext:previousContext===audioContext,
|
|
6736
|
+
gapSeconds:previousContext===audioContext
|
|
6737
|
+
?scheduledStart-this.speechScheduleTime:null
|
|
6738
|
+
});
|
|
6596
6739
|
scheduled=true;
|
|
6597
6740
|
this.speechScheduleContext=audioContext;
|
|
6598
6741
|
if(job.scheduledEnd===null){
|
|
@@ -6610,7 +6753,7 @@ class AI {
|
|
|
6610
6753
|
if(activeJob){
|
|
6611
6754
|
this.#failSpeechJob(activeJob,error,'playback-start');
|
|
6612
6755
|
}else if(!this.muted&&!isAIRequestAbort(error)){
|
|
6613
|
-
|
|
6756
|
+
arcaneLogging.error('AI audio playback failed without an active speech job.',error);
|
|
6614
6757
|
}
|
|
6615
6758
|
return false;
|
|
6616
6759
|
}finally{
|
|
@@ -6640,6 +6783,7 @@ class AI {
|
|
|
6640
6783
|
}
|
|
6641
6784
|
|
|
6642
6785
|
nextSentance(job=this.currentSpeechJob){
|
|
6786
|
+
this.#traceSpeech('nextSentance.call',{jobId:job?.diagnosticId});
|
|
6643
6787
|
if(!job||job.generation!==this.speechGeneration){
|
|
6644
6788
|
return false;
|
|
6645
6789
|
}
|
|
@@ -6654,6 +6798,7 @@ class AI {
|
|
|
6654
6798
|
(Number(job.audioContext?.currentTime)||0)+job.pauseAfterMs/1000;
|
|
6655
6799
|
}
|
|
6656
6800
|
job.state='complete';
|
|
6801
|
+
this.#traceSpeechJob('queue.complete',job);
|
|
6657
6802
|
this.#removeSpeechJob(job);
|
|
6658
6803
|
|
|
6659
6804
|
if(this.currentSpeechJob===job){
|
|
@@ -6676,6 +6821,7 @@ class AI {
|
|
|
6676
6821
|
#cancelSpeechJob(job){
|
|
6677
6822
|
job.abortController?.abort();
|
|
6678
6823
|
job.state='cancelled';
|
|
6824
|
+
this.#traceSpeechJob('queue.cancelled',job);
|
|
6679
6825
|
this.#removeSpeechJob(job);
|
|
6680
6826
|
return false;
|
|
6681
6827
|
}
|
|
@@ -6705,7 +6851,7 @@ class AI {
|
|
|
6705
6851
|
const operationId=
|
|
6706
6852
|
`${this.#events.instanceId}:tts-failure:${(++this.#speechFailureSequence).toString(36)}`;
|
|
6707
6853
|
|
|
6708
|
-
|
|
6854
|
+
arcaneLogging.error(`AI speech ${normalizedBoundary} failed.`,error);
|
|
6709
6855
|
|
|
6710
6856
|
try{
|
|
6711
6857
|
const {occurrence}=this.#events.dispatch(
|
|
@@ -6729,7 +6875,7 @@ class AI {
|
|
|
6729
6875
|
);
|
|
6730
6876
|
projectArcaneDOMEvent(window,occurrence);
|
|
6731
6877
|
}catch(reportingError){
|
|
6732
|
-
|
|
6878
|
+
arcaneLogging.error(
|
|
6733
6879
|
'AI speech failure could not be published to the runtime event boundary.',
|
|
6734
6880
|
reportingError
|
|
6735
6881
|
);
|
|
@@ -6744,6 +6890,7 @@ class AI {
|
|
|
6744
6890
|
}
|
|
6745
6891
|
|
|
6746
6892
|
job.state='failed';
|
|
6893
|
+
this.#traceSpeechJob('queue.failed',job,{boundary,error});
|
|
6747
6894
|
|
|
6748
6895
|
this.#publishTTSFailure(error,{
|
|
6749
6896
|
boundary,
|
|
@@ -6795,6 +6942,9 @@ class AI {
|
|
|
6795
6942
|
if(this.speechUnlockHandler){
|
|
6796
6943
|
return false;
|
|
6797
6944
|
}
|
|
6945
|
+
this.#traceSpeech('playback.waitingForGesture',{
|
|
6946
|
+
error,audioContextState:audioContext?.state
|
|
6947
|
+
});
|
|
6798
6948
|
|
|
6799
6949
|
const runtime=this;
|
|
6800
6950
|
const target=window;
|
|
@@ -6817,7 +6967,7 @@ class AI {
|
|
|
6817
6967
|
);
|
|
6818
6968
|
|
|
6819
6969
|
if(error?.name&&error.name!=='NotAllowedError'){
|
|
6820
|
-
|
|
6970
|
+
arcaneLogging.info('AI speech is waiting for audio playback permission.');
|
|
6821
6971
|
}
|
|
6822
6972
|
|
|
6823
6973
|
return true;
|