arcane-os 0.76.0 → 0.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +1 -1
- package/browser-runtime/ai/dbopfs-model-resources.mjs +1 -1
- package/browser-runtime/ai/speech-worker-runtime.mjs +79 -3
- package/docs/reference/ai/browser-speech.md +144 -5
- package/docs/reference/inventory/package-api.json +1 -1
- package/docs/reference/inventory/runtime-modules.json +10 -8
- package/docs/reference/runtime-modules.md +114 -14
- package/package.json +2 -2
- package/runtime/arcane/modules/AI.js +346 -31
- package/runtime/arcane/modules/DBOPFS.js +24 -6
- package/runtime/arcane/modules/PreparedSpeech.js +111 -6
|
@@ -26,7 +26,7 @@ import {
|
|
|
26
26
|
openAIResponseFormat
|
|
27
27
|
} from 'arcane-os/ai/twin-cloud';
|
|
28
28
|
import {MarkdownSpeech,stripSpeechFormatting} from 'arcane-os/speech-text';
|
|
29
|
-
import {prepareSpeech} from './PreparedSpeech.js';
|
|
29
|
+
import {prepareSpeech, readPreparedSpeech} from './PreparedSpeech.js';
|
|
30
30
|
import {createToolTextObserver} from 'arcane-os/ai/tool-text-stream';
|
|
31
31
|
|
|
32
32
|
const completeValue=(value)=>value;
|
|
@@ -51,6 +51,7 @@ credentials='omit';
|
|
|
51
51
|
const BUILT_IN_AI_SERVICES=new Set(['TWIN','OLLAMA','LOCAL_SPEACH']);
|
|
52
52
|
const AI_REASONING_EFFORTS=new Set(['none','low','medium','high','max']);
|
|
53
53
|
export const AI_READY_EVENT='ai-ready';
|
|
54
|
+
export const AI_AUDIO_OUTPUT_STATE_EVENT='ai-audio-output-state';
|
|
54
55
|
const AI_TTS_FAILURE_EVENT='ai-tts-failure';
|
|
55
56
|
export const AI_INITIALIZATION_ERROR_CODES=completeValue({
|
|
56
57
|
userReadyRegistrationCollision:
|
|
@@ -1128,6 +1129,7 @@ class AI {
|
|
|
1128
1129
|
eventTypes:completeValue(
|
|
1129
1130
|
[
|
|
1130
1131
|
AI_READY_EVENT,
|
|
1132
|
+
AI_AUDIO_OUTPUT_STATE_EVENT,
|
|
1131
1133
|
AI_TTS_FAILURE_EVENT,
|
|
1132
1134
|
...Object.values(AI_BROWSER_SPEECH_EVENT_TYPES)
|
|
1133
1135
|
]
|
|
@@ -1183,8 +1185,12 @@ class AI {
|
|
|
1183
1185
|
#stopOllamaReady=null;
|
|
1184
1186
|
#ttsSegmentation={...DEFAULT_TTS_SEGMENTATION};
|
|
1185
1187
|
#preparedSpeechPlaybacks = new Set();
|
|
1188
|
+
#activePreparedSpeechPlayback = null;
|
|
1186
1189
|
#nativeSpeechPlaybacks = new Set();
|
|
1187
1190
|
#speechUnlockContext = null;
|
|
1191
|
+
#speechResumeContext = null;
|
|
1192
|
+
#audioOutput = null;
|
|
1193
|
+
#audioOutputSequence = 0;
|
|
1188
1194
|
|
|
1189
1195
|
#traceSpeech(phase,detail={}){
|
|
1190
1196
|
if(!arcaneLogging.enabled)return null;
|
|
@@ -6335,16 +6341,15 @@ class AI {
|
|
|
6335
6341
|
return result;
|
|
6336
6342
|
}
|
|
6337
6343
|
|
|
6338
|
-
|
|
6344
|
+
#preparedSpeechInput(parts, textFormat, operation) {
|
|
6339
6345
|
if(this.#providerSpeechModel()?.speech?.playback==='native'){
|
|
6340
6346
|
const error=new Error('Native browser speech has no audio file to prepare or store; use prepareTTSPlayback.');
|
|
6341
6347
|
error.code='ARCANE_AI_TTS_AUDIO_EXPORT_UNAVAILABLE';
|
|
6342
6348
|
throw error;
|
|
6343
6349
|
}
|
|
6344
6350
|
if(!is.array(parts)) {
|
|
6345
|
-
throw new TypeError(
|
|
6351
|
+
throw new TypeError(`${operation} parts must be an array.`);
|
|
6346
6352
|
}
|
|
6347
|
-
const runtime = this;
|
|
6348
6353
|
const selected = this.#providerRuntime.selection('tts');
|
|
6349
6354
|
const configuredSpeech = this.browserSpeechConfiguration?.tts;
|
|
6350
6355
|
const configuration = configuredSpeech
|
|
@@ -6361,12 +6366,9 @@ class AI {
|
|
|
6361
6366
|
responseFormat: selected ? this.#providerSpeechResponseFormat() : this.audioFormat,
|
|
6362
6367
|
configuration
|
|
6363
6368
|
};
|
|
6364
|
-
const browserGeneration = this.#browserSpeechGeneration;
|
|
6365
|
-
const controlGeneration = this.#speechControlGeneration;
|
|
6366
6369
|
const defaultVoice = selected
|
|
6367
6370
|
? this.#providerSpeechVoice()
|
|
6368
6371
|
: this.#builtInSpeechDefaultVoice('tts', this.ttsService);
|
|
6369
|
-
const responseFormat = selection.responseFormat;
|
|
6370
6372
|
const segmentation = this.ttsSegmentation;
|
|
6371
6373
|
const segments = [];
|
|
6372
6374
|
for(const part of parts) {
|
|
@@ -6392,6 +6394,33 @@ class AI {
|
|
|
6392
6394
|
);
|
|
6393
6395
|
}
|
|
6394
6396
|
}
|
|
6397
|
+
return {segments, selection, segmentation, selected};
|
|
6398
|
+
}
|
|
6399
|
+
|
|
6400
|
+
async readPreparedTTS({parts, textFormat, storage = null, identity = null, signal = null, onState = null} = {}) {
|
|
6401
|
+
const {segments, selection, segmentation} = this.#preparedSpeechInput(parts, textFormat, 'AI.readPreparedTTS');
|
|
6402
|
+
this.#traceSpeech('readPreparedTTS.call', {parts, segments, selection, segmentation, identity, storage});
|
|
6403
|
+
return readPreparedSpeech(
|
|
6404
|
+
{
|
|
6405
|
+
owner: this,
|
|
6406
|
+
parts: segments,
|
|
6407
|
+
originalParts: parts,
|
|
6408
|
+
selection,
|
|
6409
|
+
segmentation,
|
|
6410
|
+
storage,
|
|
6411
|
+
identity,
|
|
6412
|
+
signal,
|
|
6413
|
+
onState
|
|
6414
|
+
}
|
|
6415
|
+
);
|
|
6416
|
+
}
|
|
6417
|
+
|
|
6418
|
+
prepareTTS({parts, textFormat, storage = null, identity = null, signal = null, onState = null} = {}) {
|
|
6419
|
+
const {segments, selection, segmentation, selected} = this.#preparedSpeechInput(parts, textFormat, 'AI.prepareTTS');
|
|
6420
|
+
const runtime = this;
|
|
6421
|
+
const browserGeneration = this.#browserSpeechGeneration;
|
|
6422
|
+
const controlGeneration = this.#speechControlGeneration;
|
|
6423
|
+
const responseFormat = selection.responseFormat;
|
|
6395
6424
|
this.#traceSpeech('prepareTTS.call', {parts, segments, selection, segmentation, identity, storage});
|
|
6396
6425
|
async function synthesizePreparedSpeech(part, requestSignal) {
|
|
6397
6426
|
function assertPreparedSpeechSelection() {
|
|
@@ -6434,7 +6463,7 @@ class AI {
|
|
|
6434
6463
|
);
|
|
6435
6464
|
}
|
|
6436
6465
|
|
|
6437
|
-
playPreparedTTS(prepared, {signal = null, onState = null} = {}) {
|
|
6466
|
+
playPreparedTTS(prepared, {signal = null, onState = null, audioOutput = null} = {}) {
|
|
6438
6467
|
if(!is.array(prepared?.segments) || !is.function(prepared?.getAudio)) {
|
|
6439
6468
|
throw new TypeError('AI.playPreparedTTS requires a speech preparation handle.');
|
|
6440
6469
|
}
|
|
@@ -6442,9 +6471,19 @@ class AI {
|
|
|
6442
6471
|
if(onState !== null && !is.function(onState)) {
|
|
6443
6472
|
throw new TypeError('Prepared playback onState must be a function when provided.');
|
|
6444
6473
|
}
|
|
6474
|
+
const output = audioOutput === null ? null : this.#audioOutput;
|
|
6475
|
+
if(audioOutput !== null && (output?.handle !== audioOutput
|
|
6476
|
+
|| output.disposing || output.disposed || !output.context
|
|
6477
|
+
|| output.context.state === 'closed')) {
|
|
6478
|
+
throw new TypeError('Prepared playback requires a live audio output from this AI.');
|
|
6479
|
+
}
|
|
6445
6480
|
// One playback lane per AI; detached preparation has its own lifetime.
|
|
6446
6481
|
this.stopAudio();
|
|
6447
|
-
const group = {
|
|
6482
|
+
const group = {
|
|
6483
|
+
jobs: [], context: output?.context ?? null, output,
|
|
6484
|
+
paused: false, stopped: false, state: 'waiting', error: null
|
|
6485
|
+
};
|
|
6486
|
+
this.#activePreparedSpeechPlayback = group;
|
|
6448
6487
|
const completions = [];
|
|
6449
6488
|
const generation = this.speechGeneration;
|
|
6450
6489
|
this.#traceSpeech('playPreparedTTS.call', {segments: prepared.segments});
|
|
@@ -6464,8 +6503,16 @@ class AI {
|
|
|
6464
6503
|
function stopPreparedPlayback() {
|
|
6465
6504
|
if(group.stopped) return false;
|
|
6466
6505
|
group.stopped = true;
|
|
6467
|
-
runtime
|
|
6468
|
-
|
|
6506
|
+
if(runtime.#activePreparedSpeechPlayback === group) {
|
|
6507
|
+
runtime.#activePreparedSpeechPlayback = null;
|
|
6508
|
+
runtime.speechResumeAttempt += 1;
|
|
6509
|
+
runtime.speechResumePending = false;
|
|
6510
|
+
if(runtime.#speechUnlockContext === group.context) runtime.#clearSpeechUnlock();
|
|
6511
|
+
if(runtime.speechScheduleContext === group.context) {
|
|
6512
|
+
runtime.speechScheduleContext = null;
|
|
6513
|
+
runtime.speechScheduleTime = 0;
|
|
6514
|
+
}
|
|
6515
|
+
}
|
|
6469
6516
|
publishPreparedPlaybackState('stopped');
|
|
6470
6517
|
for(const job of group.jobs) {
|
|
6471
6518
|
if(job.state === 'complete') continue;
|
|
@@ -6481,13 +6528,10 @@ class AI {
|
|
|
6481
6528
|
}
|
|
6482
6529
|
runtime.#cancelSpeechJob(job);
|
|
6483
6530
|
}
|
|
6484
|
-
if(runtime.speechScheduleContext === group.context) {
|
|
6485
|
-
runtime.speechScheduleContext = null;
|
|
6486
|
-
runtime.speechScheduleTime = 0;
|
|
6487
|
-
}
|
|
6488
6531
|
runtime.#requestSpeechPlayback();
|
|
6489
6532
|
return true;
|
|
6490
6533
|
}
|
|
6534
|
+
group.stop = stopPreparedPlayback;
|
|
6491
6535
|
const control = {
|
|
6492
6536
|
finished: null,
|
|
6493
6537
|
get state() { return group.state; },
|
|
@@ -6496,9 +6540,23 @@ class AI {
|
|
|
6496
6540
|
if(group.stopped || !group.context || group.context.state === 'closed') return false;
|
|
6497
6541
|
group.paused = true;
|
|
6498
6542
|
if(runtime.#speechUnlockContext === group.context) runtime.#clearSpeechUnlock();
|
|
6499
|
-
|
|
6500
|
-
|
|
6501
|
-
|
|
6543
|
+
let suspension;
|
|
6544
|
+
try {
|
|
6545
|
+
suspension = Promise.resolve(group.context.suspend());
|
|
6546
|
+
if(output) output.suspending = suspension;
|
|
6547
|
+
await suspension;
|
|
6548
|
+
if(group.stopped || generation !== runtime.speechGeneration) return false;
|
|
6549
|
+
publishPreparedPlaybackState('paused');
|
|
6550
|
+
return true;
|
|
6551
|
+
} catch(error) {
|
|
6552
|
+
if(output) {
|
|
6553
|
+
output.reportError(error);
|
|
6554
|
+
if(!group.stopped) group.fail(error);
|
|
6555
|
+
}
|
|
6556
|
+
throw error;
|
|
6557
|
+
} finally {
|
|
6558
|
+
if(output?.suspending === suspension) output.suspending = null;
|
|
6559
|
+
}
|
|
6502
6560
|
},
|
|
6503
6561
|
async resume() {
|
|
6504
6562
|
if(group.stopped || !group.context || group.context.state === 'closed') return false;
|
|
@@ -6558,15 +6616,18 @@ class AI {
|
|
|
6558
6616
|
async function finishPreparedPlayback(results) {
|
|
6559
6617
|
const completed = !group.stopped && results.every(Boolean);
|
|
6560
6618
|
group.stopped = true;
|
|
6619
|
+
if(runtime.#activePreparedSpeechPlayback === group) {
|
|
6620
|
+
runtime.#activePreparedSpeechPlayback = null;
|
|
6621
|
+
if(runtime.#speechUnlockContext === group.context) runtime.#clearSpeechUnlock();
|
|
6622
|
+
if(runtime.speechScheduleContext === group.context) {
|
|
6623
|
+
runtime.speechScheduleContext = null;
|
|
6624
|
+
runtime.speechScheduleTime = 0;
|
|
6625
|
+
}
|
|
6626
|
+
}
|
|
6561
6627
|
publishPreparedPlaybackState(completed ? 'complete' : group.error ? 'error' : 'stopped');
|
|
6562
6628
|
signal?.removeEventListener('abort', stopPreparedPlayback);
|
|
6563
6629
|
runtime.#preparedSpeechPlaybacks.delete(control);
|
|
6564
|
-
if(
|
|
6565
|
-
if(runtime.speechScheduleContext === group.context) {
|
|
6566
|
-
runtime.speechScheduleContext = null;
|
|
6567
|
-
runtime.speechScheduleTime = 0;
|
|
6568
|
-
}
|
|
6569
|
-
if(group.context && group.context.state !== 'closed') {
|
|
6630
|
+
if(!output && group.context && group.context.state !== 'closed') {
|
|
6570
6631
|
try {
|
|
6571
6632
|
await group.context.close();
|
|
6572
6633
|
} catch(error) {
|
|
@@ -6577,17 +6638,25 @@ class AI {
|
|
|
6577
6638
|
return completed;
|
|
6578
6639
|
}
|
|
6579
6640
|
);
|
|
6641
|
+
group.finished = control.finished;
|
|
6580
6642
|
signal?.addEventListener('abort', stopPreparedPlayback, {once: true});
|
|
6581
6643
|
publishPreparedPlaybackState('waiting');
|
|
6644
|
+
if(group.stopped) return control;
|
|
6582
6645
|
if(signal?.aborted) {
|
|
6583
6646
|
stopPreparedPlayback();
|
|
6584
6647
|
} else if(group.jobs.length) {
|
|
6585
6648
|
// Playing saved audio does not require activating a synthesis provider.
|
|
6586
6649
|
this.muted = false;
|
|
6587
6650
|
try {
|
|
6588
|
-
|
|
6589
|
-
|
|
6590
|
-
|
|
6651
|
+
if(output) {
|
|
6652
|
+
if(output.disposing || output.disposed || output.context.state === 'closed') {
|
|
6653
|
+
throw new Error('The prepared playback audio output has been disposed.');
|
|
6654
|
+
}
|
|
6655
|
+
} else {
|
|
6656
|
+
const AudioContext = globalThis.AudioContext || globalThis.webkitAudioContext;
|
|
6657
|
+
if(!is.function(AudioContext)) throw new TypeError('Audio playback is unavailable in this browser.');
|
|
6658
|
+
group.context = new AudioContext();
|
|
6659
|
+
}
|
|
6591
6660
|
this.resumeAudio(group.context, true).catch(
|
|
6592
6661
|
function failPreparedSpeechUnlock(error) {
|
|
6593
6662
|
for(const job of group.jobs) runtime.#failSpeechJob(job, error, 'playback-resume');
|
|
@@ -7096,12 +7165,209 @@ class AI {
|
|
|
7096
7165
|
return new Blob(normalized.chunks,{type:normalized.type||this.audioType});
|
|
7097
7166
|
}
|
|
7098
7167
|
|
|
7168
|
+
prepareAudioOutput(){
|
|
7169
|
+
if(this.#audioOutput && !this.#audioOutput.disposed
|
|
7170
|
+
&& this.#audioOutput.context?.state !== 'closed') {
|
|
7171
|
+
return this.#audioOutput.handle;
|
|
7172
|
+
}
|
|
7173
|
+
const runtime = this;
|
|
7174
|
+
const output = {
|
|
7175
|
+
context: null, handle: null, error: null,
|
|
7176
|
+
disposing: false, disposed: false, suspending: null,
|
|
7177
|
+
resumeAttempt: 0, disposal: null, reportError: null, publish: null
|
|
7178
|
+
};
|
|
7179
|
+
const operationId = `${this.#events.instanceId}:audio-output:${++this.#audioOutputSequence}`;
|
|
7180
|
+
const subscriptions = new Set();
|
|
7181
|
+
function snapshot(){
|
|
7182
|
+
const contextState = output.context?.state ?? null;
|
|
7183
|
+
const state = output.disposed ? 'disposed'
|
|
7184
|
+
: output.disposing ? 'disposing'
|
|
7185
|
+
: !output.context ? 'unavailable'
|
|
7186
|
+
: contextState === 'closed' ? 'closed'
|
|
7187
|
+
: output.error?.name === 'NotAllowedError' && contextState !== 'running' ? 'gesture-required'
|
|
7188
|
+
: output.error ? 'error'
|
|
7189
|
+
: contextState === 'running' ? 'ready'
|
|
7190
|
+
: contextState;
|
|
7191
|
+
return {kind: 'web-audio', state, contextState, error: output.error};
|
|
7192
|
+
}
|
|
7193
|
+
function publish(){
|
|
7194
|
+
runtime.#events.dispatch(AI_AUDIO_OUTPUT_STATE_EVENT, snapshot(), {operationId});
|
|
7195
|
+
}
|
|
7196
|
+
output.publish = publish;
|
|
7197
|
+
function reportError(error){
|
|
7198
|
+
output.error = error;
|
|
7199
|
+
publish();
|
|
7200
|
+
if(error?.name !== 'NotAllowedError') {
|
|
7201
|
+
arcaneLogging.error('AI audio output operation failed.', error);
|
|
7202
|
+
}
|
|
7203
|
+
}
|
|
7204
|
+
output.reportError = reportError;
|
|
7205
|
+
function observeContextState(){
|
|
7206
|
+
if(output.context.state === 'running') output.error = null;
|
|
7207
|
+
publish();
|
|
7208
|
+
if(output.context.state === 'closed' && !output.disposing) {
|
|
7209
|
+
output.context.removeEventListener?.('statechange', observeContextState);
|
|
7210
|
+
for(const unsubscribe of [...subscriptions]) unsubscribe();
|
|
7211
|
+
}
|
|
7212
|
+
}
|
|
7213
|
+
function observeRejection(promise, attempt = null){
|
|
7214
|
+
// The owner retains failures even when a caller observes state only.
|
|
7215
|
+
promise.catch(function observeOutputFailure(error){
|
|
7216
|
+
if(attempt === null || (attempt === output.resumeAttempt && !output.disposing && !output.disposed)) {
|
|
7217
|
+
reportError(error);
|
|
7218
|
+
} else {
|
|
7219
|
+
arcaneLogging.error('A retired AI audio output resume failed.', error);
|
|
7220
|
+
}
|
|
7221
|
+
});
|
|
7222
|
+
return promise;
|
|
7223
|
+
}
|
|
7224
|
+
async function resumeOutput(attempt){
|
|
7225
|
+
const context = output.context;
|
|
7226
|
+
if(!context || output.disposing || output.disposed || context.state === 'closed') return false;
|
|
7227
|
+
// A pending suspend targets this exact context; resume must follow it.
|
|
7228
|
+
if(output.suspending) await output.suspending;
|
|
7229
|
+
if(output.disposing || output.disposed || attempt !== output.resumeAttempt) return false;
|
|
7230
|
+
const paused = runtime.speechJobs.some(function hasPausedOutputPlayback(job){
|
|
7231
|
+
return job.preparedPlayback?.context === context && job.preparedPlayback.paused;
|
|
7232
|
+
});
|
|
7233
|
+
if(paused) return false;
|
|
7234
|
+
await context.resume();
|
|
7235
|
+
if(output.disposing || output.disposed || attempt !== output.resumeAttempt) return false;
|
|
7236
|
+
output.error = null;
|
|
7237
|
+
publish();
|
|
7238
|
+
if(context.state !== 'running') return false;
|
|
7239
|
+
if(runtime.#speechUnlockContext === context) runtime.#clearSpeechUnlock();
|
|
7240
|
+
if(runtime.speechJobs.some(function usesResumedOutput(job){
|
|
7241
|
+
return job.audioContext === context || job.preparedPlayback?.context === context;
|
|
7242
|
+
})) runtime.#requestSpeechPlayback();
|
|
7243
|
+
return true;
|
|
7244
|
+
}
|
|
7245
|
+
async function disposeOutput(){
|
|
7246
|
+
const context = output.context;
|
|
7247
|
+
const errors = [];
|
|
7248
|
+
const groups = new Set();
|
|
7249
|
+
output.disposing = true;
|
|
7250
|
+
output.resumeAttempt += 1;
|
|
7251
|
+
publish();
|
|
7252
|
+
if(context) {
|
|
7253
|
+
if(runtime.#speechUnlockContext === context) runtime.#clearSpeechUnlock();
|
|
7254
|
+
if(runtime.#speechResumeContext === context) {
|
|
7255
|
+
runtime.speechResumeAttempt += 1;
|
|
7256
|
+
runtime.speechResumePending = false;
|
|
7257
|
+
runtime.#speechResumeContext = null;
|
|
7258
|
+
}
|
|
7259
|
+
for(const job of [...runtime.speechJobs]) {
|
|
7260
|
+
if(job.preparedPlayback?.context === context) {
|
|
7261
|
+
groups.add(job.preparedPlayback);
|
|
7262
|
+
} else if(job.audioContext === context || job.sourceNode?.context === context
|
|
7263
|
+
|| job.precedingSpeechContext === context) {
|
|
7264
|
+
if(job.sourceNode) {
|
|
7265
|
+
job.sourceNode.onended = null;
|
|
7266
|
+
if(job.sourceNode.__arcaneStarted) {
|
|
7267
|
+
try { job.sourceNode.stop(); }
|
|
7268
|
+
catch(error) { errors.push(error); }
|
|
7269
|
+
}
|
|
7270
|
+
}
|
|
7271
|
+
try { runtime.#cancelSpeechJob(job); }
|
|
7272
|
+
catch(error) { errors.push(error); }
|
|
7273
|
+
}
|
|
7274
|
+
}
|
|
7275
|
+
for(const group of groups) {
|
|
7276
|
+
try { group.stop(); }
|
|
7277
|
+
catch(error) { errors.push(error); }
|
|
7278
|
+
}
|
|
7279
|
+
if(runtime.speechScheduleContext === context) {
|
|
7280
|
+
runtime.speechScheduleContext = null;
|
|
7281
|
+
runtime.speechScheduleTime = 0;
|
|
7282
|
+
}
|
|
7283
|
+
if(runtime.audioContext === context) runtime.audioContext = null;
|
|
7284
|
+
const completions = await Promise.allSettled(
|
|
7285
|
+
[...groups].map(function finishOutputPlayback(group){ return group.finished; })
|
|
7286
|
+
);
|
|
7287
|
+
for(const completion of completions) {
|
|
7288
|
+
if(completion.status === 'rejected') errors.push(completion.reason);
|
|
7289
|
+
}
|
|
7290
|
+
try {
|
|
7291
|
+
if(context.state !== 'closed') await context.close();
|
|
7292
|
+
} catch(error) {
|
|
7293
|
+
errors.push(error);
|
|
7294
|
+
output.disposing = false;
|
|
7295
|
+
output.disposal = null;
|
|
7296
|
+
if(runtime.#audioOutput === output && runtime.audioContext === null) {
|
|
7297
|
+
runtime.audioContext = context;
|
|
7298
|
+
}
|
|
7299
|
+
throw errors.length === 1 ? errors[0] : new AggregateError(errors, 'AI audio output disposal failed.');
|
|
7300
|
+
}
|
|
7301
|
+
context.removeEventListener?.('statechange', observeContextState);
|
|
7302
|
+
}
|
|
7303
|
+
output.disposing = false;
|
|
7304
|
+
output.disposed = true;
|
|
7305
|
+
output.error = !errors.length ? null : errors.length === 1
|
|
7306
|
+
? errors[0] : new AggregateError(errors, 'AI audio output playback cleanup failed.');
|
|
7307
|
+
publish();
|
|
7308
|
+
for(const unsubscribe of [...subscriptions]) unsubscribe();
|
|
7309
|
+
runtime.#requestSpeechPlayback();
|
|
7310
|
+
if(errors.length) throw output.error;
|
|
7311
|
+
return true;
|
|
7312
|
+
}
|
|
7313
|
+
output.handle = {
|
|
7314
|
+
kind: 'web-audio',
|
|
7315
|
+
get state(){ return snapshot().state; },
|
|
7316
|
+
get contextState(){ return output.context?.state ?? null; },
|
|
7317
|
+
get error(){ return output.error; },
|
|
7318
|
+
subscribe(listener){
|
|
7319
|
+
if(!is.function(listener)) throw new TypeError('Audio output subscriber must be a function.');
|
|
7320
|
+
const remove = output.disposed || output.context?.state === 'closed' ? null : runtime.#events.subscribe(
|
|
7321
|
+
AI_AUDIO_OUTPUT_STATE_EVENT,
|
|
7322
|
+
function observeAudioOutput(occurrence){
|
|
7323
|
+
if(occurrence.operationId === operationId) listener(occurrence.detail);
|
|
7324
|
+
}
|
|
7325
|
+
);
|
|
7326
|
+
function unsubscribe(){
|
|
7327
|
+
remove?.();
|
|
7328
|
+
subscriptions.delete(unsubscribe);
|
|
7329
|
+
}
|
|
7330
|
+
if(remove) subscriptions.add(unsubscribe);
|
|
7331
|
+
try { listener(snapshot()); }
|
|
7332
|
+
catch(error) { arcaneLogging.error('AI audio output state observer failed.', error); }
|
|
7333
|
+
return unsubscribe;
|
|
7334
|
+
},
|
|
7335
|
+
resume(){
|
|
7336
|
+
const attempt = ++output.resumeAttempt;
|
|
7337
|
+
return observeRejection(resumeOutput(attempt), attempt);
|
|
7338
|
+
},
|
|
7339
|
+
dispose(){
|
|
7340
|
+
if(output.disposed) return output.disposal ?? Promise.resolve(true);
|
|
7341
|
+
if(!output.disposal) {
|
|
7342
|
+
let resolve;
|
|
7343
|
+
let reject;
|
|
7344
|
+
output.disposal = observeRejection(new Promise(function retainOutputDisposal(accept, decline){
|
|
7345
|
+
resolve = accept;
|
|
7346
|
+
reject = decline;
|
|
7347
|
+
}));
|
|
7348
|
+
disposeOutput().then(resolve, reject);
|
|
7349
|
+
}
|
|
7350
|
+
return output.disposal;
|
|
7351
|
+
}
|
|
7352
|
+
};
|
|
7353
|
+
this.#audioOutput = output;
|
|
7354
|
+
try {
|
|
7355
|
+
output.context = this.#getSpeechAudioContext();
|
|
7356
|
+
output.context.addEventListener?.('statechange', observeContextState);
|
|
7357
|
+
publish();
|
|
7358
|
+
} catch(error) {
|
|
7359
|
+
reportError(error);
|
|
7360
|
+
}
|
|
7361
|
+
return output.handle;
|
|
7362
|
+
}
|
|
7363
|
+
|
|
7099
7364
|
#getSpeechAudioContext(){
|
|
7100
7365
|
if(this.audioContext&&this.audioContext.state!=='closed'){
|
|
7101
7366
|
return this.audioContext;
|
|
7102
7367
|
}
|
|
7103
7368
|
|
|
7104
|
-
const AudioContext=
|
|
7369
|
+
const AudioContext=globalThis.AudioContext||globalThis.webkitAudioContext
|
|
7370
|
+
||window.AudioContext||window.webkitAudioContext;
|
|
7105
7371
|
|
|
7106
7372
|
if(!is.function(AudioContext)){
|
|
7107
7373
|
throw new TypeError('Audio playback is unavailable in this browser.');
|
|
@@ -7396,6 +7662,7 @@ class AI {
|
|
|
7396
7662
|
this.speechGeneration+=1;
|
|
7397
7663
|
this.speechResumeAttempt+=1;
|
|
7398
7664
|
this.speechResumePending=false;
|
|
7665
|
+
this.#speechResumeContext=null;
|
|
7399
7666
|
this.speechScheduleGeneration=this.speechGeneration;
|
|
7400
7667
|
this.speechScheduleContext=null;
|
|
7401
7668
|
this.speechScheduleTime=0;
|
|
@@ -7461,13 +7728,20 @@ class AI {
|
|
|
7461
7728
|
let attempt=0;
|
|
7462
7729
|
let context;
|
|
7463
7730
|
let preparedGroup = null;
|
|
7731
|
+
let output = null;
|
|
7464
7732
|
|
|
7465
7733
|
try{
|
|
7466
7734
|
context=audioContext||this.#getSpeechAudioContext();
|
|
7735
|
+
output = this.#audioOutput?.context === context ? this.#audioOutput : null;
|
|
7736
|
+
if(output?.disposing || output?.disposed) return false;
|
|
7467
7737
|
preparedGroup = this.speechJobs.find(function findPreparedResumeOwner(job) {
|
|
7468
7738
|
return job.preparedPlayback?.context === context;
|
|
7469
7739
|
})?.preparedPlayback ?? null;
|
|
7470
7740
|
if(preparedGroup?.paused || preparedGroup?.stopped) return false;
|
|
7741
|
+
if(output?.suspending) {
|
|
7742
|
+
await output.suspending;
|
|
7743
|
+
if(output.disposing || output.disposed || preparedGroup?.paused || preparedGroup?.stopped) return false;
|
|
7744
|
+
}
|
|
7471
7745
|
this.#traceSpeech('resumeAudio.context',{
|
|
7472
7746
|
callId,state:context.state,audioTime:context.currentTime,
|
|
7473
7747
|
sampleRate:context.sampleRate
|
|
@@ -7490,11 +7764,26 @@ class AI {
|
|
|
7490
7764
|
|
|
7491
7765
|
attempt=++this.speechResumeAttempt;
|
|
7492
7766
|
this.speechResumePending=true;
|
|
7767
|
+
this.#speechResumeContext=context;
|
|
7493
7768
|
await context.resume();
|
|
7769
|
+
if(output?.disposing || output?.disposed) return false;
|
|
7494
7770
|
if(preparedGroup?.stopped) return false;
|
|
7495
7771
|
if(preparedGroup?.paused) {
|
|
7496
|
-
|
|
7497
|
-
|
|
7772
|
+
let suspension;
|
|
7773
|
+
try {
|
|
7774
|
+
suspension = Promise.resolve(context.suspend());
|
|
7775
|
+
if(output) output.suspending = suspension;
|
|
7776
|
+
await suspension;
|
|
7777
|
+
} catch(error) {
|
|
7778
|
+
output?.reportError(error);
|
|
7779
|
+
throw error;
|
|
7780
|
+
} finally {
|
|
7781
|
+
if(output?.suspending === suspension) output.suspending = null;
|
|
7782
|
+
}
|
|
7783
|
+
if(attempt === this.speechResumeAttempt) {
|
|
7784
|
+
this.speechResumePending = false;
|
|
7785
|
+
this.#speechResumeContext = null;
|
|
7786
|
+
}
|
|
7498
7787
|
return false;
|
|
7499
7788
|
}
|
|
7500
7789
|
this.#traceSpeech('resumeAudio.resumed',{
|
|
@@ -7509,6 +7798,11 @@ class AI {
|
|
|
7509
7798
|
}
|
|
7510
7799
|
|
|
7511
7800
|
this.speechResumePending=false;
|
|
7801
|
+
this.#speechResumeContext=null;
|
|
7802
|
+
if(output) {
|
|
7803
|
+
output.error = null;
|
|
7804
|
+
output.publish();
|
|
7805
|
+
}
|
|
7512
7806
|
|
|
7513
7807
|
if(context.state==='running'){
|
|
7514
7808
|
this.#clearSpeechUnlock();
|
|
@@ -7518,6 +7812,7 @@ class AI {
|
|
|
7518
7812
|
}
|
|
7519
7813
|
}catch(error){
|
|
7520
7814
|
this.#traceSpeech('resumeAudio.error',{callId,error,state:context?.state});
|
|
7815
|
+
if(output?.disposing || output?.disposed) return false;
|
|
7521
7816
|
if(preparedGroup?.stopped || preparedGroup?.paused) return false;
|
|
7522
7817
|
if(attempt&&attempt!==this.speechResumeAttempt){
|
|
7523
7818
|
return context?.state==='running';
|
|
@@ -7525,7 +7820,9 @@ class AI {
|
|
|
7525
7820
|
|
|
7526
7821
|
if(attempt===this.speechResumeAttempt){
|
|
7527
7822
|
this.speechResumePending=false;
|
|
7823
|
+
this.#speechResumeContext=null;
|
|
7528
7824
|
}
|
|
7825
|
+
output?.reportError(error);
|
|
7529
7826
|
if(preparedGroup && error?.name !== 'NotAllowedError') {
|
|
7530
7827
|
for(const job of preparedGroup.jobs) this.#failSpeechJob(job, error, 'playback-resume');
|
|
7531
7828
|
return false;
|
|
@@ -7600,6 +7897,9 @@ class AI {
|
|
|
7600
7897
|
const playbackContext=sourceNode?.context
|
|
7601
7898
|
||audioContext
|
|
7602
7899
|
||this.#getSpeechAudioContext();
|
|
7900
|
+
// Retain the actual output before asynchronous decoding so disposal
|
|
7901
|
+
// can cancel this job without cancelling unrelated synthesis.
|
|
7902
|
+
job.audioContext=playbackContext;
|
|
7603
7903
|
const audioBlob=new Blob(audioChunks,{type:audioType});
|
|
7604
7904
|
const arrayBuffer=await audioBlob.arrayBuffer();
|
|
7605
7905
|
const audioBuffer=await playbackContext.decodeAudioData(arrayBuffer);
|
|
@@ -7683,12 +7983,17 @@ class AI {
|
|
|
7683
7983
|
const deadline=this.speechScheduleTime;
|
|
7684
7984
|
if(!context||deadline<=context.currentTime)return Promise.resolve(true);
|
|
7685
7985
|
const signal=job.abortController.signal;
|
|
7986
|
+
job.precedingSpeechContext=context;
|
|
7686
7987
|
return new Promise(function waitForScheduledSpeechGap(resolve,reject){
|
|
7687
7988
|
let timer=null;
|
|
7989
|
+
let settled=false;
|
|
7688
7990
|
function finish(value,error=null){
|
|
7991
|
+
if(settled)return;
|
|
7992
|
+
settled=true;
|
|
7689
7993
|
globalThis.clearTimeout(timer);
|
|
7690
7994
|
signal.removeEventListener('abort',cancel);
|
|
7691
7995
|
context.removeEventListener('statechange',schedule);
|
|
7996
|
+
if(job.precedingSpeechContext===context)job.precedingSpeechContext=null;
|
|
7692
7997
|
if(error)reject(error);
|
|
7693
7998
|
else resolve(value);
|
|
7694
7999
|
}
|
|
@@ -7755,7 +8060,8 @@ class AI {
|
|
|
7755
8060
|
// the next turn, including when it follows buffered audio.
|
|
7756
8061
|
if(index!==0||job.state!=='ready')break;
|
|
7757
8062
|
if(!await this.#waitForPrecedingSpeechClock(job)
|
|
7758
|
-
||job.generation!==this.speechGeneration||this.muted
|
|
8063
|
+
||job.generation!==this.speechGeneration||this.muted
|
|
8064
|
+
||['cancelled','failed'].includes(job.state)){
|
|
7759
8065
|
return this.#cancelSpeechJob(job);
|
|
7760
8066
|
}
|
|
7761
8067
|
job.state='scheduled';
|
|
@@ -7783,6 +8089,10 @@ class AI {
|
|
|
7783
8089
|
const audioContext=job.sourceNode.context||job.audioContext;
|
|
7784
8090
|
|
|
7785
8091
|
if(job.preparedPlayback?.paused) break;
|
|
8092
|
+
if(this.#audioOutput?.context === audioContext && this.#audioOutput.suspending) {
|
|
8093
|
+
this.resumeAudio(audioContext, false);
|
|
8094
|
+
return scheduled;
|
|
8095
|
+
}
|
|
7786
8096
|
|
|
7787
8097
|
if(audioContext.state!=='running'){
|
|
7788
8098
|
this.#waitForSpeechGesture(null,audioContext);
|
|
@@ -7871,6 +8181,7 @@ class AI {
|
|
|
7871
8181
|
return false;
|
|
7872
8182
|
}finally{
|
|
7873
8183
|
this.speechPlaybackStarting=false;
|
|
8184
|
+
const output = this.#audioOutput;
|
|
7874
8185
|
const nextJob=this.speechJobs.find(function findFirstUnscheduledSpeechJob(job){
|
|
7875
8186
|
return job.state!=='scheduled';
|
|
7876
8187
|
});
|
|
@@ -7879,6 +8190,10 @@ class AI {
|
|
|
7879
8190
|
!this.muted
|
|
7880
8191
|
&&!this.speechAwaitingGesture
|
|
7881
8192
|
&&!this.speechResumePending
|
|
8193
|
+
&&!this.speechJobs.some(function hasPendingOutputSuspension(job){
|
|
8194
|
+
return output?.suspending && (job.audioContext === output.context
|
|
8195
|
+
|| job.preparedPlayback?.context === output.context);
|
|
8196
|
+
})
|
|
7882
8197
|
&&!this.speechJobs.some(function hasPausedPreparedPlayback(job) {
|
|
7883
8198
|
return job.preparedPlayback?.paused;
|
|
7884
8199
|
})
|
|
@@ -554,12 +554,13 @@ class DBOPFS {
|
|
|
554
554
|
|
|
555
555
|
/**
|
|
556
556
|
* Gets the directory handle for a table.
|
|
557
|
-
* Creates the table if it does not exist.
|
|
557
|
+
* Creates the table if it does not exist unless create is false.
|
|
558
558
|
*
|
|
559
559
|
* @param {string} tableName
|
|
560
|
+
* @param {{create?:boolean}} options A false create performs an existing-table lookup only.
|
|
560
561
|
* @returns {Promise<FileSystemDirectoryHandle>}
|
|
561
562
|
*/
|
|
562
|
-
async getTableHandle(tableName=''){
|
|
563
|
+
async getTableHandle(tableName='',{create=true}={}){
|
|
563
564
|
if(!this.ready){
|
|
564
565
|
await this.readyPromise;
|
|
565
566
|
}
|
|
@@ -582,6 +583,14 @@ class DBOPFS {
|
|
|
582
583
|
return existingHandle;
|
|
583
584
|
}
|
|
584
585
|
|
|
586
|
+
if(!create){
|
|
587
|
+
// A read-existing lookup must not wait for another caller to create the table.
|
|
588
|
+
return this.#db.getDirectoryHandle(
|
|
589
|
+
directoryName,
|
|
590
|
+
{create:false}
|
|
591
|
+
);
|
|
592
|
+
}
|
|
593
|
+
|
|
585
594
|
if(!this.#tableHandlePromises[registeredTableName]){
|
|
586
595
|
const handlePromise=this.#db.getDirectoryHandle(
|
|
587
596
|
directoryName,
|
|
@@ -736,10 +745,14 @@ class DBOPFS {
|
|
|
736
745
|
*
|
|
737
746
|
* @param {string} tableName
|
|
738
747
|
* @param {string} fileName
|
|
748
|
+
* @param {{createTable?:boolean}} options A false createTable preserves an absent table.
|
|
739
749
|
* @returns {Promise<File>}
|
|
740
750
|
*/
|
|
741
|
-
async readFile(tableName='',fileName=''){
|
|
742
|
-
const table=await this.getTableHandle(
|
|
751
|
+
async readFile(tableName='',fileName='',{createTable=true}={}){
|
|
752
|
+
const table=await this.getTableHandle(
|
|
753
|
+
tableName,
|
|
754
|
+
{create:createTable}
|
|
755
|
+
)
|
|
743
756
|
const handle=await table.getFileHandle(fileName,{create:false})
|
|
744
757
|
|
|
745
758
|
if(is.function(handle.getFile)){
|
|
@@ -818,14 +831,19 @@ class DBOPFS {
|
|
|
818
831
|
* @param {string} tableName
|
|
819
832
|
* @param {string} fileName
|
|
820
833
|
* @param {boolean} force
|
|
834
|
+
* @param {{createTable?:boolean}} options A false createTable preserves an absent table.
|
|
821
835
|
* @returns {Promise<*>}
|
|
822
836
|
*/
|
|
823
|
-
async get(tableName='',fileName='',force=false){
|
|
837
|
+
async get(tableName='',fileName='',force=false,{createTable=true}={}){
|
|
824
838
|
if(force||!this.#tables[tableName]?.[fileName]){
|
|
825
839
|
const cacheState=this.#fileCacheState(tableName,fileName);
|
|
826
840
|
const cacheRevision=cacheState.revision;
|
|
827
841
|
try{
|
|
828
|
-
const file=await this.readFile(
|
|
842
|
+
const file=await this.readFile(
|
|
843
|
+
tableName,
|
|
844
|
+
fileName,
|
|
845
|
+
{createTable}
|
|
846
|
+
)
|
|
829
847
|
const textContent=await file.text()
|
|
830
848
|
|
|
831
849
|
if(!this.#tables[tableName]){
|