arcane-os 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +1 -1
- package/browser-runtime/ai/browser-speech-artifacts.mjs +4 -0
- package/browser-runtime/ai/browser-speech-providers.mjs +1 -1
- package/browser-runtime/ai/browser-speech-synthesis.mjs +568 -0
- package/browser-runtime/ai/browser-speech.mjs +1 -0
- package/browser-runtime/ai/twin-cloud-speech.mjs +148 -55
- package/docs/reference/ai/browser-speech.md +97 -12
- package/docs/reference/runtime-modules.md +34 -5
- package/package.json +1 -1
- package/runtime/arcane/components/speech.html +11 -4
- package/runtime/arcane/modules/AI.js +179 -17
- package/runtime/arcane/modules/SpeechPlayback.js +242 -36
|
@@ -3,7 +3,7 @@ import Queue from '../dependencies/js-queue/queue.js';
|
|
|
3
3
|
|
|
4
4
|
const is = new Is(false);
|
|
5
5
|
const INVOKE_URL = 'https://inference.do-ai.run/v1/async-invoke';
|
|
6
|
-
const
|
|
6
|
+
const RATE_LIMIT_RETRY_DELAY_MS = 1000;
|
|
7
7
|
|
|
8
8
|
function speechError(message, code, cause) {
|
|
9
9
|
const error = cause === undefined
|
|
@@ -36,30 +36,6 @@ function assertIdentifier(value, label) {
|
|
|
36
36
|
}
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
function waitForStatus(delay, signal) {
|
|
40
|
-
assertNotAborted(signal);
|
|
41
|
-
return new Promise(
|
|
42
|
-
function waitForCloudSpeechStatus(resolve, reject) {
|
|
43
|
-
const timer = setTimeout(finishStatusWait, delay);
|
|
44
|
-
function finishStatusWait() {
|
|
45
|
-
signal.removeEventListener('abort', cancelStatusWait);
|
|
46
|
-
resolve();
|
|
47
|
-
}
|
|
48
|
-
function cancelStatusWait() {
|
|
49
|
-
clearTimeout(timer);
|
|
50
|
-
signal.removeEventListener('abort', cancelStatusWait);
|
|
51
|
-
reject(abortedSpeech(signal));
|
|
52
|
-
}
|
|
53
|
-
signal.addEventListener(
|
|
54
|
-
'abort',
|
|
55
|
-
cancelStatusWait,
|
|
56
|
-
{once: true}
|
|
57
|
-
);
|
|
58
|
-
if (signal.aborted) cancelStatusWait();
|
|
59
|
-
}
|
|
60
|
-
);
|
|
61
|
-
}
|
|
62
|
-
|
|
63
39
|
function retryAfterDelay(response) {
|
|
64
40
|
const value = response.headers?.get?.('Retry-After');
|
|
65
41
|
if (!value?.trim()) return null;
|
|
@@ -69,11 +45,11 @@ function retryAfterDelay(response) {
|
|
|
69
45
|
return is.finite(date) ? Math.max(0, date - Date.now()) : null;
|
|
70
46
|
}
|
|
71
47
|
|
|
72
|
-
async function
|
|
73
|
-
if (response.ok) return
|
|
48
|
+
async function readResponseError(response, signal, operation) {
|
|
49
|
+
if (response.ok) return null;
|
|
74
50
|
const detail = await response.text();
|
|
75
51
|
assertNotAborted(signal);
|
|
76
|
-
|
|
52
|
+
return speechError(
|
|
77
53
|
`TWiN Cloud speech ${operation} failed with HTTP ${response.status}${detail ? `: ${detail}` : '.'}`,
|
|
78
54
|
'ARCANE_AI_CLOUD_SPEECH_HTTP_ERROR'
|
|
79
55
|
);
|
|
@@ -83,8 +59,8 @@ async function requireSuccessfulResponse(response, signal, operation) {
|
|
|
83
59
|
* A remote provider/2 TTS adapter. The application selects the model and voice,
|
|
84
60
|
* and getApiKey returns its current credential or the application's existing
|
|
85
61
|
* refresh promise. Credentials never enter status or configuration. The runtime
|
|
86
|
-
* owns
|
|
87
|
-
*
|
|
62
|
+
* owns playback order. This provider queues whole synthesis jobs, submission
|
|
63
|
+
* attempts, and follow-up HTTP/body operations under one shared 429 cooldown.
|
|
88
64
|
*/
|
|
89
65
|
export function createTwinCloudTTSProvider({
|
|
90
66
|
id,
|
|
@@ -115,18 +91,132 @@ export function createTwinCloudTTSProvider({
|
|
|
115
91
|
|
|
116
92
|
const modelId = model.id;
|
|
117
93
|
const defaultVoice = model.defaultVoice;
|
|
94
|
+
const requests = new Set();
|
|
118
95
|
const activeRequests = new Set();
|
|
119
96
|
const credentialWaits = new Set();
|
|
120
97
|
let state = 'unloaded';
|
|
121
98
|
let generation = 0;
|
|
122
99
|
let unloadOperation = null;
|
|
123
100
|
let disposeOperation = null;
|
|
101
|
+
const jobs = new Queue();
|
|
102
|
+
jobs.autoRun = false;
|
|
103
|
+
const submissions = new Queue();
|
|
104
|
+
submissions.autoRun = false;
|
|
124
105
|
const followUps = new Queue();
|
|
125
106
|
followUps.autoRun = false;
|
|
107
|
+
let cooldownUntil = 0;
|
|
108
|
+
let submissionTimer = null;
|
|
126
109
|
let activeFollowUps = 0;
|
|
127
110
|
let nextFollowUpStart = 0;
|
|
128
111
|
let followUpTimer = null;
|
|
129
112
|
|
|
113
|
+
function dispatchJobs() {
|
|
114
|
+
while (state === 'ready' && activeRequests.size < maxConcurrentRequests && jobs.size) {
|
|
115
|
+
jobs.next();
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function queueSynthesis(request, payload, requestGeneration) {
|
|
120
|
+
const signal = request.controller.signal;
|
|
121
|
+
assertNotAborted(signal);
|
|
122
|
+
return new Promise(
|
|
123
|
+
function enqueueCloudSpeechJob(resolve, reject) {
|
|
124
|
+
function cancelQueuedJob() {
|
|
125
|
+
jobs.contents = jobs.contents.filter(
|
|
126
|
+
function keepOtherJob(task) {return task !== startJob;}
|
|
127
|
+
);
|
|
128
|
+
signal.removeEventListener('abort', cancelQueuedJob);
|
|
129
|
+
reject(abortedSpeech(signal));
|
|
130
|
+
dispatchJobs();
|
|
131
|
+
}
|
|
132
|
+
function finishJob() {
|
|
133
|
+
activeRequests.delete(request);
|
|
134
|
+
dispatchJobs();
|
|
135
|
+
}
|
|
136
|
+
function startJob() {
|
|
137
|
+
signal.removeEventListener('abort', cancelQueuedJob);
|
|
138
|
+
activeRequests.add(request);
|
|
139
|
+
// A slot belongs to the entire synthesis, including its
|
|
140
|
+
// accepted remote job, every read, and the complete audio.
|
|
141
|
+
synthesize(payload, request.controller, requestGeneration).then(
|
|
142
|
+
function acceptCloudSpeechJob(audio) {
|
|
143
|
+
finishJob();
|
|
144
|
+
resolve(audio);
|
|
145
|
+
},
|
|
146
|
+
function rejectCloudSpeechJob(error) {
|
|
147
|
+
finishJob();
|
|
148
|
+
reject(error);
|
|
149
|
+
}
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
jobs.add(startJob);
|
|
153
|
+
signal.addEventListener(
|
|
154
|
+
'abort',
|
|
155
|
+
cancelQueuedJob,
|
|
156
|
+
{once: true}
|
|
157
|
+
);
|
|
158
|
+
if (signal.aborted) cancelQueuedJob();
|
|
159
|
+
else dispatchJobs();
|
|
160
|
+
}
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function dispatchSubmissions() {
|
|
165
|
+
clearTimeout(submissionTimer);
|
|
166
|
+
submissionTimer = null;
|
|
167
|
+
if (state !== 'ready' || !submissions.size) return;
|
|
168
|
+
const delay = cooldownUntil - Date.now();
|
|
169
|
+
if (delay > 0) {
|
|
170
|
+
submissionTimer = setTimeout(dispatchSubmissions, Math.min(2147483647, delay));
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
// Each admitted job awaits its current attempt before requeueing, so
|
|
174
|
+
// whole-job capacity also bounds concurrent submission attempts.
|
|
175
|
+
while (state === 'ready' && submissions.size) submissions.next();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function queueSubmission(readResponse, signal) {
|
|
179
|
+
assertNotAborted(signal);
|
|
180
|
+
return new Promise(
|
|
181
|
+
function enqueueCloudSpeechSubmission(resolve, reject) {
|
|
182
|
+
function cancelQueuedSubmission() {
|
|
183
|
+
submissions.contents = submissions.contents.filter(
|
|
184
|
+
function keepOtherSubmission(task) {return task !== startSubmission;}
|
|
185
|
+
);
|
|
186
|
+
signal.removeEventListener('abort', cancelQueuedSubmission);
|
|
187
|
+
reject(abortedSpeech(signal));
|
|
188
|
+
dispatchSubmissions();
|
|
189
|
+
}
|
|
190
|
+
async function consumeSubmission() {
|
|
191
|
+
assertNotAborted(signal);
|
|
192
|
+
return await readResponse();
|
|
193
|
+
}
|
|
194
|
+
function startSubmission() {
|
|
195
|
+
signal.removeEventListener('abort', cancelQueuedSubmission);
|
|
196
|
+
consumeSubmission().then(resolve, reject);
|
|
197
|
+
}
|
|
198
|
+
submissions.add(startSubmission);
|
|
199
|
+
signal.addEventListener(
|
|
200
|
+
'abort',
|
|
201
|
+
cancelQueuedSubmission,
|
|
202
|
+
{once: true}
|
|
203
|
+
);
|
|
204
|
+
if (signal.aborted) cancelQueuedSubmission();
|
|
205
|
+
else dispatchSubmissions();
|
|
206
|
+
}
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function observeRateLimit(response) {
|
|
211
|
+
if (response.status !== 429) return;
|
|
212
|
+
cooldownUntil = Math.max(
|
|
213
|
+
cooldownUntil,
|
|
214
|
+
Date.now() + Math.max(RATE_LIMIT_RETRY_DELAY_MS, retryAfterDelay(response) ?? 0)
|
|
215
|
+
);
|
|
216
|
+
dispatchSubmissions();
|
|
217
|
+
dispatchFollowUps();
|
|
218
|
+
}
|
|
219
|
+
|
|
130
220
|
function dispatchFollowUps() {
|
|
131
221
|
clearTimeout(followUpTimer);
|
|
132
222
|
followUpTimer = null;
|
|
@@ -136,18 +226,20 @@ export function createTwinCloudTTSProvider({
|
|
|
136
226
|
const eligibleIndex = followUps.contents.findIndex(
|
|
137
227
|
function eligibleFollowUp(task) {return task.eligibleAt <= now;}
|
|
138
228
|
);
|
|
139
|
-
|
|
229
|
+
const nextStart = Math.max(nextFollowUpStart, cooldownUntil);
|
|
230
|
+
if (now < nextStart || eligibleIndex === -1) {
|
|
140
231
|
const firstEligibility = followUps.contents.reduce(
|
|
141
232
|
function earliestFollowUp(time, task) {return Math.min(time, task.eligibleAt);},
|
|
142
233
|
Infinity
|
|
143
234
|
);
|
|
144
235
|
// Long Retry-After dates may exceed the platform timer's range.
|
|
145
236
|
// Revisit at that boundary without shortening the job's eligibility.
|
|
146
|
-
followUpTimer = setTimeout(dispatchFollowUps, Math.min(2147483647, Math.max(
|
|
237
|
+
followUpTimer = setTimeout(dispatchFollowUps, Math.min(2147483647, Math.max(nextStart, firstEligibility) - now));
|
|
147
238
|
return;
|
|
148
239
|
}
|
|
149
|
-
//
|
|
150
|
-
//
|
|
240
|
+
// Successful pending responses retain job-local Retry-After timing.
|
|
241
|
+
// HTTP 429 also applies the shared cooldown before any queued start.
|
|
242
|
+
// Queue owns pending tasks in FIFO order among eligible entries.
|
|
151
243
|
if (eligibleIndex > 0) {
|
|
152
244
|
const [eligibleTask] = followUps.contents.splice(eligibleIndex, 1);
|
|
153
245
|
followUps.contents.unshift(eligibleTask);
|
|
@@ -287,7 +379,7 @@ export function createTwinCloudTTSProvider({
|
|
|
287
379
|
localOnly: false,
|
|
288
380
|
state,
|
|
289
381
|
loaded,
|
|
290
|
-
busy:
|
|
382
|
+
busy: requests.size > 0,
|
|
291
383
|
execution: {
|
|
292
384
|
requestedDevice: 'remote',
|
|
293
385
|
selectedDevice: loaded ? 'remote' : null,
|
|
@@ -310,11 +402,12 @@ export function createTwinCloudTTSProvider({
|
|
|
310
402
|
// an identified job's status read can retry a network failure.
|
|
311
403
|
return {error, retryable: options.method === 'GET' && error instanceof TypeError};
|
|
312
404
|
}
|
|
405
|
+
assertNotAborted(signal);
|
|
406
|
+
observeRateLimit(response);
|
|
313
407
|
const nextEligibility = Date.now() + (retryAfterDelay(response) ?? 0);
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
assertNotAborted(signal);
|
|
408
|
+
const error = await readResponseError(response, signal, operation);
|
|
409
|
+
assertNotAborted(signal);
|
|
410
|
+
if (error) {
|
|
318
411
|
return {error, retryable: response.status === 429, eligibleAt: nextEligibility};
|
|
319
412
|
}
|
|
320
413
|
// Parsing errors are not Fetch failures and must not replay a job.
|
|
@@ -325,7 +418,7 @@ export function createTwinCloudTTSProvider({
|
|
|
325
418
|
while (true) {
|
|
326
419
|
assertNotAborted(signal);
|
|
327
420
|
const outcome = options.method === 'POST'
|
|
328
|
-
? await readInferenceResponse
|
|
421
|
+
? await queueSubmission(readInferenceResponse, signal)
|
|
329
422
|
: await queueFollowUp(readInferenceResponse, signal, eligibleAt);
|
|
330
423
|
assertNotAborted(signal);
|
|
331
424
|
if (!outcome.error) return outcome;
|
|
@@ -333,11 +426,6 @@ export function createTwinCloudTTSProvider({
|
|
|
333
426
|
retryError = outcome.error;
|
|
334
427
|
console.warn(`Retrying the cloud speech ${operation} once.`, retryError);
|
|
335
428
|
eligibleAt = outcome.eligibleAt ?? 0;
|
|
336
|
-
if (options.method === 'POST') {
|
|
337
|
-
// A readable submission rejection retains its own bounded retry
|
|
338
|
-
// delay. It never delays another job or enters the GET queue.
|
|
339
|
-
await waitForStatus(Math.max(SUBMISSION_RETRY_DELAY_MS, eligibleAt - Date.now()), signal);
|
|
340
|
-
}
|
|
341
429
|
}
|
|
342
430
|
}
|
|
343
431
|
|
|
@@ -402,7 +490,11 @@ export function createTwinCloudTTSProvider({
|
|
|
402
490
|
const audio = await queueFollowUp(
|
|
403
491
|
async function downloadCloudSpeechAudio() {
|
|
404
492
|
const audioResponse = await fetchImpl(result.output.audio.url, {signal});
|
|
405
|
-
|
|
493
|
+
assertNotAborted(signal);
|
|
494
|
+
observeRateLimit(audioResponse);
|
|
495
|
+
const error = await readResponseError(audioResponse, signal, 'audio download');
|
|
496
|
+
assertNotAborted(signal);
|
|
497
|
+
if (error) throw error;
|
|
406
498
|
return await audioResponse.blob();
|
|
407
499
|
},
|
|
408
500
|
signal,
|
|
@@ -491,9 +583,6 @@ export function createTwinCloudTTSProvider({
|
|
|
491
583
|
if (state !== 'ready' || unloadOperation || disposeOperation) {
|
|
492
584
|
throw speechError('The cloud speech provider is not ready.', 'ARCANE_AI_NOT_READY');
|
|
493
585
|
}
|
|
494
|
-
if (activeRequests.size >= maxConcurrentRequests) {
|
|
495
|
-
throw speechError('The cloud speech provider is at capacity.', 'ARCANE_AI_PROVIDER_BUSY');
|
|
496
|
-
}
|
|
497
586
|
const payload = context.payload;
|
|
498
587
|
if (!is.string(payload?.input) || !payload.input.trim()) {
|
|
499
588
|
throw new TypeError('Cloud speech input must be a nonempty string.');
|
|
@@ -523,16 +612,16 @@ export function createTwinCloudTTSProvider({
|
|
|
523
612
|
);
|
|
524
613
|
if (context.signal?.aborted) cancelCloudSpeechRequest();
|
|
525
614
|
const request = {controller, promise: null};
|
|
526
|
-
|
|
527
|
-
request.promise = synthesize(payload, controller, generation);
|
|
615
|
+
requests.add(request);
|
|
528
616
|
try {
|
|
617
|
+
request.promise = queueSynthesis(request, payload, generation);
|
|
529
618
|
return await request.promise;
|
|
530
619
|
} catch (error) {
|
|
531
620
|
if (controller.signal.aborted) throw abortedSpeech(controller.signal);
|
|
532
621
|
throw error;
|
|
533
622
|
} finally {
|
|
534
623
|
context.signal?.removeEventListener('abort', cancelCloudSpeechRequest);
|
|
535
|
-
|
|
624
|
+
requests.delete(request);
|
|
536
625
|
}
|
|
537
626
|
},
|
|
538
627
|
|
|
@@ -546,11 +635,15 @@ export function createTwinCloudTTSProvider({
|
|
|
546
635
|
if (unloadOperation) return unloadOperation;
|
|
547
636
|
state = 'unloading';
|
|
548
637
|
generation += 1;
|
|
638
|
+
clearTimeout(submissionTimer);
|
|
639
|
+
submissionTimer = null;
|
|
640
|
+
clearTimeout(followUpTimer);
|
|
641
|
+
followUpTimer = null;
|
|
549
642
|
for (const cancelCredentialWait of [...credentialWaits]) cancelCredentialWait();
|
|
550
|
-
const
|
|
551
|
-
for (const request of
|
|
643
|
+
const pendingRequests = [...requests];
|
|
644
|
+
for (const request of pendingRequests) request.controller.abort('Cloud speech provider unloaded.');
|
|
552
645
|
unloadOperation = Promise.allSettled(
|
|
553
|
-
|
|
646
|
+
pendingRequests.map(
|
|
554
647
|
function pendingCloudSpeechRequest(request) {
|
|
555
648
|
return request.promise;
|
|
556
649
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Browser speech providers
|
|
2
2
|
|
|
3
3
|
`arcane-os/ai/browser-speech` supplies caller-selected Whisper or native Web
|
|
4
|
-
Speech recognition, Kokoro text-to-speech, and TWiN Cloud text-to-speech providers. Its local path
|
|
4
|
+
Speech recognition and synthesis, Kokoro text-to-speech, and TWiN Cloud text-to-speech providers. Its local path
|
|
5
5
|
provides artifact storage, live module routing, role Workers, provider/2
|
|
6
6
|
adapters, bounded parallel TTS synthesis, audio normalization, cancellation,
|
|
7
7
|
and cleanup.
|
|
@@ -82,6 +82,78 @@ honestly rather than recording again or silently switching providers. See the
|
|
|
82
82
|
[Web Speech specification](https://webaudio.github.io/web-speech-api/#speechreco-methods)
|
|
83
83
|
for native stop/abort behavior.
|
|
84
84
|
|
|
85
|
+
## Native browser text-to-speech and voice previews
|
|
86
|
+
|
|
87
|
+
`createBrowserSpeechSynthesisProvider({id,model:{id,name?,defaultVoice?},language?})`
|
|
88
|
+
uses the host browser's `speechSynthesis` and `SpeechSynthesisUtterance`.
|
|
89
|
+
The application selects this provider as its default when desired; constructing
|
|
90
|
+
it does not change global preferences, activate Chat speech, or play audio.
|
|
91
|
+
The provider declares `localOnly:false` because a browser voice may use a remote
|
|
92
|
+
service. Individual catalog records expose the browser's `localService` value.
|
|
93
|
+
|
|
94
|
+
```javascript
|
|
95
|
+
import {createBrowserSpeechSynthesisProvider} from 'arcane-os/ai/browser-speech';
|
|
96
|
+
|
|
97
|
+
const voicePreview = createBrowserSpeechSynthesisProvider({
|
|
98
|
+
id: 'browser-preview',
|
|
99
|
+
model: {id: 'browser-voices', name: 'Browser voices'}
|
|
100
|
+
});
|
|
101
|
+
const unsubscribe = voicePreview.subscribeCatalog(function showVoices(models) {
|
|
102
|
+
// Pass these real records to the application's language/voice selector.
|
|
103
|
+
console.log(models[0].voices);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
// Call only from the application's explicit Play action.
|
|
107
|
+
function playIntroduction(input, voice) {
|
|
108
|
+
return voicePreview.play({input, voice});
|
|
109
|
+
}
|
|
110
|
+
// The owning view unsubscribes and disposes voicePreview when it closes.
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
`catalog()` synchronously returns an array of model records. Each model has
|
|
114
|
+
`speech:{playback:'native'}` and `voices:[{id,name,lang,default,localService}]`;
|
|
115
|
+
`id` is the exact native `voiceURI`. `subscribeCatalog(callback)` immediately
|
|
116
|
+
replays that same shape and updates it on `voiceschanged`, returning an
|
|
117
|
+
unsubscribe function. An initially empty inventory is ordinary pending browser
|
|
118
|
+
state. There is no polling, discovery utterance, model download, or page-load
|
|
119
|
+
barrier. Language selectors derive their choices from this actual inventory.
|
|
120
|
+
|
|
121
|
+
`prepare({input,voice?,language?,speed?},{signal?})` returns an inert
|
|
122
|
+
`{kind:'native-speech',input,voice,language,speed,play(options)}` descriptor.
|
|
123
|
+
`play(payload,{signal,onState})` is the explicit-preview convenience. Both
|
|
124
|
+
preserve complete input. Omitted voice uses the configured `model.defaultVoice`
|
|
125
|
+
or the actual browser default; a supplied voice is never replaced with a named
|
|
126
|
+
SDK alias. Language comes from that voice unless explicitly provided. Neither
|
|
127
|
+
API reads the conversation's language preference. A saved voice waits for
|
|
128
|
+
`voiceschanged` if the inventory is empty; a nonempty inventory missing it
|
|
129
|
+
reports `ARCANE_AI_SPEECH_VOICE_UNAVAILABLE`.
|
|
130
|
+
|
|
131
|
+
Playback returns `{finished,released,state,error,pause(),resume(),stop()}`.
|
|
132
|
+
`finished` resolves `true` on actual native end, `false` on stop/abort, and
|
|
133
|
+
rejects genuine native failures. `released` resolves when native resource
|
|
134
|
+
ownership actually ends: a failed pause/resume/cancel can report an error before
|
|
135
|
+
the utterance terminates. `onState({state,error})` observes queued,
|
|
136
|
+
waiting-for-voices, playing, paused, complete, stopped, or error. Keep complete
|
|
137
|
+
errors in diagnostics and ordinary status in the interface.
|
|
138
|
+
|
|
139
|
+
Preview does not depend on AI mute or model activation. Use a separate provider
|
|
140
|
+
instance when the preview should have a lifetime independent of Chat's selected
|
|
141
|
+
provider. Provider unload/dispose stop their own playback, invalidate prepared
|
|
142
|
+
descriptors, and disposal removes catalog listeners. Web Speech pause/resume/cancel
|
|
143
|
+
operate on the native synthesis object globally; the SDK serializes its own
|
|
144
|
+
utterances and does not claim control isolation from unrelated direct native
|
|
145
|
+
callers. See the [native synthesis contract](https://webaudio.github.io/web-speech-api/#tts-section).
|
|
146
|
+
|
|
147
|
+
For Chat, configure this provider with `ai.configureSpeechProvider('tts',provider)`
|
|
148
|
+
and retain the existing explicit mute/unmute lifecycle. `AI.streamTTS()` and
|
|
149
|
+
`SpeechPlayback` use actual utterance completion and cancellation in original
|
|
150
|
+
order. `AI.prepareTTSPlayback(payload,signal)` returns either real Blob audio or
|
|
151
|
+
the native descriptor without speaking; its signal remains attached by default
|
|
152
|
+
when the returned descriptor is played. `AI.fetchTTS()` and durable
|
|
153
|
+
`AI.prepareTTS()` remain audio-file APIs and report
|
|
154
|
+
`ARCANE_AI_TTS_AUDIO_EXPORT_UNAVAILABLE` for native synthesis, which exposes no
|
|
155
|
+
downloadable audio file.
|
|
156
|
+
|
|
85
157
|
<a id="digitalocean-fal-text-to-speech"></a>
|
|
86
158
|
|
|
87
159
|
## TWiN Cloud text-to-speech
|
|
@@ -138,11 +210,13 @@ playback remain the only owners. Use
|
|
|
138
210
|
punctuation boundary should submit promptly. This is incremental playback of
|
|
139
211
|
completed synthesis jobs, not a live audio stream from TWiN Cloud.
|
|
140
212
|
|
|
141
|
-
The
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
213
|
+
The provider queues each complete synthesis job, including initial
|
|
214
|
+
`POST https://inference.do-ai.run/v1/async-invoke` and any eligible submission
|
|
215
|
+
retry. `maxConcurrentRequests` defaults to four whole jobs. Each slot remains
|
|
216
|
+
occupied through credential readiness, submission, queued/running status, and
|
|
217
|
+
the complete final audio response. Additional requests wait in order; cancelling
|
|
218
|
+
a waiting job removes it before credential reading or submission. This capacity
|
|
219
|
+
is separate from `followUpQueue.maxConcurrentRequests`.
|
|
146
220
|
|
|
147
221
|
One provider-local `js-queue` owns only follow-up status/result GETs and the final
|
|
148
222
|
audio GET. `followUpQueue` defaults to `{maxConcurrentRequests:4,intervalMs:250}`;
|
|
@@ -153,20 +227,31 @@ job awaits its current response and body before immediately enqueueing another
|
|
|
153
227
|
request only when incomplete or eligible for a retry. There is no additional
|
|
154
228
|
per-job polling delay.
|
|
155
229
|
|
|
156
|
-
A
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
230
|
+
A successful pending response's `Retry-After` changes that job's next eligibility.
|
|
231
|
+
A `429` applies a shared cooldown to all queued submission and follow-up starts,
|
|
232
|
+
using a one-second minimum and any longer readable `Retry-After`. A completely
|
|
233
|
+
readable `429` permits one retry of that submission or status read, within the
|
|
234
|
+
same whole-job slot. Already dispatched operations retain their response ownership.
|
|
160
235
|
An accepted job's status Fetch network failure may retry once for the same job;
|
|
161
|
-
|
|
236
|
+
Ambiguous POST network, JSON parsing, and error-body read failures never replay
|
|
237
|
+
the submission. Repeated failures surface their
|
|
162
238
|
complete diagnostics without an endless retry loop; a failed job does not cancel
|
|
163
239
|
siblings. Retry diagnostics remain in the developer console, outside conversation
|
|
164
240
|
history.
|
|
165
241
|
|
|
166
|
-
The
|
|
242
|
+
The provider-local queues are created when this cloud factory is called, not when the browser
|
|
167
243
|
speech entrypoint is imported. Local Kokoro and Whisper do not use this queue;
|
|
168
244
|
their setup, Workers, synthesis/transcription, and lifecycle remain independent.
|
|
169
245
|
|
|
246
|
+
Ordinary Kokoro descriptors may supply `model.voices` as complete display records,
|
|
247
|
+
for example `{id:'af_heart',name:'Heart',lang:'en-US'}`. The synchronous
|
|
248
|
+
`provider.catalog()[0].voices` preserves that array and its metadata without
|
|
249
|
+
loading the model or fetching a catalog. The application supplies the inventory
|
|
250
|
+
for its selected runtime; these records neither create a synthesis allow-list
|
|
251
|
+
nor establish that a voice has loaded or played. Ordinary Worker configuration
|
|
252
|
+
does not receive this display inventory. Existing artifact-graph voice paths
|
|
253
|
+
retain their separate meaning.
|
|
254
|
+
|
|
170
255
|
The SDK pins published `js-queue` through npm. Its browser Queue entry, original
|
|
171
256
|
package metadata, and license are projected unchanged by
|
|
172
257
|
`node tools/sync-js-queue.mjs` after an SDK dependency update. The existing runtime
|
|
@@ -160,7 +160,7 @@ read-only `ttsSegmentation`, `configureTTSSegmentation()`,
|
|
|
160
160
|
`streamTTS(text='',end=false,options={})`,
|
|
161
161
|
`prepareTTS({parts,storage,identity,signal,onState})`,
|
|
162
162
|
`playPreparedTTS(prepared,{signal,onState})`,
|
|
163
|
-
`finishTTS()`, `fetchTTS()`, `fetchSTT()`, `stopAudio()`, `resumeAudio()`,
|
|
163
|
+
`finishTTS()`, `prepareTTSPlayback()`, `fetchTTS()`, `fetchSTT()`, `stopAudio()`, `resumeAudio()`,
|
|
164
164
|
`playAudio()`; consumes `user-entity-loaded` and `arcane-ollama-ready`,
|
|
165
165
|
installs `window.ai`, and emits `ai-ready` and `ai-tts-failure`.
|
|
166
166
|
|
|
@@ -374,7 +374,7 @@ marks from a cloned outbound input before delegation; the caller's payload stays
|
|
|
374
374
|
unchanged. The third `preparation` argument is reserved for SDK-internal
|
|
375
375
|
delegation, where `{speechInputPrepared:true}` prevents a second cleanup pass;
|
|
376
376
|
applications omit it. `streamTTS(text='',end=false,options={})` and
|
|
377
|
-
`finishTTS()` use
|
|
377
|
+
`finishTTS()` use the selected playback boundary. The third-argument options below
|
|
378
378
|
are available in SDK `0.5.12`. The `textFormat` compatibility extra added in
|
|
379
379
|
SDK `0.5.16` is ignored beginning in `0.5.17`:
|
|
380
380
|
|
|
@@ -386,7 +386,7 @@ SDK `0.5.16` is ignored beginning in `0.5.17`:
|
|
|
386
386
|
| `waitForPlayback` | `false` | Omission retains the preparation promise. With `true`, the promise resolves after every extracted segment reaches a terminal playback state: `true` when all naturally end, or `false` after terminal cancellation or failure. |
|
|
387
387
|
| `textFormat` | Ignored compatibility extra | Repeated same formatting marks are removed automatically from every TTS call. This value no longer selects or disables cleanup. |
|
|
388
388
|
|
|
389
|
-
|
|
389
|
+
Audio-file voice and speed use the existing `fetchTTS()` validation and error path.
|
|
390
390
|
The automatic cleanup changes only the outbound speech-input copy; displayed,
|
|
391
391
|
stored, model, and caller-owned content remains exact. It is a narrow
|
|
392
392
|
formatting-mark filter, not a full Markdown parser: links, code contents, list
|
|
@@ -405,6 +405,24 @@ for a complete passage. A call extracting no segments resolves
|
|
|
405
405
|
`true` without waiting for earlier jobs; `finishTTS()` remains a preparation
|
|
406
406
|
flush, not a queue-wide playback barrier. A muted call resolves `false`.
|
|
407
407
|
|
|
408
|
+
`prepareTTSPlayback(payload,signal,preparation={})` is the silent playback-oriented
|
|
409
|
+
request. Audio-file providers retain the `fetchTTS()` Blob contract; a selected
|
|
410
|
+
model with `speech.playback:'native'` returns an inert `native-speech` descriptor.
|
|
411
|
+
Its `play({signal,onState})` returns native completion and lifecycle controls.
|
|
412
|
+
The preparation signal remains its default playback signal. Native input stays
|
|
413
|
+
complete; per-request voice/language/speed and the configured model default own
|
|
414
|
+
selection, without reading global user voice or conversation-language fields.
|
|
415
|
+
Native streaming jobs wait for actual end in order and allocate no AudioContext.
|
|
416
|
+
When native speech follows recorded audio, the preceding audio clock still owns
|
|
417
|
+
its requested inter-part pause. Native trailing silence delays the next part,
|
|
418
|
+
not the completed playback promise; cancellation retires either pending gap.
|
|
419
|
+
Failed native control remains owned through its `released` promise, so Stop can
|
|
420
|
+
still reach it and subsequent buffered audio cannot overlap it. Explicit saved
|
|
421
|
+
Blob playback continues to use its own AudioContext even when native speech is
|
|
422
|
+
the currently selected provider. `fetchTTS()` and durable `prepareTTS()` report
|
|
423
|
+
`ARCANE_AI_TTS_AUDIO_EXPORT_UNAVAILABLE` for native selection; no audio file is
|
|
424
|
+
fabricated. See [native voice catalogs and previews](ai/browser-speech.md#native-browser-text-to-speech-and-voice-previews).
|
|
425
|
+
|
|
408
426
|
Playback completion stays pending while the browser waits for an audio-unlock
|
|
409
427
|
gesture or a recoverable resume attempt. If resuming a closed `AudioContext`
|
|
410
428
|
fails, the affected jobs terminate and their playback results settle `false`.
|
|
@@ -3731,8 +3749,10 @@ new SpeechPlayback({
|
|
|
3731
3749
|
})
|
|
3732
3750
|
```
|
|
3733
3751
|
|
|
3734
|
-
`speech`
|
|
3735
|
-
`
|
|
3752
|
+
`speech` may expose `prepareTTSPlayback(payload,signal)` returning a Blob or
|
|
3753
|
+
native descriptor, a native provider's silent `prepare(payload,{signal})`,
|
|
3754
|
+
`fetchTTS(payload, signal)`, or `synthesize(payload, {signal})`.
|
|
3755
|
+
The playback-oriented method is preferred. `SpeechPlayback` also supplies a third
|
|
3736
3756
|
SDK-internal preparation object; existing two-argument clients may ignore it.
|
|
3737
3757
|
`prepare({key,parts,model,voice,responseFormat,
|
|
3738
3758
|
speed,autoplay=true})` uses only caller-supplied model, voice, and response-format
|
|
@@ -3775,6 +3795,15 @@ releases prepared URLs. Replay keeps completed URLs and still-pending provider
|
|
|
3775
3795
|
work, then re-submits only failed missing provider segments before starting
|
|
3776
3796
|
again from index zero.
|
|
3777
3797
|
|
|
3798
|
+
Native descriptors are retained separately from Blob URLs and never assigned to
|
|
3799
|
+
an audio element. `autoplay:false` remains silent. Play, Pause, Resume, Stop and
|
|
3800
|
+
Replay use the native control; only a successful current native `finished`
|
|
3801
|
+
advances the next part. Native errors remain observable, and a failed native
|
|
3802
|
+
control stays owned until its optional `released` promise confirms cleanup.
|
|
3803
|
+
Stale completion after stop/restart/disposal cannot advance replacement content.
|
|
3804
|
+
Selecting a native part hides and clears the HTML audio source while preserving
|
|
3805
|
+
the same shared playback state events and complete original parts.
|
|
3806
|
+
|
|
3778
3807
|
Every preparation owns an operation ID and one AbortController for each active
|
|
3779
3808
|
synthesis segment or playback delay. Replacement,
|
|
3780
3809
|
`stop()`, `cancel()`, and `destroy()` abort their owned signals, suppress stale
|
package/package.json
CHANGED
|
@@ -1912,9 +1912,11 @@
|
|
|
1912
1912
|
].includes(error?.code);
|
|
1913
1913
|
}
|
|
1914
1914
|
|
|
1915
|
-
function reportTranscriptionCancellation(
|
|
1916
|
-
|
|
1917
|
-
|
|
1915
|
+
function reportTranscriptionCancellation(
|
|
1916
|
+
reason,
|
|
1917
|
+
message,
|
|
1918
|
+
operationId = transcriptionOperationId || nextSpeechOperationId('transcription')
|
|
1919
|
+
) {
|
|
1918
1920
|
localStatus = {
|
|
1919
1921
|
message,
|
|
1920
1922
|
tone: ''
|
|
@@ -1940,6 +1942,10 @@
|
|
|
1940
1942
|
}
|
|
1941
1943
|
|
|
1942
1944
|
function cancelSTTOperation(reason) {
|
|
1945
|
+
// A native error can clear the ambient ID before its capture actually ends.
|
|
1946
|
+
const operationId = nativeCaptureSession?.operationId
|
|
1947
|
+
|| captureSession?.operationId
|
|
1948
|
+
|| transcriptionOperationId;
|
|
1943
1949
|
const hadActiveOperation = recordingRequested
|
|
1944
1950
|
|| Boolean(captureSession)
|
|
1945
1951
|
|| Boolean(nativeCaptureSession)
|
|
@@ -1976,7 +1982,8 @@
|
|
|
1976
1982
|
String(reason || 'stt-transcription-cancelled'),
|
|
1977
1983
|
reason === 'component-destroyed'
|
|
1978
1984
|
? 'Transcription stopped because the speech component was destroyed.'
|
|
1979
|
-
: 'Transcription stopped because the selected service became unavailable.'
|
|
1985
|
+
: 'Transcription stopped because the selected service became unavailable.',
|
|
1986
|
+
operationId || undefined
|
|
1980
1987
|
);
|
|
1981
1988
|
}
|
|
1982
1989
|
transcriptionOperationId = null;
|