arcane-os 0.49.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +1 -1
- package/browser-runtime/ai/browser-speech-synthesis.mjs +568 -0
- package/browser-runtime/ai/browser-speech.mjs +1 -0
- package/docs/reference/ai/browser-speech.md +73 -1
- package/docs/reference/runtime-modules.md +34 -5
- package/package.json +1 -1
- package/runtime/arcane/components/speech.html +11 -4
- package/runtime/arcane/modules/AI.js +179 -17
- package/runtime/arcane/modules/SpeechPlayback.js +242 -36
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Browser speech providers
|
|
2
2
|
|
|
3
3
|
`arcane-os/ai/browser-speech` supplies caller-selected Whisper or native Web
|
|
4
|
-
Speech recognition, Kokoro text-to-speech, and TWiN Cloud text-to-speech providers. Its local path
|
|
4
|
+
Speech recognition and synthesis, Kokoro text-to-speech, and TWiN Cloud text-to-speech providers. Its local path
|
|
5
5
|
provides artifact storage, live module routing, role Workers, provider/2
|
|
6
6
|
adapters, bounded parallel TTS synthesis, audio normalization, cancellation,
|
|
7
7
|
and cleanup.
|
|
@@ -82,6 +82,78 @@ honestly rather than recording again or silently switching providers. See the
|
|
|
82
82
|
[Web Speech specification](https://webaudio.github.io/web-speech-api/#speechreco-methods)
|
|
83
83
|
for native stop/abort behavior.
|
|
84
84
|
|
|
85
|
+
## Native browser text-to-speech and voice previews
|
|
86
|
+
|
|
87
|
+
`createBrowserSpeechSynthesisProvider({id,model:{id,name?,defaultVoice?},language?})`
|
|
88
|
+
uses the host browser's `speechSynthesis` and `SpeechSynthesisUtterance`.
|
|
89
|
+
The application selects this provider as its default when desired; constructing
|
|
90
|
+
it does not change global preferences, activate Chat speech, or play audio.
|
|
91
|
+
The provider declares `localOnly:false` because a browser voice may use a remote
|
|
92
|
+
service. Individual catalog records expose the browser's `localService` value.
|
|
93
|
+
|
|
94
|
+
```javascript
|
|
95
|
+
import {createBrowserSpeechSynthesisProvider} from 'arcane-os/ai/browser-speech';
|
|
96
|
+
|
|
97
|
+
const voicePreview = createBrowserSpeechSynthesisProvider({
|
|
98
|
+
id: 'browser-preview',
|
|
99
|
+
model: {id: 'browser-voices', name: 'Browser voices'}
|
|
100
|
+
});
|
|
101
|
+
const unsubscribe = voicePreview.subscribeCatalog(function showVoices(models) {
|
|
102
|
+
// Pass these real records to the application's language/voice selector.
|
|
103
|
+
console.log(models[0].voices);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
// Call only from the application's explicit Play action.
|
|
107
|
+
function playIntroduction(input, voice) {
|
|
108
|
+
return voicePreview.play({input, voice});
|
|
109
|
+
}
|
|
110
|
+
// The owning view unsubscribes and disposes voicePreview when it closes.
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
`catalog()` synchronously returns an array of model records. Each model has
|
|
114
|
+
`speech:{playback:'native'}` and `voices:[{id,name,lang,default,localService}]`;
|
|
115
|
+
`id` is the exact native `voiceURI`. `subscribeCatalog(callback)` immediately
|
|
116
|
+
replays that same shape and updates it on `voiceschanged`, returning an
|
|
117
|
+
unsubscribe function. An initially empty inventory is ordinary pending browser
|
|
118
|
+
state. There is no polling, discovery utterance, model download, or page-load
|
|
119
|
+
barrier. Language selectors derive their choices from this actual inventory.
|
|
120
|
+
|
|
121
|
+
`prepare({input,voice?,language?,speed?},{signal?})` returns an inert
|
|
122
|
+
`{kind:'native-speech',input,voice,language,speed,play(options)}` descriptor.
|
|
123
|
+
`play(payload,{signal,onState})` is the explicit-preview convenience. Both
|
|
124
|
+
preserve complete input. Omitted voice uses the configured `model.defaultVoice`
|
|
125
|
+
or the actual browser default; a supplied voice is never replaced with a named
|
|
126
|
+
SDK alias. Language comes from that voice unless explicitly provided. Neither
|
|
127
|
+
API reads the conversation's language preference. A saved voice waits for
|
|
128
|
+
`voiceschanged` if the inventory is empty; a nonempty inventory missing it
|
|
129
|
+
reports `ARCANE_AI_SPEECH_VOICE_UNAVAILABLE`.
|
|
130
|
+
|
|
131
|
+
Playback returns `{finished,released,state,error,pause(),resume(),stop()}`.
|
|
132
|
+
`finished` resolves `true` on actual native end, `false` on stop/abort, and
|
|
133
|
+
rejects genuine native failures. `released` resolves when native resource
|
|
134
|
+
ownership actually ends: a failed pause/resume/cancel can report an error before
|
|
135
|
+
the utterance terminates. `onState({state,error})` observes queued,
|
|
136
|
+
waiting-for-voices, playing, paused, complete, stopped, or error. Keep complete
|
|
137
|
+
errors in diagnostics and ordinary status in the interface.
|
|
138
|
+
|
|
139
|
+
Preview does not depend on AI mute or model activation. Use a separate provider
|
|
140
|
+
instance when the preview should have a lifetime independent of Chat's selected
|
|
141
|
+
provider. Provider unload/dispose stop their own playback, invalidate prepared
|
|
142
|
+
descriptors, and disposal removes catalog listeners. Web Speech pause/resume/cancel
|
|
143
|
+
operate on the native synthesis object globally; the SDK serializes its own
|
|
144
|
+
utterances and does not claim control isolation from unrelated direct native
|
|
145
|
+
callers. See the [native synthesis contract](https://webaudio.github.io/web-speech-api/#tts-section).
|
|
146
|
+
|
|
147
|
+
For Chat, configure this provider with `ai.configureSpeechProvider('tts',provider)`
|
|
148
|
+
and retain the existing explicit mute/unmute lifecycle. `AI.streamTTS()` and
|
|
149
|
+
`SpeechPlayback` use actual utterance completion and cancellation in original
|
|
150
|
+
order. `AI.prepareTTSPlayback(payload,signal)` returns either real Blob audio or
|
|
151
|
+
the native descriptor without speaking; its signal remains attached by default
|
|
152
|
+
when the returned descriptor is played. `AI.fetchTTS()` and durable
|
|
153
|
+
`AI.prepareTTS()` remain audio-file APIs and report
|
|
154
|
+
`ARCANE_AI_TTS_AUDIO_EXPORT_UNAVAILABLE` for native synthesis, which exposes no
|
|
155
|
+
downloadable audio file.
|
|
156
|
+
|
|
85
157
|
<a id="digitalocean-fal-text-to-speech"></a>
|
|
86
158
|
|
|
87
159
|
## TWiN Cloud text-to-speech
|
|
@@ -160,7 +160,7 @@ read-only `ttsSegmentation`, `configureTTSSegmentation()`,
|
|
|
160
160
|
`streamTTS(text='',end=false,options={})`,
|
|
161
161
|
`prepareTTS({parts,storage,identity,signal,onState})`,
|
|
162
162
|
`playPreparedTTS(prepared,{signal,onState})`,
|
|
163
|
-
`finishTTS()`, `fetchTTS()`, `fetchSTT()`, `stopAudio()`, `resumeAudio()`,
|
|
163
|
+
`finishTTS()`, `prepareTTSPlayback()`, `fetchTTS()`, `fetchSTT()`, `stopAudio()`, `resumeAudio()`,
|
|
164
164
|
`playAudio()`; consumes `user-entity-loaded` and `arcane-ollama-ready`,
|
|
165
165
|
installs `window.ai`, and emits `ai-ready` and `ai-tts-failure`.
|
|
166
166
|
|
|
@@ -374,7 +374,7 @@ marks from a cloned outbound input before delegation; the caller's payload stays
|
|
|
374
374
|
unchanged. The third `preparation` argument is reserved for SDK-internal
|
|
375
375
|
delegation, where `{speechInputPrepared:true}` prevents a second cleanup pass;
|
|
376
376
|
applications omit it. `streamTTS(text='',end=false,options={})` and
|
|
377
|
-
`finishTTS()` use
|
|
377
|
+
`finishTTS()` use the selected playback boundary. The third-argument options below
|
|
378
378
|
are available in SDK `0.5.12`. The `textFormat` compatibility extra added in
|
|
379
379
|
SDK `0.5.16` is ignored beginning in `0.5.17`:
|
|
380
380
|
|
|
@@ -386,7 +386,7 @@ SDK `0.5.16` is ignored beginning in `0.5.17`:
|
|
|
386
386
|
| `waitForPlayback` | `false` | Omission retains the preparation promise. With `true`, the promise resolves after every extracted segment reaches a terminal playback state: `true` when all naturally end, or `false` after terminal cancellation or failure. |
|
|
387
387
|
| `textFormat` | Ignored compatibility extra | Repeated same formatting marks are removed automatically from every TTS call. This value no longer selects or disables cleanup. |
|
|
388
388
|
|
|
389
|
-
|
|
389
|
+
Audio-file voice and speed use the existing `fetchTTS()` validation and error path.
|
|
390
390
|
The automatic cleanup changes only the outbound speech-input copy; displayed,
|
|
391
391
|
stored, model, and caller-owned content remains exact. It is a narrow
|
|
392
392
|
formatting-mark filter, not a full Markdown parser: links, code contents, list
|
|
@@ -405,6 +405,24 @@ for a complete passage. A call extracting no segments resolves
|
|
|
405
405
|
`true` without waiting for earlier jobs; `finishTTS()` remains a preparation
|
|
406
406
|
flush, not a queue-wide playback barrier. A muted call resolves `false`.
|
|
407
407
|
|
|
408
|
+
`prepareTTSPlayback(payload,signal,preparation={})` is the silent playback-oriented
|
|
409
|
+
request. Audio-file providers retain the `fetchTTS()` Blob contract; a selected
|
|
410
|
+
model with `speech.playback:'native'` returns an inert `native-speech` descriptor.
|
|
411
|
+
Its `play({signal,onState})` returns native completion and lifecycle controls.
|
|
412
|
+
The preparation signal remains its default playback signal. Native input stays
|
|
413
|
+
complete; per-request voice/language/speed and the configured model default own
|
|
414
|
+
selection, without reading global user voice or conversation-language fields.
|
|
415
|
+
Native streaming jobs wait for actual end in order and allocate no AudioContext.
|
|
416
|
+
When native speech follows recorded audio, the preceding audio clock still owns
|
|
417
|
+
its requested inter-part pause. Native trailing silence delays the next part,
|
|
418
|
+
not the completed playback promise; cancellation retires either pending gap.
|
|
419
|
+
Failed native control remains owned through its `released` promise, so Stop can
|
|
420
|
+
still reach it and subsequent buffered audio cannot overlap it. Explicit saved
|
|
421
|
+
Blob playback continues to use its own AudioContext even when native speech is
|
|
422
|
+
the currently selected provider. `fetchTTS()` and durable `prepareTTS()` report
|
|
423
|
+
`ARCANE_AI_TTS_AUDIO_EXPORT_UNAVAILABLE` for native selection; no audio file is
|
|
424
|
+
fabricated. See [native voice catalogs and previews](ai/browser-speech.md#native-browser-text-to-speech-and-voice-previews).
|
|
425
|
+
|
|
408
426
|
Playback completion stays pending while the browser waits for an audio-unlock
|
|
409
427
|
gesture or a recoverable resume attempt. If resuming a closed `AudioContext`
|
|
410
428
|
fails, the affected jobs terminate and their playback results settle `false`.
|
|
@@ -3731,8 +3749,10 @@ new SpeechPlayback({
|
|
|
3731
3749
|
})
|
|
3732
3750
|
```
|
|
3733
3751
|
|
|
3734
|
-
`speech`
|
|
3735
|
-
`
|
|
3752
|
+
`speech` may expose `prepareTTSPlayback(payload,signal)` returning a Blob or
|
|
3753
|
+
native descriptor, a native provider's silent `prepare(payload,{signal})`,
|
|
3754
|
+
`fetchTTS(payload, signal)`, or `synthesize(payload, {signal})`.
|
|
3755
|
+
The playback-oriented method is preferred. `SpeechPlayback` also supplies a third
|
|
3736
3756
|
SDK-internal preparation object; existing two-argument clients may ignore it.
|
|
3737
3757
|
`prepare({key,parts,model,voice,responseFormat,
|
|
3738
3758
|
speed,autoplay=true})` uses only caller-supplied model, voice, and response-format
|
|
@@ -3775,6 +3795,15 @@ releases prepared URLs. Replay keeps completed URLs and still-pending provider
|
|
|
3775
3795
|
work, then re-submits only failed missing provider segments before starting
|
|
3776
3796
|
again from index zero.
|
|
3777
3797
|
|
|
3798
|
+
Native descriptors are retained separately from Blob URLs and never assigned to
|
|
3799
|
+
an audio element. `autoplay:false` remains silent. Play, Pause, Resume, Stop and
|
|
3800
|
+
Replay use the native control; only a successful current native `finished`
|
|
3801
|
+
advances the next part. Native errors remain observable, and a failed native
|
|
3802
|
+
control stays owned until its optional `released` promise confirms cleanup.
|
|
3803
|
+
Stale completion after stop/restart/disposal cannot advance replacement content.
|
|
3804
|
+
Selecting a native part hides and clears the HTML audio source while preserving
|
|
3805
|
+
the same shared playback state events and complete original parts.
|
|
3806
|
+
|
|
3778
3807
|
Every preparation owns an operation ID and one AbortController for each active
|
|
3779
3808
|
synthesis segment or playback delay. Replacement,
|
|
3780
3809
|
`stop()`, `cancel()`, and `destroy()` abort their owned signals, suppress stale
|
package/package.json
CHANGED
|
@@ -1912,9 +1912,11 @@
|
|
|
1912
1912
|
].includes(error?.code);
|
|
1913
1913
|
}
|
|
1914
1914
|
|
|
1915
|
-
function reportTranscriptionCancellation(
|
|
1916
|
-
|
|
1917
|
-
|
|
1915
|
+
function reportTranscriptionCancellation(
|
|
1916
|
+
reason,
|
|
1917
|
+
message,
|
|
1918
|
+
operationId = transcriptionOperationId || nextSpeechOperationId('transcription')
|
|
1919
|
+
) {
|
|
1918
1920
|
localStatus = {
|
|
1919
1921
|
message,
|
|
1920
1922
|
tone: ''
|
|
@@ -1940,6 +1942,10 @@
|
|
|
1940
1942
|
}
|
|
1941
1943
|
|
|
1942
1944
|
function cancelSTTOperation(reason) {
|
|
1945
|
+
// A native error can clear the ambient ID before its capture actually ends.
|
|
1946
|
+
const operationId = nativeCaptureSession?.operationId
|
|
1947
|
+
|| captureSession?.operationId
|
|
1948
|
+
|| transcriptionOperationId;
|
|
1943
1949
|
const hadActiveOperation = recordingRequested
|
|
1944
1950
|
|| Boolean(captureSession)
|
|
1945
1951
|
|| Boolean(nativeCaptureSession)
|
|
@@ -1976,7 +1982,8 @@
|
|
|
1976
1982
|
String(reason || 'stt-transcription-cancelled'),
|
|
1977
1983
|
reason === 'component-destroyed'
|
|
1978
1984
|
? 'Transcription stopped because the speech component was destroyed.'
|
|
1979
|
-
: 'Transcription stopped because the selected service became unavailable.'
|
|
1985
|
+
: 'Transcription stopped because the selected service became unavailable.',
|
|
1986
|
+
operationId || undefined
|
|
1980
1987
|
);
|
|
1981
1988
|
}
|
|
1982
1989
|
transcriptionOperationId = null;
|
|
@@ -1184,6 +1184,7 @@ class AI {
|
|
|
1184
1184
|
#ttsSegmentation={...DEFAULT_TTS_SEGMENTATION};
|
|
1185
1185
|
#preparedSpeechReadiness = null;
|
|
1186
1186
|
#preparedSpeechPlaybacks = new Set();
|
|
1187
|
+
#nativeSpeechPlaybacks = new Set();
|
|
1187
1188
|
#speechUnlockContext = null;
|
|
1188
1189
|
|
|
1189
1190
|
#traceSpeech(phase,detail={}){
|
|
@@ -5940,6 +5941,11 @@ class AI {
|
|
|
5940
5941
|
}
|
|
5941
5942
|
|
|
5942
5943
|
prepareTTS({parts, storage = null, identity = null, signal = null, onState = null} = {}) {
|
|
5944
|
+
if(this.#providerSpeechModel()?.speech?.playback==='native'){
|
|
5945
|
+
const error=new Error('Native browser speech has no audio file to prepare or store; use prepareTTSPlayback.');
|
|
5946
|
+
error.code='ARCANE_AI_TTS_AUDIO_EXPORT_UNAVAILABLE';
|
|
5947
|
+
throw error;
|
|
5948
|
+
}
|
|
5943
5949
|
if(!is.array(parts)) {
|
|
5944
5950
|
throw new TypeError('AI.prepareTTS parts must be an array.');
|
|
5945
5951
|
}
|
|
@@ -6460,6 +6466,8 @@ class AI {
|
|
|
6460
6466
|
scheduledStart:null,
|
|
6461
6467
|
sourceNode:null,
|
|
6462
6468
|
state:'queued',
|
|
6469
|
+
nativePlayback:null,
|
|
6470
|
+
nativeControl:null,
|
|
6463
6471
|
text,
|
|
6464
6472
|
voice:options.voice,
|
|
6465
6473
|
speed:options.speed,
|
|
@@ -6530,6 +6538,13 @@ class AI {
|
|
|
6530
6538
|
return this.#cancelSpeechJob(job);
|
|
6531
6539
|
}
|
|
6532
6540
|
|
|
6541
|
+
if(audio?.kind==='native-speech'){
|
|
6542
|
+
job.nativePlayback=audio;
|
|
6543
|
+
job.state='ready';
|
|
6544
|
+
this.#requestSpeechPlayback();
|
|
6545
|
+
return true;
|
|
6546
|
+
}
|
|
6547
|
+
|
|
6533
6548
|
job.state='decoding';
|
|
6534
6549
|
const audioContext=this.#getSpeechAudioContext();
|
|
6535
6550
|
return this.playAudio(
|
|
@@ -6544,6 +6559,7 @@ class AI {
|
|
|
6544
6559
|
async #requestSpeechAudio(job){
|
|
6545
6560
|
job.abortController??=new AbortController();
|
|
6546
6561
|
const selection=this.#providerRuntime.selection('tts');
|
|
6562
|
+
const native=this.#providerSpeechModel()?.speech?.playback==='native';
|
|
6547
6563
|
const voice=job.voice===undefined
|
|
6548
6564
|
?(selection?this.#providerSpeechVoice():null)
|
|
6549
6565
|
:job.voice;
|
|
@@ -6551,21 +6567,23 @@ class AI {
|
|
|
6551
6567
|
model:selection?.modelId||this.modelTTS,
|
|
6552
6568
|
input:job.text,
|
|
6553
6569
|
...(job.voice!==undefined||voice?{voice}:{}),
|
|
6554
|
-
responseFormat:selection
|
|
6570
|
+
...(!native?{responseFormat:selection
|
|
6555
6571
|
?this.#providerSpeechResponseFormat()
|
|
6556
|
-
:this.audioFormat,
|
|
6572
|
+
:this.audioFormat}:{}),
|
|
6557
6573
|
speed:job.speed===undefined?this.voiceSpeed:job.speed
|
|
6558
6574
|
};
|
|
6559
6575
|
this.#traceSpeechJob('generation.request',job,{payload,selection});
|
|
6560
|
-
const response=await this.
|
|
6576
|
+
const response=await this.prepareTTSPlayback(
|
|
6561
6577
|
payload,
|
|
6562
6578
|
job.abortController.signal,
|
|
6563
6579
|
{speechInputPrepared:true}
|
|
6564
6580
|
);
|
|
6565
|
-
return
|
|
6581
|
+
return response?.kind==='native-speech'
|
|
6582
|
+
?response
|
|
6583
|
+
:this.#normalizeProviderSpeechAudio(response);
|
|
6566
6584
|
}
|
|
6567
6585
|
|
|
6568
|
-
#
|
|
6586
|
+
#providerSpeechModel(){
|
|
6569
6587
|
const selection=this.#providerRuntime.selection('tts');
|
|
6570
6588
|
if(!selection){
|
|
6571
6589
|
return null;
|
|
@@ -6573,9 +6591,13 @@ class AI {
|
|
|
6573
6591
|
const provider=this.#providerRuntime.catalog('tts').find(
|
|
6574
6592
|
entry=>entry.providerId===selection.providerId
|
|
6575
6593
|
);
|
|
6576
|
-
|
|
6594
|
+
return provider?.models.find(entry=>entry?.id===selection.modelId)??null;
|
|
6595
|
+
}
|
|
6596
|
+
|
|
6597
|
+
#providerSpeechVoice(){
|
|
6598
|
+
const model=this.#providerSpeechModel();
|
|
6577
6599
|
return is.string(model?.defaultVoice)&&model.defaultVoice.trim()
|
|
6578
|
-
?model.defaultVoice.trim()
|
|
6600
|
+
?model.speech?.playback==='native'?model.defaultVoice:model.defaultVoice.trim()
|
|
6579
6601
|
:null;
|
|
6580
6602
|
}
|
|
6581
6603
|
|
|
@@ -6709,7 +6731,38 @@ class AI {
|
|
|
6709
6731
|
return this.audioContext;
|
|
6710
6732
|
}
|
|
6711
6733
|
|
|
6734
|
+
async prepareTTSPlayback(payload={},signal=null,preparation={}){
|
|
6735
|
+
const model=this.#providerSpeechModel();
|
|
6736
|
+
if(model?.speech?.playback!=='native'){
|
|
6737
|
+
return this.fetchTTS(payload,signal,preparation);
|
|
6738
|
+
}
|
|
6739
|
+
this.#assertServiceConfigured(this.ttsService,'tts');
|
|
6740
|
+
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
6741
|
+
const selection=this.#providerRuntime.selection('tts');
|
|
6742
|
+
const request={
|
|
6743
|
+
...payload,
|
|
6744
|
+
model:payload.model??selection.modelId,
|
|
6745
|
+
voice:payload.voice??this.#providerSpeechVoice(),
|
|
6746
|
+
speed:payload.speed??this.voiceSpeed
|
|
6747
|
+
};
|
|
6748
|
+
const playback=await this.#providerRuntime.request('tts',{
|
|
6749
|
+
operation:'synthesize',payload:request,localOnly:false,signal
|
|
6750
|
+
},{speechInputPrepared:true});
|
|
6751
|
+
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
6752
|
+
return {
|
|
6753
|
+
...playback,
|
|
6754
|
+
play(options={}){
|
|
6755
|
+
return playback.play({...options,signal:options.signal??signal});
|
|
6756
|
+
}
|
|
6757
|
+
};
|
|
6758
|
+
}
|
|
6759
|
+
|
|
6712
6760
|
async fetchTTS(payload={},signal=null,{speechInputPrepared=false}={}){
|
|
6761
|
+
if(this.#providerSpeechModel()?.speech?.playback==='native'){
|
|
6762
|
+
const error=new Error('Native browser speech plays through prepareTTSPlayback and does not export audio.');
|
|
6763
|
+
error.code='ARCANE_AI_TTS_AUDIO_EXPORT_UNAVAILABLE';
|
|
6764
|
+
throw error;
|
|
6765
|
+
}
|
|
6713
6766
|
const callId=this.#traceSpeech('fetchTTS.call',{payload,aborted:signal?.aborted});
|
|
6714
6767
|
try{
|
|
6715
6768
|
this.#assertServiceConfigured(this.ttsService,'tts');
|
|
@@ -6958,6 +7011,7 @@ class AI {
|
|
|
6958
7011
|
remainder:this.audioMessageChunks,queuedJobs:this.speechJobs.length
|
|
6959
7012
|
});
|
|
6960
7013
|
for(const playback of this.#preparedSpeechPlaybacks) playback.stop();
|
|
7014
|
+
for(const playback of this.#nativeSpeechPlaybacks) playback.stop();
|
|
6961
7015
|
this.speechGeneration+=1;
|
|
6962
7016
|
this.speechResumeAttempt+=1;
|
|
6963
7017
|
this.speechResumePending=false;
|
|
@@ -6970,6 +7024,7 @@ class AI {
|
|
|
6970
7024
|
|
|
6971
7025
|
for(const job of this.speechJobs){
|
|
6972
7026
|
job.abortController?.abort();
|
|
7027
|
+
job.nativeControl?.stop();
|
|
6973
7028
|
job.state='cancelled';
|
|
6974
7029
|
this.#traceSpeechJob('queue.cancelled',job,{reason:'stopAudio',callId});
|
|
6975
7030
|
job.resolvePlayback?.(false);
|
|
@@ -6997,7 +7052,7 @@ class AI {
|
|
|
6997
7052
|
this.speechJobs.splice(0);
|
|
6998
7053
|
this.sourceNodes.splice(0);
|
|
6999
7054
|
this.currentSpeechJob=null;
|
|
7000
|
-
this.isSpeaking=
|
|
7055
|
+
this.isSpeaking=this.#nativeSpeechPlaybacks.size>0;
|
|
7001
7056
|
this.#traceSpeech('stopAudio.result',{callId,result:true});
|
|
7002
7057
|
return true;
|
|
7003
7058
|
}
|
|
@@ -7011,6 +7066,13 @@ class AI {
|
|
|
7011
7066
|
return false;
|
|
7012
7067
|
}
|
|
7013
7068
|
|
|
7069
|
+
if(!audioContext&&this.#providerSpeechModel()?.speech?.playback==='native'){
|
|
7070
|
+
const control=this.currentSpeechJob?.nativeControl;
|
|
7071
|
+
if(control&&control.resume()===false)return false;
|
|
7072
|
+
this.#requestSpeechPlayback();
|
|
7073
|
+
return true;
|
|
7074
|
+
}
|
|
7075
|
+
|
|
7014
7076
|
if(fromUserGesture){
|
|
7015
7077
|
this.#clearSpeechUnlock();
|
|
7016
7078
|
}
|
|
@@ -7207,8 +7269,77 @@ class AI {
|
|
|
7207
7269
|
);
|
|
7208
7270
|
}
|
|
7209
7271
|
|
|
7272
|
+
async #completeNativeSpeechJob(job,played){
|
|
7273
|
+
if(job.generation!==this.speechGeneration||job.state!=='scheduled')return;
|
|
7274
|
+
if(!played){
|
|
7275
|
+
this.#cancelSpeechJob(job);
|
|
7276
|
+
this.#requestSpeechPlayback();
|
|
7277
|
+
return;
|
|
7278
|
+
}
|
|
7279
|
+
// Playback completion excludes the trailing gap, just as it does for buffers.
|
|
7280
|
+
job.resolvePlayback?.(true);
|
|
7281
|
+
job.resolvePlayback=null;
|
|
7282
|
+
if(job.pauseAfterMs>0){
|
|
7283
|
+
const signal=job.abortController.signal;
|
|
7284
|
+
await new Promise(function waitForNativeSpeechPause(resolve){
|
|
7285
|
+
let timer=null;
|
|
7286
|
+
function finish(){
|
|
7287
|
+
globalThis.clearTimeout(timer);
|
|
7288
|
+
signal.removeEventListener('abort',finish);
|
|
7289
|
+
resolve();
|
|
7290
|
+
}
|
|
7291
|
+
signal.addEventListener('abort',finish,{once:true});
|
|
7292
|
+
if(signal.aborted){finish();return;}
|
|
7293
|
+
timer=globalThis.setTimeout(finish,job.pauseAfterMs);
|
|
7294
|
+
});
|
|
7295
|
+
}
|
|
7296
|
+
if(job.generation!==this.speechGeneration||job.state!=='scheduled')return;
|
|
7297
|
+
this.nextSentance(job);
|
|
7298
|
+
}
|
|
7299
|
+
|
|
7300
|
+
#waitForPrecedingSpeechClock(job){
|
|
7301
|
+
const context=this.speechScheduleContext;
|
|
7302
|
+
const deadline=this.speechScheduleTime;
|
|
7303
|
+
if(!context||deadline<=context.currentTime)return Promise.resolve(true);
|
|
7304
|
+
const signal=job.abortController.signal;
|
|
7305
|
+
return new Promise(function waitForScheduledSpeechGap(resolve,reject){
|
|
7306
|
+
let timer=null;
|
|
7307
|
+
function finish(value,error=null){
|
|
7308
|
+
globalThis.clearTimeout(timer);
|
|
7309
|
+
signal.removeEventListener('abort',cancel);
|
|
7310
|
+
context.removeEventListener('statechange',schedule);
|
|
7311
|
+
if(error)reject(error);
|
|
7312
|
+
else resolve(value);
|
|
7313
|
+
}
|
|
7314
|
+
function cancel(){finish(false);}
|
|
7315
|
+
function schedule(){
|
|
7316
|
+
globalThis.clearTimeout(timer);
|
|
7317
|
+
if(signal.aborted){cancel();return;}
|
|
7318
|
+
const remaining=deadline-context.currentTime;
|
|
7319
|
+
if(remaining<=0){finish(true);return;}
|
|
7320
|
+
if(context.state==='closed'){
|
|
7321
|
+
finish(false,new Error('The preceding speech audio context closed before its queued pause ended.'));
|
|
7322
|
+
return;
|
|
7323
|
+
}
|
|
7324
|
+
// A suspended audio clock resumes through its owning state event.
|
|
7325
|
+
if(context.state==='running')timer=globalThis.setTimeout(schedule,remaining*1000);
|
|
7326
|
+
}
|
|
7327
|
+
signal.addEventListener('abort',cancel,{once:true});
|
|
7328
|
+
context.addEventListener('statechange',schedule);
|
|
7329
|
+
schedule();
|
|
7330
|
+
});
|
|
7331
|
+
}
|
|
7332
|
+
|
|
7333
|
+
#releaseNativeSpeechPlayback(control){
|
|
7334
|
+
this.#nativeSpeechPlaybacks.delete(control);
|
|
7335
|
+
this.isSpeaking=this.#nativeSpeechPlaybacks.size>0||this.speechJobs.some(
|
|
7336
|
+
function hasPlayingSpeech(job){return job.state==='scheduled';}
|
|
7337
|
+
);
|
|
7338
|
+
this.#requestSpeechPlayback();
|
|
7339
|
+
}
|
|
7340
|
+
|
|
7210
7341
|
async #pumpSpeechPlayback(){
|
|
7211
|
-
if(this.speechPlaybackStarting||this.muted){
|
|
7342
|
+
if(this.speechPlaybackStarting||this.muted||this.#nativeSpeechPlaybacks.size){
|
|
7212
7343
|
return false;
|
|
7213
7344
|
}
|
|
7214
7345
|
|
|
@@ -7238,6 +7369,32 @@ class AI {
|
|
|
7238
7369
|
continue;
|
|
7239
7370
|
}
|
|
7240
7371
|
|
|
7372
|
+
if(job.nativePlayback){
|
|
7373
|
+
// Native speech has no buffer duration: actual completion owns
|
|
7374
|
+
// the next turn, including when it follows buffered audio.
|
|
7375
|
+
if(index!==0||job.state!=='ready')break;
|
|
7376
|
+
if(!await this.#waitForPrecedingSpeechClock(job)
|
|
7377
|
+
||job.generation!==this.speechGeneration||this.muted){
|
|
7378
|
+
return this.#cancelSpeechJob(job);
|
|
7379
|
+
}
|
|
7380
|
+
job.state='scheduled';
|
|
7381
|
+
this.currentSpeechJob=job;
|
|
7382
|
+
this.isSpeaking=true;
|
|
7383
|
+
job.nativeControl=job.nativePlayback.play({signal:job.abortController.signal});
|
|
7384
|
+
const control=job.nativeControl;
|
|
7385
|
+
this.#nativeSpeechPlaybacks.add(control);
|
|
7386
|
+
// A reported native control error need not mean the browser
|
|
7387
|
+
// released its utterance. Keep Stop ownership until it does.
|
|
7388
|
+
Promise.resolve(control.released??control.finished).then(
|
|
7389
|
+
()=>this.#releaseNativeSpeechPlayback(control),
|
|
7390
|
+
()=>this.#releaseNativeSpeechPlayback(control)
|
|
7391
|
+
);
|
|
7392
|
+
job.nativeControl.finished.then(
|
|
7393
|
+
played=>this.#completeNativeSpeechJob(job,played)
|
|
7394
|
+
).catch(error=>this.#failSpeechJob(job,error,'playback-start'));
|
|
7395
|
+
return true;
|
|
7396
|
+
}
|
|
7397
|
+
|
|
7241
7398
|
if(!['ready','scheduled'].includes(job.state)||!job.sourceNode?.buffer){
|
|
7242
7399
|
break;
|
|
7243
7400
|
}
|
|
@@ -7333,6 +7490,9 @@ class AI {
|
|
|
7333
7490
|
return false;
|
|
7334
7491
|
}finally{
|
|
7335
7492
|
this.speechPlaybackStarting=false;
|
|
7493
|
+
const nextJob=this.speechJobs.find(function findFirstUnscheduledSpeechJob(job){
|
|
7494
|
+
return job.state!=='scheduled';
|
|
7495
|
+
});
|
|
7336
7496
|
|
|
7337
7497
|
if(
|
|
7338
7498
|
!this.muted
|
|
@@ -7347,11 +7507,8 @@ class AI {
|
|
|
7347
7507
|
&&job.scheduledEnd===null;
|
|
7348
7508
|
}
|
|
7349
7509
|
)
|
|
7350
|
-
&&
|
|
7351
|
-
|
|
7352
|
-
return job.state!=='scheduled';
|
|
7353
|
-
}
|
|
7354
|
-
)?.state==='ready'
|
|
7510
|
+
&&nextJob?.state==='ready'
|
|
7511
|
+
&&(!nextJob.nativePlayback||this.speechJobs[0]===nextJob)
|
|
7355
7512
|
){
|
|
7356
7513
|
this.#requestSpeechPlayback();
|
|
7357
7514
|
}
|
|
@@ -7370,7 +7527,7 @@ class AI {
|
|
|
7370
7527
|
job.sourceNode.onended=null;
|
|
7371
7528
|
}
|
|
7372
7529
|
|
|
7373
|
-
if(job.scheduledEnd===null&&job.pauseAfterMs>0){
|
|
7530
|
+
if(!job.nativePlayback&&job.scheduledEnd===null&&job.pauseAfterMs>0){
|
|
7374
7531
|
this.speechScheduleContext=job.audioContext;
|
|
7375
7532
|
this.speechScheduleTime=
|
|
7376
7533
|
(Number(job.audioContext?.currentTime)||0)+job.pauseAfterMs/1000;
|
|
@@ -7386,7 +7543,7 @@ class AI {
|
|
|
7386
7543
|
}
|
|
7387
7544
|
)||null;
|
|
7388
7545
|
}
|
|
7389
|
-
this.isSpeaking=this.speechJobs.some(
|
|
7546
|
+
this.isSpeaking=this.#nativeSpeechPlaybacks.size>0||this.speechJobs.some(
|
|
7390
7547
|
function hasScheduledSpeechJob(candidate){
|
|
7391
7548
|
return candidate.state==='scheduled';
|
|
7392
7549
|
}
|
|
@@ -7398,9 +7555,14 @@ class AI {
|
|
|
7398
7555
|
|
|
7399
7556
|
#cancelSpeechJob(job){
|
|
7400
7557
|
job.abortController?.abort();
|
|
7558
|
+
job.nativeControl?.stop();
|
|
7401
7559
|
job.state='cancelled';
|
|
7402
7560
|
this.#traceSpeechJob('queue.cancelled',job);
|
|
7403
7561
|
this.#removeSpeechJob(job);
|
|
7562
|
+
if(this.currentSpeechJob===job)this.currentSpeechJob=null;
|
|
7563
|
+
this.isSpeaking=this.#nativeSpeechPlaybacks.size>0||this.speechJobs.some(function hasActiveSpeech(candidate){
|
|
7564
|
+
return candidate.state==='scheduled';
|
|
7565
|
+
});
|
|
7404
7566
|
return false;
|
|
7405
7567
|
}
|
|
7406
7568
|
|
|
@@ -7489,7 +7651,7 @@ class AI {
|
|
|
7489
7651
|
}
|
|
7490
7652
|
)||null;
|
|
7491
7653
|
}
|
|
7492
|
-
this.isSpeaking=this.speechJobs.some(
|
|
7654
|
+
this.isSpeaking=this.#nativeSpeechPlaybacks.size>0||this.speechJobs.some(
|
|
7493
7655
|
function hasRemainingScheduledSpeechJob(candidate){
|
|
7494
7656
|
return candidate.state==='scheduled';
|
|
7495
7657
|
}
|