arcane-os 0.76.0 → 0.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +1 -1
- package/docs/reference/ai/browser-speech.md +85 -3
- package/docs/reference/inventory/package-api.json +1 -1
- package/docs/reference/inventory/runtime-modules.json +3 -2
- package/docs/reference/runtime-modules.md +36 -3
- package/package.json +2 -2
- package/runtime/arcane/modules/AI.js +316 -24
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.77.0
|
|
4
|
+
|
|
5
|
+
- Add `AI.prepareAudioOutput()` for synchronous, silent preparation of a reusable
|
|
6
|
+
WebAudio output. Observe its actual context state through immediate subscription
|
|
7
|
+
replay and `AI_AUDIO_OUTPUT_STATE_EVENT`; resume and dispose explicitly.
|
|
8
|
+
- Add optional `audioOutput` to `playPreparedTTS()` so prepared narration borrows
|
|
9
|
+
that same output across plays. Completion and stop leave the borrowed context
|
|
10
|
+
open; omission preserves the existing per-playback context lifecycle.
|
|
11
|
+
- Keep replacement, pending suspension, decoding, cancellation and disposal tied
|
|
12
|
+
to their actual output and playback owners. Cleanup failures remain observable
|
|
13
|
+
and failed context closure can be retried. Independent native speech, saved
|
|
14
|
+
audio and detached preparation retain their existing ownership.
|
|
15
|
+
- Preparation plays no sound, creates no audio source, resumes no context,
|
|
16
|
+
changes no mute setting and loads no model. A running WebAudio context is not
|
|
17
|
+
a guarantee of future autoplay, native speech or HTML audio readiness. No
|
|
18
|
+
physical-GPU placement or native-engine change is included.
|
|
19
|
+
|
|
3
20
|
## 0.76.0
|
|
4
21
|
|
|
5
22
|
- Add public `textFormat:'plain'` to the `SpeechPlayback` constructor and
|
package/README.md
CHANGED
|
@@ -20,7 +20,7 @@ selected SDK projection or an explicit live SDK source mount. Both profiles
|
|
|
20
20
|
share the theme, packaging, event, cancellation,
|
|
21
21
|
and browser run contracts while retaining their selected application layout.
|
|
22
22
|
|
|
23
|
-
This checkout defines the `0.
|
|
23
|
+
This checkout defines the `0.77.0` SDK contract. Development applications declare
|
|
24
24
|
`arcane-os` as `latest` and retain their lockfile. Exact SDK pins begin when Roshi
|
|
25
25
|
designates a production build. Follow [Consumer updates](docs/consumer-updates.md)
|
|
26
26
|
to refresh and compare the installed SDK and lockfile with the published channel;
|
|
@@ -889,7 +889,7 @@ provider cleanup remains observable through that operation's settlement.
|
|
|
889
889
|
|
|
890
890
|
### Playback controls and independent lifetimes
|
|
891
891
|
|
|
892
|
-
`ai.playPreparedTTS(prepared,{signal,onState})` returns
|
|
892
|
+
`ai.playPreparedTTS(prepared,{signal,onState,audioOutput})` returns
|
|
893
893
|
`{state,error,finished,pause(),resume(),stop()}` immediately. It uses the
|
|
894
894
|
existing AI audio-clock scheduler and waits for each earlier segment before scheduling
|
|
895
895
|
later audio. Ready adjacent buffers retain contiguous scheduling. Requested
|
|
@@ -902,8 +902,9 @@ states are `waiting`, `waiting-for-gesture`, `scheduled`, `paused`, `complete`,
|
|
|
902
902
|
`stopped`, and `error`. Use `waiting-for-gesture` to present an audio-unlock
|
|
903
903
|
control. A `false` result from `resume()` is not a first-segment-ready signal;
|
|
904
904
|
inspect `state` and `error` to distinguish a stopped or unavailable context
|
|
905
|
-
from browser gesture waiting.
|
|
906
|
-
context immediately, before the first
|
|
905
|
+
from browser gesture waiting. With `audioOutput` omitted or `null`, playback
|
|
906
|
+
creates its own audio context immediately, before the first segment is ready,
|
|
907
|
+
and closes that context after completion, stop, cancellation, or failure.
|
|
907
908
|
|
|
908
909
|
`finished` resolves `true` after natural playback completion and `false` after
|
|
909
910
|
stop, playback cancellation, or failure. Real failures also reach the existing
|
|
@@ -931,6 +932,87 @@ preparations. A page that starts several preparations owns and observes every
|
|
|
931
932
|
`ready` promise, even when it plays only one handle. Browser autoplay permission
|
|
932
933
|
still applies; neither promise proves that a person heard the audio.
|
|
933
934
|
|
|
935
|
+
### Prepare a silent reusable WebAudio output
|
|
936
|
+
|
|
937
|
+
`ai.prepareAudioOutput()` synchronously returns an AI-owned output handle.
|
|
938
|
+
It creates or reuses that AI's persistent WebAudio context and observes its
|
|
939
|
+
actual state. Preparation stays silent: it does not resume the context, unmute
|
|
940
|
+
speech, load a model, create an audio source, play a test sound, or install
|
|
941
|
+
gesture listeners. It does not wait for speech preparation or provider readiness.
|
|
942
|
+
|
|
943
|
+
The handle exposes `kind:'web-audio'`, live `state`, `contextState`, and `error`
|
|
944
|
+
getters, `subscribe(listener)`, `resume()`, and `dispose()`. Repeated preparation
|
|
945
|
+
returns the same handle until its output is disposed or its context closes.
|
|
946
|
+
`contextState` is the platform context's current state, or `null` when unavailable;
|
|
947
|
+
`error` retains the complete actual error, or `null`.
|
|
948
|
+
|
|
949
|
+
| Output state | Meaning |
|
|
950
|
+
| --- | --- |
|
|
951
|
+
| `ready` | This WebAudio context is currently `running`. |
|
|
952
|
+
| `suspended`, `interrupted`, or another platform state | The context reports that state; suspension alone does not establish a gesture requirement. |
|
|
953
|
+
| `gesture-required` | A resume operation actually reported `NotAllowedError` while the context was not running. |
|
|
954
|
+
| `unavailable` | Context creation failed; `error` contains the actual failure. |
|
|
955
|
+
| `error` | An output operation failed; inspect the complete `error`. |
|
|
956
|
+
| `disposing`, `disposed` | Owner disposal is in progress or has completed. |
|
|
957
|
+
| `closed` | The underlying context has closed outside completed owner disposal. |
|
|
958
|
+
|
|
959
|
+
`subscribe(listener)` immediately replays `{kind,state,contextState,error}` and
|
|
960
|
+
then delivers state changes through the canonical AI event owner. It returns
|
|
961
|
+
an idempotent unsubscribe function. The named export
|
|
962
|
+
`AI_AUDIO_OUTPUT_STATE_EVENT` from `AI.js` is `'ai-audio-output-state'`; its
|
|
963
|
+
canonical occurrences carry the same snapshot in `detail` and an output-specific
|
|
964
|
+
`operationId`. Handle subscriptions select only their own output. Disposal or
|
|
965
|
+
context closure releases those subscriptions after the terminal notification;
|
|
966
|
+
a later handle subscription still receives the current snapshot immediately.
|
|
967
|
+
|
|
968
|
+
`resume()` is an explicit asynchronous action for the application's user-gesture
|
|
969
|
+
flow. It resolves `true` only when the context is actually running, and `false`
|
|
970
|
+
when unavailable, disposed, superseded, or held by a paused prepared playback.
|
|
971
|
+
A platform rejection rejects with its actual error and retains that failure in
|
|
972
|
+
the output state. It does not unmute speech or load a model. Output readiness
|
|
973
|
+
does not establish HTML audio readiness, native-speech readiness, future autoplay
|
|
974
|
+
permission, available speech audio, or audible delivery.
|
|
975
|
+
|
|
976
|
+
Pass the handle explicitly to borrow that exact output for prepared playback:
|
|
977
|
+
|
|
978
|
+
```javascript
|
|
979
|
+
// ai and prepared are the same AI owner and preparation used above.
|
|
980
|
+
const audioOutput = ai.prepareAudioOutput();
|
|
981
|
+
|
|
982
|
+
function observeNarrationOutput(listener) {
|
|
983
|
+
return audioOutput.subscribe(listener);
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
function resumeNarrationOutput() {
|
|
987
|
+
// Call from the application's user action and observe the returned promise.
|
|
988
|
+
return audioOutput.resume();
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
function playNarration() {
|
|
992
|
+
return ai.playPreparedTTS(prepared, {audioOutput});
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
function disposeNarrationOutput() {
|
|
996
|
+
// Call when this owner no longer needs the output; observe its promise.
|
|
997
|
+
return audioOutput.dispose();
|
|
998
|
+
}
|
|
999
|
+
```
|
|
1000
|
+
|
|
1001
|
+
The selected handle must be live and belong to the same AI instance; another
|
|
1002
|
+
owner's, closed, or disposing output throws `TypeError`. Borrowed playback
|
|
1003
|
+
retains the same ordered scheduling, pause/resume, cancellation, and single-lane
|
|
1004
|
+
replacement behavior. Completion or stopping playback leaves the output open
|
|
1005
|
+
for reuse. Merely preparing an output does not change the omitted-option path.
|
|
1006
|
+
|
|
1007
|
+
The application owns the output lifetime and calls `dispose()` when finished
|
|
1008
|
+
with it. Disposal stops playback bound to that context and any following work
|
|
1009
|
+
waiting on its clock, observes cleanup, and awaits the actual context close.
|
|
1010
|
+
It preserves detached preparation, saved audio, unrelated contexts, and model
|
|
1011
|
+
selection. Concurrent calls share the disposal promise. Actual cleanup or close
|
|
1012
|
+
errors reject and remain observable; a failed context close permits a later
|
|
1013
|
+
disposal retry. Stop/mute controls retain their existing playback/provider scope
|
|
1014
|
+
and do not replace this explicit output disposal.
|
|
1015
|
+
|
|
934
1016
|
## Stream chunks as they arrive
|
|
935
1017
|
|
|
936
1018
|
Use the configured `ai` created above. Run this snippet from an owned user
|
|
@@ -10,13 +10,14 @@
|
|
|
10
10
|
},
|
|
11
11
|
"artifactCount": 93,
|
|
12
12
|
"javascriptArtifactCount": 91,
|
|
13
|
-
"esmExportCount":
|
|
13
|
+
"esmExportCount": 374,
|
|
14
14
|
"artifacts": [
|
|
15
15
|
{
|
|
16
16
|
"file": "runtime/arcane/modules/AI.js",
|
|
17
17
|
"name": "AI.js",
|
|
18
18
|
"kind": "esm",
|
|
19
19
|
"exports": [
|
|
20
|
+
"AI_AUDIO_OUTPUT_STATE_EVENT",
|
|
20
21
|
"AI_BROWSER_SPEECH_CONFIGURATION_PROTOCOL",
|
|
21
22
|
"AI_BROWSER_SPEECH_ERROR_CODES",
|
|
22
23
|
"AI_BROWSER_SPEECH_EVENT_TYPES",
|
|
@@ -30,7 +31,7 @@
|
|
|
30
31
|
"availability": "Browser + native bridge + TWiN Cloud",
|
|
31
32
|
"protocol": "arcane-ai-browser-speech-configuration/1, AIProviderRuntime arcane-ai-provider/2 routes, globalThis.arcaneEvents, TWiN Cloud HTTPS, Arcane.ollama, Arcane.speech, Android WebView bridge",
|
|
32
33
|
"normalization": "Complete mutable caller-owned browser speech authority, SDK-owned provider registration/replacement/disposal, explicit STT/TTS activation, normalized Whisper STT and Kokoro TTS execution with default capacities 1 and 4 respectively, explicit provider-neutral execution snapshots through status(role,{execution:true}), sticky readiness, canonical window.user readiness without compatibility-event object-identity admission or polling, exact ordered structural calls with required arguments.message, atomic nonblank all-ID tool-result sequencing, complete all-choice streaming/data validation, native Ollama structural adaptation, complete-response then all validated tool callbacks then completion ordering, observed async callbacks, Blob/File transcription, default repeated-formatting-mark removal from cloned outbound TTS input, explicit textFormat:'plain' preservation for complete synthesis/preparation while streaming cleanup remains automatic and caller/visible content stays exact, immediate exact-segment synthesis, per-call voice/speed capture, optional final-segment pause on the existing audio clock, optional per-call terminal playback completion, detached preparation with complete semantic DBOPFS audio reuse and same-owner pending request sharing; each preparation owns its pending activation interest, final-interest cancellation aborts that activation while other interests preserve it, complete cache hits start no model, and cancellation after readiness leaves the ready model loaded; independent prepared playback without a model load for cached audio, ordered audio-clock scheduling, playable audio, active-generation TTS operation-failure routing, mute, and cancellation are normalized; provider/model/runtime/voice policy remains caller-owned. Both speech roles default execution.device to auto: webnn-npu when navigator.ml.createContext is exposed, then webgpu when navigator.gpu is exposed, then wasm for CPU execution. Failed backend loads clean up the pool before trying fresh Workers with the same prepared model and dtype; explicit devices never fall back. STT capacity only accepts 1; TTS accepts 1 through 4. Execution snapshots expose requestedDevice, selectedDevice, maxConcurrentRequests, and activeRequestCount for both roles; selectedDevice is null while unloaded and names the successful upstream session backend request when loaded. It does not prove every operation used the physical NPU: WebNN unsupported operations may use WASM, and compatibility depends on the browser, driver, hardware, and graph.",
|
|
33
|
-
"surface": "Native fetchTTS(payload,signal) and fetchSTT(audio,signal) forward caller cancellation separately from their complete Core speech payloads, normalize aborts to AbortError and suppress late results; actual engine interruption remains service-owned. prepareTTSPlayback(payload,signal) silently returns a real Blob or native-speech descriptor; native controls expose finished/released/state/error/pause/resume/stop, and actual release owns the playback lane. fetchTTS and durable prepareTTS report ARCANE_AI_TTS_AUDIO_EXPORT_UNAVAILABLE for native voices. Request-local model overrides remain exact and never mutate the configured model. Browser-speech protocol/event/error/reason constants; default `AI`; read-only `providerRuntime`, `browserSpeechConfiguration`, and `browserSpeechDescriptor`; explicit `providerRuntime.status(role,{execution:true})` snapshots; `streamTTS(text,end,options={})` with optional voice, speed, pauseAfterMs, and waitForPlayback; default speech-input cleanup with SDK-internal preparation metadata and explicit complete-call textFormat:'plain' consumed before engine dispatch; `prepareTTS({parts,storage,identity,textFormat,signal,onState})` returning ordered segments, state, ready, getAudio(index), and cancel(); `playPreparedTTS(prepared,{signal,onState})` returning state, error, finished, pause(), resume(), and stop() with one playback lane per AI and observational state/error callbacks; configure/dispose speech, route lifecycle, declaration-validated chat/stream requests with exact ordered calls; optional streamRequest toolText:{name,field} and onToolText(text,call,displayId) for decoded selected argument text with actual call identity, distinct from ordinary prose and terminal tool execution; synthesis/transcription and playback controls; initializes from current canonical user readiness, installs `window.ai`, and projects `ai-ready` plus active-generation `ai-tts-failure`."
|
|
34
|
+
"surface": "Native fetchTTS(payload,signal) and fetchSTT(audio,signal) forward caller cancellation separately from their complete Core speech payloads, normalize aborts to AbortError and suppress late results; actual engine interruption remains service-owned. prepareTTSPlayback(payload,signal) silently returns a real Blob or native-speech descriptor; native controls expose finished/released/state/error/pause/resume/stop, and actual release owns the playback lane. fetchTTS and durable prepareTTS report ARCANE_AI_TTS_AUDIO_EXPORT_UNAVAILABLE for native voices. Request-local model overrides remain exact and never mutate the configured model. Browser-speech protocol/event/error/reason constants; default `AI`; read-only `providerRuntime`, `browserSpeechConfiguration`, and `browserSpeechDescriptor`; explicit `providerRuntime.status(role,{execution:true})` snapshots; `streamTTS(text,end,options={})` with optional voice, speed, pauseAfterMs, and waitForPlayback; default speech-input cleanup with SDK-internal preparation metadata and explicit complete-call textFormat:'plain' consumed before engine dispatch; `prepareTTS({parts,storage,identity,textFormat,signal,onState})` returning ordered segments, state, ready, getAudio(index), and cancel(); `prepareAudioOutput()` returning the persistent WebAudio handle {kind,state,contextState,error,subscribe,resume,dispose} without resume, unmute, model load, sources, sound, or gesture listeners; current-state replay and output-specific canonical `AI_AUDIO_OUTPUT_STATE_EVENT` (`ai-audio-output-state`), with ready only for an actually running context; explicit owner disposal observes context-bound cleanup and close; `playPreparedTTS(prepared,{signal,onState,audioOutput})` returning state, error, finished, pause(), resume(), and stop() with one playback lane per AI and observational state/error callbacks; explicit live same-AI audioOutput borrows its context until owner disposal, while omission or null preserves per-playback terminal close; configure/dispose speech, route lifecycle, declaration-validated chat/stream requests with exact ordered calls; optional streamRequest toolText:{name,field} and onToolText(text,call,displayId) for decoded selected argument text with actual call identity, distinct from ordinary prose and terminal tool execution; synthesis/transcription and playback controls; initializes from current canonical user readiness, installs `window.ai`, and projects `ai-ready` plus active-generation `ai-tts-failure`."
|
|
34
35
|
},
|
|
35
36
|
{
|
|
36
37
|
"file": "runtime/arcane/modules/AIModelSelectionController.js",
|
|
@@ -171,10 +171,11 @@ default `AI`; read-only `providerRuntime`, `speechActivationPending`, `browserSp
|
|
|
171
171
|
read-only `ttsSegmentation`, `configureTTSSegmentation()`,
|
|
172
172
|
`streamTTS(text='',end=false,options={})`,
|
|
173
173
|
`prepareTTS({parts,textFormat,storage,identity,signal,onState})`,
|
|
174
|
-
`playPreparedTTS(prepared,{signal,onState})`,
|
|
174
|
+
`prepareAudioOutput()`, `playPreparedTTS(prepared,{signal,onState,audioOutput})`,
|
|
175
175
|
`finishTTS()`, `prepareTTSPlayback()`, `fetchTTS()`, `fetchSTT()`, `stopAudio()`, `resumeAudio()`,
|
|
176
176
|
`playAudio()`; consumes `user-entity-loaded` and `arcane-ollama-ready`,
|
|
177
|
-
installs `window.ai`, and emits `ai-ready
|
|
177
|
+
installs `window.ai`, and emits `ai-ready`, `ai-tts-failure`, and
|
|
178
|
+
`AI_AUDIO_OUTPUT_STATE_EVENT` (`'ai-audio-output-state'`).
|
|
178
179
|
|
|
179
180
|
`fetch(...)` and `fetchRequest(options)` are asynchronous complete-response
|
|
180
181
|
entry points. `streamMessage(...)` and `streamRequest(options)` deliver
|
|
@@ -496,7 +497,7 @@ cancelling one handle does not cancel another active matching caller.
|
|
|
496
497
|
Preparation cancellation prevents later synthesis and releases its activation
|
|
497
498
|
interest. The provider's operation remains responsible for actual cleanup.
|
|
498
499
|
|
|
499
|
-
`playPreparedTTS(prepared,{signal,onState})` can attach immediately. Its returned
|
|
500
|
+
`playPreparedTTS(prepared,{signal,onState,audioOutput})` can attach immediately. Its returned
|
|
500
501
|
`{state,error,finished,pause(),resume(),stop()}` handle schedules segments in
|
|
501
502
|
their original order using the existing AI audio clock. `finished` resolves `true` after
|
|
502
503
|
natural completion and `false` after stop, cancellation, or failure; genuine
|
|
@@ -521,6 +522,38 @@ model even when the AI starts muted. Missing audio alone requests the shared
|
|
|
521
522
|
TTS readiness path. See [prepared narration](ai/browser-speech.md#prepare-narration-once-and-replay-stored-audio)
|
|
522
523
|
for full record shapes, storage ownership, and a complete example.
|
|
523
524
|
|
|
525
|
+
`prepareAudioOutput()` synchronously creates or reuses this AI's persistent
|
|
526
|
+
WebAudio context and returns the same handle until disposed or closed. It is
|
|
527
|
+
silent: no resume, unmute, model load, source creation, test sound, or gesture
|
|
528
|
+
listener registration. The handle exposes `kind:'web-audio'`, current
|
|
529
|
+
`state`, `contextState`, and complete `error` getters, plus
|
|
530
|
+
`subscribe(listener)`, `resume()`, and `dispose()`.
|
|
531
|
+
`subscribe` immediately replays `{kind,state,contextState,error}`, observes
|
|
532
|
+
future changes for this output, and returns an idempotent unsubscribe function.
|
|
533
|
+
The named `AI_AUDIO_OUTPUT_STATE_EVENT` export identifies canonical occurrences
|
|
534
|
+
with that snapshot as `detail` and an output-specific `operationId`.
|
|
535
|
+
|
|
536
|
+
Output `ready` means its context is actually `running`, independently of model
|
|
537
|
+
readiness, HTML audio, native speech, future autoplay permission, or audible
|
|
538
|
+
delivery. Other states are the platform's current state (including `suspended`
|
|
539
|
+
or `interrupted`), `gesture-required` after an actual `NotAllowedError`,
|
|
540
|
+
`unavailable` when context creation fails, `error`, `disposing`, `disposed`,
|
|
541
|
+
or `closed`. Explicit `resume()` resolves a boolean for actual readiness;
|
|
542
|
+
unavailable/disposed, superseded, or paused-playback cases resolve `false`.
|
|
543
|
+
Platform rejection rejects with the actual error and retains observable state.
|
|
544
|
+
|
|
545
|
+
Pass `{audioOutput}` to `playPreparedTTS` to borrow that same AI's live output;
|
|
546
|
+
another owner's, closed, or disposing handle throws `TypeError`. Borrowed
|
|
547
|
+
playback leaves the context open after completion or stop. Omission or `null`
|
|
548
|
+
preserves the existing per-playback context and terminal context close.
|
|
549
|
+
Owner `dispose()` stops context-bound playback and dependent clock work,
|
|
550
|
+
observes cleanup, and awaits actual close without cancelling detached speech
|
|
551
|
+
preparation or deleting saved audio. Concurrent disposal calls share their
|
|
552
|
+
promise; failures remain observable and a failed close can be retried.
|
|
553
|
+
Disposal/closure releases handle subscriptions after their final notification.
|
|
554
|
+
See [silent reusable output](ai/browser-speech.md#prepare-a-silent-reusable-webaudio-output)
|
|
555
|
+
for state details and usage.
|
|
556
|
+
|
|
524
557
|
Streaming speech retains sentence
|
|
525
558
|
segmentation by default. `configureTTSSegmentation({punctuation,wordCadence})`
|
|
526
559
|
accepts `punctuation:'sentence'|'any'|'none'` and a `wordCadence` that is either
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "arcane-os",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.77.0",
|
|
4
4
|
"description": "Arcane OS JavaScript SDK, project-local CLI, browser runtime, and repository-portable application packager.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.mjs",
|
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
"test:unit": "node ./bin/arcane-test.mjs test/app-descriptor.test.mjs test/app-schema.test.mjs test/app-selection.test.mjs test/contracts.test.mjs test/conversation-closing-report.test.mjs test/doctor.test.mjs test/mail-credentials.test.mjs test/mail-outbox.test.mjs test/mail-public-api.test.mjs test/mail-send.test.mjs test/mail-transport.test.mjs test/native-windows-icon.test.mjs test/targets.test.mjs test/workspace-operation-lock.test.mjs",
|
|
126
126
|
"test:functional": "node ./bin/arcane-test.mjs test/ai-ollama-readiness.test.mjs test/core-ollama-lifecycle.test.mjs test/ai-request-model.test.mjs test/ai-request-retry.test.mjs test/ai-stream-cancellation.test.mjs test/browser-decisions.test.mjs test/browser-kokoro-complete-input.test.mjs test/browser-speech-providers.test.mjs test/browser-speech-artifacts.test.mjs test/browser-speech-resource-bridge.test.mjs test/browser-wasm-gpu-notice.test.mjs test/browser-wasm-download-resume.test.mjs test/chat-input.test.mjs test/chat-stream-speech.test.mjs test/chat-transcript-scroll.test.mjs test/chat-transient-retention.test.mjs test/cli.test.mjs test/continuous-voice-capture.test.mjs test/codex-app-server.test.mjs test/mcp-stdio-server.test.mjs test/core-classic-source.test.mjs test/core-client.test.mjs test/core-client-installation.test.mjs test/core-ai-installation.test.mjs test/core-llama-lifecycle.test.mjs test/core-development.test.mjs test/core-host.test.mjs test/core-shared-host.test.mjs test/core-model-assets.test.mjs test/core-native-services.test.mjs test/core-packaged-web.test.mjs test/core-runtime.test.mjs test/core-stdio.test.mjs test/dbopfs-committed-events.test.mjs test/dbopfs-document-library.test.mjs test/dbopfs-model-resources.test.mjs test/dbopfs-png.test.mjs test/dbopfs.test.mjs test/dev-server.test.mjs test/dev-pwa.test.mjs test/dom-event-instrumentation.test.mjs test/event-manager.test.mjs test/events.test.mjs test/file-preview.test.mjs test/import-map.test.mjs test/local-image-editing.test.mjs test/mail-cli.test.mjs test/mail-runtime.test.mjs test/mail-server.test.mjs test/markdown-editor.test.mjs test/markdown-media.test.mjs test/markdown-source-map.test.mjs test/modal.test.mjs test/model-resource-bridge.test.mjs test/native-decisions.test.mjs test/native-onnx.test.mjs test/native-module-packaging.test.mjs test/native-module-resource-references.test.mjs test/packaging.test.mjs test/pwa-packaging.test.mjs test/pwa-client.test.mjs test/pwa-install.test.mjs test/pwa-worker.test.mjs test/persistent-ai-chat-session.test.mjs test/reference-completeness.test.mjs test/rendered-print.test.mjs test/runtime-api-behavior.test.mjs test/runtime.test.mjs test/scaffold.test.mjs test/speech-playback.test.mjs test/site.test.mjs test/theme-manager.test.mjs test/twin-cloud-images.test.mjs test/twin-cloud.test.mjs test/websocket-client-import-map.test.mjs test/websocket.test.mjs test/update-check.test.mjs test/user-explicit-updates.test.mjs",
|
|
127
127
|
"test:integration": "node ./bin/arcane-test.mjs test/installed-package-runtime.test.mjs test/root-app-layout.test.mjs test/integrated-shared.test.mjs test/integrated-workspace.test.mjs test/mail-browser.test.mjs test/native-macos-provider.test.mjs test/native-plan.test.mjs test/native-portable-provider.test.mjs test/native-provider-loader.test.mjs test/npm-release.test.mjs test/release-bundle.test.mjs test/release-capability-smoke.test.mjs test/shared-payload-batch.test.mjs test/tarball.test.mjs test/browser-wasm-cpu.test.mjs test/wllama-webgpu-runtime.test.mjs",
|
|
128
|
-
"test:regression": "node ./bin/arcane-test.mjs test/channel-workflows.test.mjs test/html-import-registration.test.mjs test/html-import-packaged-script.test.mjs test/logging-regression.test.mjs test/markdown-speech.test.mjs test/prepared-speech.test.mjs test/native-provider-generation.test.mjs test/speech-queue-regression.test.mjs test/testing.test.mjs test/test-sets.test.mjs",
|
|
128
|
+
"test:regression": "node ./bin/arcane-test.mjs test/channel-workflows.test.mjs test/html-import-registration.test.mjs test/html-import-packaged-script.test.mjs test/logging-regression.test.mjs test/markdown-speech.test.mjs test/prepared-speech.test.mjs test/ai-audio-output.test.mjs test/native-provider-generation.test.mjs test/speech-queue-regression.test.mjs test/testing.test.mjs test/test-sets.test.mjs",
|
|
129
129
|
"check": "node tools/check-source.mjs && npm test",
|
|
130
130
|
"check:release": "node tools/check-source.mjs --package-only && npm run test:release",
|
|
131
131
|
"pack:local": "npm pack --ignore-scripts",
|
|
@@ -51,6 +51,7 @@ credentials='omit';
|
|
|
51
51
|
const BUILT_IN_AI_SERVICES=new Set(['TWIN','OLLAMA','LOCAL_SPEACH']);
|
|
52
52
|
const AI_REASONING_EFFORTS=new Set(['none','low','medium','high','max']);
|
|
53
53
|
export const AI_READY_EVENT='ai-ready';
|
|
54
|
+
export const AI_AUDIO_OUTPUT_STATE_EVENT='ai-audio-output-state';
|
|
54
55
|
const AI_TTS_FAILURE_EVENT='ai-tts-failure';
|
|
55
56
|
export const AI_INITIALIZATION_ERROR_CODES=completeValue({
|
|
56
57
|
userReadyRegistrationCollision:
|
|
@@ -1128,6 +1129,7 @@ class AI {
|
|
|
1128
1129
|
eventTypes:completeValue(
|
|
1129
1130
|
[
|
|
1130
1131
|
AI_READY_EVENT,
|
|
1132
|
+
AI_AUDIO_OUTPUT_STATE_EVENT,
|
|
1131
1133
|
AI_TTS_FAILURE_EVENT,
|
|
1132
1134
|
...Object.values(AI_BROWSER_SPEECH_EVENT_TYPES)
|
|
1133
1135
|
]
|
|
@@ -1183,8 +1185,12 @@ class AI {
|
|
|
1183
1185
|
#stopOllamaReady=null;
|
|
1184
1186
|
#ttsSegmentation={...DEFAULT_TTS_SEGMENTATION};
|
|
1185
1187
|
#preparedSpeechPlaybacks = new Set();
|
|
1188
|
+
#activePreparedSpeechPlayback = null;
|
|
1186
1189
|
#nativeSpeechPlaybacks = new Set();
|
|
1187
1190
|
#speechUnlockContext = null;
|
|
1191
|
+
#speechResumeContext = null;
|
|
1192
|
+
#audioOutput = null;
|
|
1193
|
+
#audioOutputSequence = 0;
|
|
1188
1194
|
|
|
1189
1195
|
#traceSpeech(phase,detail={}){
|
|
1190
1196
|
if(!arcaneLogging.enabled)return null;
|
|
@@ -6434,7 +6440,7 @@ class AI {
|
|
|
6434
6440
|
);
|
|
6435
6441
|
}
|
|
6436
6442
|
|
|
6437
|
-
playPreparedTTS(prepared, {signal = null, onState = null} = {}) {
|
|
6443
|
+
playPreparedTTS(prepared, {signal = null, onState = null, audioOutput = null} = {}) {
|
|
6438
6444
|
if(!is.array(prepared?.segments) || !is.function(prepared?.getAudio)) {
|
|
6439
6445
|
throw new TypeError('AI.playPreparedTTS requires a speech preparation handle.');
|
|
6440
6446
|
}
|
|
@@ -6442,9 +6448,19 @@ class AI {
|
|
|
6442
6448
|
if(onState !== null && !is.function(onState)) {
|
|
6443
6449
|
throw new TypeError('Prepared playback onState must be a function when provided.');
|
|
6444
6450
|
}
|
|
6451
|
+
const output = audioOutput === null ? null : this.#audioOutput;
|
|
6452
|
+
if(audioOutput !== null && (output?.handle !== audioOutput
|
|
6453
|
+
|| output.disposing || output.disposed || !output.context
|
|
6454
|
+
|| output.context.state === 'closed')) {
|
|
6455
|
+
throw new TypeError('Prepared playback requires a live audio output from this AI.');
|
|
6456
|
+
}
|
|
6445
6457
|
// One playback lane per AI; detached preparation has its own lifetime.
|
|
6446
6458
|
this.stopAudio();
|
|
6447
|
-
const group = {
|
|
6459
|
+
const group = {
|
|
6460
|
+
jobs: [], context: output?.context ?? null, output,
|
|
6461
|
+
paused: false, stopped: false, state: 'waiting', error: null
|
|
6462
|
+
};
|
|
6463
|
+
this.#activePreparedSpeechPlayback = group;
|
|
6448
6464
|
const completions = [];
|
|
6449
6465
|
const generation = this.speechGeneration;
|
|
6450
6466
|
this.#traceSpeech('playPreparedTTS.call', {segments: prepared.segments});
|
|
@@ -6464,8 +6480,16 @@ class AI {
|
|
|
6464
6480
|
function stopPreparedPlayback() {
|
|
6465
6481
|
if(group.stopped) return false;
|
|
6466
6482
|
group.stopped = true;
|
|
6467
|
-
runtime
|
|
6468
|
-
|
|
6483
|
+
if(runtime.#activePreparedSpeechPlayback === group) {
|
|
6484
|
+
runtime.#activePreparedSpeechPlayback = null;
|
|
6485
|
+
runtime.speechResumeAttempt += 1;
|
|
6486
|
+
runtime.speechResumePending = false;
|
|
6487
|
+
if(runtime.#speechUnlockContext === group.context) runtime.#clearSpeechUnlock();
|
|
6488
|
+
if(runtime.speechScheduleContext === group.context) {
|
|
6489
|
+
runtime.speechScheduleContext = null;
|
|
6490
|
+
runtime.speechScheduleTime = 0;
|
|
6491
|
+
}
|
|
6492
|
+
}
|
|
6469
6493
|
publishPreparedPlaybackState('stopped');
|
|
6470
6494
|
for(const job of group.jobs) {
|
|
6471
6495
|
if(job.state === 'complete') continue;
|
|
@@ -6481,13 +6505,10 @@ class AI {
|
|
|
6481
6505
|
}
|
|
6482
6506
|
runtime.#cancelSpeechJob(job);
|
|
6483
6507
|
}
|
|
6484
|
-
if(runtime.speechScheduleContext === group.context) {
|
|
6485
|
-
runtime.speechScheduleContext = null;
|
|
6486
|
-
runtime.speechScheduleTime = 0;
|
|
6487
|
-
}
|
|
6488
6508
|
runtime.#requestSpeechPlayback();
|
|
6489
6509
|
return true;
|
|
6490
6510
|
}
|
|
6511
|
+
group.stop = stopPreparedPlayback;
|
|
6491
6512
|
const control = {
|
|
6492
6513
|
finished: null,
|
|
6493
6514
|
get state() { return group.state; },
|
|
@@ -6496,9 +6517,23 @@ class AI {
|
|
|
6496
6517
|
if(group.stopped || !group.context || group.context.state === 'closed') return false;
|
|
6497
6518
|
group.paused = true;
|
|
6498
6519
|
if(runtime.#speechUnlockContext === group.context) runtime.#clearSpeechUnlock();
|
|
6499
|
-
|
|
6500
|
-
|
|
6501
|
-
|
|
6520
|
+
let suspension;
|
|
6521
|
+
try {
|
|
6522
|
+
suspension = Promise.resolve(group.context.suspend());
|
|
6523
|
+
if(output) output.suspending = suspension;
|
|
6524
|
+
await suspension;
|
|
6525
|
+
if(group.stopped || generation !== runtime.speechGeneration) return false;
|
|
6526
|
+
publishPreparedPlaybackState('paused');
|
|
6527
|
+
return true;
|
|
6528
|
+
} catch(error) {
|
|
6529
|
+
if(output) {
|
|
6530
|
+
output.reportError(error);
|
|
6531
|
+
if(!group.stopped) group.fail(error);
|
|
6532
|
+
}
|
|
6533
|
+
throw error;
|
|
6534
|
+
} finally {
|
|
6535
|
+
if(output?.suspending === suspension) output.suspending = null;
|
|
6536
|
+
}
|
|
6502
6537
|
},
|
|
6503
6538
|
async resume() {
|
|
6504
6539
|
if(group.stopped || !group.context || group.context.state === 'closed') return false;
|
|
@@ -6558,15 +6593,18 @@ class AI {
|
|
|
6558
6593
|
async function finishPreparedPlayback(results) {
|
|
6559
6594
|
const completed = !group.stopped && results.every(Boolean);
|
|
6560
6595
|
group.stopped = true;
|
|
6596
|
+
if(runtime.#activePreparedSpeechPlayback === group) {
|
|
6597
|
+
runtime.#activePreparedSpeechPlayback = null;
|
|
6598
|
+
if(runtime.#speechUnlockContext === group.context) runtime.#clearSpeechUnlock();
|
|
6599
|
+
if(runtime.speechScheduleContext === group.context) {
|
|
6600
|
+
runtime.speechScheduleContext = null;
|
|
6601
|
+
runtime.speechScheduleTime = 0;
|
|
6602
|
+
}
|
|
6603
|
+
}
|
|
6561
6604
|
publishPreparedPlaybackState(completed ? 'complete' : group.error ? 'error' : 'stopped');
|
|
6562
6605
|
signal?.removeEventListener('abort', stopPreparedPlayback);
|
|
6563
6606
|
runtime.#preparedSpeechPlaybacks.delete(control);
|
|
6564
|
-
if(
|
|
6565
|
-
if(runtime.speechScheduleContext === group.context) {
|
|
6566
|
-
runtime.speechScheduleContext = null;
|
|
6567
|
-
runtime.speechScheduleTime = 0;
|
|
6568
|
-
}
|
|
6569
|
-
if(group.context && group.context.state !== 'closed') {
|
|
6607
|
+
if(!output && group.context && group.context.state !== 'closed') {
|
|
6570
6608
|
try {
|
|
6571
6609
|
await group.context.close();
|
|
6572
6610
|
} catch(error) {
|
|
@@ -6577,17 +6615,25 @@ class AI {
|
|
|
6577
6615
|
return completed;
|
|
6578
6616
|
}
|
|
6579
6617
|
);
|
|
6618
|
+
group.finished = control.finished;
|
|
6580
6619
|
signal?.addEventListener('abort', stopPreparedPlayback, {once: true});
|
|
6581
6620
|
publishPreparedPlaybackState('waiting');
|
|
6621
|
+
if(group.stopped) return control;
|
|
6582
6622
|
if(signal?.aborted) {
|
|
6583
6623
|
stopPreparedPlayback();
|
|
6584
6624
|
} else if(group.jobs.length) {
|
|
6585
6625
|
// Playing saved audio does not require activating a synthesis provider.
|
|
6586
6626
|
this.muted = false;
|
|
6587
6627
|
try {
|
|
6588
|
-
|
|
6589
|
-
|
|
6590
|
-
|
|
6628
|
+
if(output) {
|
|
6629
|
+
if(output.disposing || output.disposed || output.context.state === 'closed') {
|
|
6630
|
+
throw new Error('The prepared playback audio output has been disposed.');
|
|
6631
|
+
}
|
|
6632
|
+
} else {
|
|
6633
|
+
const AudioContext = globalThis.AudioContext || globalThis.webkitAudioContext;
|
|
6634
|
+
if(!is.function(AudioContext)) throw new TypeError('Audio playback is unavailable in this browser.');
|
|
6635
|
+
group.context = new AudioContext();
|
|
6636
|
+
}
|
|
6591
6637
|
this.resumeAudio(group.context, true).catch(
|
|
6592
6638
|
function failPreparedSpeechUnlock(error) {
|
|
6593
6639
|
for(const job of group.jobs) runtime.#failSpeechJob(job, error, 'playback-resume');
|
|
@@ -7096,12 +7142,209 @@ class AI {
|
|
|
7096
7142
|
return new Blob(normalized.chunks,{type:normalized.type||this.audioType});
|
|
7097
7143
|
}
|
|
7098
7144
|
|
|
7145
|
+
prepareAudioOutput(){
|
|
7146
|
+
if(this.#audioOutput && !this.#audioOutput.disposed
|
|
7147
|
+
&& this.#audioOutput.context?.state !== 'closed') {
|
|
7148
|
+
return this.#audioOutput.handle;
|
|
7149
|
+
}
|
|
7150
|
+
const runtime = this;
|
|
7151
|
+
const output = {
|
|
7152
|
+
context: null, handle: null, error: null,
|
|
7153
|
+
disposing: false, disposed: false, suspending: null,
|
|
7154
|
+
resumeAttempt: 0, disposal: null, reportError: null, publish: null
|
|
7155
|
+
};
|
|
7156
|
+
const operationId = `${this.#events.instanceId}:audio-output:${++this.#audioOutputSequence}`;
|
|
7157
|
+
const subscriptions = new Set();
|
|
7158
|
+
function snapshot(){
|
|
7159
|
+
const contextState = output.context?.state ?? null;
|
|
7160
|
+
const state = output.disposed ? 'disposed'
|
|
7161
|
+
: output.disposing ? 'disposing'
|
|
7162
|
+
: !output.context ? 'unavailable'
|
|
7163
|
+
: contextState === 'closed' ? 'closed'
|
|
7164
|
+
: output.error?.name === 'NotAllowedError' && contextState !== 'running' ? 'gesture-required'
|
|
7165
|
+
: output.error ? 'error'
|
|
7166
|
+
: contextState === 'running' ? 'ready'
|
|
7167
|
+
: contextState;
|
|
7168
|
+
return {kind: 'web-audio', state, contextState, error: output.error};
|
|
7169
|
+
}
|
|
7170
|
+
function publish(){
|
|
7171
|
+
runtime.#events.dispatch(AI_AUDIO_OUTPUT_STATE_EVENT, snapshot(), {operationId});
|
|
7172
|
+
}
|
|
7173
|
+
output.publish = publish;
|
|
7174
|
+
function reportError(error){
|
|
7175
|
+
output.error = error;
|
|
7176
|
+
publish();
|
|
7177
|
+
if(error?.name !== 'NotAllowedError') {
|
|
7178
|
+
arcaneLogging.error('AI audio output operation failed.', error);
|
|
7179
|
+
}
|
|
7180
|
+
}
|
|
7181
|
+
output.reportError = reportError;
|
|
7182
|
+
function observeContextState(){
|
|
7183
|
+
if(output.context.state === 'running') output.error = null;
|
|
7184
|
+
publish();
|
|
7185
|
+
if(output.context.state === 'closed' && !output.disposing) {
|
|
7186
|
+
output.context.removeEventListener?.('statechange', observeContextState);
|
|
7187
|
+
for(const unsubscribe of [...subscriptions]) unsubscribe();
|
|
7188
|
+
}
|
|
7189
|
+
}
|
|
7190
|
+
function observeRejection(promise, attempt = null){
|
|
7191
|
+
// The owner retains failures even when a caller observes state only.
|
|
7192
|
+
promise.catch(function observeOutputFailure(error){
|
|
7193
|
+
if(attempt === null || (attempt === output.resumeAttempt && !output.disposing && !output.disposed)) {
|
|
7194
|
+
reportError(error);
|
|
7195
|
+
} else {
|
|
7196
|
+
arcaneLogging.error('A retired AI audio output resume failed.', error);
|
|
7197
|
+
}
|
|
7198
|
+
});
|
|
7199
|
+
return promise;
|
|
7200
|
+
}
|
|
7201
|
+
async function resumeOutput(attempt){
|
|
7202
|
+
const context = output.context;
|
|
7203
|
+
if(!context || output.disposing || output.disposed || context.state === 'closed') return false;
|
|
7204
|
+
// A pending suspend targets this exact context; resume must follow it.
|
|
7205
|
+
if(output.suspending) await output.suspending;
|
|
7206
|
+
if(output.disposing || output.disposed || attempt !== output.resumeAttempt) return false;
|
|
7207
|
+
const paused = runtime.speechJobs.some(function hasPausedOutputPlayback(job){
|
|
7208
|
+
return job.preparedPlayback?.context === context && job.preparedPlayback.paused;
|
|
7209
|
+
});
|
|
7210
|
+
if(paused) return false;
|
|
7211
|
+
await context.resume();
|
|
7212
|
+
if(output.disposing || output.disposed || attempt !== output.resumeAttempt) return false;
|
|
7213
|
+
output.error = null;
|
|
7214
|
+
publish();
|
|
7215
|
+
if(context.state !== 'running') return false;
|
|
7216
|
+
if(runtime.#speechUnlockContext === context) runtime.#clearSpeechUnlock();
|
|
7217
|
+
if(runtime.speechJobs.some(function usesResumedOutput(job){
|
|
7218
|
+
return job.audioContext === context || job.preparedPlayback?.context === context;
|
|
7219
|
+
})) runtime.#requestSpeechPlayback();
|
|
7220
|
+
return true;
|
|
7221
|
+
}
|
|
7222
|
+
async function disposeOutput(){
|
|
7223
|
+
const context = output.context;
|
|
7224
|
+
const errors = [];
|
|
7225
|
+
const groups = new Set();
|
|
7226
|
+
output.disposing = true;
|
|
7227
|
+
output.resumeAttempt += 1;
|
|
7228
|
+
publish();
|
|
7229
|
+
if(context) {
|
|
7230
|
+
if(runtime.#speechUnlockContext === context) runtime.#clearSpeechUnlock();
|
|
7231
|
+
if(runtime.#speechResumeContext === context) {
|
|
7232
|
+
runtime.speechResumeAttempt += 1;
|
|
7233
|
+
runtime.speechResumePending = false;
|
|
7234
|
+
runtime.#speechResumeContext = null;
|
|
7235
|
+
}
|
|
7236
|
+
for(const job of [...runtime.speechJobs]) {
|
|
7237
|
+
if(job.preparedPlayback?.context === context) {
|
|
7238
|
+
groups.add(job.preparedPlayback);
|
|
7239
|
+
} else if(job.audioContext === context || job.sourceNode?.context === context
|
|
7240
|
+
|| job.precedingSpeechContext === context) {
|
|
7241
|
+
if(job.sourceNode) {
|
|
7242
|
+
job.sourceNode.onended = null;
|
|
7243
|
+
if(job.sourceNode.__arcaneStarted) {
|
|
7244
|
+
try { job.sourceNode.stop(); }
|
|
7245
|
+
catch(error) { errors.push(error); }
|
|
7246
|
+
}
|
|
7247
|
+
}
|
|
7248
|
+
try { runtime.#cancelSpeechJob(job); }
|
|
7249
|
+
catch(error) { errors.push(error); }
|
|
7250
|
+
}
|
|
7251
|
+
}
|
|
7252
|
+
for(const group of groups) {
|
|
7253
|
+
try { group.stop(); }
|
|
7254
|
+
catch(error) { errors.push(error); }
|
|
7255
|
+
}
|
|
7256
|
+
if(runtime.speechScheduleContext === context) {
|
|
7257
|
+
runtime.speechScheduleContext = null;
|
|
7258
|
+
runtime.speechScheduleTime = 0;
|
|
7259
|
+
}
|
|
7260
|
+
if(runtime.audioContext === context) runtime.audioContext = null;
|
|
7261
|
+
const completions = await Promise.allSettled(
|
|
7262
|
+
[...groups].map(function finishOutputPlayback(group){ return group.finished; })
|
|
7263
|
+
);
|
|
7264
|
+
for(const completion of completions) {
|
|
7265
|
+
if(completion.status === 'rejected') errors.push(completion.reason);
|
|
7266
|
+
}
|
|
7267
|
+
try {
|
|
7268
|
+
if(context.state !== 'closed') await context.close();
|
|
7269
|
+
} catch(error) {
|
|
7270
|
+
errors.push(error);
|
|
7271
|
+
output.disposing = false;
|
|
7272
|
+
output.disposal = null;
|
|
7273
|
+
if(runtime.#audioOutput === output && runtime.audioContext === null) {
|
|
7274
|
+
runtime.audioContext = context;
|
|
7275
|
+
}
|
|
7276
|
+
throw errors.length === 1 ? errors[0] : new AggregateError(errors, 'AI audio output disposal failed.');
|
|
7277
|
+
}
|
|
7278
|
+
context.removeEventListener?.('statechange', observeContextState);
|
|
7279
|
+
}
|
|
7280
|
+
output.disposing = false;
|
|
7281
|
+
output.disposed = true;
|
|
7282
|
+
output.error = !errors.length ? null : errors.length === 1
|
|
7283
|
+
? errors[0] : new AggregateError(errors, 'AI audio output playback cleanup failed.');
|
|
7284
|
+
publish();
|
|
7285
|
+
for(const unsubscribe of [...subscriptions]) unsubscribe();
|
|
7286
|
+
runtime.#requestSpeechPlayback();
|
|
7287
|
+
if(errors.length) throw output.error;
|
|
7288
|
+
return true;
|
|
7289
|
+
}
|
|
7290
|
+
output.handle = {
|
|
7291
|
+
kind: 'web-audio',
|
|
7292
|
+
get state(){ return snapshot().state; },
|
|
7293
|
+
get contextState(){ return output.context?.state ?? null; },
|
|
7294
|
+
get error(){ return output.error; },
|
|
7295
|
+
subscribe(listener){
|
|
7296
|
+
if(!is.function(listener)) throw new TypeError('Audio output subscriber must be a function.');
|
|
7297
|
+
const remove = output.disposed || output.context?.state === 'closed' ? null : runtime.#events.subscribe(
|
|
7298
|
+
AI_AUDIO_OUTPUT_STATE_EVENT,
|
|
7299
|
+
function observeAudioOutput(occurrence){
|
|
7300
|
+
if(occurrence.operationId === operationId) listener(occurrence.detail);
|
|
7301
|
+
}
|
|
7302
|
+
);
|
|
7303
|
+
function unsubscribe(){
|
|
7304
|
+
remove?.();
|
|
7305
|
+
subscriptions.delete(unsubscribe);
|
|
7306
|
+
}
|
|
7307
|
+
if(remove) subscriptions.add(unsubscribe);
|
|
7308
|
+
try { listener(snapshot()); }
|
|
7309
|
+
catch(error) { arcaneLogging.error('AI audio output state observer failed.', error); }
|
|
7310
|
+
return unsubscribe;
|
|
7311
|
+
},
|
|
7312
|
+
resume(){
|
|
7313
|
+
const attempt = ++output.resumeAttempt;
|
|
7314
|
+
return observeRejection(resumeOutput(attempt), attempt);
|
|
7315
|
+
},
|
|
7316
|
+
dispose(){
|
|
7317
|
+
if(output.disposed) return output.disposal ?? Promise.resolve(true);
|
|
7318
|
+
if(!output.disposal) {
|
|
7319
|
+
let resolve;
|
|
7320
|
+
let reject;
|
|
7321
|
+
output.disposal = observeRejection(new Promise(function retainOutputDisposal(accept, decline){
|
|
7322
|
+
resolve = accept;
|
|
7323
|
+
reject = decline;
|
|
7324
|
+
}));
|
|
7325
|
+
disposeOutput().then(resolve, reject);
|
|
7326
|
+
}
|
|
7327
|
+
return output.disposal;
|
|
7328
|
+
}
|
|
7329
|
+
};
|
|
7330
|
+
this.#audioOutput = output;
|
|
7331
|
+
try {
|
|
7332
|
+
output.context = this.#getSpeechAudioContext();
|
|
7333
|
+
output.context.addEventListener?.('statechange', observeContextState);
|
|
7334
|
+
publish();
|
|
7335
|
+
} catch(error) {
|
|
7336
|
+
reportError(error);
|
|
7337
|
+
}
|
|
7338
|
+
return output.handle;
|
|
7339
|
+
}
|
|
7340
|
+
|
|
7099
7341
|
#getSpeechAudioContext(){
|
|
7100
7342
|
if(this.audioContext&&this.audioContext.state!=='closed'){
|
|
7101
7343
|
return this.audioContext;
|
|
7102
7344
|
}
|
|
7103
7345
|
|
|
7104
|
-
const AudioContext=
|
|
7346
|
+
const AudioContext=globalThis.AudioContext||globalThis.webkitAudioContext
|
|
7347
|
+
||window.AudioContext||window.webkitAudioContext;
|
|
7105
7348
|
|
|
7106
7349
|
if(!is.function(AudioContext)){
|
|
7107
7350
|
throw new TypeError('Audio playback is unavailable in this browser.');
|
|
@@ -7396,6 +7639,7 @@ class AI {
|
|
|
7396
7639
|
this.speechGeneration+=1;
|
|
7397
7640
|
this.speechResumeAttempt+=1;
|
|
7398
7641
|
this.speechResumePending=false;
|
|
7642
|
+
this.#speechResumeContext=null;
|
|
7399
7643
|
this.speechScheduleGeneration=this.speechGeneration;
|
|
7400
7644
|
this.speechScheduleContext=null;
|
|
7401
7645
|
this.speechScheduleTime=0;
|
|
@@ -7461,13 +7705,20 @@ class AI {
|
|
|
7461
7705
|
let attempt=0;
|
|
7462
7706
|
let context;
|
|
7463
7707
|
let preparedGroup = null;
|
|
7708
|
+
let output = null;
|
|
7464
7709
|
|
|
7465
7710
|
try{
|
|
7466
7711
|
context=audioContext||this.#getSpeechAudioContext();
|
|
7712
|
+
output = this.#audioOutput?.context === context ? this.#audioOutput : null;
|
|
7713
|
+
if(output?.disposing || output?.disposed) return false;
|
|
7467
7714
|
preparedGroup = this.speechJobs.find(function findPreparedResumeOwner(job) {
|
|
7468
7715
|
return job.preparedPlayback?.context === context;
|
|
7469
7716
|
})?.preparedPlayback ?? null;
|
|
7470
7717
|
if(preparedGroup?.paused || preparedGroup?.stopped) return false;
|
|
7718
|
+
if(output?.suspending) {
|
|
7719
|
+
await output.suspending;
|
|
7720
|
+
if(output.disposing || output.disposed || preparedGroup?.paused || preparedGroup?.stopped) return false;
|
|
7721
|
+
}
|
|
7471
7722
|
this.#traceSpeech('resumeAudio.context',{
|
|
7472
7723
|
callId,state:context.state,audioTime:context.currentTime,
|
|
7473
7724
|
sampleRate:context.sampleRate
|
|
@@ -7490,11 +7741,26 @@ class AI {
|
|
|
7490
7741
|
|
|
7491
7742
|
attempt=++this.speechResumeAttempt;
|
|
7492
7743
|
this.speechResumePending=true;
|
|
7744
|
+
this.#speechResumeContext=context;
|
|
7493
7745
|
await context.resume();
|
|
7746
|
+
if(output?.disposing || output?.disposed) return false;
|
|
7494
7747
|
if(preparedGroup?.stopped) return false;
|
|
7495
7748
|
if(preparedGroup?.paused) {
|
|
7496
|
-
|
|
7497
|
-
|
|
7749
|
+
let suspension;
|
|
7750
|
+
try {
|
|
7751
|
+
suspension = Promise.resolve(context.suspend());
|
|
7752
|
+
if(output) output.suspending = suspension;
|
|
7753
|
+
await suspension;
|
|
7754
|
+
} catch(error) {
|
|
7755
|
+
output?.reportError(error);
|
|
7756
|
+
throw error;
|
|
7757
|
+
} finally {
|
|
7758
|
+
if(output?.suspending === suspension) output.suspending = null;
|
|
7759
|
+
}
|
|
7760
|
+
if(attempt === this.speechResumeAttempt) {
|
|
7761
|
+
this.speechResumePending = false;
|
|
7762
|
+
this.#speechResumeContext = null;
|
|
7763
|
+
}
|
|
7498
7764
|
return false;
|
|
7499
7765
|
}
|
|
7500
7766
|
this.#traceSpeech('resumeAudio.resumed',{
|
|
@@ -7509,6 +7775,11 @@ class AI {
|
|
|
7509
7775
|
}
|
|
7510
7776
|
|
|
7511
7777
|
this.speechResumePending=false;
|
|
7778
|
+
this.#speechResumeContext=null;
|
|
7779
|
+
if(output) {
|
|
7780
|
+
output.error = null;
|
|
7781
|
+
output.publish();
|
|
7782
|
+
}
|
|
7512
7783
|
|
|
7513
7784
|
if(context.state==='running'){
|
|
7514
7785
|
this.#clearSpeechUnlock();
|
|
@@ -7518,6 +7789,7 @@ class AI {
|
|
|
7518
7789
|
}
|
|
7519
7790
|
}catch(error){
|
|
7520
7791
|
this.#traceSpeech('resumeAudio.error',{callId,error,state:context?.state});
|
|
7792
|
+
if(output?.disposing || output?.disposed) return false;
|
|
7521
7793
|
if(preparedGroup?.stopped || preparedGroup?.paused) return false;
|
|
7522
7794
|
if(attempt&&attempt!==this.speechResumeAttempt){
|
|
7523
7795
|
return context?.state==='running';
|
|
@@ -7525,7 +7797,9 @@ class AI {
|
|
|
7525
7797
|
|
|
7526
7798
|
if(attempt===this.speechResumeAttempt){
|
|
7527
7799
|
this.speechResumePending=false;
|
|
7800
|
+
this.#speechResumeContext=null;
|
|
7528
7801
|
}
|
|
7802
|
+
output?.reportError(error);
|
|
7529
7803
|
if(preparedGroup && error?.name !== 'NotAllowedError') {
|
|
7530
7804
|
for(const job of preparedGroup.jobs) this.#failSpeechJob(job, error, 'playback-resume');
|
|
7531
7805
|
return false;
|
|
@@ -7600,6 +7874,9 @@ class AI {
|
|
|
7600
7874
|
const playbackContext=sourceNode?.context
|
|
7601
7875
|
||audioContext
|
|
7602
7876
|
||this.#getSpeechAudioContext();
|
|
7877
|
+
// Retain the actual output before asynchronous decoding so disposal
|
|
7878
|
+
// can cancel this job without cancelling unrelated synthesis.
|
|
7879
|
+
job.audioContext=playbackContext;
|
|
7603
7880
|
const audioBlob=new Blob(audioChunks,{type:audioType});
|
|
7604
7881
|
const arrayBuffer=await audioBlob.arrayBuffer();
|
|
7605
7882
|
const audioBuffer=await playbackContext.decodeAudioData(arrayBuffer);
|
|
@@ -7683,12 +7960,17 @@ class AI {
|
|
|
7683
7960
|
const deadline=this.speechScheduleTime;
|
|
7684
7961
|
if(!context||deadline<=context.currentTime)return Promise.resolve(true);
|
|
7685
7962
|
const signal=job.abortController.signal;
|
|
7963
|
+
job.precedingSpeechContext=context;
|
|
7686
7964
|
return new Promise(function waitForScheduledSpeechGap(resolve,reject){
|
|
7687
7965
|
let timer=null;
|
|
7966
|
+
let settled=false;
|
|
7688
7967
|
function finish(value,error=null){
|
|
7968
|
+
if(settled)return;
|
|
7969
|
+
settled=true;
|
|
7689
7970
|
globalThis.clearTimeout(timer);
|
|
7690
7971
|
signal.removeEventListener('abort',cancel);
|
|
7691
7972
|
context.removeEventListener('statechange',schedule);
|
|
7973
|
+
if(job.precedingSpeechContext===context)job.precedingSpeechContext=null;
|
|
7692
7974
|
if(error)reject(error);
|
|
7693
7975
|
else resolve(value);
|
|
7694
7976
|
}
|
|
@@ -7755,7 +8037,8 @@ class AI {
|
|
|
7755
8037
|
// the next turn, including when it follows buffered audio.
|
|
7756
8038
|
if(index!==0||job.state!=='ready')break;
|
|
7757
8039
|
if(!await this.#waitForPrecedingSpeechClock(job)
|
|
7758
|
-
||job.generation!==this.speechGeneration||this.muted
|
|
8040
|
+
||job.generation!==this.speechGeneration||this.muted
|
|
8041
|
+
||['cancelled','failed'].includes(job.state)){
|
|
7759
8042
|
return this.#cancelSpeechJob(job);
|
|
7760
8043
|
}
|
|
7761
8044
|
job.state='scheduled';
|
|
@@ -7783,6 +8066,10 @@ class AI {
|
|
|
7783
8066
|
const audioContext=job.sourceNode.context||job.audioContext;
|
|
7784
8067
|
|
|
7785
8068
|
if(job.preparedPlayback?.paused) break;
|
|
8069
|
+
if(this.#audioOutput?.context === audioContext && this.#audioOutput.suspending) {
|
|
8070
|
+
this.resumeAudio(audioContext, false);
|
|
8071
|
+
return scheduled;
|
|
8072
|
+
}
|
|
7786
8073
|
|
|
7787
8074
|
if(audioContext.state!=='running'){
|
|
7788
8075
|
this.#waitForSpeechGesture(null,audioContext);
|
|
@@ -7871,6 +8158,7 @@ class AI {
|
|
|
7871
8158
|
return false;
|
|
7872
8159
|
}finally{
|
|
7873
8160
|
this.speechPlaybackStarting=false;
|
|
8161
|
+
const output = this.#audioOutput;
|
|
7874
8162
|
const nextJob=this.speechJobs.find(function findFirstUnscheduledSpeechJob(job){
|
|
7875
8163
|
return job.state!=='scheduled';
|
|
7876
8164
|
});
|
|
@@ -7879,6 +8167,10 @@ class AI {
|
|
|
7879
8167
|
!this.muted
|
|
7880
8168
|
&&!this.speechAwaitingGesture
|
|
7881
8169
|
&&!this.speechResumePending
|
|
8170
|
+
&&!this.speechJobs.some(function hasPendingOutputSuspension(job){
|
|
8171
|
+
return output?.suspending && (job.audioContext === output.context
|
|
8172
|
+
|| job.preparedPlayback?.context === output.context);
|
|
8173
|
+
})
|
|
7882
8174
|
&&!this.speechJobs.some(function hasPausedPreparedPlayback(job) {
|
|
7883
8175
|
return job.preparedPlayback?.paused;
|
|
7884
8176
|
})
|