arcane-os 0.93.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +1 -1
- package/browser-runtime/ai/twin-cloud.mjs +109 -46
- package/browser-runtime/core/client.mjs +197 -0
- package/docs/reference/ai/twin-cloud.md +35 -14
- package/docs/reference/availability-and-normalization.md +1 -1
- package/docs/reference/inventory/package-api.json +96 -12
- package/docs/reference/local-ai.md +81 -5
- package/docs/reference/native-magpie.md +98 -0
- package/docs/reference/native-nemotron.md +204 -0
- package/docs/reference/native-speech.md +364 -11
- package/docs/reference/runtime-modules.md +21 -18
- package/docs/reference/sdk-api.md +152 -11
- package/package.json +6 -1
- package/runtime/arcane/modules/AI.js +114 -14
- package/runtime/arcane/modules/AIProviderRuntime.js +32 -15
- package/runtime/arcane/modules/ContinuousVoiceCapture.js +118 -21
- package/runtime/arcane/modules/NativeSpeechCapture.js +198 -0
- package/runtime/arcane/modules/NativeSpeechPlayback.js +304 -0
- package/runtime/arcane/modules/VoiceCaptureWorklet.js +62 -2
- package/schemas/arcane-app.schema.json +2 -2
- package/schemas/arcane-package.schema.json +2 -2
- package/src/core/services/speech.mjs +320 -25
- package/src/local-ai/config.mjs +12 -1
- package/src/local-ai/dev.mjs +71 -4
- package/src/local-ai/install.mjs +113 -5
- package/src/local-ai/kokoro/frontend.mjs +219 -0
- package/src/local-ai/kokoro/helper.mjs +168 -0
- package/src/local-ai/kokoro/index.mjs +645 -0
- package/src/local-ai/kokoro/native/CMakeLists.txt +45 -0
- package/src/local-ai/kokoro/native/main.cpp +249 -0
- package/src/local-ai/magpie/build.mjs +36 -0
- package/src/local-ai/magpie/helper.mjs +254 -0
- package/src/local-ai/magpie/index.mjs +488 -0
- package/src/local-ai/magpie/native/CMakeLists.txt +10 -0
- package/src/local-ai/magpie/native/main.cpp +270 -0
- package/src/local-ai/nemotron/build.mjs +53 -0
- package/src/local-ai/nemotron/helper.mjs +308 -0
- package/src/local-ai/nemotron/index.mjs +637 -0
- package/src/local-ai/nemotron/native/CMakeLists.txt +17 -0
- package/src/local-ai/nemotron/native/main.cpp +282 -0
- package/src/native/portable-layout.mjs +67 -9
|
@@ -82,6 +82,7 @@ runtime layouts.
|
|
|
82
82
|
| `arcane-os/core/image` | Native image Core service with independent runtime preparation and retained model ownership. |
|
|
83
83
|
| `arcane-os/core/speech` | Independent injected STT/TTS engine loading, request cancellation and joined Core service lifetime. |
|
|
84
84
|
| `arcane-os/local-ai/whisper` | Retained native Whisper model, complete recordings and joined helper lifetime. |
|
|
85
|
+
| `arcane-os/local-ai/kokoro` | Retained native Kokoro synthesis using the existing ONNX/model-resource owners and a compiled phonemization/Opus helper. |
|
|
85
86
|
| `arcane-os/local-ai/whisper/build` | First-party persistent helper build against selected installed Whisper headers and libraries. |
|
|
86
87
|
| `arcane-os/local-ai/whisper/openvino-build` | Optional Windows x64 Intel NPU encoder runtime producer using selected installed sources and toolchain. |
|
|
87
88
|
| `arcane-os/ai/core-model-assets` | Complete stored browser model members projected through Core into native working files. |
|
|
@@ -469,6 +470,12 @@ The remaining data-only subpaths are eight JSON Schemas and package metadata.
|
|
|
469
470
|
| `createLocalImageService()` | function | `arcane-os/core/image` | Core local images | Node Core with selected stable-diffusion.cpp requirements |
|
|
470
471
|
| `createLocalImageService default export` | function | `arcane-os/core/image` | Core local images | Same native image service factory |
|
|
471
472
|
| `createSpeechService()` | function | `arcane-os/core/speech` | Core native speech | Node Core with explicitly supplied speech engines |
|
|
473
|
+
| `createNativeKokoroRuntime()` | function | `arcane-os/local-ai/kokoro` | Core native speech | Node with matching prepared helper/data, selected model resources and an existing ONNX owner |
|
|
474
|
+
| `createNativeKokoroRuntime default export` | function | `arcane-os/local-ai/kokoro` | Core native speech | Same retained native Kokoro factory |
|
|
475
|
+
| `KOKORO_MODEL` | constant | `arcane-os/local-ai/kokoro` | Core native speech | Selected upstream model identifier |
|
|
476
|
+
| `KOKORO_REVISION` | constant | `arcane-os/local-ai/kokoro` | Core native speech | Selected upstream model revision |
|
|
477
|
+
| `KOKORO_SAMPLE_RATE` | constant | `arcane-os/local-ai/kokoro` | Core native speech | Native PCM sample rate |
|
|
478
|
+
| `KOKORO_VOICES` | constant | `arcane-os/local-ai/kokoro` | Core native speech | Complete 28-voice US/GB selection |
|
|
472
479
|
| `createWhisperRuntime()` | function | `arcane-os/local-ai/whisper` | Core native speech | Node with a prepared matching helper, runtime, decoder and selected model |
|
|
473
480
|
| `createWhisperRuntime default export` | function | `arcane-os/local-ai/whisper` | Core native speech | Same retained native engine factory |
|
|
474
481
|
| `buildWhisperHelper()` | function | `arcane-os/local-ai/whisper/build` | Core native speech | Node with installed CMake/C++17 toolchain and matching Whisper source/runtime |
|
|
@@ -8842,6 +8849,118 @@ selected model and native engine.
|
|
|
8842
8849
|
The default export from `arcane-os/core/image` is the same
|
|
8843
8850
|
`createLocalImageService` factory and lifecycle described above.
|
|
8844
8851
|
|
|
8852
|
+
## createNativeKokoroRuntime()
|
|
8853
|
+
|
|
8854
|
+
### Overview and signature
|
|
8855
|
+
|
|
8856
|
+
The named export from `arcane-os/local-ai/kokoro` composes the selected Kokoro
|
|
8857
|
+
FP32 graph with the host's existing native ONNX owner, model-resource owner
|
|
8858
|
+
and prepared persistent phonemization/Opus helper. Construction starts no
|
|
8859
|
+
native work and acquires no model or runtime.
|
|
8860
|
+
|
|
8861
|
+
```javascript
|
|
8862
|
+
createNativeKokoroRuntime({
|
|
8863
|
+
onnx, modelAssets, runtime, prepare, modelId = 'kokoro',
|
|
8864
|
+
model = KOKORO_MODEL, revision = KOKORO_REVISION, dtype = 'fp32',
|
|
8865
|
+
paths, assetProjectionId, resourcePaths, sessionOptions,
|
|
8866
|
+
executionPreference = 'gpu', executionTarget, signal, onEvent
|
|
8867
|
+
} = {})
|
|
8868
|
+
```
|
|
8869
|
+
|
|
8870
|
+
Supply the existing `onnx` runtime or a `prepare` callback that provides it.
|
|
8871
|
+
`runtime` describes the matching native helper and data. `paths` supplies
|
|
8872
|
+
complete prepared graph/tokenizer/voice resources; a completed
|
|
8873
|
+
`assetProjectionId` instead retains projected resources through `modelAssets`.
|
|
8874
|
+
The selected model/revision/FP32 contract is fixed by this frontend.
|
|
8875
|
+
See [native Kokoro](native-speech.md) for exact resource mappings, preparation,
|
|
8876
|
+
runtime distribution and model ownership.
|
|
8877
|
+
|
|
8878
|
+
### Members and results
|
|
8879
|
+
|
|
8880
|
+
- `current()` returns complete model/lifecycle state, execution configuration,
|
|
8881
|
+
current request, queue, progress, error, default voice and supported voices.
|
|
8882
|
+
- `subscribe(listener,{replay=true,signal}={})` returns an unsubscribe function
|
|
8883
|
+
and replays current state by default.
|
|
8884
|
+
- `load(options={})` activates the selected resources, retaining omitted
|
|
8885
|
+
selections. Explicit projection/resource mapping or execution-target changes
|
|
8886
|
+
join prior native retirement before loading the new activation.
|
|
8887
|
+
- `synthesize(request,{signal,onProgress,requestId=null}={})` requires a loaded
|
|
8888
|
+
activation. `request` contains the complete `input` string, optional `model`
|
|
8889
|
+
matching `modelId`, `voice` (default `af_heart`), positive finite `speed`
|
|
8890
|
+
(default `1`) and `responseFormat` (`wav` by default, `opus` or `ogg`).
|
|
8891
|
+
The result is `{audioBase64,contentType,sampleRate,channels,model,voice,speed}`;
|
|
8892
|
+
content type is `audio/wav` or `audio/ogg; codecs=opus`, with mono 24kHz PCM
|
|
8893
|
+
synthesis. The full input is processed in ordered model segments.
|
|
8894
|
+
- `unload()` retires the activation and joins owned native work. `close()`
|
|
8895
|
+
permanently closes the engine and releases its state subscriptions.
|
|
8896
|
+
|
|
8897
|
+
Queued requests retain order. Cancelling a queued request removes only that
|
|
8898
|
+
request. Active cancellation waits for actual helper/session retirement;
|
|
8899
|
+
surviving requests resume with the same accepted provider configuration.
|
|
8900
|
+
The ONNX owner may need a running native call to return before its worker exits.
|
|
8901
|
+
Readiness is lost while that retirement/reload occurs. Errors retain the actual
|
|
8902
|
+
engine and cleanup failures; unsupported explicit voices or formats are never
|
|
8903
|
+
substituted. No audio is played by this engine.
|
|
8904
|
+
|
|
8905
|
+
### Example
|
|
8906
|
+
|
|
8907
|
+
```javascript
|
|
8908
|
+
import {createNativeKokoroRuntime} from 'arcane-os/local-ai/kokoro';
|
|
8909
|
+
|
|
8910
|
+
// The host supplies its existing ONNX owner and already prepared resources.
|
|
8911
|
+
async function narrateMoonGoat({onnx, runtime, paths}) {
|
|
8912
|
+
const voice = createNativeKokoroRuntime({
|
|
8913
|
+
onnx, runtime, paths, executionPreference: 'cpu'
|
|
8914
|
+
});
|
|
8915
|
+
try {
|
|
8916
|
+
await voice.load();
|
|
8917
|
+
return await voice.synthesize({
|
|
8918
|
+
input: 'Captain Goat has landed. Guard the moon cheese!',
|
|
8919
|
+
voice: 'af_heart', responseFormat: 'wav'
|
|
8920
|
+
});
|
|
8921
|
+
} finally {
|
|
8922
|
+
await voice.close();
|
|
8923
|
+
}
|
|
8924
|
+
}
|
|
8925
|
+
```
|
|
8926
|
+
|
|
8927
|
+
Changing `voice` to `bf_emma` selects British English. `responseFormat: 'opus'`
|
|
8928
|
+
selects Ogg Opus encoding. GPU/physical-device selection uses the existing
|
|
8929
|
+
[ONNX execution contract](local-ai.md#physical-execution-targets); an accepted
|
|
8930
|
+
provider configuration is separate from observed hardware execution.
|
|
8931
|
+
The SDK publishes a Windows x64 helper archive. Other platforms need a matching
|
|
8932
|
+
custom helper distribution and supported native ONNX host. Native helper
|
|
8933
|
+
production and platform execution evidence remain separate.
|
|
8934
|
+
|
|
8935
|
+
## createNativeKokoroRuntime default export
|
|
8936
|
+
|
|
8937
|
+
The default export from `arcane-os/local-ai/kokoro` is the same
|
|
8938
|
+
`createNativeKokoroRuntime` factory with identical inputs, results and lifetime.
|
|
8939
|
+
|
|
8940
|
+
## KOKORO_MODEL
|
|
8941
|
+
|
|
8942
|
+
Named string export from `arcane-os/local-ai/kokoro`, with value
|
|
8943
|
+
`onnx-community/Kokoro-82M-v1.0-ONNX`. It identifies this frontend's model family
|
|
8944
|
+
for matching graph, tokenizer and voice resource selection.
|
|
8945
|
+
|
|
8946
|
+
## KOKORO_REVISION
|
|
8947
|
+
|
|
8948
|
+
Named string export from `arcane-os/local-ai/kokoro`, with value
|
|
8949
|
+
`1939ad2a8e416c0acfeecc08a694d14ef25f2231`. Resource preparation uses this
|
|
8950
|
+
upstream revision with `KOKORO_MODEL`.
|
|
8951
|
+
|
|
8952
|
+
## KOKORO_SAMPLE_RATE
|
|
8953
|
+
|
|
8954
|
+
Named number export from `arcane-os/local-ai/kokoro`, with value `24000`.
|
|
8955
|
+
It describes the engine's mono PCM synthesis rate in samples per second.
|
|
8956
|
+
|
|
8957
|
+
## KOKORO_VOICES
|
|
8958
|
+
|
|
8959
|
+
Named array export from `arcane-os/local-ai/kokoro` containing all 28 supported
|
|
8960
|
+
US/GB voice identifiers. The engine uses `af_heart` when the caller omits
|
|
8961
|
+
`voice`; an explicit unsupported selection reports an error. Use this complete
|
|
8962
|
+
array to populate a native Kokoro voice selector.
|
|
8963
|
+
|
|
8845
8964
|
## createWhisperRuntime()
|
|
8846
8965
|
|
|
8847
8966
|
### Overview
|
|
@@ -9018,6 +9137,13 @@ The returned object has `name: 'speech'`, `current`, `subscribe`, `close`,
|
|
|
9018
9137
|
Core RPC methods. The context supplies `signal`, `requestId` and `emit`.
|
|
9019
9138
|
The service publishes synchronous accepted `Thinking` before waiting for
|
|
9020
9139
|
its role's load, then forwards the engine's complete progress and result.
|
|
9140
|
+
- `methods['speech.load'](request,context)` and
|
|
9141
|
+
`methods['speech.unload'](request,context)` route explicit activation to the
|
|
9142
|
+
explicit `role: 'stt' | 'tts'` and forward the remaining activation
|
|
9143
|
+
fields and result through that role's configured engine. A startup load remains owned independently of callers
|
|
9144
|
+
waiting for it. The browser facade exposes `Arcane.speech.load(request,options)`
|
|
9145
|
+
and `unload(request,options)` with `signal`, `onRequest` and no elapsed-time
|
|
9146
|
+
cutoff. See [activation routing](native-speech.md) for selection and errors.
|
|
9021
9147
|
- `close()` and its `drain()` alias stop acceptance, abort the service lifetime,
|
|
9022
9148
|
close both engines concurrently and await startup and request work. Repeated
|
|
9023
9149
|
calls share one promise. `dispose()` also releases event subscriptions.
|
|
@@ -9123,7 +9249,8 @@ import createLocalAIService from 'arcane-os/core/local-ai';
|
|
|
9123
9249
|
### Overview
|
|
9124
9250
|
|
|
9125
9251
|
Reuses matching managed runtimes or installs complete selected
|
|
9126
|
-
llama.cpp/Ollama/ONNX/image distributions
|
|
9252
|
+
llama.cpp/Ollama/ONNX/image distributions, NeMo libraries or native Kokoro
|
|
9253
|
+
helper/data trees in an explicitly supplied directory.
|
|
9127
9254
|
|
|
9128
9255
|
### Signature and result
|
|
9129
9256
|
|
|
@@ -9146,6 +9273,12 @@ trees remain intact while a fresh installation is prepared. See
|
|
|
9146
9273
|
Image requirements also accept `backend` and
|
|
9147
9274
|
`models`; their pinned default runtime, CPU/Metal selection and separate model
|
|
9148
9275
|
preparation are described in [local image generation](local-image-generation.md).
|
|
9276
|
+
`kokoro-native` returns `helperExecutable`, `espeakDataDirectory` and
|
|
9277
|
+
`libraryDirectory`. Windows x64 defaults to the helper archive from the
|
|
9278
|
+
installed SDK's numeric release; an explicit `url` or `helperRoot` selects a
|
|
9279
|
+
complete custom distribution. The data directory is the parent of
|
|
9280
|
+
`espeak-ng-data`. Helper preparation loads no Kokoro graph and keeps the
|
|
9281
|
+
existing ONNX/model-resource owners separate. See [native speech](native-speech.md).
|
|
9149
9282
|
Matching concurrent
|
|
9150
9283
|
installs share their work. Downloads retain runtime libraries and surface
|
|
9151
9284
|
network/archive failures; cancellation and operation events remain observable.
|
|
@@ -9189,6 +9322,10 @@ The corrected Windows x64 ONNX selection retains its
|
|
|
9189
9322
|
is relocated with the other runtime paths; bundling introduces no separate
|
|
9190
9323
|
runtime selection. See [ONNX runtime preparation](local-ai.md#development).
|
|
9191
9324
|
|
|
9325
|
+
Kokoro bundling retains the complete helper/data tree and relocates
|
|
9326
|
+
`helperExecutable`, `espeakDataDirectory` and `libraryDirectory`. Its selected
|
|
9327
|
+
ONNX runtime and model resources retain their existing independent ownership.
|
|
9328
|
+
|
|
9192
9329
|
Image records relocate the complete library, binding and variant paths. Model
|
|
9193
9330
|
URL descriptors retain their original acquisition metadata for preparation
|
|
9194
9331
|
after launch. Runtime-contained native model paths become artifact-relative;
|
|
@@ -11683,10 +11820,11 @@ the message or response callback payload.
|
|
|
11683
11820
|
|
|
11684
11821
|
**Node and Browser; Cloud transport.** Uses standard Fetch and cancellation,
|
|
11685
11822
|
without DOM, browser profiles, user singletons, or storage initialization.
|
|
11686
|
-
Import starts no request. HTTP `429`
|
|
11687
|
-
|
|
11688
|
-
`
|
|
11689
|
-
|
|
11823
|
+
Import starts no request. Rejected Fetch and all HTTP `429`/`529` share three
|
|
11824
|
+
retries. Quota recovery honors `Retry-After`, otherwise waits `3000`, `6000`,
|
|
11825
|
+
or `12000` milliseconds by retry number. Active same-endpoint/credential calls
|
|
11826
|
+
share quota cooldown and paced dispatch. Complete error bodies remain unchanged
|
|
11827
|
+
on exhaustion; other HTTP failures are not retried. A missing key throws
|
|
11690
11828
|
`AI_PROVIDER_NOT_CONFIGURED`; a missing model throws `TypeError`. `signal`
|
|
11691
11829
|
cancellation during transport, body reading, retry waiting,
|
|
11692
11830
|
or callback settlement prevents successful return and uses
|
|
@@ -11836,21 +11974,24 @@ Ordinary TWiN callers use `fetchRequest()` instead.
|
|
|
11836
11974
|
### Signature and result
|
|
11837
11975
|
|
|
11838
11976
|
```text
|
|
11839
|
-
async fetchHTTPResponse(url,options)
|
|
11977
|
+
async fetchHTTPResponse(url,options,{onRetry=null}={})
|
|
11840
11978
|
```
|
|
11841
11979
|
|
|
11842
11980
|
Returns the successful Fetch `Response` without consuming its body. Both URL
|
|
11843
11981
|
and Fetch options are caller-supplied; this helper adds no key, model, or
|
|
11844
11982
|
request envelope. Non-success responses are read completely as JSON when the
|
|
11845
|
-
content type contains `application/json`, otherwise as text.
|
|
11846
|
-
|
|
11983
|
+
content type contains `application/json`, otherwise as text. Rejected Fetch
|
|
11984
|
+
and all `429`/`529` share three retries with complete terminal errors.
|
|
11985
|
+
`429` honors `Retry-After` or exponential delay and coordinates a transient
|
|
11986
|
+
same-endpoint/credential cooldown. See [shared request behavior](ai/twin-cloud.md#shared-request-behavior).
|
|
11847
11987
|
|
|
11848
11988
|
### Availability and normalization
|
|
11849
11989
|
|
|
11850
11990
|
**Node and Browser.** Exported from `arcane-os/ai/twin-cloud` for shared SDK
|
|
11851
|
-
integration. `options.signal` cancels Fetch and
|
|
11991
|
+
integration. `options.signal` cancels Fetch and recovery/pacing waits and is checked
|
|
11852
11992
|
after response/error-body reads. Cancellation uses
|
|
11853
|
-
`ARCANE_AI_REQUEST_ABORTED`.
|
|
11993
|
+
`ARCANE_AI_REQUEST_ABORTED`. Retry warnings use the existing shared logger;
|
|
11994
|
+
`onRetry` observes complete waiting/requesting records outside the payload.
|
|
11854
11995
|
|
|
11855
11996
|
### Example
|
|
11856
11997
|
|
|
@@ -11870,7 +12011,7 @@ Shared complete JSON-body reader built on `fetchHTTPResponse()`.
|
|
|
11870
12011
|
### Signature and result
|
|
11871
12012
|
|
|
11872
12013
|
```text
|
|
11873
|
-
async fetchJSONResponse(url,options)
|
|
12014
|
+
async fetchJSONResponse(url,options,{onRetry=null}={})
|
|
11874
12015
|
```
|
|
11875
12016
|
|
|
11876
12017
|
Returns the full parsed JSON value. A successful response whose content type
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "arcane-os",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.95.0",
|
|
4
4
|
"description": "Arcane OS JavaScript SDK, project-local CLI, browser runtime, and repository-portable application packager.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.mjs",
|
|
@@ -89,6 +89,11 @@
|
|
|
89
89
|
"./local-ai/decisions": "./src/local-ai/decisions.mjs",
|
|
90
90
|
"./local-ai/image": "./src/local-ai/image-runtime.mjs",
|
|
91
91
|
"./local-ai/whisper": "./src/local-ai/whisper/index.mjs",
|
|
92
|
+
"./local-ai/kokoro": "./src/local-ai/kokoro/index.mjs",
|
|
93
|
+
"./local-ai/nemotron": "./src/local-ai/nemotron/index.mjs",
|
|
94
|
+
"./local-ai/nemotron/build": "./src/local-ai/nemotron/build.mjs",
|
|
95
|
+
"./local-ai/magpie": "./src/local-ai/magpie/index.mjs",
|
|
96
|
+
"./local-ai/magpie/build": "./src/local-ai/magpie/build.mjs",
|
|
92
97
|
"./local-ai/whisper/build": "./src/local-ai/whisper/build.mjs",
|
|
93
98
|
"./local-ai/whisper/openvino-build": "./src/local-ai/whisper/openvino-build.mjs",
|
|
94
99
|
"./integrated-provider": "./src/integrated-provider-loader.mjs",
|
|
@@ -28,6 +28,8 @@ import {
|
|
|
28
28
|
import {MarkdownSpeech,stripSpeechFormatting} from 'arcane-os/speech-text';
|
|
29
29
|
import {prepareSpeech, readPreparedSpeech} from './PreparedSpeech.js';
|
|
30
30
|
import {createToolTextObserver} from 'arcane-os/ai/tool-text-stream';
|
|
31
|
+
import createNativeSpeechCapture from './NativeSpeechCapture.js';
|
|
32
|
+
import prepareNativeSpeechPlayback from './NativeSpeechPlayback.js';
|
|
31
33
|
|
|
32
34
|
const completeValue=(value)=>value;
|
|
33
35
|
|
|
@@ -1150,14 +1152,18 @@ class AI {
|
|
|
1150
1152
|
}
|
|
1151
1153
|
);
|
|
1152
1154
|
|
|
1155
|
+
const speechOnly=llmService===null&&model===null;
|
|
1153
1156
|
const preferences=[
|
|
1154
|
-
llmService||'TWIN',
|
|
1157
|
+
speechOnly?'':llmService||'TWIN',
|
|
1155
1158
|
sttService||'LOCAL_SPEACH',
|
|
1156
1159
|
ttsService||'LOCAL_SPEACH',
|
|
1157
|
-
model||'TWIN',
|
|
1160
|
+
speechOnly?'':model||'TWIN',
|
|
1158
1161
|
modelTTS||'LOCAL_SPEACH',
|
|
1159
1162
|
modelSTT||'LOCAL_SPEACH'
|
|
1160
1163
|
];
|
|
1164
|
+
// Explicitly absent LLM selection must not briefly register or load a
|
|
1165
|
+
// default provider. setAI retains empty entries from this initial tuple.
|
|
1166
|
+
if(speechOnly)this.#preferenceTuple=preferences.slice();
|
|
1161
1167
|
this.setAI(
|
|
1162
1168
|
...preferences
|
|
1163
1169
|
);
|
|
@@ -2008,13 +2014,15 @@ class AI {
|
|
|
2008
2014
|
if(runtime.#builtInSpeechService(role)!==providerId||!model){
|
|
2009
2015
|
return completeValue([]);
|
|
2010
2016
|
}
|
|
2011
|
-
const defaultVoice=runtime.#builtInSpeechDefaultVoice(
|
|
2017
|
+
const defaultVoice=nativeWatch?.snapshot?.roles?.[role]?.defaultVoice??runtime.#builtInSpeechDefaultVoice(
|
|
2012
2018
|
role,
|
|
2013
2019
|
providerId
|
|
2014
2020
|
);
|
|
2015
2021
|
return completeValue([
|
|
2016
2022
|
completeValue({
|
|
2017
2023
|
id:model,
|
|
2024
|
+
...(nativeWatch?.snapshot?.roles?.[role]?.speech
|
|
2025
|
+
?{speech:nativeWatch.snapshot.roles[role].speech}:{}),
|
|
2018
2026
|
...(defaultVoice?{defaultVoice}:{})
|
|
2019
2027
|
})
|
|
2020
2028
|
]);
|
|
@@ -2026,6 +2034,39 @@ class AI {
|
|
|
2026
2034
|
return runtime.#builtInSpeechInspection(role,providerId,selection);
|
|
2027
2035
|
},
|
|
2028
2036
|
status:statusBuiltInSpeechProvider,
|
|
2037
|
+
...(role==='stt'?{
|
|
2038
|
+
supportsCapture:function supportsNativeCapture(){
|
|
2039
|
+
return nativeWatch?.snapshot?.roles?.stt?.supportsStreaming===true;
|
|
2040
|
+
},
|
|
2041
|
+
createCapture:function createBuiltInNativeCapture(options={}){
|
|
2042
|
+
if(nativeWatch?.snapshot?.roles?.stt?.supportsStreaming!==true)return null;
|
|
2043
|
+
const capture=createNativeSpeechCapture({
|
|
2044
|
+
...options,
|
|
2045
|
+
speech:nativeWatch.speech,
|
|
2046
|
+
model:nativeWatch.modelId
|
|
2047
|
+
});
|
|
2048
|
+
const operation={watch:nativeWatch,requestId:null,signal:null};
|
|
2049
|
+
const done=capture.done.finally(function releaseNativeCapture(){
|
|
2050
|
+
if(nativeRequest===operation)releaseBuiltInSpeechRequest();
|
|
2051
|
+
});
|
|
2052
|
+
done.catch(function observeNativeCaptureCompletion(error){
|
|
2053
|
+
if(!isAIRequestAbort(error))arcaneLogging.error('Native transcription capture failed.',error);
|
|
2054
|
+
});
|
|
2055
|
+
return {
|
|
2056
|
+
...capture,
|
|
2057
|
+
done,
|
|
2058
|
+
start:function startBuiltInNativeCapture(options={}){
|
|
2059
|
+
if(!nativeSpeechReady()||busy){
|
|
2060
|
+
throw aiProviderError('The native transcription model is not ready.','ARCANE_AI_ROLE_NOT_READY');
|
|
2061
|
+
}
|
|
2062
|
+
busy=true;
|
|
2063
|
+
operation.signal=options.signal;
|
|
2064
|
+
nativeRequest=operation;
|
|
2065
|
+
return capture.start(options);
|
|
2066
|
+
}
|
|
2067
|
+
};
|
|
2068
|
+
}
|
|
2069
|
+
}:{}),
|
|
2029
2070
|
load:async function loadBuiltInSpeechProvider(context={}){
|
|
2030
2071
|
if(context.signal?.aborted){
|
|
2031
2072
|
throw normalizeAIRequestAbort(context.signal.reason);
|
|
@@ -2086,7 +2127,7 @@ class AI {
|
|
|
2086
2127
|
throw error;
|
|
2087
2128
|
}
|
|
2088
2129
|
},
|
|
2089
|
-
request:function requestBuiltInSpeechProvider(context={}){
|
|
2130
|
+
request:function requestBuiltInSpeechProvider(context={}, {speechStream=null}={}){
|
|
2090
2131
|
if(context.signal?.aborted){
|
|
2091
2132
|
throw normalizeAIRequestAbort(context.signal.reason);
|
|
2092
2133
|
}
|
|
@@ -2126,7 +2167,8 @@ class AI {
|
|
|
2126
2167
|
)
|
|
2127
2168
|
:runtime.#requestBuiltInSpeechSynthesis(
|
|
2128
2169
|
context.payload,
|
|
2129
|
-
context.signal
|
|
2170
|
+
context.signal,
|
|
2171
|
+
speechStream
|
|
2130
2172
|
);
|
|
2131
2173
|
return Promise.resolve(request).finally(releaseBuiltInSpeechRequest);
|
|
2132
2174
|
},
|
|
@@ -2548,6 +2590,7 @@ class AI {
|
|
|
2548
2590
|
|
|
2549
2591
|
audioMessageChunks='';
|
|
2550
2592
|
#markdownSpeech=new MarkdownSpeech();
|
|
2593
|
+
#streamSpeechOptions=null;
|
|
2551
2594
|
sourceNodes=[];
|
|
2552
2595
|
isSpeaking=false;
|
|
2553
2596
|
audioContext=null;
|
|
@@ -4684,7 +4727,7 @@ class AI {
|
|
|
4684
4727
|
return response.text();
|
|
4685
4728
|
}
|
|
4686
4729
|
|
|
4687
|
-
async #requestBuiltInSpeechSynthesis(payload={},signal=null){
|
|
4730
|
+
async #requestBuiltInSpeechSynthesis(payload={},signal=null,speechStream=null){
|
|
4688
4731
|
const input=is.string(payload?.input)?payload.input:'';
|
|
4689
4732
|
if(!input){
|
|
4690
4733
|
throw new TypeError('Speech synthesis requires nonempty input.');
|
|
@@ -4703,9 +4746,15 @@ class AI {
|
|
|
4703
4746
|
const speed=is.finite(payload.speed)?payload.speed:this.voiceSpeed;
|
|
4704
4747
|
const nativeSpeech=this.#nativeSpeech(this.ttsService,'tts');
|
|
4705
4748
|
if(nativeSpeech){
|
|
4749
|
+
if(speechStream){
|
|
4750
|
+
return nativeSpeech.synthesizeStream({
|
|
4751
|
+
...payload,model,voice,input,speed
|
|
4752
|
+
},{...speechStream,signal});
|
|
4753
|
+
}
|
|
4706
4754
|
let response;
|
|
4707
4755
|
try{
|
|
4708
4756
|
response=await nativeSpeech.synthesize({
|
|
4757
|
+
...payload,
|
|
4709
4758
|
model,
|
|
4710
4759
|
voice,
|
|
4711
4760
|
input,
|
|
@@ -6871,12 +6920,24 @@ class AI {
|
|
|
6871
6920
|
if(end){
|
|
6872
6921
|
this.audioMessageChunks='';
|
|
6873
6922
|
this.#markdownSpeech.reset();
|
|
6923
|
+
this.#streamSpeechOptions=null;
|
|
6874
6924
|
}
|
|
6875
6925
|
this.#traceSpeech('streamTTS.result',{callId,result:false,reason:'muted'});
|
|
6876
6926
|
return Promise.resolve(false);
|
|
6877
6927
|
}
|
|
6878
6928
|
|
|
6879
|
-
|
|
6929
|
+
// A final flush belongs to the same utterance selection. An explicit
|
|
6930
|
+
// language change drains the previous text before accepting new text.
|
|
6931
|
+
let preceding=null;
|
|
6932
|
+
if(this.#streamSpeechOptions&&Object.hasOwn(options,'languageCode')
|
|
6933
|
+
&&options.languageCode!==this.#streamSpeechOptions.languageCode){
|
|
6934
|
+
preceding=this.streamTTS('',true,this.#streamSpeechOptions);
|
|
6935
|
+
}
|
|
6936
|
+
const selectedOptions={...this.#streamSpeechOptions,...options};
|
|
6937
|
+
this.#streamSpeechOptions={
|
|
6938
|
+
voice:selectedOptions.voice,speed:selectedOptions.speed,languageCode:selectedOptions.languageCode
|
|
6939
|
+
};
|
|
6940
|
+
const {voice,speed,languageCode,waitForPlayback=false}=selectedOptions;
|
|
6880
6941
|
const pauseAfterMs=Number(options.pauseAfterMs??0);
|
|
6881
6942
|
if(!is.finite(pauseAfterMs)||pauseAfterMs<0){
|
|
6882
6943
|
throw new RangeError('Speech pauses must be a nonnegative number of milliseconds.');
|
|
@@ -6885,6 +6946,7 @@ class AI {
|
|
|
6885
6946
|
const speechText=this.#markdownSpeech.append(String(text||''),end);
|
|
6886
6947
|
this.audioMessageChunks+=speechText;
|
|
6887
6948
|
const outputs=this.#extractSpeechSegments(end);
|
|
6949
|
+
if(end)this.#streamSpeechOptions=null;
|
|
6888
6950
|
this.#traceSpeech('streamTTS.segments',{
|
|
6889
6951
|
callId,segments:outputs,remainder:this.audioMessageChunks,
|
|
6890
6952
|
segmentation:this.ttsSegmentation,speechText
|
|
@@ -6892,7 +6954,7 @@ class AI {
|
|
|
6892
6954
|
|
|
6893
6955
|
if(!outputs.length){
|
|
6894
6956
|
this.#traceSpeech('streamTTS.result',{callId,result:true,reason:'buffered'});
|
|
6895
|
-
return Promise.resolve(true);
|
|
6957
|
+
return preceding??Promise.resolve(true);
|
|
6896
6958
|
}
|
|
6897
6959
|
|
|
6898
6960
|
try{
|
|
@@ -6909,7 +6971,7 @@ class AI {
|
|
|
6909
6971
|
}
|
|
6910
6972
|
|
|
6911
6973
|
const generation=this.speechGeneration;
|
|
6912
|
-
const jobs=[];
|
|
6974
|
+
const jobs=preceding?[preceding]:[];
|
|
6913
6975
|
const runtime=this;
|
|
6914
6976
|
|
|
6915
6977
|
for(const [index,output] of outputs.entries()){
|
|
@@ -6917,6 +6979,7 @@ class AI {
|
|
|
6917
6979
|
diagnosticCallId:callId,
|
|
6918
6980
|
voice,
|
|
6919
6981
|
speed,
|
|
6982
|
+
languageCode,
|
|
6920
6983
|
waitForPlayback,
|
|
6921
6984
|
pauseAfterMs:index===outputs.length-1?pauseAfterMs:0
|
|
6922
6985
|
}));
|
|
@@ -6938,6 +7001,7 @@ class AI {
|
|
|
6938
7001
|
for(const playback of this.#preparedSpeechPlaybacks) playback.stop();
|
|
6939
7002
|
this.audioMessageChunks='';
|
|
6940
7003
|
this.#markdownSpeech.reset();
|
|
7004
|
+
this.#streamSpeechOptions=null;
|
|
6941
7005
|
return;
|
|
6942
7006
|
}
|
|
6943
7007
|
this.finishTTS();
|
|
@@ -7111,6 +7175,7 @@ class AI {
|
|
|
7111
7175
|
text,
|
|
7112
7176
|
voice:options.voice,
|
|
7113
7177
|
speed:options.speed,
|
|
7178
|
+
languageCode:options.languageCode,
|
|
7114
7179
|
pauseAfterMs:options.pauseAfterMs??0,
|
|
7115
7180
|
resolvePlayback:null
|
|
7116
7181
|
};
|
|
@@ -7206,6 +7271,7 @@ class AI {
|
|
|
7206
7271
|
const payload={
|
|
7207
7272
|
model:selection?.modelId||this.modelTTS,
|
|
7208
7273
|
input:job.text,
|
|
7274
|
+
...(job.languageCode!==undefined?{languageCode:job.languageCode}:{}),
|
|
7209
7275
|
...(job.voice!==undefined||voice?{voice}:{}),
|
|
7210
7276
|
...(!native?{responseFormat:selection
|
|
7211
7277
|
?this.#providerSpeechResponseFormat()
|
|
@@ -7570,6 +7636,34 @@ class AI {
|
|
|
7570
7636
|
|
|
7571
7637
|
async prepareTTSPlayback(payload={},signal=null,preparation={}){
|
|
7572
7638
|
const model=this.#providerSpeechModel();
|
|
7639
|
+
if(model?.speech?.streaming===true){
|
|
7640
|
+
this.#assertServiceConfigured(this.ttsService,'tts');
|
|
7641
|
+
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
7642
|
+
const runtime=this;
|
|
7643
|
+
const selection=this.#providerRuntime.selection('tts');
|
|
7644
|
+
return prepareNativeSpeechPlayback({
|
|
7645
|
+
request:{
|
|
7646
|
+
...payload,
|
|
7647
|
+
model:payload.model??selection.modelId,
|
|
7648
|
+
voice:payload.voice??this.#providerSpeechVoice(),
|
|
7649
|
+
speed:payload.speed??this.voiceSpeed
|
|
7650
|
+
},
|
|
7651
|
+
signal,
|
|
7652
|
+
getAudioContext:function getOwnedSpeechOutput(){
|
|
7653
|
+
return runtime.#getSpeechAudioContext();
|
|
7654
|
+
},
|
|
7655
|
+
speech:{
|
|
7656
|
+
synthesizeStream:function requestNativeSpeechStream(request,options){
|
|
7657
|
+
return runtime.#providerRuntime.request('tts',{
|
|
7658
|
+
operation:'synthesize',payload:request,localOnly:false,signal:options.signal
|
|
7659
|
+
},{
|
|
7660
|
+
...preparation,
|
|
7661
|
+
speechStream:{onAudio:options.onAudio,onProgress:options.onProgress,onRequest:options.onRequest}
|
|
7662
|
+
});
|
|
7663
|
+
}
|
|
7664
|
+
}
|
|
7665
|
+
});
|
|
7666
|
+
}
|
|
7573
7667
|
if(model?.speech?.playback!=='native'){
|
|
7574
7668
|
return this.fetchTTS(payload,signal,preparation);
|
|
7575
7669
|
}
|
|
@@ -7618,6 +7712,7 @@ class AI {
|
|
|
7618
7712
|
'input',
|
|
7619
7713
|
'textFormat',
|
|
7620
7714
|
'responseFormat',
|
|
7715
|
+
'languageCode',
|
|
7621
7716
|
'speed'
|
|
7622
7717
|
]);
|
|
7623
7718
|
for(const key of Reflect.ownKeys(descriptors)){
|
|
@@ -7625,7 +7720,7 @@ class AI {
|
|
|
7625
7720
|
||!acceptedKeys.has(key)
|
|
7626
7721
|
||!Object.hasOwn(descriptors[key],'value')){
|
|
7627
7722
|
const error=new TypeError(
|
|
7628
|
-
'AI.fetchTTS accepts only model, voice, input, textFormat, responseFormat, and speed data properties.'
|
|
7723
|
+
'AI.fetchTTS accepts only model, voice, input, textFormat, responseFormat, languageCode, and speed data properties.'
|
|
7629
7724
|
);
|
|
7630
7725
|
error.code='ARCANE_AI_TTS_REQUEST_INVALID';
|
|
7631
7726
|
throw error;
|
|
@@ -7732,15 +7827,19 @@ class AI {
|
|
|
7732
7827
|
throw error;
|
|
7733
7828
|
}
|
|
7734
7829
|
|
|
7830
|
+
const speechRequest={
|
|
7831
|
+
model,voice,input,responseFormat,speed,
|
|
7832
|
+
...(descriptors.languageCode?{languageCode:descriptors.languageCode.value}:{})
|
|
7833
|
+
};
|
|
7735
7834
|
if(selection){
|
|
7736
7835
|
this.#traceSpeech('fetchTTS.dispatch',{
|
|
7737
|
-
callId,selection,payload:
|
|
7836
|
+
callId,selection,payload:speechRequest
|
|
7738
7837
|
});
|
|
7739
7838
|
const response=await this.#providerRuntime.request(
|
|
7740
7839
|
'tts',
|
|
7741
7840
|
{
|
|
7742
7841
|
operation:'synthesize',
|
|
7743
|
-
payload:
|
|
7842
|
+
payload:speechRequest,
|
|
7744
7843
|
localOnly:false,
|
|
7745
7844
|
signal
|
|
7746
7845
|
},
|
|
@@ -7755,10 +7854,10 @@ class AI {
|
|
|
7755
7854
|
}
|
|
7756
7855
|
|
|
7757
7856
|
this.#traceSpeech('fetchTTS.dispatch',{
|
|
7758
|
-
callId,service:this.ttsService,payload:
|
|
7857
|
+
callId,service:this.ttsService,payload:speechRequest
|
|
7759
7858
|
});
|
|
7760
7859
|
const audio=await this.#requestBuiltInSpeechSynthesis(
|
|
7761
|
-
|
|
7860
|
+
speechRequest,
|
|
7762
7861
|
signal
|
|
7763
7862
|
);
|
|
7764
7863
|
this.#traceSpeech('fetchTTS.result',{callId,audio});
|
|
@@ -7859,6 +7958,7 @@ class AI {
|
|
|
7859
7958
|
this.speechScheduleTime=0;
|
|
7860
7959
|
this.audioMessageChunks='';
|
|
7861
7960
|
this.#markdownSpeech.reset();
|
|
7961
|
+
this.#streamSpeechOptions=null;
|
|
7862
7962
|
this.#clearSpeechUnlock();
|
|
7863
7963
|
|
|
7864
7964
|
for(const job of this.speechJobs){
|