arcane-os 0.93.0 → 0.95.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.md +1 -1
  3. package/browser-runtime/ai/twin-cloud.mjs +109 -46
  4. package/browser-runtime/core/client.mjs +197 -0
  5. package/docs/reference/ai/twin-cloud.md +35 -14
  6. package/docs/reference/availability-and-normalization.md +1 -1
  7. package/docs/reference/inventory/package-api.json +96 -12
  8. package/docs/reference/local-ai.md +81 -5
  9. package/docs/reference/native-magpie.md +98 -0
  10. package/docs/reference/native-nemotron.md +204 -0
  11. package/docs/reference/native-speech.md +364 -11
  12. package/docs/reference/runtime-modules.md +21 -18
  13. package/docs/reference/sdk-api.md +152 -11
  14. package/package.json +6 -1
  15. package/runtime/arcane/modules/AI.js +114 -14
  16. package/runtime/arcane/modules/AIProviderRuntime.js +32 -15
  17. package/runtime/arcane/modules/ContinuousVoiceCapture.js +118 -21
  18. package/runtime/arcane/modules/NativeSpeechCapture.js +198 -0
  19. package/runtime/arcane/modules/NativeSpeechPlayback.js +304 -0
  20. package/runtime/arcane/modules/VoiceCaptureWorklet.js +62 -2
  21. package/schemas/arcane-app.schema.json +2 -2
  22. package/schemas/arcane-package.schema.json +2 -2
  23. package/src/core/services/speech.mjs +320 -25
  24. package/src/local-ai/config.mjs +12 -1
  25. package/src/local-ai/dev.mjs +71 -4
  26. package/src/local-ai/install.mjs +113 -5
  27. package/src/local-ai/kokoro/frontend.mjs +219 -0
  28. package/src/local-ai/kokoro/helper.mjs +168 -0
  29. package/src/local-ai/kokoro/index.mjs +645 -0
  30. package/src/local-ai/kokoro/native/CMakeLists.txt +45 -0
  31. package/src/local-ai/kokoro/native/main.cpp +249 -0
  32. package/src/local-ai/magpie/build.mjs +36 -0
  33. package/src/local-ai/magpie/helper.mjs +254 -0
  34. package/src/local-ai/magpie/index.mjs +488 -0
  35. package/src/local-ai/magpie/native/CMakeLists.txt +10 -0
  36. package/src/local-ai/magpie/native/main.cpp +270 -0
  37. package/src/local-ai/nemotron/build.mjs +53 -0
  38. package/src/local-ai/nemotron/helper.mjs +308 -0
  39. package/src/local-ai/nemotron/index.mjs +637 -0
  40. package/src/local-ai/nemotron/native/CMakeLists.txt +17 -0
  41. package/src/local-ai/nemotron/native/main.cpp +282 -0
  42. package/src/native/portable-layout.mjs +67 -9
@@ -82,6 +82,7 @@ runtime layouts.
82
82
  | `arcane-os/core/image` | Native image Core service with independent runtime preparation and retained model ownership. |
83
83
  | `arcane-os/core/speech` | Independent injected STT/TTS engine loading, request cancellation and joined Core service lifetime. |
84
84
  | `arcane-os/local-ai/whisper` | Retained native Whisper model, complete recordings and joined helper lifetime. |
85
+ | `arcane-os/local-ai/kokoro` | Retained native Kokoro synthesis using the existing ONNX/model-resource owners and a compiled phonemization/Opus helper. |
85
86
  | `arcane-os/local-ai/whisper/build` | First-party persistent helper build against selected installed Whisper headers and libraries. |
86
87
  | `arcane-os/local-ai/whisper/openvino-build` | Optional Windows x64 Intel NPU encoder runtime producer using selected installed sources and toolchain. |
87
88
  | `arcane-os/ai/core-model-assets` | Complete stored browser model members projected through Core into native working files. |
@@ -469,6 +470,12 @@ The remaining data-only subpaths are eight JSON Schemas and package metadata.
469
470
  | `createLocalImageService()` | function | `arcane-os/core/image` | Core local images | Node Core with selected stable-diffusion.cpp requirements |
470
471
  | `createLocalImageService default export` | function | `arcane-os/core/image` | Core local images | Same native image service factory |
471
472
  | `createSpeechService()` | function | `arcane-os/core/speech` | Core native speech | Node Core with explicitly supplied speech engines |
473
+ | `createNativeKokoroRuntime()` | function | `arcane-os/local-ai/kokoro` | Core native speech | Node with matching prepared helper/data, selected model resources and an existing ONNX owner |
474
+ | `createNativeKokoroRuntime default export` | function | `arcane-os/local-ai/kokoro` | Core native speech | Same retained native Kokoro factory |
475
+ | `KOKORO_MODEL` | constant | `arcane-os/local-ai/kokoro` | Core native speech | Selected upstream model identifier |
476
+ | `KOKORO_REVISION` | constant | `arcane-os/local-ai/kokoro` | Core native speech | Selected upstream model revision |
477
+ | `KOKORO_SAMPLE_RATE` | constant | `arcane-os/local-ai/kokoro` | Core native speech | Native PCM sample rate |
478
+ | `KOKORO_VOICES` | constant | `arcane-os/local-ai/kokoro` | Core native speech | Complete 28-voice US/GB selection |
472
479
  | `createWhisperRuntime()` | function | `arcane-os/local-ai/whisper` | Core native speech | Node with a prepared matching helper, runtime, decoder and selected model |
473
480
  | `createWhisperRuntime default export` | function | `arcane-os/local-ai/whisper` | Core native speech | Same retained native engine factory |
474
481
  | `buildWhisperHelper()` | function | `arcane-os/local-ai/whisper/build` | Core native speech | Node with installed CMake/C++17 toolchain and matching Whisper source/runtime |
@@ -8842,6 +8849,118 @@ selected model and native engine.
8842
8849
  The default export from `arcane-os/core/image` is the same
8843
8850
  `createLocalImageService` factory and lifecycle described above.
8844
8851
 
8852
+ ## createNativeKokoroRuntime()
8853
+
8854
+ ### Overview and signature
8855
+
8856
+ The named export from `arcane-os/local-ai/kokoro` composes the selected Kokoro
8857
+ FP32 graph with the host's existing native ONNX owner, model-resource owner
8858
+ and prepared persistent phonemization/Opus helper. Construction starts no
8859
+ native work and acquires no model or runtime.
8860
+
8861
+ ```javascript
8862
+ createNativeKokoroRuntime({
8863
+ onnx, modelAssets, runtime, prepare, modelId = 'kokoro',
8864
+ model = KOKORO_MODEL, revision = KOKORO_REVISION, dtype = 'fp32',
8865
+ paths, assetProjectionId, resourcePaths, sessionOptions,
8866
+ executionPreference = 'gpu', executionTarget, signal, onEvent
8867
+ } = {})
8868
+ ```
8869
+
8870
+ Supply the existing `onnx` runtime or a `prepare` callback that provides it.
8871
+ `runtime` describes the matching native helper and data. `paths` supplies
8872
+ complete prepared graph/tokenizer/voice resources; a completed
8873
+ `assetProjectionId` instead retains projected resources through `modelAssets`.
8874
+ The selected model/revision/FP32 contract is fixed by this frontend.
8875
+ See [native Kokoro](native-speech.md) for exact resource mappings, preparation,
8876
+ runtime distribution and model ownership.
8877
+
8878
+ ### Members and results
8879
+
8880
+ - `current()` returns complete model/lifecycle state, execution configuration,
8881
+ current request, queue, progress, error, default voice and supported voices.
8882
+ - `subscribe(listener,{replay=true,signal}={})` returns an unsubscribe function
8883
+ and replays current state by default.
8884
+ - `load(options={})` activates the selected resources, retaining omitted
8885
+ selections. Explicit projection/resource mapping or execution-target changes
8886
+ join prior native retirement before loading the new activation.
8887
+ - `synthesize(request,{signal,onProgress,requestId=null}={})` requires a loaded
8888
+ activation. `request` contains the complete `input` string, optional `model`
8889
+ matching `modelId`, `voice` (default `af_heart`), positive finite `speed`
8890
+ (default `1`) and `responseFormat` (`wav` by default, `opus` or `ogg`).
8891
+ The result is `{audioBase64,contentType,sampleRate,channels,model,voice,speed}`;
8892
+ content type is `audio/wav` or `audio/ogg; codecs=opus`, with mono 24kHz PCM
8893
+ synthesis. The full input is processed in ordered model segments.
8894
+ - `unload()` retires the activation and joins owned native work. `close()`
8895
+ permanently closes the engine and releases its state subscriptions.
8896
+
8897
+ Queued requests retain order. Cancelling a queued request removes only that
8898
+ request. Active cancellation waits for actual helper/session retirement;
8899
+ surviving requests resume with the same accepted provider configuration.
8900
+ The ONNX owner may need a running native call to return before its worker exits.
8901
+ Readiness is lost while that retirement/reload occurs. Errors retain the actual
8902
+ engine and cleanup failures; unsupported explicit voices or formats are never
8903
+ substituted. No audio is played by this engine.
8904
+
8905
+ ### Example
8906
+
8907
+ ```javascript
8908
+ import {createNativeKokoroRuntime} from 'arcane-os/local-ai/kokoro';
8909
+
8910
+ // The host supplies its existing ONNX owner and already prepared resources.
8911
+ async function narrateMoonGoat({onnx, runtime, paths}) {
8912
+ const voice = createNativeKokoroRuntime({
8913
+ onnx, runtime, paths, executionPreference: 'cpu'
8914
+ });
8915
+ try {
8916
+ await voice.load();
8917
+ return await voice.synthesize({
8918
+ input: 'Captain Goat has landed. Guard the moon cheese!',
8919
+ voice: 'af_heart', responseFormat: 'wav'
8920
+ });
8921
+ } finally {
8922
+ await voice.close();
8923
+ }
8924
+ }
8925
+ ```
8926
+
8927
+ Changing `voice` to `bf_emma` selects British English. `responseFormat: 'opus'`
8928
+ selects Ogg Opus encoding. GPU/physical-device selection uses the existing
8929
+ [ONNX execution contract](local-ai.md#physical-execution-targets); an accepted
8930
+ provider configuration is separate from observed hardware execution.
8931
+ The SDK publishes a Windows x64 helper archive. Other platforms need a matching
8932
+ custom helper distribution and supported native ONNX host. Native helper
8933
+ production and platform execution evidence remain separate.
8934
+
8935
+ ## createNativeKokoroRuntime default export
8936
+
8937
+ The default export from `arcane-os/local-ai/kokoro` is the same
8938
+ `createNativeKokoroRuntime` factory with identical inputs, results and lifetime.
8939
+
8940
+ ## KOKORO_MODEL
8941
+
8942
+ Named string export from `arcane-os/local-ai/kokoro`, with value
8943
+ `onnx-community/Kokoro-82M-v1.0-ONNX`. It identifies this frontend's model family
8944
+ for matching graph, tokenizer and voice resource selection.
8945
+
8946
+ ## KOKORO_REVISION
8947
+
8948
+ Named string export from `arcane-os/local-ai/kokoro`, with value
8949
+ `1939ad2a8e416c0acfeecc08a694d14ef25f2231`. Resource preparation uses this
8950
+ upstream revision with `KOKORO_MODEL`.
8951
+
8952
+ ## KOKORO_SAMPLE_RATE
8953
+
8954
+ Named number export from `arcane-os/local-ai/kokoro`, with value `24000`.
8955
+ It describes the engine's mono PCM synthesis rate in samples per second.
8956
+
8957
+ ## KOKORO_VOICES
8958
+
8959
+ Named array export from `arcane-os/local-ai/kokoro` containing all 28 supported
8960
+ US/GB voice identifiers. The engine uses `af_heart` when the caller omits
8961
+ `voice`; an explicit unsupported selection reports an error. Use this complete
8962
+ array to populate a native Kokoro voice selector.
8963
+
8845
8964
  ## createWhisperRuntime()
8846
8965
 
8847
8966
  ### Overview
@@ -9018,6 +9137,13 @@ The returned object has `name: 'speech'`, `current`, `subscribe`, `close`,
9018
9137
  Core RPC methods. The context supplies `signal`, `requestId` and `emit`.
9019
9138
  The service publishes synchronous accepted `Thinking` before waiting for
9020
9139
  its role's load, then forwards the engine's complete progress and result.
9140
+ - `methods['speech.load'](request,context)` and
9141
+ `methods['speech.unload'](request,context)` route explicit activation to the
9142
+ explicit `role: 'stt' | 'tts'` and forward the remaining activation
9143
+ fields and result through that role's configured engine. A startup load remains owned independently of callers
9144
+ waiting for it. The browser facade exposes `Arcane.speech.load(request,options)`
9145
+ and `unload(request,options)` with `signal`, `onRequest` and no elapsed-time
9146
+ cutoff. See [activation routing](native-speech.md) for selection and errors.
9021
9147
  - `close()` and its `drain()` alias stop acceptance, abort the service lifetime,
9022
9148
  close both engines concurrently and await startup and request work. Repeated
9023
9149
  calls share one promise. `dispose()` also releases event subscriptions.
@@ -9123,7 +9249,8 @@ import createLocalAIService from 'arcane-os/core/local-ai';
9123
9249
  ### Overview
9124
9250
 
9125
9251
  Reuses matching managed runtimes or installs complete selected
9126
- llama.cpp/Ollama/ONNX/image distributions or NeMo libraries in an explicitly supplied directory.
9252
+ llama.cpp/Ollama/ONNX/image distributions, NeMo libraries or native Kokoro
9253
+ helper/data trees in an explicitly supplied directory.
9127
9254
 
9128
9255
  ### Signature and result
9129
9256
 
@@ -9146,6 +9273,12 @@ trees remain intact while a fresh installation is prepared. See
9146
9273
  Image requirements also accept `backend` and
9147
9274
  `models`; their pinned default runtime, CPU/Metal selection and separate model
9148
9275
  preparation are described in [local image generation](local-image-generation.md).
9276
+ `kokoro-native` returns `helperExecutable`, `espeakDataDirectory` and
9277
+ `libraryDirectory`. Windows x64 defaults to the helper archive from the
9278
+ installed SDK's numeric release; an explicit `url` or `helperRoot` selects a
9279
+ complete custom distribution. The data directory is the parent of
9280
+ `espeak-ng-data`. Helper preparation loads no Kokoro graph and keeps the
9281
+ existing ONNX/model-resource owners separate. See [native speech](native-speech.md).
9149
9282
  Matching concurrent
9150
9283
  installs share their work. Downloads retain runtime libraries and surface
9151
9284
  network/archive failures; cancellation and operation events remain observable.
@@ -9189,6 +9322,10 @@ The corrected Windows x64 ONNX selection retains its
9189
9322
  is relocated with the other runtime paths; bundling introduces no separate
9190
9323
  runtime selection. See [ONNX runtime preparation](local-ai.md#development).
9191
9324
 
9325
+ Kokoro bundling retains the complete helper/data tree and relocates
9326
+ `helperExecutable`, `espeakDataDirectory` and `libraryDirectory`. Its selected
9327
+ ONNX runtime and model resources retain their existing independent ownership.
9328
+
9192
9329
  Image records relocate the complete library, binding and variant paths. Model
9193
9330
  URL descriptors retain their original acquisition metadata for preparation
9194
9331
  after launch. Runtime-contained native model paths become artifact-relative;
@@ -11683,10 +11820,11 @@ the message or response callback payload.
11683
11820
 
11684
11821
  **Node and Browser; Cloud transport.** Uses standard Fetch and cancellation,
11685
11822
  without DOM, browser profiles, user singletons, or storage initialization.
11686
- Import starts no request. HTTP `429` whose message contains `overload`
11687
- (case-insensitive) waits
11688
- `3000` milliseconds and retries the complete request; other HTTP failures
11689
- throw their complete parsed JSON or text bodies. A missing key throws
11823
+ Import starts no request. Rejected Fetch and all HTTP `429`/`529` share three
11824
+ retries. Quota recovery honors `Retry-After`, otherwise waits `3000`, `6000`,
11825
+ or `12000` milliseconds by retry number. Active same-endpoint/credential calls
11826
+ share quota cooldown and paced dispatch. Complete error bodies remain unchanged
11827
+ on exhaustion; other HTTP failures are not retried. A missing key throws
11690
11828
  `AI_PROVIDER_NOT_CONFIGURED`; a missing model throws `TypeError`. `signal`
11691
11829
  cancellation during transport, body reading, retry waiting,
11692
11830
  or callback settlement prevents successful return and uses
@@ -11836,21 +11974,24 @@ Ordinary TWiN callers use `fetchRequest()` instead.
11836
11974
  ### Signature and result
11837
11975
 
11838
11976
  ```text
11839
- async fetchHTTPResponse(url,options)
11977
+ async fetchHTTPResponse(url,options,{onRetry=null}={})
11840
11978
  ```
11841
11979
 
11842
11980
  Returns the successful Fetch `Response` without consuming its body. Both URL
11843
11981
  and Fetch options are caller-supplied; this helper adds no key, model, or
11844
11982
  request envelope. Non-success responses are read completely as JSON when the
11845
- content type contains `application/json`, otherwise as text. Only status 429
11846
- with an overload message repeats after 3000 ms; other error bodies are thrown.
11983
+ content type contains `application/json`, otherwise as text. Rejected Fetch
11984
+ and all `429`/`529` share three retries with complete terminal errors.
11985
+ `429` honors `Retry-After` or exponential delay and coordinates a transient
11986
+ same-endpoint/credential cooldown. See [shared request behavior](ai/twin-cloud.md#shared-request-behavior).
11847
11987
 
11848
11988
  ### Availability and normalization
11849
11989
 
11850
11990
  **Node and Browser.** Exported from `arcane-os/ai/twin-cloud` for shared SDK
11851
- integration. `options.signal` cancels Fetch and the overload wait and is checked
11991
+ integration. `options.signal` cancels Fetch and recovery/pacing waits and is checked
11852
11992
  after response/error-body reads. Cancellation uses
11853
- `ARCANE_AI_REQUEST_ABORTED`. Overload warnings use the existing shared logger.
11993
+ `ARCANE_AI_REQUEST_ABORTED`. Retry warnings use the existing shared logger;
11994
+ `onRetry` observes complete waiting/requesting records outside the payload.
11854
11995
 
11855
11996
  ### Example
11856
11997
 
@@ -11870,7 +12011,7 @@ Shared complete JSON-body reader built on `fetchHTTPResponse()`.
11870
12011
  ### Signature and result
11871
12012
 
11872
12013
  ```text
11873
- async fetchJSONResponse(url,options)
12014
+ async fetchJSONResponse(url,options,{onRetry=null}={})
11874
12015
  ```
11875
12016
 
11876
12017
  Returns the full parsed JSON value. A successful response whose content type
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "arcane-os",
3
- "version": "0.93.0",
3
+ "version": "0.95.0",
4
4
  "description": "Arcane OS JavaScript SDK, project-local CLI, browser runtime, and repository-portable application packager.",
5
5
  "type": "module",
6
6
  "main": "./src/index.mjs",
@@ -89,6 +89,11 @@
89
89
  "./local-ai/decisions": "./src/local-ai/decisions.mjs",
90
90
  "./local-ai/image": "./src/local-ai/image-runtime.mjs",
91
91
  "./local-ai/whisper": "./src/local-ai/whisper/index.mjs",
92
+ "./local-ai/kokoro": "./src/local-ai/kokoro/index.mjs",
93
+ "./local-ai/nemotron": "./src/local-ai/nemotron/index.mjs",
94
+ "./local-ai/nemotron/build": "./src/local-ai/nemotron/build.mjs",
95
+ "./local-ai/magpie": "./src/local-ai/magpie/index.mjs",
96
+ "./local-ai/magpie/build": "./src/local-ai/magpie/build.mjs",
92
97
  "./local-ai/whisper/build": "./src/local-ai/whisper/build.mjs",
93
98
  "./local-ai/whisper/openvino-build": "./src/local-ai/whisper/openvino-build.mjs",
94
99
  "./integrated-provider": "./src/integrated-provider-loader.mjs",
@@ -28,6 +28,8 @@ import {
28
28
  import {MarkdownSpeech,stripSpeechFormatting} from 'arcane-os/speech-text';
29
29
  import {prepareSpeech, readPreparedSpeech} from './PreparedSpeech.js';
30
30
  import {createToolTextObserver} from 'arcane-os/ai/tool-text-stream';
31
+ import createNativeSpeechCapture from './NativeSpeechCapture.js';
32
+ import prepareNativeSpeechPlayback from './NativeSpeechPlayback.js';
31
33
 
32
34
  const completeValue=(value)=>value;
33
35
 
@@ -1150,14 +1152,18 @@ class AI {
1150
1152
  }
1151
1153
  );
1152
1154
 
1155
+ const speechOnly=llmService===null&&model===null;
1153
1156
  const preferences=[
1154
- llmService||'TWIN',
1157
+ speechOnly?'':llmService||'TWIN',
1155
1158
  sttService||'LOCAL_SPEACH',
1156
1159
  ttsService||'LOCAL_SPEACH',
1157
- model||'TWIN',
1160
+ speechOnly?'':model||'TWIN',
1158
1161
  modelTTS||'LOCAL_SPEACH',
1159
1162
  modelSTT||'LOCAL_SPEACH'
1160
1163
  ];
1164
+ // Explicitly absent LLM selection must not briefly register or load a
1165
+ // default provider. setAI retains empty entries from this initial tuple.
1166
+ if(speechOnly)this.#preferenceTuple=preferences.slice();
1161
1167
  this.setAI(
1162
1168
  ...preferences
1163
1169
  );
@@ -2008,13 +2014,15 @@ class AI {
2008
2014
  if(runtime.#builtInSpeechService(role)!==providerId||!model){
2009
2015
  return completeValue([]);
2010
2016
  }
2011
- const defaultVoice=runtime.#builtInSpeechDefaultVoice(
2017
+ const defaultVoice=nativeWatch?.snapshot?.roles?.[role]?.defaultVoice??runtime.#builtInSpeechDefaultVoice(
2012
2018
  role,
2013
2019
  providerId
2014
2020
  );
2015
2021
  return completeValue([
2016
2022
  completeValue({
2017
2023
  id:model,
2024
+ ...(nativeWatch?.snapshot?.roles?.[role]?.speech
2025
+ ?{speech:nativeWatch.snapshot.roles[role].speech}:{}),
2018
2026
  ...(defaultVoice?{defaultVoice}:{})
2019
2027
  })
2020
2028
  ]);
@@ -2026,6 +2034,39 @@ class AI {
2026
2034
  return runtime.#builtInSpeechInspection(role,providerId,selection);
2027
2035
  },
2028
2036
  status:statusBuiltInSpeechProvider,
2037
+ ...(role==='stt'?{
2038
+ supportsCapture:function supportsNativeCapture(){
2039
+ return nativeWatch?.snapshot?.roles?.stt?.supportsStreaming===true;
2040
+ },
2041
+ createCapture:function createBuiltInNativeCapture(options={}){
2042
+ if(nativeWatch?.snapshot?.roles?.stt?.supportsStreaming!==true)return null;
2043
+ const capture=createNativeSpeechCapture({
2044
+ ...options,
2045
+ speech:nativeWatch.speech,
2046
+ model:nativeWatch.modelId
2047
+ });
2048
+ const operation={watch:nativeWatch,requestId:null,signal:null};
2049
+ const done=capture.done.finally(function releaseNativeCapture(){
2050
+ if(nativeRequest===operation)releaseBuiltInSpeechRequest();
2051
+ });
2052
+ done.catch(function observeNativeCaptureCompletion(error){
2053
+ if(!isAIRequestAbort(error))arcaneLogging.error('Native transcription capture failed.',error);
2054
+ });
2055
+ return {
2056
+ ...capture,
2057
+ done,
2058
+ start:function startBuiltInNativeCapture(options={}){
2059
+ if(!nativeSpeechReady()||busy){
2060
+ throw aiProviderError('The native transcription model is not ready.','ARCANE_AI_ROLE_NOT_READY');
2061
+ }
2062
+ busy=true;
2063
+ operation.signal=options.signal;
2064
+ nativeRequest=operation;
2065
+ return capture.start(options);
2066
+ }
2067
+ };
2068
+ }
2069
+ }:{}),
2029
2070
  load:async function loadBuiltInSpeechProvider(context={}){
2030
2071
  if(context.signal?.aborted){
2031
2072
  throw normalizeAIRequestAbort(context.signal.reason);
@@ -2086,7 +2127,7 @@ class AI {
2086
2127
  throw error;
2087
2128
  }
2088
2129
  },
2089
- request:function requestBuiltInSpeechProvider(context={}){
2130
+ request:function requestBuiltInSpeechProvider(context={}, {speechStream=null}={}){
2090
2131
  if(context.signal?.aborted){
2091
2132
  throw normalizeAIRequestAbort(context.signal.reason);
2092
2133
  }
@@ -2126,7 +2167,8 @@ class AI {
2126
2167
  )
2127
2168
  :runtime.#requestBuiltInSpeechSynthesis(
2128
2169
  context.payload,
2129
- context.signal
2170
+ context.signal,
2171
+ speechStream
2130
2172
  );
2131
2173
  return Promise.resolve(request).finally(releaseBuiltInSpeechRequest);
2132
2174
  },
@@ -2548,6 +2590,7 @@ class AI {
2548
2590
 
2549
2591
  audioMessageChunks='';
2550
2592
  #markdownSpeech=new MarkdownSpeech();
2593
+ #streamSpeechOptions=null;
2551
2594
  sourceNodes=[];
2552
2595
  isSpeaking=false;
2553
2596
  audioContext=null;
@@ -4684,7 +4727,7 @@ class AI {
4684
4727
  return response.text();
4685
4728
  }
4686
4729
 
4687
- async #requestBuiltInSpeechSynthesis(payload={},signal=null){
4730
+ async #requestBuiltInSpeechSynthesis(payload={},signal=null,speechStream=null){
4688
4731
  const input=is.string(payload?.input)?payload.input:'';
4689
4732
  if(!input){
4690
4733
  throw new TypeError('Speech synthesis requires nonempty input.');
@@ -4703,9 +4746,15 @@ class AI {
4703
4746
  const speed=is.finite(payload.speed)?payload.speed:this.voiceSpeed;
4704
4747
  const nativeSpeech=this.#nativeSpeech(this.ttsService,'tts');
4705
4748
  if(nativeSpeech){
4749
+ if(speechStream){
4750
+ return nativeSpeech.synthesizeStream({
4751
+ ...payload,model,voice,input,speed
4752
+ },{...speechStream,signal});
4753
+ }
4706
4754
  let response;
4707
4755
  try{
4708
4756
  response=await nativeSpeech.synthesize({
4757
+ ...payload,
4709
4758
  model,
4710
4759
  voice,
4711
4760
  input,
@@ -6871,12 +6920,24 @@ class AI {
6871
6920
  if(end){
6872
6921
  this.audioMessageChunks='';
6873
6922
  this.#markdownSpeech.reset();
6923
+ this.#streamSpeechOptions=null;
6874
6924
  }
6875
6925
  this.#traceSpeech('streamTTS.result',{callId,result:false,reason:'muted'});
6876
6926
  return Promise.resolve(false);
6877
6927
  }
6878
6928
 
6879
- const {voice,speed,waitForPlayback=false}=options;
6929
+ // A final flush belongs to the same utterance selection. An explicit
6930
+ // language change drains the previous text before accepting new text.
6931
+ let preceding=null;
6932
+ if(this.#streamSpeechOptions&&Object.hasOwn(options,'languageCode')
6933
+ &&options.languageCode!==this.#streamSpeechOptions.languageCode){
6934
+ preceding=this.streamTTS('',true,this.#streamSpeechOptions);
6935
+ }
6936
+ const selectedOptions={...this.#streamSpeechOptions,...options};
6937
+ this.#streamSpeechOptions={
6938
+ voice:selectedOptions.voice,speed:selectedOptions.speed,languageCode:selectedOptions.languageCode
6939
+ };
6940
+ const {voice,speed,languageCode,waitForPlayback=false}=selectedOptions;
6880
6941
  const pauseAfterMs=Number(options.pauseAfterMs??0);
6881
6942
  if(!is.finite(pauseAfterMs)||pauseAfterMs<0){
6882
6943
  throw new RangeError('Speech pauses must be a nonnegative number of milliseconds.');
@@ -6885,6 +6946,7 @@ class AI {
6885
6946
  const speechText=this.#markdownSpeech.append(String(text||''),end);
6886
6947
  this.audioMessageChunks+=speechText;
6887
6948
  const outputs=this.#extractSpeechSegments(end);
6949
+ if(end)this.#streamSpeechOptions=null;
6888
6950
  this.#traceSpeech('streamTTS.segments',{
6889
6951
  callId,segments:outputs,remainder:this.audioMessageChunks,
6890
6952
  segmentation:this.ttsSegmentation,speechText
@@ -6892,7 +6954,7 @@ class AI {
6892
6954
 
6893
6955
  if(!outputs.length){
6894
6956
  this.#traceSpeech('streamTTS.result',{callId,result:true,reason:'buffered'});
6895
- return Promise.resolve(true);
6957
+ return preceding??Promise.resolve(true);
6896
6958
  }
6897
6959
 
6898
6960
  try{
@@ -6909,7 +6971,7 @@ class AI {
6909
6971
  }
6910
6972
 
6911
6973
  const generation=this.speechGeneration;
6912
- const jobs=[];
6974
+ const jobs=preceding?[preceding]:[];
6913
6975
  const runtime=this;
6914
6976
 
6915
6977
  for(const [index,output] of outputs.entries()){
@@ -6917,6 +6979,7 @@ class AI {
6917
6979
  diagnosticCallId:callId,
6918
6980
  voice,
6919
6981
  speed,
6982
+ languageCode,
6920
6983
  waitForPlayback,
6921
6984
  pauseAfterMs:index===outputs.length-1?pauseAfterMs:0
6922
6985
  }));
@@ -6938,6 +7001,7 @@ class AI {
6938
7001
  for(const playback of this.#preparedSpeechPlaybacks) playback.stop();
6939
7002
  this.audioMessageChunks='';
6940
7003
  this.#markdownSpeech.reset();
7004
+ this.#streamSpeechOptions=null;
6941
7005
  return;
6942
7006
  }
6943
7007
  this.finishTTS();
@@ -7111,6 +7175,7 @@ class AI {
7111
7175
  text,
7112
7176
  voice:options.voice,
7113
7177
  speed:options.speed,
7178
+ languageCode:options.languageCode,
7114
7179
  pauseAfterMs:options.pauseAfterMs??0,
7115
7180
  resolvePlayback:null
7116
7181
  };
@@ -7206,6 +7271,7 @@ class AI {
7206
7271
  const payload={
7207
7272
  model:selection?.modelId||this.modelTTS,
7208
7273
  input:job.text,
7274
+ ...(job.languageCode!==undefined?{languageCode:job.languageCode}:{}),
7209
7275
  ...(job.voice!==undefined||voice?{voice}:{}),
7210
7276
  ...(!native?{responseFormat:selection
7211
7277
  ?this.#providerSpeechResponseFormat()
@@ -7570,6 +7636,34 @@ class AI {
7570
7636
 
7571
7637
  async prepareTTSPlayback(payload={},signal=null,preparation={}){
7572
7638
  const model=this.#providerSpeechModel();
7639
+ if(model?.speech?.streaming===true){
7640
+ this.#assertServiceConfigured(this.ttsService,'tts');
7641
+ if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
7642
+ const runtime=this;
7643
+ const selection=this.#providerRuntime.selection('tts');
7644
+ return prepareNativeSpeechPlayback({
7645
+ request:{
7646
+ ...payload,
7647
+ model:payload.model??selection.modelId,
7648
+ voice:payload.voice??this.#providerSpeechVoice(),
7649
+ speed:payload.speed??this.voiceSpeed
7650
+ },
7651
+ signal,
7652
+ getAudioContext:function getOwnedSpeechOutput(){
7653
+ return runtime.#getSpeechAudioContext();
7654
+ },
7655
+ speech:{
7656
+ synthesizeStream:function requestNativeSpeechStream(request,options){
7657
+ return runtime.#providerRuntime.request('tts',{
7658
+ operation:'synthesize',payload:request,localOnly:false,signal:options.signal
7659
+ },{
7660
+ ...preparation,
7661
+ speechStream:{onAudio:options.onAudio,onProgress:options.onProgress,onRequest:options.onRequest}
7662
+ });
7663
+ }
7664
+ }
7665
+ });
7666
+ }
7573
7667
  if(model?.speech?.playback!=='native'){
7574
7668
  return this.fetchTTS(payload,signal,preparation);
7575
7669
  }
@@ -7618,6 +7712,7 @@ class AI {
7618
7712
  'input',
7619
7713
  'textFormat',
7620
7714
  'responseFormat',
7715
+ 'languageCode',
7621
7716
  'speed'
7622
7717
  ]);
7623
7718
  for(const key of Reflect.ownKeys(descriptors)){
@@ -7625,7 +7720,7 @@ class AI {
7625
7720
  ||!acceptedKeys.has(key)
7626
7721
  ||!Object.hasOwn(descriptors[key],'value')){
7627
7722
  const error=new TypeError(
7628
- 'AI.fetchTTS accepts only model, voice, input, textFormat, responseFormat, and speed data properties.'
7723
+ 'AI.fetchTTS accepts only model, voice, input, textFormat, responseFormat, languageCode, and speed data properties.'
7629
7724
  );
7630
7725
  error.code='ARCANE_AI_TTS_REQUEST_INVALID';
7631
7726
  throw error;
@@ -7732,15 +7827,19 @@ class AI {
7732
7827
  throw error;
7733
7828
  }
7734
7829
 
7830
+ const speechRequest={
7831
+ model,voice,input,responseFormat,speed,
7832
+ ...(descriptors.languageCode?{languageCode:descriptors.languageCode.value}:{})
7833
+ };
7735
7834
  if(selection){
7736
7835
  this.#traceSpeech('fetchTTS.dispatch',{
7737
- callId,selection,payload:{model,voice,input,responseFormat,speed}
7836
+ callId,selection,payload:speechRequest
7738
7837
  });
7739
7838
  const response=await this.#providerRuntime.request(
7740
7839
  'tts',
7741
7840
  {
7742
7841
  operation:'synthesize',
7743
- payload:{model,voice,input,responseFormat,speed},
7842
+ payload:speechRequest,
7744
7843
  localOnly:false,
7745
7844
  signal
7746
7845
  },
@@ -7755,10 +7854,10 @@ class AI {
7755
7854
  }
7756
7855
 
7757
7856
  this.#traceSpeech('fetchTTS.dispatch',{
7758
- callId,service:this.ttsService,payload:{model,voice,input,responseFormat,speed}
7857
+ callId,service:this.ttsService,payload:speechRequest
7759
7858
  });
7760
7859
  const audio=await this.#requestBuiltInSpeechSynthesis(
7761
- {model,voice,input,responseFormat,speed},
7860
+ speechRequest,
7762
7861
  signal
7763
7862
  );
7764
7863
  this.#traceSpeech('fetchTTS.result',{callId,audio});
@@ -7859,6 +7958,7 @@ class AI {
7859
7958
  this.speechScheduleTime=0;
7860
7959
  this.audioMessageChunks='';
7861
7960
  this.#markdownSpeech.reset();
7961
+ this.#streamSpeechOptions=null;
7862
7962
  this.#clearSpeechUnlock();
7863
7963
 
7864
7964
  for(const job of this.speechJobs){