arcane-os 0.93.0 → 0.95.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.md +1 -1
  3. package/browser-runtime/ai/twin-cloud.mjs +109 -46
  4. package/browser-runtime/core/client.mjs +197 -0
  5. package/docs/reference/ai/twin-cloud.md +35 -14
  6. package/docs/reference/availability-and-normalization.md +1 -1
  7. package/docs/reference/inventory/package-api.json +96 -12
  8. package/docs/reference/local-ai.md +81 -5
  9. package/docs/reference/native-magpie.md +98 -0
  10. package/docs/reference/native-nemotron.md +204 -0
  11. package/docs/reference/native-speech.md +364 -11
  12. package/docs/reference/runtime-modules.md +21 -18
  13. package/docs/reference/sdk-api.md +152 -11
  14. package/package.json +6 -1
  15. package/runtime/arcane/modules/AI.js +114 -14
  16. package/runtime/arcane/modules/AIProviderRuntime.js +32 -15
  17. package/runtime/arcane/modules/ContinuousVoiceCapture.js +118 -21
  18. package/runtime/arcane/modules/NativeSpeechCapture.js +198 -0
  19. package/runtime/arcane/modules/NativeSpeechPlayback.js +304 -0
  20. package/runtime/arcane/modules/VoiceCaptureWorklet.js +62 -2
  21. package/schemas/arcane-app.schema.json +2 -2
  22. package/schemas/arcane-package.schema.json +2 -2
  23. package/src/core/services/speech.mjs +320 -25
  24. package/src/local-ai/config.mjs +12 -1
  25. package/src/local-ai/dev.mjs +71 -4
  26. package/src/local-ai/install.mjs +113 -5
  27. package/src/local-ai/kokoro/frontend.mjs +219 -0
  28. package/src/local-ai/kokoro/helper.mjs +168 -0
  29. package/src/local-ai/kokoro/index.mjs +645 -0
  30. package/src/local-ai/kokoro/native/CMakeLists.txt +45 -0
  31. package/src/local-ai/kokoro/native/main.cpp +249 -0
  32. package/src/local-ai/magpie/build.mjs +36 -0
  33. package/src/local-ai/magpie/helper.mjs +254 -0
  34. package/src/local-ai/magpie/index.mjs +488 -0
  35. package/src/local-ai/magpie/native/CMakeLists.txt +10 -0
  36. package/src/local-ai/magpie/native/main.cpp +270 -0
  37. package/src/local-ai/nemotron/build.mjs +53 -0
  38. package/src/local-ai/nemotron/helper.mjs +308 -0
  39. package/src/local-ai/nemotron/index.mjs +637 -0
  40. package/src/local-ai/nemotron/native/CMakeLists.txt +17 -0
  41. package/src/local-ai/nemotron/native/main.cpp +282 -0
  42. package/src/native/portable-layout.mjs +67 -9
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "schemaVersion": 1,
3
- "sdkVersion": "0.92.0",
3
+ "sdkVersion": "0.94.0",
4
4
  "environment": {
5
5
  "runtime": "Node.js for Node entrypoints; browser for browser-only entrypoints",
6
6
  "minimumVersion": "22.23.2 for Node entrypoints",
7
7
  "moduleSystem": "ESM"
8
8
  },
9
- "memberCount": 308,
9
+ "memberCount": 314,
10
10
  "runtimeSubpathPatterns": {
11
11
  "arcane-os/modules/*": "./runtime/arcane/modules/*",
12
12
  "arcane-os/entities/*": "./runtime/arcane/entities/*"
@@ -166,6 +166,90 @@
166
166
  "protocol": "arcane/1 service definition",
167
167
  "normalization": "Binding identity equals the named createNativeDecisionService export"
168
168
  },
169
+ {
170
+ "id": "local-ai-kokoro:createNativeKokoroRuntime",
171
+ "name": "createNativeKokoroRuntime",
172
+ "displayName": "createNativeKokoroRuntime()",
173
+ "kind": "function",
174
+ "signature": "createNativeKokoroRuntime({onnx,modelAssets,runtime,prepare,modelId='kokoro',model=KOKORO_MODEL,revision=KOKORO_REVISION,dtype='fp32',paths,assetProjectionId,resourcePaths,sessionOptions,executionPreference='gpu',executionTarget,signal,onEvent}={})",
175
+ "entrypoints": ["arcane-os/local-ai/kokoro"],
176
+ "primaryImport": "arcane-os/local-ai/kokoro",
177
+ "group": "Core native speech",
178
+ "summary": "Composes the selected Kokoro FP32 graph with the host's existing ONNX/model-resource owners and a persistent native phonemization/Opus helper.",
179
+ "availability": "Node with matching prepared helper/data and ONNX resources; SDK helper archive for Windows x64, other hosts require a matching custom helper distribution",
180
+ "protocol": "Replaying kokoro.state, retained ONNX activation and complete ordered native speech requests",
181
+ "normalization": "Returns current/subscribe/load/synthesize/unload/close; construction performs no preparation or inference. Explicit load selects a DBOPFS projection or complete native resource paths; synthesis retains complete ordered input, selected US/GB voice and 24kHz mono WAV or Ogg Opus output. Active cancellation joins owned helper/session retirement and preserves surviving queued work on the same configured provider; ONNX native interruption may wait for the current call. No alternate model, voice or provider retry is introduced. See native-speech.md for exact activation, selection and result contracts."
182
+ },
183
+ {
184
+ "id": "local-ai-kokoro:default",
185
+ "name": "default",
186
+ "displayName": "createNativeKokoroRuntime default export",
187
+ "kind": "function",
188
+ "signature": "default as createNativeKokoroRuntime",
189
+ "entrypoints": ["arcane-os/local-ai/kokoro"],
190
+ "primaryImport": "arcane-os/local-ai/kokoro",
191
+ "group": "Core native speech",
192
+ "summary": "Default binding for the same native Kokoro engine factory.",
193
+ "availability": "Node with matching prepared native resources",
194
+ "protocol": "Native Kokoro engine lifecycle",
195
+ "normalization": "Binding identity equals the named createNativeKokoroRuntime export"
196
+ },
197
+ {
198
+ "id": "local-ai-kokoro:KOKORO_MODEL",
199
+ "name": "KOKORO_MODEL",
200
+ "displayName": "KOKORO_MODEL",
201
+ "kind": "constant",
202
+ "signature": "const KOKORO_MODEL",
203
+ "entrypoints": ["arcane-os/local-ai/kokoro"],
204
+ "primaryImport": "arcane-os/local-ai/kokoro",
205
+ "group": "Core native speech",
206
+ "summary": "Identifies the Kokoro model implemented by the native frontend.",
207
+ "availability": "Node",
208
+ "protocol": "Native model selection",
209
+ "normalization": "Value is onnx-community/Kokoro-82M-v1.0-ONNX; constructor selection preserves this model's FP32 graph and resource contract"
210
+ },
211
+ {
212
+ "id": "local-ai-kokoro:KOKORO_REVISION",
213
+ "name": "KOKORO_REVISION",
214
+ "displayName": "KOKORO_REVISION",
215
+ "kind": "constant",
216
+ "signature": "const KOKORO_REVISION",
217
+ "entrypoints": ["arcane-os/local-ai/kokoro"],
218
+ "primaryImport": "arcane-os/local-ai/kokoro",
219
+ "group": "Core native speech",
220
+ "summary": "Identifies the selected upstream Kokoro model revision.",
221
+ "availability": "Node",
222
+ "protocol": "Native model resource selection",
223
+ "normalization": "Value is 1939ad2a8e416c0acfeecc08a694d14ef25f2231; graph, tokenizer and voice resources use this same revision"
224
+ },
225
+ {
226
+ "id": "local-ai-kokoro:KOKORO_SAMPLE_RATE",
227
+ "name": "KOKORO_SAMPLE_RATE",
228
+ "displayName": "KOKORO_SAMPLE_RATE",
229
+ "kind": "constant",
230
+ "signature": "const KOKORO_SAMPLE_RATE",
231
+ "entrypoints": ["arcane-os/local-ai/kokoro"],
232
+ "primaryImport": "arcane-os/local-ai/kokoro",
233
+ "group": "Core native speech",
234
+ "summary": "Native Kokoro PCM synthesis rate in samples per second.",
235
+ "availability": "Node",
236
+ "protocol": "Audio format",
237
+ "normalization": "Value is 24000; the selected output is mono WAV or Ogg Opus with the complete synthesized sample sequence"
238
+ },
239
+ {
240
+ "id": "local-ai-kokoro:KOKORO_VOICES",
241
+ "name": "KOKORO_VOICES",
242
+ "displayName": "KOKORO_VOICES",
243
+ "kind": "constant",
244
+ "signature": "const KOKORO_VOICES",
245
+ "entrypoints": ["arcane-os/local-ai/kokoro"],
246
+ "primaryImport": "arcane-os/local-ai/kokoro",
247
+ "group": "Core native speech",
248
+ "summary": "The 28 supported US and British English voice identifiers for the selected native Kokoro resources.",
249
+ "availability": "Node",
250
+ "protocol": "Native speech voice selection",
251
+ "normalization": "Complete ordered voice array exported by the frontend; af_heart is the omitted voice default. An unsupported explicit voice reports its actual incompatibility without substitution."
252
+ },
169
253
  {
170
254
  "id": "local-ai-whisper:createWhisperRuntime",
171
255
  "name": "createWhisperRuntime",
@@ -345,8 +429,8 @@
345
429
  "group": "Core native speech",
346
430
  "summary": "Composes host-supplied transcription and synthesis engines with independent loading and role readiness.",
347
431
  "availability": "Node Core with explicitly supplied engines; engine platform, model and format support remain engine-owned",
348
- "protocol": "arcane/1 speech.status/transcribe/synthesize with speech.state/progress",
349
- "normalization": "Returns current/subscribe/close/start/methods/drain/dispose; forwards complete requests and results unchanged; replaying service state, synchronous accepted Thinking, combined request/lifetime cancellation and concurrent joined engine shutdown; no engine installation or automatic provider selection"
432
+ "protocol": "arcane/1 speech.status/load/unload/transcribe/synthesize with speech.state/progress",
433
+ "normalization": "Returns current/subscribe/close/start/methods/drain/dispose; forwards complete requests and results unchanged; replaying service state, synchronous accepted Thinking, combined request/lifetime cancellation and concurrent joined engine shutdown. Explicit load/unload require role stt or tts and forward the remaining activation fields and complete result through that role's configured engine; supported preparation remains with that engine. Startup waiting is independently owned from caller cancellation; no automatic provider selection is introduced."
350
434
  },
351
435
  {
352
436
  "id": "core-speech:default",
@@ -609,10 +693,10 @@
609
693
  "entrypoints": ["arcane-os/local-ai"],
610
694
  "primaryImport": "arcane-os/local-ai",
611
695
  "group": "Core local AI",
612
- "summary": "Reuses selected managed native runtimes or installs complete llama.cpp/Ollama/image distributions, ONNX npm runtime trees with selected SDK native libraries, or NeMo libraries.",
696
+ "summary": "Reuses selected managed native runtimes or installs complete llama.cpp/Ollama/image distributions, ONNX npm runtime trees with selected SDK native libraries, NeMo libraries, or Kokoro native helper/data trees.",
613
697
  "availability": "Node; selected distribution platform and architecture assets determine runtime availability",
614
698
  "protocol": "Upstream release archives and selected ONNX/Koffi npm installations; corrected Windows x64 ONNX native libraries from the installed SDK numeric release",
615
- "normalization": "Returns absolute runtime records, including image library/binding/variant/model metadata; Windows x64 ONNX with no explicit URL and omitted version or 1.30.0 adds nativeDistributionRevision 1.30.0-directml-reshape-1, reuses that revision or prepares a fresh installation while preserving older trees; explicit URLs and other ONNX versions/platforms retain their upstream package path; shares matching installs, preserves libraries, reports failures without changing global PATH or services and prepares no model files"
699
+ "normalization": "Returns absolute runtime records, including image library/binding/variant/model metadata; Windows x64 ONNX with no explicit URL and omitted version or 1.30.0 adds nativeDistributionRevision 1.30.0-directml-reshape-1, reuses that revision or prepares a fresh installation while preserving older trees; explicit URLs and other ONNX versions/platforms retain their upstream package path; shares matching installs, preserves libraries, reports failures without changing global PATH or services and prepares no model files. Kokoro returns helperExecutable, espeakDataDirectory and libraryDirectory for the complete selected helper tree; Windows x64 defaults to the installed SDK numeric release, while explicit url/helperRoot supplies another prepared distribution and preserves model/ONNX ownership."
616
700
  },
617
701
  {
618
702
  "id": "local-ai:bundleLocalAIRuntimes",
@@ -626,7 +710,7 @@
626
710
  "summary": "Copies selected complete runtime distributions into a fresh native artifact staging root.",
627
711
  "availability": "Node; supports bundling selected published platform assets independently of the build host",
628
712
  "protocol": "Artifact-relative runtime/local-ai layout",
629
- "normalization": "Returns {runtimes,files} with artifact-relative runtime/variant paths and complete inventory; corrected Windows x64 ONNX retains nativeDistributionRevision and complete native libraries while modulePath becomes artifact-relative; image URL descriptors remain for post-launch preparation and external model working paths report portability incompatibility"
713
+ "normalization": "Returns {runtimes,files} with artifact-relative runtime/variant paths and complete inventory; corrected Windows x64 ONNX retains nativeDistributionRevision and complete native libraries while modulePath becomes artifact-relative; image URL descriptors remain for post-launch preparation and external model working paths report portability incompatibility. Kokoro retains the complete helper/data tree and relocates helperExecutable, espeakDataDirectory and libraryDirectory; ONNX and model resources retain their separate owners."
630
714
  },
631
715
  {
632
716
  "id": "root:WS",
@@ -4122,7 +4206,7 @@
4122
4206
  "summary": "Sends one complete TWiN Cloud request with an explicit caller-owned key and model.",
4123
4207
  "availability": "Node and Browser; remote HTTPS provider",
4124
4208
  "protocol": "TWiN Cloud complete-response chat",
4125
- "normalization": "Preserves complete messages and parsed provider JSON; maps explicit structured-output, tool, and reasoning options; shares overload-only HTTP 429 retry after 3000 ms and cancellation; retains no browser profile, storage, or request history"
4209
+ "normalization": "Preserves complete messages and provider JSON; Fetch rejection and all HTTP 429/529 share three retries; quota recovery honors Retry-After or exponential delay with transient same-endpoint/credential pacing, complete terminal errors and cancellation; retains no browser profile, storage, or history"
4126
4210
  },
4127
4211
  {
4128
4212
  "id": "twin-cloud:fetchSystemOneRequest",
@@ -4171,21 +4255,21 @@
4171
4255
  "name": "fetchHTTPResponse",
4172
4256
  "displayName": "fetchHTTPResponse()",
4173
4257
  "kind": "function",
4174
- "signature": "async fetchHTTPResponse(url,options)",
4258
+ "signature": "async fetchHTTPResponse(url,options,{onRetry=null}={})",
4175
4259
  "entrypoints": ["arcane-os/ai/twin-cloud"],
4176
4260
  "primaryImport": "arcane-os/ai/twin-cloud",
4177
4261
  "group": "Shared AI transport integration",
4178
4262
  "summary": "Returns a successful unconsumed Fetch Response through the shared AI HTTP owner.",
4179
4263
  "availability": "Node and Browser",
4180
4264
  "protocol": "Shared AI HTTP transport",
4181
- "normalization": "Preserves caller Fetch options and complete error bodies; repeats only overload HTTP 429 after 3000 ms; signal cancellation uses ARCANE_AI_REQUEST_ABORTED"
4265
+ "normalization": "Preserves caller Fetch options and complete errors; Fetch rejection and all HTTP 429/529 share three retries; Retry-After or exponential quota delay and transient same-endpoint/credential pacing remain abortable; onRetry observes waiting/requesting; cancellation uses ARCANE_AI_REQUEST_ABORTED"
4182
4266
  },
4183
4267
  {
4184
4268
  "id": "twin-cloud:fetchJSONResponse",
4185
4269
  "name": "fetchJSONResponse",
4186
4270
  "displayName": "fetchJSONResponse()",
4187
4271
  "kind": "function",
4188
- "signature": "async fetchJSONResponse(url,options)",
4272
+ "signature": "async fetchJSONResponse(url,options,{onRetry=null}={})",
4189
4273
  "entrypoints": ["arcane-os/ai/twin-cloud"],
4190
4274
  "primaryImport": "arcane-os/ai/twin-cloud",
4191
4275
  "group": "Shared AI transport integration",
@@ -4360,7 +4444,7 @@
4360
4444
  "summary": "Projects a Core client into the established application-facing Arcane namespace.",
4361
4445
  "availability": "Browser and native WebView JavaScript hosts with a Core client",
4362
4446
  "protocol": "arcane/1 compatibility facade",
4363
- "normalization": "Routes namespace methods through client.invoke while retaining operation-specific parameters, correlation, streaming callbacks, timeouts and cancellation; window.state({signal}?) reads the requesting window and window.setState({state},{signal}?) selects normal, maximized or fullscreen. The matching Windows host returns {platform:'windows',supported:true,state}, including observed minimized state, restores normal bounds/frame controls on leaving fullscreen and prevents activation/focus assignment during visible state transitions without changing TopMost, Core, origin or profile. Saved Settings remain application-owned; cancellation does not undo an applied change and unavailable hosts retain their actual errors without fallback. window.setTheme(presentation,{signal}?) forwards the complete per-window backgroundColor/textColor record, with omitted fields unchanged and null reset, and returns the host's actual applied/unsupported result without changing system appearance. notifications.status/show/state/close forward complete caller requests and standard invoke options; status normalizes only METHOD_NOT_ALLOWED with core-namespace-unavailable while retaining the complete original error. Matching native window hosts retain complete notification records and ordered revisions across renderer navigation, with notifications.state events and subscribe-before-snapshot recovery. Submitted means native Show returned, not displayed or read; cancellation after submission does not remove an accepted notice, and explicit close retains its record. Recipient, text, trigger, deduplication and navigation policy remains application-owned"
4447
+ "normalization": "Routes namespace methods through client.invoke while retaining operation-specific parameters, correlation, streaming callbacks, timeouts and cancellation; window.state({signal}?) reads the requesting window and window.setState({state},{signal}?) selects normal, maximized or fullscreen. The matching Windows host returns {platform:'windows',supported:true,state}, including observed minimized state, restores normal bounds/frame controls on leaving fullscreen and prevents activation/focus assignment during visible state transitions without changing TopMost, Core, origin or profile. Saved Settings remain application-owned; cancellation does not undo an applied change and unavailable hosts retain their actual errors without fallback. window.setTheme(presentation,{signal}?) forwards the complete per-window backgroundColor/textColor record, with omitted fields unchanged and null reset, and returns the host's actual applied/unsupported result without changing system appearance. notifications.status/show/state/close forward complete caller requests and standard invoke options; status normalizes only METHOD_NOT_ALLOWED with core-namespace-unavailable while retaining the complete original error. Matching native window hosts retain complete notification records and ordered revisions across renderer navigation, with notifications.state events and subscribe-before-snapshot recovery. Submitted means native Show returned, not displayed or read; cancellation after submission does not remove an accepted notice, and explicit close retains its record. Recipient, text, trigger, deduplication and navigation policy remains application-owned. speech.load(request,options) and speech.unload(request,options) forward complete activation requests, cancellation and request observers with no elapsed-time cutoff; selected engines own preparation and native retirement."
4364
4448
  },
4365
4449
  {
4366
4450
  "id": "core-client:installCoreClient",
@@ -1,13 +1,16 @@
1
1
  # Local AI through Core
2
2
 
3
3
  Core owns native llama.cpp and Ollama processes, ONNX worker sessions,
4
- selected local image generation and retained Whisper transcription. The browser retains its own
4
+ selected local image generation, retained Whisper transcription and native
5
+ Kokoro synthesis. The browser retains its own
5
6
  Wllama and ONNX implementations, and can also use an explicitly connected Core
6
7
  when that system service is available. Runtime installation, model loading and
7
8
  inference are separate operations.
8
9
 
9
10
  Native ONNX runs caller-selected graphs with complete named input/output tensors.
10
- Applications own preprocessing, tokenization and model-specific pipelines.
11
+ Applications own preprocessing, tokenization and model-specific pipelines unless
12
+ they explicitly select an SDK engine such as native Kokoro, which owns its
13
+ versioned speech frontend.
11
14
  Browser ONNX continues through its existing browser implementation.
12
15
 
13
16
  ## Application requirements
@@ -113,6 +116,71 @@ the SDK does not guess a companion location. Installation records distinguish
113
116
  the distribution's `backend` from its `requestedBackend` execution mode.
114
117
  Selecting an unavailable distribution reports its actual error.
115
118
 
119
+ ### Native Kokoro selection
120
+
121
+ Select both the existing ONNX runtime and the native speech helper:
122
+
123
+ ```json
124
+ {
125
+ "runtimes": [
126
+ {"id": "onnx"},
127
+ {"id": "kokoro-native", "modelId": "kokoro"}
128
+ ]
129
+ }
130
+ ```
131
+
132
+ `kokoro-native` is the installable helper ID, `kokoro-onnx` is the speech
133
+ provider ID, and `kokoro` is the default model alias. The requirement also
134
+ preserves `model`, `revision`, `dtype`, prepared `paths`, `assetProjectionId`,
135
+ `resourcePaths`, `sessionOptions`, `executionPreference` and `executionTarget`
136
+ for the engine. The implemented model/revision, complete resource mapping,
137
+ voices, synthesis formats and cancellation contract are in
138
+ [native speech](native-speech.md#native-kokoro-engine).
139
+
140
+ With no `helperRoot` or `url`, Windows x64 preparation selects
141
+ `arcane-kokoro-windows-x64.tar.gz` from the installed SDK's numeric release:
142
+ `https://github.com/TheWizardNexus/arcane-os-sdk/releases/download/<sdk-version>/arcane-kokoro-windows-x64.tar.gz`.
143
+ That SDK version selects the default helper release; an authored runtime
144
+ `version` alone does not select another SDK release's helper.
145
+ An explicit `url` selects a complete platform-matching archive, with optional
146
+ `version` metadata. Alternatively, `helperRoot` supplies an already prepared
147
+ complete tree copied into the new installation. Select one of these sources.
148
+ Linux and macOS currently require that explicit archive or prepared tree;
149
+ the native CMake source is available, with no claim of an exercised helper
150
+ build on those platforms.
151
+
152
+ The helper record accepts `helperExecutable`, `espeakDataDirectory` and
153
+ `libraryDirectory` overrides within the selected tree. Defaults are
154
+ `bin/arcane-kokoro.exe` on Windows (`bin/arcane-kokoro` elsewhere), `share`, and
155
+ `bin`. The data field identifies the parent containing `espeak-ng-data`.
156
+ Overrides are relative to the selected tree or absolute locations inside
157
+ `helperRoot`; all three paths relocate with that complete tree. Installation
158
+ prepares the helper and its matching data, independently of ONNX and without
159
+ downloading model weights, tokenizer files or voices.
160
+
161
+ Development and generated native composition register this engine as the TTS
162
+ role of `createSpeechService`, sharing the existing model-assets service and
163
+ the selected local-ai ONNX owner. Whisper, when selected, retains the separate
164
+ STT role and device choice. Construction and no-assets startup leave Kokoro
165
+ unloaded without helper preparation. An explicit activation with prepared
166
+ resources acquires the helper in development and waits only for that existing
167
+ ONNX owner. Native packaging has already bundled the helper. These waits stay
168
+ inside speech activation; page rendering and independent services continue.
169
+ Omitting the explicit `onnx` requirement reports unavailability on activation.
170
+ After an explicit `localai.services.recover({runtimes: ['onnx']})` completes,
171
+ the next explicit `speech.load({role: 'tts', ...selection})` reacquires the
172
+ current ONNX owner through existing preparation, after prior activation cleanup.
173
+
174
+ For saved browser assets, prepare the complete projection through the existing
175
+ [`model-assets` owner](model-assets.md), then call
176
+ `Arcane.speech.load({role: 'tts', modelId: 'kokoro', assetProjectionId,
177
+ resourcePaths, executionTarget}, {signal})`. Core retains the incoming
178
+ projection before replacing its activation and releases it only after native
179
+ exit. Preparing/loading a model remains separate from installing the helper.
180
+ Raw constructor `paths` are caller-prepared native paths; portable assembly
181
+ does not turn external model paths or a prior process's projection ID into
182
+ packaged resources. Prepare the intended model resources at their actual host.
183
+
116
184
  ## Development
117
185
 
118
186
  ```sh
@@ -125,6 +193,8 @@ configuration. Subsequent `arcane dev` runs reuse the saved requirements.
125
193
  application selects its `models` in the runtime record before requesting a
126
194
  load or generation. The development owner composes separate image and
127
195
  model-assets services alongside the existing chat/ONNX service.
196
+ `--local-ai onnx,kokoro-native` records the native TTS requirements above;
197
+ its model activation remains explicit when no assets are configured.
128
198
  Development checks selected endpoints and existing executable paths, then
129
199
  installs missing runtimes under the application's `.arcane/local-ai/runtimes`.
130
200
  Runtime preparation runs alongside HTTP serving. Browser rendering continues
@@ -174,8 +244,8 @@ processes. An already running external service remains running.
174
244
  handle with `ready`, `current()`, `handler()` and `close()`. Its `ready` promise
175
245
  means Core composition has started, not that installation or a selected model
176
246
  is ready. Runtime preparation continues at each owning service. `current()`
177
- includes separate `core`, `localAI` and optional `image` state; image status
178
- remains callable during preparation. Composition failures reject `ready`;
247
+ includes separate `core`, `localAI` and optional `image` and `speech` state;
248
+ image status remains callable during preparation. Composition failures reject `ready`;
179
249
  later service preparation failures remain on Core lifecycle and diagnostic
180
250
  events. Page rendering waits for neither.
181
251
 
@@ -227,7 +297,8 @@ owner; it neither prepares another runtime nor loads a model. An unselected
227
297
  engine reports `LOCAL_AI_RUNTIME_NOT_SELECTED`; an unavailable or recovering
228
298
  owner reports `LOCAL_AI_RUNTIME_UNAVAILABLE`, and an aborted service lifetime
229
299
  retains its abort reason. Development and generated native compositions register
230
- one shared model-assets service when ONNX or image generation is selected.
300
+ one shared model-assets service when ONNX, llama.cpp, image generation or
301
+ native Kokoro is selected.
231
302
 
232
303
  Reacquire `getONNXRuntime()` on each explicit model load because runtime recovery
233
304
  replaces that owner. Retain the acquired handle for that session's cleanup and
@@ -757,6 +828,11 @@ retains that metadata with the complete native libraries. Image records include
757
828
  `libraryPath`,
758
829
  `bindingModulePath`, requested `backend`, installed `variants` and selected
759
830
  `models` metadata. Variant records carry backend, runtime root and library path.
831
+ Kokoro records include `helperExecutable`, `espeakDataDirectory` and
832
+ `libraryDirectory`; bundling preserves the complete helper tree under
833
+ `runtime/local-ai/kokoro-native` and makes all three paths artifact-relative.
834
+ The generated entry resolves them on the destination host and obtains the
835
+ existing ONNX owner when the selected TTS activation needs it.
760
836
  Bundle returns `{runtimes,files}`,
761
837
  with artifact-relative runtime paths and a complete emitted file inventory.
762
838
  Bundling requires a fresh native staging destination, preserving existing
@@ -0,0 +1,98 @@
1
+ # Native Magpie speech
2
+
3
+ `arcane-os/local-ai/magpie` exports `createMagpieRuntime` as a named and default export. It retains the selected Magpie model and NanoCodec decoder in one owned native process. It produces complete encoded audio or ordered PCM callbacks; the caller owns playback. Existing Kokoro, browser speech, and complete-audio APIs remain separate capabilities.
4
+
5
+ ## Prepare and select the runtime
6
+
7
+ Supply a prepared native executable and the external NeMo shared-library directory:
8
+
9
+ ```js
10
+ import {createMagpieRuntime} from 'arcane-os/local-ai/magpie';
11
+
12
+ const speech = createMagpieRuntime({
13
+ runtime: {helperExecutable, libraryDirectory},
14
+ paths: {
15
+ model: magpieModel,
16
+ codec: nanoCodecModel,
17
+ tokenizerDirectory
18
+ },
19
+ executionTarget: {backend: 'cuda', cudaVisibleDevices: selectedCudaUuid},
20
+ voice: 'Aria',
21
+ languageCode: 'en-US'
22
+ });
23
+ await speech.load();
24
+ ```
25
+
26
+ The paths identify an already selected Magpie GGUF, matching tokenizer assets, and matching NanoCodec decoder. Nothing is downloaded or copied by this factory. `executionTarget.backend` must explicitly select `cuda` or `cpu`. For CUDA, the native device owner supplies its selected visible device or UUID through `cudaVisibleDevices`; the child receives that process-local environment before loading CUDA. The adapter does not invent a GPU ordinal, select another device, or fall back to another provider.
27
+
28
+ Instead of `runtime`, supply `prepare({signal})`, returning the prepared descriptor directly: `{helperExecutable, libraryDirectory, ...}`. Preparation runs when loading needs the executable, alongside model metadata preparation. It must return the descriptor, not a `{runtime}` wrapper. Without `libraryDirectory`, the executable's directory is used.
29
+
30
+ Applications using model-assets ownership may replace `paths` with `modelAssets`, `assetProjectionId`, and `resourcePaths: {model, codec, tokenizerConfig}`. The resource paths identify members of the retained projection; the tokenizer directory is derived from the selected configuration member. The projection stays retained until its native helper closes.
31
+
32
+ Other factory options are `modelId` (default `magpie`), a lifetime `signal`, and `onEvent` for the existing process/event diagnostics. Factory defaults are voice `Aria` and language `en-US`; both are exposed by `current()`.
33
+
34
+ ## State and lifecycle
35
+
36
+ - `current()` returns provider/model identity, `state`, `loaded`, `busy`, active `requestId`, `queuedRequests`, `defaultVoice`, `languageCode`, model-owned `voices`, configured execution information, `sampleRate`, progress, and a serialized error. Its speech catalog is `{streaming: true, responseFormats: ['wav', 'pcm'], defaultResponseFormat: 'wav'}`. Configured execution is not evidence of actual GPU inference.
37
+ - `subscribe(listener, {replay: true, signal})` observes lifecycle state and replays the current state by default; it returns an unsubscribe function.
38
+ - `load({paths, assetProjectionId, resourcePaths, executionTarget, modelId, signal})` loads or replaces the selected resources. Reusing the same selection reuses its loaded engine. The optional signal cancels that caller's load wait; the factory lifetime signal owns engine cancellation. Load the engine before requesting speech.
39
+ - `unload()` cancels owned work, joins native and callback cleanup, releases retained assets, and leaves the factory reusable.
40
+ - `close()` joins unloading and disposes the factory. Repeated close calls share completion.
41
+
42
+ Requests accepted by one engine are processed in order. Unloading or replacing that engine cancels its owned work; it does not cancel another engine or audio owner.
43
+
44
+ ## Exact requests, voices, and languages
45
+
46
+ Both synthesis methods accept the same request:
47
+
48
+ ```js
49
+ const request = {
50
+ input: 'The moon is closed for dragon maintenance.',
51
+ voice: 'Aria',
52
+ languageCode: 'en-US'
53
+ };
54
+ ```
55
+
56
+ `input` is passed unchanged: no trimming, sentence splitting, prompt injection, tempo stage, or output cap. A NUL or an unpaired surrogate cannot be represented by the native UTF-8 C-string interface and produces an explicit incompatibility error.
57
+
58
+ Optional `model` must match the configured `modelId`. `voice` defaults to the factory selection and is resolved case-insensitively against the matching tokenizer's `model_config.yaml` speaker-map reference and its model-owned JSON IDs. Embedded GGUF voice labels are not used as the selection map.
59
+
60
+ `languageCode` (or `language_code`) is separate per-request control information; it never changes the authored input. Omission uses the configured language. The application/model owns language selection. Installed tokenizer support determines whether a language can run; a language listed in configuration is not proof of runtime support. Native errors are returned rather than switching languages or engines. The public native ABI supplies the natural speaking rate only; an optional `speed` must be `1`.
61
+
62
+ ## Complete audio and streaming audio
63
+
64
+ `synthesize(request, {signal, requestId, onProgress})` resolves with complete `audioBase64`, `contentType`, `format`, `sampleFormat: 's16le'`, sample rate, channel count, model, voice, language code, speed, and request ID. Set `request.responseFormat` to `wav` (default) or `pcm`. WAV uses the external RIFF framing; when a complete result cannot fit that format, select PCM or streaming instead of truncating it.
65
+
66
+ `synthesizeStream(request, {signal, requestId, onProgress, onAudio})` requires `onAudio` and awaits each callback in original order:
67
+
68
+ ```js
69
+ const controller = new AbortController();
70
+ const completion = speech.synthesizeStream(request, {
71
+ signal: controller.signal,
72
+ requestId: 'dragon-announcement',
73
+ async onAudio({pcm, format, sampleRate, channels, requestId}) {
74
+ await audioOutput.write({pcm, format, sampleRate, channels, requestId});
75
+ }
76
+ });
77
+ const result = await completion;
78
+ ```
79
+
80
+ `pcm` is a Node `Buffer` of signed 16-bit little-endian interleaved PCM; the current native engine returns mono audio. The callback includes `format: 's16le'`, `sampleRate`, `channels`, and `requestId`. Awaited callback delivery applies backpressure to the native helper. The streaming terminal result contains metadata and `complete: true`, without accumulating or returning the audio again. Browser/Core transport and playback have their own delivery and cleanup owners.
81
+
82
+ `onProgress` receives synthesis progress. Callback failures cancel the matching utterance and remain observable. A caller-supplied `requestId` identifies its active or queued request; otherwise the runtime creates one.
83
+
84
+ ## Cancellation and callback ownership
85
+
86
+ `await speech.stop({requestId})` stops only that active or queued utterance and joins native completion and callback cleanup. It returns `{requestId, stopped: true}`, or `stopped: false` when the ID is no longer owned. Cancellation of an active native call is observed at a PCM callback boundary: this ABI has no asynchronous pre-first-PCM interrupt. Model creation can be cancelled by retiring its owned process.
87
+
88
+ An output callback must return before joining its own stop, unload, close, or selection replacement. Those self-joining calls reject with `MAGPIE_CALLBACK_WAIT`; their return types do not change inside callbacks. To cancel from `onAudio` or `onProgress`, synchronously call the caller-owned `controller.abort()` and return. The external owner then awaits the synthesis completion and cleanup. Likewise, do not await a new queued synthesis from a callback that the current synthesis is awaiting.
89
+
90
+ The synthesis promise rejects on cancellation or failure. Native errors, callback failures, and cleanup failures remain observable; combined failures are preserved in aggregates. Complete audio is never reported for an interrupted request.
91
+
92
+ ## Build the first-party helper
93
+
94
+ `arcane-os/local-ai/magpie/build` exports named/default `buildMagpieHelper({runtime, outputRoot, signal, onEvent})`. The selected upstream installation supplies `runtime.cmakeDirectory`, containing `NeMoSpeechConfig.cmake`, its public TTS headers, and its platform-matching libraries. Build prerequisites are CMake 3.26 or newer, a C++17 compiler, platform threads support, and that prepared NeMo TTS installation.
95
+
96
+ The builder configures, builds, and installs the SDK's `arcane-magpie` executable under `outputRoot`. It returns `{platform, architecture, root, helperExecutable, diagnostics: {configure, compile, install}}`. Combine its executable with the selected upstream `libraryDirectory` for the runtime descriptor. The SDK does not vendor NeMo libraries, models, tokenizer assets, or voices.
97
+
98
+ The source targets Windows, Linux, and macOS; Android requires its host adaptation and compatible upstream runtime. On October 17, 2026, the selected Windows x64 helper configured, compiled, and installed successfully against installed NeMo-Speech.cpp 0.2.0 using CMake 4.4.3, Visual Studio 2022, MSVC 19.44.35217, and Windows SDK 10.0.26100. Each operation exited 0, with complete diagnostics retained in the SDK's ignored selected-output directory. The helper was not launched: model inference, microphone capture, and audio playback remain unexecuted for this adapter. Linux and macOS compilation and execution remain unverified. Build success, actual selected-device inference, and audible playback are separate evidence boundaries.
@@ -0,0 +1,204 @@
1
+ # Native Nemotron streaming transcription
2
+
3
+ `arcane-os/local-ai/nemotron` provides `createNemotronRuntime`, a Node/Core
4
+ adapter for NVIDIA's public NeMo Speech ASR C ABI. One owned helper retains the
5
+ selected recognizer and optional diarization model. One active audio stream
6
+ belongs to that engine at a time. Existing Whisper transcription and standalone
7
+ diarization remain separate supported capabilities.
8
+
9
+ This adapter has no microphone, playback, conversational endpoint detector,
10
+ keyword classifier, cloud fallback or automatic engine replacement. The host
11
+ owns capture and the model owns conversational interpretation. A final ASR
12
+ segment does not establish that the person has finished thinking, yielded the
13
+ conversation, approved an action or issued an instruction.
14
+
15
+ ## Preparation and selection
16
+
17
+ Select an upstream NeMo Speech runtime with its public ASR headers, library,
18
+ runtime dependencies and CMake package configuration. This implementation was
19
+ source-reviewed against the installed NVIDIA NeMo-Speech.cpp 0.2.0 public ABI,
20
+ `nemotron-3.5-asr-streaming-0.6b` and `Nemotron-3-Diarization`. Acquire runtime and
21
+ model assets through their upstream distribution; the SDK adapter does not
22
+ download or redistribute them.
23
+
24
+ `arcane-os/local-ai/nemotron/build` exports `buildNemotronHelper`:
25
+
26
+ ```js
27
+ import {buildNemotronHelper} from 'arcane-os/local-ai/nemotron/build';
28
+ import {createNemotronRuntime} from 'arcane-os/local-ai/nemotron';
29
+
30
+ const built = await buildNemotronHelper(
31
+ {
32
+ runtime: {cmakeDirectory: selectedNeMoCmakeDirectory},
33
+ outputRoot: selectedHelperOutputDirectory,
34
+ signal,
35
+ onEvent
36
+ }
37
+ );
38
+ const engine = createNemotronRuntime(
39
+ {
40
+ runtime: {
41
+ helperExecutable: built.helperExecutable,
42
+ binaryDirectory: selectedNeMoBinaryDirectory,
43
+ libraryDirectory: selectedNeMoLibraryDirectory
44
+ },
45
+ modelPath: selectedAsrModelPath,
46
+ diarizationModelPath: selectedDiarizationModelPath,
47
+ gpu: selectedNativeGpuOrdinal,
48
+ onEvent
49
+ }
50
+ );
51
+ await engine.load();
52
+ ```
53
+
54
+ The builder requires CMake 3.26 or later and a platform C++17 toolchain. It
55
+ configures, builds and installs only the first-party helper, returning
56
+ `helperExecutable`, its `executable` alias, `root`, `platform`, `architecture`
57
+ and complete configure/build/install diagnostics. The caller selects the output
58
+ directory and supplies the appropriate installed upstream runtime. Windows
59
+ loads its runtime through the child's `PATH`; Linux uses `LD_LIBRARY_PATH` and
60
+ macOS uses `DYLD_LIBRARY_PATH`, alongside the helper's loader-relative paths.
61
+ `runtime.env` supplies additional child-only environment settings.
62
+
63
+ `gpu` is required: supply the native runtime's selected GPU ordinal or explicitly
64
+ select `-1` for CPU. Device selection belongs to the host. When the host changes
65
+ `CUDA_VISIBLE_DEVICES`, the ordinal must describe that child's visible devices.
66
+ The adapter reports `requestedGpu`; `observedBackend` remains `null` because a
67
+ requested device alone does not prove GPU inference.
68
+
69
+ Instead of `runtime`, supply an asynchronous
70
+ `prepare({signal, onEvent})` function returning that prepared descriptor.
71
+ Preparation runs during `load`, never at construction. `modelPath`,
72
+ `diarizationModelPath` and `gpu` may also be supplied by the returned descriptor.
73
+ Omit `diarizationModelPath` when speaker labels are not requested.
74
+
75
+ ## Engine lifecycle
76
+
77
+ - `current()` returns selected provider/model, lifecycle `state`, `loaded`,
78
+ `busy`, native version, requested device, active speaker scope, errors and any
79
+ explicit reset requirement.
80
+ - `subscribe(listener, {replay: true, signal} = {})` immediately replays current
81
+ state and observes subsequent lifecycle events; it returns an unsubscribe
82
+ function.
83
+ - `load({modelId, modelPath, diarizationModelPath, gpu, signal} = {})` loads the
84
+ selected models and resolves only when native readiness is reported.
85
+ `modelId` defaults to `nemotron-3.5-asr-streaming-0.6b`. Concurrent callers share
86
+ that load; one caller's signal cancels its wait, not the shared load. Unload
87
+ before replacing an active selection.
88
+ - `unload()` cancels and joins the active stream and model preparation, closes
89
+ the owned helper, and retains selection for a later explicit load.
90
+ - `close()` joins unload and disposes lifecycle subscriptions.
91
+
92
+ The host can compose this engine into the existing
93
+ [`createSpeechService`](./native-speech.md) STT engine seam. Readiness follows
94
+ the selected engine, not an application startup barrier.
95
+
96
+ ## Ordered streaming
97
+
98
+ ```js
99
+ const stream = await engine.openStream(
100
+ {
101
+ model: 'nemotron-3.5-asr-streaming-0.6b',
102
+ sampleRate: actualCaptureSampleRate,
103
+ signal,
104
+ onUpdate: async function showRecognition(result) {
105
+ await displayCompleteSnapshot(result);
106
+ }
107
+ }
108
+ );
109
+ await stream.push(monoFloat32Samples);
110
+ await stream.forceEndpoint();
111
+ // The same stream accepts the next utterance, preserving its speaker scope.
112
+ await stream.push(nextMonoFloat32Samples);
113
+ await stream.finish();
114
+ await stream.completion;
115
+ ```
116
+
117
+ `openStream({sampleRate = 16000, languageCode, model, onUpdate, signal} = {})`
118
+ requires the selected model to be ready, or waits for its already-started load.
119
+ It never silently reloads after a reset. An explicit `model` must match the
120
+ loaded selection. `sampleRate` is the actual input rate: an integer from 8000
121
+ through 96000, or `0` for the model rate. It is passed unchanged to the native
122
+ API. The transport's ordered 100 ms scheduling does not reinterpret a 48 kHz
123
+ recording as 16 kHz or discard audio.
124
+
125
+ The returned handle contains `push`, `forceEndpoint`, `finish`, `cancel`,
126
+ `completion`, `current`, `sampleRate` and `speakerScope`:
127
+
128
+ - `push(audio)` accepts mono `Float32Array` or
129
+ `{encoding: 'f32le', data: BASE64}` and resolves to all ordered snapshots from
130
+ that input. The encoded form carries exact little-endian float32 samples.
131
+ Keep a supplied array unchanged until its push settles. Other encoded
132
+ recordings produce `NEMOTRON_AUDIO_FORMAT_UNSUPPORTED`; caller-owned decoding
133
+ is required.
134
+ - `forceEndpoint()` drains the current ASR utterance while retaining the native
135
+ stream, recognizer and diarization scope. Capture closure and conversational
136
+ turn completion remain the host's decisions.
137
+ - `finish()` queues behind all accepted input, terminally finishes and drains
138
+ the native stream, delivers remaining snapshots, and resolves with that final
139
+ drain's snapshot array. The loaded engine remains available for another
140
+ stream.
141
+ - `current()` returns the latest delivered snapshot, or `null` before one is
142
+ available.
143
+ - `completion` settles after terminal finish or joined cancellation. Successful
144
+ finish resolves to the final drain's snapshots; earlier snapshots have already
145
+ been delivered through `onUpdate` and each operation's result.
146
+
147
+ Each snapshot includes `sequence`, `final`, `audioProcessedSeconds`,
148
+ `channelTag`, `speakerScope`, and every native `alternatives` record with its
149
+ exact `text`, `confidence`, `languageCodes` and `words`. Every word preserves
150
+ `word`, `startTimeMs`, `endTimeMs`, `confidence` and numeric `speaker`. Top-level
151
+ `text` and `words` expose the first alternative without changing its content.
152
+
153
+ Partial snapshots are provisional replacements, not appended transcript text.
154
+ Native endpoint finals describe individual utterances in a continuing stream;
155
+ they are forwarded unchanged. Keep their boundaries distinct. Speaker numbers
156
+ distinguish voices within the reported scope, not named people or biometric
157
+ identity. A new stream has a new scope. Timestamp values retain the native
158
+ runtime's units and origin without an invented offset.
159
+
160
+ Callbacks run serially in native result order and may be asynchronous. A
161
+ callback failure stops the stream, remains observable, and retains complete
162
+ native result records in diagnostics. Inside an active `onUpdate` callback,
163
+ calling `push`, `forceEndpoint`, `finish`, `cancel`, `unload` or `close` raises
164
+ `NEMOTRON_CALLBACK_WAIT`: those joined operations would wait for the callback
165
+ itself. Abort the caller-owned signal synchronously if cancellation is needed
166
+ there, return from the callback, then join through the external owner.
167
+
168
+ ## Cancellation and complete recordings
169
+
170
+ `cancel(reason)` and an aborted stream signal first request ordinary native
171
+ stream closure. If a noninterruptible public-ABI call is in flight, cancellation
172
+ terminates only this engine's owned helper and joins it. It also joins accepted
173
+ JavaScript operations and output callbacks before releasing stream ownership.
174
+ Explicit cancellation resolves `completion` and `cancel` with
175
+ `{cancelled: true, resetRequired, speakerScope, diagnostics?}`. Actual native,
176
+ callback or cleanup failures reject and retain their diagnostic causes.
177
+
178
+ After helper termination, `current()` reports `resetRequired: true`, a reset
179
+ record and `state: 'unloaded'`. The next caller must explicitly `load()`;
180
+ recognizer replacement and speaker-scope reset are never silent. Ordinary
181
+ stream closure keeps the loaded helper. No cancellation claims that unfinished
182
+ recognition produced a complete final.
183
+
184
+ `transcribe(request, {signal, onProgress, onUpdate} = {})` uses the same stream
185
+ implementation for complete recordings. Supply `{audio, sampleRate, model?,
186
+ languageCode?}` or a direct supported audio value at the default sample rate.
187
+ It returns `{text, words, results, speakerScope}`, retaining all final records
188
+ in `results`, and emits accepted/recognizing progress. Its final text is the
189
+ ordered concatenation of native final text without inserted separators. It
190
+ does not decode WAV, MP3 or other encoded recording formats.
191
+
192
+ ## Evidence boundary
193
+
194
+ The adapter, transport and platform paths have been source-reviewed. On October
195
+ 17, 2026, the selected Windows x64 helper configured, compiled and installed
196
+ successfully against installed NeMo-Speech.cpp 0.2.0 using CMake 4.4.3, Visual
197
+ Studio 2022, MSVC 19.44.35217 and Windows SDK 10.0.26100. Each operation exited 0;
198
+ complete diagnostics remain in the SDK's ignored selected-output directory. The
199
+ helper was not launched, and there is no measured latency, GPU-inference,
200
+ microphone, diarization, audio playback or multi-utterance acceptance result for
201
+ this adapter. Linux and macOS compilation and execution remain unverified.
202
+ Compilation, header review and upstream documentation remain distinct from a
203
+ built native application's observed behavior. Android requires the necessary
204
+ host integration and a matching native runtime.