arcane-os 0.62.0 → 0.64.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +40 -0
  2. package/README.md +1 -1
  3. package/browser-runtime/core/client.mjs +7 -2
  4. package/docs/reference/README.md +7 -0
  5. package/docs/reference/cli.md +5 -1
  6. package/docs/reference/core-client.md +11 -2
  7. package/docs/reference/core-native-packaging.md +57 -8
  8. package/docs/reference/core-repositories.md +149 -0
  9. package/docs/reference/inventory/package-api.json +163 -9
  10. package/docs/reference/local-ai.md +51 -7
  11. package/docs/reference/local-image-flux.md +154 -0
  12. package/docs/reference/local-image-generation.md +12 -0
  13. package/docs/reference/local-image-sdxl.md +139 -0
  14. package/docs/reference/local-whisper.md +219 -0
  15. package/docs/reference/native-speech.md +163 -0
  16. package/docs/reference/runtime-modules.md +16 -2
  17. package/docs/reference/sdk-api.md +335 -6
  18. package/package.json +5 -1
  19. package/runtime/arcane/components/speech.html +15 -3
  20. package/runtime/arcane/components/voice-transcription.html +19 -3
  21. package/runtime/arcane/modules/AI.js +250 -27
  22. package/runtime/arcane/modules/AIProviderRuntime.js +20 -3
  23. package/runtime/arcane/modules/AIRuntimeState.js +15 -2
  24. package/runtime/arcane/modules/ComponentContracts.js +3 -0
  25. package/schemas/arcane-app.schema.json +53 -2
  26. package/schemas/arcane-package.schema.json +53 -2
  27. package/src/core/repositories.mjs +210 -0
  28. package/src/core/services/speech.mjs +361 -0
  29. package/src/local-ai/config.mjs +9 -3
  30. package/src/local-ai/dev.mjs +32 -2
  31. package/src/local-ai/image-library.mjs +9 -0
  32. package/src/local-ai/image-models/flux.mjs +31 -0
  33. package/src/local-ai/image-models/sdxl.mjs +18 -0
  34. package/src/local-ai/install.mjs +26 -7
  35. package/src/local-ai/whisper/build.mjs +48 -0
  36. package/src/local-ai/whisper/config.mjs +61 -0
  37. package/src/local-ai/whisper/index.mjs +515 -0
  38. package/src/local-ai/whisper/install.mjs +274 -0
  39. package/src/local-ai/whisper/native/CMakeLists.txt +98 -0
  40. package/src/local-ai/whisper/native/NOTICE +18 -0
  41. package/src/local-ai/whisper/native/README.md +104 -0
  42. package/src/local-ai/whisper/native/main.cpp +489 -0
  43. package/src/native/portable-layout.mjs +37 -7
  44. package/src/native/windows-provider.mjs +39 -15
@@ -1,17 +1,171 @@
1
1
  {
2
2
  "schemaVersion": 1,
3
- "sdkVersion": "0.57.0",
3
+ "sdkVersion": "0.63.0",
4
4
  "environment": {
5
5
  "runtime": "Node.js for Node entrypoints; browser for browser-only entrypoints",
6
6
  "minimumVersion": "22.23.2 for Node entrypoints",
7
7
  "moduleSystem": "ESM"
8
8
  },
9
- "memberCount": 267,
9
+ "memberCount": 278,
10
10
  "runtimeSubpathPatterns": {
11
11
  "arcane-os/modules/*": "./runtime/arcane/modules/*",
12
12
  "arcane-os/entities/*": "./runtime/arcane/entities/*"
13
13
  },
14
14
  "members": [
15
+ {
16
+ "id": "core-repositories:resolveArcaneDataPaths",
17
+ "name": "resolveArcaneDataPaths",
18
+ "displayName": "resolveArcaneDataPaths()",
19
+ "kind": "function",
20
+ "signature": "resolveArcaneDataPaths({dataRoot}={})",
21
+ "entrypoints": ["arcane-os/core/repositories"],
22
+ "primaryImport": "arcane-os/core/repositories",
23
+ "group": "Native repository workspaces",
24
+ "summary": "Resolves persistent per-user ArcaneData and Repos locations without I/O or migration.",
25
+ "availability": "Node on Windows, Linux and macOS; Android supplies its persistent dataRoot",
26
+ "protocol": "Native filesystem paths",
27
+ "normalization": "Returns absolute dataRoot and repositoriesRoot; explicit dataRoot takes precedence, otherwise uses LocalAppData, XDG data home or user Application Support. Changes no launch, preference, workspace or user data."
28
+ },
29
+ {
30
+ "id": "core-repositories:createRepositoryWorkspace",
31
+ "name": "createRepositoryWorkspace",
32
+ "displayName": "createRepositoryWorkspace()",
33
+ "kind": "function",
34
+ "signature": "createRepositoryWorkspace({name,directory,dataRoot,remote,branch,onEvent,run=runProcess}={})",
35
+ "entrypoints": ["arcane-os/core/repositories"],
36
+ "primaryImport": "arcane-os/core/repositories",
37
+ "group": "Native repository workspaces",
38
+ "summary": "Owns one selected native working checkout with ArcaneData/Repos defaults and existing Git/process lifecycle.",
39
+ "availability": "Node with Git on Windows, Linux and macOS; Android requires host process adaptation",
40
+ "protocol": "Git working repository and SDK process adapter",
41
+ "normalization": "Returns directory, open, status, pull, push and identical close/drain/dispose hooks. Only missing or empty destinations clone; existing roots are inspected without rewriting, resetting or pulling. Explicit paths remain selected. Same-directory operations serialize within the process; cancellation and shutdown join accepted process work."
42
+ },
43
+ {
44
+ "id": "core-speech:createSpeechService",
45
+ "name": "createSpeechService",
46
+ "displayName": "createSpeechService()",
47
+ "kind": "function",
48
+ "signature": "createSpeechService({stt,tts,signal}={})",
49
+ "entrypoints": ["arcane-os/core/speech"],
50
+ "primaryImport": "arcane-os/core/speech",
51
+ "group": "Core native speech",
52
+ "summary": "Composes host-supplied transcription and synthesis engines with independent loading and role readiness.",
53
+ "availability": "Node Core with explicitly supplied engines; engine platform, model and format support remain engine-owned",
54
+ "protocol": "arcane/1 speech.status/transcribe/synthesize with speech.state/progress",
55
+ "normalization": "Returns current/subscribe/close/start/methods/drain/dispose; forwards complete requests and results unchanged; replaying service state, synchronous accepted Thinking, combined request/lifetime cancellation and concurrent joined engine shutdown; no engine installation or automatic provider selection"
56
+ },
57
+ {
58
+ "id": "core-speech:default",
59
+ "name": "default",
60
+ "displayName": "createSpeechService default export",
61
+ "kind": "function",
62
+ "signature": "default as createSpeechService",
63
+ "entrypoints": ["arcane-os/core/speech"],
64
+ "primaryImport": "arcane-os/core/speech",
65
+ "group": "Core native speech",
66
+ "summary": "Default binding for the same injected-engine native speech service factory.",
67
+ "availability": "Node Core with host-supplied engines",
68
+ "protocol": "arcane/1 service definition",
69
+ "normalization": "Binding identity equals the named createSpeechService export"
70
+ },
71
+ {
72
+ "id": "local-ai-image:createImageRuntime",
73
+ "name": "createImageRuntime",
74
+ "displayName": "createImageRuntime()",
75
+ "kind": "function",
76
+ "signature": "createImageRuntime({libraryPath,bindingModulePath,variants=[],models=[],backend='auto',signal,onEvent}={})",
77
+ "entrypoints": ["arcane-os/local-ai/image"],
78
+ "primaryImport": "arcane-os/local-ai/image",
79
+ "group": "Core local images",
80
+ "summary": "Retains selected stable-diffusion.cpp generation and PNG img2img execution in a shared native worker and returns complete PNG results.",
81
+ "availability": "Node with prepared upstream image library, Koffi binding and caller-selected model resources",
82
+ "protocol": "Shared per-library worker/context lifetime with ordered native operations and replaying state",
83
+ "normalization": "Returns load/generate/edit/unload/current/subscribe/close; edit copies complete PNG input at acceptance and takes explicit strength; prompts remain complete and cancellation/close join native completion before resource release"
84
+ },
85
+ {
86
+ "id": "core-image:createCoreImageRuntime",
87
+ "name": "createCoreImageRuntime",
88
+ "displayName": "createCoreImageRuntime()",
89
+ "kind": "function",
90
+ "signature": "createCoreImageRuntime({client=getInstalledCoreClient(),signal,onEvent}={})",
91
+ "entrypoints": ["arcane-os/ai/core-image"],
92
+ "primaryImport": "arcane-os/ai/core-image",
93
+ "group": "Core local images",
94
+ "summary": "Uses an available Core image service for generation and original-PNG editing with synchronous Thinking acknowledgement, lifecycle replay and complete PNG Blobs.",
95
+ "availability": "Browser/native WebView with an available Core image service; construction opens no connection or model",
96
+ "protocol": "arcane/1 image.status/load/generate/edit/unload and image.state/progress",
97
+ "normalization": "Returns load/generate/edit/unload/inspect/current/subscribe/close; edit preserves the complete original PNG Blob/File through transport with explicit strength; timeoutMs is zero, close cancels only this accessor and explicit unload owns the shared model"
98
+ },
99
+ {
100
+ "id": "core-image-service:createLocalImageService",
101
+ "name": "createLocalImageService",
102
+ "displayName": "createLocalImageService()",
103
+ "kind": "function",
104
+ "signature": "createLocalImageService(configuration,{appRoot,runtimes=[],signal,onEvent,prepare,modelAssets}={})",
105
+ "entrypoints": ["arcane-os/core/image"],
106
+ "primaryImport": "arcane-os/core/image",
107
+ "group": "Core local images",
108
+ "summary": "Composes independent image-runtime preparation, retained native model use and complete PNG transport as a Core service.",
109
+ "availability": "Node Core with selected stable-diffusion.cpp configuration and prepared runtime or preparation callback",
110
+ "protocol": "arcane/1 image lifecycle, generate/edit and progress; base64 framing only at complete PNG input/result transport boundaries",
111
+ "normalization": "Status remains callable during preparation; shared model-assets handles remain retained through actual model unload, worker completion and disposal"
112
+ },
113
+ {
114
+ "id": "core-image-service:default",
115
+ "name": "default",
116
+ "displayName": "createLocalImageService default export",
117
+ "kind": "function",
118
+ "signature": "default as createLocalImageService",
119
+ "entrypoints": ["arcane-os/core/image"],
120
+ "primaryImport": "arcane-os/core/image",
121
+ "group": "Core local images",
122
+ "summary": "Default binding for the same native image service factory.",
123
+ "availability": "Node Core runtime",
124
+ "protocol": "arcane/1 service definition",
125
+ "normalization": "Binding identity equals the named createLocalImageService export"
126
+ },
127
+ {
128
+ "id": "core-model-assets:prepareCoreModelAssets",
129
+ "name": "prepareCoreModelAssets",
130
+ "displayName": "prepareCoreModelAssets()",
131
+ "kind": "function",
132
+ "signature": "async prepareCoreModelAssets({client=getInstalledCoreClient(),workingDirectory,members,signal,onProgress}={})",
133
+ "entrypoints": ["arcane-os/ai/core-model-assets"],
134
+ "primaryImport": "arcane-os/ai/core-model-assets",
135
+ "group": "Core model assets",
136
+ "summary": "Projects complete caller-owned Blob/File members through Core into native working files without changing their durable store.",
137
+ "availability": "Browser with an available model-assets Core service and complete selected members",
138
+ "protocol": "arcane/1 modelAssets open/write/complete/release operations with transport-local base64",
139
+ "normalization": "Returns id/directory/ordered members/release; independent files transfer concurrently, each file writes in order, and cancellation joins cleanup"
140
+ },
141
+ {
142
+ "id": "core-model-assets-service:createModelAssetService",
143
+ "name": "createModelAssetService",
144
+ "displayName": "createModelAssetService()",
145
+ "kind": "function",
146
+ "signature": "createModelAssetService({appRoot=process.cwd()}={})",
147
+ "entrypoints": ["arcane-os/core/model-assets"],
148
+ "primaryImport": "arcane-os/core/model-assets",
149
+ "group": "Core model assets",
150
+ "summary": "Owns selected native working-file projections and engine retain handles while original stored model files remain unchanged.",
151
+ "availability": "Node Core with an application-selected working directory",
152
+ "protocol": "arcane/1 modelAssets.open/write/complete/release/status and modelAssets.state",
153
+ "normalization": "Returns current/retain/dispose and Core service lifecycle; final native release joins writes and closes before deleting only its working projection"
154
+ },
155
+ {
156
+ "id": "core-model-assets-service:default",
157
+ "name": "default",
158
+ "displayName": "createModelAssetService default export",
159
+ "kind": "function",
160
+ "signature": "default as createModelAssetService",
161
+ "entrypoints": ["arcane-os/core/model-assets"],
162
+ "primaryImport": "arcane-os/core/model-assets",
163
+ "group": "Core model assets",
164
+ "summary": "Default binding for the same native working-file service factory.",
165
+ "availability": "Node Core runtime",
166
+ "protocol": "arcane/1 service definition",
167
+ "normalization": "Binding identity equals the named createModelAssetService export"
168
+ },
15
169
  {
16
170
  "id": "local-ai-onnx:createONNXRuntime",
17
171
  "name": "createONNXRuntime",
@@ -129,14 +283,14 @@
129
283
  "name": "createLocalAIService",
130
284
  "displayName": "createLocalAIService()",
131
285
  "kind": "function",
132
- "signature": "createLocalAIService(configuration,{appRoot,runtimes=[],signal,onEvent}={})",
286
+ "signature": "createLocalAIService(configuration,{appRoot,runtimes=[],signal,onEvent,prepare}={})",
133
287
  "entrypoints": ["arcane-os/core/local-ai"],
134
288
  "primaryImport": "arcane-os/core/local-ai",
135
289
  "group": "Core local AI",
136
290
  "summary": "Composes selected native llama.cpp, Ollama and ONNX engines as a Core service with retained process/worker ownership and model lifecycle.",
137
291
  "availability": "Node with selected upstream runtime distributions or available configured system services",
138
292
  "protocol": "arcane/1 methods and events; llama.cpp OpenAI HTTP/SSE, Ollama native HTTP/NDJSON and ONNX tensor transport",
139
- "normalization": "Application-owned runtime and model requirements; independent startup, complete responses and exact owned-process cleanup"
293
+ "normalization": "Application-owned runtime and model requirements; optional prepare supplies installed records during this service startup; independent services, complete responses and exact owned-process cleanup"
140
294
  },
141
295
  {
142
296
  "id": "core-local-ai:default",
@@ -161,10 +315,10 @@
161
315
  "entrypoints": ["arcane-os/local-ai"],
162
316
  "primaryImport": "arcane-os/local-ai",
163
317
  "group": "Core local AI",
164
- "summary": "Reuses selected managed native runtimes or installs complete official llama.cpp/Ollama distributions, ONNX npm runtime trees or NeMo libraries.",
318
+ "summary": "Reuses selected managed native runtimes or installs complete official llama.cpp/Ollama/image distributions, ONNX npm runtime trees or NeMo libraries.",
165
319
  "availability": "Node; upstream platform and architecture assets determine runtime availability",
166
- "protocol": "Official upstream release archives or selected ONNX npm installation",
167
- "normalization": "Returns absolute runtime records; shares matching concurrent installs, preserves libraries and reports failures without changing global PATH or services"
320
+ "protocol": "Official upstream release archives and selected ONNX/Koffi npm runtime installations",
321
+ "normalization": "Returns absolute runtime records, including image library/binding/variant/model metadata; shares matching installs, preserves libraries, reports failures without changing global PATH or services and prepares no model files"
168
322
  },
169
323
  {
170
324
  "id": "local-ai:bundleLocalAIRuntimes",
@@ -178,7 +332,7 @@
178
332
  "summary": "Copies selected complete runtime distributions into a fresh native artifact staging root.",
179
333
  "availability": "Node; supports bundling published upstream platform assets independently of the build host",
180
334
  "protocol": "Artifact-relative runtime/local-ai layout",
181
- "normalization": "Returns {runtimes,files} with relative runtime records and the complete emitted inventory; model resources remain application-owned"
335
+ "normalization": "Returns {runtimes,files} with artifact-relative runtime/variant paths and complete inventory; image URL descriptors remain for post-launch preparation and external model working paths report portability incompatibility"
182
336
  },
183
337
  {
184
338
  "id": "root:WS",
@@ -4052,7 +4206,7 @@
4052
4206
  "summary": "Assembles selected application services and browser content with the SDK-owned Windows x64 WebView2 and SEA host.",
4053
4207
  "availability": "Node for assembly; Windows x64 with WebView2 Runtime for execution",
4054
4208
  "protocol": "arcane-native-builder/1 windows-x64 provider",
4055
- "normalization": "Uses a selected hostDirectory or the exact installed SDK version's numeric release asset; preserves complete payloads; native.icon PNG/ICO derives the copied executable and host.icon without an app compiler, while other formats retain assembly with native.icon.unsupported and executableIcon:false; the matching host loads the selected window/taskbar icon and handles window.setTheme with per-field applied/unsupported results; run cancellation closes stdin and waits for host-owned graceful shutdown"
4209
+ "normalization": "Uses a selected hostDirectory or the exact installed SDK version's numeric release asset; preserves complete payloads; app displayName plus .exe (app ID fallback) and matching .exe.config use Windows filename substitutions only; manifest.host.executable owns inventory, icon, verify and run while generic cache/Core names remain unchanged; native.icon PNG/ICO derives the copied executable and host.icon without an app compiler, while other formats retain assembly with native.icon.unsupported and executableIcon:false; the matching host loads the selected window/taskbar icon and handles window.setTheme with per-field applied/unsupported results; run cancellation closes stdin and waits for host-owned graceful shutdown"
4056
4210
  },
4057
4211
  {
4058
4212
  "id": "native-windows-provider:arcaneNativeBuilderProvider",
@@ -1,6 +1,7 @@
1
1
  # Local AI through Core
2
2
 
3
- Core owns native llama.cpp and Ollama processes and ONNX worker sessions. The browser retains its own
3
+ Core owns native llama.cpp and Ollama processes, ONNX worker sessions and
4
+ selected local image generation. The browser retains its own
4
5
  Wllama and ONNX implementations, and can also use an explicitly connected Core
5
6
  when that system service is available. Runtime installation, model loading and
6
7
  inference are separate operations.
@@ -20,7 +21,8 @@ Existing native fields remain alongside this record:
20
21
  "runtimes": [
21
22
  {"id": "llama.cpp"},
22
23
  {"id": "ollama"},
23
- {"id": "onnx"}
24
+ {"id": "onnx"},
25
+ {"id": "stable-diffusion.cpp", "backend": "auto", "models": ["sd14"]}
24
26
  ],
25
27
  "llamaCpp": {
26
28
  "modelsDirectory": "models/llama"
@@ -34,11 +36,25 @@ Existing native fields remain alongside this record:
34
36
  Each runtime requirement accepts `id`, optional upstream `version`, and an
35
37
  optional runtime archive `url`. ONNX resolves its official npm package or an
36
38
  explicitly supplied package archive. A string runtime ID is also accepted.
37
- Omitting the version allows reuse of an available runtime. Explicit versions
39
+ For chat runtimes, omitting the version allows reuse of an available runtime. Explicit versions
38
40
  select that upstream distribution; an existing service must match the requested
39
41
  version before it is reused. A service occupying the selected address is
40
42
  preserved when its version cannot be established.
41
43
 
44
+ Both public application/package schemas accept `stable-diffusion.cpp` as a
45
+ string or runtime-record ID. Its record additionally accepts `backend`
46
+ (`auto`, `cpu` or `metal`) and `models`, an array of `sd14`, `flux2-klein-4b` or
47
+ complete model descriptors. Omitted image version selects `master-929-3f8527a`; omitted backend
48
+ selects `auto`; omitted models leaves model selection empty. Windows/Linux
49
+ distribution defaults use CPU; the selected universal Mac archive supports
50
+ Metal with CPU fallback. Installation obtains the upstream library and its
51
+ managed `koffi@3.3.2` binding, without installing a model. Complete model
52
+ descriptors keep their resources, context, defaults and operations separate.
53
+ See [image selection and model preparation](local-image-generation.md#selection-and-model-assets).
54
+ The [FLUX.2 Klein 4B guide](local-image-flux.md) describes its diffusion model,
55
+ VAE and Qwen text-conditioning resources, four-step defaults and `txt2img`
56
+ operation. Each model retains its own supported operation set.
57
+
42
58
  `llamaCpp` accepts `url`, `model`, `modelsDirectory`, and an `args` string array.
43
59
  `model` selects a single GGUF file; `modelsDirectory` selects llama.cpp router
44
60
  mode. Paths resolve from the application root. Ollama accepts `url`,
@@ -55,6 +71,10 @@ arcane dev --local-ai llama.cpp,ollama,onnx
55
71
 
56
72
  The option adds those runtime requirements to the authored application
57
73
  configuration. Subsequent `arcane dev` runs reuse the saved requirements.
74
+ `--local-ai stable-diffusion.cpp` also selects image runtime preparation; the
75
+ application selects its `models` in the runtime record before requesting a
76
+ load or generation. The development owner composes separate image and
77
+ model-assets services alongside the existing chat/ONNX service.
58
78
  Development checks selected endpoints and existing executable paths, then
59
79
  installs missing runtimes under the application's `.arcane/local-ai/runtimes`.
60
80
  Runtime preparation runs alongside HTTP serving. Browser rendering continues
@@ -89,9 +109,12 @@ processes. An already running external service remains running.
89
109
 
90
110
  `developApplication()` returns its usual server handle and an optional `localAI`
91
111
  handle with `ready`, `current()`, `handler()` and `close()`. Its `ready` promise
92
- means installation and Core composition completed. Runtime/model readiness is
93
- reported separately by Core status and lifecycle events. Preparation failures
94
- remain on that promise and the operation event stream.
112
+ means Core composition has started, not that installation or a selected model
113
+ is ready. Runtime preparation continues at each owning service. `current()`
114
+ includes separate `core`, `localAI` and optional `image` state; image status
115
+ remains callable during preparation. Composition failures reject `ready`;
116
+ later service preparation failures remain on Core lifecycle and diagnostic
117
+ events. Page rendering waits for neither.
95
118
 
96
119
  ## Core service
97
120
 
@@ -112,6 +135,14 @@ application. `current()` returns the current snapshot without starting another
112
135
  process. The service starts each selected runtime independently, retains its
113
136
  process between requests, and joins cleanup during disposal.
114
137
 
138
+ Optional `prepare({signal,onEvent})` returns resolved runtime records before
139
+ this service starts its selected engines. The managed development owner uses
140
+ that callback without delaying the HTTP listener or independent services.
141
+ This factory accepts chat/ONNX requirements; the separate
142
+ [`createLocalImageService()`](local-image-generation.md#core-service-and-transport)
143
+ owns image requirements. The development and generated native composition
144
+ split the shared application record accordingly.
145
+
115
146
  `localai.status` returns `{runtimes}` and, when Ollama is selected, the existing
116
147
  `ollama` and `models.ollama` catalog fields. Runtime records distinguish
117
148
  `installed`, `available`, `state`, `models`, ownership and errors. A listening
@@ -305,11 +336,24 @@ archive in this installer.
305
336
 
306
337
  Ensure returns absolute `{id,version,platform,architecture,root}` records.
307
338
  Server runtimes also include `executable`; ONNX includes `modulePath` pointing
308
- to its public package entry. Bundle returns `{runtimes,files}`,
339
+ to its public package entry. Image records include `libraryPath`,
340
+ `bindingModulePath`, requested `backend`, installed `variants` and selected
341
+ `models` metadata. Variant records carry backend, runtime root and library path.
342
+ Bundle returns `{runtimes,files}`,
309
343
  with artifact-relative runtime paths and a complete emitted file inventory.
310
344
  Bundling requires a fresh native staging destination, preserving existing
311
345
  completed outputs.
312
346
 
347
+ Image assembly preserves the complete runtime/binding tree and relocates
348
+ library, binding and variant paths into the artifact. It retains original
349
+ model URL descriptors for preparation after launch. A model file already
350
+ inside the runtime tree becomes artifact-relative; an external native working
351
+ path reports its portability incompatibility. Use the explicit native-resource
352
+ workflow or [model-assets preparation](model-assets.md) for those files.
353
+ Model acquisition, context loading and inference remain separate from runtime
354
+ assembly. Platform availability depends on the selected upstream archive;
355
+ this contract does not establish execution on every supported host.
356
+
313
357
  ### NeMo Speech library runtime
314
358
 
315
359
  The runtime helpers also accept `{id: 'nemo-speech', version: '0.2.0'}` to
@@ -0,0 +1,154 @@
1
+ # FLUX.2 Klein 4B through Core
2
+
3
+ The `flux2-klein-4b` model definition selects distilled FLUX.2 Klein 4B for
4
+ text-to-image generation through the shared
5
+ [local image runtime](local-image-generation.md). It uses the same retained
6
+ native context, progress, cancellation and complete PNG result contract as
7
+ other selected image models. Applications own prompts, image associations and
8
+ saved results.
9
+
10
+ ## Model selection
11
+
12
+ Select this runtime record under `native.localAI` in `arcane-app.json`, or
13
+ the corresponding package projection's `localAI` record:
14
+
15
+ ```json
16
+ {
17
+ "runtimes": [
18
+ {
19
+ "id": "stable-diffusion.cpp",
20
+ "version": "master-929-3f8527a",
21
+ "backend": "auto",
22
+ "models": ["flux2-klein-4b"]
23
+ }
24
+ ]
25
+ }
26
+ ```
27
+
28
+ The definition is a single `FLUX2_KLEIN_4B_MODEL` record in
29
+ `src/local-ai/image-models/flux.mjs`. The built-in model ID supplies that record
30
+ through normal runtime configuration; the shared service exposes its resource
31
+ metadata through `image.status` and `images.inspect()`.
32
+
33
+ Backend selection belongs to the shared runtime. This definition adds no CPU
34
+ offload setting or device override. `auto` uses the available runtime's
35
+ accelerated backend with CPU fallback; actual support depends on the installed
36
+ runtime distribution. See the shared guide for platform distribution
37
+ availability. Selecting the model does not establish that it has loaded or run
38
+ on a GPU, CPU or NPU.
39
+
40
+ ## Required model resources
41
+
42
+ Prepare all three resources together. They are complementary parts of one
43
+ model pipeline, rather than three independent image generators.
44
+
45
+ | Native role | Resource | Purpose |
46
+ | --- | --- | --- |
47
+ | `diffusion_model` | [flux-2-klein-4b.safetensors](https://huggingface.co/black-forest-labs/FLUX.2-klein-4B/resolve/e7b7dc27f91deacad38e78976d1f2b499d76a294/flux-2-klein-4b.safetensors) | Distilled image generation model. |
48
+ | `vae` | [`full_encoder_small_decoder.safetensors`](https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/resolve/a3efc24f613ef42d9428af62fdbd6f5fd8856c4a/full_encoder_small_decoder.safetensors) | Complete image encoder and the smaller FLUX.2 decoder. |
49
+ | `llm` | [Qwen3-4B-Q8_0.gguf](https://huggingface.co/Qwen/Qwen3-4B-GGUF/resolve/bc640142c66e1fdd12af0bd68f40445458f3869b/Qwen3-4B-Q8_0.gguf) | Qwen3-4B text conditioning inside the image engine. |
50
+
51
+ The selected upstream records identify these resources as Apache-2.0. The
52
+ engine supplies its embedded Qwen tokenizer. This definition needs no separate
53
+ tokenizer download, chat server or LoRA. The full encoder in the selected VAE
54
+ also retains the model's image-encoding capability; a decoder-only file is a
55
+ different resource.
56
+
57
+ The existing DBOPFS model store owns authoritative model files and resumable
58
+ downloads. The shared [model-assets service](model-assets.md) prepares complete
59
+ native working files. Runtime installation does not download another copy of
60
+ these models. Resource roles map to the exact filenames returned by that
61
+ preparation.
62
+
63
+ The following extends the shared generation guide. It uses an existing
64
+ `images` accessor, DBOPFS-backed `modelStore`, application-selected
65
+ `workingDirectory` and request `signal`:
66
+
67
+ ```javascript
68
+ import {createBrowserModelSource} from 'arcane-os/ai/browser-wasm';
69
+ import {prepareCoreModelAssets} from 'arcane-os/ai/core-model-assets';
70
+
71
+ const selected = (await images.inspect({signal})).models.find(
72
+ function selectedKlein(model) {
73
+ return model.id === 'flux2-klein-4b';
74
+ }
75
+ );
76
+ const resources = Object.entries(selected.resources);
77
+ const source = createBrowserModelSource({
78
+ id: selected.id,
79
+ files: resources.map(function modelFile([role, resource]) {
80
+ return {name: resource.filename, url: resource.url};
81
+ })
82
+ });
83
+ const stored = await modelStore.ensure(source, {signal});
84
+ const projection = await prepareCoreModelAssets({
85
+ workingDirectory,
86
+ members: source.files.map(function workingMember(member, index) {
87
+ return {path: member.name, file: stored.files[index]};
88
+ }),
89
+ signal
90
+ });
91
+
92
+ try {
93
+ await images.load({
94
+ model: selected.id,
95
+ assetProjectionId: projection.id,
96
+ resourcePaths: Object.fromEntries(
97
+ resources.map(function nativeRole([role, resource]) {
98
+ return [role, resource.filename];
99
+ })
100
+ ),
101
+ signal
102
+ });
103
+ } finally {
104
+ await projection.release();
105
+ }
106
+
107
+ const result = await images.generate({
108
+ model: selected.id,
109
+ prompt: 'A dignified raccoon astronaut watering a moon cactus in a teacup.',
110
+ parameters: {width: 768, height: 768},
111
+ signal
112
+ });
113
+ // Display result.images using the shared guide's Blob preview example.
114
+ ```
115
+
116
+ Change the prompt to request another scene and the dimensions to select its
117
+ canvas. Successful loading retains the working projection inside the native
118
+ model owner; releasing the caller's preparation handle leaves that native
119
+ retain intact. Repeated generation reuses the loaded context. Explicitly
120
+ unload it when the application no longer needs it, following the shared
121
+ lifecycle contract.
122
+
123
+ ## Defaults and model behavior
124
+
125
+ The definition supplies four sampling steps, the Euler sampler and
126
+ `sample_params.guidance.txt_cfg: 1`. Omitted native options retain their native
127
+ initializer defaults, including automatic FLUX.2 scheduler selection. These
128
+ settings belong to the distilled 4B checkpoint; the separate Klein base and
129
+ 9B variants have their own model requirements.
130
+
131
+ The native default canvas is 512 by 512. Klein's native preparation aligns
132
+ dimensions upward to multiples of 16; use the returned image dimensions when
133
+ displaying or saving results. The example already selects aligned dimensions.
134
+ At the default CFG value of 1, the native text-to-image path does not apply
135
+ negative-prompt conditioning.
136
+
137
+ The SDK forwards the caller's complete prompt to the native API. The engine
138
+ then applies its prompt-weighting syntax and Qwen conditioning format.
139
+ Parentheses and brackets can affect weighting. The selected native API has no
140
+ public switch for literal text conditioning, so exact SDK transport does not
141
+ imply literal interpretation by the model. The shared guide describes inputs
142
+ that cannot pass unchanged through its native C-string boundary.
143
+
144
+ This definition advertises `txt2img`. The upstream Klein model also supports
145
+ reference-image editing, while the SDK's separate whole-image `img2img`
146
+ operation uses a different input path. Reference editing becomes an SDK
147
+ capability when the shared runtime implements its reference-image dispatch;
148
+ selecting this model alone does not add that operation.
149
+
150
+ These settings and resource roles follow the
151
+ [tagged FLUX.2 instructions](https://github.com/leejet/stable-diffusion.cpp/blob/master-929-3f8527a/docs/flux2.md),
152
+ the [official Klein model](https://huggingface.co/black-forest-labs/FLUX.2-klein-4B),
153
+ the [official small decoder](https://huggingface.co/black-forest-labs/FLUX.2-small-decoder)
154
+ and the [official Qwen model files](https://huggingface.co/Qwen/Qwen3-4B-GGUF).
@@ -5,6 +5,12 @@ retained native context. The browser uses Core for model lifecycle and complete
5
5
  PNG results. Applications own image prompts, style choices, character identity,
6
6
  associations and saved results.
7
7
 
8
+ The same public runtime also exposes whole-image `edit()` for an original PNG,
9
+ complete prompt and explicit strength. It reuses this model/context lifecycle;
10
+ applications retain the original and choose whether to save the separate
11
+ result. See [local image editing](local-image-editing.md) for supported inputs,
12
+ lossless transport, native preprocessing and cancellation.
13
+
8
14
  Runtime installation, model asset preparation, model loading and image
9
15
  generation are separate operations. The service starts promptly and retains
10
16
  runtime preparation in the background. `image.status` remains callable during
@@ -35,6 +41,12 @@ model's resource metadata; runtime installation does not independently download
35
41
  or bundle model files. The shared model asset owner prepares selected browser
36
42
  assets and supplies a Core working projection for native path-based loading.
37
43
 
44
+ `flux2-klein-4b` selects the three-resource distilled FLUX.2 Klein definition.
45
+ Its [family guide](local-image-flux.md) covers diffusion/VAE/Qwen preparation,
46
+ four-step defaults and the advertised `txt2img` operation. Keep each model's
47
+ supported operations distinct: selecting FLUX does not enable reference-image
48
+ editing or imply the same input path as SD1.4 img2img.
49
+
38
50
  The first published distribution path selects CPU on Windows and Linux.
39
51
  On macOS, `auto` selects the same upstream universal runtime's built-in Metal
40
52
  backend, with CPU fallback; explicit `cpu` remains available. A caller-supplied