arcane-os 0.61.0 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +1 -1
- package/browser-runtime/ai/core-image.mjs +37 -5
- package/docs/reference/README.md +5 -0
- package/docs/reference/cli.md +5 -1
- package/docs/reference/core-native-packaging.md +22 -2
- package/docs/reference/inventory/package-api.json +106 -8
- package/docs/reference/local-ai.md +48 -7
- package/docs/reference/local-image-editing.md +102 -16
- package/docs/reference/local-image-flux.md +154 -0
- package/docs/reference/local-image-generation.md +6 -0
- package/docs/reference/model-assets.md +4 -4
- package/docs/reference/native-speech.md +114 -0
- package/docs/reference/sdk-api.md +173 -4
- package/package.json +2 -1
- package/src/core/services/image.mjs +42 -0
- package/src/core/services/speech.mjs +360 -0
- package/src/local-ai/config.mjs +4 -2
- package/src/local-ai/image-models/flux.mjs +31 -0
- package/src/local-ai/image-models/sd14.mjs +1 -1
- package/src/local-ai/image-runtime.mjs +23 -1
- package/src/local-ai/image-worker.mjs +14 -0
- package/src/native/windows-provider.mjs +39 -15
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,40 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.63.0
|
|
4
|
+
|
|
5
|
+
- Add the approved FLUX.2 Klein 4B text-to-image model descriptor to the
|
|
6
|
+
existing retained local image runtime, with its separate diffusion, VAE
|
|
7
|
+
and text-encoder resources and four-step Euler defaults. Applications keep
|
|
8
|
+
their model choice, complete prompts and image persistence.
|
|
9
|
+
- Name assembled Windows launchers from the application display name, with a
|
|
10
|
+
matching executable configuration and manifest-selected launch path.
|
|
11
|
+
Preserve supported spaces, case and Unicode, existing generic artifacts,
|
|
12
|
+
icon embedding and the unchanged shared host archive.
|
|
13
|
+
- Add `createSpeechService({stt, tts, signal})` through `arcane-os/core/speech`
|
|
14
|
+
for independently supplied native engines, immediate service registration,
|
|
15
|
+
sticky role state, complete requests/results, cancellation and joined close.
|
|
16
|
+
This service does not install or select a speech engine.
|
|
17
|
+
- Preserve existing image generation/editing and native host behavior. No
|
|
18
|
+
additional GPU setup, SDXL, Whisper engine, Kokoro/Sherpa adoption or browser
|
|
19
|
+
AI native-speech readiness correction is included in this increment.
|
|
20
|
+
|
|
21
|
+
## 0.62.0
|
|
22
|
+
|
|
23
|
+
- Add whole-image PNG editing through the existing local image runtime, Core
|
|
24
|
+
service and browser accessor: `edit({model, image, prompt, strength, ...})`
|
|
25
|
+
and `image.edit` share the retained generation context and lifecycle.
|
|
26
|
+
- Preserve complete original PNG input and prompts through lossless transport,
|
|
27
|
+
copy queued native input, and pass explicit finite strength without clamping.
|
|
28
|
+
Return a separate complete PNG result; applications own persistence.
|
|
29
|
+
- Preserve synchronous Thinking, progress, ordered context work, cancellation,
|
|
30
|
+
unload and close. SD1.4 now advertises `img2img` beside `txt2img`.
|
|
31
|
+
- Support static non-interlaced 8-bit RGB/RGBA PNG input using the existing
|
|
32
|
+
worker decoder. Unsupported formats report their actual limitation; model
|
|
33
|
+
cropping, resizing and alpha behavior remain explicit in the editing guide.
|
|
34
|
+
- Reuse the approved runtime and model workflow with no dependency change.
|
|
35
|
+
This increment does not add masks, reference editing, new models or GPU
|
|
36
|
+
installation, and does not claim exact visual instruction adherence.
|
|
37
|
+
|
|
3
38
|
## 0.61.0
|
|
4
39
|
|
|
5
40
|
- Add retained local image generation through Core, with public native runtime,
|
package/README.md
CHANGED
|
@@ -20,7 +20,7 @@ selected SDK projection or an explicit live SDK source mount. Both profiles
|
|
|
20
20
|
share the theme, packaging, event, cancellation,
|
|
21
21
|
and browser run contracts while retaining their selected application layout.
|
|
22
22
|
|
|
23
|
-
This checkout defines the `0.
|
|
23
|
+
This checkout defines the `0.63.0` SDK contract. Development applications declare
|
|
24
24
|
`arcane-os` as `latest` and retain their lockfile. Exact SDK pins begin when Roshi
|
|
25
25
|
designates a production build. Follow [Consumer updates](docs/consumer-updates.md)
|
|
26
26
|
to refresh and compare the installed SDK and lockfile with the published channel;
|
|
@@ -30,6 +30,24 @@ function decodeImage(image) {
|
|
|
30
30
|
return {...metadata, blob};
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
/** Preserve the complete encoded input at the JSON transport boundary. */
|
|
34
|
+
async function encodeImage(image, signal) {
|
|
35
|
+
if (!(image instanceof Blob)) {
|
|
36
|
+
throw new TypeError('Image editing requires the original PNG Blob or File.');
|
|
37
|
+
}
|
|
38
|
+
const content = new Uint8Array(await image.arrayBuffer());
|
|
39
|
+
signal.throwIfAborted();
|
|
40
|
+
let data;
|
|
41
|
+
if (is.function(content.toBase64)) {
|
|
42
|
+
data = content.toBase64();
|
|
43
|
+
} else {
|
|
44
|
+
let binary = '';
|
|
45
|
+
for (const value of content) binary += String.fromCharCode(value);
|
|
46
|
+
data = globalThis.btoa(binary);
|
|
47
|
+
}
|
|
48
|
+
return {data, encoding: 'base64', mediaType: image.type};
|
|
49
|
+
}
|
|
50
|
+
|
|
33
51
|
/** Browser access to the selected native image service; no result persistence. */
|
|
34
52
|
export function createCoreImageRuntime(
|
|
35
53
|
{client = getInstalledCoreClient(), signal, onEvent} = {}
|
|
@@ -145,7 +163,7 @@ export function createCoreImageRuntime(
|
|
|
145
163
|
method,
|
|
146
164
|
revision,
|
|
147
165
|
streamId: parameters.streamId,
|
|
148
|
-
status: method === 'image.generate' ? 'Thinking' : null,
|
|
166
|
+
status: method === 'image.generate' || method === 'image.edit' ? 'Thinking' : null,
|
|
149
167
|
progress: null,
|
|
150
168
|
task: null
|
|
151
169
|
};
|
|
@@ -176,18 +194,21 @@ export function createCoreImageRuntime(
|
|
|
176
194
|
request.task = Promise.resolve().then(
|
|
177
195
|
async function executeImageRequest() {
|
|
178
196
|
operationSignal.throwIfAborted();
|
|
197
|
+
const requestParameters = method === 'image.edit'
|
|
198
|
+
? {...parameters, image: await encodeImage(parameters.image, operationSignal)} : parameters;
|
|
199
|
+
operationSignal.throwIfAborted();
|
|
179
200
|
observe(
|
|
180
201
|
onEvent,
|
|
181
|
-
{type: 'image.request', data: {method, parameters}}
|
|
202
|
+
{type: 'image.request', data: {method, parameters: requestParameters}}
|
|
182
203
|
);
|
|
183
204
|
operationSignal.throwIfAborted();
|
|
184
205
|
const result = await client.invoke(
|
|
185
206
|
method,
|
|
186
|
-
|
|
207
|
+
requestParameters,
|
|
187
208
|
{signal: operationSignal, timeoutMs: 0}
|
|
188
209
|
);
|
|
189
210
|
operationSignal.throwIfAborted();
|
|
190
|
-
if (method === 'image.generate') {
|
|
211
|
+
if (method === 'image.generate' || method === 'image.edit') {
|
|
191
212
|
const images = result.images.map(decodeImage);
|
|
192
213
|
operationSignal.throwIfAborted();
|
|
193
214
|
const decoded = {...result, images};
|
|
@@ -274,6 +295,17 @@ export function createCoreImageRuntime(
|
|
|
274
295
|
);
|
|
275
296
|
}
|
|
276
297
|
|
|
298
|
+
function edit(
|
|
299
|
+
{model, image, prompt, strength, parameters, assetProjectionId, resourcePaths, signal: requestSignal, onProgress} = {}
|
|
300
|
+
) {
|
|
301
|
+
const streamId = client?.uuid?.() ?? globalThis.crypto.randomUUID();
|
|
302
|
+
return invoke(
|
|
303
|
+
'image.edit',
|
|
304
|
+
{model, image, prompt, strength, parameters, streamId, assetProjectionId, resourcePaths},
|
|
305
|
+
{signal: requestSignal, onProgress}
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
|
|
277
309
|
async function unload({signal: requestSignal} = {}) {
|
|
278
310
|
const before = revision;
|
|
279
311
|
const result = await invoke(
|
|
@@ -356,5 +388,5 @@ export function createCoreImageRuntime(
|
|
|
356
388
|
);
|
|
357
389
|
}
|
|
358
390
|
|
|
359
|
-
return {load, generate, unload, inspect, current, subscribe, close};
|
|
391
|
+
return {load, generate, edit, unload, inspect, current, subscribe, close};
|
|
360
392
|
}
|
package/docs/reference/README.md
CHANGED
|
@@ -55,6 +55,9 @@ alone does not make bare module names resolve in a browser.
|
|
|
55
55
|
| Cancel one Core speech synthesis request without changing its payload | [Speech request lifetime](core-client.md#events-and-request-lifetime) and [Core facade](sdk-api.md#createcorefacade) |
|
|
56
56
|
| Compose native Core dispatch, service lifecycle, and stdio hosting | [Native Core runtime](core-runtime.md) |
|
|
57
57
|
| Run application-selected llama.cpp, Ollama or ONNX through Core | [Local AI through Core](local-ai.md) |
|
|
58
|
+
| Generate complete local PNG images with a retained native model | [Local image generation](local-image-generation.md) |
|
|
59
|
+
| Edit an original PNG with a complete prompt and explicit strength | [Local image editing](local-image-editing.md) |
|
|
60
|
+
| Project complete stored model files into native working files | [Model assets through Core](model-assets.md) |
|
|
58
61
|
| Retain a Nemotron model and diarize caller-fed audio streams | [Native speaker diarization](diarization.md) |
|
|
59
62
|
| Persist application-selected preferences through Core | [Core preferences service](core-preferences.md) |
|
|
60
63
|
| Read complete selected Git text without changing a checkout | [Git text snapshots](git-text-snapshot.md) |
|
|
@@ -223,6 +226,8 @@ Ollama, Wllama, Whisper, Kokoro, native, or cloud transport:
|
|
|
223
226
|
| Run a caller-selected GGUF LLM locally | [`arcane-os/ai/browser-wasm`](ai/browser-wasm.md) | Browser secure context with WebAssembly and OPFS/DBOPFS; explicit CPU selection or WebGPU with full offload and no automatic CPU fallback. |
|
|
224
227
|
| Run application-selected native llama.cpp/Ollama chat | [`arcane-os/ai/core-local`, `arcane-os/core/local-ai`](local-ai.md) | Explicit Core connection and selected native runtimes; installation, availability and loaded-model readiness remain distinct. |
|
|
225
228
|
| Run a local ONNX graph with named tensors | [`arcane-os/local-ai/onnx`, `arcane-os/ai/core-onnx`](local-ai.md#native-onnx-sessions) | Retained native workers; browser access requires an available Core ONNX service. |
|
|
229
|
+
| Generate local images from complete prompts and selected model resources | [`arcane-os/local-ai/image`, `arcane-os/ai/core-image`](local-image-generation.md) | Prepared native image runtime; browser access requires its Core service. CPU Windows/Linux and Mac Metal/CPU paths have distinct actual host requirements. |
|
|
230
|
+
| Supply complete stored model members to a native engine | [`arcane-os/ai/core-model-assets`, `arcane-os/core/model-assets`](model-assets.md) | Available Core service; DBOPFS originals stay authoritative while native owners retain working files. |
|
|
226
231
|
| Run caller-selected Whisper/Kokoro locally | [`arcane-os/ai/browser-speech`](ai/browser-speech.md) | Browser with DBOPFS, Web Locks, Workers and selected upstream sources; automatic WebNN NPU, WebGPU, then WASM loading, or an explicit backend. |
|
|
227
232
|
| Use native browser recognition/voices or selected cloud TTS | [Speech provider choices](ai/browser-speech.md) | Native APIs depend on the browser and may use remote services; cloud TTS requires the application key and endpoint access. |
|
|
228
233
|
| Send complete chat, System One or image requests without retained history | [`arcane-os/ai/twin-cloud`](ai/twin-cloud.md) | Node or browser with Fetch and explicit model/credentials; image results contain every returned image as a Blob. |
|
package/docs/reference/cli.md
CHANGED
|
@@ -49,7 +49,7 @@ meaning and cardinality rules:
|
|
|
49
49
|
| `--app` | app id or label | Workspace/app operations except shared scope and `verify-bundle`; optional diagnostic label for `mail serve`. |
|
|
50
50
|
| `--apps-root` | `.` or `apps` | `new`, `init`; standalone setup defaults to the repository root. |
|
|
51
51
|
| `--sdk-runtime-source` | SDK checkout directory | `dev` only; maps live SDK source through the selected browser destinations for that server process. |
|
|
52
|
-
| `--local-ai` | comma-separated runtime IDs | `dev` only; adds `llama.cpp`, `ollama` and/or `
|
|
52
|
+
| `--local-ai` | comma-separated runtime IDs | `dev` only; adds `llama.cpp`, `ollama`, `onnx` and/or `stable-diffusion.cpp` requirements to application configuration and prepares missing runtimes alongside serving. See [local AI through Core](local-ai.md). |
|
|
53
53
|
| `--arcane-root` | directory | `doctor`, native `build`/`run`, `native-doctor`, `native-prepare` |
|
|
54
54
|
| `--host` / `--port` | host / integer 0–65535 | Browser `dev`/`run` default to HTTPS at `127.0.0.1:8000`; `mail serve` defaults to HTTPS with HTTP/2 at `0.0.0.0:4433` and accepts an explicit bind host. |
|
|
55
55
|
| `--http-port` | integer 0–65535 | Browser `dev`/`run` HTTP redirect listener; defaults to `0`, which selects an available port. |
|
|
@@ -375,6 +375,10 @@ arcane dev [--app <id>] [--public] [--http | --https] [--cert <file> --key <file
|
|
|
375
375
|
reuse them. Available services and executables are reused; missing runtimes
|
|
376
376
|
are installed inside the application while serving starts independently.
|
|
377
377
|
Applications choose their model files and optional browser Core connection.
|
|
378
|
+
`--local-ai stable-diffusion.cpp` prepares the selected native image runtime;
|
|
379
|
+
configure its `models` separately in the application's runtime record.
|
|
380
|
+
The [image service](local-image-generation.md) reports installation and model
|
|
381
|
+
readiness independently, with complete PNG results and explicit cancellation.
|
|
378
382
|
See [local AI through Core](local-ai.md) for configuration and native bundling.
|
|
379
383
|
|
|
380
384
|
### Lifecycle
|
|
@@ -248,8 +248,9 @@ to the artifact root. Only the explicitly configured entry imports
|
|
|
248
248
|
`createLocalAIService` from `arcane-os/core/local-ai`. It resolves those bundled
|
|
249
249
|
locations against the installed artifact and supplies them in the launch
|
|
250
250
|
context's `runtimes` array, alongside `appRoot`, before applying explicit launch
|
|
251
|
-
configuration. It passes
|
|
252
|
-
|
|
251
|
+
configuration. It passes chat/ONNX requirements to that factory and routes
|
|
252
|
+
selected image requirements to their separate service as described below.
|
|
253
|
+
Applications without that selection have no generated local-AI import
|
|
253
254
|
or service. Availability of a particular engine belongs to the selected
|
|
254
255
|
local-AI installer and service; a generic ONNX runtime is not a speech model.
|
|
255
256
|
|
|
@@ -260,6 +261,25 @@ service exposes retained worker sessions through `onnx.load/run/unload/status`;
|
|
|
260
261
|
model files remain caller-selected application resources. See
|
|
261
262
|
[native ONNX sessions](local-ai.md#native-onnx-sessions).
|
|
262
263
|
|
|
264
|
+
Selected `stable-diffusion.cpp` additionally composes
|
|
265
|
+
`createLocalImageService` from `arcane-os/core/image` and one shared
|
|
266
|
+
`createModelAssetService` from `arcane-os/core/model-assets`. The image factory
|
|
267
|
+
receives the full authored selection plus that same model-assets instance.
|
|
268
|
+
Library, Koffi binding and variant paths resolve from the artifact root;
|
|
269
|
+
ordinary browser imports use `arcane-os/ai/core-image` and
|
|
270
|
+
`arcane-os/ai/core-model-assets` through the managed import map.
|
|
271
|
+
|
|
272
|
+
The runtime bundle preserves the complete selected native distribution and
|
|
273
|
+
binding tree. It does not load or download a model. Original model URL
|
|
274
|
+
descriptors remain available for preparation after launch; runtime-contained
|
|
275
|
+
model paths relocate with the artifact. An external native working path
|
|
276
|
+
reports its portability incompatibility. Supply those files through the
|
|
277
|
+
application's native resources or the shared
|
|
278
|
+
[stored-model projection](model-assets.md). Native use retains that projection
|
|
279
|
+
until actual unload or shutdown completes. See
|
|
280
|
+
[local image generation](local-image-generation.md) for selection, complete
|
|
281
|
+
results and CPU/Metal platform boundaries.
|
|
282
|
+
|
|
263
283
|
## Windows executable
|
|
264
284
|
|
|
265
285
|
The ordinary application command is:
|
|
@@ -1,17 +1,115 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": 1,
|
|
3
|
-
"sdkVersion": "0.
|
|
3
|
+
"sdkVersion": "0.62.0",
|
|
4
4
|
"environment": {
|
|
5
5
|
"runtime": "Node.js for Node entrypoints; browser for browser-only entrypoints",
|
|
6
6
|
"minimumVersion": "22.23.2 for Node entrypoints",
|
|
7
7
|
"moduleSystem": "ESM"
|
|
8
8
|
},
|
|
9
|
-
"memberCount":
|
|
9
|
+
"memberCount": 274,
|
|
10
10
|
"runtimeSubpathPatterns": {
|
|
11
11
|
"arcane-os/modules/*": "./runtime/arcane/modules/*",
|
|
12
12
|
"arcane-os/entities/*": "./runtime/arcane/entities/*"
|
|
13
13
|
},
|
|
14
14
|
"members": [
|
|
15
|
+
{
|
|
16
|
+
"id": "local-ai-image:createImageRuntime",
|
|
17
|
+
"name": "createImageRuntime",
|
|
18
|
+
"displayName": "createImageRuntime()",
|
|
19
|
+
"kind": "function",
|
|
20
|
+
"signature": "createImageRuntime({libraryPath,bindingModulePath,variants=[],models=[],backend='auto',signal,onEvent}={})",
|
|
21
|
+
"entrypoints": ["arcane-os/local-ai/image"],
|
|
22
|
+
"primaryImport": "arcane-os/local-ai/image",
|
|
23
|
+
"group": "Core local images",
|
|
24
|
+
"summary": "Retains selected stable-diffusion.cpp generation and PNG img2img execution in a shared native worker and returns complete PNG results.",
|
|
25
|
+
"availability": "Node with prepared upstream image library, Koffi binding and caller-selected model resources",
|
|
26
|
+
"protocol": "Shared per-library worker/context lifetime with ordered native operations and replaying state",
|
|
27
|
+
"normalization": "Returns load/generate/edit/unload/current/subscribe/close; edit copies complete PNG input at acceptance and takes explicit strength; prompts remain complete and cancellation/close join native completion before resource release"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"id": "core-image:createCoreImageRuntime",
|
|
31
|
+
"name": "createCoreImageRuntime",
|
|
32
|
+
"displayName": "createCoreImageRuntime()",
|
|
33
|
+
"kind": "function",
|
|
34
|
+
"signature": "createCoreImageRuntime({client=getInstalledCoreClient(),signal,onEvent}={})",
|
|
35
|
+
"entrypoints": ["arcane-os/ai/core-image"],
|
|
36
|
+
"primaryImport": "arcane-os/ai/core-image",
|
|
37
|
+
"group": "Core local images",
|
|
38
|
+
"summary": "Uses an available Core image service for generation and original-PNG editing with synchronous Thinking acknowledgement, lifecycle replay and complete PNG Blobs.",
|
|
39
|
+
"availability": "Browser/native WebView with an available Core image service; construction opens no connection or model",
|
|
40
|
+
"protocol": "arcane/1 image.status/load/generate/edit/unload and image.state/progress",
|
|
41
|
+
"normalization": "Returns load/generate/edit/unload/inspect/current/subscribe/close; edit preserves the complete original PNG Blob/File through transport with explicit strength; timeoutMs is zero, close cancels only this accessor and explicit unload owns the shared model"
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"id": "core-image-service:createLocalImageService",
|
|
45
|
+
"name": "createLocalImageService",
|
|
46
|
+
"displayName": "createLocalImageService()",
|
|
47
|
+
"kind": "function",
|
|
48
|
+
"signature": "createLocalImageService(configuration,{appRoot,runtimes=[],signal,onEvent,prepare,modelAssets}={})",
|
|
49
|
+
"entrypoints": ["arcane-os/core/image"],
|
|
50
|
+
"primaryImport": "arcane-os/core/image",
|
|
51
|
+
"group": "Core local images",
|
|
52
|
+
"summary": "Composes independent image-runtime preparation, retained native model use and complete PNG transport as a Core service.",
|
|
53
|
+
"availability": "Node Core with selected stable-diffusion.cpp configuration and prepared runtime or preparation callback",
|
|
54
|
+
"protocol": "arcane/1 image lifecycle, generate/edit and progress; base64 framing only at complete PNG input/result transport boundaries",
|
|
55
|
+
"normalization": "Status remains callable during preparation; shared model-assets handles remain retained through actual model unload, worker completion and disposal"
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"id": "core-image-service:default",
|
|
59
|
+
"name": "default",
|
|
60
|
+
"displayName": "createLocalImageService default export",
|
|
61
|
+
"kind": "function",
|
|
62
|
+
"signature": "default as createLocalImageService",
|
|
63
|
+
"entrypoints": ["arcane-os/core/image"],
|
|
64
|
+
"primaryImport": "arcane-os/core/image",
|
|
65
|
+
"group": "Core local images",
|
|
66
|
+
"summary": "Default binding for the same native image service factory.",
|
|
67
|
+
"availability": "Node Core runtime",
|
|
68
|
+
"protocol": "arcane/1 service definition",
|
|
69
|
+
"normalization": "Binding identity equals the named createLocalImageService export"
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"id": "core-model-assets:prepareCoreModelAssets",
|
|
73
|
+
"name": "prepareCoreModelAssets",
|
|
74
|
+
"displayName": "prepareCoreModelAssets()",
|
|
75
|
+
"kind": "function",
|
|
76
|
+
"signature": "async prepareCoreModelAssets({client=getInstalledCoreClient(),workingDirectory,members,signal,onProgress}={})",
|
|
77
|
+
"entrypoints": ["arcane-os/ai/core-model-assets"],
|
|
78
|
+
"primaryImport": "arcane-os/ai/core-model-assets",
|
|
79
|
+
"group": "Core model assets",
|
|
80
|
+
"summary": "Projects complete caller-owned Blob/File members through Core into native working files without changing their durable store.",
|
|
81
|
+
"availability": "Browser with an available model-assets Core service and complete selected members",
|
|
82
|
+
"protocol": "arcane/1 modelAssets open/write/complete/release operations with transport-local base64",
|
|
83
|
+
"normalization": "Returns id/directory/ordered members/release; independent files transfer concurrently, each file writes in order, and cancellation joins cleanup"
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"id": "core-model-assets-service:createModelAssetService",
|
|
87
|
+
"name": "createModelAssetService",
|
|
88
|
+
"displayName": "createModelAssetService()",
|
|
89
|
+
"kind": "function",
|
|
90
|
+
"signature": "createModelAssetService({appRoot=process.cwd()}={})",
|
|
91
|
+
"entrypoints": ["arcane-os/core/model-assets"],
|
|
92
|
+
"primaryImport": "arcane-os/core/model-assets",
|
|
93
|
+
"group": "Core model assets",
|
|
94
|
+
"summary": "Owns selected native working-file projections and engine retain handles while original stored model files remain unchanged.",
|
|
95
|
+
"availability": "Node Core with an application-selected working directory",
|
|
96
|
+
"protocol": "arcane/1 modelAssets.open/write/complete/release/status and modelAssets.state",
|
|
97
|
+
"normalization": "Returns current/retain/dispose and Core service lifecycle; final native release joins writes and closes before deleting only its working projection"
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": "core-model-assets-service:default",
|
|
101
|
+
"name": "default",
|
|
102
|
+
"displayName": "createModelAssetService default export",
|
|
103
|
+
"kind": "function",
|
|
104
|
+
"signature": "default as createModelAssetService",
|
|
105
|
+
"entrypoints": ["arcane-os/core/model-assets"],
|
|
106
|
+
"primaryImport": "arcane-os/core/model-assets",
|
|
107
|
+
"group": "Core model assets",
|
|
108
|
+
"summary": "Default binding for the same native working-file service factory.",
|
|
109
|
+
"availability": "Node Core runtime",
|
|
110
|
+
"protocol": "arcane/1 service definition",
|
|
111
|
+
"normalization": "Binding identity equals the named createModelAssetService export"
|
|
112
|
+
},
|
|
15
113
|
{
|
|
16
114
|
"id": "local-ai-onnx:createONNXRuntime",
|
|
17
115
|
"name": "createONNXRuntime",
|
|
@@ -129,14 +227,14 @@
|
|
|
129
227
|
"name": "createLocalAIService",
|
|
130
228
|
"displayName": "createLocalAIService()",
|
|
131
229
|
"kind": "function",
|
|
132
|
-
"signature": "createLocalAIService(configuration,{appRoot,runtimes=[],signal,onEvent}={})",
|
|
230
|
+
"signature": "createLocalAIService(configuration,{appRoot,runtimes=[],signal,onEvent,prepare}={})",
|
|
133
231
|
"entrypoints": ["arcane-os/core/local-ai"],
|
|
134
232
|
"primaryImport": "arcane-os/core/local-ai",
|
|
135
233
|
"group": "Core local AI",
|
|
136
234
|
"summary": "Composes selected native llama.cpp, Ollama and ONNX engines as a Core service with retained process/worker ownership and model lifecycle.",
|
|
137
235
|
"availability": "Node with selected upstream runtime distributions or available configured system services",
|
|
138
236
|
"protocol": "arcane/1 methods and events; llama.cpp OpenAI HTTP/SSE, Ollama native HTTP/NDJSON and ONNX tensor transport",
|
|
139
|
-
"normalization": "Application-owned runtime and model requirements; independent
|
|
237
|
+
"normalization": "Application-owned runtime and model requirements; optional prepare supplies installed records during this service startup; independent services, complete responses and exact owned-process cleanup"
|
|
140
238
|
},
|
|
141
239
|
{
|
|
142
240
|
"id": "core-local-ai:default",
|
|
@@ -161,10 +259,10 @@
|
|
|
161
259
|
"entrypoints": ["arcane-os/local-ai"],
|
|
162
260
|
"primaryImport": "arcane-os/local-ai",
|
|
163
261
|
"group": "Core local AI",
|
|
164
|
-
"summary": "Reuses selected managed native runtimes or installs complete official llama.cpp/Ollama distributions, ONNX npm runtime trees or NeMo libraries.",
|
|
262
|
+
"summary": "Reuses selected managed native runtimes or installs complete official llama.cpp/Ollama/image distributions, ONNX npm runtime trees or NeMo libraries.",
|
|
165
263
|
"availability": "Node; upstream platform and architecture assets determine runtime availability",
|
|
166
|
-
"protocol": "Official upstream release archives
|
|
167
|
-
"normalization": "Returns absolute runtime records; shares matching
|
|
264
|
+
"protocol": "Official upstream release archives and selected ONNX/Koffi npm runtime installations",
|
|
265
|
+
"normalization": "Returns absolute runtime records, including image library/binding/variant/model metadata; shares matching installs, preserves libraries, reports failures without changing global PATH or services and prepares no model files"
|
|
168
266
|
},
|
|
169
267
|
{
|
|
170
268
|
"id": "local-ai:bundleLocalAIRuntimes",
|
|
@@ -178,7 +276,7 @@
|
|
|
178
276
|
"summary": "Copies selected complete runtime distributions into a fresh native artifact staging root.",
|
|
179
277
|
"availability": "Node; supports bundling published upstream platform assets independently of the build host",
|
|
180
278
|
"protocol": "Artifact-relative runtime/local-ai layout",
|
|
181
|
-
"normalization": "Returns {runtimes,files} with relative runtime
|
|
279
|
+
"normalization": "Returns {runtimes,files} with artifact-relative runtime/variant paths and complete inventory; image URL descriptors remain for post-launch preparation and external model working paths report portability incompatibility"
|
|
182
280
|
},
|
|
183
281
|
{
|
|
184
282
|
"id": "root:WS",
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# Local AI through Core
|
|
2
2
|
|
|
3
|
-
Core owns native llama.cpp and Ollama processes
|
|
3
|
+
Core owns native llama.cpp and Ollama processes, ONNX worker sessions and
|
|
4
|
+
selected local image generation. The browser retains its own
|
|
4
5
|
Wllama and ONNX implementations, and can also use an explicitly connected Core
|
|
5
6
|
when that system service is available. Runtime installation, model loading and
|
|
6
7
|
inference are separate operations.
|
|
@@ -20,7 +21,8 @@ Existing native fields remain alongside this record:
|
|
|
20
21
|
"runtimes": [
|
|
21
22
|
{"id": "llama.cpp"},
|
|
22
23
|
{"id": "ollama"},
|
|
23
|
-
{"id": "onnx"}
|
|
24
|
+
{"id": "onnx"},
|
|
25
|
+
{"id": "stable-diffusion.cpp", "backend": "auto", "models": ["sd14"]}
|
|
24
26
|
],
|
|
25
27
|
"llamaCpp": {
|
|
26
28
|
"modelsDirectory": "models/llama"
|
|
@@ -34,11 +36,22 @@ Existing native fields remain alongside this record:
|
|
|
34
36
|
Each runtime requirement accepts `id`, optional upstream `version`, and an
|
|
35
37
|
optional runtime archive `url`. ONNX resolves its official npm package or an
|
|
36
38
|
explicitly supplied package archive. A string runtime ID is also accepted.
|
|
37
|
-
|
|
39
|
+
For chat runtimes, omitting the version allows reuse of an available runtime. Explicit versions
|
|
38
40
|
select that upstream distribution; an existing service must match the requested
|
|
39
41
|
version before it is reused. A service occupying the selected address is
|
|
40
42
|
preserved when its version cannot be established.
|
|
41
43
|
|
|
44
|
+
Both public application/package schemas accept `stable-diffusion.cpp` as a
|
|
45
|
+
string or runtime-record ID. Its record additionally accepts `backend`
|
|
46
|
+
(`auto`, `cpu` or `metal`) and `models`, an array of `sd14` or complete model
|
|
47
|
+
descriptors. Omitted image version selects `master-929-3f8527a`; omitted backend
|
|
48
|
+
selects `auto`; omitted models leaves model selection empty. Windows/Linux
|
|
49
|
+
distribution defaults use CPU; the selected universal Mac archive supports
|
|
50
|
+
Metal with CPU fallback. Installation obtains the upstream library and its
|
|
51
|
+
managed `koffi@3.3.2` binding, without installing a model. Complete model
|
|
52
|
+
descriptors keep their resources, context, defaults and operations separate.
|
|
53
|
+
See [image selection and model preparation](local-image-generation.md#selection-and-model-assets).
|
|
54
|
+
|
|
42
55
|
`llamaCpp` accepts `url`, `model`, `modelsDirectory`, and an `args` string array.
|
|
43
56
|
`model` selects a single GGUF file; `modelsDirectory` selects llama.cpp router
|
|
44
57
|
mode. Paths resolve from the application root. Ollama accepts `url`,
|
|
@@ -55,6 +68,10 @@ arcane dev --local-ai llama.cpp,ollama,onnx
|
|
|
55
68
|
|
|
56
69
|
The option adds those runtime requirements to the authored application
|
|
57
70
|
configuration. Subsequent `arcane dev` runs reuse the saved requirements.
|
|
71
|
+
`--local-ai stable-diffusion.cpp` also selects image runtime preparation; the
|
|
72
|
+
application selects its `models` in the runtime record before requesting a
|
|
73
|
+
load or generation. The development owner composes separate image and
|
|
74
|
+
model-assets services alongside the existing chat/ONNX service.
|
|
58
75
|
Development checks selected endpoints and existing executable paths, then
|
|
59
76
|
installs missing runtimes under the application's `.arcane/local-ai/runtimes`.
|
|
60
77
|
Runtime preparation runs alongside HTTP serving. Browser rendering continues
|
|
@@ -89,9 +106,12 @@ processes. An already running external service remains running.
|
|
|
89
106
|
|
|
90
107
|
`developApplication()` returns its usual server handle and an optional `localAI`
|
|
91
108
|
handle with `ready`, `current()`, `handler()` and `close()`. Its `ready` promise
|
|
92
|
-
means
|
|
93
|
-
|
|
94
|
-
|
|
109
|
+
means Core composition has started, not that installation or a selected model
|
|
110
|
+
is ready. Runtime preparation continues at each owning service. `current()`
|
|
111
|
+
includes separate `core`, `localAI` and optional `image` state; image status
|
|
112
|
+
remains callable during preparation. Composition failures reject `ready`;
|
|
113
|
+
later service preparation failures remain on Core lifecycle and diagnostic
|
|
114
|
+
events. Page rendering waits for neither.
|
|
95
115
|
|
|
96
116
|
## Core service
|
|
97
117
|
|
|
@@ -112,6 +132,14 @@ application. `current()` returns the current snapshot without starting another
|
|
|
112
132
|
process. The service starts each selected runtime independently, retains its
|
|
113
133
|
process between requests, and joins cleanup during disposal.
|
|
114
134
|
|
|
135
|
+
Optional `prepare({signal,onEvent})` returns resolved runtime records before
|
|
136
|
+
this service starts its selected engines. The managed development owner uses
|
|
137
|
+
that callback without delaying the HTTP listener or independent services.
|
|
138
|
+
This factory accepts chat/ONNX requirements; the separate
|
|
139
|
+
[`createLocalImageService()`](local-image-generation.md#core-service-and-transport)
|
|
140
|
+
owns image requirements. The development and generated native composition
|
|
141
|
+
split the shared application record accordingly.
|
|
142
|
+
|
|
115
143
|
`localai.status` returns `{runtimes}` and, when Ollama is selected, the existing
|
|
116
144
|
`ollama` and `models.ollama` catalog fields. Runtime records distinguish
|
|
117
145
|
`installed`, `available`, `state`, `models`, ownership and errors. A listening
|
|
@@ -305,11 +333,24 @@ archive in this installer.
|
|
|
305
333
|
|
|
306
334
|
Ensure returns absolute `{id,version,platform,architecture,root}` records.
|
|
307
335
|
Server runtimes also include `executable`; ONNX includes `modulePath` pointing
|
|
308
|
-
to its public package entry.
|
|
336
|
+
to its public package entry. Image records include `libraryPath`,
|
|
337
|
+
`bindingModulePath`, requested `backend`, installed `variants` and selected
|
|
338
|
+
`models` metadata. Variant records carry backend, runtime root and library path.
|
|
339
|
+
Bundle returns `{runtimes,files}`,
|
|
309
340
|
with artifact-relative runtime paths and a complete emitted file inventory.
|
|
310
341
|
Bundling requires a fresh native staging destination, preserving existing
|
|
311
342
|
completed outputs.
|
|
312
343
|
|
|
344
|
+
Image assembly preserves the complete runtime/binding tree and relocates
|
|
345
|
+
library, binding and variant paths into the artifact. It retains original
|
|
346
|
+
model URL descriptors for preparation after launch. A model file already
|
|
347
|
+
inside the runtime tree becomes artifact-relative; an external native working
|
|
348
|
+
path reports its portability incompatibility. Use the explicit native-resource
|
|
349
|
+
workflow or [model-assets preparation](model-assets.md) for those files.
|
|
350
|
+
Model acquisition, context loading and inference remain separate from runtime
|
|
351
|
+
assembly. Platform availability depends on the selected upstream archive;
|
|
352
|
+
this contract does not establish execution on every supported host.
|
|
353
|
+
|
|
313
354
|
### NeMo Speech library runtime
|
|
314
355
|
|
|
315
356
|
The runtime helpers also accept `{id: 'nemo-speech', version: '0.2.0'}` to
|
|
@@ -1,10 +1,97 @@
|
|
|
1
|
-
# Local image
|
|
1
|
+
# Local image editing through Core
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
`createCoreImageRuntime()` exposes whole-image img2img through `edit()`, using
|
|
4
|
+
the same retained native context as generation. The selected SD1.4 model
|
|
5
|
+
advertises `txt2img` and `img2img`. Applications supply the original PNG, the
|
|
6
|
+
complete prompt and explicit strength, retain the original, and decide whether
|
|
7
|
+
to save the separate generated result.
|
|
8
|
+
|
|
9
|
+
## Browser accessor
|
|
10
|
+
|
|
11
|
+
Prepare and load the selected model using the
|
|
12
|
+
[shared generation and model-asset lifecycle](local-image-generation.md).
|
|
13
|
+
Then pass a generated PNG Blob or a supported PNG File directly:
|
|
14
|
+
|
|
15
|
+
```js
|
|
16
|
+
import {createCoreImageRuntime} from 'arcane-os/ai/core-image';
|
|
17
|
+
|
|
18
|
+
const images = createCoreImageRuntime({signal: page.signal});
|
|
19
|
+
images.subscribe(function showImageStatus(snapshot) {
|
|
20
|
+
statusElement.textContent = snapshot.status ?? snapshot.state;
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
const result = await images.edit({
|
|
24
|
+
model: 'sd14',
|
|
25
|
+
image: originalPNG,
|
|
26
|
+
prompt: 'A dignified raccoon astronaut wearing a purple knitted helmet, holding a tiny moon cactus.',
|
|
27
|
+
strength: 0.6,
|
|
28
|
+
parameters: {
|
|
29
|
+
width: 512,
|
|
30
|
+
height: 512,
|
|
31
|
+
sample_params: {sample_steps: 20}
|
|
32
|
+
},
|
|
33
|
+
signal: page.signal,
|
|
34
|
+
onProgress(progress) {
|
|
35
|
+
console.log(progress);
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const editedPNG = result.images[0].blob;
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The example uses the application's existing `page` AbortController,
|
|
43
|
+
`statusElement` and original `originalPNG` Blob or File. It assumes that the
|
|
44
|
+
selected model has already been loaded. `edit()` can also prepare a selected
|
|
45
|
+
model on demand using the same `assetProjectionId` and `resourcePaths` fields
|
|
46
|
+
as `generate()`.
|
|
47
|
+
|
|
48
|
+
| Member or field | Contract |
|
|
49
|
+
| --- | --- |
|
|
50
|
+
| `edit({model, image, prompt, strength, parameters?, assetProjectionId?, resourcePaths?, signal?, onProgress?})` | Returns the complete result with `images: [{blob, mediaType, width, height}]`, preserving additional runtime result metadata. |
|
|
51
|
+
| `image` | Original complete PNG `Blob` or `File`, with `image/png` or an empty declared media type. Supported PNG encodings are listed below. |
|
|
52
|
+
| `prompt` | Complete caller-authored string, passed unchanged. Native C-string incompatibilities retain the generation error contract. |
|
|
53
|
+
| `strength` | Explicit finite number from `0` through `1`, mapped to native img2img strength. It takes precedence over a model default or `parameters.strength`; it is never clamped. |
|
|
54
|
+
| `parameters` | Native generation options, including canvas dimensions, seed, sampling and `image_preprocess`. The top-level image supplies native `init_image`; the top-level strength supplies native `strength`. Other options retain the generation contract. |
|
|
55
|
+
|
|
56
|
+
`Thinking` is published synchronously when the browser accepts the edit, before
|
|
57
|
+
reading the Blob, sending the request, preparing the runtime or loading the
|
|
58
|
+
model. Native sampling progress uses the existing progress surface. Complete
|
|
59
|
+
PNG results arrive after native completion; the SDK fabricates no image chunks
|
|
60
|
+
or assistant turns. Closing the accessor cancels and joins its requests while
|
|
61
|
+
preserving the shared Core client and model ownership.
|
|
62
|
+
|
|
63
|
+
The original encoded input is preserved through lossless Core JSON framing.
|
|
64
|
+
The browser does not decode, draw, re-encode, resize, crop, composite or save it.
|
|
65
|
+
The request signal reaches input preparation, worker decoding and cooperative
|
|
66
|
+
native cancellation. Cancellation during `Blob.arrayBuffer()` is observed
|
|
67
|
+
after that platform operation completes. An aborted edit produces no late
|
|
68
|
+
success or progress callback. Request invocation retains `timeoutMs: 0`.
|
|
69
|
+
|
|
70
|
+
## Core and native request
|
|
71
|
+
|
|
72
|
+
The `arcane-os/core/image` service adds `image.edit` with
|
|
73
|
+
`{model, image, prompt, strength, parameters?, streamId?, assetProjectionId?, resourcePaths?}`.
|
|
74
|
+
At this JSON boundary only, `image` is
|
|
75
|
+
`{data: completeBase64PNG, encoding: 'base64', mediaType}`. The service emits
|
|
76
|
+
the existing accepted `image.progress` event before preparation, then forwards
|
|
77
|
+
the decoded transport content to the shared image worker. Results use the
|
|
78
|
+
same complete PNG transport records as `image.generate`.
|
|
79
|
+
|
|
80
|
+
The native runtime's `edit()` takes the same operation fields, with
|
|
81
|
+
`image: {data: Uint8Array, mediaType}` and no Core projection arguments.
|
|
82
|
+
It copies the supplied data view at acceptance so caller reuse cannot change
|
|
83
|
+
an input waiting behind another operation.
|
|
84
|
+
The worker decodes once, loads or reuses the exact selected context, and passes
|
|
85
|
+
the raster as native `init_image` with explicit `strength`. The existing native
|
|
86
|
+
binding retains input memory through completion and copies complete output
|
|
87
|
+
rasters before native cleanup. The same queue, cancellation, unload and close
|
|
88
|
+
owners serve generation and editing; no second engine or model store is created.
|
|
89
|
+
|
|
90
|
+
Unreadable or unsupported input rejects the edit without substituting an
|
|
91
|
+
image. Errors retain their complete details on rejected promises and diagnostic
|
|
92
|
+
events. Applications keep those engineering diagnostics outside ordinary status
|
|
93
|
+
surfaces and durable chat history. No prompt, input or result is persisted by
|
|
94
|
+
this operation.
|
|
8
95
|
|
|
9
96
|
## Decoder contract
|
|
10
97
|
|
|
@@ -56,22 +143,21 @@ failure never returns partial imagery or substitutes a different image.
|
|
|
56
143
|
|
|
57
144
|
## Editing semantics at the native owner
|
|
58
145
|
|
|
59
|
-
Whole-image img2img uses an initial image, prompt and strength.
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
with its base SD1.4 model in the
|
|
146
|
+
Whole-image img2img uses an initial image, prompt and strength. This `edit()`
|
|
147
|
+
contract selects that operation. Masked and reference editing require their
|
|
148
|
+
own model-specific contracts and are outside this PNG img2img operation.
|
|
149
|
+
Ordinary img2img does not require LoRA training. The selected engine documents
|
|
150
|
+
this directly with its base SD1.4 model in the
|
|
65
151
|
[tagged img2img example](https://github.com/leejet/stable-diffusion.cpp/blob/master-929-3f8527a/docs/sd.md#img2img-example).
|
|
66
152
|
|
|
67
153
|
Source dimensions are decoder output, not an implicit request to change the
|
|
68
|
-
generation dimensions. The
|
|
69
|
-
and reports actual output dimensions.
|
|
70
|
-
|
|
154
|
+
generation dimensions. The engine center-crops and resizes temporary initial
|
|
155
|
+
images to the selected canvas by default and reports actual output dimensions.
|
|
156
|
+
Caller images remain unchanged;
|
|
71
157
|
even `mode=none` does not suppress all later model adaptation. See the
|
|
72
158
|
[native preprocessing contract](https://github.com/leejet/stable-diffusion.cpp/blob/master-929-3f8527a/docs/image_preprocessing.md).
|
|
73
159
|
|
|
74
|
-
An RGB-only model such as the selected SD1.4
|
|
160
|
+
An RGB-only model such as the selected SD1.4 consumes the stored RGB
|
|
75
161
|
and does not preserve alpha. The SDK must not silently choose a compositing
|
|
76
162
|
background or claim that passing RGBA preserves its displayed appearance.
|
|
77
163
|
Strength zero is also not an exact-copy operation: native img2img still uses
|