arcane-os 0.60.0 → 0.62.0

package/src/local-ai/image-models/sd14.mjs

@@ -0,0 +1,15 @@
+/** Official complete checkpoint; acquisition belongs to the model asset owner. */
+export const SD14_MODEL = {
+    id: 'sd14',
+    name: 'Stable Diffusion 1.4',
+    family: 'sd',
+    resources: {
+        model: {
+            url: 'https://huggingface.co/CompVis/stable-diffusion-v-1-4-original/resolve/c96766dd476698a0ece589565635fafc88ebecae/sd-v1-4.ckpt',
+            filename: 'sd-v1-4.ckpt'
+        }
+    },
+    context: {},
+    defaults: {},
+    operations: ['txt2img', 'img2img']
+};

package/src/local-ai/image-png.mjs

@@ -0,0 +1,74 @@
+import {Buffer} from 'node:buffer';
+import {crc32, deflate} from 'node:zlib';
+
+/** Encode complete native RGB/RGBA pixels in the owning image worker. */
+export async function encodePNG(
+    {width, height, channel, data, signal}
+) {
+    signal?.throwIfAborted();
+    if (channel !== 3 && channel !== 4) {
+        throw new TypeError('PNG image output requires RGB or RGBA pixels.');
+    }
+
+    const pixels = Buffer.from(data.buffer, data.byteOffset, data.byteLength);
+    const rowLength = width * channel;
+    // PNG scanlines must represent every supplied pixel exactly once.
+    if (pixels.length !== rowLength * height) {
+        throw new TypeError('The pixel data does not match the image dimensions.');
+    }
+    const scanlines = Buffer.alloc((rowLength + 1) * height);
+    for (let row = 0; row < height; row++) {
+        signal?.throwIfAborted();
+        const start = row * rowLength;
+        pixels.copy(scanlines, row * (rowLength + 1) + 1, start, start + rowLength);
+    }
+
+    // The zero filter prefix preserves each channel; zlib runs asynchronously.
+    const compressed = await new Promise(
+        function compressImage(resolve, reject) {
+            deflate(
+                scanlines,
+                function imageCompressed(error, result) {
+                    if (error) {
+                        reject(error);
+                    } else {
+                        resolve(result);
+                    }
+                }
+            );
+        }
+    );
+    signal?.throwIfAborted();
+
+    const header = Buffer.alloc(13);
+    header.writeUInt32BE(width, 0);
+    header.writeUInt32BE(height, 4);
+    header[8] = 8;
+    header[9] = channel === 3 ? 2 : 6;
+    const chunks = [
+        Buffer.from(
+            [137, 80, 78, 71, 13, 10, 26, 10]
+        ),
+        pngChunk('IHDR', header)
+    ];
+
+    // PNG limits each chunk's framing field; continuation retains all data.
+    for (let offset = 0; offset < compressed.length; offset += 0x7fffffff) {
+        signal?.throwIfAborted();
+        chunks.push(pngChunk('IDAT', compressed.subarray(offset, offset + 0x7fffffff)));
+    }
+    chunks.push(pngChunk('IEND', Buffer.alloc(0)));
+    return Buffer.concat(chunks);
+}
+
+function pngChunk(name, data) {
+    const type = Buffer.from(name, 'ascii');
+    const length = Buffer.alloc(4);
+    length.writeUInt32BE(data.length);
+    // CRC is required PNG framing, confined to this image encoder.
+    const crc = Buffer.alloc(4);
+    crc.writeUInt32BE(crc32(data, crc32(type)));
+    return Buffer.concat(
+        [length, type, data, crc]
+    );
+}

package/src/local-ai/image-runtime.mjs

@@ -0,0 +1,358 @@
+import {Worker} from 'node:worker_threads';
+import {isDeepStrictEqual} from 'node:util';
+import Is from 'strong-type';
+import {createArcaneEventSource} from '../event-manager.mjs';
+
+const is = new Is(false);
+const brokers = new Map();
+
+/** Share the native library owner between Core services in this host. */
+export function createImageRuntime(
+    {libraryPath, bindingModulePath, variants = [], models = [], backend = 'auto', signal, onEvent} = {}
+) {
+    let broker = brokers.get(libraryPath);
+    if (!broker || broker.closed || broker.closing) {
+        const predecessor = broker && !broker.closed ? broker.close() : undefined;
+        broker = createBroker({libraryPath, bindingModulePath, variants, backend}, predecessor);
+        brokers.set(libraryPath, broker);
+    }
+    broker.references++;
+    const lifetime = new AbortController();
+    const lifetimeSignal = signal ? AbortSignal.any([signal, lifetime.signal]) : lifetime.signal;
+    const operations = new Set();
+    const definitions = new Map();
+    const selections = new WeakMap();
+    let closing;
+    for (const model of models) definitions.set(model.id, model);
+
+    const stopDiagnostic = broker.events.on(
+        'image.diagnostic',
+        function runtimeDiagnostic(event) {
+            if (onEvent) {
+                observe(onEvent, event.detail);
+            }
+        }
+    );
+
+    function selectedModel(model) {
+        let definition = typeof model === 'string' ? definitions.get(model) : model;
+        if (!definition) {
+            throw failure('LOCAL_IMAGE_MODEL_UNAVAILABLE', `The image model ${String(model)} has not been prepared.`);
+        }
+        const previous = definitions.get(definition.id);
+        if (previous && isDeepStrictEqual(previous, definition)) definition = previous;
+        definitions.set(definition.id, definition);
+        let selection = selections.get(definition);
+        if (!selection) {
+            selection = {model: definition, backend, selection: ++broker.nextSelection};
+            selections.set(definition, selection);
+        }
+        return selection;
+    }
+
+    function operation(name, parameters, operationSignal, onProgress) {
+        if (closing) return Promise.reject(failure('LOCAL_IMAGE_CLOSED', 'The image runtime is closing.'));
+        const combined = operationSignal ? AbortSignal.any([lifetimeSignal, operationSignal]) : lifetimeSignal;
+        combined.throwIfAborted();
+        if (name === 'unload') {
+            broker.cancelPending(failure('LOCAL_IMAGE_UNLOADED', 'The image model is being unloaded.'));
+        }
+        const task = broker.enqueue(name, parameters, combined, onProgress);
+        operations.add(task);
+        function settled() { operations.delete(task); }
+        task.then(settled, settled);
+        return task;
+    }
+
+    function load({model, context, signal: operationSignal} = {}) {
+        let selected = selectedModel(model);
+        if (context !== undefined) {
+            selected = selectedModel(
+                {...selected.model, context: {...selected.model.context, ...context}}
+            );
+        }
+        return operation('load', selected, operationSignal);
+    }
+
+    function generate({model, prompt, parameters, signal: operationSignal, onProgress} = {}) {
+        return operation(
+            'generate',
+            {...selectedModel(model), prompt, parameters},
+            operationSignal,
+            onProgress
+        );
+    }
+
+    function edit({model, image, prompt, strength, parameters, signal: operationSignal, onProgress} = {}) {
+        if (!is.number(strength) || !is.finite(strength)) {
+            throw new TypeError('Image editing requires an explicit finite strength.');
+        }
+        if (strength < 0 || strength > 1) {
+            throw new RangeError('Image editing strength must be between 0 and 1.');
+        }
+        // The accepted input must survive caller reuse while this request queues.
+        const input = {
+            ...image,
+            data: image?.data instanceof Uint8Array ? new Uint8Array(image.data) : image?.data
+        };
+        return operation(
+            'edit',
+            {...selectedModel(model), image: input, prompt, strength, parameters},
+            operationSignal,
+            onProgress
+        );
+    }
+
+    function unload({signal: operationSignal} = {}) {
+        return operation('unload', {}, operationSignal);
+    }
+
+    function subscribe(listener, {replay = true} = {}) {
+        const stop = broker.events.on('image.state', function stateChanged(event) { observe(listener, event.detail); });
+        if (replay) observe(listener, broker.current());
+        return stop;
+    }
+
+    function close() {
+        if (closing) return closing;
+        closing = Promise.resolve().then(async function releaseRuntime() {
+            await Promise.allSettled([...operations]);
+            const errors = [];
+            try {
+                // Native accessor closure releases the shared model before its
+                // service may release working files. Browser accessor closure
+                // does not invoke this native ownership boundary.
+                if (broker.current().available && !broker.closing) {
+                    await broker.enqueue('unload', {});
+                } else {
+                    await broker.close();
+                }
+            } catch (error) {
+                errors.push(error);
+            }
+            stopDiagnostic();
+            lifetimeSignal.removeEventListener('abort', abortRuntime);
+            broker.references--;
+            if (broker.references === 0 || errors.length) {
+                try {
+                    await broker.close();
+                } catch (error) {
+                    if (!errors.includes(error)) errors.push(error);
+                } finally {
+                    if (brokers.get(libraryPath) === broker) brokers.delete(libraryPath);
+                }
+            }
+            if (errors.length === 1) throw errors[0];
+            if (errors.length > 1) {
+                throw new AggregateError(errors, 'Image model and native owner cleanup failed.', {cause: errors[0]});
+            }
+            return broker.current();
+        });
+        lifetime.abort();
+        return closing;
+    }
+
+    function abortRuntime() { close().catch(reportError); }
+    lifetimeSignal.addEventListener('abort', abortRuntime, {once: true});
+    if (lifetimeSignal.aborted) abortRuntime();
+    return {load, generate, edit, unload, current: broker.current, subscribe, close};
+}
+
+function createBroker(configuration, predecessor) {
+    const events = createArcaneEventSource(
+        {},
+        {source: 'local-image', eventTypes: ['image.state', 'image.diagnostic']}
+    );
+    const queue = [];
+    let state = {
+        state: 'unloaded', available: true, selectedModel: null, loaded: false,
+        backend: null, devices: [], resources: {}, progress: null, error: null, closed: false
+    };
+    let worker;
+    let starting;
+    let workerReady = false;
+    let workerExited = false;
+    let terminalError;
+    let workerExit;
+    let active;
+    let nextRequest = 0;
+    let closeTask;
+    const broker = {references: 0, nextSelection: 0, closing: false, closed: false, events, current, enqueue, cancelPending, close};
+
+    function current() { return structuredClone(state); }
+
+    function publish(fields) {
+        state = {...state, ...fields};
+        events.dispatch('image.state', current());
+    }
+
+    function finish(operation, error, result) {
+        if (operation.settled) return;
+        operation.settled = true;
+        operation.signal?.removeEventListener('abort', operation.abort);
+        if (active === operation) active = undefined;
+        if (error) operation.reject(error);
+        else operation.resolve(result);
+    }
+
+    function failed(error) {
+        terminalError = error;
+        publish({state: 'error', available: false, error: describeError(error)});
+        if (active) finish(active, error);
+        for (const operation of queue.splice(0)) finish(operation, error);
+    }
+
+    async function startWorker() {
+        if (predecessor) {
+            try {
+                await predecessor;
+            } catch (error) {
+                // Retirement has joined the prior native worker even when its
+                // cleanup reports an error. Preserve that diagnostic while the
+                // new owner starts its independent lifetime.
+                events.dispatch('image.diagnostic', {type: 'local-image.previous-close.failed', error: describeError(error)});
+            }
+        }
+        worker = new Worker(new URL('./image-worker.mjs', import.meta.url), {workerData: configuration});
+        workerExit = new Promise(function awaitWorkerExit(resolve) {
+            worker.once('exit', function imageWorkerExited(code) {
+                workerExited = true;
+                workerReady = false;
+                if (!broker.closed && !closeTask) {
+                    failed(failure('LOCAL_IMAGE_WORKER_EXITED', `The native image worker exited with code ${code}.`));
+                }
+                resolve();
+            });
+        });
+        worker.on('error', failed);
+        worker.on('message', function workerMessage(message) {
+            if (message.type === 'ready') {
+                workerReady = true;
+                publish(message.state);
+                pump();
+            } else if (message.type === 'state') {
+                publish(message.state);
+            } else if (message.type === 'diagnostic') {
+                events.dispatch('image.diagnostic', message.event);
+            } else if (message.type === 'progress') {
+                publish({progress: message.progress});
+                if (active?.id === message.id && active.onProgress) {
+                    try {
+                        Promise.resolve(active.onProgress(message.progress)).catch(reportError);
+                    } catch (error) {
+                        reportError(error);
+                    }
+                }
+            } else if (message.type === 'result') {
+                if (active?.id !== message.id) return;
+                finish(active, message.error ? restoreError(message.error) : null, message.result);
+                pump();
+            } else if (message.type === 'fatal') {
+                failed(restoreError(message.error));
+            }
+        });
+    }
+
+    function pump() {
+        if (active || !queue.length) return;
+        if (!worker) {
+            if (!starting) {
+                publish({state: 'starting'});
+                starting = startWorker();
+                starting.catch(failed);
+            }
+            return;
+        }
+        if (!workerReady) return;
+        active = queue.shift();
+        if (active.signal?.aborted) {
+            finish(active, active.signal.reason);
+            pump();
+            return;
+        }
+        try {
+            worker.postMessage({type: 'operation', id: active.id, operation: active.name, parameters: active.parameters});
+        } catch (error) {
+            finish(active, error);
+            pump();
+        }
+    }
+
+    function enqueue(name, parameters, signal, onProgress) {
+        if (broker.closed) return Promise.reject(failure('LOCAL_IMAGE_CLOSED', 'The image runtime is closed.'));
+        if (broker.closing && name !== 'close') {
+            return Promise.reject(failure('LOCAL_IMAGE_CLOSED', 'The image runtime is closing.'));
+        }
+        if (terminalError) return Promise.reject(terminalError);
+        return new Promise(function enqueueImage(resolve, reject) {
+            const operation = {id: ++nextRequest, name, parameters, signal, onProgress, resolve, reject, settled: false};
+            operation.abort = function cancelOperation() {
+                if (active === operation) {
+                    publish({state: 'cancelling'});
+                    worker.postMessage({type: 'cancel', id: operation.id});
+                } else {
+                    const index = queue.indexOf(operation);
+                    if (index !== -1) queue.splice(index, 1);
+                    finish(operation, signal.reason);
+                }
+            };
+            signal?.addEventListener('abort', operation.abort, {once: true});
+            queue.push(operation);
+            if (signal?.aborted) operation.abort();
+            pump();
+        });
+    }
+
+    function cancelPending(reason) {
+        for (const operation of queue.splice(0)) finish(operation, reason);
+        if (active) {
+            publish({state: 'cancelling'});
+            worker.postMessage({type: 'cancel', id: active.id});
+        }
+    }
+
+    function close() {
+        if (closeTask) return closeTask;
+        broker.closing = true;
+        closeTask = (async function closeImageOwner() {
+            try {
+                if (worker || starting) {
+                    if (!workerExited && !terminalError) await enqueue('close', {});
+                }
+            } finally {
+                await starting?.catch(function startupClosed() {});
+                await workerExit;
+                broker.closed = true;
+                publish({state: 'closed', available: false, loaded: false, resources: {}, closed: true});
+                events.dispose();
+            }
+        })();
+        return closeTask;
+    }
+    return broker;
+}
+
+function failure(code, message) {
+    return Object.assign(new Error(message), {code});
+}
+
+function describeError(error) {
+    if (!(error instanceof Error)) return {name: 'Error', message: String(error), cause: error};
+    return {
+        ...error, name: error.name, message: error.message, stack: error.stack,
+        ...(error.cause !== undefined ? {cause: describeError(error.cause)} : {}),
+        ...(error instanceof AggregateError ? {errors: error.errors.map(describeError)} : {})
+    };
+}
+
+function restoreError(record) { return Object.assign(new Error(record.message), record); }
+
+function reportError(error) { console.error('Local image observation failed.', error); }
+
+function observe(listener, detail) {
+    try {
+        Promise.resolve(listener(detail)).catch(reportError);
+    } catch (error) {
+        reportError(error);
+    }
+}

package/src/local-ai/image-worker.mjs

@@ -0,0 +1,232 @@
+import {parentPort, workerData} from 'node:worker_threads';
+import {createImageLibrary} from './image-library.mjs';
+import {encodePNG} from './image-png.mjs';
+import {decodeImage} from './image-editing.mjs';
+
+let library;
+let active;
+let selection;
+let selectedBackend;
+let state = {
+    state: 'unloaded', available: true, selectedModel: null, loaded: false,
+    backend: null, devices: [], resources: {}, progress: null, error: null, closed: false
+};
+
+function publish(fields) {
+    state = {...state, ...fields};
+    parentPort.postMessage({type: 'state', state});
+}
+
+function diagnostic(event) { parentPort.postMessage({type: 'diagnostic', event}); }
+
+function progress(value) {
+    if (!active) return;
+    parentPort.postMessage({type: 'progress', id: active.id, progress: value});
+}
+
+async function initialize() {
+    const variants = workerData.variants.length ? workerData.variants : [
+        {backend: workerData.backend, libraryPath: workerData.libraryPath}
+    ];
+    const failures = [];
+    for (const variant of variants) {
+        try {
+            library = createImageLibrary(
+                {
+                    libraryPath: variant.libraryPath,
+                    bindingModulePath: workerData.bindingModulePath,
+                    onLog: diagnostic,
+                    onProgress: progress
+                }
+            );
+        } catch (error) {
+            if (error.code !== 'LOCAL_IMAGE_LIBRARY_UNAVAILABLE') throw error;
+            failures.push(error);
+            diagnostic({type: 'local-image.backend.unavailable', backend: variant.backend, error: describeError(error)});
+            continue;
+        }
+        const inventory = await library.listDevices();
+        const requested = workerData.backend;
+        const device = inventory.devices.find(function preferredDevice(value) {
+            if (requested === 'cpu' || variant.backend === 'cpu') return /^cpu/iu.test(value.name);
+            return /^(cuda|vulkan|metal)/iu.test(value.name);
+        });
+        if (device) {
+            selectedBackend = device.name;
+            state = {...state, backend: selectedBackend, devices: inventory.devices};
+            diagnostic({type: 'local-image.devices', ...inventory, backend: selectedBackend});
+            return;
+        }
+        await library.close();
+        library = undefined;
+        diagnostic({type: 'local-image.backend.unavailable', backend: variant.backend, ...inventory});
+    }
+    if (failures.length) throw new AggregateError(failures, 'The selected image runtime libraries could not load.');
+    throw Object.assign(new Error('No compatible image generation device is available.'), {code: 'LOCAL_IMAGE_BACKEND_UNAVAILABLE'});
+}
+
+async function unload() {
+    if (!state.loaded) return;
+    publish({state: 'unloading'});
+    await library.unload();
+    selection = undefined;
+    publish({state: 'unloaded', selectedModel: null, loaded: false, resources: {}, progress: null});
+}
+
+async function load(parameters, signal) {
+    signal.throwIfAborted();
+    if (selection === parameters.selection && state.loaded) return;
+    await unload();
+    signal.throwIfAborted();
+    const model = parameters.model;
+    const requestedBackend = parameters.backend ?? 'auto';
+    const requestedDevice = state.devices.find(function requestedModelDevice(device) {
+        if (requestedBackend === 'auto') return device.name === selectedBackend;
+        return device.name.toLowerCase().startsWith(requestedBackend.toLowerCase());
+    }) ?? state.devices.find(function fallbackCPU(device) {
+        return /^cpu/iu.test(device.name);
+    });
+    if (!requestedDevice && model.context?.backend === undefined) {
+        throw Object.assign(new Error(`No available device can run the requested ${requestedBackend} image backend.`), {code: 'LOCAL_IMAGE_BACKEND_UNAVAILABLE'});
+    }
+    publish({state: 'loading', selectedModel: model.id, error: null});
+    try {
+        const loaded = await library.load(
+            {
+                resources: model.resources,
+                context: {...model.context, backend: model.context?.backend ?? requestedDevice.name},
+                signal
+            }
+        );
+        signal.throwIfAborted();
+        selection = parameters.selection;
+        publish(
+            {
+                state: 'ready', loaded: true, resources: model.resources,
+                backend: loaded.backend ?? selectedBackend, modelVersion: loaded.modelVersion
+            }
+        );
+    } catch (error) {
+        await library.unload();
+        selection = undefined;
+        publish({state: signal.aborted ? 'unloaded' : 'error', loaded: false, resources: {}, error: describeError(error)});
+        throw error;
+    }
+}
+
+async function execute(message, signal) {
+    if (message.operation === 'close') {
+        const errors = [];
+        try {
+            await unload();
+        } catch (error) {
+            errors.push(error);
+        }
+        try {
+            await library.close();
+        } catch (error) {
+            errors.push(error);
+        }
+        if (errors.length === 1) throw errors[0];
+        if (errors.length > 1) throw new AggregateError(errors, 'The native image engine could not finish cleanup.');
+        publish({state: 'closed', available: false, closed: true});
+        return state;
+    }
+    if (message.operation === 'unload') {
+        await unload();
+        return state;
+    }
+    const parameters = message.parameters;
+    let image;
+    if (message.operation === 'edit') {
+        try {
+            image = await decodeImage({...parameters.image, signal});
+        } catch (error) {
+            // Decoding leaves the selected context unchanged. Replay its actual
+            // state so a broker cancellation cannot remain visibly unsettled.
+            publish({error: describeError(error)});
+            throw error;
+        }
+    }
+    await load(parameters, signal);
+    if (message.operation === 'load') return state;
+    signal.throwIfAborted();
+    publish({state: 'generating', progress: null, error: null});
+    try {
+        const defaults = parameters.model.defaults ?? {};
+        const supplied = parameters.parameters ?? {};
+        const rasterImages = await library.generate(
+            {
+                prompt: parameters.prompt,
+                parameters: {
+                    ...defaults, ...supplied,
+                    ...(message.operation === 'edit' ? {strength: parameters.strength} : {}),
+                    sample_params: {
+                        ...defaults.sample_params, ...supplied.sample_params,
+                        guidance: {...defaults.sample_params?.guidance, ...supplied.sample_params?.guidance}
+                    }
+                },
+                image,
+                signal
+            }
+        );
+        signal.throwIfAborted();
+        const images = [];
+        for (const image of rasterImages) {
+            const data = await encodePNG({...image, signal});
+            images.push({data, mediaType: 'image/png', width: image.width, height: image.height});
+        }
+        signal.throwIfAborted();
+        publish({state: 'ready', progress: null});
+        return {images};
+    } catch (error) {
+        publish({state: state.loaded ? 'ready' : 'error', progress: null, error: describeError(error)});
+        throw error;
+    }
+}
+
+parentPort.on('message', function incoming(message) {
+    if (message.type === 'cancel') {
+        if (active?.id === message.id) active.controller.abort();
+        return;
+    }
+    if (message.type !== 'operation') return;
+    const controller = new AbortController();
+    active = {id: message.id, controller};
+    const task = execute(message, controller.signal);
+    task.then(
+        function completed(result) {
+            active = undefined;
+            parentPort.postMessage({type: 'result', id: message.id, result});
+            if (message.operation === 'close') parentPort.close();
+        },
+        function operationFailed(error) {
+            active = undefined;
+            parentPort.postMessage({type: 'result', id: message.id, error: describeError(error)});
+            if (message.operation === 'close') parentPort.close();
+        }
+    );
+});
+
+function describeError(error) {
+    if (!(error instanceof Error)) return {name: 'Error', message: String(error), cause: error};
+    return {
+        ...error, name: error.name, message: error.message, stack: error.stack,
+        ...(error.cause !== undefined ? {cause: describeError(error.cause)} : {}),
+        ...(error instanceof AggregateError ? {errors: error.errors.map(describeError)} : {})
+    };
+}
+
+try {
+    await initialize();
+    parentPort.postMessage({type: 'ready', state});
+} catch (error) {
+    let failure = error;
+    try {
+        await library?.close();
+    } catch (cleanupError) {
+        failure = new AggregateError([error, cleanupError], 'Image runtime initialization and cleanup failed.');
+    }
+    parentPort.postMessage({type: 'fatal', error: describeError(failure)});
+    parentPort.close();
+}