arcane-os 0.63.0 → 0.65.0

package/src/core/repositories.mjs

@@ -0,0 +1,375 @@
+import {mkdir, readdir, writeFile} from 'node:fs/promises';
+import {homedir} from 'node:os';
+import path from 'node:path';
+import Is from 'strong-type';
+import {ArcaneError, throwIfAborted} from '../errors.mjs';
+import {runProcess} from '../process.mjs';
+import {repositoryPull, repositoryPush, repositoryStatus} from '../repository.mjs';
+
+const is = new Is(false);
+const directoryOperations = new Map();
+
+/** Resolve persistent native data without creating or migrating anything. */
+export function resolveArcaneDataPaths({dataRoot} = {}) {
+    let root = dataRoot;
+    if (root === undefined) {
+        const home = homedir();
+        switch (process.platform) {
+            case 'win32':
+                root = path.join(process.env.LOCALAPPDATA || path.join(home, 'AppData', 'Local'), 'ArcaneData');
+                break;
+            case 'darwin':
+                root = path.join(home, 'Library', 'Application Support', 'ArcaneData');
+                break;
+            case 'linux': {
+                const xdg = process.env.XDG_DATA_HOME;
+                const base = xdg && path.isAbsolute(xdg) ? xdg : path.join(home, '.local', 'share');
+                root = path.join(base, 'ArcaneData');
+                break;
+            }
+            default:
+                throw new ArcaneError(
+                    'ARCANE_DATA_ROOT_REQUIRED',
+                    'This native host must supply its persistent ArcaneData directory through dataRoot.'
+                );
+        }
+    }
+    requireString(root, 'dataRoot');
+    const resolved = path.resolve(root);
+    return {dataRoot: resolved, repositoriesRoot: path.join(resolved, 'Repos')};
+}
+
+/** One connected checkout, composed into an application's own Core service. */
+export function createRepositoryWorkspace(
+    {name, directory, dataRoot, remote, branch, onEvent, run = runProcess} = {}
+) {
+    let selected = directory;
+    if (selected === undefined) {
+        requireString(name, 'name');
+        if (name === '.' || name === '..' || /[/\\]/u.test(name)) {
+            throw new TypeError('name must be one directory name; use directory for an explicit repository path.');
+        }
+        selected = path.join(resolveArcaneDataPaths({dataRoot}).repositoriesRoot, name);
+    }
+    requireString(selected, 'directory');
+    if (remote !== undefined) requireString(remote, 'remote');
+    if (branch !== undefined) requireString(branch, 'branch');
+    if (!is.function(run)) throw new TypeError('run must implement the SDK process adapter.');
+    const repositoryDirectory = path.resolve(selected);
+    const pending = new Set();
+    let opened = false;
+    let closing = null;
+
+    async function prepare(signal, execute = run) {
+        throwIfAborted(signal);
+        if (opened) return {directory: repositoryDirectory, cloned: false};
+        let entries;
+        try {
+            entries = await readdir(repositoryDirectory);
+        } catch (error) {
+            if (error.code !== 'ENOENT') throw error;
+            entries = [];
+        }
+        throwIfAborted(signal);
+        if (entries.length) {
+            // A parent checkout or a bare object cache is not this connection's
+            // working root. Inspection never changes an existing destination.
+            const result = await execute(
+                'git',
+                ['rev-parse', '--is-inside-work-tree', '--show-prefix'],
+                {cwd: repositoryDirectory, signal, onEvent}
+            );
+            throwIfAborted(signal);
+            const [inside, prefix] = result.stdout.split(/\r?\n/u);
+            if (inside !== 'true' || prefix !== '') {
+                throw new ArcaneError(
+                    'ARCANE_REPOSITORY_DIRECTORY_INVALID',
+                    'The selected nonempty directory is not a Git working repository root.',
+                    {details: result}
+                );
+            }
+            opened = true;
+            return {directory: repositoryDirectory, cloned: false};
+        }
+        requireString(remote, 'remote');
+        await mkdir(path.dirname(repositoryDirectory), {recursive: true});
+        throwIfAborted(signal);
+        const arguments_ = ['clone', '--progress'];
+        if (branch !== undefined) arguments_.push('--branch', branch);
+        arguments_.push('--', remote, repositoryDirectory);
+        const result = await execute(
+            'git',
+            arguments_,
+            {cwd: path.dirname(repositoryDirectory), signal, onEvent}
+        );
+        throwIfAborted(signal);
+        opened = true;
+        return {directory: repositoryDirectory, cloned: true, stdout: result.stdout, stderr: result.stderr};
+    }
+
+    function accept(operation, signal) {
+        try {
+            throwIfAborted(signal);
+            if (closing) throw new ArcaneError('CORE_CLOSING', 'The repository workspace is closing.');
+        } catch (error) {
+            return Promise.reject(error);
+        }
+        const result = orderDirectoryOperation(
+            repositoryDirectory,
+            async function runRepositoryOperation() {
+                throwIfAborted(signal);
+                return operation();
+            }
+        );
+        pending.add(result);
+        function settled() {
+            pending.delete(result);
+        }
+        result.then(settled, settled);
+        return result;
+    }
+
+    function open({signal} = {}) {
+        return accept(
+            function openRepository() {
+                return prepare(signal);
+            },
+            signal
+        );
+    }
+
+    function status({signal} = {}) {
+        return accept(
+            async function readRepositoryStatus() {
+                await prepare(signal);
+                return repositoryStatus({workspaceRoot: repositoryDirectory, signal, onEvent, run});
+            },
+            signal
+        );
+    }
+
+    function pull({signal} = {}) {
+        return accept(
+            async function pullRepository() {
+                await prepare(signal);
+                return repositoryPull({workspaceRoot: repositoryDirectory, signal, onEvent, run});
+            },
+            signal
+        );
+    }
+
+    function push({signal} = {}) {
+        return accept(
+            async function pushRepository() {
+                await prepare(signal);
+                return repositoryPush({workspaceRoot: repositoryDirectory, signal, onEvent, run});
+            },
+            signal
+        );
+    }
+
+    function write({files, message, signal} = {}) {
+        let selectedFiles;
+        try {
+            requireString(message, 'message');
+            if (!message.isWellFormed()) throw new TypeError('message cannot be represented completely as UTF-8 text.');
+            if (!is.array(files) || files.length === 0) {
+                throw new TypeError('write requires at least one explicitly selected file.');
+            }
+            selectedFiles = Array.from(
+                files,
+                function snapshotFile(file) {
+                    const {path: filename, content} = file;
+                    requireString(filename, 'file.path');
+                    if (!filename.isWellFormed() || filename.includes('\0')) {
+                        throw new TypeError('file.path cannot be represented completely as a native filename.');
+                    }
+                    if (!is.string(content) || !content.isWellFormed()) {
+                        throw new TypeError('file.content must be complete UTF-8-representable text.');
+                    }
+                    const destination = path.resolve(repositoryDirectory, filename);
+                    const relative = path.relative(repositoryDirectory, destination);
+                    const first = relative.split(path.sep)[0];
+                    if (path.isAbsolute(filename) || !relative || relative === '..'
+                        || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)
+                        || first.toLowerCase() === '.git') {
+                        throw new TypeError('file.path must name a working file relative to this repository, outside Git metadata.');
+                    }
+                    return {path: filename, content, destination};
+                }
+            );
+        } catch (error) {
+            return Promise.reject(error);
+        }
+
+        return accept(
+            async function writeRepositoryFiles() {
+                const outcome = {
+                    directory: repositoryDirectory,
+                    state: 'local',
+                    stage: 'prepare',
+                    paths: selectedFiles.map(function selectedPath(file) { return file.path; }),
+                    writtenPaths: [],
+                    written: false,
+                    staged: false,
+                    committed: false,
+                    pushed: false,
+                    outputs: []
+                };
+                let pendingWrite = false;
+
+                async function executeGit(stage, args, input, options = {}) {
+                    throwIfAborted(signal);
+                    outcome.stage = stage;
+                    const output = {stage, stdout: '', stderr: ''};
+                    const observed = {stdout: false, stderr: false};
+                    outcome.outputs.push(output);
+                    let beforeStartFailed = false;
+
+                    function consumeOutput({stream, chunk}) {
+                        observed[stream] = true;
+                        output[stream] += chunk;
+                    }
+
+                    async function observeEvent(event) {
+                        try {
+                            await onEvent?.(event);
+                        } catch (error) {
+                            if (event.type === 'process.starting') beforeStartFailed = true;
+                            throw error;
+                        }
+                    }
+
+                    function recordCompletion(result) {
+                        if (!result || !is.integer(result.code)) return;
+                        output.code = result.code;
+                        output.signal = result.signal ?? null;
+                        for (const stream of ['stdout', 'stderr']) {
+                            if (!observed[stream] && is.string(result[stream])) output[stream] = result[stream];
+                        }
+                        if (result.code !== 0) return;
+                        if (stage === 'stage') outcome.staged = true;
+                        if (stage === 'commit') {
+                            outcome.committed = true;
+                            outcome.state = 'committed';
+                        }
+                        if (stage === 'push') {
+                            outcome.pushed = true;
+                            outcome.state = 'pushed';
+                        }
+                    }
+
+                    try {
+                        const result = await run(
+                            'git',
+                            args,
+                            {cwd: options.cwd ?? repositoryDirectory, signal, onEvent: observeEvent,
+                                onOutput: consumeOutput, captureOutput: false, input}
+                        );
+                        recordCompletion(result);
+                        if (result.code !== 0) {
+                            throw new ArcaneError(
+                                'ARCANE_OPERATION_FAILED',
+                                `Git ${stage} exited with code ${String(result.code)}.`,
+                                {details: result}
+                            );
+                        }
+                        return {...result, stdout: output.stdout, stderr: output.stderr};
+                    } catch (error) {
+                        if (!beforeStartFailed) recordCompletion(error?.details);
+                        const notStarted = beforeStartFailed
+                            || (error?.code === 'ARCANE_PREREQUISITE_MISSING' && !error?.details)
+                            || (error?.code === 'ARCANE_CANCELLED' && !error?.details);
+                        if (!notStarted && output.code !== 0 && (stage === 'commit' || stage === 'push')) {
+                            outcome.state = 'uncertain';
+                        }
+                        throw error;
+                    }
+                }
+
+                async function* commitMessage() {
+                    yield message;
+                }
+
+                try {
+                    await prepare(
+                        signal,
+                        function prepareGit(command, args, options) {
+                            return executeGit('prepare', args, undefined, options);
+                        }
+                    );
+                    outcome.stage = 'write';
+                    for (const file of selectedFiles) {
+                        throwIfAborted(signal);
+                        outcome.path = file.path;
+                        await mkdir(path.dirname(file.destination), {recursive: true});
+                        throwIfAborted(signal);
+                        pendingWrite = true;
+                        await writeFile(file.destination, file.content, {encoding: 'utf8', signal});
+                        pendingWrite = false;
+                        outcome.writtenPaths.push(file.path);
+                    }
+                    outcome.written = true;
+                    delete outcome.path;
+                    await executeGit('stage', ['--literal-pathspecs', 'add', '--', ...outcome.paths]);
+                    await executeGit(
+                        'commit',
+                        ['--literal-pathspecs', 'commit', '--only', '--cleanup=verbatim', '--file', '-', '--', ...outcome.paths],
+                        commitMessage()
+                    );
+                    await executeGit('push', ['push']);
+                    outcome.stage = 'complete';
+                    return outcome;
+                } catch (cause) {
+                    if (pendingWrite) outcome.state = 'uncertain';
+                    throw new ArcaneError(
+                        cause?.code ?? 'ARCANE_REPOSITORY_WRITE_FAILED',
+                        cause instanceof Error ? cause.message : String(cause),
+                        {cause, details: outcome, exitCode: cause?.exitCode}
+                    );
+                }
+            },
+            signal
+        );
+    }
+
+    function close() {
+        if (!closing) {
+            closing = Promise.allSettled([...pending]).then(
+                function operationsDrained(results) {
+                    const failures = results.filter(
+                        function failed(result) {
+                            return result.status === 'rejected';
+                        }
+                    ).map(
+                        function reason(result) {
+                            return result.reason;
+                        }
+                    );
+                    if (failures.length) throw new AggregateError(failures, 'Repository operations failed while draining.');
+                }
+            );
+        }
+        return closing;
+    }
+
+    return {directory: repositoryDirectory, open, status, pull, push, write, close, drain: close, dispose: close};
+}
+
+function requireString(value, name) {
+    if (!is.string(value) || value === '') throw new TypeError(`${name} must be a nonempty string.`);
+}
+
+function orderDirectoryOperation(directory, operation) {
+    const key = process.platform === 'win32' ? directory.toLowerCase() : directory;
+    const previous = directoryOperations.get(key) ?? Promise.resolve();
+    const result = previous.then(operation, operation);
+    function releaseQueue() {
+        if (directoryOperations.get(key) === tail) directoryOperations.delete(key);
+    }
+    // Only the scheduling tail settles successfully after an error. Each caller
+    // retains its complete failure; a later operation can retry the same path.
+    const tail = result.then(releaseQueue, releaseQueue);
+    directoryOperations.set(key, tail);
+    return result;
+}

package/src/core/services/local-ai.mjs

@@ -208,17 +208,54 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
         publish();
     }
 
+    function beginOllamaObservation(engine) {
+        return {revision: ++engine.ollamaRevision, signal: engine.signal};
+    }
+
+    function ownsOllamaObservation(engine, observation) {
+        return observation.signal === engine.signal && !observation.signal.aborted
+            && engine.server.current.available && observation.revision >= engine.ollamaAppliedRevision;
+    }
+
+    function recordOllamaFailure(engine, error, observation) {
+        if (!ownsOllamaObservation(engine, observation)) return;
+        engine.ollamaAppliedRevision = observation.revision;
+        recordFailure(engine, error);
+    }
+
+    function recordOllamaModels(engine, catalog, running, observation) {
+        if (!ownsOllamaObservation(engine, observation)) return;
+        engine.ollamaAppliedRevision = observation.revision;
+        engine.state = {...engine.state, ...engine.server.current, error: null};
+        const resident = new Map((running.models ?? []).map(function residentModel(model) {
+            return [model.model ?? model.name, model];
+        }));
+        const models = new Map((catalog.models ?? []).map(function catalogModel(model) {
+            const id = model.model ?? model.name ?? model.id;
+            return [id, {...model, id, loaded: resident.has(id)}];
+        }));
+        for (const [id, model] of resident) {
+            if (!models.has(id)) models.set(id, {...model, id, loaded: true});
+        }
+        engine.models = [...models.values()];
+    }
+
     async function refreshModels(engine, signal) {
         const revision = engine.revision;
         if (engine.id === 'ollama') {
-            const [catalog, running] = await Promise.all([
-                requestLocalJSON({url: engine.server.url, path: '/api/tags', signal}),
-                requestLocalJSON({url: engine.server.url, path: '/api/ps', signal})
-            ]);
-            const loaded = new Set((running.models ?? []).map(function loadedId(model) { return model.model ?? model.name; }));
-            engine.models = (catalog.models ?? []).map(function ollamaModel(model) {
-                return {...model, id: model.model ?? model.name, loaded: loaded.has(model.model ?? model.name)};
-            });
+            const observation = beginOllamaObservation(engine);
+            const observationSignal = signal ? AbortSignal.any([signal, observation.signal]) : observation.signal;
+            try {
+                const [catalog, running] = await Promise.all([
+                    requestLocalJSON({url: engine.server.url, path: '/api/tags', signal: observationSignal}),
+                    requestLocalJSON({url: engine.server.url, path: '/api/ps', signal: observationSignal})
+                ]);
+                observationSignal.throwIfAborted();
+                recordOllamaModels(engine, catalog, running, observation);
+            } catch (error) {
+                if (!observationSignal.aborted) recordOllamaFailure(engine, error, observation);
+                throw error;
+            }
         } else {
             const properties = await requestLocalJSON({url: engine.server.url, path: '/props', signal});
             engine.router = properties.role === 'router';
@@ -427,6 +464,34 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
 
     async function ollamaRequest(operation, parameters = {}, request, method = 'POST') {
         const engine = await engineReady('ollama', request.signal);
+        const engineSignal = engine.signal;
+        const observation = operation === 'ps' ? beginOllamaObservation(engine) : null;
+        try {
+            const result = await ollamaResponse(engine, operation, parameters, request, method);
+            if (operation === 'ps') {
+                recordOllamaModels(engine, {models: engine.models}, result, observation);
+                publish();
+            }
+            return result;
+        } catch (error) {
+            if (operation === 'ps' && !request.signal.aborted) recordOllamaFailure(engine, error, observation);
+            throw error;
+        } finally {
+            // Every owned operation that can change residency/catalog leaves
+            // a real upstream observation, including cancellation or failure.
+            if (['chat', 'generate', 'embed', 'pull', 'create', 'copy', 'delete'].includes(operation)
+                && engine.signal === engineSignal && !engineSignal.aborted) {
+                try {
+                    await refreshModels(engine, engineSignal);
+                } catch {
+                    // The refresh publishes only its current owned failure;
+                    // retain the operation's original result or error.
+                }
+            }
+        }
+    }
+
+    async function ollamaResponse(engine, operation, parameters, request, method) {
         const signal = AbortSignal.any([request.signal, engine.signal]);
         const {streamId, ...payload} = parameters;
         if (payload.stream !== true) {
@@ -472,6 +537,8 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
     function startEngine(engine) {
         engine.controller = new AbortController();
         engine.signal = AbortSignal.any([lifetimeSignal, engine.controller.signal]);
+        engine.ollamaRevision = 0;
+        engine.ollamaAppliedRevision = 0;
         const signal = engine.signal;
         engine.server = createLocalAIServer({
             id: engine.id, appRoot, runtime: runtimes.find(function runtimeRecord(record) { return record.id === engine.id; }),
@@ -487,7 +554,10 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
             }
             return engine;
         });
-        engine.ready.catch(function startupFailed(error) { if (!signal.aborted) recordFailure(engine, error); });
+        engine.ready.catch(function startupFailed(error) {
+            // Ollama refreshes and the server already publish their own failures.
+            if (!signal.aborted && engine.id !== 'ollama') recordFailure(engine, error);
+        });
     }
 
     const service = {
@@ -557,7 +627,7 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
                         await refreshModels(engine, request.signal);
                     } catch (error) {
                         request.signal.throwIfAborted();
-                        recordFailure(engine, error);
+                        if (engine.id !== 'ollama') recordFailure(engine, error);
                     }
                 }));
                 return snapshot();

package/src/core/services/speech.mjs

@@ -196,6 +196,7 @@ export function createSpeechService({stt, tts, signal} = {}) {
                 parameters,
                 {
                     signal: operationSignal,
+                    requestId: request.requestId,
                     onProgress(progress) {
                         if (!operationSignal.aborted) {
                             request.emit?.(

package/src/dev-server.mjs

@@ -855,6 +855,7 @@ async function startOwnedDevServer({
     certPath,
     keyPath,
     localAI,
+    core = localAI,
     signal,
     sdkRuntimeSourceRoot
 }={},events,releaseSignal){
@@ -1261,7 +1262,7 @@ async function startOwnedDevServer({
                 response.end();
                 return;
             }
-            if (localAI && await localAI.handler(request, response)) return;
+            if (core && await core.handler(request, response)) return;
             if(request.method!=='GET'&&request.method!=='HEAD'){
                 deny(response,405,'Method not allowed.');
                 return;
@@ -1558,7 +1559,7 @@ async function startOwnedDevServer({
         }
         if(!closeInitiated){
             closeInitiated=true;
-            localAIClosing=Promise.resolve().then(function closeLocalAI(){return localAI?.close(error);});
+            localAIClosing=Promise.resolve().then(function closeCore(){return core?.close(error);});
             localAIClosing.catch(function observeLocalAICloseFailure(closeError){operationalError??=closeError;});
             try{
                 closeDevelopmentListeners(fileServer, tlsServer).then(

package/src/index.mjs

@@ -146,6 +146,7 @@ export {
 } from './workspace.mjs';
 export {materializeInstalledSdkRuntime} from './installed-sdk-runtime.mjs';
 export {startDevServer} from './dev-server.mjs';
+export {createDevelopmentCore} from './core/development.mjs';
 export {startSourceExampleServer} from './source-server.mjs';
 export {acceptWebSocket} from './websocket.mjs';
 export {WS} from '../browser-runtime/websocket-client.mjs';

package/src/local-ai/config.mjs

@@ -4,9 +4,11 @@ import Is from 'strong-type';
 import {ArcaneError, ERROR_CODES, throwIfAborted} from '../errors.mjs';
 import {SD14_MODEL} from './image-models/sd14.mjs';
 import {FLUX2_KLEIN_4B_MODEL} from './image-models/flux.mjs';
+import {SDXL_BASE_1_0_MODEL} from './image-models/sdxl.mjs';
+import {normalizeWhisperRuntimeRequirement} from './whisper/config.mjs';
 
 const is = new Is(false);
-export const LOCAL_AI_RUNTIME_IDS = ['llama.cpp', 'ollama', 'onnx', 'stable-diffusion.cpp'];
+export const LOCAL_AI_RUNTIME_IDS = ['llama.cpp', 'ollama', 'onnx', 'stable-diffusion.cpp', 'whisper.cpp'];
 
 /** Image requirements describe assets without acquiring or loading a model. */
 export function normalizeImageRuntimeRequirement(record) {
@@ -20,9 +22,10 @@ export function normalizeImageRuntimeRequirement(record) {
     const models = (record.models ?? []).map(
         function selectedImageModel(value) {
             const model = value === 'sd14' ? SD14_MODEL
-                : value === 'flux2-klein-4b' ? FLUX2_KLEIN_4B_MODEL : value;
+                : value === 'flux2-klein-4b' ? FLUX2_KLEIN_4B_MODEL
+                : value === 'sdxl-base-1.0' ? SDXL_BASE_1_0_MODEL : value;
             if (!model || !is.object(model) || is.array(model) || !is.string(model.id) || !model.id) {
-                throw new ArcaneError(ERROR_CODES.usage, 'An image model needs a descriptor with an id, or a built-in model ID (sd14 or flux2-klein-4b).');
+                throw new ArcaneError(ERROR_CODES.usage, 'An image model needs a descriptor with an id, or a built-in model ID (sd14, flux2-klein-4b, or sdxl-base-1.0).');
             }
             if (!model.resources || !is.object(model.resources) || is.array(model.resources)) {
                 throw new ArcaneError(ERROR_CODES.usage, `Image model ${model.id} needs resources keyed by model role.`);
@@ -59,6 +62,7 @@ export function normalizeLocalAIConfig(value) {
             throw new ArcaneError(ERROR_CODES.usage, `Unknown local AI runtime: ${String(record?.id ?? item)}.`);
         }
         if (record.id === 'stable-diffusion.cpp') return normalizeImageRuntimeRequirement(record);
+        if (record.id === 'whisper.cpp') return normalizeWhisperRuntimeRequirement(record);
         return {...record};
     });
     return {...value, runtimes};