arcane-os 0.66.0 → 0.67.0

package/src/core/services/decisions.mjs

@@ -0,0 +1,144 @@
+import path from 'node:path';
+import {createNativeDecisionModel} from '../../local-ai/decisions.mjs';
+import {createArcaneEventSource} from '../../event-manager.mjs';
+import {CoreError} from '../../../browser-runtime/core/contracts.mjs';
+import {encodeTensorMap} from '../../../browser-runtime/ai/onnx-tensors.mjs';
+
+/** App-owned selection; shared typed-decision execution, without a renderer. */
+export function createNativeDecisionService(configuration = {}, {appRoot = process.cwd()} = {}) {
+    const events = createArcaneEventSource({}, {source: 'core-decisions', eventTypes: ['decisions.state']});
+    const lifetime = new AbortController();
+    let context;
+    let model;
+    let loading;
+    let loadController;
+    let releasing;
+    let closing;
+    let unsubscribe;
+
+    function current() {
+        return model?.current() ?? {
+            family: 'laya', model: configuration.model ?? 'onnx-community/laya-typed-decisions-ONNX',
+            revision: configuration.revision ?? 'main', dtype: configuration.dtype ?? 'fp32',
+            state: closing ? 'disposed' : 'unloaded', loaded: false, busy: Boolean(loading),
+            activeRequests: 0, progress: null, error: null, execution: null
+        };
+    }
+
+    function publish(value = current()) {
+        events.dispatch('decisions.state', value);
+        context?.emit('decisions.state', value);
+    }
+
+    function subscribe(listener, {emitCurrent = true, signal} = {}) {
+        const stop = events.on('decisions.state', function changed(event) { listener(event.detail); }, {signal});
+        try { if (emitCurrent && !signal?.aborted) listener(current()); }
+        catch (error) { stop(); throw error; }
+        return stop;
+    }
+
+    function operationSignal(signal) { return signal ? AbortSignal.any([signal, lifetime.signal]) : lifetime.signal; }
+
+    function load({signal} = {}) {
+        operationSignal(signal).throwIfAborted();
+        if (!context) throw new CoreError({code: 'CORE_NOT_READY', message: 'Register and start the decision service with Core before loading.'});
+        if (loading) return observeLoad(loading, loadController, signal);
+        if (model?.current().loaded) return Promise.resolve(model.current());
+        loadController = new AbortController();
+        const controller = loadController;
+        const selectedSignal = AbortSignal.any([lifetime.signal, controller.signal]);
+        const precedingRelease = releasing;
+        loading = Promise.resolve().then(async function loadSelectedModel() {
+            selectedSignal.throwIfAborted();
+            if (precedingRelease) await precedingRelease;
+            selectedSignal.throwIfAborted();
+            if (model) await model.dispose();
+            unsubscribe?.();
+            // Only the explicit load waits for its native dependencies. Merely
+            // registering this service does not download a model or delay UI.
+            const [localAI, modelAssets] = await Promise.all([
+                context.getService('local-ai'),
+                configuration.paths ? undefined : context.getService('model-assets')
+            ]);
+            selectedSignal.throwIfAborted();
+            const paths = configuration.paths && Object.fromEntries(
+                Object.entries(configuration.paths).map(function nativePath([name, value]) { return [name, path.resolve(appRoot, value)]; })
+            );
+            model = createNativeDecisionModel({
+                ...configuration, paths, onnx: localAI.getONNXRuntime(), modelAssets,
+                signal: lifetime.signal
+            });
+            unsubscribe = model.subscribe(publish);
+            return model.load({signal: selectedSignal});
+        });
+        function settled() { loading = undefined; loadController = undefined; publish(); }
+        loading.then(settled, settled);
+        publish();
+        return observeLoad(loading, controller, signal);
+    }
+
+    function observeLoad(task, controller, signal) {
+        function abort() { controller.abort(signal.reason); }
+        signal?.addEventListener('abort', abort, {once: true});
+        if (signal?.aborted) abort();
+        return task.finally(function detachLoadCaller() { signal?.removeEventListener('abort', abort); });
+    }
+
+    async function evaluate(rows, {signal, runOptions} = {}) {
+        const selectedSignal = operationSignal(signal);
+        selectedSignal.throwIfAborted();
+        if (!model) throw new CoreError({code: 'ARCANE_DECISION_NOT_LOADED', message: 'Load the selected native decision model before evaluating.'});
+        return model.evaluate(rows, {signal: selectedSignal, runOptions});
+    }
+
+    function unload() {
+        if (releasing) return releasing;
+        // An unload during model preparation must cancel that accepted load,
+        // rather than wait for it to download and activate first.
+        loadController?.abort(new CoreError({name: 'AbortError', code: 'ARCANE_AI_REQUEST_ABORTED', message: 'The native decision load was cancelled.'}));
+        const task = model?.unload() ?? Promise.resolve();
+        const acceptedLoad = loading;
+        releasing = Promise.resolve().then(async function releaseSelectedModel() {
+            const results = await Promise.allSettled([task, acceptedLoad]);
+            if (results[0].status === 'rejected') throw results[0].reason;
+            return current();
+        });
+        function released() { releasing = undefined; }
+        releasing.then(released, released);
+        return releasing;
+    }
+
+    function dispose() {
+        if (closing) return closing;
+        const reason = new CoreError({name: 'AbortError', code: 'ARCANE_AI_REQUEST_ABORTED', message: 'The native decision service is closing.'});
+        closing = Promise.resolve().then(async function closeDecisionService() {
+            const failures = [];
+            try { await loading; } catch (error) { if (!lifetime.signal.aborted) failures.push(error); }
+            try { await model?.dispose(); } catch (error) { failures.push(error); }
+            unsubscribe?.();
+            publish();
+            events.dispose();
+            if (failures.length) throw new AggregateError(failures, 'Closing the decision service failed.');
+        });
+        lifetime.abort(reason);
+        return closing;
+    }
+
+    return {
+        name: configuration.name ?? 'decisions', current, subscribe, load, evaluate, classify: evaluate, unload, dispose,
+        start(serviceContext) { context = serviceContext; },
+        methods: {
+            'decisions.status': current,
+            'decisions.load': function loadRequest(parameters, request) { return load({signal: request.signal}); },
+            'decisions.evaluate': async function evaluateRequest({rows, runOptions}, request) {
+                const result = await evaluate(rows, {runOptions, signal: request.signal});
+                // Typed arrays retain their complete values through Core's
+                // existing JSON tensor encoding, only at this transport edge.
+                return {...result, outputs: encodeTensorMap(result.outputs)};
+            },
+            'decisions.unload': {lifetime: 'service', handle: unload}
+        }
+    };
+}
+
+export default createNativeDecisionService;

package/src/core/services/local-ai.mjs

@@ -149,8 +149,13 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
         };
     }
 
+    function runtimeRecord(engine) {
+        return {...engine.state, models: [...engine.models],
+            ...(engine.id === 'llama.cpp' ? {released: engine.released === true} : {})};
+    }
+
     function snapshot() {
-        const records = [...engines.values()].map(function runtimeRecord(engine) { return {...engine.state, models: [...engine.models]}; });
+        const records = [...engines.values()].map(runtimeRecord);
         if (onnxSelected) records.push(onnxStatus());
         const ollama = records.find(function ollamaRecord(record) { return record.id === 'ollama'; });
         return {
@@ -283,7 +288,7 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
             engine.models = [...merged.values()];
         }
         publish();
-        return {...engine.state, models: [...engine.models]};
+        return runtimeRecord(engine);
     }
 
     function observeModels(engine) {
@@ -347,7 +352,7 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
         return recovering;
     }
 
-    async function engineReady(id, signal, {retryLoading = false} = {}) {
+    async function engineReady(id, signal, {retryLoading = false, resumeReleased = true} = {}) {
         signal?.throwIfAborted();
         const engine = engines.get(id);
         if (!engine) throw failure('LOCAL_AI_RUNTIME_NOT_SELECTED', `${id} is not selected in this application's localAI configuration.`);
@@ -358,13 +363,28 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
         if (retryLoading && engine.state.state === 'loading') {
             await awaitReadiness(restartEngine(engine), signal);
         }
-        if (engine.released) { engine.released = false; startEngine(engine); }
+        if (engine.released) {
+            if (!resumeReleased) return engine;
+            engine.released = false;
+            startEngine(engine);
+        }
         await awaitReadiness(engine.ready, signal);
         signal?.throwIfAborted();
         if (!engine.state.available) throw failure('LOCAL_AI_RUNTIME_UNAVAILABLE', `${id} is unavailable.`);
         return engine;
     }
 
+    function observeLlama(signal) {
+        signal?.throwIfAborted();
+        const engine = engines.get('llama.cpp');
+        if (!engine) throw failure('LOCAL_AI_RUNTIME_NOT_SELECTED', "llama.cpp is not selected in this application's localAI configuration.");
+        // Observation neither waits for startup nor starts a released engine.
+        if (!engine.state.available || engine.released || engine.stopping || engine.recovering || engine.signal.aborted) {
+            return runtimeRecord(engine);
+        }
+        return refreshModels(engine, signal);
+    }
+
     function waitForModel(engine, model, signal) {
         return new Promise(function awaitModel(resolve, reject) {
             let unsubscribe;
@@ -563,6 +583,9 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
     const service = {
         name: 'local-ai',
         current: snapshot,
+        getONNXRuntime() {
+            return requireONNX();
+        },
         async start(currentContext) {
             context = currentContext;
             if (prepare) {
@@ -633,17 +656,16 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
                 return snapshot();
             },
             'llama.status': async function llamaStatus(_parameters, request) {
-                const engine = await engineReady('llama.cpp', request.signal);
-                return refreshModels(engine, request.signal);
+                return observeLlama(request.signal);
             },
             'llama.models': async function llamaModels(_parameters, request) {
-                const engine = await engineReady('llama.cpp', request.signal);
-                const status = await refreshModels(engine, request.signal);
+                const status = await observeLlama(request.signal);
                 return {models: status.models};
             },
             'llama.load': loadLlama,
             'llama.unload': async function unloadLlama({model}, request) {
-                const engine = await engineReady('llama.cpp', request.signal);
+                const engine = await engineReady('llama.cpp', request.signal, {resumeReleased: false});
+                if (engine.released) return {...engine.state, models: [...engine.models], released: true, unloaded: true};
                 engine.loadControllers.get(model)?.abort(failure('LOCAL_AI_MODEL_UNLOADED', `The application unloaded ${model}.`));
                 if (engine.loads.has(model)) await Promise.allSettled([engine.loads.get(model)]);
                 if (!engine.router) {

package/src/core/services/model-assets.mjs

@@ -10,7 +10,7 @@ function failure(code, message) {
     return new CoreError({code, message});
 }
 
-/** Working files for native engines. The browser's stored assets stay authoritative. */
+/** Working files for native engines. Caller-selected source assets stay authoritative. */
 export function createModelAssetService({appRoot = process.cwd()} = {}) {
     const projections = new Map();
     let context;
@@ -63,6 +63,7 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
             const errors = [];
             let filesClosed = false;
             try { await record.opening; } catch (error) { record.error ??= error; }
+            try { await record.downloading; } catch (error) { record.error ??= error; }
             try { await record.finishing; } catch (error) { record.error ??= error; }
             try { await closeMembers(record); filesClosed = true; } catch (error) { errors.push(error); }
             if (record.directory && filesClosed) {
@@ -87,10 +88,16 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
 
     function releasePreparation(record) {
         record.preparationOwned = false;
+        record.downloadController?.abort(failure('MODEL_ASSET_PROJECTION_RELEASED', 'Model asset preparation was released.'));
         publish(record);
         return cleanup(record);
     }
 
+    function release(id) {
+        const record = projections.get(id);
+        return record ? releasePreparation(record) : Promise.resolve({id, state: 'released'});
+    }
+
     async function failedOperation(record, error) {
         record.error = error;
         record.state = 'error';
@@ -165,19 +172,98 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
         const member = record.members[memberIndex];
         if (!member) throw new TypeError('Select an existing model asset member.');
         try {
-            const task = member.tail.then(async function appendOriginalContent() {
-                request.signal.throwIfAborted();
-                if (!record.preparationOwned) throw failure('MODEL_ASSET_PROJECTION_RELEASED', 'Model asset preparation was released.');
-                await member.handle.appendFile(Buffer.from(contentBase64, 'base64'));
-                request.signal.throwIfAborted();
-            });
-            // File writes remain ordered; unrelated member files write independently.
-            member.tail = task.catch(function observeWriteFailure() {});
-            await task;
+            await appendMemberContent(record, member, Buffer.from(contentBase64, 'base64'), request.signal);
             return {id, memberIndex, written: true};
         } catch (error) { return failedOperation(record, error); }
     }
 
+    function appendMemberContent(record, member, content, signal) {
+        const task = member.tail.then(async function appendOriginalContent() {
+            signal.throwIfAborted();
+            if (!record.preparationOwned) throw failure('MODEL_ASSET_PROJECTION_RELEASED', 'Model asset preparation was released.');
+            await member.handle.appendFile(content);
+            signal.throwIfAborted();
+        });
+        // File writes remain ordered; unrelated member files write independently.
+        member.tail = task.catch(function observeWriteFailure() {});
+        return task;
+    }
+
+    async function prepare({id, workingDirectory, members, signal, onProgress} = {}) {
+        const controller = new AbortController();
+        const operationSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
+        const request = {signal: operationSignal};
+        let record;
+        let completed = 0;
+        let firstFailure;
+
+        async function progress(phase, memberIndex) {
+            operationSignal.throwIfAborted();
+            await onProgress?.({
+                phase, completed, total: members.length, unit: 'files',
+                ...(memberIndex === undefined ? {} : {memberIndex, path: members[memberIndex].path})
+            });
+            operationSignal.throwIfAborted();
+        }
+
+        async function downloadMember(member, memberIndex) {
+            try {
+                await progress('download', memberIndex);
+                const response = await fetch(member.url, {signal: operationSignal});
+                if (!response.ok) {
+                    const error = failure('MODEL_ASSET_DOWNLOAD_FAILED', `Model asset request failed: ${response.status} ${response.statusText}`);
+                    error.url = member.url;
+                    error.status = response.status;
+                    error.response = await response.text();
+                    throw error;
+                }
+                if (!response.body) throw failure('MODEL_ASSET_DOWNLOAD_FAILED', `Model asset response has no content stream: ${member.url}`);
+                for await (const content of response.body) {
+                    operationSignal.throwIfAborted();
+                    await appendMemberContent(record, record.members[memberIndex], content, operationSignal);
+                    await progress('download', memberIndex);
+                }
+                completed += 1;
+                await progress('download', memberIndex);
+            } catch (error) {
+                firstFailure ??= error;
+                controller.abort(error);
+                throw error;
+            }
+        }
+
+        try {
+            await openProjection({id, workingDirectory, members}, request);
+            record = requireProjection(id);
+            operationSignal.throwIfAborted();
+            if (!record.preparationOwned || closing) {
+                throw failure('MODEL_ASSET_PROJECTION_RELEASED', 'Model asset preparation was released.');
+            }
+            record.downloadController = controller;
+            // Cleanup joins this transfer-only task. Its rejection handler below
+            // may release preparation without waiting on itself.
+            record.downloading = Promise.resolve().then(async function downloadMembers() {
+                await progress('open');
+                const results = await Promise.allSettled(members.map(downloadMember));
+                const errors = results.filter(function rejected(result) { return result.status === 'rejected'; })
+                    .map(function reason(result) { return result.reason; });
+                if (errors.length > 1) {
+                    throw new AggregateError(errors, 'Downloading model asset members failed.', {cause: firstFailure});
+                }
+                if (errors.length) throw errors[0];
+            });
+            await record.downloading;
+            await progress('complete');
+            const projection = await completeProjection({id}, request);
+            await progress('ready');
+            return projection;
+        } catch (error) {
+            controller.abort(error);
+            if (record) return failedOperation(record, error);
+            throw error;
+        }
+    }
+
     async function completeProjection({id}, request) {
         const record = requireProjection(id);
         await record.opening;
@@ -245,7 +331,7 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
     }
 
     return {
-        name: 'model-assets', current, retain,
+        name: 'model-assets', current, prepare, retain, release,
         start(serviceContext) { context = serviceContext; },
         methods: {
             'modelAssets.open': openProjection,
@@ -255,8 +341,7 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
             'modelAssets.release': {
                 lifetime: 'service',
                 handle({id}) {
-                    const record = projections.get(id);
-                    return record ? releasePreparation(record) : {id, state: 'released'};
+                    return release(id);
                 }
             }
         },

package/src/git-identity.mjs

@@ -3,6 +3,7 @@ import Is from 'strong-type';
 import {ArcaneError, throwIfAborted} from './errors.mjs';
 import {runProcess} from './process.mjs';
 import {createEventQueue} from './event-queue.mjs';
+import {readGitText} from './repository.mjs';
 
 const is = new Is(false);
 
@@ -16,67 +17,30 @@ export async function readGitIdentity({directory, signal, onEvent, run = runProc
         const args = ['config'];
         if (scope !== 'effective') args.push(`--${scope}`);
         args.push('--includes', '--null', '--get-regexp', '^(user[.](name|email)|github[.]user)$');
-        const chunks = [];
-        try {
-            const result = await run('git', args, {
-                cwd, signal, onEvent: events.send, allowNonzero: true,
-                outputEncoding: {stdout: null},
-                captureOutput: {stdout: false},
-                emitOutputEvents: {stdout: false},
-                // Git configuration may contain non-UTF-8 values. Keep its raw
-                // output until close rather than silently replacing characters.
-                onOutput: function observeOutput({stream, chunk}) {
-                    if (stream === 'stdout') chunks.push(chunk);
-                }
-            });
-            try { throwIfAborted(signal); }
-            catch (error) { error.details = result; throw error; }
-            const identity = {name: null, email: null, githubUser: null};
-            if (result.code === 1) return identity;
-            if (result.code !== 0) {
-                throw new ArcaneError('ARCANE_OPERATION_FAILED',
-                    `Git ${scope} identity lookup exited with code ${String(result.code)}.`, {details: result});
-            }
-            let output;
-            try { output = new TextDecoder('utf-8', {fatal: true, ignoreBOM: true}).decode(Buffer.concat(chunks)); }
-            catch (cause) {
-                throw new ArcaneError('ARCANE_GIT_IDENTITY_NOT_TEXT',
-                    `Git ${scope} identity configuration cannot be represented as UTF-8 text.`, {cause, details: result});
-            }
-            const fields = {'user.name': 'name', 'user.email': 'email', 'github.user': 'githubUser'};
-            // Git's NUL record framing preserves complete values, including newlines
-            // and empty strings. The last occurrence is the effective scalar value.
-            for (const record of output.split('\0')) {
-                if (record === '') continue;
-                const separator = record.indexOf('\n');
-                const key = separator === -1 ? record : record.slice(0, separator);
-                if (Object.hasOwn(fields, key)) {
-                    identity[fields[key]] = separator === -1 ? '' : record.slice(separator + 1);
-                }
-            }
-            return identity;
-        } catch (error) {
-            // Keep the process error and its native details intact. Raw stdout
-            // belongs to this reader because the process owner did not decode it.
-            const rawStdout = Buffer.concat(chunks);
-            let attached = false;
-            let attachmentError;
-            try {
-                if (error !== null && (is.object(error) || is.function(error)) && !('rawStdout' in error)) {
-                    Object.defineProperty(error, 'rawStdout', {
-                        value: rawStdout, writable: true, enumerable: true, configurable: true
-                    });
-                    attached = true;
-                }
-            } catch (cause) { attachmentError = cause; }
-            if (attached) throw error;
-            const failure = new ArcaneError(error?.code ?? 'ARCANE_OPERATION_FAILED',
-                error instanceof Error ? error.message : String(error),
-                {cause: error, details: error?.details, exitCode: error?.exitCode});
+        const {result, output, rawStdout} = await readGitText(args, {
+            cwd, signal, onEvent: events.send, run, allowNonzero: true,
+            notTextCode: 'ARCANE_GIT_IDENTITY_NOT_TEXT', label: `Git ${scope} identity configuration`
+        });
+        const identity = {name: null, email: null, githubUser: null};
+        if (result.code === 1) return identity;
+        if (result.code !== 0) {
+            const failure = new ArcaneError('ARCANE_OPERATION_FAILED',
+                `Git ${scope} identity lookup exited with code ${String(result.code)}.`, {details: result});
             failure.rawStdout = rawStdout;
-            if (attachmentError) failure.attachmentError = attachmentError;
             throw failure;
         }
+        const fields = {'user.name': 'name', 'user.email': 'email', 'github.user': 'githubUser'};
+        // Git's NUL record framing preserves complete values, including newlines
+        // and empty strings. The last occurrence is the effective scalar value.
+        for (const record of output.split('\0')) {
+            if (record === '') continue;
+            const separator = record.indexOf('\n');
+            const key = separator === -1 ? record : record.slice(0, separator);
+            if (Object.hasOwn(fields, key)) {
+                identity[fields[key]] = separator === -1 ? '' : record.slice(separator + 1);
+            }
+        }
+        return identity;
     }
 
     const scopes = cwd === undefined ? ['global'] : ['global', 'local', 'effective'];

package/src/local-ai/decision-tokenizer-worker.mjs

@@ -0,0 +1,31 @@
+import {readFile} from 'node:fs/promises';
+import {parentPort, workerData} from 'node:worker_threads';
+import {Tokenizer} from '@huggingface/tokenizers';
+import {createDecisionInputs, encodeDecisionRows} from '../../browser-runtime/ai/decision-runtime.mjs';
+import {serializeCoreError} from '../../browser-runtime/core/contracts.mjs';
+
+// Parsing and tokenization run here, not on Core's request/rendering thread.
+try {
+    const [tokenizerText, configurationText] = await Promise.all([
+        readFile(workerData.tokenizerPath, 'utf8'),
+        readFile(workerData.tokenizerConfigPath, 'utf8')
+    ]);
+    const tokenizer = new Tokenizer(JSON.parse(tokenizerText), JSON.parse(configurationText));
+    const interfaceAdapter = {
+        encode(text, options) { return tokenizer.encode(text, options).ids; },
+        convert_tokens_to_ids(token) { return tokenizer.token_to_id(token); }
+    };
+    parentPort.on('message', function encodeRequest({id, rows}) {
+        try {
+            const {encoded, pad} = encodeDecisionRows(interfaceAdapter, rows, 'laya');
+            const inputs = createDecisionInputs(encoded, pad);
+            parentPort.postMessage({id, result: inputs});
+        } catch (error) {
+            parentPort.postMessage({id, error: serializeCoreError(error)});
+        }
+    });
+    parentPort.postMessage({id: 0, result: {ready: true}});
+} catch (error) {
+    parentPort.postMessage({id: 0, error: serializeCoreError(error)});
+    parentPort.close();
+}

package/src/local-ai/decision-tokenizer.mjs

@@ -0,0 +1,109 @@
+import {Worker} from 'node:worker_threads';
+import {CoreError} from '../../browser-runtime/core/contracts.mjs';
+
+function cancelled() {
+    return new CoreError({name: 'AbortError', code: 'ARCANE_AI_REQUEST_ABORTED', message: 'Decision tokenization was cancelled.'});
+}
+
+/** One tokenizer worker owned by one native model activation. */
+export function createDecisionTokenizer({tokenizerPath, tokenizerConfigPath, signal, onError}) {
+    signal?.throwIfAborted();
+    const pending = new Map();
+    let nextId = 0;
+    let closing;
+    let terminalError;
+    let exited = false;
+    const worker = new Worker(new URL('./decision-tokenizer-worker.mjs', import.meta.url), {
+        workerData: {tokenizerPath, tokenizerConfigPath}
+    });
+    let resolveExit;
+    const exit = new Promise(function tokenizerExit(resolve) { resolveExit = resolve; });
+
+    function rejectPending(error) {
+        terminalError ??= error;
+        for (const operation of pending.values()) operation.reject(error);
+        pending.clear();
+    }
+
+    function operation(id) {
+        return new Promise(function retainOperation(resolve, reject) {
+            pending.set(id, {resolve, reject});
+        });
+    }
+
+    const ready = operation(0);
+    // The native owner observes ready together with ONNX load. Retain an error
+    // handler immediately when the other owner is still starting.
+    ready.catch(function observeTokenizerLoadFailure() {});
+    worker.on('message', function tokenizerMessage(message) {
+        const request = pending.get(message.id);
+        if (!request) return;
+        pending.delete(message.id);
+        if (Object.hasOwn(message, 'error')) {
+            const error = new CoreError(message.error);
+            request.reject(error);
+            if (message.id === 0) failed(error);
+        }
+        else request.resolve(message.result);
+    });
+    worker.on('error', failed);
+    worker.on('messageerror', failed);
+    worker.once('exit', function tokenizerExited(code) {
+        exited = true;
+        signal?.removeEventListener('abort', abort);
+        const error = terminalError ?? new CoreError({
+            code: 'ARCANE_DECISION_TOKENIZER_CLOSED',
+            message: `The decision tokenizer worker exited with code ${code}.`
+        });
+        if (!closing) failed(error);
+        else rejectPending(error);
+        resolveExit(code);
+    });
+
+    function failed(error) {
+        const firstFailure = !terminalError && !closing;
+        rejectPending(error);
+        close(error).catch(function reportTokenizerFailure(value) { console.error(value); });
+        if (firstFailure) onError?.(error);
+    }
+
+    function close(reason = cancelled()) {
+        if (closing) return closing;
+        rejectPending(reason);
+        signal?.removeEventListener('abort', abort);
+        closing = Promise.resolve().then(async function releaseTokenizer() {
+            let terminationError;
+            try { if (!exited) await worker.terminate(); }
+            catch (error) {
+                terminationError = error;
+                console.error('Terminating the decision tokenizer failed.', error);
+            }
+            // A rejected termination request is not evidence that the worker
+            // has stopped reading its files. Retain ownership until real exit.
+            await exit;
+            if (terminationError) throw terminationError;
+        });
+        return closing;
+    }
+
+    function abort() {
+        close(signal.reason).catch(function reportTokenizerCloseFailure(error) { console.error(error); });
+    }
+    signal?.addEventListener('abort', abort, {once: true});
+    if (signal?.aborted) abort();
+
+    async function encode(rows) {
+        await ready;
+        if (terminalError || closing || exited) throw terminalError ?? cancelled();
+        const id = ++nextId;
+        const result = operation(id);
+        try { worker.postMessage({id, rows}); }
+        catch (error) {
+            pending.get(id).reject(error);
+            pending.delete(id);
+        }
+        return result;
+    }
+
+    return {ready, encode, close, get exited() { return exited; }};
+}