arcane-os 0.93.0 → 0.95.0

package/src/core/services/speech.mjs

@@ -44,8 +44,8 @@ function waitForLoad(task, signal) {
 /** Hosts supply selected engines; this service owns their independent lifetimes. */
 export function createSpeechService({stt, tts, signal} = {}) {
     const roles = {
-        stt: {engine: stt, operation: 'transcribe', state: null, loading: null, unsubscribe: null},
-        tts: {engine: tts, operation: 'synthesize', state: null, loading: null, unsubscribe: null}
+        stt: {engine: stt, operation: 'transcribe', state: null, loading: null, changing: null, unsubscribe: null},
+        tts: {engine: tts, operation: 'synthesize', state: null, loading: null, changing: null, unsubscribe: null}
     };
     for (const [name, role] of Object.entries(roles)) {
         if (role.engine == null) {
@@ -68,6 +68,8 @@ export function createSpeechService({stt, tts, signal} = {}) {
         {source: 'core-speech', eventTypes: ['speech.state']}
     );
     const jobs = new Set();
+    const streams = new Map();
+    const syntheses = new Map();
     const publications = [];
     let context;
     let started = false;
@@ -91,8 +93,10 @@ export function createSpeechService({stt, tts, signal} = {}) {
             ttsEngine: roles.tts.state.providerId ?? '',
             sttEngine: roles.stt.state.providerId ?? '',
             roles: {
-                stt: {...roles.stt.state, available: transcriptionAvailable},
-                tts: {...roles.tts.state, available: synthesisAvailable}
+                stt: {...roles.stt.state, available: transcriptionAvailable,
+                    supportsStreaming: is.function(roles.stt.engine?.openStream)},
+                tts: {...roles.tts.state, available: synthesisAvailable,
+                    supportsStreaming: is.function(roles.tts.engine?.synthesizeStream)}
             },
             closed
         };
@@ -124,6 +128,36 @@ export function createSpeechService({stt, tts, signal} = {}) {
         publish();
     }
 
+    function observeRoleTask(name, task, startup = false) {
+        const role = roles[name];
+        role.changing = task;
+        // Explicit load cancellation belongs to that caller's wait, not another
+        // inference request. Only the service-owned startup is a shared wait.
+        role.loading = startup ? task : null;
+        jobs.add(task);
+        function settled(error, failed) {
+            jobs.delete(task);
+            // An older startup or activation must not replace the selected state.
+            if (role.loading === task) role.loading = null;
+            if (role.changing !== task) return;
+            role.changing = null;
+            if (closed) return;
+            if (failed && startup && !closing) {
+                roleFailed(name, error);
+            } else {
+                role.state = role.engine.current();
+                publish();
+            }
+        }
+        task.then(
+            function speechRoleSettled() { settled(undefined, false); },
+            function speechRoleFailed(error) { settled(error, true); }
+        ).catch(function speechRoleObserverFailed(error) {
+            console.error('Speech role state observer failed.', error);
+        });
+        return task;
+    }
+
     function startRole(name) {
         const role = roles[name];
         if (!role.engine || closing) {
@@ -142,34 +176,60 @@ export function createSpeechService({stt, tts, signal} = {}) {
             roleFailed(name, error);
             return;
         }
+        // Replay can synchronously select an activation or close the service.
+        if (role.changing || closing || closed) return;
         const task = Promise.resolve().then(
             async function loadSpeechEngine() {
                 lifetimeSignal.throwIfAborted();
-                await role.engine.load(
+                if (role.changing !== task) return;
+                return role.engine.load(
                     {signal: lifetimeSignal}
                 );
-                if (!closed) {
-                    role.state = role.engine.current();
-                    publish();
-                }
-            }
-        );
-        role.loading = task;
-        task.then(
-            function speechEngineLoaded() {
-                role.loading = null;
-            },
-            function speechEngineLoadFailed(error) {
-                role.loading = null;
-                if (!closing) {
-                    roleFailed(name, error);
-                }
-            }
-        ).catch(
-            function speechLoadObserverFailed(error) {
-                console.error('Speech load state observer failed.', error);
             }
         );
+        observeRoleTask(name, task, true);
+    }
+
+    function changeRole(operation, parameters, request) {
+        if (closing || closed) {
+            throw failure('CORE_CLOSING', 'The speech service is closing.');
+        }
+        if (!parameters || !is.object(parameters) || is.array(parameters)
+            || !['stt', 'tts'].includes(parameters.role)) {
+            throw failure('SPEECH_ROLE_INVALID', 'Select the stt or tts speech role.');
+        }
+        const {role: name, ...selection} = parameters;
+        const role = roles[name];
+        const operationSignal = request.signal
+            ? AbortSignal.any([lifetimeSignal, request.signal]) : lifetimeSignal;
+        operationSignal.throwIfAborted();
+        if (!role.engine) {
+            throw failure('SPEECH_ENGINE_UNAVAILABLE', `No ${name} speech engine is configured.`);
+        }
+        if (!is.function(role.engine[operation])) {
+            throw failure('SPEECH_OPERATION_UNAVAILABLE', `The ${name} speech engine does not support ${operation}().`);
+        }
+        let acknowledgementFailed = false;
+        let acknowledgementError;
+        const task = Promise.resolve().then(async function changeSpeechRole() {
+            if (acknowledgementFailed) throw acknowledgementError;
+            operationSignal.throwIfAborted();
+            const result = await role.engine[operation]({...selection, signal: operationSignal});
+            operationSignal.throwIfAborted();
+            return result;
+        });
+        // Retain ownership before an acknowledgement observer can close or reenter.
+        observeRoleTask(name, task);
+        try {
+            request.emit?.('speech.progress', {
+                requestId: request.requestId, role: name, operation,
+                status: 'Thinking', progress: {phase: 'accepted'}
+            });
+        } catch (error) {
+            acknowledgementFailed = true;
+            acknowledgementError = error;
+        }
+        return task;
     }
 
     function invoke(name, parameters, request) {
@@ -229,6 +289,201 @@ export function createSpeechService({stt, tts, signal} = {}) {
         }
     }
 
+    function streamRole(name, operation, request, streamId) {
+        if (closing || closed) throw failure('CORE_CLOSING', 'The speech service is closing.');
+        const role = roles[name];
+        if (!is.function(role.engine?.[operation])) {
+            throw failure('SPEECH_OPERATION_UNAVAILABLE', `The ${name} speech engine does not support ${operation}().`);
+        }
+        const operationSignal = request.signal
+            ? AbortSignal.any([lifetimeSignal, request.signal]) : lifetimeSignal;
+        operationSignal.throwIfAborted();
+        request.emit?.('speech.progress', {
+            streamId, requestId: request.requestId, role: name, status: 'Thinking', progress: {phase: 'accepted'}
+        });
+        return {role, operationSignal};
+    }
+
+    function retainStreamTask(task) {
+        jobs.add(task);
+        function releaseSpeechStreamTask() { jobs.delete(task); }
+        task.then(releaseSpeechStreamTask, releaseSpeechStreamTask);
+        return task;
+    }
+
+    function openStream({streamId, options = {}} = {}, request) {
+        const activation = streamRole('stt', 'openStream', request, streamId);
+        const role = activation.role;
+        const controller = new AbortController();
+        const operationSignal = AbortSignal.any([activation.operationSignal, controller.signal]);
+        if (!is.string(streamId) || !streamId) throw new TypeError('A speech stream requires a streamId.');
+        if (streams.has(streamId)) throw failure('SPEECH_STREAM_ACTIVE', 'The speech stream is already open.');
+        const record = {stream: null, controller, queue: Promise.resolve(), ending: false, cancellation: null, task: null};
+        streams.set(streamId, record);
+
+        function cancelNativeStream() {
+            record.ending = true;
+            if (record.stream && !record.cancellation) {
+                record.cancellation = Promise.resolve().then(function cancelOwnedNativeStream() {
+                    return record.stream.cancel();
+                });
+            }
+            return record.cancellation;
+        }
+        function abortedStream() {
+            cancelNativeStream()?.catch(function reportNativeStreamCancellation(error) {
+                console.error('Native speech stream cancellation failed.', error);
+            });
+        }
+        record.cancel = cancelNativeStream;
+        operationSignal.addEventListener('abort', abortedStream, {once: true});
+        const task = (async function ownTranscriptionStream() {
+            try {
+                if (role.loading) await waitForLoad(role.loading, operationSignal);
+                operationSignal.throwIfAborted();
+                record.stream = await role.engine.openStream({
+                    ...options,
+                    signal: operationSignal,
+                    onUpdate: function reportTranscriptionUpdate(result) {
+                        if (!operationSignal.aborted) request.emit?.('speech.transcription', {streamId, result});
+                    }
+                });
+                if (operationSignal.aborted) {
+                    await cancelNativeStream();
+                    operationSignal.throwIfAborted();
+                }
+                request.emit?.('speech.opened', {
+                    streamId,
+                    ...(options.sampleRate === undefined ? {} : {sampleRate: options.sampleRate})
+                });
+                const result = await record.stream.completion;
+                operationSignal.throwIfAborted();
+                return result;
+            } catch (error) {
+                try { await cancelNativeStream(); } catch (cleanupError) {
+                    throw new AggregateError([error, cleanupError], 'Speech stream and cleanup failed.');
+                }
+                throw error;
+            } finally {
+                operationSignal.removeEventListener('abort', abortedStream);
+                if (streams.get(streamId) === record) streams.delete(streamId);
+            }
+        })();
+        record.task = task;
+        return retainStreamTask(task);
+    }
+
+    function streamOperation(operation, {streamId, audio} = {}, request) {
+        request.signal?.throwIfAborted();
+        const record = streams.get(streamId);
+        if (operation === 'cancel') {
+            if (!record) return {streamId, cancelled: true};
+            record.controller.abort();
+            return record.task.then(
+                function streamFinishedBeforeCancellation() { return {streamId, cancelled: true}; },
+                function streamCancellationSettled(error) {
+                    if (error?.name !== 'AbortError') throw error;
+                    return {streamId, cancelled: true};
+                }
+            );
+        }
+        if (!record?.stream) throw failure('SPEECH_STREAM_UNAVAILABLE', 'The speech stream is not open.');
+        if (record.ending) throw failure('SPEECH_STREAM_ENDING', 'The speech stream is finishing.');
+        if (operation === 'finish') record.ending = true;
+        const task = record.queue.then(function performSpeechStreamOperation() {
+            request.signal?.throwIfAborted();
+            if (operation === 'push') return record.stream.push(decodeFloatAudio(audio));
+            return record.stream[operation]();
+        });
+        record.queue = task;
+        task.catch(function cancelFailedStreamOperation() {
+            record.cancel()?.catch(function reportStreamOperationCleanup(error) {
+                console.error('Failed speech stream cleanup failed.', error);
+            });
+        });
+        return retainStreamTask(task);
+    }
+
+    function synthesizeStream({streamId, request: payload} = {}, request) {
+        const activation = streamRole('tts', 'synthesizeStream', request, streamId);
+        const role = activation.role;
+        const controller = new AbortController();
+        const operationSignal = AbortSignal.any([activation.operationSignal, controller.signal]);
+        if (!is.string(streamId) || !streamId) throw new TypeError('Speech synthesis requires a streamId.');
+        if (syntheses.has(streamId)) throw failure('SPEECH_STREAM_ACTIVE', 'The speech synthesis is already active.');
+        const record = {controller, task: null, sequence: 0, acknowledgements: new Map()};
+        syntheses.set(streamId, record);
+        function cancelAudioDelivery() {
+            for (const acknowledgement of record.acknowledgements.values()) acknowledgement.reject(operationSignal.reason);
+            record.acknowledgements.clear();
+        }
+        operationSignal.addEventListener('abort', cancelAudioDelivery, {once: true});
+        const task = (async function ownSynthesisStream() {
+            try {
+                if (role.loading) await waitForLoad(role.loading, operationSignal);
+                operationSignal.throwIfAborted();
+                const result = await role.engine.synthesizeStream(payload, {
+                    signal: operationSignal,
+                    requestId: request.requestId,
+                    onProgress: function reportSynthesisProgress(progress) {
+                        if (!operationSignal.aborted) request.emit?.('speech.progress', {
+                            streamId, requestId: request.requestId, role: 'tts', status: 'Thinking', progress
+                        });
+                    },
+                    onAudio: async function reportSynthesisAudio(audio) {
+                        operationSignal.throwIfAborted();
+                        const sequence = ++record.sequence;
+                        const consumed = new Promise(function waitForAudioConsumer(resolve, reject) {
+                            record.acknowledgements.set(sequence, {resolve, reject});
+                        });
+                        // The native producer waits here, never the event parser.
+                        // Cancellation releases this wait before joining native work.
+                        try {
+                            const pcm = audio.pcm;
+                            const encoded = Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+                            request.emit?.('speech.audio', {
+                                streamId, sequence,
+                                audio: {...audio, pcm: {encoding: 'base64', data: encoded.toString('base64')}}
+                            });
+                            await consumed;
+                            operationSignal.throwIfAborted();
+                        } finally {
+                            record.acknowledgements.delete(sequence);
+                        }
+                    }
+                });
+                operationSignal.throwIfAborted();
+                return result;
+            } finally {
+                operationSignal.removeEventListener('abort', cancelAudioDelivery);
+                if (syntheses.get(streamId) === record) syntheses.delete(streamId);
+            }
+        })();
+        record.task = task;
+        return retainStreamTask(task);
+    }
+
+    function synthesisControl(operation, {streamId, sequence} = {}, request) {
+        request.signal?.throwIfAborted();
+        const record = syntheses.get(streamId);
+        if (operation === 'cancel') {
+            if (!record) return {streamId, cancelled: true};
+            record.controller.abort();
+            return record.task.then(
+                function synthesisFinishedBeforeCancellation() { return {streamId, cancelled: true}; },
+                function synthesisCancellationSettled(error) {
+                    if (error !== record.controller.signal.reason) throw error;
+                    return {streamId, cancelled: true};
+                }
+            );
+        }
+        const acknowledgement = record?.acknowledgements.get(sequence);
+        if (!acknowledgement) throw failure('SPEECH_AUDIO_UNAVAILABLE', 'That speech audio contribution is not awaiting consumption.');
+        record.acknowledgements.delete(sequence);
+        acknowledgement.resolve();
+        return {streamId, sequence};
+    }
+
     function subscribe(listener, {replay = true, signal: subscriptionSignal} = {}) {
         if (!is.function(listener)) {
             throw new TypeError('A speech state listener must be a function.');
@@ -340,11 +595,39 @@ export function createSpeechService({stt, tts, signal} = {}) {
         },
         methods: {
             'speech.status': current,
+            'speech.load': function load(parameters, request = {}) {
+                return changeRole('load', parameters, request);
+            },
+            'speech.unload': function unload(parameters, request = {}) {
+                return changeRole('unload', parameters, request);
+            },
             'speech.transcribe': function transcribe(parameters, request = {}) {
                 return invoke('stt', parameters, request);
             },
             'speech.synthesize': function synthesize(parameters, request = {}) {
                 return invoke('tts', parameters, request);
+            },
+            // The open call stays pending so normal connection cancellation owns
+            // the native stream; controls never extend it to service lifetime.
+            'speech.open': openStream,
+            'speech.push': function push(parameters, request) {
+                return streamOperation('push', parameters, request);
+            },
+            'speech.endpoint': function endpoint(parameters, request) {
+                return streamOperation('forceEndpoint', parameters, request);
+            },
+            'speech.finish': function finish(parameters, request) {
+                return streamOperation('finish', parameters, request);
+            },
+            'speech.cancel': function cancel(parameters, request) {
+                return streamOperation('cancel', parameters, request);
+            },
+            'speech.synthesizeStream': synthesizeStream,
+            'speech.audioConsumed': function audioConsumed(parameters, request) {
+                return synthesisControl('consumed', parameters, request);
+            },
+            'speech.cancelSynthesis': function cancelSynthesis(parameters, request) {
+                return synthesisControl('cancel', parameters, request);
             }
         },
         drain: close,
@@ -358,4 +641,16 @@ export function createSpeechService({stt, tts, signal} = {}) {
     };
 }
 
+function decodeFloatAudio(audio) {
+    if (audio?.encoding !== 'f32le' || !is.string(audio.data)) {
+        throw new TypeError('Core streaming audio requires {encoding: "f32le", data: BASE64}.');
+    }
+    const encoded = Buffer.from(audio.data, 'base64');
+    if (encoded.length % 4 !== 0) throw new TypeError('The f32le transport ends during a sample.');
+    const samples = new Float32Array(encoded.length / 4);
+    const bits = new Uint32Array(samples.buffer);
+    for (let index = 0; index < samples.length; index += 1) bits[index] = encoded.readUInt32LE(index * 4);
+    return samples;
+}
+
 export default createSpeechService;

package/src/local-ai/config.mjs

@@ -8,7 +8,17 @@ import {SDXL_BASE_1_0_MODEL} from './image-models/sdxl.mjs';
 import {normalizeWhisperRuntimeRequirement} from './whisper/config.mjs';
 
 const is = new Is(false);
-export const LOCAL_AI_RUNTIME_IDS = ['llama.cpp', 'ollama', 'onnx', 'stable-diffusion.cpp', 'whisper.cpp'];
+export const LOCAL_AI_RUNTIME_IDS = ['llama.cpp', 'ollama', 'onnx', 'stable-diffusion.cpp', 'whisper.cpp', 'kokoro-native'];
+
+/** Describe the selected helper without acquiring it or selecting model assets. */
+export function normalizeKokoroRuntimeRequirement(record) {
+    for (const field of ['helperRoot', 'helperExecutable', 'espeakDataDirectory', 'libraryDirectory']) {
+        if (record[field] !== undefined && (!is.string(record[field]) || record[field] === '')) {
+            throw new ArcaneError(ERROR_CODES.usage, `kokoro-native ${field} must be a nonempty path.`);
+        }
+    }
+    return {...record};
+}
 
 /** Select a llama distribution without discovering hardware or acquiring it. */
 export function normalizeLlamaRuntimeRequirement(record) {
@@ -79,6 +89,7 @@ export function normalizeLocalAIConfig(value) {
         if (record.id === 'llama.cpp') return normalizeLlamaRuntimeRequirement(record);
         if (record.id === 'stable-diffusion.cpp') return normalizeImageRuntimeRequirement(record);
         if (record.id === 'whisper.cpp') return normalizeWhisperRuntimeRequirement(record);
+        if (record.id === 'kokoro-native') return normalizeKokoroRuntimeRequirement(record);
         return {...record};
     });
     return {...value, runtimes};

package/src/local-ai/dev.mjs

@@ -9,7 +9,9 @@ import {normalizeLocalAIConfig} from './config.mjs';
 import {discoverLocalAIRuntimes} from './discover.mjs';
 import {ensureLocalAIRuntimes} from './install.mjs';
 import {createWhisperRuntime} from './whisper/index.mjs';
+import {createNativeKokoroRuntime} from './kokoro/index.mjs';
 import {createExecutionDeviceCatalog} from './execution-devices.mjs';
+import {ArcaneError, ERROR_CODES} from '../errors.mjs';
 
 /** Local engines share the ordinary app-service development transport. */
 export function createDevelopmentLocalAI({
@@ -24,7 +26,7 @@ export function createDevelopmentLocalAI({
     const localConfig = {
         ...selected,
         runtimes: selected.runtimes.filter(function selectedChatRuntime(requirement) {
-            return requirement.id !== 'stable-diffusion.cpp' && requirement.id !== 'whisper.cpp';
+            return !['stable-diffusion.cpp', 'whisper.cpp', 'kokoro-native'].includes(requirement.id);
         })
     };
     const service = createLocalAIService(
@@ -48,7 +50,7 @@ export function createDevelopmentLocalAI({
         return requirement.id === 'stable-diffusion.cpp';
     });
     if (imageSelected || selected.runtimes.some(function selectedProjectedRuntime(requirement) {
-        return ['onnx', 'llama.cpp'].includes(requirement.id);
+        return ['onnx', 'llama.cpp', 'kokoro-native'].includes(requirement.id);
     })) {
         modelAssets = createModelAssetService({appRoot});
         definitions.push(modelAssets);
@@ -71,8 +73,13 @@ export function createDevelopmentLocalAI({
     const whisperRequirement = selected.runtimes.find(function selectedWhisperRuntime(requirement) {
         return requirement.id === 'whisper.cpp';
     });
+    const kokoroRequirement = selected.runtimes.find(function selectedKokoroRuntime(requirement) {
+        return requirement.id === 'kokoro-native';
+    });
+    let stt;
+    let tts;
     if (whisperRequirement) {
-        const stt = createWhisperRuntime(
+        stt = createWhisperRuntime(
             {
                 modelId: whisperRequirement.modelId,
                 temporaryDirectory: path.join(appRoot, '.arcane', 'speech', 'whisper'),
@@ -85,7 +92,49 @@ export function createDevelopmentLocalAI({
                 }
             }
         );
-        speechService = createSpeechService({stt, signal});
+    }
+    if (kokoroRequirement) {
+        tts = createNativeKokoroRuntime({
+            modelId: kokoroRequirement.modelId,
+            model: kokoroRequirement.model,
+            revision: kokoroRequirement.revision,
+            dtype: kokoroRequirement.dtype,
+            paths: kokoroRequirement.paths,
+            assetProjectionId: kokoroRequirement.assetProjectionId,
+            resourcePaths: kokoroRequirement.resourcePaths,
+            sessionOptions: kokoroRequirement.sessionOptions,
+            executionPreference: kokoroRequirement.executionPreference,
+            executionTarget: kokoroRequirement.executionTarget,
+            modelAssets, signal, onEvent,
+            async prepare({signal: activationSignal}) {
+                if (!localConfig.runtimes.some(function selectedONNX(requirement) { return requirement.id === 'onnx'; })) {
+                    throw new ArcaneError(ERROR_CODES.targetUnavailable, 'Native Kokoro requires an explicitly selected onnx runtime.');
+                }
+                const preparation = new AbortController();
+                const preparationSignal = AbortSignal.any([activationSignal, preparation.signal]);
+                function preparationFailed(error) {
+                    preparation.abort(error);
+                    throw error;
+                }
+                // Helper acquisition and the existing ONNX owner's startup are
+                // independent. Join both before native activation can continue.
+                const outcomes = await Promise.allSettled([
+                    waitForKokoroONNX(preparationSignal).catch(preparationFailed),
+                    ensureLocalAIRuntimes({
+                        runtimes: [kokoroRequirement], directory, signal: preparationSignal, onEvent
+                    }).catch(preparationFailed)
+                ]);
+                const failures = outcomes.filter(function failed(result) { return result.status === 'rejected'; })
+                    .map(function cause(result) { return result.reason; });
+                if (failures.length === 1) throw failures[0];
+                if (failures.length) throw new AggregateError(failures, 'Native Kokoro preparation failed.');
+                preparationSignal.throwIfAborted();
+                return {onnx: outcomes[0].value, runtime: outcomes[1].value[0]};
+            }
+        });
+    }
+    if (stt || tts) {
+        speechService = createSpeechService({stt, tts, signal});
         definitions.push(speechService);
     }
     const core = createDevelopmentCore(
@@ -101,6 +150,24 @@ export function createDevelopmentLocalAI({
             }
         }
     );
+    function waitForKokoroONNX(activationSignal) {
+        activationSignal.throwIfAborted();
+        let cancel;
+        const cancelled = new Promise(function observePreparationCancellation(_resolve, reject) {
+            cancel = function stopWaitingForONNX() { reject(activationSignal.reason); };
+            activationSignal.addEventListener('abort', cancel, {once: true});
+        });
+        const ready = core.ready.then(async function existingONNXOwner(runtime) {
+            activationSignal.throwIfAborted();
+            const owner = await runtime.getService('local-ai');
+            activationSignal.throwIfAborted();
+            return owner.getONNXRuntime();
+        });
+        // Cancel this selected activation's wait without stopping the shared owner.
+        return Promise.race([ready, cancelled]).finally(function releaseONNXWait() {
+            activationSignal.removeEventListener('abort', cancel);
+        });
+    }
     function current() {
         return {
             ...core.current(),