arcane-os 0.94.0 → 0.96.0

package/runtime/arcane/modules/NativeSpeechPlayback.js

@@ -0,0 +1,304 @@
+import Is from 'strong-type';
+import {arcaneLogging} from 'arcane-os/logging';
+
+const is = new Is(false);
+
+/** Prepare exact native speech for the existing utterance queue, without starting synthesis. */
+export default function prepareNativeSpeechPlayback({speech, request, getAudioContext, signal = null}) {
+    if (!is.function(speech?.synthesizeStream) || !is.function(getAudioContext)) {
+        throw new TypeError('Native speech playback requires streaming speech and an AudioContext owner.');
+    }
+    return {
+        kind: 'native-speech',
+        input: request.input,
+        voice: request.voice,
+        language: request.languageCode ?? request.language_code ?? request.language,
+        speed: request.speed ?? 1,
+        play(options = {}) {
+            return startPlayback(speech, request, getAudioContext, signal, options);
+        }
+    };
+}
+
+function startPlayback(speech, request, getAudioContext, preparationSignal, {signal = null, onState = null} = {}) {
+    const controller = new AbortController();
+    const signals = new Set([preparationSignal, signal].filter(Boolean));
+    const segments = [];
+    const failures = [];
+    const completion = deferred();
+    const release = deferred();
+    let context = null;
+    let playbackState = 'queued';
+    let playbackError = null;
+    let paused = false;
+    let settled = false;
+    let synthesisSettled = true;
+    let releaseSettled = false;
+    let nextStart = 0;
+    let playbackReady = null;
+
+    function publish(nextState, error = playbackError) {
+        playbackState = nextState;
+        playbackError = error;
+        if (!is.function(onState)) return;
+        function observerFailed(cause) {
+            arcaneLogging.error('Native speech playback observer failed.', cause);
+        }
+        try { Promise.resolve(onState({state: nextState, error})).catch(observerFailed); }
+        catch (cause) { observerFailed(cause); }
+    }
+
+    function retainFailure(error) {
+        if (!failures.includes(error)) failures.push(error);
+        playbackError = failures.length === 1 ? failures[0]
+            : new AggregateError(failures, 'Native speech playback and cleanup failed.');
+    }
+
+    function releaseIfFinished() {
+        if (!settled || !synthesisSettled || releaseSettled) return;
+        releaseSettled = true;
+        if (failures.length) release.reject(playbackError);
+        else release.resolve();
+    }
+
+    function removeSegment(segment) {
+        const position = segments.indexOf(segment);
+        if (position >= 0) segments.splice(position, 1);
+        segment.buffer = null;
+        segment.ended.resolve();
+    }
+
+    function detachSource(segment, stop) {
+        const node = segment.node;
+        if (!node) return;
+        segment.node = null;
+        node.onended = null;
+        if (stop && segment.started) {
+            try { node.stop(); }
+            catch (error) { retainFailure(error); }
+        }
+        segment.started = false;
+        try { node.disconnect(); }
+        catch (error) { retainFailure(error); }
+    }
+
+    function wakePlayback() {
+        if (!settled && (paused || context?.state !== 'running')) return;
+        playbackReady?.resolve();
+        playbackReady = null;
+    }
+
+    function finish(completed, error = null) {
+        if (error) retainFailure(error);
+        if (settled) {
+            if (error) publish('error');
+            releaseIfFinished();
+            return;
+        }
+        settled = true;
+        for (const requestSignal of signals) requestSignal.removeEventListener('abort', stop);
+        context?.removeEventListener('statechange', contextChanged);
+        if (!completed && !controller.signal.aborted) {
+            controller.abort(error ?? cancellation());
+        }
+        for (const segment of [...segments]) {
+            detachSource(segment, true);
+            removeSegment(segment);
+        }
+        wakePlayback();
+        publish(playbackError ? 'error' : completed ? 'complete' : 'stopped');
+        if (playbackError) completion.reject(playbackError);
+        else completion.resolve(completed);
+        releaseIfFinished();
+    }
+
+    function updatePlaybackState() {
+        if (settled) return;
+        publish(paused ? 'paused'
+            : context?.state === 'running' && segments.length ? 'playing' : 'waiting-for-audio');
+    }
+
+    function contextChanged() {
+        if (settled) return;
+        if (context.state === 'closed') {
+            finish(false, playbackFailure('The shared audio output closed during native speech.'));
+            return;
+        }
+        wakePlayback();
+        updatePlaybackState();
+    }
+
+    function schedule(segment) {
+        if (settled || paused || segment.node) return;
+        const node = context.createBufferSource();
+        segment.node = node;
+        node.buffer = segment.buffer;
+        node.connect(context.destination);
+        segment.startAt = Math.max(context.currentTime, nextStart);
+        nextStart = segment.startAt + segment.buffer.duration - segment.offset;
+        node.onended = function segmentEnded() {
+            if (segment.node !== node || settled) return;
+            detachSource(segment, false);
+            removeSegment(segment);
+            if (failures.length) finish(false);
+            else if (synthesisSettled && !segments.length) finish(true);
+            else updatePlaybackState();
+        };
+        node.start(segment.startAt, segment.offset);
+        segment.started = true;
+    }
+
+    async function receiveAudio({pcm, format, sampleRate, channels}) {
+        try {
+            await playAudio(pcm, format, sampleRate, channels);
+        } catch (error) {
+            if (error !== controller.signal.reason) finish(false, error);
+            throw error;
+        }
+    }
+
+    async function playAudio(pcm, format, sampleRate, channels) {
+        controller.signal.throwIfAborted();
+        if (format !== 's16le' || !(pcm instanceof Uint8Array)) {
+            throw playbackFailure('Native speech requires signed 16-bit little-endian PCM.');
+        }
+        // PCM sample framing belongs to this audio transport, not request admission.
+        const frames = pcm.byteLength / (2 * channels);
+        if (!Number.isInteger(frames)) {
+            throw playbackFailure('Native speech returned an incomplete PCM frame.');
+        }
+        if (!frames) return;
+        const buffer = context.createBuffer(channels, frames, sampleRate);
+        const samples = new DataView(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+        for (let channel = 0; channel < channels; channel += 1) {
+            const output = buffer.getChannelData(channel);
+            for (let frame = 0; frame < frames; frame += 1) {
+                output[frame] = samples.getInt16((frame * channels + channel) * 2, true) / 32768;
+            }
+        }
+        const preceding = segments[segments.length - 1];
+        const segment = {buffer, offset: 0, startAt: 0, node: null, started: false, ended: deferred()};
+        segments.push(segment);
+        schedule(segment);
+        updatePlaybackState();
+        controller.signal.throwIfAborted();
+        if (paused || context.state !== 'running') {
+            playbackReady ??= deferred();
+            await playbackReady.promise;
+        }
+        // Keep the current and next chunk scheduled. The awaited callback applies
+        // playback backpressure without another utterance queue or a complete clip.
+        if (preceding) await preceding.ended.promise;
+        controller.signal.throwIfAborted();
+    }
+
+    function stop() {
+        if (settled) return false;
+        finish(false);
+        return playbackError === null;
+    }
+
+    function pause() {
+        if (settled || paused) return false;
+        paused = true;
+        const now = context?.currentTime ?? 0;
+        for (const segment of [...segments]) {
+            if (segment.node) {
+                segment.offset = Math.min(segment.buffer.duration,
+                    segment.offset + Math.max(0, now - segment.startAt));
+                detachSource(segment, true);
+            }
+            if (segment.offset >= segment.buffer.duration) removeSegment(segment);
+        }
+        nextStart = now;
+        if (failures.length) { finish(false); return false; }
+        if (synthesisSettled && !segments.length) { finish(true); return false; }
+        publish('paused');
+        return true;
+    }
+
+    function resume() {
+        if (settled || !paused) return false;
+        paused = false;
+        try {
+            nextStart = context?.currentTime ?? 0;
+            for (const segment of segments) schedule(segment);
+            wakePlayback();
+            updatePlaybackState();
+            return true;
+        } catch (error) {
+            finish(false, error);
+            return false;
+        }
+    }
+
+    const control = {
+        finished: completion.promise,
+        released: release.promise,
+        get state() { return playbackState; },
+        get error() { return playbackError; },
+        pause,
+        resume,
+        stop
+    };
+    for (const requestSignal of signals) requestSignal.addEventListener('abort', stop, {once: true});
+    publish('queued');
+    if ([...signals].some(function alreadyAborted(requestSignal) { return requestSignal.aborted; })) stop();
+    if (!settled) {
+        try {
+            context = getAudioContext();
+            if (context.state === 'closed') throw playbackFailure('The shared audio output is closed.');
+            context.addEventListener('statechange', contextChanged);
+            synthesisSettled = false;
+            updatePlaybackState();
+            const synthesis = Promise.resolve().then(function synthesizeOwnedUtterance() {
+                controller.signal.throwIfAborted();
+                return speech.synthesizeStream(request, {signal: controller.signal, onAudio: receiveAudio});
+            });
+            synthesis.then(function synthesisCompleted() {
+                synthesisSettled = true;
+                if (!settled && !segments.length) finish(true);
+                releaseIfFinished();
+            }, function synthesisFailed(error) {
+                synthesisSettled = true;
+                if (cancelledByOwner(error, controller.signal)) finish(false);
+                else finish(false, error);
+                releaseIfFinished();
+            });
+        } catch (error) {
+            finish(false, error);
+        }
+    }
+    return control;
+}
+
+function deferred() {
+    const result = {};
+    result.promise = new Promise(function retainCompletion(resolve, reject) {
+        result.resolve = resolve;
+        result.reject = reject;
+    });
+    result.promise.catch(function observeRetainedFailure() {});
+    return result;
+}
+
+function cancellation() {
+    const error = new Error('The owning caller stopped this native utterance.');
+    error.name = 'AbortError';
+    error.code = 'ARCANE_AI_REQUEST_ABORTED';
+    return error;
+}
+
+function cancelledByOwner(error, signal) {
+    if (!signal.aborted) return false;
+    if (error === signal.reason) return true;
+    // Core represents the matching request cancellation with its transport error.
+    // A separate failure or cleanup aggregate remains observable.
+    return error?.code === 'ARCANE_REQUEST_ABORTED' && !error.cause && !error.errors;
+}
+
+function playbackFailure(message) {
+    const error = new Error(message);
+    error.code = 'ARCANE_AI_SPEECH_PLAYBACK_FAILED';
+    return error;
+}

package/runtime/arcane/modules/VoiceCaptureWorklet.js

@@ -18,10 +18,21 @@ class ArcaneVoiceCaptureProcessor extends AudioWorkletProcessor {
     #buffer = null;
     #view = null;
     #stopped = false;
+    #pcmMode = false;
+    #pcmBuffer = null;
+    #pcmOffset = 0;
+    #pcmFrames = 0;
+    #observedActivity = false;
+    #observedFrames = 0;
 
     constructor(options) {
         super();
         const configuration = options.processorOptions;
+        this.#pcmMode = configuration.mode === 'pcm';
+        if (this.#pcmMode) {
+            this.#pcmFrames = Math.max(1, Math.round(configuration.pcmChunkMs * sampleRate / 1000));
+            this.#pcmBuffer = new Float32Array(this.#pcmFrames);
+        }
         this.#preRoll = new Float32Array(
             Math.round(configuration.preRollMs * sampleRate / 1000)
         );
@@ -35,11 +46,23 @@ class ArcaneVoiceCaptureProcessor extends AudioWorkletProcessor {
     }
 
     #receiveControl(event) {
-        if (this.#stopped || event.data.type !== 'stop') return;
+        if (this.#stopped) return;
+        if (event.data.type === 'flush') {
+            try {
+                this.#flushAnalysis();
+                if (this.#pcmMode) this.#emitPCM();
+                this.port.postMessage({type: 'flushed', id: event.data.id});
+            } catch (error) {
+                this.#fail(error);
+            }
+            return;
+        }
+        if (event.data.type !== 'stop') return;
         this.#stopped = true;
         try {
             this.#flushAnalysis();
-            this.#emit('stop');
+            if (this.#pcmMode) this.#emitPCM();
+            else this.#emit('stop');
             this.#release();
             this.port.postMessage(
                 {type: 'stopped'}
@@ -74,6 +97,25 @@ class ArcaneVoiceCaptureProcessor extends AudioWorkletProcessor {
     #flushAnalysis() {
         if (!this.#analysisCount) return;
         const activity = Math.sqrt(this.#analysisEnergy / this.#analysisCount) > this.#activityThreshold;
+        if (this.#pcmMode) {
+            if (activity !== this.#observedActivity) {
+                this.#observedActivity = activity;
+                this.port.postMessage({
+                    type: 'activity',
+                    active: activity,
+                    audioTime: this.#observedFrames / sampleRate
+                });
+            }
+            for (let index = 0; index < this.#analysisCount; index += 1) {
+                this.#pcmBuffer[this.#pcmOffset] = this.#analysisSamples[index];
+                this.#pcmOffset += 1;
+                if (this.#pcmOffset === this.#pcmFrames) this.#emitPCM();
+            }
+            this.#observedFrames += this.#analysisCount;
+            this.#analysisCount = 0;
+            this.#analysisEnergy = 0;
+            return;
+        }
         // Classification selects boundaries only. Every original sample still
         // enters capture in order, including a partial final analysis window.
         for (let index = 0; index < this.#analysisCount; index += 1) {
@@ -123,6 +165,22 @@ class ArcaneVoiceCaptureProcessor extends AudioWorkletProcessor {
         this.#preRollLength = Math.min(this.#preRollLength + 1, this.#preRoll.length);
     }
 
+    #emitPCM() {
+        if (!this.#pcmOffset) return;
+        const samples = this.#pcmOffset === this.#pcmFrames
+            ? this.#pcmBuffer
+            : this.#pcmBuffer.slice(0, this.#pcmOffset);
+        this.#sequence += 1;
+        this.port.postMessage({
+            type: 'pcm',
+            audio: samples.buffer,
+            sampleRate,
+            sequence: this.#sequence
+        }, [samples.buffer]);
+        this.#pcmBuffer = new Float32Array(this.#pcmFrames);
+        this.#pcmOffset = 0;
+    }
+
     #beginSegment() {
         // Preserve all requested pre-roll even when it exceeds chunkMs. After a
         // periodic boundary, retain quiet audio until activity or the pause so
@@ -206,6 +264,8 @@ class ArcaneVoiceCaptureProcessor extends AudioWorkletProcessor {
     }
 
     #release() {
+        this.#pcmBuffer = null;
+        this.#pcmOffset = 0;
         this.#buffer = null;
         this.#view = null;
         this.#preRoll = new Float32Array(0);

package/src/core/host.mjs

@@ -39,6 +39,7 @@ export function resolveNativeLaunchContext({appId, context = {}} = {}) {
     const resolved = {
         ...context,
         stateRoot,
+        modelsDirectory: context.modelsDirectory === undefined ? path.join(root, 'Models') : context.modelsDirectory,
         workspaceRoot: context.workspaceRoot === undefined ? path.join(root, 'Workspace') : context.workspaceRoot
     };
     if (context.sharedHost !== undefined) {

package/src/core/services/local-ai.mjs

@@ -121,7 +121,7 @@ function collectCompletion(completion, chunk) {
 }
 
 /** Core owns native processes; applications own runtime and model selection. */
-export function createLocalAIService(configuration, {appRoot, runtimes = [], signal, onEvent, prepare, executionDevices} = {}) {
+export function createLocalAIService(configuration, {appRoot, runtimes = [], signal, onEvent, prepare, executionDevices, autoStart = true} = {}) {
     const config = normalizeLocalAIConfig(configuration) ?? {runtimes: []};
     const lifetime = new AbortController();
     const lifetimeSignal = signal ? AbortSignal.any([signal, lifetime.signal]) : lifetime.signal;
@@ -130,6 +130,8 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
     const owner = {};
     const events = createArcaneEventSource(owner, {source: 'core-local-ai', eventTypes: ['localai.state']});
     let context;
+    let preparing;
+    let prepared = false;
     let modelAssetOwner;
     let closed = false;
     let onnx;
@@ -738,17 +740,20 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
         });
     }
 
-    const service = {
-        name: 'local-ai',
-        current: snapshot,
-        getONNXRuntime() {
-            return requireONNX();
-        },
-        async start(currentContext) {
-            context = currentContext;
+    function prepareEngines(requestSignal) {
+        lifetimeSignal.throwIfAborted();
+        requestSignal?.throwIfAborted();
+        if (prepared) return Promise.resolve(snapshot());
+        if (preparing) return awaitReadiness(preparing, requestSignal ?? lifetimeSignal);
+        const operationSignal = requestSignal ? AbortSignal.any([lifetimeSignal, requestSignal]) : lifetimeSignal;
+        const task = Promise.resolve().then(async function prepareSelectedEngines() {
+            operationSignal.throwIfAborted();
             if (prepare) {
-                runtimes = await prepare({signal: lifetimeSignal, onEvent});
-                lifetimeSignal.throwIfAborted();
+                runtimes = await prepare({signal: operationSignal, onEvent: function preparationProgress(event) {
+                    context?.emit('localai.preparation', event);
+                    return onEvent?.(event);
+                }});
+                operationSignal.throwIfAborted();
                 onnxRecord = runtimes.find(function installedONNX(runtime) { return runtime.id === 'onnx'; });
             }
             for (const requirement of config.runtimes) {
@@ -764,11 +769,40 @@ export function createLocalAIService(configuration, {appRoot, runtimes = [], sig
                     state: {id: requirement.id, installed: runtimes.some(function installed(record) { return record.id === requirement.id && record.executable; }), available: false, state: 'starting'}
                 };
                 engines.set(engine.id, engine);
-                startEngine(engine);
+                try { startEngine(engine); }
+                catch (error) {
+                    engines.delete(engine.id);
+                    throw error;
+                }
             }
             publish();
+            prepared = true;
+        });
+        preparing = task;
+        jobs.add(task);
+        function releasePreparation() {
+            jobs.delete(task);
+            if (preparing === task) preparing = null;
+        }
+        task.then(releasePreparation, releasePreparation);
+        return task;
+    }
+
+    const service = {
+        name: 'local-ai',
+        current: snapshot,
+        getONNXRuntime() {
+            return requireONNX();
+        },
+        async start(currentContext) {
+            context = currentContext;
+            if (autoStart === false) return;
+            return prepareEngines(lifetimeSignal);
         },
         methods: {
+            'localai.prepare': function prepareSelectedRuntime(_parameters, request) {
+                return prepareEngines(request.signal).then(snapshot);
+            },
             'localai.services.recover': async function recoverServices({runtimes: selected = [...engines.keys(), ...(onnxSelected ? ['onnx'] : [])]} = {}, request) {
                 request.signal.throwIfAborted();
                 const selectedEngines = selected.map(function selectedEngine(id) {