arcane-os 0.94.0 → 0.95.0

package/runtime/arcane/modules/ContinuousVoiceCapture.js

@@ -4,8 +4,8 @@ import {arcaneLogging} from 'arcane-os/logging';
 const is = new Is(false);
 
 /**
- * Browser microphone capture with activity-selected, non-overlapping WAV clips.
- * Callbacks are observational and never delay the microphone or worklet.
+ * Browser microphone capture with activity-selected WAV clips or continuous PCM.
+ * PCM callbacks settle in capture order without delaying the audio worklet.
  * stop() delivers the final active clip; cancel() abandons pending delivery.
  */
 export default class ContinuousVoiceCapture {
@@ -13,6 +13,8 @@ export default class ContinuousVoiceCapture {
     #onSegment;
     #onError;
     #onState;
+    #onPCM;
+    #onActivity;
     #session = null;
 
     constructor(
@@ -21,7 +23,11 @@ export default class ContinuousVoiceCapture {
             quietMs = 2000,
             chunkMs = 30000,
             activityThreshold = 0.02,
+            mode = 'segments',
+            pcmChunkMs = 100,
             onSegment,
+            onPCM,
+            onActivity,
             onError,
             onState
         } = {}
@@ -38,8 +44,16 @@ export default class ContinuousVoiceCapture {
         if (!is.finite(activityThreshold) || activityThreshold < 0) {
             throw new RangeError('activityThreshold must be a finite nonnegative amplitude.');
         }
-        this.#options = {preRollMs, quietMs, chunkMs, activityThreshold};
+        if (!['segments', 'pcm'].includes(mode)) {
+            throw new TypeError('Voice capture mode must be segments or pcm.');
+        }
+        if (!is.finite(pcmChunkMs) || pcmChunkMs <= 0) {
+            throw new RangeError('pcmChunkMs must be a finite positive duration.');
+        }
+        this.#options = {preRollMs, quietMs, chunkMs, activityThreshold, mode, pcmChunkMs};
         this.#onSegment = onSegment;
+        this.#onPCM = onPCM;
+        this.#onActivity = onActivity;
         this.#onError = onError;
         this.#onState = onState;
     }
@@ -47,10 +61,11 @@ export default class ContinuousVoiceCapture {
     async start(
         {
             mediaConstraints = {audio: true},
+            sampleRate,
             signal
         } = {}
     ) {
-        if ((this.#session && !this.#session.settled) || signal?.aborted) return false;
+        if (this.#session || signal?.aborted) return false;
 
         const session = {
             phase: 'starting',
@@ -62,8 +77,12 @@ export default class ContinuousVoiceCapture {
             node: null,
             cleanup: [],
             errors: [],
+            pcmDelivery: Promise.resolve(),
+            flushes: new Map(),
+            flushSequence: 0,
             resolveStartup: null,
-            resolveStop: null
+            resolveStop: null,
+            rejectStop: null
         };
         session.startupCancelled = new Promise(
             function ownStartupCancellation(resolve) {
@@ -71,10 +90,12 @@ export default class ContinuousVoiceCapture {
             }
         );
         session.stopped = new Promise(
-            function ownCaptureStop(resolve) {
+            function ownCaptureStop(resolve, reject) {
                 session.resolveStop = resolve;
+                session.rejectStop = reject;
             }
         );
+        session.stopped.catch(function observeCaptureStopFailure() {});
         this.#session = session;
 
         const owner = this;
@@ -101,7 +122,7 @@ export default class ContinuousVoiceCapture {
         this.#notify(session, this.#onState, 'starting');
         if (!this.#starting(session)) return false;
 
-        const preparation = this.#prepare(session, mediaConstraints);
+        const preparation = this.#prepare(session, mediaConstraints, sampleRate);
         return Promise.race(
             [preparation, session.startupCancelled]
         );
@@ -136,27 +157,46 @@ export default class ContinuousVoiceCapture {
 
     cancel() {
         const session = this.#session;
-        if (!session) return;
+        if (!session) return Promise.resolve();
         session.cancelled = true;
         this.#settle(session, 'stopped');
+        return session.stopped;
+    }
+
+    flush() {
+        const session = this.#session;
+        if (!session || session.phase !== 'listening') return session?.pcmDelivery ?? Promise.resolve();
+        session.flushSequence += 1;
+        const id = session.flushSequence;
+        const task = new Promise(function ownCaptureFlush(resolve, reject) {
+            session.flushes.set(id, {resolve, reject});
+        });
+        try {
+            session.node.port.postMessage({type: 'flush', id});
+        } catch (error) {
+            session.flushes.delete(id);
+            this.#fail(session, error);
+            return Promise.reject(error);
+        }
+        return task;
     }
 
     destroy() {
-        this.cancel();
+        return this.cancel();
     }
 
     #starting(session) {
         return this.#session === session && session.phase === 'starting';
     }
 
-    async #prepare(session, mediaConstraints) {
+    async #prepare(session, mediaConstraints, sampleRate) {
         try {
             if (!is.function(globalThis.AudioContext)
                 || !is.function(globalThis.AudioWorkletNode)
                 || !is.function(globalThis.navigator?.mediaDevices?.getUserMedia)) {
                 throw new Error('Continuous voice capture requires microphone access and AudioWorklet.');
             }
-            const context = new globalThis.AudioContext();
+            const context = new globalThis.AudioContext(sampleRate === undefined ? undefined : {sampleRate});
             session.context = context;
             if (!context.audioWorklet) {
                 throw new Error('This browser does not expose AudioWorklet microphone processing.');
@@ -226,6 +266,22 @@ export default class ContinuousVoiceCapture {
                                 durationMs: message.durationMs
                             }
                         );
+                    } else if (message.type === 'pcm') {
+                        owner.#deliverPCM(session, {
+                            samples: new Float32Array(message.audio),
+                            sampleRate: message.sampleRate,
+                            channels: 1,
+                            sequence: message.sequence
+                        });
+                    } else if (message.type === 'activity') {
+                        owner.#notify(session, owner.#onActivity, {
+                            active: message.active,
+                            audioTime: message.audioTime
+                        });
+                    } else if (message.type === 'flushed') {
+                        const flush = session.flushes.get(message.id);
+                        session.flushes.delete(message.id);
+                        if (flush) session.pcmDelivery.then(flush.resolve, flush.reject);
                     } else if (message.type === 'stopped') {
                         owner.#settle(session, 'stopped');
                     } else if (message.type === 'error') {
@@ -321,6 +377,23 @@ export default class ContinuousVoiceCapture {
         await context.resume();
     }
 
+    #deliverPCM(session, value) {
+        const owner = this;
+        // Capture and inference run concurrently. This ordered delivery chain
+        // retains every sample while the consumer catches up; no activity gate
+        // selects or discards audio in continuous mode.
+        session.pcmDelivery = session.pcmDelivery.then(async function deliverCapturedPCM() {
+            if (!session.cancelled && is.function(owner.#onPCM)) {
+                await owner.#onPCM(value);
+            }
+        });
+        // Observe failure without converting the delivery/flush contract to success.
+        session.pcmDelivery.catch(function reportPCMDeliveryFailure(error) {
+            if (!session.settled) owner.#fail(session, error);
+            else owner.#reportError(session, error, true);
+        });
+    }
+
     #notify(session, callback, value) {
         if (session.cancelled
             || (this.#session && this.#session !== session)
@@ -424,23 +497,47 @@ export default class ContinuousVoiceCapture {
             }
         }
         const context = session.context;
+        let contextClosing = Promise.resolve();
         if (context && context.state !== 'closed') {
-            const owner = this;
-            function observeContextCloseFailure(error) {
-                owner.#reportError(session, error);
-            }
             try {
-                Promise.resolve(context.close()).catch(observeContextCloseFailure);
+                contextClosing = Promise.resolve(context.close());
             } catch (error) {
                 session.errors.push(error);
             }
         }
         session.resolveStartup(false);
-        for (const error of session.errors) {
-            this.#reportError(session, error, true);
+        for (const flush of session.flushes.values()) {
+            if (session.errors.length) flush.reject(session.errors[0]);
+            else session.pcmDelivery.then(flush.resolve, flush.reject);
         }
-        this.#notify(session, this.#onState, session.errors.length ? 'interrupted' : state);
-        if (this.#session === session) this.#session = null;
-        session.resolveStop();
+        session.flushes.clear();
+        // stop() drains callbacks already accepted before the worklet's stop
+        // acknowledgment. cancel() explicitly abandons subsequent delivery;
+        // both operations join the context's actual asynchronous close.
+        const owner = this;
+        Promise.allSettled(
+            [session.pcmDelivery, contextClosing]
+        ).then(
+            function finishCapturedCleanup(results) {
+                for (const result of results) {
+                    if (result.status === 'rejected' && !session.errors.includes(result.reason)) {
+                        session.errors.push(result.reason);
+                    }
+                }
+                if (owner.#session === session) owner.#session = null;
+                if (session.errors.length) {
+                    session.rejectStop(
+                        session.errors.length === 1 ? session.errors[0] : new AggregateError(
+                            session.errors,
+                            'Voice capture delivery or cleanup failed.'
+                        )
+                    );
+                } else {
+                    session.resolveStop();
+                }
+                for (const error of session.errors) owner.#reportError(session, error, true);
+                owner.#notify(session, owner.#onState, session.errors.length ? 'interrupted' : state);
+            }
+        );
     }
 }

package/runtime/arcane/modules/NativeSpeechCapture.js

@@ -0,0 +1,198 @@
+import ContinuousVoiceCapture from './ContinuousVoiceCapture.js';
+
+/**
+ * One microphone lifetime and one scoped Core ASR stream.
+ * Recognition callbacks receive (result, {requestStop}). They request a drain
+ * without awaiting it, then return; the external owner joins stop() or done.
+ * Awaiting outer stop()/done/forceEndpoint() from a recognition callback waits on that
+ * callback itself. Segments do not submit chat turns.
+ */
+export default function createNativeSpeechCapture({speech, model, ...options}) {
+    const controller = new AbortController();
+    let stream;
+    let started = false;
+    let stopping;
+    let settled = false;
+    let cancelled = false;
+    let cleanup;
+    let detachSignal;
+    let resolveDone;
+    let rejectDone;
+    const done = new Promise(function ownNativeCapture(resolve, reject) {
+        resolveDone = resolve;
+        rejectDone = reject;
+    });
+    // The caller still receives rejection through done; this observer prevents
+    // a synchronous device failure from becoming an unhandled rejection first.
+    done.catch(function observeNativeCaptureFailure() {});
+
+    function finish(error, result) {
+        if (settled) return;
+        settled = true;
+        detachSignal?.();
+        if (error) rejectDone(error);
+        else resolveDone(result);
+    }
+
+    function fail(error) {
+        if (settled || cleanup) return cleanup;
+        return endCapture(error, error);
+    }
+
+    function endCapture(reason, primaryError) {
+        if (settled || cleanup) return cleanup ?? done;
+        const captureCleanup = capture.cancel();
+        // Join the explicit native cancellation operation, not the immediately
+        // aborted browser request, before releasing the capture/provider lane.
+        // Native cancellation proceeds while capture joins its pending PCM.
+        const nativeCleanup = Promise.resolve().then(function cancelNativeStream() {
+            return stream?.cancel(reason);
+        });
+        cleanup = Promise.allSettled(
+            [captureCleanup, nativeCleanup]
+        ).then(
+            function nativeCaptureReleased(results) {
+                const errors = primaryError ? [primaryError] : [];
+                for (const result of results) {
+                    if (result.status === 'rejected' && !errors.includes(result.reason)) errors.push(result.reason);
+                }
+                const error = errors.length > 1
+                    ? new AggregateError(errors, 'Native capture failed during cleanup.')
+                    : errors[0];
+                finish(error, results[1].value);
+                if (error) return options.onError?.(error);
+            }
+        );
+        cleanup.catch(function observeCaptureErrorCallback(error) {
+            console.error('Native speech capture error callback failed.', error);
+        });
+        controller.abort(reason);
+        return cleanup;
+    }
+
+    const capture = new ContinuousVoiceCapture({
+        mode: 'pcm',
+        pcmChunkMs: options.pcmChunkMs ?? 100,
+        activityThreshold: options.activityThreshold ?? 0.02,
+        async onPCM(record) {
+            try {
+                await stream.ready;
+                controller.signal.throwIfAborted();
+                const sampleRate = options.streamOptions?.sampleRate ?? options.sampleRate ?? 16000;
+                if (record.sampleRate !== sampleRate) {
+                    throw new Error(`Microphone PCM sample rate ${record.sampleRate} differs from native stream rate ${sampleRate}.`);
+                }
+                await stream.push(record.samples);
+            } catch (error) {
+                // The owned cancellation ends an in-flight browser request;
+                // that expected abort is distinct from actual delivery failure.
+                if (controller.signal.aborted && (
+                    error === controller.signal.reason || error?.name === 'AbortError' || error?.code === 'ARCANE_CANCELLED'
+                )) return;
+                throw error;
+            }
+        },
+        onActivity: options.onActivity,
+        onState: options.onState,
+        onError: fail
+    });
+
+    function cancel(reason) {
+        if (settled) return done;
+        cancelled = true;
+        endCapture(reason);
+        return done;
+    }
+
+    function stop() {
+        if (stopping || settled) return stopping ?? done;
+        if (!stream) return cancel();
+        async function drainNativeCapture() {
+            await capture.stop();
+            if (!controller.signal.aborted) await stream.finish();
+            return done;
+        }
+        stopping = drainNativeCapture().catch(
+            function nativeCaptureStopFailed(error) {
+                fail(error);
+                return done;
+            }
+        );
+        return stopping;
+    }
+
+    function requestStop() {
+        // A callback requests the same terminal drain without joining itself.
+        // done retains its success or failure for the external owning caller.
+        stop().catch(function observeRequestedCaptureStop() {});
+    }
+
+    const callbackControls = {requestStop};
+
+    return {
+        done,
+        async start({signal, mediaConstraints} = {}) {
+            if (started || settled) return false;
+            started = true;
+            function externalCancellation() { cancel(signal.reason); }
+            if (signal) {
+                signal.addEventListener('abort', externalCancellation, {once: true});
+                detachSignal = function removeCaptureSignal() {
+                    signal.removeEventListener('abort', externalCancellation);
+                };
+                if (signal.aborted) {
+                    cancel(signal.reason);
+                    return false;
+                }
+            }
+            try {
+                stream = speech.openStream({model, sampleRate: options.sampleRate ?? 16000, ...options.streamOptions}, {
+                    signal: controller.signal,
+                    onProgress: options.onProgress,
+                    onRequest: options.onRequest,
+                    async onUpdate(result) {
+                        if (controller.signal.aborted) return;
+                        await options.onUpdate?.(result, callbackControls);
+                        // These are ASR snapshot callbacks, not semantic turn-end
+                        // decisions. Preserve every field and replacement partial.
+                        if (result.final) await options.onSegment?.(result, callbackControls);
+                        else await options.onInterim?.(result, callbackControls);
+                    }
+                });
+                stream.completion.then(async function nativeStreamFinished(result) {
+                    if (cleanup) return;
+                    await capture.cancel();
+                    if (cleanup) return cleanup;
+                    finish(undefined, result);
+                }, fail).catch(function captureObserverFailed(error) {
+                    return fail(error);
+                });
+                // Microphone permission remains in the initiating user gesture;
+                // its samples queue while the native stream becomes ready.
+                const capturing = capture.start({
+                    signal: controller.signal,
+                    sampleRate: options.streamOptions?.sampleRate ?? options.sampleRate ?? 16000,
+                    mediaConstraints: mediaConstraints ?? options.mediaConstraints ?? {audio: {echoCancellation: true}}
+                });
+                const [ready] = await Promise.all([capturing, stream.ready]);
+                if (!ready && !settled) cancel();
+                return ready && !settled;
+            } catch (error) {
+                if (!cancelled) await fail(error);
+                else await cleanup;
+                return false;
+            }
+        },
+        async forceEndpoint() {
+            if (!started || settled || stopping) return [];
+            await capture.flush();
+            controller.signal.throwIfAborted();
+            return stream.forceEndpoint();
+        },
+        stop,
+        requestStop,
+        cancel,
+        destroy: cancel,
+        current() { return stream?.current() ?? null; }
+    };
+}