@happyvertical/smrt-ui 0.51.38 → 0.52.0

package/dist/components/forms/dictation.svelte.js

@@ -0,0 +1,686 @@
+/**
+ * Dictation: speak into a text field.
+ *
+ * The state machine behind the microphone button and "long-press a field to
+ * speak". It does not recognise speech itself: the host hands it a speech
+ * source, normally smrt-svelte's browser speech-to-text adapter
+ * (`createSttDictationSource` in `@happyvertical/smrt-svelte/browser-ai`),
+ * whose adapters already satisfy `DictationSpeechSource`. That keeps smrt-ui
+ * free of the Provider and the speech engines, and gives one speech engine
+ * for forms, chat, and anything else.
+ *
+ * States: `idle` → `starting` → `listening` → `stopping` → `idle`, and
+ * `error` (with `errorKind`) from any of them. A "ready" beep plays when the
+ * microphone is actually listening. Final phrases go to `onText` (put them
+ * in the field with `insertTextAtCursor`); the phrase in progress is
+ * `interim`.
+ *
+ * How listening stops: tap the microphone again (`toggle`/`stop`), send the
+ * form, press Escape, or pause talking (the browser ends it). A long press
+ * starts listening and it keeps listening after the finger comes up, so the
+ * person can press, hear the beep, let go, and talk.
+ *
+ * On first use it asks for the microphone (`getUserMedia`) and waits for the
+ * answer before it starts recognising, so the permission prompt cannot cut
+ * the recogniser off and a "no" shows as "the microphone is blocked".
+ *
+ * It never stops silently. Every recogniser error becomes an `error` state
+ * with a plain `errorKind` (and the raw `errorCode`, and a console warning);
+ * a recogniser that ends on its own within `earlyEndMs` of starting without
+ * hearing anything (Brave's speech recognition does this) is an error too.
+ *
+ * Recording fallback: give it a `transcribe` function and, when the browser
+ * has no speech recognition (Firefox), or it fails with no speech service
+ * behind it (Brave: Web Speech `network` / `service-not-allowed`, or an end
+ * straight away), it records the message with `MediaRecorder` instead
+ * (WebM/Opus, or MP4 on Safari; capped at `maxDurationMs` and `maxBytes`).
+ * When the person stops, the state is `transcribing` ("Writing it down…")
+ * while `transcribe(audio, { mimeType, language })` turns the recording into
+ * text, which goes to `onText` like a spoken phrase. Once the browser's
+ * recogniser has failed, this `Dictation` records straight away next time.
+ */
+import { canCaptureDictationAudio, createMediaRecorderCapture, DictationError, } from './audio-capture.js';
+import { playReadyBeep } from './ready-beep.js';
+const DICTATION_ERROR_KINDS = new Set([
+    'unsupported',
+    'denied',
+    'no-speech',
+    'microphone',
+    'interrupted',
+    'too-long',
+    'not-transcribed',
+    'unavailable',
+    'forbidden',
+    'failed',
+]);
+const TRANSCRIBE_ERROR_KINDS = new Set([
+    'too-long',
+    'not-transcribed',
+    'unavailable',
+    'forbidden',
+    'no-speech',
+]);
+const SPEECH_ERROR_KINDS = {
+    network: 'unsupported',
+    'service-not-allowed': 'unsupported',
+    'language-not-supported': 'unsupported',
+    'not-allowed': 'denied',
+    'no-speech': 'no-speech',
+    'audio-capture': 'microphone',
+    aborted: 'interrupted',
+};
+/** The raw code of a speech or media error, when it has one. */
+export function dictationErrorCode(error) {
+    if (error instanceof DictationError)
+        return error.code;
+    const e = (error ?? {});
+    if (typeof e.speechError === 'string')
+        return e.speechError;
+    // A raw SpeechRecognitionErrorEvent carries the code as `error`.
+    if (typeof e.error === 'string')
+        return e.error;
+    if (typeof e.code === 'string')
+        return e.code;
+    if (typeof e.name === 'string' && e.name !== 'Error')
+        return e.name;
+    return null;
+}
+/**
+ * Sort a speech error into a `DictationErrorKind`. Reads the error's `code`
+ * and `name` (smrt-svelte browser-ai errors: `PERMISSION_DENIED`,
+ * `CAPABILITY_NOT_AVAILABLE`) and the Web Speech error names.
+ */
+export function classifyDictationError(error) {
+    const known = error?.dictationKind;
+    if (typeof known === 'string' &&
+        DICTATION_ERROR_KINDS.has(known)) {
+        return known;
+    }
+    const e = (error ?? {});
+    const speechCode = typeof e.speechError === 'string'
+        ? e.speechError
+        : typeof e.error === 'string'
+            ? e.error
+            : '';
+    if (speechCode && SPEECH_ERROR_KINDS[speechCode]) {
+        return SPEECH_ERROR_KINDS[speechCode];
+    }
+    const code = typeof e.code === 'string' ? e.code : '';
+    const name = typeof e.name === 'string' ? e.name : '';
+    const message = typeof e.message === 'string'
+        ? e.message.toLowerCase()
+        : String(error ?? '').toLowerCase();
+    if (code === 'PERMISSION_DENIED' ||
+        name === 'PermissionDeniedError' ||
+        name === 'NotAllowedError' ||
+        /not-allowed|service-not-allowed|permission denied/.test(message)) {
+        return 'denied';
+    }
+    if (code === 'CAPABILITY_NOT_AVAILABLE' ||
+        code === 'UNSUPPORTED_ADAPTER' ||
+        name === 'CapabilityNotAvailableError' ||
+        name === 'NotSupportedError' ||
+        /not available|not supported|unsupported/.test(message)) {
+        return 'unsupported';
+    }
+    if (name === 'NotFoundError' ||
+        name === 'NotReadableError' ||
+        name === 'OverconstrainedError' ||
+        /audio capture|audio-capture/.test(message)) {
+        return 'microphone';
+    }
+    if (/no speech|no-speech/.test(message))
+        return 'no-speech';
+    if (/network/.test(message))
+        return 'unsupported';
+    if (/aborted/.test(message))
+        return 'interrupted';
+    return 'failed';
+}
+function pageLanguage() {
+    if (typeof document !== 'undefined') {
+        const lang = document.documentElement.getAttribute('lang');
+        if (lang)
+            return lang;
+    }
+    return 'en-US';
+}
+function defaultLog(event) {
+    // biome-ignore lint/suspicious/noConsole: dictation must never fail silently; this is the diagnostic trail.
+    console.warn(`[dictation] stopped (${event.kind}) at ${event.stage}` +
+        (event.code ? ` [${event.code}]` : '') +
+        `: ${event.message}` +
+        (event.fallback === 'recording' ? ' (recording instead)' : ''));
+}
+export class Dictation {
+    /** Where dictation is (see the module comment). */
+    state = $state('idle');
+    /** Why it stopped, while `state` is `error`. */
+    errorKind = $state(null);
+    /** The raw error code behind `errorKind` (`network`, `NotAllowedError`…). */
+    errorCode = $state(null);
+    /** The phrase being spoken right now. */
+    interim = $state('');
+    /** The ready beep could not play yet (no user gesture); `unlock` plays it. */
+    beepPending = $state(false);
+    /**
+     * `true` while this message is being recorded (the fallback) rather than
+     * recognised by the browser.
+     */
+    recording = $state(false);
+    #options;
+    #source = null;
+    #sourcePromise = null;
+    #unsubscribe = [];
+    #session = 0;
+    #stopTimer = null;
+    #disposed = false;
+    #microphoneReady = false;
+    /** When the source said it is listening (for the early-end check). */
+    #listeningAt = 0;
+    /** A result arrived in this session. */
+    #heard = false;
+    /** The browser's recogniser failed for want of a speech service. */
+    #speechBroken = false;
+    /** The recorder for this message, in the recording fallback. */
+    #capture = null;
+    /** The recorder is recording (its `start` resolved). */
+    #captureReady = false;
+    #captureOff = null;
+    /** Cancels writing a recording down. */
+    #transcribeAbort = null;
+    constructor(options) {
+        this.#options = options;
+    }
+    /** A speech source or a `transcribe` was given (the browser may still turn out unable). */
+    get available() {
+        return Boolean(this.#options.source || this.#options.transcribe);
+    }
+    /** Starting or listening. */
+    get active() {
+        return this.state === 'starting' || this.state === 'listening';
+    }
+    /** Replace options (for example a new `onText`). */
+    setOptions(options) {
+        this.#options = { ...this.#options, ...options };
+    }
+    /** Start listening. A no-op while already starting, listening, or writing down. */
+    async start() {
+        if (this.#disposed ||
+            this.active ||
+            this.state === 'stopping' ||
+            this.state === 'transcribing')
+            return;
+        this.errorKind = null;
+        this.errorCode = null;
+        this.interim = '';
+        this.#heard = false;
+        this.#listeningAt = 0;
+        this.recording = false;
+        this.state = 'starting';
+        const session = ++this.#session;
+        const provider = this.#options.source;
+        if (this.#canRecord() && (this.#speechBroken || !provider)) {
+            await this.#startRecording(session);
+            return;
+        }
+        if (!provider) {
+            this.#fail('unsupported', null, 'No speech source', 'source');
+            return;
+        }
+        let source;
+        try {
+            source = await this.#resolveSource(provider);
+        }
+        catch (error) {
+            if (session === this.#session)
+                this.#failOrRecord(error, 'source', session);
+            return;
+        }
+        if (session !== this.#session || this.#disposed)
+            return;
+        // Ask for the microphone first and wait for the answer: the recogniser
+        // then starts with the permission settled instead of racing the prompt.
+        try {
+            await this.#ensureMicrophone();
+        }
+        catch (error) {
+            if (session === this.#session)
+                this.#failWith(error, 'microphone');
+            return;
+        }
+        if (session !== this.#session ||
+            this.#disposed ||
+            this.state !== 'starting')
+            return;
+        try {
+            await source.start({
+                language: this.#language(),
+                continuous: true,
+                interimResults: true,
+            });
+        }
+        catch (error) {
+            if (session === this.#session)
+                this.#failOrRecord(error, 'start', session);
+            return;
+        }
+        if (session !== this.#session || this.recording)
+            return;
+        if (!source.onStart && this.state === 'starting')
+            this.#listening();
+    }
+    /**
+     * Stop listening; the phrase in progress is kept if the source finishes it.
+     * A recording is written down (`transcribing`) and its text handed over.
+     */
+    async stop() {
+        if (!this.active)
+            return;
+        if (this.recording) {
+            await this.#finishRecording();
+            return;
+        }
+        this.state = 'stopping';
+        const session = this.#session;
+        this.#clearStopTimer();
+        this.#stopTimer = setTimeout(() => {
+            if (session === this.#session && this.state === 'stopping')
+                this.#idle();
+        }, this.#options.stopTimeoutMs ?? 1500);
+        try {
+            await this.#source?.stop();
+        }
+        catch {
+            if (session === this.#session)
+                this.#idle();
+        }
+    }
+    /** Tap on the microphone: start, or stop while listening. */
+    async toggle() {
+        if (this.active)
+            await this.stop();
+        else
+            await this.start();
+    }
+    /** Clear an error so the field looks normal again. */
+    clearError() {
+        if (this.state === 'error') {
+            this.state = 'idle';
+            this.errorKind = null;
+            this.errorCode = null;
+        }
+    }
+    /**
+     * Call from a user gesture (a tap, a long press's release): plays the
+     * ready beep if it could not play when listening started.
+     */
+    async unlock() {
+        if (!this.beepPending)
+            return;
+        const played = await this.#playBeep();
+        if (played)
+            this.beepPending = false;
+    }
+    dispose() {
+        this.#disposed = true;
+        this.#session++;
+        this.#clearStopTimer();
+        if (this.active && !this.recording)
+            void this.#source?.stop().catch(() => undefined);
+        this.#dropCapture(true);
+        this.#transcribeAbort?.abort();
+        this.#transcribeAbort = null;
+        for (const off of this.#unsubscribe)
+            off();
+        this.#unsubscribe = [];
+        this.state = 'idle';
+        this.recording = false;
+    }
+    #language() {
+        return this.#options.language ?? pageLanguage();
+    }
+    /** A `transcribe` was given and this browser (or host) can record. */
+    #canRecord() {
+        return Boolean(this.#options.transcribe &&
+            (this.#options.capture || canCaptureDictationAudio()));
+    }
+    /**
+     * The browser's recogniser failed in a way that means it has no speech
+     * service (or none at all): record instead, if we can and nothing was
+     * heard yet.
+     */
+    #shouldRecordInstead(kind, stage) {
+        if (!this.#canRecord() || this.#heard)
+            return false;
+        return (kind === 'unsupported' || (kind === 'interrupted' && stage === 'end'));
+    }
+    #failOrRecord(error, stage, session) {
+        const kind = classifyDictationError(error);
+        if (this.#shouldRecordInstead(kind, stage)) {
+            this.#log(kind, dictationErrorCode(error), errorMessage(error), stage, true);
+            void this.#switchToRecording(session);
+            return;
+        }
+        this.#failWith(error, stage);
+    }
+    /** Leave the browser's recogniser and record this message instead. */
+    async #switchToRecording(session) {
+        this.#speechBroken = true;
+        this.interim = '';
+        this.#clearStopTimer();
+        const source = this.#source;
+        this.recording = true;
+        if (source)
+            void source.stop().catch(() => undefined);
+        await this.#startRecording(session);
+    }
+    async #startRecording(session) {
+        this.recording = true;
+        const factory = this.#options.capture ?? createMediaRecorderCapture;
+        let capture;
+        try {
+            capture = factory({
+                maxDurationMs: this.#options.maxDurationMs,
+                maxBytes: this.#options.maxBytes,
+            });
+        }
+        catch (error) {
+            if (session === this.#session)
+                this.#failWith(error, 'record');
+            return;
+        }
+        this.#dropCapture(true);
+        this.#capture = capture;
+        // Reaching the time cap writes the recording down; growing past the size
+        // cap fails (`stop` then rejects with `too-long`). Either way, stop.
+        this.#captureOff = capture.onLimit(() => {
+            if (session === this.#session && this.state === 'listening') {
+                void this.#finishRecording();
+            }
+        });
+        try {
+            await capture.start();
+            if (this.#capture === capture)
+                this.#captureReady = true;
+        }
+        catch (error) {
+            if (session === this.#session && this.#capture === capture) {
+                this.#dropCapture(true);
+                this.#failWith(error, 'record');
+            }
+            else {
+                capture.cancel();
+            }
+            return;
+        }
+        if (session !== this.#session ||
+            this.#disposed ||
+            this.#capture !== capture ||
+            !this.active) {
+            if (this.#capture === capture)
+                this.#dropCapture(true);
+            else
+                capture.cancel();
+            return;
+        }
+        // Recording from the start: it is listening now (and beeps). Switched
+        // over mid-message: it already said "Listening…".
+        if (this.state === 'starting')
+            this.#listening();
+    }
+    /** Stop recording, write it down, and hand the text over. */
+    async #finishRecording() {
+        const session = this.#session;
+        const capture = this.#capture;
+        if (!capture || !this.#captureReady) {
+            // Still opening the microphone: nothing recorded yet, just stop.
+            this.#dropCapture(true);
+            this.#session++;
+            this.#idle();
+            return;
+        }
+        this.#dropCapture(false);
+        this.state = 'transcribing';
+        this.interim = '';
+        this.beepPending = false;
+        let recording;
+        try {
+            recording = await capture.stop();
+        }
+        catch (error) {
+            if (session === this.#session)
+                this.#failWith(error, 'record');
+            return;
+        }
+        if (session !== this.#session || this.#disposed)
+            return;
+        if (recording.audio.size === 0) {
+            this.#fail('no-speech', null, 'The recording is empty', 'record');
+            return;
+        }
+        const transcribe = this.#options.transcribe;
+        if (!transcribe) {
+            this.#fail('unavailable', null, 'No transcribe function', 'transcribe');
+            return;
+        }
+        const controller = new AbortController();
+        this.#transcribeAbort = controller;
+        let timedOut = false;
+        const timer = setTimeout(() => {
+            timedOut = true;
+            controller.abort();
+        }, this.#options.transcribeTimeoutMs ?? 60_000);
+        let text = '';
+        try {
+            const result = await transcribe(recording.audio, {
+                mimeType: recording.mimeType,
+                language: this.#language(),
+                durationMs: recording.durationMs,
+                signal: controller.signal,
+            });
+            text = (typeof result === 'string' ? result : (result?.text ?? '')).trim();
+        }
+        catch (error) {
+            if (session !== this.#session || this.#disposed)
+                return;
+            if (timedOut) {
+                this.#fail('not-transcribed', 'timeout', 'Writing the recording down took too long', 'transcribe');
+            }
+            else {
+                // Only the writing-down kinds make sense here; a provider's
+                // "network" error is not the browser lacking speech recognition.
+                const kind = classifyDictationError(error);
+                this.#fail(TRANSCRIBE_ERROR_KINDS.has(kind) ? kind : 'not-transcribed', dictationErrorCode(error), errorMessage(error), 'transcribe');
+            }
+            return;
+        }
+        finally {
+            clearTimeout(timer);
+            if (this.#transcribeAbort === controller)
+                this.#transcribeAbort = null;
+        }
+        if (session !== this.#session || this.#disposed)
+            return;
+        if (!text) {
+            this.#fail('no-speech', null, 'Nothing was written down', 'transcribe');
+            return;
+        }
+        this.#idle();
+        this.#options.onText(text);
+    }
+    /** Forget the recorder; `cancel` throws its audio away too. */
+    #dropCapture(cancel) {
+        this.#captureOff?.();
+        this.#captureOff = null;
+        if (cancel)
+            this.#capture?.cancel();
+        this.#capture = null;
+        this.#captureReady = false;
+    }
+    async #ensureMicrophone() {
+        if (this.#microphoneReady || this.#options.requestMicrophone === false)
+            return;
+        const media = typeof navigator !== 'undefined' ? navigator.mediaDevices : undefined;
+        if (!media || typeof media.getUserMedia !== 'function')
+            return;
+        const stream = await media.getUserMedia({ audio: true });
+        // Only the permission was wanted; the recogniser opens its own stream.
+        for (const track of stream.getTracks())
+            track.stop();
+        this.#microphoneReady = true;
+    }
+    async #resolveSource(provider) {
+        if (this.#source)
+            return this.#source;
+        this.#sourcePromise ??= Promise.resolve().then(provider);
+        try {
+            const source = await this.#sourcePromise;
+            if (!this.#source) {
+                this.#source = source;
+                this.#subscribe(source);
+            }
+            return source;
+        }
+        catch (error) {
+            this.#sourcePromise = null;
+            throw error;
+        }
+    }
+    #subscribe(source) {
+        this.#unsubscribe.push(source.onResult((result) => {
+            if (this.#disposed || this.recording)
+                return;
+            if (!(this.active || this.state === 'stopping'))
+                return;
+            this.#heard = true;
+            if (result.isFinal) {
+                this.interim = '';
+                this.#options.onInterim?.('');
+                const text = result.text.trim();
+                if (text)
+                    this.#options.onText(text);
+            }
+            else {
+                this.interim = result.text;
+                this.#options.onInterim?.(result.text);
+            }
+        }), source.onError((error) => {
+            if (this.#disposed || this.recording)
+                return;
+            if (this.state === 'idle' || this.state === 'error')
+                return;
+            // Cut off after the person asked it to stop: that is the stop.
+            if (this.state === 'stopping' &&
+                classifyDictationError(error) === 'interrupted')
+                return;
+            if (this.active) {
+                this.#failOrRecord(error, 'error', this.#session);
+                return;
+            }
+            this.#failWith(error, 'error');
+        }), source.onEnd(() => {
+            if (this.#disposed || this.recording)
+                return;
+            if (this.state === 'error' || this.state === 'idle')
+                return;
+            if (this.state === 'stopping') {
+                this.#idle();
+                return;
+            }
+            // Ended by itself, with no error. Before it ever listened, or within
+            // `earlyEndMs` without hearing anything, it did not really work
+            // (Brave ends like this): say so rather than going quiet, or record
+            // instead when we can.
+            const early = this.#options.earlyEndMs ?? 1000;
+            if (this.state === 'starting' ||
+                (early > 0 && !this.#heard && Date.now() - this.#listeningAt < early)) {
+                const message = 'Speech recognition ended straight away';
+                if (this.#shouldRecordInstead('interrupted', 'end')) {
+                    this.#log('interrupted', null, message, 'end', true);
+                    void this.#switchToRecording(this.#session);
+                    return;
+                }
+                this.#fail('interrupted', null, message, 'end');
+                return;
+            }
+            this.#idle();
+        }));
+        if (source.onStart) {
+            this.#unsubscribe.push(source.onStart(() => {
+                if (this.recording)
+                    return;
+                if (!this.#disposed && this.state === 'starting') {
+                    this.#listening();
+                    return;
+                }
+                if (!this.#disposed && this.state === 'listening')
+                    return;
+                // Stopped (or disposed) before the microphone opened: close it again.
+                void source.stop().catch(() => undefined);
+            }));
+        }
+    }
+    #listening() {
+        this.state = 'listening';
+        this.#listeningAt = Date.now();
+        void this.#playBeep().then((played) => {
+            this.beepPending = !played && this.state === 'listening';
+        });
+    }
+    async #playBeep() {
+        const beep = this.#options.beep ?? true;
+        if (beep === false)
+            return true;
+        try {
+            return beep === true ? await playReadyBeep() : Boolean(await beep());
+        }
+        catch {
+            return false;
+        }
+    }
+    #idle() {
+        this.#clearStopTimer();
+        this.state = 'idle';
+        this.interim = '';
+        this.beepPending = false;
+        this.recording = false;
+    }
+    #failWith(error, stage) {
+        this.#fail(classifyDictationError(error), dictationErrorCode(error), errorMessage(error), stage);
+    }
+    #log(kind, code, message, stage, fallback = false) {
+        const log = this.#options.log ?? defaultLog;
+        try {
+            log({
+                kind,
+                code,
+                message,
+                stage,
+                ...(fallback ? { fallback: 'recording' } : {}),
+            });
+        }
+        catch {
+            // A broken logger must not hide the error from the person.
+        }
+    }
+    #fail(kind, code, message, stage) {
+        this.#clearStopTimer();
+        this.#dropCapture(true);
+        this.#log(kind, code, message, stage);
+        this.errorKind = kind;
+        this.errorCode = code;
+        this.state = 'error';
+        this.interim = '';
+        this.beepPending = false;
+        this.recording = false;
+    }
+    #clearStopTimer() {
+        if (this.#stopTimer)
+            clearTimeout(this.#stopTimer);
+        this.#stopTimer = null;
+    }
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error ?? 'unknown');
+}