@the-inclusionist/engine 9.0.0 → 11.0.0

package/dist-pkg/ui/voice-control.js

@@ -0,0 +1,276 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// ui/voice-control — PLAYING BY VOICE, PUT TOGETHER (ADR-0189, ADR-0193, ADR-0194, ADR-0204 erratum; issues #184, #190).
+//
+// The four pieces that existed separately, joined: the Dev's vocabulary and the firing rule (`input/voice-map`), the recogniser
+// opened from the delivery (`platform/vosk-runtime`), the microphone that stays open (`platform/voice-listener`), and the
+// presses on the virtual controller with the source `fala` (ADR-0111). The 👄 of the quick bar drives it; what cannot start is
+// SAID, written once in `problems`, and puts the icon back to off — a child must never meet a button that does nothing.
+//
+// 📌 THE GRAMMAR FOLLOWS THE MENU (ADR-0194): saying the name of an item activates it, so the words the open menu is showing
+// join the closed grammar while it is open and leave with it.
+//
+// 🎯 A NAME HEARD IS A CURSOR AND A CONFIRM, NOT A CLICK (ADR-0194 §2): the root's menu navigation puts the cursor on the
+// named item (`pointAt`), and the confirm position is pressed on the virtual controller like any other spoken word — so the
+// item is activated by the same path, with the same spoken feedback, as a child confirming it with the cursor on it. WHEN a
+// name fires is the reader's (`input/voice-map` over `platform/speech-recognition`): at once, unless another item's name
+// continues it («voltar» / «voltar ao jogo»), which waits for the end of the utterance (§3). A LOCKED item's name is heard
+// too and takes the same path: the confirm reaches the item's own press, which says its reason and does nothing (§5, ADR-0161).
+// A name with a word the loaded model lacks cannot be heard at all, and is a line of `problems` (§4).
+// ⚠️ EXCEPT WITH ONE BUTTON ONLY ON (ADR-0218 §4): then every word heard, a name included, is the child's switch and nothing more
+// — one press, which the controller takes as «take the one shown», and no cursor put anywhere first.
+//
+// ⚠️ A SPOKEN COMMAND IS A TAP, NOT A HOLD. The word arrives, the position is pressed and let go — one press, one release.
+// Whether that tap keeps a direction going is the latch's, and on speech the latch starts as the GAME answers `holdsKeys()`
+// (ADR-0249, resolved by the root when the press arrives): a platform game keeps walking on «direita», a quiz moves once on
+// «abaixo», and the child may change either for her voice.
+import { voiceGrammar, createVoiceCommands } from '../input/voice-map.js';
+import { loadVoskRuntime } from '../platform/vosk-runtime.js';
+import { startVoiceListening } from '../platform/voice-listener.js';
+import { spokenText } from '../platform/speech-recognition.js';
+import { MENU_CONFIRM } from './menu-intent.js';
+/** How long a spoken position stays pressed. The same pulse the scan uses: long enough for a game to see a press and a release. */
+export const VOICE_PULSE_MS = 400;
+/** The base of a language tag: `en-US` and `en` are the same model. */
+const baseOf = (tag) => tag.split('-')[0].toLowerCase();
+export function createVoiceControl(d) {
+    const { t } = d;
+    const loadRuntime = d.loadRuntime ?? loadVoskRuntime;
+    const listen = d.listen ?? startVoiceListening;
+    const said = new Set();
+    let on = false, starting = null;
+    let listener = null;
+    let commands = null;
+    let grammarGiven = '';
+    /** The words the model now listening knows, once read (`platform/vosk-vocabulary`); `null` while unknown. */
+    let vocabulary = null;
+    /** Bumped by every stop, so a vocabulary that arrives for a recogniser already replaced is dropped. */
+    let generation = 0;
+    /**
+     * 🔴 THE LANGUAGE THE LISTENING STARTED IN, read ONCE. The model and the grammar must come from the same reading: `start()` used
+     * to read the language again after the model had loaded, and a page booting in English (pt-BR until its stored language
+     * applies) opened the Portuguese model with the English grammar — 15 «missing in vocabulary», measured on 2026-09-27.
+     */
+    let listeningLanguage = '';
+    /** A choice among the game's options borrowing this ear (ADR-0256): what is heard goes to it, and no command is obeyed. */
+    let lent = null;
+    /** The open menu's names that can enter the grammar. A one-letter name stays out: in a closed grammar every short noise lands on it. */
+    const sayableNames = () => d.menuWords().filter((n) => spokenText(n).length > 1);
+    /**
+     * 🔴 A NAME THE MODEL CANNOT HEAR IS SAID TO THE ADULT (ADR-0194 §4, ADR-0169): Kaldi drops a grammar word its vocabulary lacks,
+     * in silence, and the item goes mute for the child — she can still walk to it with the direction words. One line per item and
+     * language, naming the words, the cost and the fix. Nothing is said while the vocabulary is unknown: no line beats a false one.
+     */
+    const reportUnsayable = (names) => {
+        const known = vocabulary;
+        if (!known)
+            return;
+        const language = listeningLanguage;
+        for (const name of names) {
+            const lacking = [...new Set(spokenText(name).split(' '))].filter((w) => w && !known.has(w));
+            const kind = `unsayable:${language}:${name}`;
+            if (!lacking.length || said.has(kind))
+                continue;
+            said.add(kind);
+            d.report(`voice control: the menu item "${name}" has ${lacking.length === 1 ? 'a word' : 'words'} the ${language} speech `
+                + `model does not know (${lacking.map((w) => `"${w}"`).join(', ')}) — the child cannot choose this item by saying its name, `
+                + `only by walking to it with the direction words; reword the item's label in the ${language} dictionary with words the `
+                + 'model knows (ADR-0194 §4)');
+        }
+    };
+    /** The open menu's names, handed to the reader and checked against the model, and the grammar with them. */
+    const followMenu = () => {
+        const names = sayableNames();
+        commands?.items(names);
+        reportUnsayable(names);
+        return voiceGrammar(listeningLanguage, names);
+    };
+    const command = (action) => {
+        d.controller.press(action, 'fala');
+        d.after(() => d.controller.release(action, 'fala'), VOICE_PULSE_MS);
+    };
+    /*
+     * 🔴 ONE WORD HEARD IS ONE ACTION (ADR-0111 errata, ADR-0218 §4). With one button only on, the controller takes every press as
+     * «take the one shown», so a word may do nothing BESIDES its press: asked here, first, for every word — a position, a name, any
+     * kind a later reader adds. 📏 Before, a name was still pointed at: with the chip on «próximo» the cursor jumped to the named
+     * item and then stepped on from it; on «cancelar» the press meant nothing and the cursor had moved anyway.
+     */
+    const obey = (heard) => {
+        for (const c of heard ?? []) {
+            if (d.oneButtonOnly())
+                command(c.kind === 'position' ? c.action : MENU_CONFIRM);
+            else if (c.kind === 'position')
+                command(c.action);
+            else if (d.pointAt(c.name))
+                command(MENU_CONFIRM);
+        }
+    };
+    const stop = () => {
+        const going = listener;
+        listener = null;
+        commands = null;
+        lent = null;
+        vocabulary = null;
+        generation += 1;
+        void going?.stop();
+    };
+    /**
+     * Nothing started: the child hears why, the adult reads it once, and the icon goes back to off. A failure that arrives after
+     * the 👄 was turned off (or the root disposed, ADR-0220) has nobody waiting for it and says and writes nothing.
+     */
+    const failed = (kind, line, spoken) => {
+        if (!on)
+            return;
+        if (!said.has(kind)) {
+            said.add(kind);
+            d.report(line);
+        }
+        d.alert(spoken);
+        d.turnOff();
+    };
+    const start = async () => {
+        /*
+         * 🔴 THE LOAD IS INSIDE A `try` AND THIS WAS MEASURED, not foreseen: on 2026-09-21 the delivery's bundle turned out to be an
+         * ES module and the loader threw. Nothing here caught it, so the rejection died as an unhandled promise and the 👄 STAYED
+         * LIT over a microphone that had never opened — the exact defect this module exists to prevent, one layer above where it
+         * was being prevented. A failure that has no name is still a failure the child has to be told about.
+         */
+        let load;
+        const language = d.language();
+        try {
+            load = await loadRuntime({
+                base: d.base, language, hasFile: d.hasFile, loadBundle: d.loadBundle, ...(d.fetch ? { fetch: d.fetch } : {}),
+            });
+        }
+        catch (e) {
+            failed('runtime', `voice control: the recogniser did not open (${e instanceof Error ? e.message : String(e)}) — the child `
+                + 'cannot play by speaking; check that the delivery carries the command files', t('sr.voice.failed'));
+            return;
+        }
+        if (!load.ok) {
+            /*
+             * 📌 ONE LINE PER LANGUAGE, NAMING THAT LANGUAGE'S FIX (ADR-0225 and its erratum, ADR-0169). The delivery carries every
+             * language unless its `--commands` list narrowed it, and the install fetches every one the delivery carries — so a model
+             * missing here was left out of the delivery or has not come down yet, and «open it online» alone sends the adult after a
+             * download that cannot happen in the first case. The reading's line names its two fixes the same way.
+             */
+            const lang = language.split('-')[0].toLowerCase();
+            failed(`files:${lang}`, `voice control: ${load.missing.join(', ')} not on this device for ${language} — the child cannot play `
+                + `by speaking in this language; build the delivery with \`npx inclusionist-heavy\` without \`--commands\` (it carries every `
+                + `language) or with \`--commands ${lang}\` in its list, and open the game once online so the install fetches them`, t('sr.voice.needsInternet'));
+            return;
+        }
+        // the language changed while the model loaded (a page booting before its stored language applies): start again in it
+        if (baseOf(d.language()) !== baseOf(language)) {
+            await start();
+            return;
+        }
+        listeningLanguage = language;
+        commands = createVoiceCommands(language);
+        const grammar = followMenu();
+        grammarGiven = grammar.join('\n');
+        try {
+            listener = await listen({
+                model: load.model,
+                grammar,
+                getUserMedia: d.getUserMedia,
+                createContext: d.createContext,
+                onPartial: (text) => { if (lent)
+                    lent(text, false);
+                else
+                    obey(commands?.partial(text)); },
+                onFinal: (text) => { if (lent)
+                    lent(text, true);
+                else
+                    obey(commands?.final(text)); },
+            });
+        }
+        catch {
+            failed('microphone', 'voice control: the microphone did not open — the child cannot play by speaking; allow the microphone '
+                + 'for this page, or plug one in', t('sr.voice.noMicrophone'));
+            return;
+        }
+        // ⚠️ TURNED OFF WHILE IT WAS STARTING: the microphone opened after the child let go of the icon, and a listener nobody asked
+        // for would go on hearing her. The check is here and not only in `apply`, because starting is not instantaneous.
+        if (!on) {
+            stop();
+            return;
+        }
+        // a menu that opened or closed while the microphone was opening changed the names after the grammar above was built
+        api.refreshGrammar();
+        d.say(t('sr.voice.ready'));
+        // 📌 THE VOCABULARY IS READ AFTER THE CHILD CAN SPEAK, never before: it is a report for the adult, and reading the model's
+        // archive a second time must not delay the first command. It checks the names showing when it arrives and every later menu.
+        const mine = generation;
+        void load.vocabulary?.().then((words) => {
+            if (mine !== generation || !listener)
+                return;
+            vocabulary = words;
+            reportUnsayable(sayableNames());
+        });
+    };
+    const api = {
+        async apply(next) {
+            on = next;
+            if (!next) {
+                stop();
+                return;
+            }
+            if (listener || starting)
+                return;
+            starting = start().finally(() => { starting = null; });
+            await starting;
+        },
+        refreshGrammar() {
+            // lent to a choice: the options are the grammar until it gives the ear back
+            if (!listener || lent)
+                return;
+            // 📌 A new grammar is a new recogniser (`platform/voice-listener`), and the root asks on every menu change it sees —
+            // most of which change no name. Only a grammar that differs is handed on.
+            const grammar = followMenu();
+            const given = grammar.join('\n');
+            if (given === grammarGiven)
+                return;
+            grammarGiven = given;
+            listener.setGrammar(grammar);
+        },
+        async languageChanged() {
+            /*
+             * 🔴 THE RECOGNISER STOPS CHOOSING ITS LANGUAGE ONCE (ADR-0225). `start()` reads `d.language()` for the MODEL and for the
+             * vocabulary, and it used to read it only at the moment the 👄 was switched on — so a child who changed language while
+             * listening went on being heard in the old one.
+             * ⚠️ AND THAT IS WORSE THAN STOPPING, which is why this is not cosmetic: the grammar DOES follow, because it is rebuilt
+             * whenever a menu opens (ADR-0194). So after a change the new language's words were fed to the old language's model, and
+             * a closed grammar answers with the nearest candidate — measured in the lab on 2026-09-14, where «configurações de
+             * inclusão» came back as «quatro». A microphone that acts on a word the child did not say is the defect here.
+             * 📌 OFF IS ALREADY RIGHT: the next start reads the language then. This only has work to do while it is listening.
+             */
+            if (!on)
+                return;
+            // ⚠️ A START IN FLIGHT IS IN THE OLD LANGUAGE. Letting it finish and replacing it costs one opening; skipping it would
+            // leave `starting` to install the old model on top of the new one, which is the race the `if (!on)` in `start` exists for.
+            await starting;
+            // a start that noticed the change itself is already listening in the new language: nothing to redo
+            if (listener && baseOf(listeningLanguage) === baseOf(d.language()))
+                return;
+            stop();
+            await api.apply(true);
+        },
+        borrow(language, grammar, onHeard) {
+            if (!listener || lent || baseOf(language) !== baseOf(listeningLanguage))
+                return null;
+            const held = listener;
+            lent = onHeard;
+            held.setGrammar(grammar);
+            grammarGiven = '';
+            return {
+                async stop() {
+                    if (lent !== onHeard)
+                        return;
+                    lent = null;
+                    api.refreshGrammar(); // the menu's grammar back
+                },
+            };
+        },
+    };
+    return api;
+}

package/dist-pkg/ui/voice-settings.d.ts

@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import type { Translator } from '../core/i18n.js';
+import type { DomQuery } from '../core/dom-query.js';
+import { type AudioCatState } from './audio-choices.js';
+export interface TtsPanelEngine {
+    id: string;
+    speak: (text: string) => void;
+}
+/** Minimal shape of the injected `tts` (platform/tts.ts's createTts() instance) this section drives. */
+export interface TtsPanel {
+    getEngineSel: () => string;
+    setEngineSel: (v: string) => void;
+    getEngine: () => TtsPanelEngine | null;
+    getVoiceObj: () => SpeechSynthesisVoice | null;
+    setVoiceObj: (v: SpeechSynthesisVoice | null) => void;
+    loadTTS: () => void;
+    narrate: (text: string) => void;
+    /**
+     * Does this assembly have a neural engine (the game's Kokoro port, ADR-0198)? Optional, absent reads `true`: a panel test
+     * fake written before this field has no opinion on neural engines.
+     */
+    neuralAvailable?: boolean;
+    /** The voices of the language (ADR-0185). Optional: a panel driven without them offers no «Voz» list and locks nothing. */
+    voices?: () => readonly PanelVoice[];
+    currentVoice?: () => PanelVoice | null;
+    setVoice?: (id: string) => boolean;
+}
+/** A voice as this panel lists it: `webspeech:<name>` for the browser's, `xx_name` for Kokoro's. */
+export interface PanelVoice {
+    readonly voice: string;
+    readonly engine?: string;
+    readonly recommended?: boolean;
+}
+/**
+ * The browser, as four functions.
+ *
+ * 🔴 A module that reaches `document` or `window` undoes a decision (ADR-0178), and the health ratchet refuses a NEW one that
+ * does. These four are what the voice section actually needs from the browser, and naming them is what makes the section
+ * testable: hand it a fake `doc`, a list of voices and two no-ops, and every branch below runs without a speech engine.
+ */
+export interface VoicePorts {
+    /** Makes one <option> — the only element this section builds. A FUNCTION and not the document: a port read at wiring
+     *  time would make a panel unmountable where there is no document yet, and the node suite mounts exactly there. */
+    readonly newOption: () => HTMLOptionElement;
+    /** The system voices, or an empty list where the browser has none or throws. */
+    readonly systemVoices: () => readonly SpeechSynthesisVoice[];
+    /** Speak the sample through the BROWSER's own synthesiser — the neural engine speaks through `tts` instead. */
+    readonly speakSample: (text: string, chosen: SpeechSynthesisVoice | null) => void;
+    /** Called again whenever the browser finishes loading its voice list: they arrive late, and often empty first.
+     *  ⚠️ The parameter is `again` and not `run`, which is a GAME ACTION word this layer does not say (the boundary of 2026-09-06). */
+    readonly whenVoicesChange: (again: () => void) => void;
+}
+/**
+ * What this section needs from the panel's ctx.
+ *
+ * ⚠️ The store shape is written here and NOT imported from `ui/settings-audio`, which declares the same two methods: that
+ * module imports this one, and importing it back would be the cycle ADR-0173 forbids — for a type as much as for a value.
+ */
+/**
+ * THE SETTINGS THIS SECTION READS AND WRITES — the page's settings store, built by the root (ADR-0232 D2c, issue #207): the
+ * spoken index (ADR-0044 item 3) and the speech rate (ADR-0183). The names are `core/state`'s, so a root passes the store
+ * itself; the reads are LIVE, because the quick bar and a second screen change them too.
+ */
+export interface VoiceSettingsStore {
+    readonly menuIndexOn: boolean;
+    setMenuIndexOnValue(on: boolean): void;
+    readonly speechPpm: number;
+    setSpeechPpmValue(ppm: number): void;
+}
+export interface VoiceSettingsCtx {
+    /**
+     * The root's translator (ADR-0232 D3): its `t`, and the page's language as a BCP-47 tag, which picks the system voices the
+     * list offers. A Pick and not a bare `t`, because the voice list reads the language.
+     */
+    translator: Pick<Translator, 't' | 'bcp47'>;
+    /** The page's settings store (see `VoiceSettingsStore`). REQUIRED: a section reading from nowhere would show the defaults. */
+    settings: VoiceSettingsStore;
+    $: DomQuery;
+    srSay: (msg: string) => void;
+    toggleBtn: (btn: HTMLElement, on: boolean) => void;
+    store: {
+        get(key: string, fallback?: string | null): string | null;
+        set(key: string, value: string | number | boolean): boolean;
+    };
+    tts: TtsPanel;
+    getAudioCat: () => Record<string, AudioCatState> | null;
+    setCatGain: (k: string) => void;
+}
+export interface VoiceSettings {
+    /** Everything the section shows, refreshed from the stored state. Called by the panel's own render. */
+    render: () => void;
+    /** Just the narration switch, its volume and the chosen engine. Exported because the root reflects it from outside. */
+    reflectTts: () => void;
+}
+export declare function createVoiceSettings(ctx: VoiceSettingsCtx, ports: VoicePorts): VoiceSettings;