@the-inclusionist/engine 8.0.0 → 10.0.0

package/dist-pkg/platform/tts.js

@@ -1,66 +1,111 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-// platform/tts — narração por voz. Motor NEURAL pt-BR carregado LAZY pela PORTA que o jogo fornece (ADR-0094; modelo
-// em cache OPFS → a 2ª sessão fala offline), com FALLBACK imediato p/ a voz nativa do navegador (Web Speech) — que é
-// também o que se ouve quando não há porta nenhuma, e é por isso que a ausência dela não cala nada. narrate() é o
-// ponto de entrada, gated pelo toggle 'Narração (TTS)' do mixer (audioCat.tts.on) — independe das legendas. As funções de
-// PAINEL (populateTTSEngines/Voices/reflectTTS) ficam no game.js (→ ui/settings-audio, #38→#54) e usam get/setEngineSel +
-// get/setVoiceObj daqui. Injeção por closure. Ver docs/plano-tts-fase-f5.md + docs/5-Refactoring/plano-modularizacao-mapa.md.
-import * as store from './storage.js';
-import { t, bcp47 } from '../core/i18n.js';
-import { criarFalaInterrompivel } from './interruptible-speech.js';
-// ⚠️ MIGRAÇÃO PENDENTE (ADR-0022): a implementação neural de hoje é o @mintplex-labs/piper-tts-web e será SUBSTITUÍDA
-// por sherpa-onnx-wasm (loader universal, modelos VITS/Piper + Kokoro-multi-lang carregados de qq URL R2 em runtime via
-// FS.writeFile; lazy-fetch; eSpeak/Web Speech de fallback).
-// 🔴 O MOTIVO ESCRITO AQUI ERA «@mintplex-labs foi descontinuado e só carrega 2 vozes pt-BR», E AS DUAS METADES
-// FORAM MEDIDAS FALSAS EM 2026-09-09 — não pela leitura de um registo, que é como elas se propagaram por seis
-// sítios, mas pelo registo npm e pelo pacote instalado:
-//   · DESCONTINUADO: nenhuma versão tem campo `deprecated`, e a 1.0.5 foi publicada em 2026-08-11 — depois de
-//     o ADR-0022 (2026-07-06) a dar por morta. A frase é de Julho e está atribuída ao Dev; o registo não a nega
-//     no passado, nega-a HOJE.
-//   · DUAS VOZES pt-BR: o `VoiceId` da 1.0.4 instalada traz 118 vozes, e entre elas as QUATRO exactas do
-//     ADR-0110 — `pt_BR-faber-medium`, `en_US-ryan-medium`, `en_US-amy-medium`, `es_MX-claude-high`.
-// ⚠️ O QUE CONTINUA VERDADEIRO, e é mais afiado do que o motivo velho: o bundle prende `HF_BASE` a
-// `huggingface.co/diffusionstudio/piper-voices` COM UMA GUARDA (`if (!url.match("https://huggingface.co")) return`),
-// logo nenhum espelho de escola é alcançável por ele; e ele busca o PRÓPRIO runtime de cdnjs/jsDelivr por
-// omissão, que é o que o ADR-0114 retira. A montante, o Piper mudou para `OHF-Voice/piper1-gpl` e a Open Home
-// Foundation procura mantenedores — o risco existe, mas está noutro sítio.
-// 📌 A ESCOLHA CONTINUA A SER A #129, e este comentário não a toma: descreve o que foi medido para que o
-// próximo leitor não herde o motivo velho como se fosse medição.
-// ⚠️ E DESDE O ADR-0094 ESTE MÓDULO NÃO NOMEIA FORNECEDOR NENHUM — quem o nomeia é o JOGO, por `ctx.carregarVozNeural`.
-// O nome estava aqui num `import()` e o pacote em `devDependencies`, o que publicou uma engine que não compilava
-// (ADR-0093); pô-lo em `dependencies` consertava o build e obrigava todo consumidor a 135,4 MB de `onnxruntime-web`,
-// que o fornecedor traz como peer não-opcional. A porta é o que faz a troca do ADR-0022 não ser um segundo abalo.
-/**
- * Voz NEURAL por idioma, indexada pela etiqueta BCP-47 de core/i18n. Só o pt-BR tem uma hoje, e essa ausência
- * é agora CONSULTÁVEL em vez de presumida: `loadTTS` pergunta à tabela se existe voz para o idioma corrente,
- * em vez de carregar a portuguesa aconteça o que acontecer.
- *
- * Era `TTS_SOURCES['pt-BR']` escrito à mão no ponto de uso, o que fazia o jogo em inglês baixar 25,6 MB de
- * voz portuguesa para ler texto em inglês com fonética errada — pior que não ter voz neural nenhuma, porque
- * gasta a banda da escola para entregar algo ininteligível.
- */
-const TTS_SOURCES = {
-    'pt-BR': { engine: 'piper', voice: 'pt_BR-faber-medium' },
-};
+// platform/tts — spoken narration. The browser's voice speaks first (Web Speech, ADR-0200); the neural fallback is Kokoro, which
+// THE ENGINE ITSELF loads from the delivery (ADR-0216 §1) the first time a child picks it — the game only says it wants one; the
+// loader, the browser's speech, the clock and the `<audio>` maker are the root's, received here (ADR-0232 D4). narrate()
+// is the entry point, gated by the mixer's narration toggle (audioCat.tts.on) — independent of captions. The PANEL's side (the
+// engine and voice lists) lives in `ui/voice-settings` and reads and writes the choice through this module. Injection by closure.
+import { KEYS } from './storage-keys.js';
+import { createInterruptibleSpeech } from './interruptible-speech.js';
+import { voicesForLocale } from './voice-plan.js';
+import { spokenWords, speechSeconds, speechPlaybackRate } from '../core/speech-rate.js';
+import { KOKORO_VOICES, tokenize, sentenceStyle, eFala, wavDe, } from './kokoro.js';
+// 📌 The only neural engine is Kokoro (ADR-0207): the engine's earlier neural engine left when the licence chain of its voices came
+// to light. 📌 And ADR-0094's fear — «a provider named here lands its runtime in every consumer's `node_modules`» — is answered by the
+// delivery and not by a port: the runtime is catalogued, fetched by the build and read from `heavy/`, never imported from npm.
+/** An engine the child may have stored before it left the engine (ADR-0207): read as no explicit choice, so the voice in use speaks. */
+const ENGINES_THAT_LEFT = ['piper'];
 export function createTts(ctx) {
-    let ttsEngine = null, ttsLoading = false, ttsFailed = false, _ttsPct = 0, _narrateCount = 0;
-    let _ttsVoiceObj = null; // voz do Web Speech selecionada
-    let ttsEngineSel = store.get(store.KEYS.ttsEngine, null) || 'webspeech'; // webspeech | piper | kokoro | kitten | espeak
+    const { t, bcp47 } = ctx.translator;
+    let ttsEngine = null, ttsLoading = false, ttsFailed = false, _narrateCount = 0;
+    let _ttsVoiceObj = null; // the selected Web Speech voice
+    // An engine set explicitly (stored, or by the panel) wins; otherwise the engine of the voice in use, which is the browser's when it
+    // offers one for the language (ADR-0200) — measured on the device at every call, since the browser lists its voices late.
+    const stored = ctx.store.get(KEYS.ttsEngine, null) || null; // webspeech | kokoro | kitten | espeak
+    let ttsEngineSelExplicito = stored && !ENGINES_THAT_LEFT.includes(stored) ? stored : null;
+    const engineSel = () => {
+        if (ttsEngineSelExplicito)
+            return ttsEngineSelExplicito;
+        const e = voiceInUse()?.engine;
+        // a fallback this game did not ask for is not the default: undeclared, the browser's voice (no «not bundled» alert)
+        return e === 'kokoro' && ctx.neuralVoice ? e : 'webspeech';
+    };
+    let kokoroBackend = null;
+    let loadedVoice = null; // the voice the loaded engine speaks; another choice reloads it
+    // Each voice's words per minute, averaged over the utterances long enough to measure (`core/speech-rate`): what a short
+    // utterance («Voltar») is played against. Neural voices by id; browser voices by name.
+    const voiceAverage = new Map();
+    const media = (voiceName, ppm) => {
+        if (!ppm)
+            return;
+        const before = voiceAverage.get(voiceName);
+        voiceAverage.set(voiceName, before ? before * 0.7 + ppm * 0.3 : ppm);
+    };
+    /**
+     * WEB SPEECH FIRST (ADR-0200; issue #190): the browser's voices for the language lead the list, so where the device already speaks
+     * no neural voice is loaded; then Kokoro's where the game declared it speaks neurally (ADR-0216 §3) — the fallback.
+     */
+    const browserVoices = () => {
+        let list = [];
+        try {
+            list = ctx.speech.synth()?.getVoices() ?? [];
+        }
+        catch { /* no speech synthesis here */ }
+        return voicesForLocale(bcp47(), list.map((v) => ({ locale: v.lang.replace('_', '-'), engine: 'webspeech', voice: 'webspeech:' + v.name })));
+    };
+    const availableVoices = () => [...browserVoices(), ...(ctx.neuralVoice ? voicesForLocale(bcp47(), KOKORO_VOICES) : [])];
+    /** The browser voice an entry names, or null. */
+    const browserVoice = (id) => {
+        if (!id?.startsWith('webspeech:'))
+            return null;
+        try {
+            return ctx.speech.synth()?.getVoices().find((v) => 'webspeech:' + v.name === id) ?? null;
+        }
+        catch {
+            return null;
+        }
+    };
+    function voiceInUse() {
+        const list = availableVoices();
+        const storedVoiceId = ctx.store.get(KEYS.ttsVoz, null);
+        return list.find((v) => v.voice === storedVoiceId) ?? list[0] ?? null;
+    }
+    function chooseVoice(id) {
+        if (!availableVoices().some((v) => v.voice === id))
+            return false;
+        ctx.store.set(KEYS.ttsVoz, id);
+        ttsEngineSelExplicito = availableVoices().find((v) => v.voice === id).engine;
+        _ttsVoiceObj = browserVoice(id);
+        return true;
+    }
     function speakWebSpeech(text) {
         try {
-            const ss = window.speechSynthesis;
+            const ss = ctx.speech.synth();
             if (!ss)
                 return false;
             ss.cancel();
-            // `u.lang` era 'pt-BR' fixo. Com o jogo em inglês ou espanhol isso pedia ao navegador uma voz
-            // PORTUGUESA para um texto que não é português — e o resultado não é sotaque, é ininteligível: a
-            // fonética errada aplicada às letras erradas. Segue o idioma do jogo.
-            const u = new SpeechSynthesisUtterance(text);
+            // `u.lang` follows the game's language. Fixed at pt-BR, a game in English or Spanish would ask the browser for a
+            // PORTUGUESE voice for a text that is not Portuguese — and the result is not an accent, it is unintelligible: the
+            // wrong phonetics applied to the wrong letters.
+            const voiceName = _ttsVoiceObj ?? browserVoice(voiceInUse()?.voice);
+            const u = ctx.speech.utterance(text);
             u.lang = bcp47();
-            if (_ttsVoiceObj)
-                u.voice = _ttsVoiceObj;
-            u.rate = 1;
+            if (voiceName)
+                u.voice = voiceName;
             u.volume = Math.min(1, ctx.getVolume() * 1.4);
+            // ADR-0183 §1: the browser's `rate` is a multiplier. The voice's words a minute at rate 1 is measured on its own utterances
+            // (start to end, so the silent ends count — an approximation the neural path does not need); until one is measured, 1.
+            const voiceIdOf = 'webspeech:' + (voiceName?.name ?? bcp47());
+            const words = spokenWords(text);
+            const voiceBase = voiceAverage.get(voiceIdOf) ?? null;
+            u.rate = ctx.getSpeechPpm && voiceBase ? speechPlaybackRate(0, 0, ctx.getSpeechPpm(), voiceBase).rate : 1;
+            let startAt = 0;
+            u.onstart = () => { startAt = ctx.now(); };
+            u.onend = () => {
+                const seconds = startAt ? (ctx.now() - startAt) / 1000 : 0;
+                const measured = speechPlaybackRate(words, seconds, 150, null).voiceWpm;
+                if (measured)
+                    media(voiceIdOf, measured / u.rate);
+            };
             ss.speak(u);
             return true;
         }
@@ -68,104 +113,157 @@ export function createTts(ctx) {
             return false;
         }
     }
+    /**
+     * A neural voice's utterances, whatever made the WAV (ADR-0183 §1): every utterance is measured — its words over its speech time,
+     * silent ends trimmed — and played at the child's rate over the voice's, through a media element that keeps the pitch.
+     */
+    function speakByWav(voiceId, produce) {
+        return createInterruptibleSpeech({
+            synthesize: async (text) => {
+                const bytes = await produce(text);
+                const ac = ctx.ensureAC();
+                if (!ac)
+                    throw new Error('AudioContext unavailable'); // the `catch` there handles it: this item is silent, the engine lives
+                let speedRatio = 1;
+                if (ctx.getSpeechPpm) {
+                    const buf = await ac.decodeAudioData(bytes.slice(0));
+                    const measure = speechPlaybackRate(spokenWords(text), speechSeconds(buf.getChannelData(0), buf.sampleRate), ctx.getSpeechPpm(), voiceAverage.get(voiceId) ?? null);
+                    media(voiceId, measure.voiceWpm);
+                    speedRatio = measure.rate;
+                }
+                return { url: URL.createObjectURL(new Blob([bytes], { type: 'audio/wav' })), rate: speedRatio };
+            },
+            play: (sound, onFinish) => {
+                const ac = ctx.ensureAC();
+                if (!ac) {
+                    URL.revokeObjectURL(sound.url);
+                    return null;
+                }
+                const el = ctx.createAudio();
+                el.preservesPitch = true;
+                el.src = sound.url;
+                el.playbackRate = sound.rate;
+                try {
+                    ac.createMediaElementSource(el).connect(ctx.catNode('tts') || ctx.audioOut() || ac.destination);
+                }
+                catch { /* already routed */ }
+                el.onended = () => { URL.revokeObjectURL(sound.url); onFinish(); };
+                void el.play().catch(() => { URL.revokeObjectURL(sound.url); onFinish(); });
+                return el;
+            },
+            stop: (el) => { el.pause(); URL.revokeObjectURL(el.src); },
+        });
+    }
+    /**
+     * KOKORO (ADR-0186, ADR-0216; issues #181, #200): the voice's text becomes espeak-ng phonemes in its language, then the model's
+     * ids and a waveform. WebGPU only where a short test synthesis is speech — measured, WebGPU can run and return noise — and WASM
+     * with threads otherwise (ADR-0192).
+     */
+    function loadKokoroEngine(forVoice) {
+        const load = ctx.loadKokoro;
+        const kv = KOKORO_VOICES.find((v) => v.voice === forVoice.voice);
+        if (!kv) {
+            ttsFailed = true;
+            ctx.srAlert(t('sr.tts.neuralNotBundled'));
+            return;
+        }
+        ttsLoading = true;
+        loadedVoice = forVoice.voice;
+        const t0 = ctx.now();
+        ctx.srSay(t('sr.tts.downloading'));
+        load().then(async (mod) => {
+            const [symbolIds, table] = await Promise.all([mod.vocabulary(), mod.voice(kv.voice)]);
+            const synthesiseWith = async (kokoroSession, text) => {
+                const ids = tokenize(await mod.phonemize(text, kv.espeak), symbolIds);
+                return kokoroSession.synthesize(ids, sentenceStyle(table, ids.length - 2));
+            };
+            let kokoroSession = null;
+            try {
+                const gpu = await mod.session('webgpu');
+                if (eFala(await synthesiseWith(gpu, t('tts.kokoro.teste')))) {
+                    kokoroSession = gpu;
+                    kokoroBackend = 'webgpu';
+                }
+            }
+            catch { /* no WebGPU here: WASM below */ }
+            if (!kokoroSession) {
+                kokoroSession = await mod.session('wasm');
+                kokoroBackend = 'wasm';
+            }
+            const used = kokoroSession;
+            const fala = speakByWav(kv.voice, async (text) => wavDe(await synthesiseWith(used, text)));
+            ttsEngine = { id: 'kokoro', speak: (text) => { fala.speak(text); } };
+            ttsLoading = false;
+            try {
+                ctx.speech.synth()?.cancel();
+            }
+            catch (e) { /* noop */ }
+            narrate(t('sr.tts.ready', { s: ((ctx.now() - t0) / 1000).toFixed(0) }));
+        }).catch(() => { ttsLoading = false; ttsFailed = true; ctx.srAlert(t('sr.tts.loadFailed')); });
+    }
     function loadTTS() {
         if (ttsEngine || ttsLoading || ttsFailed)
             return;
-        if (ttsEngineSel !== 'piper') { // Kokoro/Kitten/eSpeak NG ainda não entraram
-            if (ttsEngineSel !== 'webspeech')
+        const sel = engineSel();
+        if (sel !== 'kokoro') { // Kitten and eSpeak NG are not built
+            if (sel !== 'webspeech')
                 ctx.srAlert(t('sr.tts.engineNoLanguage'));
             return;
         }
-        // ESTA MONTAGEM NÃO TRAZ MOTOR NEURAL (ADR-0094). Vem ANTES da pergunta do idioma de propósito: sem
-        // porta, não há voz neural em idioma nenhum, e dizer «não há voz para o teu idioma» faria a criança
-        // pensar que trocar de idioma resolveria. `ttsFailed` para não repetir a pergunta a cada fala.
-        const carregar = ctx.carregarVozNeural;
-        if (!carregar) {
+        // THIS GAME DID NOT ASK FOR A NEURAL VOICE (ADR-0216 §3). It comes BEFORE the language question on purpose: without
+        // the declaration there is no neural voice in any language — the delivery does not carry the model — and saying "there
+        // is no voice for your language" would make the child think switching language would fix it. `ttsFailed` so it is not
+        // repeated on every utterance.
+        if (!ctx.neuralVoice) {
             ttsFailed = true;
             ctx.srAlert(t('sr.tts.neuralNotBundled'));
             return;
         }
-        // O Piper só tem voz para os idiomas em TTS_SOURCES. Sem voz para o idioma corrente, NÃO se baixa a de
-        // outro idioma: avisa e segue na voz do navegador, que fala a língua certa.
-        const fonte = TTS_SOURCES[bcp47()];
-        if (!fonte) {
+        // The voice in use for the current language (ADR-0185); a browser voice in use with Kokoro set explicitly gives way to the
+        // language's first Kokoro voice. None: another language's voice is NOT fetched — it would read this text with the wrong phonetics.
+        const current = voiceInUse();
+        const forVoice = current?.engine === 'kokoro' ? current : voicesForLocale(bcp47(), KOKORO_VOICES)[0] ?? null;
+        if (!forVoice) {
             ctx.srAlert(t('sr.tts.noNeuralForLanguage'));
             return;
         }
-        ttsLoading = true;
-        const t0 = performance.now();
-        ctx.srSay(t('sr.tts.downloading'));
-        carregar().then(async (mod) => {
-            const session = await mod.TtsSession.create({ voiceId: fonte.voice,
-                progress: (p) => { if (!p || !p.total)
-                    return; const pct = Math.round(p.loaded * 100 / p.total); if (pct >= _ttsPct + 25 && pct < 100) {
-                    _ttsPct = pct;
-                    ctx.srSay(t('sr.tts.progress', { pct }));
-                } },
-                logger: () => { } });
-            // A FILA DE UM SAIU (ADR-0044, item 2). Ela era `if (busy) next = text; else speakNow(text)`, e o
-            // efeito, varrendo cinco itens de menu, era ouvir o PRIMEIRO inteiro e depois o ÚLTIMO — os três do
-            // meio sumiam, porque cada pedido sobrescrevia o `next`. Lento e lacunar, e quem não enxerga navega
-            // POR ESCUTA: a escuta ficava vários itens atrás do foco.
-            //
-            // A política agora mora em `platform/interruptible-speech`, pura e testada com falsos. Aqui só se
-            // diz COMO sintetizar, tocar e parar — o quando é lá, com a guarda de geração que impede uma síntese
-            // lenta de atropelar a mais nova.
-            const fala = criarFalaInterrompivel({
-                sintetizar: async (texto) => {
-                    const wav = await session.predict(texto);
-                    const ac = ctx.ensureAC();
-                    if (!ac)
-                        throw new Error('AudioContext unavailable'); // o `catch` de lá trata: silêncio deste item, motor vivo
-                    return ac.decodeAudioData(await wav.arrayBuffer());
-                },
-                tocar: (buf, aoTerminar) => {
-                    const ac = ctx.ensureAC();
-                    if (!ac)
-                        return null;
-                    const src = ac.createBufferSource();
-                    src.buffer = buf;
-                    src.connect(ctx.catNode('tts') || ctx.audioOut() || ac.destination);
-                    src.onended = aoTerminar;
-                    src.start();
-                    return src;
-                },
-                parar: (src) => { src.stop(); },
-            });
-            ttsEngine = { id: 'piper', speak: (text) => { fala.falar(text); } };
-            ttsLoading = false;
-            try {
-                window.speechSynthesis && window.speechSynthesis.cancel();
-            }
-            catch (e) { /* noop */ }
-            narrate(t('sr.tts.ready', { s: ((performance.now() - t0) / 1000).toFixed(0) })); // já sai NA voz nova
-        }).catch(() => { ttsLoading = false; ttsFailed = true; ctx.srAlert(t('sr.tts.loadFailed')); });
+        loadKokoroEngine(forVoice);
     }
     function ttsSpeak(text) {
-        if (ttsEngineSel !== 'webspeech') {
-            if (ttsEngine && ttsEngine.id === ttsEngineSel && ttsEngine.speak) {
+        if (engineSel() !== 'webspeech') {
+            // another voice picked, or the language changed, since the engine loaded: load the voice in use
+            const inUse = voiceInUse();
+            if (ttsEngine && inUse?.engine !== 'webspeech' && loadedVoice !== inUse?.voice) {
+                ttsEngine = null;
+                ttsFailed = false;
+            }
+            if (ttsEngine && ttsEngine.speak) {
                 try {
                     ttsEngine.speak(text);
                 }
                 catch (e) { /* noop */ }
                 return true;
             }
-            loadTTS(); // motor neural (baixando/indisponível) → cai no fallback
+            loadTTS(); // the neural engine is downloading or unavailable → falls back
         }
-        return speakWebSpeech(text); // fallback imediato: Web Speech, no idioma do jogo
+        return speakWebSpeech(text); // immediate fallback: Web Speech, in the game's language
     }
     function narrate(text) {
         const cat = ctx.getAudioCat();
         if (!ctx.getSoundOn() || !cat || !cat.tts || !cat.tts.on || !text)
             return;
+        if (availableVoices().length === 0)
+            return; // no voice speaks this language: narration is locked (ADR-0185 §4)
         _narrateCount++;
         ttsSpeak(text);
     }
     return {
         narrate, ttsSpeak, loadTTS, speakWebSpeech,
-        getEngineSel: () => ttsEngineSel, setEngineSel: (v) => { ttsEngineSel = v; },
+        getEngineSel: engineSel, setEngineSel: (v) => { ttsEngineSelExplicito = v; },
         getEngine: () => ttsEngine, getVoiceObj: () => _ttsVoiceObj, setVoiceObj: (v) => { _ttsVoiceObj = v; },
         get loading() { return ttsLoading; }, get failed() { return ttsFailed; }, get narrateCount() { return _narrateCount; },
-        get neuralDisponivel() { return !!ctx.carregarVozNeural; },
+        get neuralAvailable() { return !!ctx.neuralVoice; },
+        get kokoroDevice() { return kokoroBackend; },
+        voices: availableVoices, currentVoice: voiceInUse, setVoice: chooseVoice,
     };
 }

package/dist-pkg/platform/vision-loop.d.ts

@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+export type LoopHealth = 'ok' | 'slow' | 'stalled';
+export interface VisionLoopDeps {
+    readonly requestFrame: (cb: (ms: number) => void) => number;
+    readonly cancelFrame: (handle: number) => void;
+    readonly now: () => number;
+    readonly every: (cb: () => void, ms: number) => number;
+    readonly stopEvery: (handle: number) => void;
+}
+export interface VisionLoopOptions {
+    readonly onFrame: (ms: number) => void;
+    readonly onError?: (error: unknown) => void;
+    readonly onHealth?: (health: LoopHealth, framesPerSecond: number) => void;
+    readonly floorFps?: number;
+    readonly stallMs?: number;
+    readonly watchMs?: number;
+}
+export interface VisionLoop {
+    start(): void;
+    stop(): void;
+    health(): LoopHealth;
+}
+export declare function createVisionLoop(deps: VisionLoopDeps, { onFrame, onError, onHealth, floorFps, stallMs, watchMs }: VisionLoopOptions): VisionLoop;

package/dist-pkg/platform/vision-loop.js

@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// platform/vision-loop — THE CAMERA'S FRAME LOOP, WHICH CANNOT STOP IN SILENCE (ADR-0213; issue #196).
+//
+// · The next frame is booked BEFORE anything that could throw. The lab's loop booked it at the end, one item threw on every frame, and the
+//   eye control stopped on its first frame with nothing on screen saying so — every measurement taken after that was of a stopped page.
+// · A frame that throws is handed to `onError` and the loop goes on.
+// · A watchdog on its OWN clock (a stopped loop cannot report itself) says whether frames are arriving: `stalled` when none came for
+//   `stallMs`, `slow` under `floorFps` in the last second, `ok` otherwise. A reading taken while it is not `ok` is not a reading, and the
+//   child must be told, not left gazing at a control that no longer listens. It reports on change only.
+export function createVisionLoop(deps, { onFrame, onError = () => { }, onHealth = () => { }, floorFps = 8, stallMs = 1000, watchMs = 500 }) {
+    let frame = null, watch = null, health = 'ok', startedAt = 0;
+    const times = [];
+    const tick = (ms) => {
+        frame = deps.requestFrame(tick);
+        times.push(deps.now());
+        try {
+            onFrame(ms);
+        }
+        catch (e) {
+            onError(e);
+        }
+    };
+    const check = () => {
+        const now = deps.now();
+        while (times.length && now - times[0] > 1000)
+            times.shift();
+        const last = times.at(-1) ?? startedAt;
+        if (health === 'stalled' && now - last < stallMs)
+            startedAt = now; // back from a stall: judge the rate on a full second of the new frames
+        const next = now - last >= stallMs ? 'stalled' : now - startedAt >= 1000 && times.length < floorFps ? 'slow' : 'ok';
+        if (next !== health) {
+            health = next;
+            onHealth(health, times.length);
+        }
+    };
+    return {
+        start() {
+            if (frame !== null)
+                return;
+            startedAt = deps.now();
+            health = 'ok';
+            times.length = 0;
+            frame = deps.requestFrame(tick);
+            watch = deps.every(check, watchMs);
+        },
+        stop() {
+            if (frame !== null)
+                deps.cancelFrame(frame);
+            if (watch !== null)
+                deps.stopEvery(watch);
+            frame = null;
+            watch = null;
+        },
+        health: () => health,
+    };
+}

package/dist-pkg/platform/vision.d.ts

@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/** The files the face reader needs, by catalogue id. */
+export declare const FACE_VISION_FILES: readonly ["visao:runtime", "visao:runtime:cola", "visao:runtime:wasm", "visao:modelo:rosto"];
+/** The files the hand reader needs (ADR-0210 hands map: the Gesture Recognizer gives the canned gestures and the landmarks). */
+export declare const HAND_VISION_FILES: readonly ["visao:runtime", "visao:runtime:cola", "visao:runtime:wasm", "visao:modelo:gestos"];
+/** The part of one detection the eye control reads. */
+export interface FaceDetection {
+    readonly faceBlendshapes?: ReadonlyArray<{
+        readonly categories: ReadonlyArray<{
+            readonly categoryName: string;
+            readonly score: number;
+        }>;
+    }>;
+    readonly facialTransformationMatrixes?: ReadonlyArray<{
+        readonly data: ArrayLike<number>;
+    }>;
+    readonly faceLandmarks?: ReadonlyArray<ReadonlyArray<{
+        readonly x: number;
+        readonly y: number;
+        readonly z: number;
+    }>>;
+}
+interface Landmarker {
+    detectForVideo(frame: unknown, ms: number): FaceDetection;
+    close(): void;
+}
+/** The part of one Gesture Recognizer result the hands map reads: each hand's categories, best first, and its landmarks. */
+export interface HandDetection {
+    readonly gestures?: ReadonlyArray<ReadonlyArray<{
+        readonly categoryName: string;
+        readonly score: number;
+    }>>;
+    readonly landmarks?: ReadonlyArray<ReadonlyArray<{
+        readonly x: number;
+        readonly y: number;
+        readonly z: number;
+    }>>;
+}
+interface Recognizer {
+    recognizeForVideo(frame: unknown, ms: number): HandDetection;
+    close(): void;
+}
+export type Connections = ReadonlyArray<{
+    readonly start: number;
+    readonly end: number;
+}>;
+/** The shape of the `tasks-vision` bundle this module uses. */
+export interface TasksVision {
+    readonly FilesetResolver: {
+        forVisionTasks(wasmBase: string): Promise<unknown>;
+    };
+    readonly FaceLandmarker: {
+        createFromOptions(fileset: unknown, options: object): Promise<Landmarker>;
+        readonly FACE_LANDMARKS_LEFT_EYE: Connections;
+        readonly FACE_LANDMARKS_RIGHT_EYE: Connections;
+        readonly FACE_LANDMARKS_LEFT_IRIS: Connections;
+        readonly FACE_LANDMARKS_RIGHT_IRIS: Connections;
+        readonly FACE_LANDMARKS_LEFT_EYEBROW: Connections;
+        readonly FACE_LANDMARKS_RIGHT_EYEBROW: Connections;
+        readonly FACE_LANDMARKS_LIPS: Connections;
+    };
+    readonly GestureRecognizer: {
+        createFromOptions(fileset: unknown, options: object): Promise<Recognizer>;
+        readonly HAND_CONNECTIONS: Connections;
+    };
+}
+/** The lines the eye control highlights (ADR-0213): the eyes with their irises, and the brows — never the camera's picture (ADR-0212). */
+export interface EyeLines {
+    readonly eyes: Connections;
+    readonly brows: Connections;
+}
+/** The face mode also draws the lips (ADR-0212 §3). */
+export interface FaceLines extends EyeLines {
+    readonly lips: Connections;
+}
+export interface VisionDeps {
+    /** The page's address, to make the delivery paths absolute. */
+    readonly base: string;
+    /**
+     * Whether a catalogue file (by its upstream address) is in the checked cache — `platform/heavy`'s `checkedCacheHas`, over
+     * the host's `caches`. REQUIRED (ADR-0232 D4): the root lends the cache; this module reaches none.
+     */
+    readonly hasFile: (upstreamUrl: string) => Promise<boolean>;
+    readonly importBundle?: (absoluteUrl: string) => Promise<TasksVision>;
+}
+export type Delegate = 'GPU' | 'CPU';
+export interface FaceTracker {
+    /** One frame; null while it has nothing to say (no face, or the CPU landmarker still being built after a GPU failure). */
+    detect(frame: unknown, ms: number): FaceDetection | null;
+    delegate(): Delegate;
+    /** The connection sets the bundle ships for the eyes, irises and brows, to draw over the game. */
+    readonly eyeLines: EyeLines;
+    /** The same lines and the lips, for the face mode. */
+    readonly faceLines: FaceLines;
+    close(): void;
+}
+export type FaceTrackerLoad = {
+    readonly ok: true;
+    readonly tracker: FaceTracker;
+} | {
+    readonly ok: false;
+    readonly missing: readonly string[];
+};
+export interface HandTracker {
+    /** The hand's bones, as the bundle ships them, to draw over the game (ADR-0215). */
+    readonly handLines: Connections;
+    /** One frame; null while it has nothing to say. */
+    detect(frame: unknown, ms: number): HandDetection | null;
+    delegate(): Delegate;
+    close(): void;
+}
+export type HandTrackerLoad = {
+    readonly ok: true;
+    readonly tracker: HandTracker;
+} | {
+    readonly ok: false;
+    readonly missing: readonly string[];
+};
+export declare function loadFaceTracker(deps: VisionDeps): Promise<FaceTrackerLoad>;
+/** The Gesture Recognizer, one hand, in video mode — as the lab's map rounds ran it. */
+export declare function loadHandTracker(deps: VisionDeps): Promise<HandTrackerLoad>;
+/** The camera, video only, at the size the readings were measured. */
+export declare function openCamera(media: {
+    getUserMedia(c: MediaStreamConstraints): Promise<MediaStream>;
+}): Promise<MediaStream>;
+/** Let the camera go: every track stops, so the device's light goes off. */
+export declare function closeCamera(stream: {
+    getTracks(): ReadonlyArray<{
+        stop(): void;
+    }>;
+} | null | undefined): void;
+export {};