@the-inclusionist/engine 9.0.0 → 11.0.0

package/dist-pkg/input/face-map.js

@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// input/face-map — THE FACE AS A CONTROLLER: THIRTEEN EXPRESSIONS AND THE ACTIONS THEY PRESS (ADR-0210 and its erratum; issue #191).
+//
+// The map is the Dev's, settled in twelve lab rounds (the last clean: every position read, nothing read that was not asked, nothing read in
+// 26 s of «no position»). Each expression is measured as a LEVEL — how far its blendshapes rose from the calibrated rest, 0..1 — and the
+// levels go through the rules those rounds found before a level becomes a press:
+// · exclusions: lips pressed lift the smile, and the mouth pushed to a side projects the lips, so the specific one clears the other; a
+//   squeezed eye pulls the brow down, so a squeeze at START's mark clears the frown;
+// · the pucker has its own mark (0.6): a relaxed mouth sits near 0.33;
+// · the crossed look: a look to one side peaking at 0.6, then the other side within 0.8 s and held 300 ms; a turned head cancels it;
+// · START is one eye squeezed at 0.5 for 0.7 s; the mouth expressions hold 300 ms more, because talking moves the mouth in short syllables;
+// · then a level at 0.3 held 300 ms presses, and stays pressed while it holds.
+// ⚠️ The levels are keyed by EXPRESSION, never by action: a number per action is the analogue grammar ADR-0112 refused for input/. The
+// only thing that reaches the action vocabulary is which actions are held.
+/** The Dev's map (ADR-0210 erratum): each expression and the action it presses. SELECT has no expression. */
+export const FACE_MAP = {
+    browsUp: 'up', frown: 'down', headLeft: 'left', headRight: 'right', squeeze: 'start',
+    pressLips: 'action1', openMouth: 'action2', smile: 'action3', pucker: 'action4',
+    lookRightThenLeft: 'leftShoulder', mouthLeft: 'leftTrigger', lookLeftThenRight: 'rightShoulder', mouthRight: 'rightTrigger',
+};
+const EXPRESSIONS = Object.keys(FACE_MAP);
+/** The blendshapes the map reads — a closed list. */
+export const FACE_BLENDSHAPES = [
+    'browInnerUp', 'browOuterUpLeft', 'browOuterUpRight', 'browDownLeft', 'browDownRight',
+    'mouthPressLeft', 'mouthPressRight', 'jawOpen', 'mouthSmileLeft', 'mouthSmileRight', 'mouthUpperUpLeft', 'mouthUpperUpRight',
+    'mouthPucker', 'mouthLeft', 'mouthRight',
+    'eyeLookOutLeft', 'eyeLookInLeft', 'eyeLookOutRight', 'eyeLookInRight', 'eyeBlinkLeft', 'eyeBlinkRight', 'eyeSquintLeft', 'eyeSquintRight',
+];
+const READ = new Set(FACE_BLENDSHAPES);
+/** The Face Landmarker's categories, keeping the ones the map reads. */
+export function faceScoresFromCategories(categories) {
+    return Object.fromEntries((categories ?? []).filter((c) => READ.has(c.categoryName)).map((c) => [c.categoryName, c.score]));
+}
+const rise = (s, rest, n) => {
+    const b = rest[n] ?? 0;
+    return Math.max(0, ((s[n] ?? 0) - b) / Math.max(1e-6, 1 - b));
+};
+/**
+ * The head turned, from the nose tip's offset against the middle of the face's two sides (landmarks 1, 234, 454), in face widths, from the
+ * rest. In the camera's unmirrored image the player's left is +x. `FULL_TURN` 0.65 puts the 0.3 mark near 18° (the Dev asked for a less
+ * sensitive head; geometry, not measured on a face).
+ */
+export const FULL_TURN = 0.65;
+export function headTurn(landmarks, restX = 0) {
+    if (!landmarks || landmarks.length < 455)
+        return { x: null, left: 0, right: 0 };
+    const nose = landmarks[1], a = landmarks[234], b = landmarks[454];
+    const width = Math.hypot(b.x - a.x, b.y - a.y) || 1e-6;
+    const x = (nose.x - (a.x + b.x) / 2) / width;
+    const clamp = (v) => Math.min(1, Math.max(0, v));
+    return { x, left: clamp((x - restX) / FULL_TURN), right: clamp((restX - x) / FULL_TURN) };
+}
+/** The raw levels of one frame, from the scores and the head turn, against the rest. */
+export function faceLevels(s, rest, turn) {
+    const r = (...names) => Math.max(0, ...names.map((n) => rise(s, rest.scores, n)));
+    return {
+        browsUp: r('browInnerUp', 'browOuterUpLeft', 'browOuterUpRight'),
+        frown: r('browDownLeft', 'browDownRight'),
+        headLeft: turn.left, headRight: turn.right,
+        // one eye squeezed is enough, the other squeezed or not: the more closed eye, by blink or squint
+        squeeze: r('eyeBlinkLeft', 'eyeSquintLeft', 'eyeBlinkRight', 'eyeSquintRight'),
+        pressLips: r('mouthPressLeft', 'mouthPressRight'),
+        openMouth: r('jawOpen'),
+        smile: r('mouthSmileLeft', 'mouthSmileRight', 'mouthUpperUpLeft', 'mouthUpperUpRight'),
+        pucker: r('mouthPucker'),
+        // a look to the player's left moves the left eye out and the right eye in, and both must rise: the weaker of the pair
+        lookRightThenLeft: Math.min(rise(s, rest.scores, 'eyeLookOutLeft'), rise(s, rest.scores, 'eyeLookInRight')),
+        lookLeftThenRight: Math.min(rise(s, rest.scores, 'eyeLookOutRight'), rise(s, rest.scores, 'eyeLookInLeft')),
+        mouthLeft: r('mouthLeft'), mouthRight: r('mouthRight'),
+    };
+}
+/** START's squeeze mark: above the relaxed narrowing (0.30–0.34) and the eye a real frown squeezes (0.36–0.49) in the Dev's runs. */
+export const SQUEEZE_MARK = 0.5;
+const DOMINATES = { pressLips: ['smile'], mouthLeft: ['pucker'], mouthRight: ['pucker'] };
+/** The exclusions: a held specific expression clears the generic one it drags along; a squeeze at START's mark clears the frown. */
+export function resolveFace(levels, mark) {
+    const out = { ...levels };
+    if ((levels.squeeze ?? 0) >= SQUEEZE_MARK)
+        out.frown = 0;
+    for (const e of EXPRESSIONS) {
+        if ((levels[e] ?? 0) >= mark)
+            for (const weak of DOMINATES[e] ?? [])
+                out[weak] = 0;
+    }
+    return out;
+}
+/** The pucker's own mark (0.6): below it, it is not a pucker. */
+export const PUCKER_MARK = 0.6;
+export function ownMarks(levels) {
+    return (levels.pucker ?? 0) < PUCKER_MARK ? { ...levels, pucker: 0 } : levels;
+}
+const LOOK_BUTTONS = ['lookRightThenLeft', 'lookLeftThenRight'];
+/** Each button's first half is the other side. */
+const FIRST_SIDE = { lookRightThenLeft: 'lookLeftThenRight', lookLeftThenRight: 'lookRightThenLeft' };
+/**
+ * The crossed look. In: the look to each side (in the levels, under the button it ends on). Out: the button levels. A first side counts
+ * only if it peaked at `peakMin` and then left the mark; the second must reach `peakMin` within `windowMs` and stay `holdMs`; held, it
+ * stays down while the look stays at the mark; a turned head cancels any crossing.
+ */
+export function createCrossedLook({ mark, peakMin = 0.6, windowMs = 800, holdMs = 300 }) {
+    const side = {
+        lookRightThenLeft: { on: false, peak: 0, leftAt: null }, lookLeftThenRight: { on: false, peak: 0, leftAt: null },
+    };
+    const press = {
+        lookRightThenLeft: { start: null, heldSince: null }, lookLeftThenRight: { start: null, heldSince: null },
+    };
+    /** A side peaks while it is at the mark; leaving after a peak of `peakMin` leaves a first half for the other button. */
+    function trackSide(s, v, ms, turned) {
+        if (v >= mark) {
+            if (!s.on) {
+                s.on = true;
+                s.peak = 0;
+            } // zeroed on arrival, so a weak visit never inherits a strong one's peak
+            s.peak = Math.max(s.peak, v);
+        }
+        else if (s.on) {
+            s.on = false;
+            if (s.peak >= peakMin)
+                s.leftAt = ms;
+        }
+        if (turned) {
+            s.leftAt = null;
+            s.peak = 0;
+        }
+    }
+    /** The second side starts a press only if the first left within `windowMs` — and it spends that first half either way. */
+    function startOf(ms, before) {
+        if (before.leftAt === null)
+            return -1;
+        const start = ms - before.leftAt <= windowMs ? ms : -1;
+        before.leftAt = null;
+        return start;
+    }
+    /** One button's level: held, it stays down while the look stays at the mark; otherwise the second side must reach
+     *  `peakMin` with the head still, and after `holdMs` the press is held. Letting go clears it whole. */
+    function stepPress(st, v, ms, before, turned) {
+        if (st.heldSince !== null) {
+            if (v >= mark)
+                return 1;
+            st.heldSince = null;
+            st.start = null;
+            return 0;
+        }
+        if (v < peakMin || turned) {
+            st.start = null;
+            return 0;
+        }
+        st.start ??= startOf(ms, before);
+        if (st.start >= 0 && ms - st.start >= holdMs)
+            st.heldSince = st.start;
+        return st.start >= 0 ? 1 : 0;
+    }
+    return (ms, levels) => {
+        const out = { ...levels };
+        const turned = Math.max(levels.headLeft ?? 0, levels.headRight ?? 0) >= mark;
+        // every side is tracked before any button reads one: a button's first half is the OTHER side's exit
+        for (const b of LOOK_BUTTONS)
+            trackSide(side[b], levels[b] ?? 0, ms, turned);
+        for (const b of LOOK_BUTTONS)
+            out[b] = stepPress(press[b], levels[b] ?? 0, ms, side[FIRST_SIDE[b]], turned);
+        return out;
+    };
+}
+/** START's long squeeze: its level passes only after staying at `SQUEEZE_MARK` for `ms` (the usual hold comes on top: about 1 s). */
+export function createLongSqueeze(ms = 700) {
+    let since = null;
+    return (now, levels) => {
+        if ((levels.squeeze ?? 0) < SQUEEZE_MARK) {
+            since = null;
+            return { ...levels, squeeze: 0 };
+        }
+        since ??= now;
+        return now - since >= ms ? levels : { ...levels, squeeze: 0 };
+    };
+}
+/** The mouth expressions, held `ms` longer: talking moves the mouth past the mark in short syllables. */
+const MOUTH_EXPRESSIONS = ['pressLips', 'openMouth', 'smile', 'pucker', 'mouthLeft', 'mouthRight'];
+export function createMouthHold({ mark, ms = 300 }) {
+    const since = new Map();
+    return (now, levels) => {
+        const out = { ...levels };
+        for (const e of MOUTH_EXPRESSIONS) {
+            if ((levels[e] ?? 0) < mark) {
+                since.delete(e);
+                continue;
+            }
+            if (!since.has(e))
+                since.set(e, now);
+            if (now - since.get(e) < ms)
+                out[e] = 0;
+        }
+        return out;
+    };
+}
+/** The whole reader: a frame's scores and landmarks in, the actions held and the ones that just started out. */
+export function createFaceMapReader({ rest, mark = 0.3, holdMs = 300 }) {
+    const crossed = createCrossedLook({ mark }), squeeze = createLongSqueeze(), mouth = createMouthHold({ mark });
+    const aboveSince = new Map(), held = new Set();
+    return (ms, scores, landmarks) => {
+        const turn = headTurn(landmarks, rest.headX);
+        const levels = mouth(ms, squeeze(ms, crossed(ms, ownMarks(resolveFace(faceLevels(scores, rest, turn), mark)))));
+        const started = [];
+        for (const e of EXPRESSIONS) {
+            if ((levels[e] ?? 0) >= mark) {
+                if (!aboveSince.has(e))
+                    aboveSince.set(e, ms);
+                if (!held.has(e) && ms - aboveSince.get(e) >= holdMs) {
+                    held.add(e);
+                    started.push(FACE_MAP[e]);
+                }
+            }
+            else {
+                aboveSince.delete(e);
+                held.delete(e);
+            }
+        }
+        return { held: [...held].map((e) => FACE_MAP[e]), started };
+    };
+}
+/**
+ * THE FACE'S REST, MEASURED BY ITSELF (the Dev, 2026-09-16: «sozinho, quando o rosto fica parado olhando para o centro da tela por 3 s+»).
+ * Frames gather while the face stays still — every blendshape and the nose within `maxSpread` of their median — for `restMs` in a row;
+ * any movement restarts the count, so the rest is never taken from a child who is talking, reading or looking around.
+ */
+export function createFaceRest({ restMs = 3000, maxSpread = 0.08 } = {}) {
+    let since = null;
+    const samples = [];
+    const median = (xs) => { const a = [...xs].sort((p, q) => p - q); return a[Math.floor(a.length / 2)] ?? 0; };
+    return (ms, scores, headX) => {
+        since ??= ms;
+        samples.push({ scores, headX });
+        const moving = [...FACE_BLENDSHAPES.map((n) => samples.map((s) => s.scores[n] ?? 0)), samples.map((s) => s.headX ?? 0)]
+            .some((xs) => { const m = median(xs); return xs.some((x) => Math.abs(x - m) > maxSpread); });
+        if (moving) {
+            samples.length = 0;
+            samples.push({ scores, headX });
+            since = ms;
+            return { rest: null, stillMs: 0 };
+        }
+        if (ms - since < restMs)
+            return { rest: null, stillMs: ms - since };
+        return { rest: faceRestFromSamples(samples), stillMs: ms - since };
+    };
+}
+/** The rest from a few seconds of samples: each blendshape's median, and the nose's. */
+export function faceRestFromSamples(samples) {
+    const median = (xs) => { const a = [...xs].sort((p, q) => p - q); return a.length ? a[Math.floor(a.length / 2)] : 0; };
+    const scores = {};
+    for (const n of FACE_BLENDSHAPES) {
+        const xs = samples.map((s) => s.scores[n]).filter((v) => typeof v === 'number');
+        if (xs.length)
+            scores[n] = median(xs);
+    }
+    return { scores, headX: median(samples.map((s) => s.headX).filter((v) => v !== null)) };
+}

package/dist-pkg/input/face-signals.d.ts

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+export interface HeadPose {
+    readonly yaw: number;
+    readonly pitch: number;
+    readonly roll: number;
+    /** The face's position in the camera's metric space, in cm: the camera at the origin looking down −z, so a face in front has tz < 0. */
+    readonly tx: number;
+    readonly ty: number;
+    readonly tz: number;
+}
+/** Degrees from the Face Landmarker's 4×4 column-major matrix. Null without one. */
+export declare function headPoseFromMatrix(m: ArrayLike<number> | null | undefined): HeadPose | null;
+/** The blendshapes the eye control reads — a closed list, so a score kept here can never be a number per action (ADR-0112). */
+export declare const EYE_BLENDSHAPES: readonly ["eyeLookInLeft", "eyeLookOutLeft", "eyeLookUpLeft", "eyeLookDownLeft", "eyeLookInRight", "eyeLookOutRight", "eyeLookUpRight", "eyeLookDownRight", "eyeBlinkLeft", "eyeBlinkRight"];
+export type EyeBlendshape = (typeof EYE_BLENDSHAPES)[number];
+/** Scores of the eye blendshapes by name; one the detection did not bring is absent. */
+export type BlendshapeScores = Partial<Readonly<Record<EyeBlendshape, number>>>;
+/** The Face Landmarker gives all 52 blendshapes as `{ categoryName, score }` pairs; this keeps the eye ones, by name. */
+export declare function scoresFromCategories(categories: ReadonlyArray<{
+    readonly categoryName: string;
+    readonly score: number;
+}> | null | undefined): BlendshapeScores;
+export interface EyeGaze {
+    /** Grows as the eye turns towards the PLAYER'S LEFT. */
+    readonly h: number;
+    /** Grows as the eye turns DOWN. */
+    readonly v: number;
+}
+/**
+ * Each eye's gaze. `left` and `right` are the subject's own eyes, as MediaPipe names them; the right eye turning to the player's left moves
+ * towards the nose (in), the left eye moves out. A missing score reads as zero.
+ */
+export declare function eyeGazeFromScores(s: BlendshapeScores | null | undefined): {
+    readonly left: EyeGaze;
+    readonly right: EyeGaze;
+};
+/** The mark a blink score must reach on BOTH eyes for them to count as closed — START is both eyes closed (ADR-0213 §5). */
+export declare const EYES_CLOSED_MARK = 0.5;
+/** Both eyes closed: the LESS closed of the two reaches the mark, so one eye squeezed shut is not a close. */
+export declare function bothEyesClosed(s: BlendshapeScores | null | undefined, mark?: number): boolean;

package/dist-pkg/input/face-signals.js

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// input/face-signals — WHAT THE EYE CONTROL READS FROM THE FACE LANDMARKER (ADR-0213; issue #194).
+//
+// Three readings and nothing else: the head pose from the facial transformation matrix, each eye's gaze from the ready-made blendshapes, and
+// whether both eyes are closed. Nothing here knows where the screen is — the eye control measures how far the gaze moves from its own rest
+// (input/gaze-relative), and uses the pose only to notice a head that turns fast.
+//
+// ⚠️ THE SIGN OF PITCH IS NOT MEASURED against a real head. Only the pose's rate of change reaches a command, so the sign does not.
+/** Degrees from the Face Landmarker's 4×4 column-major matrix. Null without one. */
+export function headPoseFromMatrix(m) {
+    if (!m || m.length < 16)
+        return null;
+    const deg = 180 / Math.PI;
+    return {
+        yaw: Math.atan2(m[8], m[10]) * deg,
+        pitch: Math.asin(Math.max(-1, Math.min(1, -m[9]))) * deg,
+        roll: Math.atan2(m[1], m[5]) * deg,
+        tx: m[12], ty: m[13], tz: m[14],
+    };
+}
+/** The blendshapes the eye control reads — a closed list, so a score kept here can never be a number per action (ADR-0112). */
+export const EYE_BLENDSHAPES = [
+    'eyeLookInLeft', 'eyeLookOutLeft', 'eyeLookUpLeft', 'eyeLookDownLeft',
+    'eyeLookInRight', 'eyeLookOutRight', 'eyeLookUpRight', 'eyeLookDownRight',
+    'eyeBlinkLeft', 'eyeBlinkRight',
+];
+const READ = new Set(EYE_BLENDSHAPES);
+/** The Face Landmarker gives all 52 blendshapes as `{ categoryName, score }` pairs; this keeps the eye ones, by name. */
+export function scoresFromCategories(categories) {
+    return Object.fromEntries((categories ?? []).filter((c) => READ.has(c.categoryName)).map((c) => [c.categoryName, c.score]));
+}
+/**
+ * Each eye's gaze. `left` and `right` are the subject's own eyes, as MediaPipe names them; the right eye turning to the player's left moves
+ * towards the nose (in), the left eye moves out. A missing score reads as zero.
+ */
+export function eyeGazeFromScores(s) {
+    const q = (n) => s?.[n] ?? 0;
+    return {
+        left: { h: q('eyeLookOutLeft') - q('eyeLookInLeft'), v: q('eyeLookDownLeft') - q('eyeLookUpLeft') },
+        right: { h: q('eyeLookInRight') - q('eyeLookOutRight'), v: q('eyeLookDownRight') - q('eyeLookUpRight') },
+    };
+}
+/** The mark a blink score must reach on BOTH eyes for them to count as closed — START is both eyes closed (ADR-0213 §5). */
+export const EYES_CLOSED_MARK = 0.5;
+/** Both eyes closed: the LESS closed of the two reaches the mark, so one eye squeezed shut is not a close. */
+export function bothEyesClosed(s, mark = EYES_CLOSED_MARK) {
+    return Math.min(s?.['eyeBlinkLeft'] ?? 0, s?.['eyeBlinkRight'] ?? 0) >= mark;
+}