@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/web/audio.ts

@@ -0,0 +1,315 @@
+// The play page's sound: a music bed, the place's ambience, loops that pan and fade with where the hero
+// stands, footsteps on the floor under him, and one-shots for what happens in the story. Web Audio only.
+import { mouthAt, type LipSync } from '../src/play/lipsync.ts';
+import { personPx, type GeoThing, type Geometry } from '../src/play/geometry.ts';
+import { levelGain, placeSound, spokenLine, zoneAt, type SoundData } from '../src/play/sound.ts';
+import { holds, type State } from '../src/story/engine.ts';
+import { Synth } from './synth.ts';
+
+/** What the mixer reads from the game each frame. */
+export type Listened = {
+  data: { sound?: SoundData; world?: { hero: { key: string } } };
+  place?: { key: string; g: Geometry; hots: Array<{ thing: GeoThing }>; weather?: { w: { fall: string; strength: number } }; water?: unknown };
+  hero: { x: number; y: number; phase: number; path: unknown[] };
+  state: State;
+  viewX: number;
+  speech: unknown;
+  voiceStarted?(speech: unknown): void;
+  voiceEnded?(speech: unknown): void;
+  gone(): Set<string>;
+  on(fn: (event: string, detail: Record<string, unknown>) => void): void;
+};
+
+type Voice = { src: AudioBufferSourceNode; gain: GainNode; pan: StereoPannerNode };
+
+const FADE = 0.6;
+
+export class Mixer {
+  readonly ctx = new AudioContext();
+  private master = this.ctx.createGain();
+  private musicBus = this.ctx.createGain();
+  private voiceBus = this.ctx.createGain();
+  private synth = new Synth(this.ctx, this.musicBus, this.master);
+  private limiter = new DynamicsCompressorNode(this.ctx, { threshold: -14, knee: 8, ratio: 8, attack: 0.004, release: 0.25 });
+  private buffers = new Map<string, AudioBuffer>();
+  private loading = new Map<string, Promise<AudioBuffer | undefined>>();
+  /** Running loops by id: the music, the bed, and the things of this place. */
+  private voices = new Map<string, Voice>();
+  private lastStep = 0;
+  private lastUsed: string | undefined;
+  private seen: { items: string[]; flags: string[]; talk: boolean } | undefined;
+  private recorder: MediaRecorder | undefined;
+  private chunks: Blob[] = [];
+  muted = false;
+  /** What started when, on the audio clock: for QA and for laying sound under a recording. */
+  readonly log: Array<{ at: number; id: string; kind: string }> = [];
+
+  constructor(private game: Listened, private url: (file: string) => string) {
+    // A soft limiter at the end: one-shots on top of loops and music never clip.
+    this.musicBus.connect(this.master);
+    this.voiceBus.gain.value = 1.1;
+    this.voiceBus.connect(this.master);
+    this.master.connect(this.limiter).connect(this.ctx.destination);
+    // Browsers start audio only after a gesture; the first click or key resumes it.
+    const resume = () => void this.ctx.resume();
+    addEventListener('pointerdown', resume, { capture: true });
+    addEventListener('keydown', resume, { capture: true });
+    // Buttons (inventory, conversation choices) click.
+    addEventListener('click', (e) => {
+      if ((e.target as HTMLElement | null)?.closest?.('button')) this.shot('click');
+    }, { capture: true });
+    game.on((event, detail) => {
+      if (event === 'enter') void this.enter(String(detail.place));
+      if (event === 'use') this.lastUsed = String(detail.thing);
+      if (event === 'exit' && this.lastUsed) this.shot(`exit:${this.lastUsed}`);
+      if (event === 'ending') this.shot('ending');
+      if (event === 'say') void this.speak(String(detail.text), false, typeof detail.who === 'string' ? detail.who : 'narrator');
+      if (event === 'silent') this.hush();
+      if (event === 'enter') this.hush();
+    });
+  }
+
+  get sound(): SoundData | undefined {
+    return this.game.data.sound;
+  }
+
+  private buffer(id: string): Promise<AudioBuffer | undefined> {
+    const entry = this.sound?.files[id];
+    if (!entry) return Promise.resolve(undefined);
+    if (!this.loading.has(id)) {
+      this.loading.set(id, fetch(this.url(entry.file))
+        .then((r) => r.arrayBuffer())
+        .then((bytes) => this.ctx.decodeAudioData(bytes))
+        .then((buf) => (this.buffers.set(id, buf), buf))
+        .catch(() => undefined));
+    }
+    return this.loading.get(id)!;
+  }
+
+  private async start(id: string, gain: number, bus: AudioNode = this.master) {
+    if (this.voices.has(id)) return;
+    const buf = await this.buffer(id);
+    if (!buf || this.voices.has(id)) return;
+    const src = this.ctx.createBufferSource();
+    src.buffer = buf;
+    src.loop = true;
+    const g = this.ctx.createGain();
+    const pan = this.ctx.createStereoPanner();
+    g.gain.value = 0;
+    g.gain.setTargetAtTime(gain, this.ctx.currentTime, FADE / 3);
+    src.connect(g).connect(pan).connect(bus);
+    // Start somewhere inside the loop, so two places with the same take never sound in step.
+    src.start(0, Math.random() * buf.duration);
+    this.voices.set(id, { src, gain: g, pan });
+    this.log.push({ at: this.ctx.currentTime, id, kind: 'loop' });
+  }
+
+  private stop(id: string) {
+    const v = this.voices.get(id);
+    if (!v) return;
+    this.voices.delete(id);
+    v.gain.gain.setTargetAtTime(0, this.ctx.currentTime, FADE / 3);
+    v.src.stop(this.ctx.currentTime + FADE * 2);
+  }
+
+  /** A new place: its bed and loops fade in, the last place's fade out; the music carries on. */
+  async enter(place: string) {
+    const sound = this.sound;
+    // No music of its own (none made, or every take rejected): a quiet one made here. No bed for the place: its
+    // rain and its sea, from what the page already knows of it.
+    if (!sound?.music) this.synth.startMusic();
+    const at = this.game.place?.key === place ? this.game.place : undefined;
+    const bedless = !sound?.places[place]?.bed;
+    this.synth.ambience(bedless && at?.weather?.w.fall === 'rain' ? at.weather.w.strength : 0, bedless && !!at?.water);
+    if (!sound) return;
+    // Everything loads on the first place, so later one-shots and footsteps are ready when they fire.
+    if (!this.loading.size) for (const id of Object.keys(sound.files)) void this.buffer(id);
+    if (sound.music) void this.start(sound.music, levelGain('music', 2), this.musicBus);
+    const here = sound.places[place];
+    const keep = new Set([sound.music, here?.bed].filter(Boolean));
+    for (const id of [...this.voices.keys()]) if (!keep.has(id)) this.stop(id);
+    if (here?.bed) void this.start(here.bed, levelGain('bed', 2));
+  }
+
+  private speaking: AudioBufferSourceNode | undefined;
+  private articulation: { speech: unknown; start: number; sync: LipSync } | undefined;
+
+  /** The active source owns both articulation and time; loading, mute, pause and interruption close the lips. */
+  mouth(speech: unknown) {
+    const active = this.articulation;
+    return mouthAt(active?.sync, active && active.speech === speech && !this.muted && this.ctx.state === 'running'
+      ? this.ctx.currentTime - active.start : undefined);
+  }
+  private lines = new Map<string, Promise<AudioBuffer | undefined>>();
+
+  /** ?voices=npc: only the people the hero meets speak (for a film with a voice-over); ?voices=0: nobody. */
+  private voiced = new URLSearchParams(location.search).get('voices') ?? 'all';
+
+  /** Says an answer card aloud as it is pointed at, even with no line up (the cards are voiced as the hero's lines). */
+  hear(text: string) {
+    void this.speak(text, true, this.game.data.world?.hero.key);
+  }
+
+  /** Says a line in its cast voice, cutting off the one before. */
+  private async speak(text: string, card = false, speaker?: string) {
+    this.hush();
+    const request = this.speechRequest;
+    const speech = this.game.speech;
+    const line = spokenLine(this.sound, text, speaker);
+    if (!line || this.voiced === '0') return;
+    if (this.voiced === 'npc' && (line.speaker === 'narrator' || line.speaker === this.game.data.world?.hero.key)) return;
+    if (!this.lines.has(line.file)) this.lines.set(line.file, fetch(this.url(line.file)).then((r) => r.arrayBuffer()).then((b) => this.ctx.decodeAudioData(b)).catch(() => undefined));
+    const buf = await this.lines.get(line.file);
+    if (!buf || request !== this.speechRequest || (this.game.speech === null && !card)) return;
+    const src = this.ctx.createBufferSource();
+    src.buffer = buf;
+    src.connect(this.voiceBus);
+    const start = this.ctx.currentTime;
+    src.start(start);
+    this.speaking = src;
+    if (!card && line.lipsync) this.articulation = { speech, start, sync: line.lipsync };
+    if (!card && !this.muted && this.ctx.state === 'running') this.game.voiceStarted?.(speech);
+    src.onended = () => {
+      if (this.speaking === src) { this.speaking = undefined; this.articulation = undefined; }
+      if (!card) this.game.voiceEnded?.(speech);
+    };
+    this.log.push({ at: this.ctx.currentTime, id: text.slice(0, 40), kind: 'line' });
+  }
+
+  private speechRequest = 0;
+  private hush() {
+    this.speechRequest++;
+    this.articulation = undefined;
+    try {
+      this.speaking?.stop();
+    } catch {
+      // already ended
+    }
+    this.speaking = undefined;
+  }
+
+  /** A one-shot, panned to where it happens. */
+  shot(trigger: string, x?: number) {
+    const id = this.sound?.shots[trigger];
+    const buf = id ? this.buffers.get(id) : undefined;
+    if (!id || !buf) return;
+    this.play(id, buf, levelGain('shot', this.sound!.files[id].level), x === undefined ? 0 : this.panAt(x));
+  }
+
+  private play(id: string, buf: AudioBuffer, gain: number, pan: number, offset = 0, duration?: number, rate = 1) {
+    this.log.push({ at: this.ctx.currentTime, id, kind: duration ? 'step' : 'shot' });
+    const src = this.ctx.createBufferSource();
+    src.buffer = buf;
+    src.playbackRate.value = rate;
+    const g = this.ctx.createGain();
+    g.gain.value = gain;
+    const p = this.ctx.createStereoPanner();
+    p.pan.value = pan;
+    src.connect(g).connect(p).connect(this.master);
+    if (duration) {
+      // A short fade at the cut, so a sliced footstep never clicks.
+      g.gain.setValueAtTime(gain, this.ctx.currentTime + duration / rate - 0.03);
+      g.gain.linearRampToValueAtTime(0, this.ctx.currentTime + duration / rate);
+    }
+    src.start(0, offset, duration);
+  }
+
+  private panAt(x: number): number {
+    const g = this.game.place?.g;
+    if (!g) return 0;
+    return placeSound(x, 0, x, 0, this.game.viewX || g.frame.width / 2, g.frame.width, 1).pan;
+  }
+
+  /** Every frame: loops follow the hero, footsteps follow his walk, story changes sound. */
+  update() {
+    const { game } = this;
+    const sound = this.sound;
+    const place = game.place;
+    if (!sound || !place || this.ctx.state !== 'running') return;
+    const here = sound.places[place.key];
+    const g = place.g;
+    const now = this.ctx.currentTime;
+    const heroPx = personPx(g, game.hero.y);
+    const gone = game.gone();
+
+    for (const loop of here?.loops ?? []) {
+      const hot = place.hots.find((h) => h.thing.key === loop.thing);
+      const on = !!hot && !gone.has(loop.thing) && holds(game.state, loop.when);
+      if (!on) {
+        this.stop(loop.id);
+        continue;
+      }
+      const [x0, , x1, y1] = hot.thing.box;
+      const at = placeSound((x0 + x1) / 2, y1, game.hero.x, game.hero.y, game.viewX || g.frame.width / 2, g.frame.width, heroPx);
+      const voice = this.voices.get(loop.id);
+      if (!voice) void this.start(loop.id, at.gain * levelGain('loop', loop.level));
+      else {
+        voice.gain.gain.setTargetAtTime(at.gain * levelGain('loop', loop.level), now, 0.15);
+        voice.pan.pan.setTargetAtTime(at.pan, now, 0.15);
+      }
+    }
+    // Music steps back while someone speaks, further when the line is voiced.
+    this.musicBus.gain.setTargetAtTime(game.speech ? (this.speaking ? 0.35 : 0.55) : 1, now, 0.3);
+
+    // Two footsteps per walk cycle; the floor under the hero picks the surface.
+    const step = Math.floor(game.hero.phase * 2);
+    if (step !== this.lastStep) {
+      this.lastStep = step;
+      const id = here?.zones[zoneAt(g.zone, g.cell, game.hero.x, game.hero.y)] ?? here?.zones.find(Boolean);
+      const buf = id ? this.buffers.get(id) : undefined;
+      const slices = id ? sound.files[id].slices : undefined;
+      if (buf && slices?.length) {
+        const [offset, duration] = slices[Math.floor(Math.random() * slices.length)];
+        // Farther away is quieter; a little pitch and level jitter so no two steps repeat exactly.
+        const depth = Math.min(1, heroPx / (g.frame.height * 0.45));
+        this.play(id!, buf, levelGain('step', 2) * (0.45 + 0.55 * depth) * (0.85 + Math.random() * 0.3), this.panAt(game.hero.x) * 0.6, offset, duration, 0.93 + Math.random() * 0.14);
+      }
+    }
+
+    // What the story just did: an item taken, a thing used to change something, a conversation opened.
+    const s = game.state;
+    const seen = this.seen;
+    if (seen) {
+      const hot = place.hots.find((h) => h.thing.key === this.lastUsed);
+      const x = hot ? (hot.thing.box[0] + hot.thing.box[2]) / 2 : undefined;
+      if (s.items.some((i) => !seen.items.includes(i))) this.shot('take', x);
+      else if (s.flags.some((f) => !seen.flags.includes(f)) && this.lastUsed) this.shot(`use:${this.lastUsed}`, x);
+      if (s.talk && !seen.talk) this.shot('talk', x);
+    }
+    this.seen = { items: [...s.items], flags: [...s.flags], talk: !!s.talk };
+  }
+
+  /** Starts the sound if the browser lets the page do it without a click (a gamepad press is not one everywhere). */
+  resume() {
+    if (this.ctx.state === 'suspended') void this.ctx.resume().catch(() => undefined);
+  }
+
+  toggleMute() {
+    this.muted = !this.muted;
+    this.master.gain.setTargetAtTime(this.muted ? 0 : 1, this.ctx.currentTime, 0.05);
+  }
+
+  /** For recordings: everything the page plays, into a webm the recorder can lay under its video. */
+  record() {
+    const out = this.ctx.createMediaStreamDestination();
+    this.limiter.connect(out);
+    this.chunks = [];
+    this.recorder = new MediaRecorder(out.stream, { mimeType: 'audio/webm;codecs=opus' });
+    this.recorder.ondataavailable = (e) => this.chunks.push(e.data);
+    this.recorder.start(1000);
+  }
+
+  /** Stops recording; the webm as base64. */
+  async stopRecording(): Promise<string> {
+    const rec = this.recorder;
+    if (!rec) return '';
+    await new Promise<void>((resolve) => {
+      rec.onstop = () => resolve();
+      rec.stop();
+    });
+    const bytes = new Uint8Array(await new Blob(this.chunks, { type: 'audio/webm' }).arrayBuffer());
+    let bin = '';
+    for (let i = 0; i < bytes.length; i += 0x8000) bin += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+    return btoa(bin);
+  }
+}

package/app/web/face.ts

@@ -0,0 +1,131 @@
+// A small, registered mouth patch keeps the existing painted face, cap and body intact.
+import { faceAt, type Articulation, type Box, type FaceDefinition, type FacePose, type FaceState, type Mouth } from '../src/play/lipsync.ts';
+
+type Surface = HTMLImageElement | HTMLCanvasElement;
+
+/** Draw a full-size mask so pixels outside the gradient's rectangle are cleared too. */
+function feather(c: CanvasRenderingContext2D, cx: number, cy: number, rx: number, ry: number) {
+  const mask = Object.assign(document.createElement('canvas'), { width: c.canvas.width, height: c.canvas.height });
+  const m = mask.getContext('2d')!;
+  m.translate(cx, cy); m.scale(rx, ry);
+  const fade = m.createRadialGradient(0, 0, 0.68, 0, 0, 1);
+  fade.addColorStop(0, '#fff'); fade.addColorStop(1, '#fff0');
+  m.fillStyle = fade; m.fillRect(-1, -1, 2, 2);
+  c.globalCompositeOperation = 'destination-in'; c.drawImage(mask, 0, 0);
+  c.globalCompositeOperation = 'source-over';
+}
+export class PaintedFace {
+  private patches = new Map<string, Map<Mouth, HTMLCanvasElement>>();
+  private canvas = document.createElement('canvas');
+  private cache = '';
+  private revision = 0;
+  get version() { return this.revision; }
+  private blend = document.createElement('canvas');
+  private neutral = new Map<string, HTMLCanvasElement>();
+  private closed = new Map<string, HTMLCanvasElement>();
+  private closure = document.createElement('canvas');
+
+  constructor(private info: FaceDefinition, atlas: HTMLImageElement, only?: readonly string[]) {
+    for (const [view, mouths] of Object.entries({ default: info.mouths, ...info.atlases })) {
+    if (only && !only.includes(view)) continue;
+    const patches = new Map<Mouth, HTMLCanvasElement>();
+    for (const [mouth, [x, y, width, height]] of Object.entries(mouths)) {
+      const patch = Object.assign(document.createElement('canvas'), { width, height });
+      const c = patch.getContext('2d')!;
+      const anchor = info.anchors?.[view], frame = info.frames?.[view]?.[mouth as Mouth] ?? anchor;
+      const dx = anchor && frame ? (anchor[0] + anchor[2] / 2 - frame[0] - frame[2] / 2) * width : 0;
+      const dy = anchor && frame ? (anchor[1] - frame[1]) * height : 0;
+      c.drawImage(atlas, x, y, width, height, dx, dy, width, height);
+      if(mouth==='X'){
+        const raw=Object.assign(document.createElement('canvas'),{width,height});
+        raw.getContext('2d')!.drawImage(patch,0,0);this.neutral.set(view,raw);
+      }
+      // Feather only the outer beard/skin; the lips in the centre stay opaque.
+      const [cx, cy, rx, ry] = anchor && frame ? [
+        (anchor[0] + anchor[2] / 2) * width, (anchor[1] + frame[3] / 2) * height,
+        Math.max(frame[2] / 2 + anchor[2] * 0.12, anchor[2] * 0.65) * width,
+        (frame[3] / 2 + anchor[2] * 0.12) * height,
+      ] : [width / 2, height / 2, width / 2, height / 2];
+      feather(c, cx, cy, rx, ry);
+      patches.set(mouth as Mouth, patch);
+    }
+    this.patches.set(view, patches);
+    }
+  }
+
+  draw(image: Surface, sheet: string, cell: number, cols: number, rows: number, articulation: Articulation, time: number, sourceCell = cell, override?: FacePose, performance?: FaceState): CanvasImageSource {
+    const pose = override ?? (sheet === '$image' ? this.info.imagePose : this.info.sheets[sheet]?.[cell]);
+    const width = image.width / cols, height = image.height / rows;
+    // Only the explicitly registered poses are touched.
+    if (!pose) return image;
+    const state = performance ?? faceAt(articulation, time, this.info.seed);
+    const { blink, brow: lift } = state;
+    const patches = this.patches.get(pose.atlas ?? 'default');
+    if (!patches) return image;
+    const key = [sheet, cell, pose.atlas, image instanceof HTMLCanvasElement ? Math.floor(time * 60) : '', pose.mouth.join(','), pose.lips?.join(','), pose.sourceOpen, articulation.mouth, articulation.previous, articulation.mix.toFixed(2), blink.toFixed(2), lift.toFixed(2)].join(':');
+    if (key === this.cache) return this.canvas;
+    this.cache = key;
+    this.revision++;
+    if (this.canvas.width !== width || this.canvas.height !== height) Object.assign(this.canvas, { width, height });
+    const c = this.canvas.getContext('2d')!;
+    c.clearRect(0, 0, width, height);
+    const ox = (sourceCell % cols) * width, oy = Math.floor(sourceCell / cols) * height;
+    c.drawImage(image, ox, oy, width, height, 0, 0, width, height);
+    const [x, y, w, h] = pose.mouth;
+    const view = pose.atlas ?? 'default', anchor = this.info.anchors?.[view];
+    const neutral = patches.get('X')!;
+    const destination: Box = pose.lips && anchor ? (() => {
+      // Uniform scale from the ORIGINAL mouth corners. The nose, jaw and beard stay as painted.
+      const [lx,ly,lw] = pose.lips, scale = lw / (anchor[2] * neutral.width);
+      const pw = neutral.width * scale, ph = neutral.height * scale;
+      return [lx + lw / 2 - (anchor[0] + anchor[2] / 2) * pw, ly - anchor[1] * ph, pw, ph];
+    })() : [x,y,w,h];
+    if(pose.sourceOpen && pose.lips && anchor && this.info.frames?.[view]){
+      const height=pose.lips[3]/(destination[2]/neutral.width),key=`${view}:${height.toFixed(2)}`;
+      let closed=this.closed.get(key);
+      if(!closed){
+        const raw=this.neutral.get(view)!;
+        closed=Object.assign(document.createElement('canvas'),{width:raw.width,height:raw.height});
+        const n=closed.getContext('2d')!;n.drawImage(raw,0,0);
+        const h=Math.max(height,anchor[3]*raw.height);
+        feather(n, (anchor[0]+anchor[2]/2)*raw.width, anchor[1]*raw.height+h/2,
+          anchor[2]*raw.width*.7, h/2+anchor[2]*raw.width*.12);
+        this.closed.set(key,closed);
+      }
+      if(this.closure.width!==neutral.width||this.closure.height!==neutral.height)Object.assign(this.closure,{width:neutral.width,height:neutral.height});
+      const n=this.closure.getContext('2d')!;n.clearRect(0,0,this.closure.width,this.closure.height);n.drawImage(closed,0,0);
+      n.globalCompositeOperation='destination-in';n.drawImage(image,ox+destination[0],oy+destination[1],destination[2],destination[3],0,0,this.closure.width,this.closure.height);n.globalCompositeOperation='source-over';
+      c.drawImage(this.closure,...destination);
+    }
+    if (this.blend.width !== neutral.width || this.blend.height !== neutral.height) Object.assign(this.blend, { width: neutral.width, height: neutral.height });
+    const blend = this.blend.getContext('2d')!;
+    blend.clearRect(0,0,this.blend.width,this.blend.height);
+    // A single premultiplied crossfade avoids stacking two feathered lower faces over the original.
+    blend.globalCompositeOperation = 'lighter';
+    for (const [mouth,weight] of [[articulation.previous,1-articulation.mix],[articulation.mouth,articulation.mix]] as Array<[Mouth,number]>) {
+      // Closed source art stays exact; originally open gestures have the neutral underlay above.
+      if (!pose.lips || !anchor || !this.info.frames?.[view] || mouth === 'X' || mouth === 'A' || weight <= 0) continue;
+      blend.globalAlpha = weight;
+      blend.drawImage(patches.get(mouth)!,0,0,this.blend.width,this.blend.height);
+    }
+    blend.globalAlpha = 1;
+    // Lip paint cannot grow past the source silhouette, including a puckered profile.
+    blend.globalCompositeOperation = 'destination-in';
+    blend.drawImage(image,ox+destination[0],oy+destination[1],destination[2],destination[3],0,0,this.blend.width,this.blend.height);
+    blend.globalCompositeOperation = 'source-over';
+    c.drawImage(this.blend, ...destination);
+    for (const eye of pose.eyes) this.eye(c, image, ox, oy, eye, blink, lift);
+    return this.canvas;
+  }
+
+  /** Compress the original painted eye between its lids; retain the surrounding skin and brow texture. */
+  private eye(c: CanvasRenderingContext2D, image: Surface, ox: number, oy: number, [x, y, w, h]: Box, blink: number, lift: number) {
+    if (blink < 0.01 && lift < 0.01) return;
+    const middle = h * 0.5, half = h * 0.18, open = Math.max(0.025, half * (1 - blink));
+    const source = [0, h * 0.2, middle - half, middle + half, h];
+    const target = [0, h * 0.2 - lift, middle - open, middle + open, h];
+    for (let i = 0; i < source.length - 1; i++) {
+      c.drawImage(image, ox + x, oy + y + source[i], w, source[i + 1] - source[i], x, y + target[i], w, target[i + 1] - target[i]);
+    }
+  }
+}

package/app/web/face3d.ts

@@ -0,0 +1,35 @@
+import * as THREE from 'three';
+import { facialControl, faceWeight, isViseme, facialPriority, type FaceControl } from '../src/play/face3d.ts';
+import type { FaceState } from '../src/play/lipsync.ts';
+
+/** Only authored, recognized morph targets are driven. A body skeleton alone does not pretend to articulate. */
+export class ModelFace {
+  private bindings: Array<{ mesh: THREE.Mesh; index: number; control: FaceControl; name: string; selected: boolean }> = [];
+  readonly mouth: boolean;
+  readonly eyes: boolean;
+  private nativeVisemes = new Map<THREE.Mesh, Set<FaceControl>>();
+  constructor(root: THREE.Object3D) {
+    root.traverse(object => {
+      if (!(object as THREE.Mesh).isMesh) return;
+      const mesh = object as THREE.Mesh;
+      for (const [name, index] of Object.entries(mesh.morphTargetDictionary ?? {})) {
+        const control = facialControl(name);
+        if (control && mesh.morphTargetInfluences) this.bindings.push({ mesh, index, control, name, selected: true });
+      }
+    });
+    for (const binding of this.bindings.filter(b => isViseme(b.control))) {
+      const group = this.bindings.filter(b => b.mesh === binding.mesh && b.control === binding.control).sort((a,b) => facialPriority(a.name) - facialPriority(b.name));
+      binding.selected = group[0] === binding;
+      if (binding.selected) {
+        const controls = this.nativeVisemes.get(binding.mesh) ?? new Set<FaceControl>();
+        controls.add(binding.control);
+        this.nativeVisemes.set(binding.mesh, controls);
+      }
+    }
+    this.mouth = [...this.nativeVisemes.values()].some(controls => [...controls].some(c => c !== 'sil')) || this.bindings.some(b => b.control === 'jaw');
+    this.eyes = this.bindings.some(b => b.control.startsWith('blink'));
+  }
+  apply(state: FaceState) {
+    for (const { mesh, index, control, selected } of this.bindings) mesh.morphTargetInfluences![index] = selected ? faceWeight(control, state, this.nativeVisemes.get(mesh) ?? false) : 0;
+  }
+}