@convai/embody 1.0.1 → 1.1.0-beta.0

package/dist/speech/lipsync.js

@@ -1,5 +1,7 @@
-import { createLipsyncProcessor, mhaChannelIndex, } from '@convai/web-sdk/lipsync-helpers';
+import { createLipsyncProcessor, mhaChannelIndex, MHA_PROFILE, } from '@convai/web-sdk/lipsync-helpers';
 import { applyCorrectiveMap } from './correctiveMap.js';
+import { MouthFinish } from './mouthFinish.js';
+import { Expression } from './expression.js';
 /**
  * Teeth values are the ones approved on a convai-mha-1 head (2026-10-13) and
  * re-checked on the two default bodies: the teeth stay behind the lips at
@@ -15,13 +17,47 @@ const DEFAULTS = {
     teethLowerRest: 0.18,
     teethLowerDrop: 0.35,
     visualLeadSeconds: 0.1,
+    voiceLeadSeconds: 0.2,
+    finish: true,
+    expression: 'neutral',
     staleReleaseSeconds: 0.35,
     // 0.45 is the SDK's recommended gain; 0.28 damped articulation.
     gainOverrides: { mouthLowerLipDepressL: 0.45, mouthLowerLipDepressR: 0.45 },
 };
+/**
+ * How far into a reply's stream its first mouth movement is looked for, s.
+ * The MHA stream opens ~150 ms into the first syllable; a frame further in
+ * than this is not the onset, and the stream is taken from its first frame.
+ */
+const MAX_LEAD_IN = 0.35;
+/** A mouth channel this far from its value in the stream's first frame has moved. */
+const MOVED = 0.04;
+/** Voice level (0–1) below which the voice is between sounds. */
+const QUIET = 0.02;
+/** Frames arriving under a voice that never pauses are late: they start after this, s. */
+const LATE_GRACE = 0.35;
 /** Wall-clock seconds, for catching up when a caller caps or skips its delta. */
 const nowSeconds = () => (typeof performance !== 'undefined' ? performance.now() : Date.now()) / 1000;
 export class LipsyncSystem {
+    /**
+     * Where the stream's motion starts: the first frame that moves the mouth,
+     * any mouth channel away from its rest in the first frame, within the
+     * lead-in. Keyed on the jaw alone, a reply opening on rounded or closed
+     * lips ("Who knew the moon", "Peter") matched a second into the stream
+     * (63 and 80 frames measured), the whole reply ran that far early, and its
+     * last words had no frames left.
+     */
+    leadIn(q, fps) {
+        const limit = Math.min(q.length, Math.round(MAX_LEAD_IN * fps));
+        const rest = q.getFrame(0);
+        let first = 0;
+        for (; rest && first < limit; first++) {
+            const f = q.getFrame(first);
+            if (f && this.ch.mouth.some((i) => Math.abs(f[i] - rest[i]) > MOVED))
+                break;
+        }
+        return first < limit ? first : 0;
+    }
     constructor(tuning = {}, correctiveMap = null) {
         this.bindings = [];
         this.accum = 0;
@@ -37,6 +73,27 @@ export class LipsyncSystem {
         this.speaking = false;
         this.mapped = null;
         this.unpinned = null;
+        /**
+         * The voice's own clock, when the received audio can be read. Set by
+         * createCharacter from the session; null keeps the wall-clock path.
+         */
+        this.voice = null;
+        /** Frames consumed this turn, and the turn frame where speech starts. */
+        this.turnFrame = 0;
+        this.anchor = null;
+        /** Frames waiting when the voice's packets stopped, or null while they flow. */
+        this.holdWaiting = null;
+        /** The queue ran dry after the anchor: the next frames are a new start. */
+        this.dry = false;
+        /** Since it ran dry: whether the voice went quiet, and how long frames have waited. */
+        this.gapSeen = false;
+        this.lateFor = 0;
+        this.blend = null;
+        this.finisher = new MouthFinish();
+        this.wasSpeakingClock = false;
+        /** The last whole frame consumed on the voice clock, uncorrected. */
+        this.rawHeld = null;
+        this.zeros = null;
         this.correctiveMap = correctiveMap;
         const { teethLowerTuck, ...rest } = tuning;
         this.t = {
@@ -45,6 +102,7 @@ export class LipsyncSystem {
             ...rest,
             gainOverrides: { ...DEFAULTS.gainOverrides, ...tuning.gainOverrides },
         };
+        this.expression = this.t.expression === false ? null : new Expression(this.t.expression);
         this.processor = createLipsyncProcessor('mha', {
             skipEyeChannels: true,
             gainOverrides: this.t.gainOverrides,
@@ -61,6 +119,8 @@ export class LipsyncSystem {
             teethUpUpper: idx('teethUpU'),
             teethDownLower: idx('teethDownD'),
             noseUpper: idxs('noseWrinkleUpperL', 'noseWrinkleUpperR'),
+            // Every mouth and jaw channel; the profile's names carry the CTRL_expressions_ prefix.
+            mouth: MHA_PROFILE.channelNames.map((n, i) => (/^(mouth|jaw)/.test(n.replace(/^CTRL_expressions_/, '')) ? i : -1)).filter((i) => i >= 0),
         };
     }
     /** Bind every morph-bearing mesh under root. Returns matched channel count. */
@@ -91,6 +151,108 @@ export class LipsyncSystem {
             this.mapped = new Float32Array(frame.length);
         return applyCorrectiveMap(frame, map, this.mapped);
     }
+    /**
+     * Consume the queue up to where the voice is, and return the stream at that
+     * exact time: the last whole frame blended toward the next by the fraction
+     * left over. Embody draws once per render tick, so at ~46 fps against a
+     * 60 fps stream whole frames stepped 1, 2 or 3 at a time, an uneven speed
+     * that read as jitter. Null until the voice is audible, or when nothing new
+     * is due.
+     */
+    followVoice(q, elapsed, fps) {
+        const voice = this.voice;
+        const played = voice.update(elapsed);
+        if (played === null)
+            return null;
+        // A break in the packets is a stall between the voice's chunks or a
+        // silence the sender suppressed. Frames exist only for audio that was
+        // made: a suppressed silence has its closed-mouth frames waiting when the
+        // packets stop; a stall has none, and its frames (the next chunk's) come
+        // with the next packets. So after a break the clock may move the mouth
+        // only as far as the frames that were waiting when it began; the rest of
+        // the jump is taken off the anchor. Measured on an OpenAI voice: 0.3 s
+        // stalls between chunks, 23 frames of the next phrase skipped on each.
+        if (voice.holding) {
+            if (this.holdWaiting === null)
+                this.holdWaiting = q.length;
+        }
+        else if (this.holdWaiting !== null) {
+            const excess = Math.max(0, voice.resumedBy - this.holdWaiting / fps);
+            if (this.anchor !== null && excess > 0)
+                this.anchor -= excess * fps;
+            this.holdWaiting = null;
+        }
+        const loud = voice.level === null || voice.level > QUIET;
+        if (this.anchor === null) {
+            this.anchor = this.turnFrame + this.leadIn(q, fps);
+        }
+        else if (this.dry) {
+            // Frames after a dry spell are the next chunk's, or late. The voice's
+            // chunks and their frames are made together; the lead empties the queue
+            // ~0.2 s before a chunk's sound ends, and the next chunk's frames can
+            // land then, or a second before their sound. Catching up in a burst
+            // showed a phrase's first shapes in one tick before its sound (17
+            // frames in 90 ms measured); starting on arrival while the last chunk
+            // still sounded ran 0.25 s early. So they start at the voice's next
+            // onset, as a reply does: once it has gone quiet and come back. A voice
+            // that never drops means the frames are late under continuous speech,
+            // and they start after a short grace; an unreadable level starts them
+            // at once.
+            if (!loud)
+                this.gapSeen = true;
+            if (q.hasFrames()) {
+                this.lateFor += elapsed;
+                const start = voice.level === null || (this.gapSeen ? loud : this.lateFor > LATE_GRACE);
+                if (!start)
+                    return null;
+                this.anchor = this.turnFrame + this.leadIn(q, fps) - played * fps;
+                this.dry = false;
+            }
+        }
+        const target = this.anchor + (played + this.t.voiceLeadSeconds) * fps;
+        let last = null;
+        while (this.turnFrame <= Math.floor(target) && q.hasFrames()) {
+            last = q.getFrameWithAlpha(0);
+            q.consumeFrames(1);
+            this.turnFrame++;
+        }
+        if (!q.hasFrames() && !this.dry) {
+            this.dry = true;
+            this.gapSeen = false;
+            this.lateFor = 0;
+        }
+        if (last) {
+            // Copied: the queue's frames are its own and may be reused.
+            if (this.rawHeld?.length !== last.length)
+                this.rawHeld = new Float32Array(last.length);
+            this.rawHeld.set(last);
+        }
+        const held = this.rawHeld;
+        if (!held)
+            return null;
+        const next = q.getFrame(0);
+        const a = target - Math.floor(target);
+        // Nothing new consumed and nothing to move toward: not fresh, so a
+        // starved queue still releases the mouth after staleReleaseSeconds.
+        if (!next)
+            return last ? held : null;
+        if (a <= 0)
+            return held;
+        if (this.blend?.length !== held.length)
+            this.blend = new Float32Array(held.length);
+        for (let i = 0; i < held.length; i++)
+            this.blend[i] = held[i] + ((next[i] ?? held[i]) - held[i]) * a;
+        return this.blend;
+    }
+    /** A rest frame in the stream's layout, for the finisher to add to. */
+    zero() {
+        const n = MHA_PROFILE.channelNames.length;
+        if (this.zeros?.length !== n)
+            this.zeros = new Float32Array(n);
+        else
+            this.zeros.fill(0);
+        return this.zeros;
+    }
     /** Tick each frame. Pass the queue only if this character owns the turn. */
     update(delta, q) {
         if (!this.bindings.length)
@@ -108,7 +270,14 @@ export class LipsyncSystem {
             // or the body layer holds "talking" forever.
             this.speaking = false;
             this.energy += (0 - this.energy) * (1 - Math.exp(-delta / 0.22));
-            this.processor.apply(this.processor.process(null, delta), this.bindings);
+            let idle = this.processor.process(null, delta);
+            if (!idle && this.t.finish && this.finisher.active)
+                idle = this.zero();
+            if (idle && this.t.finish)
+                this.finisher.apply(idle, delta, { raw: null, speaking: false, level: null, envelope: 0 });
+            if (idle)
+                this.expression?.apply(idle, delta, false);
+            this.processor.apply(idle, this.bindings);
             return;
         }
         const fps = q.getPlaybackFps?.() || 60;
@@ -119,8 +288,28 @@ export class LipsyncSystem {
             this.sinceFresh = 0;
         }
         this.wasSpeaking = speaking;
+        const clocked = speaking && !!this.voice?.available;
+        if (speaking && !this.wasSpeakingClock) {
+            this.voice?.reset();
+            this.turnFrame = 0;
+            this.anchor = null;
+            this.holdWaiting = null;
+            this.dry = false;
+            this.rawHeld = null;
+        }
+        this.wasSpeakingClock = speaking;
         let fresh = null;
-        if (speaking) {
+        if (clocked) {
+            fresh = this.followVoice(q, elapsed, fps);
+            if (fresh) {
+                this.lastFrame = this.correct(fresh);
+                this.sinceFresh = 0;
+            }
+            else if (this.anchor !== null) {
+                this.sinceFresh += elapsed;
+            }
+        }
+        else if (speaking) {
             this.accum += elapsed;
             while (this.accum >= frameTime && q.hasFrames()) {
                 const frame = q.getFrameWithAlpha(0);
@@ -145,12 +334,23 @@ export class LipsyncSystem {
         // Holding the last shape through a stall reads as a frozen mouth; let the
         // envelope settle it instead, and pick up again on the next frame.
         const stale = this.sinceFresh > this.t.staleReleaseSeconds;
-        const raw = speaking && !stale ? this.lastFrame : null;
-        const out = this.processor.process(raw, delta);
+        // On the voice clock the mouth holds at rest until the voice is audible.
+        const waiting = clocked && this.anchor === null;
+        const raw = speaking && !stale && !waiting ? this.lastFrame : null;
+        let out = this.processor.process(raw, delta);
+        // The processor returns nothing once settled; the finisher's own motion
+        // (a voice-driven cover) must still be written out until it is gone.
+        if (!out && this.t.finish && this.finisher.active)
+            out = this.zero();
         // mirror the processor's envelope so the seal fades with speech
         const target = raw ? 1 : 0;
         const tau = target > this.env ? 0.09 : 0.18;
         this.env += (target - this.env) * (1 - Math.exp(-delta / tau));
+        if (out && this.t.finish) {
+            this.finisher.apply(out, delta, { raw, speaking, level: this.voice?.level ?? null, envelope: this.env });
+        }
+        if (out)
+            this.expression?.apply(out, delta, speaking);
         if (out) {
             const { jawCap, sealCeiling, sealOnset, teethRetract, teethUpperRest, teethUpperLift, teethLowerRest, teethLowerDrop } = this.t;
             const ch = this.ch;

package/dist/speech/mouthFinish.d.ts

@@ -0,0 +1,46 @@
+export interface MouthFinishState {
+    /** The frame the stream is showing (before shaping), or null. */
+    raw: Float32Array | null;
+    speaking: boolean;
+    /** The voice's level now (0–1), or null when it cannot be read. */
+    level: number | null;
+    /** The processor's speech envelope, 0–1. */
+    envelope: number;
+}
+export declare class MouthFinish {
+    private readonly jaw;
+    private readonly peel;
+    private readonly funnel;
+    private readonly contact;
+    private readonly lipsTogether;
+    /** Every mouth and jaw channel. The profile's names carry the CTRL_expressions_ prefix. */
+    private readonly mouth;
+    /** Current bilabial closure, 0–1. */
+    seal: number;
+    private rest;
+    private pause;
+    private quiet;
+    private voiceEnv;
+    private print;
+    private still;
+    /** Covering a held stream, and how far the cover is in (0–1). */
+    private covering;
+    private cover;
+    private drive;
+    private starved;
+    private heard;
+    private closedFor;
+    private capped;
+    private readonly sx;
+    private readonly sv;
+    private primed;
+    /**
+     * Whether the finisher is still adding to the face: the stall cover, which
+     * the voice drives. The processor returns no frame once its own envelope
+     * has settled, and whatever was last written would then stay on the face;
+     * while this is true the caller must keep feeding frames.
+     */
+    get active(): boolean;
+    /** Finish `out` in place. */
+    apply(out: Float32Array, dt: number, s: MouthFinishState): void;
+}

package/dist/speech/mouthFinish.js

@@ -0,0 +1,235 @@
+/**
+ * Mouth finishing: what the MHA stream needs before it reads as a person
+ * talking, applied to the shaped frame (MHA channel order) before it is
+ * written to the face. The motion is the stream's own; smoothing harder,
+ * rate caps and voice-driven closures were all tried in the coffee-shop
+ * studio and each blurred articulation or stuttered. What remains:
+ *
+ *  · Bilabials. The stream closes the lips with mouthLipsTogether*, a
+ *    MetaHuman corrective the published bodies do not carry, so m/b/p never
+ *    closed. On a closure the lip peel, funnel and most of the jaw are taken
+ *    away as the stream asks, and the lips touch with the sticky shapes,
+ *    bridging only the gap that is left. A closure longer than a real stop
+ *    (the stream runs neighbouring m/b/p together: "Peter Piper picked a peck
+ *    of pickled peppers" sealed half the line) lets go while the voice goes on.
+ *  · Breaks. The stream keeps the mouth moving through the voice's pauses;
+ *    from 150 ms of silence it drifts most of the way to rest, and parts again
+ *    as the voice returns, at a human pace in both directions.
+ *  · Stalls. When the voice plays and the stream gives nothing — frozen on
+ *    one shape, or run dry — the voice's level drives the jaw until frames
+ *    return: no voice ever plays on a still mouth. The cover eases in and
+ *    out and holds until the stream moves again.
+ *  · Onset. The processor's envelope fades speech in over ~90 ms, which showed
+ *    the first syllable of a reply at a quarter of its size, after the voice;
+ *    it is divided back out while it rises.
+ *  · A soft knee on the odd over-wide opening (peaks of 0.5–0.57 against a
+ *    typical high of 0.18 read as aggressive), lips scaled with the jaw.
+ *  · Light critically damped smoothing (~33 ms), then every channel clamped
+ *    to [0, 1] (past 1 a morph extrapolates and the lips cross).
+ *
+ * Voice level comes from the voice clock; without it, breaks and stalls are
+ * left alone (the gates stay off). Values are the studio's.
+ */
+import { MHA_PROFILE, mhaChannelIndex } from '@convai/web-sdk/lipsync-helpers';
+const LT_ON = 0.09;
+const LT_FULL = 0.3;
+/**
+ * A hold this long is a stall, s. Longer than the mouth's lead on the voice
+ * (LipsyncSystem's voiceLeadSeconds, 0.2) plus slack: the stream runs out
+ * that far ahead of the voice at the end of EVERY reply, and its last shape
+ * held under the voice's tail is not a stall (measured: 190 ms). At 60 ms
+ * the cover fired on every last word and the voice popped the jaw open.
+ * 0.34 s was seen on a live reply's end, so the line sits well past it.
+ */
+const STALL = 0.5;
+/** Voice with no stream at all for this long is a dry queue, s. */
+const STARVED = 0.15;
+const MAX_CLOSURE = 0.12;
+const SMOOTH = 60;
+const JAW_KNEE = 0.3;
+const JAW_RATIO = 2.5;
+const PAUSE = 0.15;
+const PAUSE_FULL = 0.4;
+const PAUSE_DEPTH = 0.85;
+const smoothstep = (a, b, x) => {
+    const t = Math.min(1, Math.max(0, (x - a) / (b - a)));
+    return t * t * (3 - 2 * t);
+};
+/** Sticky that just closes the gap left at this jaw; ×0.7 (at full, the lips clung like magnets). */
+const stickyFor = (jaw) => 0.7 * Math.min(0.7, 0.35 + 3.5 * jaw);
+const ix = (...names) => names.map((n) => mhaChannelIndex(n)).filter((i) => i !== undefined);
+export class MouthFinish {
+    constructor() {
+        this.jaw = mhaChannelIndex('jawOpen') ?? -1;
+        this.peel = ix('mouthUpperLipRaiseL', 'mouthUpperLipRaiseR', 'mouthLowerLipDepressL', 'mouthLowerLipDepressR');
+        this.funnel = ix('mouthFunnelUL', 'mouthFunnelUR', 'mouthFunnelDL', 'mouthFunnelDR');
+        this.contact = ix('mouthStickyUC', 'mouthStickyDC', 'mouthStickyUINL', 'mouthStickyUINR', 'mouthStickyDINL', 'mouthStickyDINR');
+        this.lipsTogether = ix('mouthLipsTogetherUL', 'mouthLipsTogetherUR');
+        /** Every mouth and jaw channel. The profile's names carry the CTRL_expressions_ prefix. */
+        this.mouth = MHA_PROFILE.channelNames
+            .map((n, i) => (/^(mouth|jaw)/.test(n.replace(/^CTRL_expressions_/, '')) ? i : -1))
+            .filter((i) => i >= 0);
+        /** Current bilabial closure, 0–1. */
+        this.seal = 0;
+        this.rest = 1;
+        this.pause = 0;
+        this.quiet = 0;
+        this.voiceEnv = 0;
+        this.print = NaN;
+        this.still = 0;
+        /** Covering a held stream, and how far the cover is in (0–1). */
+        this.covering = false;
+        this.cover = 0;
+        this.drive = 0;
+        this.starved = 0;
+        this.heard = false;
+        this.closedFor = 0;
+        this.capped = false;
+        this.sx = new Float32Array(this.mouth.length);
+        this.sv = new Float32Array(this.mouth.length);
+        this.primed = false;
+    }
+    /**
+     * Whether the finisher is still adding to the face: the stall cover, which
+     * the voice drives. The processor returns no frame once its own envelope
+     * has settled, and whatever was last written would then stay on the face;
+     * while this is true the caller must keep feeding frames.
+     */
+    get active() {
+        return this.cover > 0.002 || this.drive > 0.002;
+    }
+    /** Finish `out` in place. */
+    apply(out, dt, s) {
+        if (this.jaw < 0 || dt <= 0)
+            return;
+        // Speaking once the voice is heard: the client flags a reply ~0.45 s
+        // before its audio arrives, and until then the mouth rests, ready.
+        if (!s.speaking)
+            this.heard = false;
+        else if (s.level === null || s.level > 0.02)
+            this.heard = true;
+        const speaking = s.speaking && this.heard;
+        let raw = s.raw;
+        // Stall: a stream holding exactly still while the voice plays.
+        this.voiceEnv += ((s.level === null ? 0 : smoothstep(0.008, 0.09, s.level)) - this.voiceEnv) * (1 - Math.exp(-dt / 0.05));
+        // The cover blends in and out (never switches), and once on it stays on
+        // until the stream moves again, speaking or not. The commonest hold is
+        // the end of a reply: the stream stops on its last shape while the
+        // voice's tail plays. Switched, the cover shut the mouth there, then let
+        // go as the reply ended and the held shape opened it again before the
+        // rest closed it: a flutter on every last word.
+        if (raw) {
+            const print = raw[this.jaw] + 3 * raw[0] + 7 * raw[raw.length >> 1] + 13 * raw[raw.length - 1];
+            if (print !== this.print) {
+                this.still = 0;
+                this.covering = false;
+            }
+            else
+                this.still += dt;
+            this.print = print;
+            this.starved = 0;
+            if (speaking && s.level !== null && this.still > STALL)
+                this.covering = true;
+        }
+        else {
+            // No stream. Under an audible voice that is a dry queue; the end of a
+            // reply, where the stream (which leads) runs out ~0.2 s before the
+            // voice, reads as a fading level and drives nothing.
+            this.starved = speaking && s.level !== null && s.level > 0.02 ? this.starved + dt : 0;
+            if (this.starved > STARVED)
+                this.covering = true;
+            if (!s.speaking && !this.covering)
+                this.still = 0;
+        }
+        if (!s.speaking)
+            this.covering = false;
+        this.cover += ((this.covering ? 1 : 0) - this.cover) * (1 - Math.exp(-dt / (this.covering ? 0.08 : 0.12)));
+        this.drive += ((this.covering ? this.voiceEnv : 0) - this.drive) * (1 - Math.exp(-dt / 0.1));
+        if (this.cover > 0.002) {
+            const k = this.cover;
+            for (const i of this.mouth)
+                out[i] *= 1 - k;
+            out[this.jaw] += 0.18 * this.drive * k;
+            for (const i of this.peel)
+                out[i] += 0.08 * this.drive * k;
+            if (this.covering)
+                raw = null;
+        }
+        // Onset: the first syllable at full size.
+        if (speaking && raw && s.envelope < 0.98) {
+            this.rest = 0;
+            const g = 1 / Math.max(s.envelope, 0.2);
+            for (const i of this.mouth)
+                out[i] *= g;
+        }
+        // Soft knee on the odd over-wide opening.
+        const j0 = out[this.jaw];
+        if (j0 > JAW_KNEE) {
+            const r = (JAW_KNEE + (j0 - JAW_KNEE) / JAW_RATIO) / j0;
+            out[this.jaw] = j0 * r;
+            for (const i of this.peel)
+                out[i] *= r;
+        }
+        // Rest after a reply, and in its breaks.
+        this.rest += ((speaking ? 0 : 1) - this.rest) * (1 - Math.exp(-dt / (speaking ? 0.05 : 0.3)));
+        this.quiet = speaking && s.level !== null && s.level < 0.006 ? this.quiet + dt : 0;
+        const gate = PAUSE_DEPTH * smoothstep(PAUSE, PAUSE_FULL, this.quiet);
+        this.pause += (gate - this.pause) * (1 - Math.exp(-dt / (gate > this.pause ? 0.09 : 0.08)));
+        const settle = Math.max(this.rest, this.pause);
+        if (settle > 0.002)
+            for (const i of this.mouth)
+                out[i] *= 1 - settle;
+        // Bilabials, on the stream's own timing.
+        let lt = 0;
+        if (speaking && raw)
+            for (const i of this.lipsTogether)
+                lt = Math.max(lt, raw[i]);
+        let target = smoothstep(LT_ON, LT_FULL, lt);
+        if (speaking && target > 0.5) {
+            this.closedFor += dt;
+            if (this.closedFor > MAX_CLOSURE && s.level !== null && s.level > 0.01)
+                this.capped = true;
+        }
+        else if (target < 0.3) {
+            this.closedFor = 0;
+            this.capped = false;
+        }
+        if (this.capped)
+            target = 0;
+        this.seal += (target - this.seal) * (1 - Math.exp(-dt / (target > this.seal ? 0.05 : 0.08)));
+        const c = this.seal;
+        if (c > 0.002) {
+            for (const i of this.peel)
+                out[i] *= 1 - c;
+            for (const i of this.funnel)
+                out[i] *= 1 - c;
+            out[this.jaw] *= 1 - 0.9 * c;
+        }
+        // Smoothing, exact for any frame time.
+        const ex = Math.exp(-SMOOTH * dt);
+        for (let k = 0; k < this.mouth.length; k++) {
+            const i = this.mouth[k];
+            const goal = out[i];
+            if (!this.primed) {
+                this.sx[k] = goal;
+                this.sv[k] = 0;
+                continue;
+            }
+            const y = this.sx[k] - goal;
+            const tmp = (this.sv[k] + SMOOTH * y) * dt;
+            this.sx[k] = goal + (y + tmp) * ex;
+            this.sv[k] = (this.sv[k] - SMOOTH * tmp) * ex;
+            out[i] = this.sx[k];
+        }
+        this.primed = true;
+        // Range, then the lips touch: on a bilabial, and lightly at rest.
+        for (const i of this.mouth)
+            out[i] = Math.min(1, Math.max(0, out[i]));
+        const touch = Math.max(c, settle);
+        if (touch > 0.002) {
+            const v = stickyFor(out[this.jaw]) * touch;
+            for (const i of this.contact)
+                out[i] = Math.max(out[i], v);
+        }
+    }
+}

package/dist/speech/voiceClock.d.ts

@@ -0,0 +1,94 @@
+/**
+ * The voice's own clock, for lipsync.
+ *
+ * Mouth frames arrive paced for the wall clock (60 per second from the moment
+ * the character starts speaking). WebRTC does not play the voice on the wall
+ * clock: its jitter buffer holds it (~180 ms measured) and stretches and
+ * squeezes it to absorb network jitter (a 25.25 s reply played in 24.9 s).
+ * Against the pauses in the voice, a wall-clocked mouth started ~230 ms late
+ * and ended ~250 ms early in a 20 s reply: it moved through silences.
+ *
+ * So the mouth follows the voice. This reads the received audio track's RTP
+ * synchronization source — the timestamp of the sample playing now (48 kHz)
+ * and its level — and reports how much voice has played since the first
+ * audible sample:
+ *
+ *  · until the voice is audible it reports null (the mouth holds);
+ *  · after that it advances by the voice's own RTP steps, so stretching and
+ *    squeezing move the mouth with the voice;
+ *  · while no packets arrive (a network break, a server stall, silence
+ *    suppression) it holds, and when they return advances by the voice time
+ *    that actually passed — a wall-clocked mouth stayed ahead of the voice by
+ *    every break's length for the rest of the reply.
+ *
+ * The level comes from the source when the sender negotiated the RTP
+ * audio-level extension. Convai's voice does not (measured on staging:
+ * sources carry no `audioLevel`), so otherwise it comes from the receiver's
+ * inbound-rtp stats: the level of the decoded audio, refreshed every
+ * ~100–200 ms in Chrome. Coarse, but breaks are 150 ms and longer. One stats
+ * request is in flight at a time; a stats level older than 0.5 s is not used.
+ *
+ * A level is believed only once it has read above zero. A voice the page
+ * does not play out (autoplay blocked before a user gesture, a muted or
+ * detached element) still arrives and still advances the clock, but Chrome
+ * measures its level as exactly 0 throughout (measured: 306 speaking frames,
+ * peak 0, totalAudioEnergy 0). Taken at its word, that silence held every
+ * break gate shut and pulled the mouth to rest against the stream every
+ * 150 ms of a reply. Until the voice is shown to be playing, the level is
+ * unknown (null), and the gates that depend on it stay off.
+ *
+ * No AudioContext: neither level needs a user gesture.
+ */
+/** What the clock needs of an RTCRtpReceiver. */
+export interface VoiceReceiver {
+    getSynchronizationSources(): {
+        rtpTimestamp: number;
+        timestamp: number;
+        audioLevel?: number;
+    }[];
+    getStats?(): Promise<{
+        forEach(cb: (s: {
+            type: string;
+            kind?: string;
+            audioLevel?: number;
+        }) => void): void;
+    }>;
+}
+export declare class VoiceClock {
+    private readonly receiver;
+    private readonly now;
+    private started;
+    private played;
+    private lastRtp;
+    private lastGoodRtp;
+    private held;
+    private waited;
+    /** Seconds of packets seen while waiting for the voice. */
+    private packets;
+    private stats;
+    private polling;
+    /** Whether any level has read above zero: the voice is being played out. */
+    private playing;
+    /** Audio level of the voice this tick (0–1), or null without a receiver. */
+    level: number | null;
+    /** True while no packets arrive: the clock is holding. */
+    holding: boolean;
+    /** Seconds the clock jumped on the tick packets came back; 0 on every other tick. */
+    resumedBy: number;
+    constructor(receiver: () => VoiceReceiver | null, now?: () => number);
+    /** Whether a receiver is available to clock against. */
+    get available(): boolean;
+    /** Start of a turn: forget the last one. */
+    reset(): void;
+    /**
+     * The latest inbound-rtp level, and ask for the next one. `undefined` while
+     * the first answer is still coming (a level is readable, just not yet);
+     * null when the receiver has no stats at all.
+     */
+    private statsLevel;
+    /**
+     * Advance by `wallStep` seconds of render time. Returns seconds of voice
+     * played since it became audible, or null while it is not audible yet.
+     */
+    update(wallStep: number): number | null;
+}