@convai/embody 1.0.1-beta.0 → 1.1.0-beta.0

package/dist/speech/voiceClock.js

@@ -0,0 +1,176 @@
+/**
+ * The voice's own clock, for lipsync.
+ *
+ * Mouth frames arrive paced for the wall clock (60 per second from the moment
+ * the character starts speaking). WebRTC does not play the voice on the wall
+ * clock: its jitter buffer holds it (~180 ms measured) and stretches and
+ * squeezes it to absorb network jitter (a 25.25 s reply played in 24.9 s).
+ * Against the pauses in the voice, a wall-clocked mouth started ~230 ms late
+ * and ended ~250 ms early in a 20 s reply: it moved through silences.
+ *
+ * So the mouth follows the voice. This reads the received audio track's RTP
+ * synchronization source — the timestamp of the sample playing now (48 kHz)
+ * and its level — and reports how much voice has played since the first
+ * audible sample:
+ *
+ *  · until the voice is audible it reports null (the mouth holds);
+ *  · after that it advances by the voice's own RTP steps, so stretching and
+ *    squeezing move the mouth with the voice;
+ *  · while no packets arrive (a network break, a server stall, silence
+ *    suppression) it holds, and when they return advances by the voice time
+ *    that actually passed — a wall-clocked mouth stayed ahead of the voice by
+ *    every break's length for the rest of the reply.
+ *
+ * The level comes from the source when the sender negotiated the RTP
+ * audio-level extension. Convai's voice does not (measured on staging:
+ * sources carry no `audioLevel`), so otherwise it comes from the receiver's
+ * inbound-rtp stats: the level of the decoded audio, refreshed every
+ * ~100–200 ms in Chrome. Coarse, but breaks are 150 ms and longer. One stats
+ * request is in flight at a time; a stats level older than 0.5 s is not used.
+ *
+ * A level is believed only once it has read above zero. A voice the page
+ * does not play out (autoplay blocked before a user gesture, a muted or
+ * detached element) still arrives and still advances the clock, but Chrome
+ * measures its level as exactly 0 throughout (measured: 306 speaking frames,
+ * peak 0, totalAudioEnergy 0). Taken at its word, that silence held every
+ * break gate shut and pulled the mouth to rest against the stream every
+ * 150 ms of a reply. Until the voice is shown to be playing, the level is
+ * unknown (null), and the gates that depend on it stay off.
+ *
+ * No AudioContext: neither level needs a user gesture.
+ */
+/** RTP clock rate of Opus, the voice codec. */
+const RATE = 48000;
+/** Audio level (0–1) that counts as audible. */
+const ONSET = 0.01;
+/** A source older than this has stopped playing, s. */
+const STALE = 0.06;
+/** Longest the clock waits for the voice before starting anyway, s. */
+const ANCHOR_TIMEOUT = 15;
+/** Packets for this long with no level ever above zero: the voice is not played out, s. */
+const SILENT_PLAYOUT = 1;
+/** A stats level older than this is not used, s. */
+const STATS_STALE = 0.5;
+export class VoiceClock {
+    constructor(receiver, now = () => Date.now()) {
+        this.receiver = receiver;
+        this.now = now;
+        this.started = false;
+        this.played = 0;
+        this.lastRtp = null;
+        this.lastGoodRtp = null;
+        this.held = 0;
+        this.waited = 0;
+        /** Seconds of packets seen while waiting for the voice. */
+        this.packets = 0;
+        this.stats = null;
+        this.polling = false;
+        /** Whether any level has read above zero: the voice is being played out. */
+        this.playing = false;
+        /** Audio level of the voice this tick (0–1), or null without a receiver. */
+        this.level = null;
+        /** True while no packets arrive: the clock is holding. */
+        this.holding = false;
+        /** Seconds the clock jumped on the tick packets came back; 0 on every other tick. */
+        this.resumedBy = 0;
+    }
+    /** Whether a receiver is available to clock against. */
+    get available() {
+        return this.receiver() !== null;
+    }
+    /** Start of a turn: forget the last one. */
+    reset() {
+        this.started = false;
+        this.played = 0;
+        this.lastRtp = this.lastGoodRtp = null;
+        this.held = 0;
+        this.waited = 0;
+        this.packets = 0;
+        this.holding = false;
+        this.resumedBy = 0;
+    }
+    /**
+     * The latest inbound-rtp level, and ask for the next one. `undefined` while
+     * the first answer is still coming (a level is readable, just not yet);
+     * null when the receiver has no stats at all.
+     */
+    statsLevel(r) {
+        if (r.getStats && !this.polling) {
+            this.polling = true;
+            r.getStats()
+                .then((report) => {
+                report.forEach((s) => {
+                    if (s.type === 'inbound-rtp' && s.kind === 'audio' && typeof s.audioLevel === 'number')
+                        this.stats = { level: s.audioLevel, at: this.now() };
+                });
+            })
+                .catch(() => { })
+                .finally(() => { this.polling = false; });
+        }
+        if (!r.getStats)
+            return null;
+        const st = this.stats;
+        return st && (this.now() - st.at) / 1000 <= STATS_STALE ? st.level : undefined;
+    }
+    /**
+     * Advance by `wallStep` seconds of render time. Returns seconds of voice
+     * played since it became audible, or null while it is not audible yet.
+     */
+    update(wallStep) {
+        const r = this.receiver();
+        const src = r?.getSynchronizationSources()[0];
+        const age = src ? (this.now() - src.timestamp) / 1000 : Infinity;
+        const rtp = src && age <= STALE ? src.rtpTimestamp + Math.max(0, age) * RATE : null;
+        const read = src ? (age <= STALE ? src.audioLevel ?? this.statsLevel(r) : 0) : null;
+        const level = read === undefined ? null : read;
+        if (level !== null && level > 0)
+            this.playing = true;
+        this.level = this.playing ? level : null;
+        if (!this.started) {
+            this.waited += wallStep;
+            if (rtp !== null)
+                this.packets += wallStep;
+            // The onset reads the raw level. A level still being fetched is not
+            // "no level": between replies the stats go stale, so the first packet
+            // of every reply found none and started the clock on it, 75–200 ms
+            // before the voice was audible, a different amount each reply. Only a
+            // receiver that cannot report a level starts on packets. A voice never
+            // played out reads 0 throughout; after a second of packets with no
+            // level ever seen, start on them rather than wait out the timeout.
+            const audible = read === undefined ? false
+                : read === null ? rtp !== null
+                    : read > ONSET || (!this.playing && this.packets > SILENT_PLAYOUT);
+            if (!audible && this.waited <= ANCHOR_TIMEOUT)
+                return null;
+            this.started = true;
+            this.played = 0;
+            this.lastRtp = this.lastGoodRtp = rtp;
+            return 0;
+        }
+        let step;
+        this.resumedBy = 0;
+        if (rtp === null) {
+            // No packets: hold.
+            this.held += wallStep;
+            step = 0;
+        }
+        else if (this.held > 0) {
+            // Packets are back: advance by the voice time that passed, or by the
+            // time held when the RTP jump is not usable (a new audio segment).
+            const d = this.lastGoodRtp === null ? -1 : (rtp - this.lastGoodRtp) / RATE;
+            step = d >= 0 && d <= this.held + 1 ? d : this.held;
+            this.resumedBy = step;
+            this.held = 0;
+        }
+        else {
+            const d = this.lastRtp === null ? -1 : (rtp - this.lastRtp) / RATE;
+            step = d >= 0 && d <= 0.25 ? d : wallStep;
+        }
+        this.played += step;
+        this.holding = rtp === null;
+        if (rtp !== null)
+            this.lastGoodRtp = rtp;
+        this.lastRtp = rtp;
+        return this.played;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@convai/embody",
-  "version": "1.0.1-beta.0",
+  "version": "1.1.0-beta.0",
   "description": "Give a Convai character a body: resolve 3D characters by characterId and drive them with lipsync and animation. Layered on @convai/web-sdk.",
   "license": "Apache-2.0",
   "author": "Convai <support@convai.com>",