@voqalize/avatar 0.2.2 → 0.3.0

package/src/avatar.js

@@ -1,14 +1,18 @@
 /**
- * The avatar — a programmable talking head.
+ * The mixer — a programmable talking head.
  *
- * The public surface. Everything the server drives goes through here:
+ * Not the public surface: that is `createAvatar({ mount, client })` in
+ * `packages/avatar/client/createAvatar.ts`, and this is what it drives. Everything below is
+ * addressed by our own runtime, our tooling, and an avatar author who chose to
+ * build on the SVG renderer (`@voqalize/avatar/internal`, no semver promise).
  *
+ *   const avatar = createAvatar({ mount, face: peep })   // faces.js, or a face module
  *   avatar.setState('LISTENING', { emotion: 'warm' })
  *   avatar.setGaze('SCREEN_LEFT')
  *   avatar.speak({ audio, cues })        // cues are {t, v, i?} in ms
  *   avatar.pushCues(moreCues)            // streaming top-up
- *   avatar.interject('OKAY')
- *   avatar.gesture('HI')                 // a hand at the frame edge + its face
+ *   avatar.action('ACK_RECEIVE')
+ *   avatar.action('GESTURE_GREET')       // a hand at the frame edge + its face
  *   avatar.perform(beats, { audio })     // timed {t, do, ...} verbs, same clock
  *   avatar.setUserSpeaking(bool)         // the user has the floor, so listening
  *                                        // is contingent instead of timed
@@ -26,25 +30,33 @@
  */
 
 import { REST, CHANNELS, TAU, RANGE, GROUPS, clamp, approach } from './params.js';
-import { createFace as createPeepFace, META as peepMeta } from './face-peep.js';
-import { createFace as createWrenFace, META as wrenMeta } from './face-wren.js';
-import { createFace as createMynaFace, META as mynaMeta } from './face-myna.js';
 import { emotionPose } from './emotions.js';
-import { GazeLayer, GAZE_TARGETS } from './gaze.js';
+import { GazeLayer, GAZE_TARGETS, AVERSION } from './gaze.js';
 import { IdleLayer, ListeningEngine } from './idle.js';
 import { ClipPlayer } from './clips.js';
-import { INTERJECTIONS } from './interjections.js';
+import { ACTIONS, INTERNAL_CLIPS } from './interjections.js';
 import { VisemeTrack, shapeFor, SILENT } from './visemes.js';
 import { PerformTrack } from './perform.js';
-import { createHand, HAND_GESTURES } from './hand.js';
+import { createHand, HAND_GESTURES, HAND_ACTION_TO_FRAME_GESTURE } from './hand.js';
+import { avatarFrame, createSvgRig } from './rig.js';
 
 // Each state's `idle` is a profile for the liveness layer (see DEFAULT_PROFILE
 // in idle.js). Blink gaps come from docs/research-biomechanics.md §5: the rate
 // alone separates listening (~16/min) from thinking (~25/min) from visually
 // busy (~9/min), and it is the cheapest state signal the rig has.
 export const STATES = {
-  IDLE:               { gaze: 'USER',     emotion: 'neutral',    idle: { sway: 1.0 }, backchannel: false },
-  LISTENING:          { gaze: 'USER',     emotion: 'neutral',    backchannel: true,
+  // Idle means present but occupied with one's own quiet business. It must not
+  // compete with LISTENING's sustained user attention: the default target is
+  // away from the user and the wander only visits other non-task targets.
+  IDLE:               { gaze: 'AWAY_THINKING', emotion: 'neutral', engagement: false,
+                        idle: { sway: 0.72, blinkGap: [4.6, 6.6] },
+                        wander: { targets: ['AWAY_THINKING', 'AWAY_RIGHT', 'NOTES'], every: [3.6, 6.8] } },
+  // `aversion` is why this state does not stare. Continuous eye contact is not
+  // the attentive pose it looks like — it is a demand for more talk (Rossano)
+  // and it measures as *tense*, not attentive (Wang & Gratch). See AVERSION in
+  // gaze.js for the numbers; the mixer holds it off near a turn boundary.
+  LISTENING:          { gaze: 'USER',     emotion: 'neutral',    engagement: true,
+                        aversion: 'LISTEN',
                         idle: { sway: 1.0, blinkGap: [3.1, 4.2] },
                         pose: { browRaiseL: 0.06, browRaiseR: 0.06, lidL: -0.04, lidR: -0.04 } },
   // Faster, shallower breath is the measured cognitive-load signature, and the
@@ -53,13 +65,13 @@ export const STATES = {
   // leads DOWN (39% of measured cognitive aversions, §4.2) and wanders on the
   // ~3.5s cognitive-aversion cadence, coming back to the user roughly one
   // dwell in four — still with you, working.
-  THINKING:           { gaze: 'AWAY_DOWN', emotion: 'thoughtful', backchannel: false,
+  THINKING:           { gaze: 'AWAY_DOWN', emotion: 'thoughtful', engagement: false,
                         idle: { sway: 0.7, blinkGap: [2.1, 2.7], breathRate: 1.18, breathAmp: 0.7,
                                 hold: { every: [4.5, 9.0], dur: [0.8, 1.5] } },
                         wander: { targets: ['AWAY_DOWN', 'AWAY_DOWN', 'AWAY_THINKING', 'USER'],
                                   every: [2.6, 4.4] } },
-  SPEAKING:           { gaze: 'USER',     emotion: 'neutral',    idle: { sway: 0.55 }, backchannel: false },
-  REVIEWING_SCREEN:   { gaze: 'SCREEN_CENTER', emotion: 'thoughtful', backchannel: false,
+  SPEAKING:           { gaze: 'USER',     emotion: 'neutral',    idle: { sway: 0.55 }, engagement: false },
+  REVIEWING_SCREEN:   { gaze: 'SCREEN_CENTER', emotion: 'thoughtful', engagement: false,
                         idle: { sway: 0.8, blinkGap: [4.0, 6.5] },
                         wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_RIGHT', 'SCREEN_TOP', 'SCREEN_WORK'],
                                   every: [1.8, 5.0] } },
@@ -67,7 +79,7 @@ export const STATES = {
   // multiplier to exist at all: 0.05 here renders as 0.3° of rotation, which
   // is no tilt whatever the number says. 0.30 renders ~1.7° — visible at tile
   // size, still gentle. Every other channel in this pose read fine on screen.
-  WAITING_FOR_USER: { gaze: 'USER',     emotion: 'encouraging', backchannel: true,
+  WAITING_FOR_USER: { gaze: 'USER',     emotion: 'encouraging', engagement: true,
                         idle: { sway: 1.0, blinkGap: [3.1, 4.2] },
                         pose: { headRoll: 0.30, browRaiseL: 0.16, browRaiseR: 0.12 } },
   // Straining to hear. The one state where the amplitude constraint yields,
@@ -75,11 +87,11 @@ export const STATES = {
   // engagement ceiling (+0.16), head cheated aside on USER_EAR so an ear
   // favors the speaker while the eyes hold contact, and a concentration
   // squint with knit brows. Stillness does the rest — straining people
-  // freeze — so holds are frequent and there are NO backchannels: you don't
+  // freeze — so holds are frequent and there is no engagement lean: you don't
   // nod along to what you can't hear. Server sends it on soft/low-SNR user
   // audio, typically followed by SORRY or a "could you repeat" utterance.
   CANT_HEAR: {
-    gaze: 'USER_EAR', emotion: 'neutral', backchannel: false,
+    gaze: 'USER_EAR', emotion: 'neutral', engagement: false,
     idle: { sway: 0.5, blinkGap: [4.5, 6.5],
             hold: { every: [2.5, 5.5], dur: [1.0, 1.8] } },
     // A minimal line face swallows small deltas — the ink moves whole units
@@ -94,6 +106,33 @@ export const STATES = {
       mouthPress: 0.45, mouthCornerL: -0.22, mouthCornerR: -0.22,
     },
   },
+  // The user's microphone is closed, and the agent is the one that closed it.
+  // Authored against CANT_HEAR as its exact inverse, because that contrast is
+  // the whole read: straining leans *in* to get more of the user, this settles
+  // *back* — nothing more is coming through and the avatar knows why. The eyes
+  // stay on the user, because a deliberate hold is not inattention; what
+  // carries it is the mouth, pressed shut and staying shut, and the slow,
+  // unhurried blink of waiting rather than working. No engagement lean: you
+  // cannot nod along to a channel you have muted yourself. No filter, ever —
+  // DEGRADED and OFFLINE own "something is broken", and this is a decision.
+  MUTED: {
+    gaze: 'USER', emotion: 'neutral', engagement: false,
+    idle: { sway: 0.45, blinkGap: [4.5, 7.0],
+            hold: { every: [3.0, 6.0], dur: [0.9, 1.6] } },
+    // Line-face scaled (see CANT_HEAR): peep's resting mouth is drawn smiling,
+    // so a closed mouth has to be authored clearly past flat to read as closed
+    // at all. Corners were -0.20 first, which rendered as a straight line and
+    // at 130 px against LISTENING said nothing — the delta a viewer gets is
+    // curvature, and flat is the halfway point of it, not the end. -0.34 is as
+    // far as it goes before the hold starts reading as sulking. No squint and
+    // no lid drop: one is straining, the other was asleep. browInner carries
+    // the "one moment" without the worry lift.
+    pose: {
+      torsoLean: -0.28, headPitch: 0.04,
+      browRaiseL: 0.08, browRaiseR: 0.06, browInnerL: 0.28, browInnerR: 0.22,
+      mouthPress: 0.78, mouthCornerL: -0.34, mouthCornerR: -0.34,
+    },
+  },
   // --- application state ---------------------------------------------------
   // "Momentarily busy on the thing you asked for." No hands in frame, so the
   // whole read comes from four cheap cues (docs/research-biomechanics.md §6.4):
@@ -102,12 +141,12 @@ export const STATES = {
   // is what says activity rather than rocking — and, the important one, a
   // brief glance back up to the user every few seconds. The glance is the tell
   // that the user has not been forgotten; without it, busy is just absent.
-  TYPING: {
+  WORKING: {
     // SCREEN_WORK, not NOTES: on a steep down target the gaze layer's lid
     // follow seals the eyes, and at tile size shut eyes read as asleep, not
     // busy. A mild down-left with the head pitched into it keeps the iris in
     // the opening — eyes down but awake.
-    gaze: 'SCREEN_WORK', emotion: 'neutral', backchannel: false,
+    gaze: 'SCREEN_WORK', emotion: 'neutral', engagement: false,
     idle: { sway: 0.6, blinkGap: [6.0, 7.5], breathRate: 1.05,
             rhythm: { amp: 0.05, freq: 2.2 } },
     glance: { to: 'USER', every: [4, 7], hold: [0.7, 1.1] },
@@ -123,7 +162,7 @@ export const STATES = {
   // DEGRADED says "my feed is broken", TYPING_CHAT says "I'm working around
   // it" — a server will typically sequence DEGRADED → TYPING_CHAT.
   TYPING_CHAT: {
-    gaze: 'SCREEN_WORK', emotion: 'neutral', backchannel: false,
+    gaze: 'SCREEN_WORK', emotion: 'neutral', engagement: false,
     idle: { sway: 0.6, blinkGap: [5.5, 7.0], breathRate: 1.05,
             rhythm: { amp: 0.055, freq: 2.5 } },
     glance: { to: 'USER', every: [3.2, 5.5], hold: [1.2, 2.0] },
@@ -136,12 +175,12 @@ export const STATES = {
   },
   // Attention genuinely elsewhere. What separates this from TYPING is target
   // *stability* (§6.4): busy is one steady off-user target, distracted is
-  // wandering ones, held long (aversion >3s), with no backchannels — the
+  // wandering ones, held long (aversion >3s), with no engagement lean — the
   // missing nod is as diagnostic as the look-away. Sway is looser than
   // LISTENING because attention is what was holding the body still. The
   // widget only looks away; deciding when to snap back is the server's call.
   DISTRACTED: {
-    gaze: 'AWAY_RIGHT', emotion: 'neutral', backchannel: false,
+    gaze: 'AWAY_RIGHT', emotion: 'neutral', engagement: false,
     idle: { sway: 1.15, blinkGap: [1.8, 4.2] },
     // Sideways and up, never steep-down: lateral is where real intimacy/
     // distraction aversions live, and a steep down target seals this rig's
@@ -156,7 +195,7 @@ export const STATES = {
   // wiggle nobody makes while merely reading. Server semantics: a filler
   // while an async activity completes; the server exits it when done.
   SEARCHING_SCREEN: {
-    gaze: 'SCREEN_CENTER', emotion: 'neutral', backchannel: false,
+    gaze: 'SCREEN_CENTER', emotion: 'neutral', engagement: false,
     idle: { sway: 0.65, blinkGap: [5.0, 6.8], breathRate: 1.05,
             flick: { amp: 0.30, every: [3.5, 7.0] } },
     wander: { targets: ['SCREEN_CENTER', 'SCREEN_LEFT', 'SCREEN_TOP', 'SCREEN_WORK',
@@ -180,7 +219,7 @@ export const STATES = {
   // use to predict that someone is about to speak. The head comes *up* rather
   // than down: a lowered head is deferential and reads as yielding.
   TAKING_FLOOR: {
-    gaze: 'USER', emotion: 'neutral', idle: { sway: 0.6 }, backchannel: false,
+    gaze: 'USER', emotion: 'neutral', idle: { sway: 0.6 }, engagement: false,
     pose: {
       browRaiseL: 0.26, browRaiseR: 0.22, lidL: -0.10, lidR: -0.10,
       headPitch: -0.10, torsoLean: 0.22, shoulderL: 0.30, shoulderR: 0.30,
@@ -193,7 +232,7 @@ export const STATES = {
   // purpose — stillness is what makes it read as intent rather than as fidget),
   // leaning in, lips apart and staying apart.
   WANTS_IN: {
-    gaze: 'USER', emotion: 'neutral', idle: { sway: 0.45 }, backchannel: false,
+    gaze: 'USER', emotion: 'neutral', idle: { sway: 0.45 }, engagement: false,
     pose: {
       browRaiseL: 0.42, browRaiseR: 0.38, lidL: -0.14, lidR: -0.14,
       headPitch: -0.14, torsoLean: 0.42, shoulderL: 0.45, shoulderR: 0.45,
@@ -204,17 +243,17 @@ export const STATES = {
   // happen faster than anything else on the face — see YIELD_FLOOR, which is
   // what actually delivers the snap.
   YIELDED: {
-    gaze: 'USER', emotion: 'neutral', idle: { sway: 0.9 }, backchannel: false,
+    gaze: 'USER', emotion: 'neutral', idle: { sway: 0.9 }, engagement: false,
     pose: {
       browRaiseL: 0.10, browRaiseR: 0.08,
       torsoLean: -0.18, shoulderL: -0.12, shoulderR: -0.12,
     },
   },
 
-  DEGRADED:           { gaze: 'USER',     emotion: 'neutral',    backchannel: false,
+  DEGRADED:           { gaze: 'USER',     emotion: 'neutral',    engagement: false,
                         idle: { sway: 0.4, blinkGap: [4.0, 8.0] },
                         pose: { lidL: 0.3, lidR: 0.3 }, filter: 'grayscale(.55) brightness(.82)' },
-  OFFLINE:            { gaze: 'USER',     emotion: 'neutral',    backchannel: false,
+  OFFLINE:            { gaze: 'USER',     emotion: 'neutral',    engagement: false,
                         idle: { sway: 0.15, blinkGap: [9, 15] },
                         pose: { lidL: 0.95, lidR: 0.95, mouthCornerL: 0, mouthCornerR: 0 },
                         filter: 'grayscale(1) brightness(.6)' },
@@ -222,63 +261,26 @@ export const STATES = {
 
 export const STATE_NAMES = Object.keys(STATES);
 
-/**
- * The avatars this rig can wear: `{ create, meta }` records.
- *
- * `create` is `createFace(mount, theme) -> { svg, apply, theme, destroy }`,
- * callable standalone — the rig tooling drives faces with no mixer attached.
- * That behavioural contract is still the whole of what the *rig* needs:
- * everything else — visemes, emotions, gaze, idle, clips, the mixer — works
- * in parameter space and never learns which face it is driving.
- *
- * `meta` is the avatar descriptor (viewBox, mouthCrop — see META in any face
- * module): the things a HOST or a TOOL needs to frame a face without opening
- * it. This registry was once factories-only, on the argument that a schema
- * guessed from two faces would be wrong; the third face settled it. Every rig
- * needed exactly a framing rect and a mouth rect to stop the tooling from
- * hard-coding per-avatar tables, and nothing else — so that is all meta
- * carries.
- *
- * The key is the avatar's name, not its rank. It used to be possible to read
- * rank into it — the original rig was keyed `default`, which became a lie the
- * moment it stopped being the one we ship. DEFAULT_AVATAR below is the only
- * place the choice is made.
- *
- * Two earlier rigs, `classic` and `blue-shirt`, were removed on 2026-08-06:
- * stakeholders accepted the line-art pair and rejected both of the others, so
- * carrying them was maintenance against art nobody wanted. What they taught
- * the abstraction survives them — `face-core.js` exists because all three of
- * the first rigs wrote the same apply(), and META exists because all three
- * needed the same two rects. Their code is in git history if a lesson ever
- * needs re-reading.
- */
-export const AVATARS = {
-  peep: { create: createPeepFace, meta: peepMeta },
-  wren: { create: createWrenFace, meta: wrenMeta },
-  myna: { create: createMynaFace, meta: mynaMeta },
-};
-
-export const AVATAR_NAMES = Object.keys(AVATARS);
-
-/** The avatar a host gets when it does not ask for one. */
-export const DEFAULT_AVATAR = 'peep';
-
 export function createAvatar(opts = {}) {
   const mount = typeof opts.mount === 'string' ? document.querySelector(opts.mount) : opts.mount;
   if (!mount) throw new Error('createAvatar: mount element required');
 
-  // `opts.avatar` names one from AVATARS; `opts.face` passes a factory directly,
-  // so a host can supply an avatar the rig has never heard of. A bare factory
-  // has no descriptor, so meta falls back to what the svg itself declares.
-  const entry = opts.face ? { create: opts.face } : AVATARS[opts.avatar || DEFAULT_AVATAR];
-  if (!entry) {
-    throw new Error(`createAvatar: unknown avatar "${opts.avatar}" (have: ${AVATAR_NAMES.join(', ')})`);
+  // `opts.face` is a Face record — `{ create, meta }`, one per face module. It
+  // is passed in rather than named, because a name would need a table, and a
+  // table would need every face imported to answer any lookup: three drawings
+  // in every consumer's bundle to render one. `src/faces.js` still has that
+  // table, for tooling that genuinely wants all of them.
+  const entry = opts.rig ? null : opts.face;
+  if (!opts.rig && !entry) {
+    throw new Error('createAvatar: a `face` (see src/faces.js) or a `rig` is required');
   }
-  const face = entry.create(mount, opts.theme);
-  const meta = entry.meta || (() => {
-    const vb = (face.svg.getAttribute('viewBox') || '0 0 1 1').split(/[\s,]+/).map(Number);
-    return { viewBox: { x: vb[0], y: vb[1], w: vb[2], h: vb[3] } };
-  })();
+  const face = entry ? entry.create(mount, opts.theme) : null;
+  // A renderer-neutral rig needs no SVG descriptor: `meta` is what SVG hosts
+  // and tools frame the drawing with, never a requirement of the rig contract.
+  // A face carries its own — a `{ create }` with no META used to be tolerated
+  // here and the viewBox re-read off the produced svg, which meant a face could
+  // ship half a descriptor and nothing would say so.
+  const meta = face ? entry.meta : null;
   const gaze = new GazeLayer();
   const idle = new IdleLayer();
   const speech = new VisemeTrack();
@@ -288,19 +290,21 @@ export function createAvatar(opts = {}) {
     onGaze: (g) => { gazeOverrideByClip = g; applyGaze(); },
     onBlink: () => idle.blink(),
   });
-  const backchannel = new ListeningEngine((id) => { interject(id); emit('backchannel', id); });
+  const engagement = new ListeningEngine();
   const performTrack = new PerformTrack();
-  // The hand is a sibling of the mixer, not a layer inside it: it writes SVG
-  // directly rather than parameter channels, because a hand at the frame edge is
-  // not part of the rig's body (see hand.js, and CLAUDE.md constraint 9 for the
-  // arm chain this replaces). `hand: false` opts out — for a face drawn in some
-  // other idiom, or a tile too small to spend the pixels.
-  const hand = opts.hand === false ? null : createHand(face.svg, face.theme, meta, { dir: opts.handSide });
+  // The current SVG hand is a renderer adapter for the first-class `frame.hand`
+  // control. `hand: false` only disables its SVG rendering; gesture actions
+  // still emit the semantic hand frame for a supplied custom rig.
+  const hand = face && opts.hand !== false ? createHand(face.svg, face.theme, meta, { dir: opts.handSide }) : null;
+  // The existing SVG face and hand are one migration adapter implementing the
+  // renderer-agnostic AvatarRig contract. New renderers never need face SVG
+  // coordinates or the hand layer's private geometry.
+  const rig = opts.rig ? opts.rig(mount, opts.rigOptions) : createSvgRig(face, hand);
 
   gaze.onLargeShift = () => idle.blink();
 
   const listeners = {
-    state: [], speakEnd: [], clipEnd: [], backchannel: [], performEnd: [], gestureEnd: [],
+    state: [], speakEnd: [], clipEnd: [], performEnd: [], gestureEnd: [],
   };
   const emit = (ev, ...a) => listeners[ev] && listeners[ev].forEach((f) => f(...a));
   clip.onEnd = (c) => { if (c) emit('clipEnd', c.id); };
@@ -323,6 +327,9 @@ export function createAvatar(opts = {}) {
   // excursion changes. Values above ~1.5 saturate the open vowels against the
   // channel clamp, which is the intended ceiling rather than a bug.
   let mouthGain = opts.mouthGain ?? 1;
+  let handSide = opts.handSide === -1 ? 'left' : 'right';
+  let handAction = null;
+  const handQueue = [];
   // Gesture gain, same idea for the clip layer. A nod is ballistic — NOD_SMALL
   // peaks at 149ms — but the head smooths at a 160ms time constant, so barely
   // 60% of an authored peak is ever rendered. The keyframes were written against
@@ -340,6 +347,7 @@ export function createAvatar(opts = {}) {
   let slowBlinkAt = 0;
   let glanceAt = 0;
   let glanceUntil = 0;
+  let attendUntil = 0;
   let speakClock = null;
   let speakStart = 0;
 
@@ -358,7 +366,7 @@ export function createAvatar(opts = {}) {
   // `manual` withholds the rAF loop so a tool can drive frames itself. The
   // baseline pages could already step a ClipPlayer by hand, but nothing could
   // step the *mixer* — which is where idle, gaze and engagement actually
-  // compose — so motion had no reproducible render. See tools/motion.mjs.
+  // compose — so motion had no reproducible render. See apps/authoring/tools/motion.mjs.
   const manual = !!opts.manual;
 
   function frame(now) {
@@ -427,23 +435,31 @@ export function createAvatar(opts = {}) {
         setGaze(gl.to);
       }
     }
-    backchannel.enabled = !!st.backchannel && !clip.playing;
-    backchannel.update(dt);
+    // Aversion is a property of the state, but it is held off around a turn
+    // boundary: the floor is handed over under mutual gaze, and an avatar that
+    // looks away exactly as the user finishes has declined it. `attend` is the
+    // mixer's one-frame veto — anything that means "the user is checking
+    // whether I am with them" sets it (see api.attend).
+    gaze.setAversion(st.aversion ? AVERSION[st.aversion] : null);
+    gaze.hold = attendUntil > elapsed || clip.playing;
+
+    engagement.enabled = !!st.engagement && !clip.playing;
+    engagement.update(dt);
     // Engagement posture: forward lean while the user holds the floor, spent
     // only in the states that are *about* the user holding the floor. The
     // research (docs/research-biomechanics.md §6.3) puts sustained attentive
     // lean at +0.15–0.25; engage glides, and torsoLean's 0.24s tau smooths
     // the state gate, so the lean arrives and leaves like weight shifting.
-    if (st.backchannel) target.torsoLean += 0.16 * backchannel.engage;
+    if (st.engagement) target.torsoLean += 0.16 * engagement.engage;
     // Straining leans harder while there is actually a faint voice to strain
     // after. engage already tracks "the user is (barely) talking", so this
     // costs nothing; with no user signal the static pose carries the state.
-    else if (stateName === 'CANT_HEAR') target.torsoLean += 0.10 * backchannel.engage;
+    else if (stateName === 'CANT_HEAR') target.torsoLean += 0.10 * engagement.engage;
 
     // 4. mouth. The server's viseme track wins; a clip's mouth track fills the
     // gaps. There is deliberately no third leg: with no cues the mouth stays
     // shut, and a still mouth under speech is the *visible* symptom of a
-    // backend that could not align — see docs/removed.md § Amplitude lipsync.
+    // backend that could not align.
     const clipOut = clip.update(dtMs);
     let mouth = speech.sample();
     let mouthOwner = mouth ? 'speech' : null;
@@ -506,17 +522,11 @@ export function createAvatar(opts = {}) {
     // 8. smooth toward the target — this is where co-articulation happens
     for (const c of CHANNELS) cur[c] = approach(cur[c], target[c], TAU[c], dt);
 
-    face.apply(cur);
-
-    // 9. the hand, outside all of the above. It writes a transform on its own
-    //    group rather than parameter channels, so it neither smooths nor
-    //    composes — its timelines are authored as delivered motion, not as
-    //    targets to chase. `elapsed` is the mixer's own clock, so a manual
-    //    stepper gets a reproducible gesture for free.
-    if (hand) {
-      const done = hand.update(elapsed * 1000);
-      if (done) emit('gestureEnd', done.id);
-    }
+    // 9. First-class hand control. The semantic frame is generated here, above
+    // every renderer, so SVG, WebGL, and video rigs receive exactly the same
+    // gesture/progress information. A handless rig simply ignores `frame.hand`.
+    const handFrame = updateHandAction(elapsed * 1000);
+    rig.apply(avatarFrame(cur, handFrame || undefined));
   }
 
   const REST_SHAPE = shapeFor(SILENT, 1);
@@ -554,9 +564,12 @@ export function createAvatar(opts = {}) {
     // fire a stale timestamp immediately.
     glanceUntil = 0;
     glanceAt = elapsed + (st.glance ? st.glance.every[0] + Math.random() * (st.glance.every[1] - st.glance.every[0]) : 0);
-    backchannel.reset(name === 'LISTENING' ? 2.2 : 4);
-    face.svg.style.filter = st.filter || '';
-    face.svg.style.transition = 'filter .5s ease';
+    // SVG's desaturation filter is a legacy renderer detail. A generic rig
+    // receives the same state pose and may express degradation its own way.
+    if (face) {
+      face.svg.style.filter = st.filter || '';
+      face.svg.style.transition = 'filter .5s ease';
+    }
     if (changed) { idle.blink(); emit('state', name); }
     return api;
   }
@@ -578,8 +591,9 @@ export function createAvatar(opts = {}) {
    * @param {() => number} [o.clock] custom ms clock, if you drive audio yourself
    */
   function speak(o = {}) {
-    // A spoken interjection must die before real speech starts.
-    if (clip.playing && clip.clip.mouthCues) clip.stop();
+    // Speech owns the mouth in the mixer, but a server action still gets to
+    // complete its physical landing on the other channels. Do not cancel it
+    // here: a hand or head cannot disappear simply because playout began.
     speakStart = performance.now();
     speakClock = o.clock
       ? o.clock
@@ -592,15 +606,48 @@ export function createAvatar(opts = {}) {
     return api;
   }
 
+  /**
+   * "The user may be checking whether I am with them — hold their eyes."
+   *
+   * This is the widget's half of the **gaze window**. In face-to-face talk a
+   * speaker periodically looks at the listener, mutual gaze is established, the
+   * listener responds inside that window, and the speaker looks away again
+   * (Bavelas, Coates & Johnson 2002) — listener responses cluster inside the
+   * window rather than being scattered across the turn.
+   *
+   * We cannot see the user, so we cannot observe the window opening. What a
+   * caller *can* do is name the moments that co-occur with it — a mid-turn
+   * pause, a tag question ("...right?", "you know?"), a completed clause with
+   * the turn analyzer's completion probability high, the user answering a
+   * question the bot asked. `attend(ms)` is how those arrive: for its duration
+   * the face stops averting and holds the user, which is the prerequisite for
+   * any response to be *seen*. Emitting the response itself stays a separate
+   * call — a window that opens and draws nothing is a real and common outcome
+   * (with every measured invitation cue present, humans respond to only ~30% of
+   * opportunities), and conflating the two would make the avatar answer
+   * everything. The explicit response remains a backend/application decision.
+   *
+   * Deliberately **not on the wire yet**: there is no `attend` command in
+   * `packages/avatar/client/types.ts`, so today this is reachable only from JS (the demo and
+   * the rig pages). Adding the command is a protocol change and waits for a
+   * server that has something real to key it off — see docs/internal-mixer.md.
+   *
+   * @param {number} [ms=1200] how long to hold. Binetti (N=498) puts preferred
+   *        mutual gaze at 3295 ± 706 ms, so this is a fraction of the ceiling.
+   */
+  function attend(ms = 1200) {
+    attendUntil = Math.max(attendUntil, elapsed + ms / 1000);
+    return api;
+  }
+
   function pushCues(cues) { speech.push(cues); return api; }
 
   function stopSpeaking() { speech.stop(); return api; }
 
   function interject(id) {
-    const c = INTERJECTIONS[id];
+    const c = INTERNAL_CLIPS[id];
     if (!c) throw new Error(`unknown interjection: ${id}`);
     clip.play(c, c.audioEl);
-    backchannel.reset(3.5);
     return api;
   }
 
@@ -610,34 +657,70 @@ export function createAvatar(opts = {}) {
    *
    * The face half is not a convenience — a hand rising to the jaw over a head
    * and shoulders sitting perfectly still is a cut-out, not a gesture. Each
-   * entry in HAND_GESTURES names an interjection that already exists and was
-   * already tuned (`WAVE`, `THUMBS_UP`, `ONE_MOMENT`); firing it here is the
+   * entry in HAND_GESTURES names the matching semantic face action; firing it here is the
    * library composing two authored things, not the client inventing motion.
    *
-   * The two halves stay separable in both directions. `interject('WAVE')` is
-   * still the face alone and is unchanged by this — a server that upgrades gets
-   * no new behaviour until it asks for one — and on an avatar mounted with
-   * `hand: false` this call degrades to exactly that interjection, which is the
-   * same graceful failure the arm removal already forced every id through.
+   * On an avatar mounted with `hand: false` this internal helper degrades to
+   * the face action alone.
    */
   function gesture(id) {
     const def = HAND_GESTURES[id];
     if (!def) throw new Error(`unknown hand gesture: ${id}`);
     if (hand) hand.play(id, elapsed * 1000);
     if (def.face) interject(def.face);
-    // A hand in frame is a deliberate move; a backchannel landing on top of it
-    // is the listening engine talking over the server.
-    backchannel.reset(def.dur / 1000 + 0.5);
     return api;
   }
 
+  /** One self-completing server action. State continues to resolve underneath;
+   * face and hand layers queue their next movement so an in-flight physical
+   * gesture always gets to land. */
+  function action(id) {
+    const handDef = HAND_GESTURES[id];
+    if (handDef) {
+      startHandAction(id, handDef);
+      if (handDef.face) {
+        const faceClip = ACTIONS[handDef.face];
+        if (faceClip) clip.play(faceClip, faceClip.audioEl, { queue: true });
+      }
+      return api;
+    }
+    const faceClip = ACTIONS[id];
+    if (!faceClip) throw new Error(`unknown action: ${id}`);
+    clip.play(faceClip, faceClip.audioEl, { queue: true });
+    return api;
+  }
+
+  function startHandAction(id, def) {
+    const gesture = HAND_ACTION_TO_FRAME_GESTURE[id];
+    if (!gesture) return;
+    if (handAction) {
+      if (handAction.id !== id && !handQueue.some((item) => item.id === id)) {
+        handQueue.push({ id, def, gesture });
+      }
+      return;
+    }
+    handAction = { id, def, gesture, start: elapsed * 1000 };
+  }
+
+  function updateHandAction(nowMs) {
+    if (!handAction) return null;
+    const progress = (nowMs - handAction.start) / handAction.def.dur;
+    if (progress >= 1) {
+      const done = handAction;
+      const next = handQueue.shift();
+      handAction = next ? { ...next, start: nowMs } : null;
+      emit('gestureEnd', done.id);
+      return handAction ? { gesture: handAction.gesture, progress: 0, side: handSide } : null;
+    }
+    return { gesture: handAction.gesture, progress: Math.max(0, progress), side: handSide };
+  }
+
   /**
-   * Tell the listening engine whether the USER holds the floor, so backchannels
-   * become contingent on their pauses instead of running on a timer. The server
-   * owns this — it has the endpointer — and sends it as the `user` command;
-   * `null` hands back to the no-signal timer fallback.
+   * Tell the listening engine whether Pipecat VAD says the user holds the
+   * floor. This changes only sustained engagement posture; it can never create
+   * a nod or acknowledgement clip.
    */
-  function setUserSpeaking(b) { backchannel.setUserSpeaking(b); return api; }
+  function setUserSpeaking(b) { engagement.setUserSpeaking(b); return api; }
 
   // What one action does when its moment comes. Enum validity is checked here,
   // where the enums live: a bad value warns and is skipped, because one stale
@@ -650,8 +733,7 @@ export function createAvatar(opts = {}) {
       if (a.do === 'state') setState(a.name, { keepGaze: a.keepGaze !== false });
       else if (a.do === 'emotion') setEmotion(a.name, a.i ?? 1);
       else if (a.do === 'gaze') setGaze(a.name);
-      else if (a.do === 'interject') interject(a.id);
-      else if (a.do === 'gesture') gesture(a.id);
+      else if (a.do === 'action') action(a.id);
     } catch (e) {
       console.warn(`perform: ${a.do} at ${a.t}ms skipped — ${e.message}`);
     }
@@ -661,8 +743,8 @@ export function createAvatar(opts = {}) {
 
   /**
    * Play a timed action track — the composition surface a server assembles
-   * turns from. Verbs: state / emotion / gaze / interject / gesture (see
-   * perform.js for hygiene, docs/contract-protocol.md for the schema).
+   * turns from. Verbs: state / emotion / gaze / action (see
+   * perform.js for hygiene, docs/internal-mixer.md for the schema).
    *
    * Clock resolution mirrors speak(): explicit `clock` fn, else the audio
    * element's own time, else ms elapsed since this call. perform() never
@@ -691,12 +773,12 @@ export function createAvatar(opts = {}) {
   }
 
   const api = {
-    setState, setEmotion, setGaze, speak, pushCues, stopSpeaking, interject,
-    gesture, perform,
+    setState, setEmotion, setGaze, speak, pushCues, stopSpeaking, attend,
+    action, perform,
     /** Which hand the character gestures with: +1 the viewer's right (its own
      *  left), -1 the other. Both are anatomically real — the thumb splays away
      *  from the body either way — so this is a character choice, not a fix. */
-    setHandSide: (d) => { if (hand) hand.setDir(d); return api; },
+    setHandSide: (d) => { handSide = d === -1 ? 'left' : 'right'; return api; },
     setUserSpeaking,
     /** Articulation gain: 1 is the VISEME_SHAPES table as authored. */
     setMouthGain: (g) => { mouthGain = g; return api; },
@@ -720,21 +802,22 @@ export function createAvatar(opts = {}) {
     get speaking() { return speech.playing; },
     get performing() { return performTrack.playing; },
     get clip() { return clip.id; },
-    /** The hand gesture in flight, or null. `null` forever if `hand: false`. */
-    get gesturing() { return hand ? hand.id : null; },
+    /** Semantic hand gesture in flight, independent of renderer capability. */
+    get gesturing() { return handAction ? handAction.id : null; },
     get params() { return cur; },
-    get userSpeaking() { return backchannel.speaking; },
-    svg: face.svg,
-    meta,
+    get userSpeaking() { return engagement.speaking; },
+    // Legacy SVG inspection fields. New AvatarRig implementations should not
+    // rely on or provide them; the renderer-neutral contract is apply/destroy.
+    svg: face?.svg || null,
+    meta: meta || null,
     /** The mounted rig's palette, merged with any `opts.theme` overrides. A
      *  host that has to paint anything *around* the widget — a tile margin, a
      *  page behind a transparent mount — needs the same colours the drawing
      *  used, and guessing them per avatar is how the two drift apart. */
-    theme: face.theme,
+    theme: face?.theme,
     destroy() {
       cancelAnimationFrame(raf);
-      if (hand) hand.destroy();
-      face.destroy();
+      rig.destroy();
     },
   };
 
@@ -743,13 +826,18 @@ export function createAvatar(opts = {}) {
   return api;
 }
 
-export { INTERJECTIONS, INTERJECTION_IDS, SPOKEN_IDS, attachAudio } from './interjections.js';
+export { ACTION_IDS, ACTIONS, attachAudio } from './interjections.js';
 export { GAZE_NAMES, GAZE_TARGETS } from './gaze.js';
 export { normalizeActions } from './perform.js';
-export { HAND_GESTURES, HAND_GESTURE_IDS, checkHandFraming } from './hand.js';
+export { checkHandFraming } from './hand.js';
 export { EMOTION_NAMES } from './emotions.js';
+// The mouth clock travels with the rest of it. Someone has to turn a cue array
+// plus a clock into "which letter is on screen right now", every renderer needs
+// exactly that, and none of them should write it twice — so it is a plain class
+// to construct, not a contract to implement.
 export {
-  VISEME_LETTERS, VISEME_SHAPES, normalizeCues, textToCues,
+  VISEME_LETTERS, VISEME_SHAPES, VisemeTrack, shapeFor, SILENT,
+  normalizeCues, textToCues,
   ARPABET_TO_VISEME, AZURE_VISEME_TO_LETTER, LEAD_MS,
 } from './visemes.js';
 // No THEME re-export: each face module owns its palette, and `api.theme` is