@voqalize/avatar 0.4.4 → 0.5.1

package/src/behavior.d.ts

@@ -1,42 +0,0 @@
-import type { AvatarStateName } from "./avatar.js";
-
-/** The two renderer calls behavior resolution actually needs. */
-export interface BehaviorAvatar {
-  setState(name: AvatarStateName): unknown;
-  /**
-   * One motion by name. The vocabulary is open — two core intents plus
-   * whatever this avatar publishes — so a name it does not have must be a
-   * no-op rather than a throw.
-   */
-  action(id: string): unknown;
-}
-
-/**
- * The whole behaviour vocabulary — the nine. Not an alias list; see behavior.js.
- *
- * Every one of these is also an `AvatarStateName`, which is why there is no
- * mapping table any more.
- */
-export type BehaviorStateId =
-  | "IDLE" | "LISTENING" | "CANT_HEAR" | "THINKING" | "WORKING"
-  | "MUTED" | "SPEAKING" | "DEGRADED" | "OFFLINE";
-
-export type BehaviorActionId = "ack" | "turn.interrupted";
-
-export const BEHAVIOR_STATE_IDS: readonly BehaviorStateId[];
-export const BEHAVIOR_ACTIONS: Readonly<Record<BehaviorActionId, { renderAction: string }>>;
-export const BEHAVIOR_ACTION_IDS: readonly BehaviorActionId[];
-export const WIRE_ACTION_TO_BEHAVIOR: Readonly<Record<string, BehaviorActionId>>;
-
-export class BehaviorController {
-  constructor(avatar: BehaviorAvatar);
-  state: BehaviorStateId | null;
-  setState(id: BehaviorStateId, options?: { force?: boolean }): this;
-  action(id: BehaviorActionId): this;
-  /**
-   * One id off the wire: a core intent through the catalog, anything else
-   * straight to the renderer. Never throws — the vocabulary is open.
-   */
-  wireAction(id: string): this;
-  destroy(): void;
-}

package/src/behavior.js

@@ -1,114 +0,0 @@
-/**
- * Behavior catalog and controller.
- *
- * This is the layer between factual/wire inputs and the bundled SVG renderer.
- * It is not a seam any avatar implementation sees: an implementation receives
- * a PipecatClient and decides for itself what a state means.
- *
- * The vocabulary is the nine core states and two core actions. The states used
- * to be seventeen — the core
- * set plus every SVG render state (TYPING, SEARCHING_SCREEN, WANTS_IN, …)
- * passed straight through, so that tooling could drive one of *those* from
- * here. That made the mixer's private state list look like part of the
- * behaviour vocabulary. Tooling that wants a render state calls
- * `avatar.setState` on the mixer, which is whose state it is.
- */
-
-/**
- * The durable states this layer resolves. Each one *is* the render state it
- * asks the bundled SVG mixer for, so there is nothing here to map.
- *
- * There used to be a second column — a per-state `renderState`, so a renderer
- * could draw `WORKING` as anything it liked and the divergence would land here
- * rather than leak. In nine rows it had exactly one non-identity entry,
- * `STRAINING -> CANT_HEAR`, and the wire redesign deleted that entry by
- * renaming the state: a state names the bot's *situation*, never how it looks,
- * and "straining" was a situation named after one drawing of it. A table whose
- * every row reads `X: X` documents nothing and invites a renderer to plug into
- * it, so it is a list now. A renderer that genuinely wants to draw one of these
- * as something else calls `avatar.setState` with the render state it wants,
- * which is whose state that is.
- */
-export const BEHAVIOR_STATE_IDS = Object.freeze([
-  'IDLE', 'LISTENING', 'CANT_HEAR', 'THINKING', 'WORKING',
-  'MUTED', 'SPEAKING', 'DEGRADED', 'OFFLINE',
-]);
-
-const STATE_IDS = new Set(BEHAVIOR_STATE_IDS);
-
-/**
- * Library action IDs: the two intents every renderer owes a server, in this
- * layer's readable spelling.
- *
- * It used to be seven, with a `sequence` command alongside for a renderer's own
- * motions. The seven were not the wrong *names* so much as the wrong idea: four
- * of them were `GESTURE_*` — a greet, a goodbye, a wave — which are things a
- * particular body does and not intents a server can hold every face to, and two
- * were the receipt and the nod, which are two *shapes* of acknowledging. So the
- * server now says only that an acknowledgement is due, the avatar picks which
- * one it makes, and the gestures live in whichever catalogue can draw them,
- * reachable by the same open `action` id a sequence used to need its own
- * command for.
- */
-export const BEHAVIOR_ACTIONS = Object.freeze({
-  ack: { renderAction: 'ACKNOWLEDGE' },
-  'turn.interrupted': { renderAction: 'RESPONSE_INTERRUPTED' },
-});
-
-export const BEHAVIOR_ACTION_IDS = Object.freeze(Object.keys(BEHAVIOR_ACTIONS));
-
-/** The two core wire names map into the behavior catalog here and nowhere else. */
-export const WIRE_ACTION_TO_BEHAVIOR = Object.freeze({
-  ACKNOWLEDGE: 'ack',
-  RESPONSE_INTERRUPTED: 'turn.interrupted',
-});
-
-/**
- * Turns behavior intent into the legacy SVG mixer's calls. A future renderer
- * receives the same intent through its own adapter; it does not need to know
- * the legacy state/action names used below.
- */
-export class BehaviorController {
-  constructor(avatar) {
-    if (!avatar || typeof avatar.setState !== 'function' || typeof avatar.action !== 'function') {
-      throw new TypeError('BehaviorController requires an avatar implementation');
-    }
-    this.avatar = avatar;
-    this.state = null;
-  }
-
-  setState(id, { force = false } = {}) {
-    if (!STATE_IDS.has(id)) throw new Error(`unknown behavior state: ${id}`);
-    if (!force && this.state === id) return this;
-    this.state = id;
-    this.avatar.setState(id);
-    return this;
-  }
-
-  destroy() {}
-
-  action(id) {
-    const def = BEHAVIOR_ACTIONS[id];
-    if (!def) throw new Error(`unknown behavior action: ${id}`);
-    this.avatar.action(def.renderAction);
-    return this;
-  }
-
-  /**
-   * One id off the wire. A core one goes through the behavior catalog; any
-   * other goes straight to the renderer, and that asymmetry is the point.
-   * `BEHAVIOR_ACTIONS` exists so a portable intent has a readable name
-   * independent of how any one face renders it. An avatar's own id has no such
-   * independence — it *is* the rendering, named by the renderer that owns it —
-   * so giving it a behavior alias would claim a portability it does not have.
-   *
-   * Neither path throws. The wire's action vocabulary is open and an id this
-   * face cannot draw is the expected case, not somebody's broken build.
-   */
-  wireAction(id) {
-    const behaviorId = WIRE_ACTION_TO_BEHAVIOR[id];
-    if (behaviorId) return this.action(behaviorId);
-    this.avatar.action(id);
-    return this;
-  }
-}

package/src/camera.js

@@ -1,29 +0,0 @@
-/**
- * The close webcam crop shared by every avatar renderer.
- *
- * The head gets most of the tile because lip sync and listening expressions
- * are the information the avatar is there to carry. The remaining lower band
- * leaves enough neck and shoulder for posture to read without turning the
- * call tile into a half-body portrait.
- */
-export const CALL_CAMERA = Object.freeze({
-  aspect: 4 / 3,
-  headroom: 0.06,
-  head: 0.70,
-  body: 0.24,
-});
-
-/** Derive a camera rectangle from three landmarks in the drawing's own units. */
-export function viewBoxForHead({ centerX, crownY, chinY }) {
-  if (![centerX, crownY, chinY].every(Number.isFinite) || chinY <= crownY) {
-    throw new TypeError('viewBoxForHead: expected a finite centre and crown above chin');
-  }
-  const h = (chinY - crownY) / CALL_CAMERA.head;
-  const w = h * CALL_CAMERA.aspect;
-  return {
-    x: centerX - w / 2,
-    y: crownY - h * CALL_CAMERA.headroom,
-    w,
-    h,
-  };
-}

package/src/clips.js

@@ -1,155 +0,0 @@
-/**
- * Gesture clip player.
- *
- * A clip is a short, multi-channel keyframed timeline. Clip channels are
- * *additive deltas*, not absolute poses — a nod adds pitch to whatever the head
- * is already doing, which is what makes "nod while speaking" work without any
- * special-casing.
- *
- * A server action is a physical movement, not an abstract state transition.
- * Queued actions finish their landing before the next action begins; their
- * underlying state may change freely while that happens. `stop()` remains for
- * real speech taking mouth ownership.
- */
-
-import { clamp } from './params.js';
-import { INTERJECTION_TRACK_TAIL_MS } from './speech-timing.js';
-import { VisemeTrack } from './visemes.js';
-
-const smoothstep = (t) => t * t * (3 - 2 * t);
-
-function sampleTrack(keys, u) {
-  if (!keys.length) return 0;
-  if (u <= keys[0][0]) return keys[0][1];
-  const last = keys[keys.length - 1];
-  if (u >= last[0]) return last[1];
-  for (let i = 1; i < keys.length; i++) {
-    if (u <= keys[i][0]) {
-      const [t0, v0] = keys[i - 1];
-      const [t1, v1] = keys[i];
-      const k = t1 === t0 ? 1 : smoothstep((u - t0) / (t1 - t0));
-      return v0 + (v1 - v0) * k;
-    }
-  }
-  return last[1];
-}
-
-const RAMP_IN = 70;
-const RAMP_OUT = 150;
-
-export class ClipPlayer {
-  /**
-   * @param {object} hooks
-   * @param {(name:string|null)=>void} hooks.onGaze  clip-scoped gaze override
-   * @param {()=>void}                 hooks.onBlink
-   */
-  constructor(hooks = {}) {
-    this.hooks = hooks;
-    this.clip = null;
-    this.t = 0;
-    this.fading = false;
-    this.fadeT = 0;
-    this.mouth = new VisemeTrack();
-    this._blinksDone = 0;
-    this.onEnd = null;
-    this.queue = [];
-  }
-
-  get playing() { return !!this.clip; }
-  get id() { return this.clip ? this.clip.id : null; }
-
-  /**
-   * @param {object} clip
-   * @param {HTMLAudioElement} [audio] if given, the clip's mouth track is
-   *        scheduled against the audio clock instead of the local timer
-   */
-  play(clip, audio, { queue = false } = {}) {
-    if (!clip) return;
-    if (this.clip && queue) {
-      if (this.clip.id === clip.id || this.queue.some((item) => item.clip.id === clip.id)) return;
-      this.queue.push({ clip, audio });
-      return;
-    }
-    this._start(clip, audio);
-  }
-
-  _start(clip, audio) {
-    this.clip = clip;
-    this.t = 0;
-    this.fading = false;
-    this.fadeT = 0;
-    this._blinksDone = 0;
-    this.audio = audio || null;
-
-    if (clip.mouthCues && clip.mouthCues.length) {
-      const clock = audio
-        ? () => audio.currentTime * 1000
-        : () => this.t;
-      this.mouth.tailMs = INTERJECTION_TRACK_TAIL_MS;
-      this.mouth.start(clip.mouthCues, clock);
-    } else {
-      this.mouth.stop();
-    }
-    if (clip.gaze && this.hooks.onGaze) this.hooks.onGaze(clip.gaze);
-    if (audio) { audio.currentTime = 0; audio.play().catch(() => {}); }
-  }
-
-  /** Barge-in. Fades rather than cutting, so the head doesn't snap. */
-  stop(immediate = false) {
-    if (!this.clip) return;
-    if (immediate) return this._end();
-    this.fading = true;
-    this.fadeT = 0;
-  }
-
-  _end() {
-    const had = this.clip;
-    this.clip = null;
-    this.fading = false;
-    this.mouth.stop();
-    if (this.audio) { this.audio.pause(); this.audio = null; }
-    if (had && had.gaze && this.hooks.onGaze) this.hooks.onGaze(null);
-    if (this.onEnd) this.onEnd(had);
-    const next = this.queue.shift();
-    if (next) this._start(next.clip, next.audio);
-  }
-
-  /**
-   * @returns {{delta: object, weight: number, mouth: object|null, ownsMouth: boolean}}
-   */
-  update(dtMs) {
-    if (!this.clip) return { delta: null, weight: 0, mouth: null, ownsMouth: false };
-    this.t += dtMs;
-    const c = this.clip;
-    const u = clamp(this.t / c.duration);
-
-    let w = Math.min(1, this.t / RAMP_IN) * Math.min(1, (c.duration - this.t) / RAMP_OUT);
-    if (this.fading) {
-      this.fadeT += dtMs;
-      w *= Math.max(0, 1 - this.fadeT / 110);
-      if (this.fadeT >= 110) { this._end(); return { delta: null, weight: 0, mouth: null, ownsMouth: false }; }
-    }
-    w = clamp(w);
-
-    if (c.blinkAt && this.hooks.onBlink) {
-      while (this._blinksDone < c.blinkAt.length && u >= c.blinkAt[this._blinksDone]) {
-        this._blinksDone++;
-        this.hooks.onBlink();
-      }
-    }
-
-    const delta = {};
-    for (const ch in c.keys) delta[ch] = sampleTrack(c.keys[ch], u) * w;
-
-    let mouth = null;
-    const ownsMouth = !!(c.mouthCues && c.mouthCues.length);
-    if (ownsMouth) mouth = this.mouth.sample();
-
-    if (this.t >= c.duration && !this.fading) {
-      // Let the mouth track finish its tail before tearing down.
-      if (!ownsMouth || !this.mouth.playing) this._end();
-    }
-
-    return { delta, weight: w, mouth, ownsMouth };
-  }
-}

package/src/conformance.js

@@ -1,119 +0,0 @@
-/**
- * The rig conformance sweep — the assertions, with no page around them.
- *
- * Drives every state, emotion, gaze and semantic action through every rig you
- * hand it, then a viseme track, and asserts the mixer keeps producing finite,
- * in-range parameters and a still-attached drawing. A NaN anywhere in the chain
- * shows up as a *frozen face*, which is exactly the defect the eye is worst at
- * catching: a still avatar looks calm, not broken.
- *
- * This is the numeric half of judging a rig, and it is the smaller half —
- * passing here says a change did not break the mixer, never that it looks good
- * (CLAUDE.md § Verifying). It is also what a face author runs against a new
- * drawing before anyone is asked to look at it.
- *
- * `advance` is the seam that lets one copy of the sweep serve both callers:
- * a rig page in the working tree lets real time pass and watches it happen, while
- * `packages/avatar/test/conformance.test.ts` steps `{manual: true}` avatars by a fixed
- * dt and finishes in milliseconds. The sweep's own clock is handed to `speak`
- * so the mouth articulates under both — a stepped avatar reading
- * `performance.now()` would sit on the first cue for the whole track.
- */
-import {
-  STATE_NAMES,
-  EMOTION_NAMES,
-  GAZE_NAMES,
-  ACTION_IDS,
-  checkHandFraming,
-} from './avatar.js';
-import { textToCues } from './visemes.js';
-
-/** Parameters live in −1..1 by contract; 2 is the slack a gain may legally add. */
-const PARAM_LIMIT = 2;
-
-const realTime = (s) => new Promise((r) => setTimeout(r, s * 1000));
-
-/**
- * @param {Array<{name: string, avatar: object}>} rigs one entry per drawing
- * @param {{advance?: (seconds: number) => void|Promise<void>}} [opts]
- * @returns {Promise<{ok: boolean, problems: string[], summary: string}>}
- */
-export async function conformanceSweep(rigs, { advance = realTime } = {}) {
-  const problems = [];
-  let ms = 0;
-  const clock = () => ms;
-  const tick = async (seconds) => { ms += seconds * 1000; await advance(seconds); };
-  const all = (fn) => rigs.forEach((r) => fn(r.avatar, r.name));
-
-  function check(label) {
-    for (const r of rigs) {
-      const p = r.avatar.params;
-      for (const k in p) {
-        if (!Number.isFinite(p[k])) problems.push(`${r.name} ${label}: ${k}=${p[k]}`);
-        else if (Math.abs(p[k]) > PARAM_LIMIT) {
-          problems.push(`${r.name} ${label}: ${k} out of range (${p[k].toFixed(2)})`);
-        }
-      }
-      // A renderer-neutral rig has no svg to detach; the SVG one must stay put.
-      if (r.avatar.svg && !r.avatar.svg.isConnected) problems.push(`${r.name} ${label}: svg detached`);
-    }
-  }
-
-  for (const s of STATE_NAMES)   { all((k) => k.setState(s));      await tick(0.09); check(`state ${s}`); }
-  for (const e of EMOTION_NAMES) { all((k) => k.setEmotion(e, 1)); await tick(0.09); check(`emotion ${e}`); }
-  for (const g of GAZE_NAMES)    { all((k) => k.setGaze(g));       await tick(0.09); check(`gaze ${g}`); }
-  for (const i of ACTION_IDS)    { all((k) => k.action(i));        await tick(0.18); check(`action ${i}`); }
-
-  // The hand. Two things are being proved and only one of them is the mixer's.
-  // Playing each gesture above checked the usual invariants — its face half runs
-  // through the clip layer like any other interjection. checkHandFraming is the
-  // one that matters per AVATAR: the hand places itself off META.viewBox, so a
-  // rig with a different window could push ink through the camera edge or lift
-  // the wrist into frame, and neither is visible in a resting still.
-  for (const r of rigs) {
-    if (!r.avatar.meta) continue;
-    try { checkHandFraming(r.avatar.meta); } catch (e) { problems.push(`${r.name} ${e.message}`); }
-  }
-  all((k) => k.setState('LISTENING'));
-  await tick(2);   // let the longest gesture leave frame before speech starts
-
-  const cues = textToCues('okay, and what happened next');
-  all((k) => k.speak({ cues, clock }));
-  for (let i = 0; i < 12; i++) { await tick(0.12); check('speaking'); }
-  all((k) => k.stopSpeaking().setState('LISTENING'));
-  await tick(0.2); check('after speech');
-
-  return {
-    ok: problems.length === 0,
-    problems,
-    summary: `${rigs.length} rigs x ${STATE_NAMES.length} states, ${EMOTION_NAMES.length} emotions, `
-      + `${GAZE_NAMES.length} gazes, ${ACTION_IDS.length} semantic actions `
-      + `(framing asserted per avatar), plus a viseme track`,
-  };
-}
-
-/**
- * A deterministic `advance` for `{manual: true}` avatars: fixed-dt stepping, no
- * wall clock, no rAF. Same rigs and same seed give the same numbers every run,
- * which is what makes a failure here reproducible rather than anecdotal.
- */
-export function stepper(rigs, dt = 1 / 60) {
-  return (seconds) => {
-    for (let t = 0; t < seconds; t += dt) rigs.forEach((r) => r.avatar.step(dt));
-  };
-}
-
-/**
- * Replace `Math.random` with a seeded LCG and return the undo.
- *
- * The sweep's assertions are invariants — they must hold for any draw — but a
- * failure you cannot reproduce is a failure you cannot fix, and idle, gaze
- * aversion and blink timing are all random. Call this *before* creating the
- * avatars: constructors draw their first phases immediately.
- */
-export function seedRandom(seed = 1) {
-  const original = Math.random;
-  let s = seed >>> 0 || 1;
-  Math.random = () => ((s = (s * 1664525 + 1013904223) >>> 0) / 4294967296);
-  return () => { Math.random = original; };
-}

package/src/emotions.js

@@ -1,58 +0,0 @@
-/**
- * Affect is a separate axis from state.
- *
- * If emotion were folded into the state enum you'd need SPEAKING_WARM,
- * SPEAKING_CONCERNED, LISTENING_WARM ... and the table would be unmaintainable
- * within a week. Instead: `setState('LISTENING', { emotion: 'curious' })`. The
- * emotion contributes a base pose that every other layer builds on top of.
- */
-
-export const EMOTIONS = {
-  // Deliberately empty: REST is genuine availability, rather than a social
-  // smile baked into every state. Positive affect belongs to a response,
-  // greeting, success beat, or explicit warm/encouraging emotion.
-  neutral: {},
-  warm: {
-    mouthCornerL: 0.48, mouthCornerR: 0.48,
-    // A real smile squints. Without this it reads as a mask. From below —
-    // the cheek pushes the lower lid up (AU6) — with no upper lid dropped on
-    // top, which on a photographic eye read as sleepy rather than warm.
-    squintL: 0.30, squintR: 0.30,
-    browRaiseL: 0.10, browRaiseR: 0.10,
-  },
-  curious: {
-    browRaiseL: 0.34, browRaiseR: 0.12,
-    browAngleL: 0.10,
-    headRoll: 0.07,
-    mouthCornerL: 0.18, mouthCornerR: 0.14,
-    lidL: -0.10, lidR: -0.10,
-  },
-  concerned: {
-    browInnerL: 0.55, browInnerR: 0.55,
-    browRaiseL: -0.08, browRaiseR: -0.08,
-    mouthCornerL: -0.22, mouthCornerR: -0.22,
-    lidL: 0.06, lidR: 0.06,
-  },
-  encouraging: {
-    mouthCornerL: 0.58, mouthCornerR: 0.58,
-    browRaiseL: 0.26, browRaiseR: 0.26,
-    squintL: 0.22, squintR: 0.22,
-    headPitch: 0.05,
-  },
-  thoughtful: {
-    browRaiseL: -0.14, browRaiseR: -0.06,
-    browInnerL: 0.18, browInnerR: 0.10,
-    mouthPress: 0.45, mouthCornerL: -0.05, mouthCornerR: 0.02,
-    lidL: 0.10, lidR: 0.10,
-  },
-};
-
-export const EMOTION_NAMES = Object.keys(EMOTIONS);
-
-/** Blend an emotion toward neutral by `intensity`. */
-export function emotionPose(name, intensity = 1) {
-  const src = EMOTIONS[name] || EMOTIONS.neutral;
-  const out = {};
-  for (const k in src) out[k] = src[k] * intensity;
-  return out;
-}