@voqalize/avatar 0.3.0 → 0.4.0

package/client/internal.ts

@@ -15,7 +15,7 @@
  *     const track = new VisemeTrack();
  *     track.start(cues, () => performance.now() - t0);
  *     // per frame:
- *     const s = track.sample();   // { letter: "D", intensity: 1 } | null
+ *     const s = track.sample();   // { letter: "D", intensity: 1, phone: "OW" } | null
  *
  * `docs/internal-rig.md` describes the pose-channel model the bundled SVG
  * renderer uses internally. It is *not* the seam to implement — see
@@ -27,6 +27,11 @@ export {
   // all three in every bundle that wanted the viseme clock.
   // `@voqalize/avatar/faces/<name>` is where a face comes from.
   createAvatar as createSvgAvatar,
+  // The pose space a custom `rig` is handed on every frame, and the rests it is
+  // measured against.
+  REST,
+  CHANNELS,
+  RANGE,
   STATES,
   STATE_NAMES,
   ACTIONS,
@@ -40,6 +45,8 @@ export {
   VISEME_SHAPES,
   SILENT,
   LEAD_MS,
+  // The body's share of a held tilt, for a page driving a rig by hand.
+  SHOULDER_TILT,
   shapeFor,
   normalizeCues,
   textToCues,
@@ -62,8 +69,18 @@ export type {
   CreateAvatarOptions as CreateSvgAvatarOptions,
 } from "../src/avatar.js";
 
+// The rig contract, for a renderer that implements `apply(frame)` directly
+// instead of wearing the SVG adapter. `docs/internal-rig.md` — and note that
+// this is the mixer's private seam, not the avatar interface.
+export type {
+  AvatarFrame,
+  AvatarRig,
+  AvatarRigFactory,
+  HandFrame,
+  RigPose,
+} from "../src/rig.js";
+
 export {
-  BEHAVIOR_STATES,
   BEHAVIOR_STATE_IDS,
   BEHAVIOR_ACTIONS,
   BEHAVIOR_ACTION_IDS,
@@ -72,9 +89,20 @@ export {
 
 export type { BehaviorStateId, BehaviorActionId } from "../src/behavior.js";
 
-export { isAvatarMessage } from "./types.js";
+export { isAvatarMessage, CORE_ACTION_IDS } from "./types.js";
+// The optional driving-UI declaration, as a type only. The bundled avatar's own
+// `supports` value lives on the public entry point and importing it here would
+// drag `peep`'s drawing in behind it — the same reason the faces are not
+// exported from this module. Compose yours from `CORE_ACTION_IDS` and
+// `ACTION_IDS` above, which cost nothing.
+export type { AvatarSupport } from "./createAvatar.js";
+// A renderer may reuse the one lifecycle/precedence ladder without gaining a
+// second public avatar interface.
+export { AvatarClient } from "./AvatarClient.js";
 export type {
   AvatarCommand,
   AvatarCue,
   AvatarCuesCmd,
+  CoreActionId,
 } from "./types.js";
+export type { AvatarClientOptions, AvatarDriver, AvatarPresenceState } from "./AvatarClient.js";

package/client/ishita.ts

@@ -13,6 +13,8 @@ const IMAGES = {
 const FACE = () => import('../src/canvas/avatars/round/face.mjs');
 
 export type { CanvasAvatarOptions };
+// Same mixer, same clip library, so the same declaration (`supports.ts`).
+export { supports } from './supports.js';
 
 export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
   return createCanvasAvatar(options, {

package/client/kabir.ts

@@ -13,6 +13,8 @@ const IMAGES = {
 const FACE = () => import('../src/canvas/avatars/round/face.mjs');
 
 export type { CanvasAvatarOptions };
+// Same mixer, same clip library, so the same declaration (`supports.ts`).
+export { supports } from './supports.js';
 
 export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
   return createCanvasAvatar(options, {

package/client/meera.ts

@@ -13,6 +13,8 @@ const IMAGES = {
 const FACE = () => import('../src/canvas/avatars/round/face.mjs');
 
 export type { CanvasAvatarOptions };
+// Same mixer, same clip library, so the same declaration (`supports.ts`).
+export { supports } from './supports.js';
 
 export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
   return createCanvasAvatar(options, {

package/client/naina.ts

@@ -13,6 +13,8 @@ const IMAGES = {
 const FACE = () => import('../src/canvas/avatars/round/face.mjs');
 
 export type { CanvasAvatarOptions };
+// Same mixer, same clip library, so the same declaration (`supports.ts`).
+export { supports } from './supports.js';
 
 export function createAvatar(options: CanvasAvatarOptions): AvatarInstance {
   return createCanvasAvatar(options, {

package/client/playout.ts

@@ -0,0 +1,95 @@
+/**
+ * When the bot's audio actually begins, heard on the track the browser plays.
+ *
+ * `BotStartedSpeaking` and the audio reach the browser by different roads —
+ * the event on the data channel, the sound over RTP and through a jitter
+ * buffer — so the event is a statement about *which* audio is starting and
+ * not about when anyone hears it. Measured on two stacks, the event arrived
+ * 57 ms after the first sample on one and 160 ms before it on the other: the
+ * mouth lagged the voice on the first and ran ahead of it on the second, and
+ * no constant corrects both. So the turn clock is anchored to the sound.
+ *
+ * What makes "the sound began" observable is that between turns the decoded
+ * track is exact digital zero — Opus decodes silence to silence — so onset is
+ * the first sample off zero, not a speech detector's guess at a threshold. That
+ * matters: cue `t: 0` is the first sample the TTS produced, lead-in included,
+ * and a detector keyed to *speech* would anchor 20-50 ms into the lead-in.
+ *
+ * What this probe hears is the sound entering the page's audio graph, not
+ * leaving the speaker. For a wired speaker the difference is about what the
+ * compositor and display add to the picture, and the two cancel. A Bluetooth
+ * headset adds 150-250 ms that nothing on the picture side matches, so the
+ * probe also reports the device latency the browser admits to, and the
+ * client delays the mouth by the excess (`AvatarClient.ts`).
+ */
+
+export interface PlayoutProbe {
+  /**
+   * Where the bot's current sound began, on the caller's `now()` clock.
+   * `null`: the track is silent right now. `undefined`: this probe cannot say —
+   * the track has been sounding for longer than it can look back, or the
+   * audio graph is not running.
+   */
+  onset(): number | null | undefined;
+  /** The output device's latency as this context reports it, in ms; 0 when it
+   * will not say. The probe shares the device the call's audio plays on. */
+  outputLatencyMs?(): number;
+  /** Ask a suspended graph to run. A browser that needs a gesture for it
+   * ignores the request outside one, so this is safe to call at any time. */
+  resume?(): void;
+  dispose(): void;
+}
+
+/** 8192 samples is 170 ms at 48 kHz: past the 57 ms the sound has been seen to
+ * lead the event by, with room for a slower network. */
+const WINDOW = 8192;
+/** -80 dBFS. The gaps between turns are exact zero; this only keeps a
+ * denormal from reading as speech. */
+const FLOOR = 1e-4;
+
+export function createPlayoutProbe(track: MediaStreamTrack, now: () => number): PlayoutProbe | null {
+  const Ctx = globalThis.AudioContext;
+  if (!Ctx) return null;
+  let ctx: AudioContext;
+  let source: MediaStreamAudioSourceNode;
+  let analyser: AnalyserNode;
+  try {
+    ctx = new Ctx();
+    source = ctx.createMediaStreamSource(new MediaStream([track]));
+    analyser = ctx.createAnalyser();
+    analyser.fftSize = WINDOW;
+    source.connect(analyser);
+  } catch {
+    return null;
+  }
+  // A context made after the user's connect click is allowed to run; one that
+  // is not simply answers `undefined`, and the event anchors the turn.
+  ctx.resume().catch(() => {});
+  const buf = new Float32Array(WINDOW);
+
+  return {
+    onset() {
+      if (ctx.state !== "running") return undefined;
+      analyser.getFloatTimeDomainData(buf);
+      const t = now();
+      let first = -1;
+      for (let i = 0; i < buf.length; i++) {
+        if (Math.abs(buf[i]!) > FLOOR) { first = i; break; }
+      }
+      if (first < 0) return null;
+      if (first === 0) return undefined;
+      return t - ((buf.length - first) / ctx.sampleRate) * 1000;
+    },
+    outputLatencyMs() {
+      const s = ctx.outputLatency;
+      return typeof s === "number" && Number.isFinite(s) ? s * 1000 : 0;
+    },
+    resume() {
+      if (ctx.state === "suspended") ctx.resume().catch(() => {});
+    },
+    dispose() {
+      source.disconnect();
+      ctx.close().catch(() => {});
+    },
+  };
+}

package/client/supports.ts

@@ -0,0 +1,28 @@
+/**
+ * `supports` for every avatar this package ships — the SVG faces and the six
+ * canvas identities alike.
+ *
+ * It is one constant and one file because they are one renderer: the canvas
+ * avatars mount the same mixer with the same clip library, so a second list
+ * would be the same list with a second chance to be wrong. It is not in
+ * `createAvatar.ts`, where the type is documented, because a value is imported
+ * for real — reading it from there would pull `peep`'s drawing into a canvas
+ * avatar's bundle.
+ *
+ * The two required ids first, then this renderer's own catalogue. `ACTION_IDS`
+ * already carries `RESPONSE_INTERRUPTED` as this renderer's shape for it, so
+ * the join dedupes rather than concatenating, and `ACKNOWLEDGE` is the one name
+ * here with no clip behind it because `action()` resolves it on the floor
+ * (docs/internal-mixer.md § Actions).
+ */
+
+import { ACTION_IDS } from "../src/avatar.js";
+import type { AvatarSupport } from "./createAvatar.js";
+import { CORE_ACTION_IDS } from "./types.js";
+
+export const supports: AvatarSupport = Object.freeze({
+  actions: Object.freeze([
+    ...CORE_ACTION_IDS,
+    ...ACTION_IDS.filter((id) => !(CORE_ACTION_IDS as readonly string[]).includes(id)),
+  ]),
+});

package/client/three/assets.ts

@@ -0,0 +1,24 @@
+/**
+ * Where the three compiled characters are, resolved against this module's own
+ * location.
+ *
+ * `new URL(…, import.meta.url)` and not a bundler's asset import: these files
+ * are fetched at runtime by a consumer's app, and the spelling has to survive
+ * Vite, webpack, Rollup, esbuild, a plain `tsc` output and a browser loading
+ * the module directly. The query-suffix form this used to carry is Vite syntax —
+ * every other toolchain, including the compiler that now builds this directory,
+ * passes it through verbatim and produces an import of a file that is not there.
+ *
+ * The depth is the same from the source tree and from the compiled one
+ * (`client/three/` and `dist/three/` are siblings), so one literal is correct in
+ * both — and literal is load-bearing, because a bundler can only follow this
+ * pattern when it can read the path without running anything.
+ */
+export const ASSETS = Object.freeze({
+  tara: new URL("../../assets/tara.glb", import.meta.url).href,
+  tushar: new URL("../../assets/tushar.glb", import.meta.url).href,
+  tanya: new URL("../../assets/tanya.glb", import.meta.url).href,
+});
+
+/** Every compiled character, in build order. */
+export type CharacterName = keyof typeof ASSETS;

package/client/three/budgets.ts

@@ -0,0 +1,27 @@
+/** Measurable limits for one 3-D avatar at its shipping surface. */
+export const SHIPPING_SURFACE = Object.freeze({
+  cssWidth: 400,
+  cssHeight: 300,
+  maxDevicePixelRatio: 2,
+});
+
+/** Keep the physical drawing buffer inside the shipping ceiling at any mount size. */
+export function pixelRatioFor(width: number, height: number, devicePixelRatio: number): number {
+  const maxWidth = SHIPPING_SURFACE.cssWidth * SHIPPING_SURFACE.maxDevicePixelRatio;
+  const maxHeight = SHIPPING_SURFACE.cssHeight * SHIPPING_SURFACE.maxDevicePixelRatio;
+  return Math.max(0.1, Math.min(
+    SHIPPING_SURFACE.maxDevicePixelRatio,
+    devicePixelRatio,
+    maxWidth / Math.max(1, width),
+    maxHeight / Math.max(1, height),
+  ));
+}
+
+/** Crossing one of these is a failed asset/runtime review, not a suggestion. */
+export const HARD_BUDGET = Object.freeze({
+  assetBytes: 6 * 1024 * 1024,
+  triangles: 60_000,
+  drawCalls: 30,
+  bones: 40,
+  morphTargets: 40,
+});

package/client/three/holds.ts

@@ -0,0 +1,64 @@
+/**
+ * How far each character may hold its head off centre, as the mixer wants it:
+ * pose units per axis, ready to hand to `createSvgAvatar` as `headHold`.
+ *
+ * The angles themselves are in `motion-limits.json` and are only
+ * there. They were read off the three faces by the person who owns how they
+ * look, one axis at a time, at the surface a call actually shows — which is
+ * the only way this kind of number can be got, and the reason nothing here
+ * recomputes, adjusts or second-guesses one. This file is the unit change and
+ * nothing else.
+ *
+ * An axis the file marks for re-measurement is not enforced: it was read off an
+ * asset that has since been fixed, and holding the driver to a defect that no
+ * longer exists is worse than the defect. It falls back to the tightest angle
+ * still live on that axis across the other characters instead of falling out of
+ * the budget altogether, which is what it used to do. That is not a way of
+ * believing the stale number by the back door — it is the same budget the
+ * faces nobody has any doubt about are already held to. Dropping the axis made
+ * the one face nobody can currently vouch for the only one driven with no
+ * budget at all, and speech asks for about nine degrees of yaw before a budget
+ * trims it.
+ */
+
+import LIMITS from "./motion-limits.json" with { type: "json" };
+import { HEAD_CLAMP, HEAD_DEG } from "./tara-rig.js";
+
+/** Degrees of each axis per pose unit, which is `HEAD_DEG` at the clamp. */
+const PER_UNIT = {
+  headYaw: HEAD_DEG.yaw / HEAD_CLAMP,
+  headPitch: HEAD_DEG.pitch / HEAD_CLAMP,
+  headRoll: HEAD_DEG.roll / HEAD_CLAMP,
+} as const;
+
+const AXIS = { headYaw: "yaw", headPitch: "pitch", headRoll: "roll" } as const;
+
+export type HeadHold = Partial<Record<keyof typeof AXIS, number>>;
+
+type Measured = { safe: number; remeasure?: string };
+
+/** What the file says about one character's axis, if it says anything live. */
+const live = (name: string, axis: string): Measured | undefined => {
+  const measured = (LIMITS.characters as Record<string, Record<string, unknown>>)[name];
+  const entry = measured?.[axis] as Measured | undefined;
+  return entry && !entry.remeasure ? entry : undefined;
+};
+
+/** The tightest angle any character is still vouched for on this axis. */
+function safest(axis: string): number | undefined {
+  const angles = Object.keys(LIMITS.characters)
+    .map((name) => live(name, axis)?.safe)
+    .filter((deg): deg is number => deg !== undefined);
+  return angles.length ? Math.min(...angles) : undefined;
+}
+
+/** `name` as the limits file spells it: tara, tushar, tanya. */
+export function headHold(name: keyof typeof LIMITS.characters): HeadHold {
+  const out: HeadHold = {};
+  for (const channel of Object.keys(AXIS) as (keyof typeof AXIS)[]) {
+    const axis = AXIS[channel];
+    const deg = live(name, axis)?.safe ?? safest(axis);
+    if (deg !== undefined) out[channel] = deg / PER_UNIT[channel];
+  }
+  return out;
+}

package/client/three/internal.ts

@@ -0,0 +1,54 @@
+/**
+ * The 2.5-D rig, below the `createAvatar` modules — `/internal`, no semver promise.
+ *
+ * `tara.ts`, `tushar.ts` and `tanya.ts` are what a consumer gets: one
+ * `createAvatar` each, the mixer behind it, nothing to configure. This is the
+ * other door, for a page whose subject is the *rig* rather than the call — the
+ * renderer with no mixer in front of it, the three compiled characters, and the
+ * head envelope their channels are scaled by. Its readers are the rig instruments
+ * in the working tree, and nothing that ships.
+ *
+ * It exists because those instruments were reaching around the package instead,
+ * by relative path into its source, and two of them carried their own copy of
+ * the head envelope — with a comment saying the copy would quietly lie the day
+ * the envelope moved. It moved on 2026-09-18. One export is cheaper than that
+ * warning, and it is the same `/internal` spelling this package already uses for
+ * the mixer.
+ */
+export { ASSETS as CHARACTERS } from "./assets.js";
+export type { CharacterName } from "./assets.js";
+
+export { createTaraRig, EYE_DEG, HEAD_CLAMP, HEAD_DEG, TARA_TUNING } from "./tara-rig.js";
+export type { TaraRigOptions } from "./tara-rig.js";
+export { HARD_BUDGET, pixelRatioFor, SHIPPING_SURFACE } from "./budgets.js";
+
+/**
+ * The authored motion tables these characters are driven by.
+ *
+ * `tara.ts` and its siblings hand them to the mixer and nobody else sees them,
+ * which is right for a consumer: a face's clips are not a configuration. An
+ * instrument that drives the mixer directly — a scripted take, where the script
+ * *is* the server — needs the same two tables, and the alternative was a
+ * relative import into the package's own source of the kind this file exists to
+ * end.
+ */
+export { BLENDER_ACTIONS, BLENDER_SEQUENCES, BLENDER_SUPPORTS } from "./sequences.js";
+
+/**
+ * The head budget each character is driven under, and the measurements it comes
+ * from.
+ *
+ * An instrument that watches a character move has to mount the *shipping*
+ * character or it is watching something else, and `headHold` is the one option
+ * `tara.ts` passes that a lab page would otherwise have to guess at — the
+ * budget is load-bearing, not a margin: speech asks for about nine degrees of
+ * yaw before it trims one. The limits themselves come along because a page that
+ * plots degrees has to draw the band, and reading it out of the same file the
+ * budget is derived from is the only way the two cannot disagree.
+ *
+ * Read-only, both of them. Nothing here recomputes a measured angle
+ * (`holds.ts`).
+ */
+export { headHold } from "./holds.js";
+export type { HeadHold } from "./holds.js";
+export { default as MOTION_LIMITS } from "./motion-limits.json" with { type: "json" };

package/client/three/motion-limits.json

@@ -0,0 +1,116 @@
+{
+ "what": "How far each character may HOLD its head off centre, in degrees, measured by eye. A sustained pose is the strict case and this is the strict number: past it a static turn stops reading as a head that turned, because the eye behind a flat projection never foreshortens. `holds.ts` converts these into the mixer's pose units and `test/three/presence.test.ts` gates LISTENING and SPEAKING against them.",
+ "versus": "This is not `HEAD_DEG` and is much smaller than it. `HEAD_DEG` (tara-rig.ts) is the envelope any motion may REACH — set by where the projection itself tears, which a rendered ladder put past 21 degrees of yaw on tara. A transient that returns is forgiven what a held pose is not, so a nod may pass through an angle this file will not let the head sit at.",
+ "units": "degrees from rest, either side, per axis driven alone from rest",
+ "surface": {
+  "cssWidth": 400,
+  "cssHeight": 300,
+  "devicePixelRatio": 2,
+  "magnification": 1
+ },
+ "measured": {
+  "on": "2026-09-18",
+  "by": "owner",
+  "using": "the hold-and-move rig instrument in the working tree, which pins one axis at a fixed angle",
+  "how": "static held pose, one axis at a time from rest, judged by eye at the shipping surface",
+  "safe": "reads as a head that turned, with nothing to notice",
+  "limit": "the last angle before something is visibly wrong"
+ },
+ "characters": {
+  "tara": {
+   "yaw": {
+    "safe": 6,
+    "limit": 7
+   },
+   "pitch": {
+    "safe": 5,
+    "limit": 6
+   },
+   "roll": {
+    "safe": 6,
+    "limit": 7
+   },
+   "defects": [
+    {
+     "axis": "yaw",
+     "from": 7,
+     "what": "the eye does not foreshorten or narrow; it is what gives the turn away first"
+    },
+    {
+     "axis": "roll",
+     "from": 0,
+     "what": "the transition reads as a hinge rather than a neck bending, at any angle"
+    }
+   ]
+  },
+  "tushar": {
+   "yaw": {
+    "safe": 6,
+    "limit": 7
+   },
+   "pitch": {
+    "safe": 5,
+    "limit": 6
+   },
+   "roll": {
+    "safe": 6,
+    "limit": 7
+   },
+   "defects": [
+    {
+     "axis": "yaw",
+     "from": 7,
+     "what": "the eye does not foreshorten or narrow; it is what gives the turn away first"
+    },
+    {
+     "axis": "roll",
+     "from": 0,
+     "what": "the transition reads as a hinge rather than a neck bending, at any angle"
+    }
+   ]
+  },
+  "tanya": {
+   "yaw": {
+    "safe": 5,
+    "limit": 5.5,
+    "remeasure": "every defect this was measured from was fixed on 2026-09-19 (8ba16e8 the neck under the jaw, 6c5c141 the hair that hangs past it), so the number is a record of the old asset and not of this one. Nothing may gate on it until the owner drives her again."
+   },
+   "pitch": {
+    "safe": 5,
+    "limit": 6
+   },
+   "roll": {
+    "safe": 1,
+    "limit": 1,
+    "remeasure": "every defect this was measured from was fixed on 2026-09-19 (8ba16e8 the neck under the jaw, 6c5c141 the hair that hangs past it), so the number is a record of the old asset and not of this one. Nothing may gate on it until the owner drives her again."
+   },
+   "defects": [
+    {
+     "axis": "yaw",
+     "from": 5.5,
+     "what": "a black area opens along the jaw line",
+     "fixed": "2026-09-19"
+    },
+    {
+     "axis": "yaw",
+     "from": 5.5,
+     "what": "something around the neck moves incorrectly",
+     "fixed": "2026-09-19"
+    },
+    {
+     "axis": "roll",
+     "from": 1,
+     "what": "a gap opens between the neck and the face on the far side",
+     "fixed": "2026-09-19"
+    },
+    {
+     "axis": "roll",
+     "from": 2,
+     "what": "the hair breaks up",
+     "fixed": "2026-09-19"
+    }
+   ]
+  }
+ },
+ "remeasure": "An axis carrying `remeasure` was measured on an asset that has since changed. It stays as written \u2014 a measurement is not edited by someone who did not make it \u2014 and a gate skips it and says so."
+}