@voqalize/avatar 0.0.1 → 0.2.0

package/src/idle.js

@@ -392,8 +392,10 @@ export class IdleLayer {
  * that acknowledges on a metronome reads as distracting
  * (docs/research-biomechanics.md §3.5). So this engine is contingent first:
  *
- *   · The host tells it about the user's voice — a coarse speaking flag
- *     (setUserSpeaking) or a measured level (observeLevel), flag wins.
+ *   · The server tells it about the user's voice — a coarse speaking flag
+ *     (setUserSpeaking), off its own endpointer. With no signal at all the
+ *     scheduler falls back to a timer, which is the weakest mode and is meant
+ *     to be: contingency is the whole point.
  *   · Acknowledgements fire at PAUSE ONSETS: when the user stops talking, a
  *     nod lands 250–600 ms later, about half the time, never more often than
  *     every 2.5 s. That timing is where a human listener's nod sits.
@@ -420,10 +422,7 @@ export class ListeningEngine {
     this._next = 0;
     // --- user-signal state
     this._hasSignal = false;
-    this._explicit = null;   // host-declared flag; null = not driven
-    this._derived = false;   // level-derived VAD
-    this._levelOn = false;   // raw hysteresis state behind _derived
-    this._flipT = 0;         // how long the level has disagreed with _levelOn
+    this._explicit = null;   // server-declared flag; null = not driven
     this._speaking = false;  // merged VAD, after hysteresis
     this._spokeAt = -1e9;    // start of the current speech stretch
     this._silentAt = 0;      // end of the last one
@@ -439,29 +438,13 @@ export class ListeningEngine {
     this._pending = -1;
   }
 
-  /** Host-declared user speech. Pass null to hand control back to the level VAD. */
+  /** Server-declared user speech, off the pipeline's own endpointer. Pass null
+   *  to hand back to the no-signal timer. */
   setUserSpeaking(b) {
     if (b !== null) this._hasSignal = true;
     this._explicit = b === null ? null : !!b;
   }
 
-  /** Feed the smoothed user audio level (an AudioFallback.level). Same scale,
-   *  thresholds and asymmetry as the demo's RMS VAD: quick in (80 ms), slow
-   *  out (250 ms) — declaring the turn over early is the expensive mistake,
-   *  and the 250 ms quiet-hold IS the pause detector the contingent
-   *  scheduler keys off. */
-  observeLevel(level) {
-    this._hasSignal = true;
-    const on = !!this._levelOn;
-    const wants = level > (on ? 0.018 : 0.030);
-    if (wants === on) this._flipT = this.t;
-    else if (this.t - this._flipT >= (on ? 0.25 : 0.08)) {
-      this._levelOn = wants;
-      this._flipT = this.t;
-    }
-    this._derived = !!this._levelOn;
-  }
-
   /** The one seam for choosing an acknowledgement. Context-aware: what the
    *  user just did decides the weight class of the reply
    *  (docs/research-biomechanics.md §3.3 — continuers co-occur with ongoing
@@ -486,7 +469,7 @@ export class ListeningEngine {
     return r < 0.55 ? 'NOD_SMALL' : r < 0.8 ? 'BROW_ACK' : 'NOD_SLOW';
   }
 
-  get speaking() { return this._explicit !== null ? this._explicit : this._derived; }
+  get speaking() { return this._explicit === true; }
 
   update(dt) {
     this.t += dt;

package/src/line-art.js

@@ -104,6 +104,32 @@ export function region(flat) {
   return d + 'Z';
 }
 
+/**
+ * On-curve points -> the polybezier form above, Catmull-Rom at the standard 1/6
+ * tension. The house idiom for a FACE is hand-authored control points: a brow or
+ * a lip contour is tuned a handle at a time and interpolation would fight that.
+ * This is for marks whose geometry is easier to read as a list of places the
+ * line goes through than as three-in-four control points — the hand's contours,
+ * where the authoring question is "how far does the thumb clear the knuckles"
+ * and every point is measured against another point. Curve control is worth
+ * less there than a shape whose numbers can be argued about.
+ */
+export function smooth(pts) {
+  const out = [pts[0]];
+  for (let i = 0; i < pts.length - 1; i++) {
+    const p0 = pts[i - 1] || pts[i];
+    const p1 = pts[i];
+    const p2 = pts[i + 1];
+    const p3 = pts[i + 2] || p2;
+    out.push(
+      [p1[0] + (p2[0] - p0[0]) / 6, p1[1] + (p2[1] - p0[1]) / 6],
+      [p2[0] - (p3[0] - p1[0]) / 6, p2[1] - (p3[1] - p1[1]) / 6],
+      p2
+    );
+  }
+  return out;
+}
+
 /** Deterministic jitter, so a drawing is the same every load. */
 export function rng(seed) {
   let s = seed >>> 0;

package/src/perform.js

@@ -3,7 +3,7 @@
  *
  * A performance is the server's choreography: timed verbs `{t, do, ...}` fired
  * against a clock, where every verb resolves to one of the widget's own enums —
- * states, emotions, gaze targets, interjections. The vocabulary is deliberately
+ * states, emotions, gaze targets, interjections, hand gestures. The vocabulary is deliberately
  * closed: the backend sequences what the rig already does well, it cannot
  * invent motion. That constraint is what makes the wire format assemblable by
  * a dialogue manager and reviewable by a human.
@@ -21,7 +21,10 @@
  * effects, and replaying a nod is worse than missing one.
  */
 
-const VERBS = new Set(['state', 'emotion', 'gaze', 'interject']);
+const VERBS = new Set(['state', 'emotion', 'gaze', 'interject', 'gesture']);
+// The verbs addressed by `id` rather than `name`. Both id verbs name a clip the
+// widget already owns; both name-verbs name an enum value.
+const ID_VERBS = new Set(['interject', 'gesture']);
 
 /**
  * Shape hygiene for action arrays, in the spirit of normalizeCues: sort by
@@ -45,8 +48,9 @@ export function normalizeActions(actions) {
       console.warn(`perform: dropped unknown verb "${a && a.do}"`, a);
       continue;
     }
-    if ((a.do === 'interject' ? a.id : a.name) == null) {
-      console.warn(`perform: dropped ${a.do} with no ${a.do === 'interject' ? 'id' : 'name'}`, a);
+    const idVerb = ID_VERBS.has(a.do);
+    if ((idVerb ? a.id : a.name) == null) {
+      console.warn(`perform: dropped ${a.do} with no ${idVerb ? 'id' : 'name'}`, a);
       continue;
     }
     out.push(a);

package/client/dist/pipecat.d.ts

@@ -1,21 +0,0 @@
-/**
- * `@voqalize/avatar/pipecat` — drive the widget from a pipecat session.
- *
- * Framework-free: everything here is plain TypeScript over the `AvatarApi` the
- * root export returns. `AvatarClient` is the whole surface — construct it
- * around a mounted widget and either `attach()` it to a live `PipecatClient`
- * or feed it messages yourself with `dispatch()`.
- *
- *     import { createAvatar } from "@voqalize/avatar";
- *     import { AvatarClient } from "@voqalize/avatar/pipecat";
- *
- *     const avatar = createAvatar({ mount: "#tile" });
- *     const detach = new AvatarClient(avatar).attach(pipecatClient);
- *
- * `@pipecat-ai/client-js` is a peer dependency of this subpath only — the root
- * export has no dependencies at all, and a host that carries avatar commands
- * over its own transport can import this module and never call `attach()`.
- */
-export { AvatarClient, type AvatarClientOptions } from "./AvatarClient.js";
-export { isAvatarMessage, AVATAR_MESSAGE_TYPE, AVATAR_PROTOCOL_VERSION, type AvatarCommand, type AvatarCue, type AvatarCuesCmd, type AvatarHintCmd, type AvatarInterjectCmd, type AvatarPerformAction, type AvatarPerformCmd, type AvatarServerMessage, type AvatarSpeechCmd, type AvatarStateCmd, type AvatarUnknownCmd, type AvatarUserCmd, } from "./types.js";
-//# sourceMappingURL=pipecat.d.ts.map

package/client/dist/pipecat.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pipecat.d.ts","sourceRoot":"","sources":["../src/pipecat.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAE3E,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,uBAAuB,EACvB,KAAK,aAAa,EAClB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,aAAa,GACnB,MAAM,YAAY,CAAC"}

package/client/dist/pipecat.js

@@ -1,21 +0,0 @@
-/**
- * `@voqalize/avatar/pipecat` — drive the widget from a pipecat session.
- *
- * Framework-free: everything here is plain TypeScript over the `AvatarApi` the
- * root export returns. `AvatarClient` is the whole surface — construct it
- * around a mounted widget and either `attach()` it to a live `PipecatClient`
- * or feed it messages yourself with `dispatch()`.
- *
- *     import { createAvatar } from "@voqalize/avatar";
- *     import { AvatarClient } from "@voqalize/avatar/pipecat";
- *
- *     const avatar = createAvatar({ mount: "#tile" });
- *     const detach = new AvatarClient(avatar).attach(pipecatClient);
- *
- * `@pipecat-ai/client-js` is a peer dependency of this subpath only — the root
- * export has no dependencies at all, and a host that carries avatar commands
- * over its own transport can import this module and never call `attach()`.
- */
-export { AvatarClient } from "./AvatarClient.js";
-export { isAvatarMessage, AVATAR_MESSAGE_TYPE, AVATAR_PROTOCOL_VERSION, } from "./types.js";
-//# sourceMappingURL=pipecat.js.map

package/client/dist/pipecat.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pipecat.js","sourceRoot":"","sources":["../src/pipecat.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAA4B,MAAM,mBAAmB,CAAC;AAE3E,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,uBAAuB,GAaxB,MAAM,YAAY,CAAC"}

package/client/dist/react.d.ts

@@ -1,16 +0,0 @@
-/**
- * `@voqalize/avatar/react` — the React binding.
- *
- *     import { Avatar } from "@voqalize/avatar/react";
- *
- *     <Avatar client={pipecatClient} className="tile" />
- *
- * Peers: `react >= 18` and `@pipecat-ai/client-js`. Everything a non-React
- * host needs is in `@voqalize/avatar/pipecat`; this module adds a mount
- * lifecycle and nothing else.
- */
-export { useAvatar, type UseAvatarHandle, type UseAvatarOptions } from "./useAvatar.js";
-export { Avatar, type AvatarProps } from "./Avatar.js";
-export { AvatarClient, type AvatarClientOptions } from "./AvatarClient.js";
-export { isAvatarMessage, AVATAR_MESSAGE_TYPE, AVATAR_PROTOCOL_VERSION, type AvatarCommand, type AvatarCue, type AvatarCuesCmd, type AvatarHintCmd, type AvatarInterjectCmd, type AvatarPerformAction, type AvatarPerformCmd, type AvatarServerMessage, type AvatarSpeechCmd, type AvatarStateCmd, type AvatarUnknownCmd, type AvatarUserCmd, } from "./types.js";
-//# sourceMappingURL=react.d.ts.map

package/client/dist/react.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"react.d.ts","sourceRoot":"","sources":["../src/react.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,eAAe,EAAE,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACxF,OAAO,EAAE,MAAM,EAAE,KAAK,WAAW,EAAE,MAAM,aAAa,CAAC;AAGvD,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAC3E,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,uBAAuB,EACvB,KAAK,aAAa,EAClB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,aAAa,GACnB,MAAM,YAAY,CAAC"}

package/client/dist/react.js

@@ -1,17 +0,0 @@
-/**
- * `@voqalize/avatar/react` — the React binding.
- *
- *     import { Avatar } from "@voqalize/avatar/react";
- *
- *     <Avatar client={pipecatClient} className="tile" />
- *
- * Peers: `react >= 18` and `@pipecat-ai/client-js`. Everything a non-React
- * host needs is in `@voqalize/avatar/pipecat`; this module adds a mount
- * lifecycle and nothing else.
- */
-export { useAvatar } from "./useAvatar.js";
-export { Avatar } from "./Avatar.js";
-// Re-exported so a React consumer needs one import for the common case.
-export { AvatarClient } from "./AvatarClient.js";
-export { isAvatarMessage, AVATAR_MESSAGE_TYPE, AVATAR_PROTOCOL_VERSION, } from "./types.js";
-//# sourceMappingURL=react.js.map

package/client/dist/react.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"react.js","sourceRoot":"","sources":["../src/react.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,SAAS,EAA+C,MAAM,gBAAgB,CAAC;AACxF,OAAO,EAAE,MAAM,EAAoB,MAAM,aAAa,CAAC;AAEvD,wEAAwE;AACxE,OAAO,EAAE,YAAY,EAA4B,MAAM,mBAAmB,CAAC;AAC3E,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,uBAAuB,GAaxB,MAAM,YAAY,CAAC"}

package/client/src/pipecat.ts

@@ -1,38 +0,0 @@
-/**
- * `@voqalize/avatar/pipecat` — drive the widget from a pipecat session.
- *
- * Framework-free: everything here is plain TypeScript over the `AvatarApi` the
- * root export returns. `AvatarClient` is the whole surface — construct it
- * around a mounted widget and either `attach()` it to a live `PipecatClient`
- * or feed it messages yourself with `dispatch()`.
- *
- *     import { createAvatar } from "@voqalize/avatar";
- *     import { AvatarClient } from "@voqalize/avatar/pipecat";
- *
- *     const avatar = createAvatar({ mount: "#tile" });
- *     const detach = new AvatarClient(avatar).attach(pipecatClient);
- *
- * `@pipecat-ai/client-js` is a peer dependency of this subpath only — the root
- * export has no dependencies at all, and a host that carries avatar commands
- * over its own transport can import this module and never call `attach()`.
- */
-
-export { AvatarClient, type AvatarClientOptions } from "./AvatarClient.js";
-
-export {
-  isAvatarMessage,
-  AVATAR_MESSAGE_TYPE,
-  AVATAR_PROTOCOL_VERSION,
-  type AvatarCommand,
-  type AvatarCue,
-  type AvatarCuesCmd,
-  type AvatarHintCmd,
-  type AvatarInterjectCmd,
-  type AvatarPerformAction,
-  type AvatarPerformCmd,
-  type AvatarServerMessage,
-  type AvatarSpeechCmd,
-  type AvatarStateCmd,
-  type AvatarUnknownCmd,
-  type AvatarUserCmd,
-} from "./types.js";

package/client/src/react.ts

@@ -1,34 +0,0 @@
-/**
- * `@voqalize/avatar/react` — the React binding.
- *
- *     import { Avatar } from "@voqalize/avatar/react";
- *
- *     <Avatar client={pipecatClient} className="tile" />
- *
- * Peers: `react >= 18` and `@pipecat-ai/client-js`. Everything a non-React
- * host needs is in `@voqalize/avatar/pipecat`; this module adds a mount
- * lifecycle and nothing else.
- */
-
-export { useAvatar, type UseAvatarHandle, type UseAvatarOptions } from "./useAvatar.js";
-export { Avatar, type AvatarProps } from "./Avatar.js";
-
-// Re-exported so a React consumer needs one import for the common case.
-export { AvatarClient, type AvatarClientOptions } from "./AvatarClient.js";
-export {
-  isAvatarMessage,
-  AVATAR_MESSAGE_TYPE,
-  AVATAR_PROTOCOL_VERSION,
-  type AvatarCommand,
-  type AvatarCue,
-  type AvatarCuesCmd,
-  type AvatarHintCmd,
-  type AvatarInterjectCmd,
-  type AvatarPerformAction,
-  type AvatarPerformCmd,
-  type AvatarServerMessage,
-  type AvatarSpeechCmd,
-  type AvatarStateCmd,
-  type AvatarUnknownCmd,
-  type AvatarUserCmd,
-} from "./types.js";

package/src/audio-fallback.js

@@ -1,100 +0,0 @@
-/**
- * Amplitude-driven lipsync — the degradation path.
- *
- * When cues are late, missing, or the TTS vendor can't emit them, we read the
- * audio directly with a WebAudio AnalyserNode and guess. It is obviously worse
- * than real visemes, but it is *far* better than a still mouth, it costs
- * nothing, and it keeps working when the network doesn't.
- *
- * The guess uses two cheap signals:
- *   · RMS  -> how far the mouth opens
- *   · spectral tilt -> which shape family (sibilant / open vowel / rounded)
- */
-
-const GATE = 0.012;
-
-export class AudioFallback {
-  constructor() {
-    this.ctx = null;
-    this.analyser = null;
-    this.time = null;
-    this.freq = null;
-    this.level = 0;
-    this.active = false;
-    this._letter = 'X';
-    this._holdUntil = 0;
-    this._t = 0;
-  }
-
-  /** @param {MediaStream|HTMLMediaElement|AudioNode} source */
-  attach(source) {
-    this.detach();
-    const Ctx = window.AudioContext || window.webkitAudioContext;
-    this.ctx = new Ctx();
-    let node;
-    if (source instanceof MediaStream) node = this.ctx.createMediaStreamSource(source);
-    else if (source instanceof AudioNode) node = source;
-    else node = this.ctx.createMediaElementSource(source);
-
-    this.analyser = this.ctx.createAnalyser();
-    this.analyser.fftSize = 1024;
-    this.analyser.smoothingTimeConstant = 0.55;
-    node.connect(this.analyser);
-    // Media-element sources must still reach the speakers.
-    if (!(source instanceof MediaStream)) this.analyser.connect(this.ctx.destination);
-
-    this.time = new Uint8Array(this.analyser.fftSize);
-    this.freq = new Uint8Array(this.analyser.frequencyBinCount);
-    this.active = true;
-    if (this.ctx.state === 'suspended') this.ctx.resume().catch(() => {});
-    return this;
-  }
-
-  detach() {
-    if (this.ctx) { try { this.ctx.close(); } catch (e) { /* already closed */ } }
-    this.ctx = null; this.analyser = null; this.active = false; this.level = 0;
-  }
-
-  /** @returns {{letter:string,intensity:number}|null} */
-  sample(dt) {
-    if (!this.active) return null;
-    this._t += dt;
-    this.analyser.getByteTimeDomainData(this.time);
-    this.analyser.getByteFrequencyData(this.freq);
-
-    let sum = 0;
-    for (let i = 0; i < this.time.length; i++) {
-      const v = (this.time[i] - 128) / 128;
-      sum += v * v;
-    }
-    const rms = Math.sqrt(sum / this.time.length);
-    this.level += (rms - this.level) * (1 - Math.exp(-dt / 0.045));
-
-    if (this.level < GATE) { this._letter = 'X'; return { letter: 'X', intensity: 1 }; }
-
-    // Bin edges assume ~48kHz; exact boundaries don't matter much here.
-    const n = this.freq.length;
-    const band = (a, b) => {
-      let s = 0;
-      const lo = Math.floor(n * a), hi = Math.floor(n * b);
-      for (let i = lo; i < hi; i++) s += this.freq[i];
-      return s / Math.max(1, hi - lo);
-    };
-    const low = band(0.00, 0.035);   // ~0-800 Hz
-    const mid = band(0.035, 0.13);   // ~800-3k
-    const high = band(0.13, 0.40);   // ~3k-9.6k
-    const total = low + mid + high + 1e-6;
-
-    let letter;
-    if (high / total > 0.34) letter = 'B';                 // sibilant
-    else if (low / total > 0.62) letter = this.level > 0.16 ? 'F' : 'E'; // rounded / back
-    else letter = this.level > 0.20 ? 'D' : this.level > 0.09 ? 'C' : 'B';
-
-    // Hold each guess briefly; frame-rate shape churn looks like chattering.
-    if (this._t < this._holdUntil) letter = this._letter;
-    else { this._letter = letter; this._holdUntil = this._t + 0.055; }
-
-    const intensity = Math.min(1, this.level / 0.22);
-    return { letter, intensity };
-  }
-}