@wibly/sdk 0.1.2 → 0.1.3

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # `@wibly/sdk` — Changelog
 
+## 0.1.3 — 2026-06-08
+
+## 0.1.3 — 2026-06-08
+
+### Added
+
+- `session.ttsPlayback` — client-side TTS playback telemetry
+  (`subscribe` / `getSnapshot`) exposing `{ isPlaying, speechLevel }`.
+  The Host shell samples speech energy from its single `<audio>` element
+  and pushes smoothed levels via `setTtsPlaybackTelemetry(session, …)` so
+  Experience bundles can drive amplitude-synced avatar animation without
+  high-frequency `voice.*` events on the wire.
+- `@wibly/sdk/react` — `useTtsPlayback()` hook wrapping the telemetry
+  store for shell and bundle React trees.
+
 ## 0.1.2 — 2026-06-08
 
 ### Added

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wibly/sdk",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Wibly @wibly/sdk",
   "type": "module",
   "main": "./src/index.ts",
@@ -18,9 +18,9 @@
     "access": "public"
   },
   "dependencies": {
-    "@wibly/internal-manifest": "0.1.2",
-    "@wibly/internal-protocol": "0.1.2",
-    "@wibly/internal-shared": "0.1.2",
+    "@wibly/internal-manifest": "0.1.3",
+    "@wibly/internal-protocol": "0.1.3",
+    "@wibly/internal-shared": "0.1.3",
     "zod": "^3.25.76"
   },
   "peerDependencies": {

package/src/client.ts

@@ -77,6 +77,11 @@ import {
   createServerTimeSource,
   type ServerTimeSource,
 } from './time.js';
+import {
+  createTtsPlaybackStore,
+  registerTtsPlaybackStore,
+  type TtsPlaybackTelemetry,
+} from './tts-playback.js';
 import {
   createTransport,
   type Transport,
@@ -204,6 +209,13 @@ export type Session = {
   };
   readonly inference: SessionInference;
   readonly voice: SessionVoice;
+  /**
+   * Client-side TTS playback telemetry (speech energy + playing
+   * flag). Populated by the Host shell from the local audio element;
+   * read by Experience bundles to drive avatar animation. Never
+   * crosses the wire. See `tts-playback.ts`.
+   */
+  readonly ttsPlayback: TtsPlaybackTelemetry;
   readonly events: Pick<EventBus, 'onEvent' | 'onAnyEvent'>;
   readonly lifecycle: LifecycleBindings;
   /** Server-time helper. See `time.ts`. */
@@ -240,6 +252,7 @@ const DEFAULT_VOICE_SPEAK_TIMEOUT_MS = 30_000;
 export const createSession = (config: SessionConfig): Session => {
   const store = createSessionStore();
   const bus = createEventBus();
+  const ttsPlaybackStore = createTtsPlaybackStore();
   const time = createServerTimeSource({ now: config.now });
   const lifecycle = createLifecycleBindings(bus);
   let isPreviewFlag: boolean = config.isPreview ?? false;
@@ -806,6 +819,10 @@ export const createSession = (config: SessionConfig): Session => {
       scheduleTimer,
       voiceSpeakTimeoutMs,
     ),
+    ttsPlayback: {
+      subscribe: ttsPlaybackStore.subscribe,
+      getSnapshot: ttsPlaybackStore.getSnapshot,
+    },
     events: { onEvent: bus.onEvent, onAnyEvent: bus.onAnyEvent },
     lifecycle,
     time: { serverNow: time.serverNow, recordEvent: time.recordEvent },
@@ -836,6 +853,8 @@ export const createSession = (config: SessionConfig): Session => {
     },
   };
 
+  registerTtsPlaybackStore(session, ttsPlaybackStore);
+
   transport.start();
   return session;
 };

package/src/index.ts

@@ -83,6 +83,15 @@ export {
   type VoiceAudioPayload,
 } from './voice.js';
 
+export {
+  createTtsPlaybackStore,
+  registerTtsPlaybackStore,
+  setTtsPlaybackTelemetry,
+  type TtsPlaybackSnapshot,
+  type TtsPlaybackStore,
+  type TtsPlaybackTelemetry,
+} from './tts-playback.js';
+
 export {
   type EventBus,
   type EventHandler,

package/src/react.ts

@@ -37,6 +37,7 @@ import type { EventPayload, LifecyclePayload } from '@wibly/internal-protocol';
 import type { ConsentDecision, ConsentRequiredPayload } from './consent.js';
 import type { Session, SessionConfig, SessionState } from './client.js';
 import { createSession } from './client.js';
+import type { TtsPlaybackSnapshot } from './tts-playback.js';
 
 const SessionContext = createContext<Session | null>(null);
 
@@ -121,6 +122,20 @@ export const useConnectionState = (): SessionState['connectionState'] => {
   return state.connectionState;
 };
 
+/**
+ * Subscribe to client-side TTS playback telemetry (speech energy +
+ * playing flag). Populated by the Host shell's audio sampler; idle
+ * until the first clip plays.
+ */
+export const useTtsPlayback = (): TtsPlaybackSnapshot => {
+  const session = useSession();
+  return useSyncExternalStore(
+    session.ttsPlayback.subscribe,
+    session.ttsPlayback.getSnapshot,
+    session.ttsPlayback.getSnapshot,
+  );
+};
+
 /**
  * Subscribe to a server-emitted event by `eventType`. The handler
  * fires on every matching `event` frame. Unsubscribes on unmount.

package/src/tts-playback.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+  createTtsPlaybackStore,
+  registerTtsPlaybackStore,
+  setTtsPlaybackTelemetry,
+} from './tts-playback.js';
+
+describe('createTtsPlaybackStore', () => {
+  it('starts idle', () => {
+    const store = createTtsPlaybackStore();
+    expect(store.getSnapshot()).toEqual({ isPlaying: false, speechLevel: 0 });
+  });
+
+  it('notifies subscribers when the snapshot changes', () => {
+    const store = createTtsPlaybackStore();
+    const listener = vi.fn();
+    const unsubscribe = store.subscribe(listener);
+
+    store.set({ isPlaying: true, speechLevel: 0.5 });
+
+    expect(listener).toHaveBeenCalledTimes(1);
+    expect(store.getSnapshot()).toEqual({ isPlaying: true, speechLevel: 0.5 });
+
+    unsubscribe();
+    store.set({ isPlaying: false, speechLevel: 0 });
+    expect(listener).toHaveBeenCalledTimes(1);
+  });
+
+  it('treats an identical set as a no-op and keeps a stable reference', () => {
+    const store = createTtsPlaybackStore();
+    const listener = vi.fn();
+    store.subscribe(listener);
+
+    const before = store.getSnapshot();
+    store.set({ isPlaying: false, speechLevel: 0 });
+
+    // useSyncExternalStore depends on a stable reference when nothing
+    // changed — otherwise it re-renders (or warns) every read.
+    expect(store.getSnapshot()).toBe(before);
+    expect(listener).not.toHaveBeenCalled();
+  });
+});
+
+describe('setTtsPlaybackTelemetry', () => {
+  it('routes a sample to the registered store', () => {
+    const session = {};
+    const store = createTtsPlaybackStore();
+    registerTtsPlaybackStore(session, store);
+
+    setTtsPlaybackTelemetry(session, { isPlaying: true, speechLevel: 0.8 });
+
+    expect(store.getSnapshot()).toEqual({ isPlaying: true, speechLevel: 0.8 });
+  });
+
+  it('is a no-op for a session with no registered store', () => {
+    expect(() =>
+      setTtsPlaybackTelemetry({}, { isPlaying: true, speechLevel: 1 }),
+    ).not.toThrow();
+  });
+});

package/src/tts-playback.ts

@@ -0,0 +1,99 @@
+/**
+ * Client-side TTS playback telemetry.
+ *
+ * This is a purely local, ephemeral signal — it never touches the
+ * WebSocket. The Host shell samples speech energy from the single
+ * `<audio>` element (Web Audio `AnalyserNode`) and pushes a smoothed
+ * `speechLevel` here at animation-frame cadence; Experience bundles
+ * read it through `session.ttsPlayback` to drive avatar animation
+ * (the amplitude-driven fallback the Platform Spec §3.5 describes for
+ * providers without phoneme metadata).
+ *
+ * Why a dedicated store and not a `voice.*` event: the signal updates
+ * ~60×/sec. Pumping that through `session.events` would flood every
+ * listener and the wire-shaped bus machinery. A tiny
+ * `useSyncExternalStore`-friendly snapshot store keeps the hot path
+ * cheap and matches the SDK's existing one-store-per-concern pattern.
+ *
+ * The read surface (`subscribe` / `getSnapshot`) is public on
+ * `Session.ttsPlayback`. The write surface lives behind
+ * `setTtsPlaybackTelemetry(session, snapshot)` so only the shell that
+ * owns the audio element mutates it — bundles consume, never produce.
+ */
+
+export type TtsPlaybackSnapshot = {
+  /** True while a TTS clip is actively playing. */
+  readonly isPlaying: boolean;
+  /** Smoothed speech energy, normalised to 0..1. */
+  readonly speechLevel: number;
+};
+
+/** Read surface exposed on `Session.ttsPlayback`. */
+export type TtsPlaybackTelemetry = {
+  readonly subscribe: (listener: () => void) => () => void;
+  readonly getSnapshot: () => TtsPlaybackSnapshot;
+};
+
+/** Full store — the read surface plus the shell-only setter. */
+export type TtsPlaybackStore = TtsPlaybackTelemetry & {
+  readonly set: (next: TtsPlaybackSnapshot) => void;
+};
+
+const IDLE: TtsPlaybackSnapshot = { isPlaying: false, speechLevel: 0 };
+
+export const createTtsPlaybackStore = (): TtsPlaybackStore => {
+  let snapshot: TtsPlaybackSnapshot = IDLE;
+  const listeners = new Set<() => void>();
+
+  return {
+    subscribe: (listener) => {
+      listeners.add(listener);
+      return () => {
+        listeners.delete(listener);
+      };
+    },
+    getSnapshot: () => snapshot,
+    set: (next) => {
+      if (
+        next.isPlaying === snapshot.isPlaying &&
+        next.speechLevel === snapshot.speechLevel
+      ) {
+        return;
+      }
+      snapshot = next;
+      for (const listener of Array.from(listeners)) {
+        try {
+          listener();
+        } catch {
+          // Listeners must not throw; swallow to avoid cascading.
+        }
+      }
+    },
+  };
+};
+
+/**
+ * Registry mapping a `Session` to its writable telemetry store. The
+ * `Session` only exposes the read surface, so the shell pushes
+ * samples through this side channel keyed by the session reference.
+ */
+const storeRegistry = new WeakMap<object, TtsPlaybackStore>();
+
+export const registerTtsPlaybackStore = (
+  session: object,
+  store: TtsPlaybackStore,
+): void => {
+  storeRegistry.set(session, store);
+};
+
+/**
+ * Push a telemetry sample for the given session. No-op if the session
+ * has no registered store (e.g. a stub session in a dev harness that
+ * supplies its own `ttsPlayback`).
+ */
+export const setTtsPlaybackTelemetry = (
+  session: object,
+  snapshot: TtsPlaybackSnapshot,
+): void => {
+  storeRegistry.get(session)?.set(snapshot);
+};