@voicethere/agent 0.7.11 → 0.7.12

package/dist/templates/webhooks-redis/agent.js

@@ -10903,6 +10903,12 @@ function isPlayStopAckMessage(value) {
   const msg = value;
   return msg.type === "play_stop_ack" && typeof msg.requestId === "string";
 }
+function isPlayPoseAckMessage(value) {
+  if (!value || typeof value !== "object")
+    return false;
+  const msg = value;
+  return msg.type === "play_pose_ack" && typeof msg.requestId === "string";
+}
 function isRecordingControlAckMessage(value) {
   if (!value || typeof value !== "object")
     return false;
@@ -10963,7 +10969,7 @@ function isParentMessage(value) {
   if (!value || typeof value !== "object")
     return false;
   const msg = value;
-  return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "play_ack" || msg.type === "play_status_ack" || msg.type === "play_stop_ack" || msg.type === "mix_control_ack" || msg.type === "stt_control_ack" || msg.type === "webhook";
+  return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "play_ack" || msg.type === "play_status_ack" || msg.type === "play_stop_ack" || msg.type === "play_pose_ack" || msg.type === "mix_control_ack" || msg.type === "stt_control_ack" || msg.type === "webhook";
 }
 function isSessionScopedParentMessage(value) {
   return isParentMessage(value) && !isWebhookMessage(value);
@@ -10985,6 +10991,7 @@ var pendingRecordingAcks = /* @__PURE__ */ new Map();
 var pendingPlayAcks = /* @__PURE__ */ new Map();
 var pendingPlayStatusAcks = /* @__PURE__ */ new Map();
 var pendingPlayStopAcks = /* @__PURE__ */ new Map();
+var pendingPlayPoseAcks = /* @__PURE__ */ new Map();
 var pendingMixAcks = /* @__PURE__ */ new Map();
 var pendingSttAcks = /* @__PURE__ */ new Map();
 function handleRecordingControlAck(message) {
@@ -11039,6 +11046,19 @@ function handlePlayStopAck(message) {
     requestId: message.requestId
   });
 }
+function handlePlayPoseAck(message) {
+  const pending = pendingPlayPoseAcks.get(message.requestId);
+  if (!pending)
+    return;
+  clearTimeout(pending.timer);
+  pendingPlayPoseAcks.delete(message.requestId);
+  pending.resolve({
+    ok: message.ok,
+    playId: message.playId,
+    reason: message.reason,
+    requestId: message.requestId
+  });
+}
 function handleMixControlAck(message) {
   const pending = pendingMixAcks.get(message.requestId);
   if (!pending)
@@ -11124,7 +11144,7 @@ function sendParentMessage(message) {
     process.send?.(message);
     return;
   }
-  if (msgType === "play" || msgType === "play_status" || msgType === "play_stop") {
+  if (msgType === "play" || msgType === "play_status" || msgType === "play_stop" || msgType === "play_pose") {
     process.send?.(message);
     return;
   }
@@ -11310,6 +11330,10 @@ function defineAgent(handlers) {
       handlePlayStopAck(message);
       return;
     }
+    if (isPlayPoseAckMessage(message)) {
+      handlePlayPoseAck(message);
+      return;
+    }
     if (isMixControlAckMessage(message)) {
       handleMixControlAck(message);
       return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicethere/agent",
-  "version": "0.7.11",
+  "version": "0.7.12",
   "description": "VoiceThere customer agent SDK — IPC types and runtime helpers for sandboxed child bundles",
   "type": "module",
   "exports": {
@@ -95,9 +95,9 @@
     "@node-webrtc-rust/sdk": ">=0.9.0"
   },
   "devDependencies": {
-    "@node-webrtc-rust/helpers": "0.9.5",
-    "@node-webrtc-rust/sdk": "0.9.5",
-    "@node-webrtc-rust/signaling": "0.9.5",
+    "@node-webrtc-rust/helpers": "0.9.9",
+    "@node-webrtc-rust/sdk": "0.9.9",
+    "@node-webrtc-rust/signaling": "0.9.9",
     "@types/node": "22.20.1",
     "http-server": "14.1.1",
     "ioredis": "5.11.1",

package/templates/README.md

@@ -15,10 +15,10 @@ import {
 
 ## Product vs e2e
 
-| Kind        | Dashboard create                                                                                                                             | Prebuilt seed bundle                 | Typical consumer        |
-| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
+| Kind        | Dashboard create                                                                                                                                                 | Prebuilt seed bundle                 | Typical consumer        |
+| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
 | **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `game-sync`, `voice-showcase`, `recording-consent`, `positional-tts`, `spatial-showcase`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
-| **e2e**     | No                                                                                                                                           | No — build from sources at test time | `voicethere/e2e` smokes |
+| **e2e**     | No                                                                                                                                                               | No — build from sources at test time | `voicethere/e2e` smokes |
 
 Product templates always set `seedOnCreate: true`. CI fails if a product template is missing its prebuilt bundle after `npm run build`.
 
@@ -83,17 +83,17 @@ Sources: `positional-tts/agent.ts` (defineAgent wiring), `positional-tts/orbit.t
 
 Spatial audio showcase for `/showcase` — one agent template with three demos selected by `{ type: "join", demo }` on the control DataChannel:
 
-| Demo | Behavior |
-| ---- | -------- |
-| `orbit` | TTS speaker orbits the listener (`setTtsPose`); `{ type: "orbit", action: "set" \| "say" }`; agent emits `{ type: "orbit_pose" }` |
-| `soundboard` | Positional clip pads — browser sends `clipId` only (never URLs); agent resolves against an allowlisted `assetOrigin` |
-| `proximity` | Shared mix room (`createMixGroup` + `setClientPose`); `{ type: "move" }`, `{ type: "mute_peer" }`; `{ type: "room_state" }` broadcast |
+| Demo         | Behavior                                                                                                                                                                   |
+| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `orbit`      | Looping sine tone + TTS orbit the listener (`setPlayPose` + `setTtsPose`); STT echo off; `{ type: "orbit", action: "set" \| "say" }`; agent emits `{ type: "orbit_pose" }` |
+| `soundboard` | Positional clip pads — browser sends `clipId` only (never URLs); agent resolves against an allowlisted `assetOrigin`                                                       |
+| `proximity`  | Shared mix room (`createMixGroup` + `setClientPose`); `{ type: "move" }`, `{ type: "mute_peer" }`; `{ type: "room_state" }` broadcast                                      |
 
 **Protocol:** inbound `join`, `orbit`, `pad`, `pad_stop`, `pad_status`, `move`, `mute_peer`, `leave`, `ping` — outbound `showcase_ack`, `orbit_pose`, `pad_progress`, `room_state`, `room_full`, `error`.
 
-**Limits:** `MAX_ROOM_PEERS = 8`, `MAX_ACTIVE_PLAYS = 4` per session. Clip ids: `chime`, `bell`, `laser`, `impact`, `footsteps`, `rain-loop`, `cafe-loop`, `jingle`.
+**Limits:** `MAX_ROOM_PEERS = 8`, `MAX_ACTIVE_PLAYS = 12` per session. Clip ids: `chime`, `bell`, `laser`, `impact`, `footsteps`, `rain-loop`, `cafe-loop`, `jingle`.
 
-Sources: `spatial-showcase/agent.ts`, `protocol.ts`, `sounds.ts`, `room.ts`, `orbit.ts`.
+Sources: `spatial-showcase/agent.ts`, `protocol.ts`, `sounds.ts`, `room.ts`, `orbit.ts`, `sine.ts`.
 
 ### `webhooks.ts` (`webhooks`)
 

package/templates/spatial-showcase/agent.ts

@@ -1,5 +1,5 @@
 /**
- * Spatial audio showcase — orbit TTS, positional soundboard, proximity room.
+ * Spatial audio showcase — orbiting sine + TTS, positional soundboard, proximity room.
  *
  * Browser joins with `{ type: "join", demo, assetOrigin? }` then sends demo-specific
  * commands. Clip URLs are resolved server-side from clipId + allowlisted assetOrigin.
@@ -21,7 +21,9 @@ import {
   sendToClient,
   setClientPose,
   setListenerMute,
+  setPlayPose,
   setPositionalMixing,
+  setSttEnabled,
   setTtsPose,
   speak,
   stopPlay,
@@ -42,16 +44,23 @@ import {
 } from "./protocol.js";
 import { ProximityRoom } from "./room.js";
 import {
+  isShowcaseClipId,
   resolveClipUrl,
   SHOWCASE_SOUND_FILES,
   type ShowcaseClipId,
 } from "./sounds.js";
+import {
+  buildOrbitSineInlineClipBase64,
+  ORBIT_SINE_INLINE_URL,
+} from "./sine.js";
 
 const ORBIT_TICK_MS = 50;
 const ORBIT_POSE_EMIT_MS = 100;
 const ROOM_STATE_TICK_MS = 100;
 const LOOP_PAD_POLL_MS = 250;
-const MAX_ACTIVE_PLAYS = 4;
+const ORBIT_SINE_POLL_MS = 250;
+const ORBIT_SINE_VOLUME = 0.25;
+export const MAX_ACTIVE_PLAYS = 12;
 const SHOWCASE_ROOM_GROUP_ID = "showcase-room";
 
 type LoopPadState = {
@@ -73,6 +82,8 @@ type SessionState = {
   playIds: Set<string>;
   loopPads: Map<string, LoopPadState>;
   loopPollTimers: Map<string, ReturnType<typeof setInterval>>;
+  orbitSinePlayId?: string;
+  orbitSinePollTimer?: ReturnType<typeof setInterval>;
   inProximityRoom: boolean;
   lastOrbitPoseEmitMs?: number;
 };
@@ -95,11 +106,15 @@ function ensureShowcaseRoomGroup(): Promise<boolean> {
   return showcaseRoomGroupReady;
 }
 
-function ack(
-  sessionId: string,
-  action: string,
-  extra?: { playId?: string },
-): void {
+type PadAckExtra = {
+  playId?: string;
+  clipId?: ShowcaseClipId;
+  x?: number;
+  z?: number;
+  placement?: PlayOptions["placement"];
+};
+
+function ack(sessionId: string, action: string, extra?: PadAckExtra): void {
   sendToClient(sessionId, {
     type: "showcase_ack",
     action,
@@ -108,12 +123,18 @@ function ack(
   });
 }
 
-function ackError(sessionId: string, action: string, error: string): void {
+function ackError(
+  sessionId: string,
+  action: string,
+  error: string,
+  extra?: { clipId?: ShowcaseClipId },
+): void {
   sendToClient(sessionId, {
     type: "showcase_ack",
     action,
     ok: false,
     error,
+    ...(extra?.clipId !== undefined ? { clipId: extra.clipId } : {}),
   });
 }
 
@@ -165,6 +186,19 @@ function clearRoomStateTimer(state: SessionState): void {
   }
 }
 
+function stopLoopPadsForClip(
+  sessionId: string,
+  state: SessionState,
+  clipId: ShowcaseClipId,
+): void {
+  for (const [playId, pad] of [...state.loopPads.entries()]) {
+    if (pad.clipId === clipId) {
+      void stopPlay(playId);
+      clearLoopPad(sessionId, state, playId);
+    }
+  }
+}
+
 function clearLoopPad(
   sessionId: string,
   state: SessionState,
@@ -194,6 +228,87 @@ function clearAllPlays(sessionId: string, state: SessionState): void {
   state.loopPollTimers.clear();
 }
 
+function currentOrbitPose(state: SessionState) {
+  if (!state.orbitStartMs) {
+    return orbitTtsPose(0, state.orbit);
+  }
+  const elapsedSec = (Date.now() - state.orbitStartMs) / 1000;
+  return orbitTtsPose(elapsedSec, state.orbit);
+}
+
+function clearOrbitSinePoll(state: SessionState): void {
+  if (state.orbitSinePollTimer) {
+    clearInterval(state.orbitSinePollTimer);
+    state.orbitSinePollTimer = undefined;
+  }
+}
+
+async function stopOrbitSine(state: SessionState): Promise<void> {
+  clearOrbitSinePoll(state);
+  const playId = state.orbitSinePlayId;
+  state.orbitSinePlayId = undefined;
+  if (playId) {
+    await stopPlay(playId);
+  }
+}
+
+function startOrbitSineLoopPoll(sessionId: string, state: SessionState): void {
+  clearOrbitSinePoll(state);
+  const playId = state.orbitSinePlayId;
+  if (!playId) {
+    return;
+  }
+
+  state.orbitSinePollTimer = setInterval(() => {
+    void (async () => {
+      if (!state.orbitSinePlayId) {
+        clearOrbitSinePoll(state);
+        return;
+      }
+      const status = await getPlay(state.orbitSinePlayId);
+      if (!status.ok) {
+        return;
+      }
+      if (status.status === "ended") {
+        const replay = await play({
+          url: ORBIT_SINE_INLINE_URL,
+          bytes: buildOrbitSineInlineClipBase64(),
+          sessionIds: [sessionId],
+          volume: ORBIT_SINE_VOLUME,
+          pose: currentOrbitPose(state),
+        });
+        if (replay.ok && replay.playId) {
+          state.orbitSinePlayId = replay.playId;
+        }
+      }
+    })();
+  }, ORBIT_SINE_POLL_MS);
+}
+
+async function startOrbitSinePlay(
+  sessionId: string,
+  state: SessionState,
+): Promise<void> {
+  await stopOrbitSine(state);
+  const result = await play({
+    url: ORBIT_SINE_INLINE_URL,
+    bytes: buildOrbitSineInlineClipBase64(),
+    sessionIds: [sessionId],
+    volume: ORBIT_SINE_VOLUME,
+    pose: currentOrbitPose(state),
+  });
+  if (!result.ok || !result.playId) {
+    agentLog(
+      "warn",
+      `spatial-showcase orbit sine play failed: ${result.reason ?? "play_failed"}`,
+      sessionId,
+    );
+    return;
+  }
+  state.orbitSinePlayId = result.playId;
+  startOrbitSineLoopPoll(sessionId, state);
+}
+
 function emitOrbitPose(sessionId: string, state: SessionState): void {
   if (!state.orbitStartMs) {
     return;
@@ -221,13 +336,19 @@ function emitOrbitPose(sessionId: string, state: SessionState): void {
 function startOrbitDemo(sessionId: string, state: SessionState): void {
   stopOrbitInterval(state);
   beginOrbitClock(state);
+  void startOrbitSinePlay(sessionId, state);
   state.orbitTimer = setInterval(() => {
     if (state.orbit.paused || !state.orbitStartMs) {
       return;
     }
-    const elapsedSec = (Date.now() - state.orbitStartMs) / 1000;
+    const pose = currentOrbitPose(state);
+    if (state.orbitSinePlayId) {
+      void safeMixCall(sessionId, "setPlayPose", () =>
+        setPlayPose(state.orbitSinePlayId!, pose),
+      );
+    }
     void safeMixCall(sessionId, "setTtsPose", () =>
-      setTtsPose(sessionId, orbitTtsPose(elapsedSec, state.orbit)),
+      setTtsPose(sessionId, pose),
     );
     emitOrbitPose(sessionId, state);
   }, ORBIT_TICK_MS);
@@ -312,6 +433,15 @@ async function startLoopPadPoll(
           clearLoopPad(sessionId, state, playId);
           state.playIds.add(replay.playId);
           state.loopPads.set(replay.playId, pad);
+          ack(sessionId, "pad", {
+            playId: replay.playId,
+            clipId: pad.clipId,
+            x: pad.x,
+            z: pad.z,
+            ...(pad.placement !== undefined
+              ? { placement: pad.placement }
+              : {}),
+          });
           void startLoopPadPoll(sessionId, state, replay.playId, pad);
         }
       } else if (status.status) {
@@ -336,24 +466,27 @@ export async function handlePadPlay(
   volume?: number,
   placement?: PlayOptions["placement"],
 ): Promise<void> {
-  if (state.playIds.size >= MAX_ACTIVE_PLAYS) {
-    ackError(sessionId, "pad", "busy");
-    return;
-  }
-
   const assetOrigin = state.assetOrigin;
   if (!assetOrigin) {
-    ackError(sessionId, "pad", "missing_asset_origin");
+    ackError(sessionId, "pad", "missing_asset_origin", { clipId });
     return;
   }
 
   const url = resolveClipUrl(assetOrigin, clipId);
   if (!url) {
-    ackError(sessionId, "pad", "invalid_asset_origin");
+    ackError(sessionId, "pad", "invalid_asset_origin", { clipId });
     return;
   }
 
   const sound = SHOWCASE_SOUND_FILES[clipId];
+  if (sound.loop) {
+    stopLoopPadsForClip(sessionId, state, clipId);
+  }
+
+  if (state.playIds.size >= MAX_ACTIVE_PLAYS) {
+    ackError(sessionId, "pad", "busy", { clipId });
+    return;
+  }
   const playVolume = volume ?? sound.defaultVolume;
   const options: PlayOptions = {
     url,
@@ -364,12 +497,18 @@ export async function handlePadPlay(
 
   const result = await play(options);
   if (!result.ok || !result.playId) {
-    ackError(sessionId, "pad", result.reason ?? "play_failed");
+    ackError(sessionId, "pad", result.reason ?? "play_failed", { clipId });
     return;
   }
 
   state.playIds.add(result.playId);
-  ack(sessionId, "pad", { playId: result.playId });
+  ack(sessionId, "pad", {
+    playId: result.playId,
+    clipId,
+    x,
+    z,
+    ...(placement !== undefined ? { placement } : {}),
+  });
 
   if (sound.loop) {
     const pad: LoopPadState = {
@@ -406,9 +545,13 @@ async function handleJoin(
   ack(sessionId, "join");
 
   switch (demo) {
-    case "orbit":
+    case "orbit": {
+      void safeMixCall(sessionId, "setSttEnabled", () =>
+        setSttEnabled({ enabled: false, sessionId }),
+      );
       startOrbitDemo(sessionId, state);
       break;
+    }
     case "soundboard":
       break;
     case "proximity": {
@@ -451,8 +594,12 @@ function teardownSession(sessionId: string): void {
 
   resetOrbitSession(state);
   clearRoomStateTimer(state);
+  void stopOrbitSine(state);
   clearAllPlays(sessionId, state);
   void safeMixCall(sessionId, "clearTtsPose", () => clearTtsPose(sessionId));
+  void safeMixCall(sessionId, "setSttEnabled", () =>
+    setSttEnabled({ enabled: true, sessionId }),
+  );
 
   if (state.inProximityRoom) {
     proximityRoom.leave(sessionId);
@@ -524,6 +671,10 @@ async function dispatchMessage(
         ackError(sessionId, "pad", "wrong_demo");
         return;
       }
+      if (!isShowcaseClipId(message.clipId)) {
+        ackError(sessionId, "pad", "invalid_clip_id");
+        return;
+      }
       await handlePadPlay(
         sessionId,
         state,
@@ -641,14 +792,6 @@ defineAgent({
     });
   },
 
-  onUserSpeechFinal({ sessionId, text }) {
-    const state = sessions.get(sessionId);
-    if (!state || state.demo !== "orbit") {
-      return;
-    }
-    speak(sessionId, `You said: ${text}`);
-  },
-
   onDataChannelMessage(ctx) {
     const message = parseShowcaseMessage(ctx.message);
     if (!message) {

package/templates/spatial-showcase/protocol.ts

@@ -164,7 +164,9 @@ export function isShowcaseDemo(value: unknown): value is ShowcaseDemo {
   return typeof value === "string" && SHOWCASE_DEMOS.has(value);
 }
 
-export function isShowcaseInbound(message: unknown): message is ShowcaseInbound {
+export function isShowcaseInbound(
+  message: unknown,
+): message is ShowcaseInbound {
   return parseShowcaseMessage(message) !== null;
 }
 

package/templates/spatial-showcase/sine.ts

@@ -0,0 +1,43 @@
+/** Inline orbit sine clip — 16 kHz mono s16le, decoded size under 64 KiB play cap. */
+
+/** 2 s keeps loop-restart gaps (250 ms poll) infrequent; 16 kHz mono still fits under 64 KiB. */
+export const ORBIT_SINE_DURATION_MS = 2000;
+export const ORBIT_SINE_FREQUENCY_HZ = 440;
+export const ORBIT_SINE_AMPLITUDE = 8000;
+export const ORBIT_SINE_INLINE_URL =
+  "https://example.com/inline-orbit-sine.wav";
+
+/** Looping ~440 Hz sine WAV as base64 for {@link play} inline bytes. */
+export function buildOrbitSineInlineClipBase64(
+  durationMs = ORBIT_SINE_DURATION_MS,
+  frequencyHz = ORBIT_SINE_FREQUENCY_HZ,
+  amplitude = ORBIT_SINE_AMPLITUDE,
+): string {
+  const sampleRate = 16_000;
+  const channels = 1;
+  const bytesPerSample = 2;
+  const numSamples = Math.floor((sampleRate * durationMs) / 1000);
+  const dataSize = numSamples * channels * bytesPerSample;
+  const buffer = Buffer.alloc(44 + dataSize);
+  buffer.write("RIFF", 0);
+  buffer.writeUInt32LE(36 + dataSize, 4);
+  buffer.write("WAVE", 8);
+  buffer.write("fmt ", 12);
+  buffer.writeUInt32LE(16, 16);
+  buffer.writeUInt16LE(1, 20);
+  buffer.writeUInt16LE(channels, 22);
+  buffer.writeUInt32LE(sampleRate, 24);
+  buffer.writeUInt32LE(sampleRate * channels * bytesPerSample, 28);
+  buffer.writeUInt16LE(channels * bytesPerSample, 32);
+  buffer.writeUInt16LE(16, 34);
+  buffer.write("data", 36);
+  buffer.writeUInt32LE(dataSize, 40);
+  for (let i = 0; i < numSamples; i++) {
+    const t = i / sampleRate;
+    const sample = Math.round(
+      amplitude * Math.sin(2 * Math.PI * frequencyHz * t),
+    );
+    buffer.writeInt16LE(sample, 44 + i * 2);
+  }
+  return buffer.toString("base64");
+}