@voicethere/agent 0.7.11 → 0.7.13

package/templates/spatial-showcase/agent.ts

@@ -1,5 +1,5 @@
 /**
- * Spatial audio showcase — orbit TTS, positional soundboard, proximity room.
+ * Spatial audio showcase — orbiting sine + TTS, positional soundboard, proximity room.
  *
  * Browser joins with `{ type: "join", demo, assetOrigin? }` then sends demo-specific
  * commands. Clip URLs are resolved server-side from clipId + allowlisted assetOrigin.
@@ -21,7 +21,9 @@ import {
   sendToClient,
   setClientPose,
   setListenerMute,
+  setPlayPose,
   setPositionalMixing,
+  setSttEnabled,
   setTtsPose,
   speak,
   stopPlay,
@@ -42,16 +44,26 @@ import {
 } from "./protocol.js";
 import { ProximityRoom } from "./room.js";
 import {
+  isShowcaseClipId,
   resolveClipUrl,
   SHOWCASE_SOUND_FILES,
   type ShowcaseClipId,
 } from "./sounds.js";
+import {
+  buildOrbitSineInlineClipBase64,
+  ORBIT_SINE_DURATION_MS,
+  ORBIT_SINE_INLINE_URL,
+  orbitSineClipDurationMs,
+} from "./sine.js";
 
 const ORBIT_TICK_MS = 50;
 const ORBIT_POSE_EMIT_MS = 100;
 const ROOM_STATE_TICK_MS = 100;
 const LOOP_PAD_POLL_MS = 250;
-const MAX_ACTIVE_PLAYS = 4;
+const ORBIT_SINE_POLL_MS = 250;
+const ORBIT_SINE_GAPLESS_LEAD_MS = 200;
+const ORBIT_SINE_VOLUME = 0.25;
+export const MAX_ACTIVE_PLAYS = 12;
 const SHOWCASE_ROOM_GROUP_ID = "showcase-room";
 
 type LoopPadState = {
@@ -73,6 +85,13 @@ type SessionState = {
   playIds: Set<string>;
   loopPads: Map<string, LoopPadState>;
   loopPollTimers: Map<string, ReturnType<typeof setInterval>>;
+  orbitSinePlayId?: string;
+  orbitSinePollTimer?: ReturnType<typeof setInterval>;
+  orbitSineRestartTimer?: ReturnType<typeof setTimeout>;
+  orbitSineFrequencyHz: number;
+  orbitSineEnabled: boolean;
+  orbitSinePhaseRad: number;
+  orbitManualPose?: { x: number; z: number };
   inProximityRoom: boolean;
   lastOrbitPoseEmitMs?: number;
 };
@@ -95,11 +114,15 @@ function ensureShowcaseRoomGroup(): Promise<boolean> {
   return showcaseRoomGroupReady;
 }
 
-function ack(
-  sessionId: string,
-  action: string,
-  extra?: { playId?: string },
-): void {
+type PadAckExtra = {
+  playId?: string;
+  clipId?: ShowcaseClipId;
+  x?: number;
+  z?: number;
+  placement?: PlayOptions["placement"];
+};
+
+function ack(sessionId: string, action: string, extra?: PadAckExtra): void {
   sendToClient(sessionId, {
     type: "showcase_ack",
     action,
@@ -108,12 +131,18 @@ function ack(
   });
 }
 
-function ackError(sessionId: string, action: string, error: string): void {
+function ackError(
+  sessionId: string,
+  action: string,
+  error: string,
+  extra?: { clipId?: ShowcaseClipId },
+): void {
   sendToClient(sessionId, {
     type: "showcase_ack",
     action,
     ok: false,
     error,
+    ...(extra?.clipId !== undefined ? { clipId: extra.clipId } : {}),
   });
 }
 
@@ -154,6 +183,9 @@ function defaultSessionState(
     playIds: new Set(),
     loopPads: new Map(),
     loopPollTimers: new Map(),
+    orbitSineFrequencyHz: 440,
+    orbitSineEnabled: true,
+    orbitSinePhaseRad: 0,
     inProximityRoom: false,
   };
 }
@@ -165,6 +197,19 @@ function clearRoomStateTimer(state: SessionState): void {
   }
 }
 
+function stopLoopPadsForClip(
+  sessionId: string,
+  state: SessionState,
+  clipId: ShowcaseClipId,
+): void {
+  for (const [playId, pad] of [...state.loopPads.entries()]) {
+    if (pad.clipId === clipId) {
+      void stopPlay(playId);
+      clearLoopPad(sessionId, state, playId);
+    }
+  }
+}
+
 function clearLoopPad(
   sessionId: string,
   state: SessionState,
@@ -194,8 +239,188 @@ function clearAllPlays(sessionId: string, state: SessionState): void {
   state.loopPollTimers.clear();
 }
 
-function emitOrbitPose(sessionId: string, state: SessionState): void {
+function currentOrbitPose(state: SessionState) {
+  if (state.orbitManualPose) {
+    const elevation = state.orbit.elevation ?? 0;
+    return {
+      position: {
+        x: state.orbitManualPose.x,
+        y: elevation,
+        z: state.orbitManualPose.z,
+      },
+      orientation: { x: 0, y: 0, z: 0, w: 1 },
+    };
+  }
   if (!state.orbitStartMs) {
+    return orbitTtsPose(0, state.orbit);
+  }
+  const elapsedSec = (Date.now() - state.orbitStartMs) / 1000;
+  return orbitTtsPose(elapsedSec, state.orbit);
+}
+
+function orbitPoseFromXz(
+  state: SessionState,
+  x: number,
+  z: number,
+): ReturnType<typeof orbitTtsPose> {
+  const elevation = state.orbit.elevation ?? 0;
+  return {
+    position: { x, y: elevation, z },
+    orientation: { x: 0, y: 0, z: 0, w: 1 },
+  };
+}
+
+function syncOrbitClockFromManualPose(state: SessionState): void {
+  const manual = state.orbitManualPose;
+  if (!manual) {
+    return;
+  }
+  const periodSec = state.orbit.periodSec ?? 2 * Math.PI;
+  const angleRad = Math.atan2(manual.z, manual.x);
+  const elapsedSec = (angleRad * periodSec) / (2 * Math.PI);
+  state.orbitStartMs = Date.now() - elapsedSec * 1000;
+  state.orbitManualPose = undefined;
+}
+
+function clearOrbitSinePoll(state: SessionState): void {
+  if (state.orbitSinePollTimer) {
+    clearInterval(state.orbitSinePollTimer);
+    state.orbitSinePollTimer = undefined;
+  }
+}
+
+function clearOrbitSineRestartTimer(state: SessionState): void {
+  if (state.orbitSineRestartTimer) {
+    clearTimeout(state.orbitSineRestartTimer);
+    state.orbitSineRestartTimer = undefined;
+  }
+}
+
+async function stopOrbitSine(state: SessionState): Promise<void> {
+  clearOrbitSinePoll(state);
+  clearOrbitSineRestartTimer(state);
+  const playId = state.orbitSinePlayId;
+  state.orbitSinePlayId = undefined;
+  if (playId) {
+    await stopPlay(playId);
+  }
+}
+
+function orbitSineClipBytes(state: SessionState): string {
+  return buildOrbitSineInlineClipBase64(
+    ORBIT_SINE_DURATION_MS,
+    state.orbitSineFrequencyHz,
+    undefined,
+    state.orbitSinePhaseRad,
+  );
+}
+
+function orbitSineClipDuration(state: SessionState): number {
+  return orbitSineClipDurationMs(
+    ORBIT_SINE_DURATION_MS,
+    state.orbitSineFrequencyHz,
+  );
+}
+
+function scheduleOrbitSineGaplessRestart(
+  sessionId: string,
+  state: SessionState,
+): void {
+  clearOrbitSineRestartTimer(state);
+  const durationMs = orbitSineClipDuration(state);
+  const delayMs = Math.max(0, durationMs - ORBIT_SINE_GAPLESS_LEAD_MS);
+  state.orbitSineRestartTimer = setTimeout(() => {
+    state.orbitSineRestartTimer = undefined;
+    void restartOrbitSineGapless(sessionId, state);
+  }, delayMs);
+}
+
+async function restartOrbitSineGapless(
+  sessionId: string,
+  state: SessionState,
+): Promise<void> {
+  if (!state.orbitSineEnabled || !state.orbitSinePlayId) {
+    return;
+  }
+  const previousPlayId = state.orbitSinePlayId;
+  const result = await play({
+    url: ORBIT_SINE_INLINE_URL,
+    bytes: orbitSineClipBytes(state),
+    sessionIds: [sessionId],
+    volume: ORBIT_SINE_VOLUME,
+    pose: currentOrbitPose(state),
+  });
+  if (!result.ok || !result.playId) {
+    agentLog(
+      "warn",
+      `spatial-showcase orbit sine gapless restart failed: ${result.reason ?? "play_failed"}`,
+      sessionId,
+    );
+    startOrbitSineLoopPoll(sessionId, state);
+    return;
+  }
+  state.orbitSinePlayId = result.playId;
+  void stopPlay(previousPlayId);
+  scheduleOrbitSineGaplessRestart(sessionId, state);
+}
+
+function startOrbitSineLoopPoll(sessionId: string, state: SessionState): void {
+  if (state.orbitSineRestartTimer) {
+    return;
+  }
+  clearOrbitSinePoll(state);
+  const playId = state.orbitSinePlayId;
+  if (!playId) {
+    return;
+  }
+
+  state.orbitSinePollTimer = setInterval(() => {
+    void (async () => {
+      if (!state.orbitSinePlayId || state.orbitSineRestartTimer) {
+        clearOrbitSinePoll(state);
+        return;
+      }
+      const status = await getPlay(state.orbitSinePlayId);
+      if (!status.ok) {
+        return;
+      }
+      if (status.status === "ended") {
+        await restartOrbitSineGapless(sessionId, state);
+      }
+    })();
+  }, ORBIT_SINE_POLL_MS);
+}
+
+async function startOrbitSinePlay(
+  sessionId: string,
+  state: SessionState,
+): Promise<void> {
+  if (!state.orbitSineEnabled) {
+    return;
+  }
+  await stopOrbitSine(state);
+  const result = await play({
+    url: ORBIT_SINE_INLINE_URL,
+    bytes: orbitSineClipBytes(state),
+    sessionIds: [sessionId],
+    volume: ORBIT_SINE_VOLUME,
+    pose: currentOrbitPose(state),
+  });
+  if (!result.ok || !result.playId) {
+    agentLog(
+      "warn",
+      `spatial-showcase orbit sine play failed: ${result.reason ?? "play_failed"}`,
+      sessionId,
+    );
+    return;
+  }
+  state.orbitSinePlayId = result.playId;
+  scheduleOrbitSineGaplessRestart(sessionId, state);
+  startOrbitSineLoopPoll(sessionId, state);
+}
+
+function emitOrbitPose(sessionId: string, state: SessionState): void {
+  if (!state.orbitStartMs && !state.orbitManualPose) {
     return;
   }
   const now = Date.now();
@@ -206,10 +431,12 @@ function emitOrbitPose(sessionId: string, state: SessionState): void {
     return;
   }
   state.lastOrbitPoseEmitMs = now;
-  const elapsedSec = (now - state.orbitStartMs) / 1000;
-  const pose = orbitTtsPose(elapsedSec, state.orbit);
-  const angleRad =
-    (2 * Math.PI * elapsedSec) / (state.orbit.periodSec ?? 2 * Math.PI);
+  const pose = currentOrbitPose(state);
+  const angleRad = state.orbitManualPose
+    ? Math.atan2(state.orbitManualPose.z, state.orbitManualPose.x)
+    : (2 * Math.PI * (now - state.orbitStartMs!)) /
+      1000 /
+      (state.orbit.periodSec ?? 2 * Math.PI);
   sendToClient(sessionId, {
     type: "orbit_pose",
     angleDeg: (angleRad * 180) / Math.PI,
@@ -218,16 +445,38 @@ function emitOrbitPose(sessionId: string, state: SessionState): void {
   });
 }
 
+function emitOrbitPlacePose(
+  sessionId: string,
+  state: SessionState,
+  x: number,
+  z: number,
+): void {
+  state.lastOrbitPoseEmitMs = Date.now();
+  const angleRad = Math.atan2(z, x);
+  sendToClient(sessionId, {
+    type: "orbit_pose",
+    angleDeg: (angleRad * 180) / Math.PI,
+    x,
+    z,
+  });
+}
+
 function startOrbitDemo(sessionId: string, state: SessionState): void {
   stopOrbitInterval(state);
   beginOrbitClock(state);
+  void startOrbitSinePlay(sessionId, state);
   state.orbitTimer = setInterval(() => {
-    if (state.orbit.paused || !state.orbitStartMs) {
+    if (state.orbit.paused || (!state.orbitStartMs && !state.orbitManualPose)) {
       return;
     }
-    const elapsedSec = (Date.now() - state.orbitStartMs) / 1000;
+    const pose = currentOrbitPose(state);
+    if (state.orbitSinePlayId) {
+      void safeMixCall(sessionId, "setPlayPose", () =>
+        setPlayPose(state.orbitSinePlayId!, pose),
+      );
+    }
     void safeMixCall(sessionId, "setTtsPose", () =>
-      setTtsPose(sessionId, orbitTtsPose(elapsedSec, state.orbit)),
+      setTtsPose(sessionId, pose),
     );
     emitOrbitPose(sessionId, state);
   }, ORBIT_TICK_MS);
@@ -312,6 +561,15 @@ async function startLoopPadPoll(
           clearLoopPad(sessionId, state, playId);
           state.playIds.add(replay.playId);
           state.loopPads.set(replay.playId, pad);
+          ack(sessionId, "pad", {
+            playId: replay.playId,
+            clipId: pad.clipId,
+            x: pad.x,
+            z: pad.z,
+            ...(pad.placement !== undefined
+              ? { placement: pad.placement }
+              : {}),
+          });
           void startLoopPadPoll(sessionId, state, replay.playId, pad);
         }
       } else if (status.status) {
@@ -336,24 +594,27 @@ export async function handlePadPlay(
   volume?: number,
   placement?: PlayOptions["placement"],
 ): Promise<void> {
-  if (state.playIds.size >= MAX_ACTIVE_PLAYS) {
-    ackError(sessionId, "pad", "busy");
-    return;
-  }
-
   const assetOrigin = state.assetOrigin;
   if (!assetOrigin) {
-    ackError(sessionId, "pad", "missing_asset_origin");
+    ackError(sessionId, "pad", "missing_asset_origin", { clipId });
     return;
   }
 
   const url = resolveClipUrl(assetOrigin, clipId);
   if (!url) {
-    ackError(sessionId, "pad", "invalid_asset_origin");
+    ackError(sessionId, "pad", "invalid_asset_origin", { clipId });
     return;
   }
 
   const sound = SHOWCASE_SOUND_FILES[clipId];
+  if (sound.loop) {
+    stopLoopPadsForClip(sessionId, state, clipId);
+  }
+
+  if (state.playIds.size >= MAX_ACTIVE_PLAYS) {
+    ackError(sessionId, "pad", "busy", { clipId });
+    return;
+  }
   const playVolume = volume ?? sound.defaultVolume;
   const options: PlayOptions = {
     url,
@@ -364,12 +625,18 @@ export async function handlePadPlay(
 
   const result = await play(options);
   if (!result.ok || !result.playId) {
-    ackError(sessionId, "pad", result.reason ?? "play_failed");
+    ackError(sessionId, "pad", result.reason ?? "play_failed", { clipId });
     return;
   }
 
   state.playIds.add(result.playId);
-  ack(sessionId, "pad", { playId: result.playId });
+  ack(sessionId, "pad", {
+    playId: result.playId,
+    clipId,
+    x,
+    z,
+    ...(placement !== undefined ? { placement } : {}),
+  });
 
   if (sound.loop) {
     const pad: LoopPadState = {
@@ -385,6 +652,34 @@ export async function handlePadPlay(
   }
 }
 
+export async function handlePadMove(
+  sessionId: string,
+  state: SessionState,
+  playId: string,
+  x: number,
+  z: number,
+): Promise<void> {
+  if (!state.playIds.has(playId)) {
+    ackError(sessionId, "pad_move", "unknown_play");
+    return;
+  }
+
+  const poseResult = await setPlayPose(playId, poseAt(x, z));
+  if (!poseResult.ok) {
+    ackError(sessionId, "pad_move", poseResult.reason ?? "set_pose_failed");
+    return;
+  }
+
+  const loopPad = state.loopPads.get(playId);
+  if (loopPad) {
+    loopPad.x = x;
+    loopPad.z = z;
+    delete loopPad.placement;
+  }
+
+  ack(sessionId, "pad_move", { playId, x, z });
+}
+
 async function handleJoin(
   sessionId: string,
   demo: ShowcaseDemo,
@@ -406,9 +701,13 @@ async function handleJoin(
   ack(sessionId, "join");
 
   switch (demo) {
-    case "orbit":
+    case "orbit": {
+      void safeMixCall(sessionId, "setSttEnabled", () =>
+        setSttEnabled({ enabled: false, sessionId }),
+      );
       startOrbitDemo(sessionId, state);
       break;
+    }
     case "soundboard":
       break;
     case "proximity": {
@@ -451,8 +750,12 @@ function teardownSession(sessionId: string): void {
 
   resetOrbitSession(state);
   clearRoomStateTimer(state);
+  void stopOrbitSine(state);
   clearAllPlays(sessionId, state);
   void safeMixCall(sessionId, "clearTtsPose", () => clearTtsPose(sessionId));
+  void safeMixCall(sessionId, "setSttEnabled", () =>
+    setSttEnabled({ enabled: true, sessionId }),
+  );
 
   if (state.inProximityRoom) {
     proximityRoom.leave(sessionId);
@@ -510,11 +813,50 @@ async function dispatchMessage(
           state.orbit.elevation = message.elevation;
         }
         if (message.paused !== undefined) {
+          if (message.paused === false && state.orbitManualPose) {
+            syncOrbitClockFromManualPose(state);
+          }
           state.orbit.paused = message.paused;
         }
+        if (message.frequencyHz !== undefined) {
+          state.orbitSineFrequencyHz = message.frequencyHz;
+          if (state.orbitSineEnabled && state.orbitSinePlayId) {
+            void startOrbitSinePlay(sessionId, state);
+          }
+        }
+        if (message.sinePlaying !== undefined) {
+          state.orbitSineEnabled = message.sinePlaying;
+          if (message.sinePlaying) {
+            if (!state.orbitSinePlayId) {
+              void startOrbitSinePlay(sessionId, state);
+            }
+          } else {
+            void stopOrbitSine(state);
+          }
+        }
         ack(sessionId, "orbit_set");
         return;
       }
+      if (message.action === "place") {
+        state.orbit.paused = true;
+        state.orbitManualPose = { x: message.x, z: message.z };
+        const pose = orbitPoseFromXz(state, message.x, message.z);
+        if (state.orbitSinePlayId) {
+          void safeMixCall(sessionId, "setPlayPose", () =>
+            setPlayPose(state.orbitSinePlayId!, pose),
+          );
+        }
+        void safeMixCall(sessionId, "setTtsPose", () =>
+          setTtsPose(sessionId, pose),
+        );
+        emitOrbitPlacePose(sessionId, state, message.x, message.z);
+        ack(sessionId, "orbit_place");
+        return;
+      }
+      sendToClient(sessionId, {
+        type: "orbit_say_status",
+        status: "playing",
+      });
       speak(sessionId, message.text);
       ack(sessionId, "orbit_say");
       return;
@@ -524,6 +866,10 @@ async function dispatchMessage(
         ackError(sessionId, "pad", "wrong_demo");
         return;
       }
+      if (!isShowcaseClipId(message.clipId)) {
+        ackError(sessionId, "pad", "invalid_clip_id");
+        return;
+      }
       await handlePadPlay(
         sessionId,
         state,
@@ -567,6 +913,20 @@ async function dispatchMessage(
       }
       return;
     }
+    case "pad_move": {
+      if (state.demo !== "soundboard") {
+        ackError(sessionId, "pad_move", "wrong_demo");
+        return;
+      }
+      await handlePadMove(
+        sessionId,
+        state,
+        message.playId,
+        message.x,
+        message.z,
+      );
+      return;
+    }
     case "move": {
       if (state.demo !== "proximity") {
         ackError(sessionId, "move", "wrong_demo");
@@ -641,14 +1001,6 @@ defineAgent({
     });
   },
 
-  onUserSpeechFinal({ sessionId, text }) {
-    const state = sessions.get(sessionId);
-    if (!state || state.demo !== "orbit") {
-      return;
-    }
-    speak(sessionId, `You said: ${text}`);
-  },
-
   onDataChannelMessage(ctx) {
     const message = parseShowcaseMessage(ctx.message);
     if (!message) {
@@ -657,6 +1009,24 @@ defineAgent({
     void dispatchMessage(ctx.sessionId, message);
   },
 
+  onSpeechEvent(ctx, event) {
+    const state = sessions.get(ctx.sessionId);
+    if (!state || state.demo !== "orbit") {
+      return;
+    }
+    if (event.type === "agent_speaking_start") {
+      sendToClient(ctx.sessionId, {
+        type: "orbit_say_status",
+        status: "playing",
+      });
+    } else if (event.type === "agent_speaking_end") {
+      sendToClient(ctx.sessionId, {
+        type: "orbit_say_status",
+        status: "ended",
+      });
+    }
+  },
+
   onSessionEnd({ sessionId }) {
     teardownSession(sessionId);
     agentLog("info", `spatial-showcase session_end ${sessionId}`, sessionId);