@voicethere/agent 0.7.12 → 0.8.0

package/templates/redis-sync/world-layout.ts

@@ -18,8 +18,37 @@ export const PEER_FIELD_X = 1;
 export const PEER_FIELD_Y = 2;
 export const PEER_FIELD_ACTIVE = 3;
 
+/** Client → agent position frame: float32le [clientIndex, x, y] (12 bytes). */
+export const POSITION_FLOAT_COUNT = 3;
+export const POSITION_BYTE_LENGTH = POSITION_FLOAT_COUNT * 4;
+
+export type PeerPosition = {
+  clientIndex: number;
+  x: number;
+  y: number;
+};
+
 export const REDIS_WORLD_KEY = "e2e:redis-sync:world";
 
+/** Atomic splice of one peer slot (16 bytes) into the world blob. */
+export const LUA_PATCH_PEER_SLOT = `
+local key = KEYS[1]
+local offset = tonumber(ARGV[1])
+local slot = ARGV[2]
+local size = tonumber(ARGV[3])
+local world = redis.call('GET', key)
+if not world then
+  world = string.rep(string.char(0), size)
+elseif #world < size then
+  world = world .. string.rep(string.char(0), size - #world)
+elseif #world > size then
+  world = string.sub(world, 1, size)
+end
+world = string.sub(world, 1, offset) .. slot .. string.sub(world, offset + #slot + 1)
+redis.call('SET', key, world)
+return size
+`;
+
 export type PeerSlot = {
   clientIndex: number;
   x: number;
@@ -53,6 +82,51 @@ export function clearPeerSlot(world: Float32Array, clientIndex: number): void {
   writePeerSlot(world, clientIndex, 0, 0, 0);
 }
 
+function asDataView(data: ArrayBufferLike | ArrayBufferView): DataView {
+  if (ArrayBuffer.isView(data)) {
+    return new DataView(data.buffer, data.byteOffset, data.byteLength);
+  }
+  return new DataView(data);
+}
+
+/** Decode a 12-byte inbound position frame. */
+export function decodePositionBuffer(
+  data: ArrayBufferLike | ArrayBufferView | null | undefined,
+): PeerPosition | null {
+  if (data == null) {
+    return null;
+  }
+  const view = asDataView(data);
+  if (view.byteLength < POSITION_BYTE_LENGTH) {
+    return null;
+  }
+  const clientIndex = view.getFloat32(0, true);
+  const x = view.getFloat32(4, true);
+  const y = view.getFloat32(8, true);
+  if (
+    !Number.isFinite(clientIndex) ||
+    clientIndex < 0 ||
+    !Number.isFinite(x) ||
+    !Number.isFinite(y)
+  ) {
+    return null;
+  }
+  return { clientIndex: Math.trunc(clientIndex), x, y };
+}
+
+/** Write [clientIndex, x, y] into `dest` (length >= 3) and return it. */
+export function encodePositionInto(
+  dest: Float32Array,
+  clientIndex: number,
+  x: number,
+  y: number,
+): Float32Array {
+  dest[0] = clientIndex;
+  dest[1] = x;
+  dest[2] = y;
+  return dest;
+}
+
 export function readPeerSlot(
   world: Float32Array,
   clientIndex: number,
@@ -71,46 +145,49 @@ export function readPeerSlot(
 }
 
 /**
- * Decode a Redis / Node Buffer (or any Uint8Array) into a world Float32Array.
+ * Copy Redis / Node Buffer bytes into `dest` (default: a new world).
  *
- * Node Buffer pools often hand out views whose `byteOffset` is not a multiple of
- * 4. `new Float32Array(buf.buffer, buf.byteOffset, …)` then throws:
- *   "start offset of Float32Array should be a multiple of 4"
- * Copy via `ArrayBuffer.slice` so the view always starts at offset 0.
+ * Node Buffer pools can have `byteOffset` not divisible by 4, so we copy via
+ * a Uint8Array view onto `dest` instead of slicing a new ArrayBuffer.
  */
 export function normalizeWorldBuffer(
   raw: Uint8Array | null | undefined,
+  dest: Float32Array = createEmptyWorldBuffer(),
 ): Float32Array {
+  const destBytes = new Uint8Array(
+    dest.buffer,
+    dest.byteOffset,
+    dest.byteLength,
+  );
   if (!raw || raw.byteLength === 0) {
-    return createEmptyWorldBuffer();
+    destBytes.fill(0);
+    return dest;
   }
-  const bytes = Math.floor(raw.byteLength / 4) * 4;
-  if (bytes === 0) {
-    return createEmptyWorldBuffer();
+  const byteCount = Math.min(
+    Math.floor(raw.byteLength / 4) * 4,
+    destBytes.byteLength,
+  );
+  if (byteCount > 0) {
+    destBytes.set(raw.subarray(0, byteCount));
   }
-  const aligned = raw.buffer.slice(raw.byteOffset, raw.byteOffset + bytes);
-  const decoded = new Float32Array(aligned);
-  if (decoded.length === WORLD_FLOAT_COUNT) {
-    return decoded;
+  if (byteCount < destBytes.byteLength) {
+    destBytes.fill(0, byteCount);
   }
-  const normalized = createEmptyWorldBuffer();
-  normalized.set(
-    decoded.subarray(0, Math.min(decoded.length, WORLD_FLOAT_COUNT)),
-  );
-  return normalized;
+  return dest;
 }
 
-export function decodeWorldBuffer(data: ArrayBuffer): Float32Array {
-  const view = new Float32Array(data);
-  if (view.length === WORLD_FLOAT_COUNT) {
-    return view;
-  }
-  const normalized = createEmptyWorldBuffer();
-  normalized.set(view.subarray(0, Math.min(view.length, WORLD_FLOAT_COUNT)));
-  return normalized;
+export function decodeWorldBuffer(
+  data: ArrayBuffer,
+  dest: Float32Array = createEmptyWorldBuffer(),
+): Float32Array {
+  return normalizeWorldBuffer(new Uint8Array(data), dest);
 }
 
+/** Same backing ArrayBuffer when the world is a standalone Float32Array. */
 export function worldBufferToArrayBuffer(world: Float32Array): ArrayBuffer {
+  if (world.byteOffset === 0 && world.byteLength === world.buffer.byteLength) {
+    return world.buffer as ArrayBuffer;
+  }
   return world.buffer.slice(
     world.byteOffset,
     world.byteOffset + world.byteLength,

package/templates/spatial-showcase/README.md

@@ -0,0 +1,13 @@
+# spatial-showcase
+
+Spatial audio showcase for `/showcase`. A `{ type: "join", demo }` DataChannel message selects `orbit`, `soundboard`, or `proximity`.
+
+## Build
+
+```bash
+npx @voicethere/agent build --entry templates/spatial-showcase/agent.ts --outfile dist/agent.js
+```
+
+## Sources
+
+- `agent.ts`, `protocol.ts`, `sounds.ts`, `room.ts`, `orbit.ts`, `orbit-session.ts`, `sine.ts`

package/templates/spatial-showcase/agent.ts

@@ -51,7 +51,9 @@ import {
 } from "./sounds.js";
 import {
   buildOrbitSineInlineClipBase64,
+  ORBIT_SINE_DURATION_MS,
   ORBIT_SINE_INLINE_URL,
+  orbitSineClipDurationMs,
 } from "./sine.js";
 
 const ORBIT_TICK_MS = 50;
@@ -59,6 +61,7 @@ const ORBIT_POSE_EMIT_MS = 100;
 const ROOM_STATE_TICK_MS = 100;
 const LOOP_PAD_POLL_MS = 250;
 const ORBIT_SINE_POLL_MS = 250;
+const ORBIT_SINE_GAPLESS_LEAD_MS = 200;
 const ORBIT_SINE_VOLUME = 0.25;
 export const MAX_ACTIVE_PLAYS = 12;
 const SHOWCASE_ROOM_GROUP_ID = "showcase-room";
@@ -84,6 +87,11 @@ type SessionState = {
   loopPollTimers: Map<string, ReturnType<typeof setInterval>>;
   orbitSinePlayId?: string;
   orbitSinePollTimer?: ReturnType<typeof setInterval>;
+  orbitSineRestartTimer?: ReturnType<typeof setTimeout>;
+  orbitSineFrequencyHz: number;
+  orbitSineEnabled: boolean;
+  orbitSinePhaseRad: number;
+  orbitManualPose?: { x: number; z: number };
   inProximityRoom: boolean;
   lastOrbitPoseEmitMs?: number;
 };
@@ -175,6 +183,9 @@ function defaultSessionState(
     playIds: new Set(),
     loopPads: new Map(),
     loopPollTimers: new Map(),
+    orbitSineFrequencyHz: 440,
+    orbitSineEnabled: true,
+    orbitSinePhaseRad: 0,
     inProximityRoom: false,
   };
 }
@@ -229,6 +240,17 @@ function clearAllPlays(sessionId: string, state: SessionState): void {
 }
 
 function currentOrbitPose(state: SessionState) {
+  if (state.orbitManualPose) {
+    const elevation = state.orbit.elevation ?? 0;
+    return {
+      position: {
+        x: state.orbitManualPose.x,
+        y: elevation,
+        z: state.orbitManualPose.z,
+      },
+      orientation: { x: 0, y: 0, z: 0, w: 1 },
+    };
+  }
   if (!state.orbitStartMs) {
     return orbitTtsPose(0, state.orbit);
   }
@@ -236,6 +258,30 @@ function currentOrbitPose(state: SessionState) {
   return orbitTtsPose(elapsedSec, state.orbit);
 }
 
+function orbitPoseFromXz(
+  state: SessionState,
+  x: number,
+  z: number,
+): ReturnType<typeof orbitTtsPose> {
+  const elevation = state.orbit.elevation ?? 0;
+  return {
+    position: { x, y: elevation, z },
+    orientation: { x: 0, y: 0, z: 0, w: 1 },
+  };
+}
+
+function syncOrbitClockFromManualPose(state: SessionState): void {
+  const manual = state.orbitManualPose;
+  if (!manual) {
+    return;
+  }
+  const periodSec = state.orbit.periodSec ?? 2 * Math.PI;
+  const angleRad = Math.atan2(manual.z, manual.x);
+  const elapsedSec = (angleRad * periodSec) / (2 * Math.PI);
+  state.orbitStartMs = Date.now() - elapsedSec * 1000;
+  state.orbitManualPose = undefined;
+}
+
 function clearOrbitSinePoll(state: SessionState): void {
   if (state.orbitSinePollTimer) {
     clearInterval(state.orbitSinePollTimer);
@@ -243,8 +289,16 @@ function clearOrbitSinePoll(state: SessionState): void {
   }
 }
 
+function clearOrbitSineRestartTimer(state: SessionState): void {
+  if (state.orbitSineRestartTimer) {
+    clearTimeout(state.orbitSineRestartTimer);
+    state.orbitSineRestartTimer = undefined;
+  }
+}
+
 async function stopOrbitSine(state: SessionState): Promise<void> {
   clearOrbitSinePoll(state);
+  clearOrbitSineRestartTimer(state);
   const playId = state.orbitSinePlayId;
   state.orbitSinePlayId = undefined;
   if (playId) {
@@ -252,7 +306,68 @@ async function stopOrbitSine(state: SessionState): Promise<void> {
   }
 }
 
+function orbitSineClipBytes(state: SessionState): string {
+  return buildOrbitSineInlineClipBase64(
+    ORBIT_SINE_DURATION_MS,
+    state.orbitSineFrequencyHz,
+    undefined,
+    state.orbitSinePhaseRad,
+  );
+}
+
+function orbitSineClipDuration(state: SessionState): number {
+  return orbitSineClipDurationMs(
+    ORBIT_SINE_DURATION_MS,
+    state.orbitSineFrequencyHz,
+  );
+}
+
+function scheduleOrbitSineGaplessRestart(
+  sessionId: string,
+  state: SessionState,
+): void {
+  clearOrbitSineRestartTimer(state);
+  const durationMs = orbitSineClipDuration(state);
+  const delayMs = Math.max(0, durationMs - ORBIT_SINE_GAPLESS_LEAD_MS);
+  state.orbitSineRestartTimer = setTimeout(() => {
+    state.orbitSineRestartTimer = undefined;
+    void restartOrbitSineGapless(sessionId, state);
+  }, delayMs);
+}
+
+async function restartOrbitSineGapless(
+  sessionId: string,
+  state: SessionState,
+): Promise<void> {
+  if (!state.orbitSineEnabled || !state.orbitSinePlayId) {
+    return;
+  }
+  const previousPlayId = state.orbitSinePlayId;
+  const result = await play({
+    url: ORBIT_SINE_INLINE_URL,
+    bytes: orbitSineClipBytes(state),
+    sessionIds: [sessionId],
+    volume: ORBIT_SINE_VOLUME,
+    pose: currentOrbitPose(state),
+  });
+  if (!result.ok || !result.playId) {
+    agentLog(
+      "warn",
+      `spatial-showcase orbit sine gapless restart failed: ${result.reason ?? "play_failed"}`,
+      sessionId,
+    );
+    startOrbitSineLoopPoll(sessionId, state);
+    return;
+  }
+  state.orbitSinePlayId = result.playId;
+  void stopPlay(previousPlayId);
+  scheduleOrbitSineGaplessRestart(sessionId, state);
+}
+
 function startOrbitSineLoopPoll(sessionId: string, state: SessionState): void {
+  if (state.orbitSineRestartTimer) {
+    return;
+  }
   clearOrbitSinePoll(state);
   const playId = state.orbitSinePlayId;
   if (!playId) {
@@ -261,7 +376,7 @@ function startOrbitSineLoopPoll(sessionId: string, state: SessionState): void {
 
   state.orbitSinePollTimer = setInterval(() => {
     void (async () => {
-      if (!state.orbitSinePlayId) {
+      if (!state.orbitSinePlayId || state.orbitSineRestartTimer) {
         clearOrbitSinePoll(state);
         return;
       }
@@ -270,16 +385,7 @@ function startOrbitSineLoopPoll(sessionId: string, state: SessionState): void {
         return;
       }
       if (status.status === "ended") {
-        const replay = await play({
-          url: ORBIT_SINE_INLINE_URL,
-          bytes: buildOrbitSineInlineClipBase64(),
-          sessionIds: [sessionId],
-          volume: ORBIT_SINE_VOLUME,
-          pose: currentOrbitPose(state),
-        });
-        if (replay.ok && replay.playId) {
-          state.orbitSinePlayId = replay.playId;
-        }
+        await restartOrbitSineGapless(sessionId, state);
       }
     })();
   }, ORBIT_SINE_POLL_MS);
@@ -289,10 +395,13 @@ async function startOrbitSinePlay(
   sessionId: string,
   state: SessionState,
 ): Promise<void> {
+  if (!state.orbitSineEnabled) {
+    return;
+  }
   await stopOrbitSine(state);
   const result = await play({
     url: ORBIT_SINE_INLINE_URL,
-    bytes: buildOrbitSineInlineClipBase64(),
+    bytes: orbitSineClipBytes(state),
     sessionIds: [sessionId],
     volume: ORBIT_SINE_VOLUME,
     pose: currentOrbitPose(state),
@@ -306,11 +415,12 @@ async function startOrbitSinePlay(
     return;
   }
   state.orbitSinePlayId = result.playId;
+  scheduleOrbitSineGaplessRestart(sessionId, state);
   startOrbitSineLoopPoll(sessionId, state);
 }
 
 function emitOrbitPose(sessionId: string, state: SessionState): void {
-  if (!state.orbitStartMs) {
+  if (!state.orbitStartMs && !state.orbitManualPose) {
     return;
   }
   const now = Date.now();
@@ -321,10 +431,12 @@ function emitOrbitPose(sessionId: string, state: SessionState): void {
     return;
   }
   state.lastOrbitPoseEmitMs = now;
-  const elapsedSec = (now - state.orbitStartMs) / 1000;
-  const pose = orbitTtsPose(elapsedSec, state.orbit);
-  const angleRad =
-    (2 * Math.PI * elapsedSec) / (state.orbit.periodSec ?? 2 * Math.PI);
+  const pose = currentOrbitPose(state);
+  const angleRad = state.orbitManualPose
+    ? Math.atan2(state.orbitManualPose.z, state.orbitManualPose.x)
+    : (2 * Math.PI * (now - state.orbitStartMs!)) /
+      1000 /
+      (state.orbit.periodSec ?? 2 * Math.PI);
   sendToClient(sessionId, {
     type: "orbit_pose",
     angleDeg: (angleRad * 180) / Math.PI,
@@ -333,12 +445,28 @@ function emitOrbitPose(sessionId: string, state: SessionState): void {
   });
 }
 
+function emitOrbitPlacePose(
+  sessionId: string,
+  state: SessionState,
+  x: number,
+  z: number,
+): void {
+  state.lastOrbitPoseEmitMs = Date.now();
+  const angleRad = Math.atan2(z, x);
+  sendToClient(sessionId, {
+    type: "orbit_pose",
+    angleDeg: (angleRad * 180) / Math.PI,
+    x,
+    z,
+  });
+}
+
 function startOrbitDemo(sessionId: string, state: SessionState): void {
   stopOrbitInterval(state);
   beginOrbitClock(state);
   void startOrbitSinePlay(sessionId, state);
   state.orbitTimer = setInterval(() => {
-    if (state.orbit.paused || !state.orbitStartMs) {
+    if (state.orbit.paused || (!state.orbitStartMs && !state.orbitManualPose)) {
       return;
     }
     const pose = currentOrbitPose(state);
@@ -524,6 +652,34 @@ export async function handlePadPlay(
   }
 }
 
+export async function handlePadMove(
+  sessionId: string,
+  state: SessionState,
+  playId: string,
+  x: number,
+  z: number,
+): Promise<void> {
+  if (!state.playIds.has(playId)) {
+    ackError(sessionId, "pad_move", "unknown_play");
+    return;
+  }
+
+  const poseResult = await setPlayPose(playId, poseAt(x, z));
+  if (!poseResult.ok) {
+    ackError(sessionId, "pad_move", poseResult.reason ?? "set_pose_failed");
+    return;
+  }
+
+  const loopPad = state.loopPads.get(playId);
+  if (loopPad) {
+    loopPad.x = x;
+    loopPad.z = z;
+    delete loopPad.placement;
+  }
+
+  ack(sessionId, "pad_move", { playId, x, z });
+}
+
 async function handleJoin(
   sessionId: string,
   demo: ShowcaseDemo,
@@ -657,11 +813,50 @@ async function dispatchMessage(
           state.orbit.elevation = message.elevation;
         }
         if (message.paused !== undefined) {
+          if (message.paused === false && state.orbitManualPose) {
+            syncOrbitClockFromManualPose(state);
+          }
           state.orbit.paused = message.paused;
         }
+        if (message.frequencyHz !== undefined) {
+          state.orbitSineFrequencyHz = message.frequencyHz;
+          if (state.orbitSineEnabled && state.orbitSinePlayId) {
+            void startOrbitSinePlay(sessionId, state);
+          }
+        }
+        if (message.sinePlaying !== undefined) {
+          state.orbitSineEnabled = message.sinePlaying;
+          if (message.sinePlaying) {
+            if (!state.orbitSinePlayId) {
+              void startOrbitSinePlay(sessionId, state);
+            }
+          } else {
+            void stopOrbitSine(state);
+          }
+        }
         ack(sessionId, "orbit_set");
         return;
       }
+      if (message.action === "place") {
+        state.orbit.paused = true;
+        state.orbitManualPose = { x: message.x, z: message.z };
+        const pose = orbitPoseFromXz(state, message.x, message.z);
+        if (state.orbitSinePlayId) {
+          void safeMixCall(sessionId, "setPlayPose", () =>
+            setPlayPose(state.orbitSinePlayId!, pose),
+          );
+        }
+        void safeMixCall(sessionId, "setTtsPose", () =>
+          setTtsPose(sessionId, pose),
+        );
+        emitOrbitPlacePose(sessionId, state, message.x, message.z);
+        ack(sessionId, "orbit_place");
+        return;
+      }
+      sendToClient(sessionId, {
+        type: "orbit_say_status",
+        status: "playing",
+      });
       speak(sessionId, message.text);
       ack(sessionId, "orbit_say");
       return;
@@ -718,6 +913,20 @@ async function dispatchMessage(
       }
       return;
     }
+    case "pad_move": {
+      if (state.demo !== "soundboard") {
+        ackError(sessionId, "pad_move", "wrong_demo");
+        return;
+      }
+      await handlePadMove(
+        sessionId,
+        state,
+        message.playId,
+        message.x,
+        message.z,
+      );
+      return;
+    }
     case "move": {
       if (state.demo !== "proximity") {
         ackError(sessionId, "move", "wrong_demo");
@@ -800,6 +1009,24 @@ defineAgent({
     void dispatchMessage(ctx.sessionId, message);
   },
 
+  onSpeechEvent(ctx, event) {
+    const state = sessions.get(ctx.sessionId);
+    if (!state || state.demo !== "orbit") {
+      return;
+    }
+    if (event.type === "agent_speaking_start") {
+      sendToClient(ctx.sessionId, {
+        type: "orbit_say_status",
+        status: "playing",
+      });
+    } else if (event.type === "agent_speaking_end") {
+      sendToClient(ctx.sessionId, {
+        type: "orbit_say_status",
+        status: "ended",
+      });
+    }
+  },
+
   onSessionEnd({ sessionId }) {
     teardownSession(sessionId);
     agentLog("info", `spatial-showcase session_end ${sessionId}`, sessionId);