@voicethere/agent 0.6.0 → 0.7.0

package/dist/templates/webhooks-redis/agent.js

@@ -10891,6 +10891,18 @@ function isRecordingControlAckMessage(value) {
   const msg = value;
   return msg.type === "recording_control_ack" && typeof msg.requestId === "string";
 }
+function isMixControlAckMessage(value) {
+  if (!value || typeof value !== "object")
+    return false;
+  const msg = value;
+  return msg.type === "mix_control_ack" && typeof msg.requestId === "string";
+}
+function isSttControlAckMessage(value) {
+  if (!value || typeof value !== "object")
+    return false;
+  const msg = value;
+  return msg.type === "stt_control_ack" && typeof msg.requestId === "string";
+}
 function isWebhookMessage(value) {
   if (!value || typeof value !== "object")
     return false;
@@ -10933,7 +10945,7 @@ function isParentMessage(value) {
   if (!value || typeof value !== "object")
     return false;
   const msg = value;
-  return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "webhook";
+  return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "mix_control_ack" || msg.type === "stt_control_ack" || msg.type === "webhook";
 }
 function isSessionScopedParentMessage(value) {
   return isParentMessage(value) && !isWebhookMessage(value);
@@ -10947,8 +10959,12 @@ function parseDataChannelPayload(raw) {
 }
 var peerEnvBySessionId = /* @__PURE__ */ new Map();
 var recordingAvailableBySessionId = /* @__PURE__ */ new Map();
+var mixAvailableBySessionId = /* @__PURE__ */ new Map();
+var ttsPoseAvailableBySessionId = /* @__PURE__ */ new Map();
 var endedSessionIds = /* @__PURE__ */ new Set();
 var pendingRecordingAcks = /* @__PURE__ */ new Map();
+var pendingMixAcks = /* @__PURE__ */ new Map();
+var pendingSttAcks = /* @__PURE__ */ new Map();
 function handleRecordingControlAck(message) {
   const pending = pendingRecordingAcks.get(message.requestId);
   if (!pending)
@@ -10961,6 +10977,30 @@ function handleRecordingControlAck(message) {
     requestId: message.requestId
   });
 }
+function handleMixControlAck(message) {
+  const pending = pendingMixAcks.get(message.requestId);
+  if (!pending)
+    return;
+  clearTimeout(pending.timer);
+  pendingMixAcks.delete(message.requestId);
+  pending.resolve({
+    ok: message.ok,
+    reason: message.reason,
+    requestId: message.requestId
+  });
+}
+function handleSttControlAck(message) {
+  const pending = pendingSttAcks.get(message.requestId);
+  if (!pending)
+    return;
+  clearTimeout(pending.timer);
+  pendingSttAcks.delete(message.requestId);
+  pending.resolve({
+    ok: message.ok,
+    reason: message.reason,
+    requestId: message.requestId
+  });
+}
 function clearPendingRecordingAcksForSession(sessionId, reason) {
   for (const [requestId, pending] of pendingRecordingAcks) {
     if (pending.sessionId !== sessionId)
@@ -11016,6 +11056,11 @@ function allowOutboundForSession(sessionId) {
   return queue.isLive(sessionId);
 }
 function sendParentMessage(message) {
+  const msgType = message && typeof message === "object" && "type" in message ? message.type : void 0;
+  if (msgType === "mix_control" || msgType === "stt_control") {
+    process.send?.(message);
+    return;
+  }
   const sessionId = message && typeof message === "object" && "sessionId" in message && typeof message.sessionId === "string" ? message.sessionId : void 0;
   if (!allowOutboundForSession(sessionId))
     return;
@@ -11097,6 +11142,8 @@ async function handleParentMessage(message, handlers) {
       endedSessionIds.delete(message.sessionId);
       peerEnvBySessionId.set(message.sessionId, message.env);
       recordingAvailableBySessionId.set(message.sessionId, message.recordingAvailable ?? false);
+      mixAvailableBySessionId.set(message.sessionId, message.mixAvailable ?? false);
+      ttsPoseAvailableBySessionId.set(message.sessionId, message.ttsPoseAvailable ?? false);
       const sessionStartInitDelayMs = resolveSessionStartInitDelayMs();
       if (sessionStartInitDelayMs > 0) {
         await new Promise((resolve) => setTimeout(resolve, sessionStartInitDelayMs));
@@ -11104,7 +11151,9 @@ async function handleParentMessage(message, handlers) {
       await (handlers.onClientJoin ?? handlers.onSessionStart)?.({
         sessionId: message.sessionId,
         env: message.env,
-        recordingAvailable: message.recordingAvailable ?? false
+        recordingAvailable: message.recordingAvailable ?? false,
+        mixAvailable: message.mixAvailable ?? false,
+        ttsPoseAvailable: message.ttsPoseAvailable ?? false
       });
       sendParentMessage({
         type: "session_start_ack",
@@ -11142,6 +11191,8 @@ async function handleParentMessage(message, handlers) {
       clearPendingRecordingAcksForSession(message.sessionId, "session_ended");
       peerEnvBySessionId.delete(message.sessionId);
       recordingAvailableBySessionId.delete(message.sessionId);
+      mixAvailableBySessionId.delete(message.sessionId);
+      ttsPoseAvailableBySessionId.delete(message.sessionId);
       await (handlers.onClientLeave ?? handlers.onSessionEnd)?.({
         sessionId: message.sessionId
       });
@@ -11161,6 +11212,14 @@ function defineAgent(handlers) {
       handleRecordingControlAck(message);
       return;
     }
+    if (isMixControlAckMessage(message)) {
+      handleMixControlAck(message);
+      return;
+    }
+    if (isSttControlAckMessage(message)) {
+      handleSttControlAck(message);
+      return;
+    }
     if (isWebhookMessage(message)) {
       void agentStartReady.then(() => handleWebhookMessage(message, handlers));
       return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicethere/agent",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "VoiceThere customer agent SDK — IPC types and runtime helpers for sandboxed child bundles",
   "type": "module",
   "exports": {
@@ -95,9 +95,9 @@
     "@node-webrtc-rust/sdk": ">=0.6.5"
   },
   "devDependencies": {
-    "@node-webrtc-rust/helpers": "0.7.4",
-    "@node-webrtc-rust/sdk": "0.7.4",
-    "@node-webrtc-rust/signaling": "0.7.4",
+    "@node-webrtc-rust/helpers": "0.8.0",
+    "@node-webrtc-rust/sdk": "0.8.0",
+    "@node-webrtc-rust/signaling": "0.8.0",
     "@types/node": "22.20.1",
     "http-server": "14.1.1",
     "ioredis": "5.11.1",

package/templates/README.md

@@ -15,10 +15,10 @@ import {
 
 ## Product vs e2e
 
-| Kind        | Dashboard create                                                                                                           | Prebuilt seed bundle                 | Typical consumer        |
-| ----------- | -------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
-| **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `game-sync`, `voice-showcase`, `recording-consent`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
-| **e2e**     | No                                                                                                                         | No — build from sources at test time | `voicethere/e2e` smokes |
+| Kind        | Dashboard create                                                                                                                             | Prebuilt seed bundle                 | Typical consumer        |
+| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
+| **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `game-sync`, `voice-showcase`, `recording-consent`, `positional-tts`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
+| **e2e**     | No                                                                                                                                           | No — build from sources at test time | `voicethere/e2e` smokes |
 
 Product templates always set `seedOnCreate: true`. CI fails if a product template is missing its prebuilt bundle after `npm run build`.
 
@@ -73,6 +73,12 @@ Demonstrates conversation recording consent on connect: asks whether recording i
 
 Sources: `recording-consent/agent.ts` (defineAgent wiring), `conversation.ts` (pure state machine).
 
+### `positional-tts/` (`positional-tts`)
+
+Voice+Data demo — enables positional mixing and orbits each listener’s TTS speaker with `setTtsPose` on a ~50 ms timer. Speaks a short greeting on connect and echoes voice finals / chat so you hear panning while TTS plays. Requires Voice+Data runner mode (`isMixAvailable`).
+
+Sources: `positional-tts/agent.ts` (defineAgent wiring), `positional-tts/orbit.ts` (pure circle helper for tests).
+
 ### `webhooks.ts` (`webhooks`)
 
 Inbound HTTP webhook sample — verifies `x-agent-webhook-signature` HMAC on the **raw body** with `AGENT_WEBHOOK_SIGNING_SECRET`, then `JSON.parse` and fans out to connected sessions via DataChannel + `speak`.

package/templates/mix-smoke.ts

@@ -0,0 +1,224 @@
+/**
+ * E2E agent for voice-data-mix-smoke — positional mix via data-channel commands.
+ *
+ * Protocol (JSON on control DC):
+ * - `{ type: "mix", action: "whoami" }`
+ * - `{ type: "mix", action: "create_group", groupId, clientIds }`
+ * - `{ type: "mix", action: "set_pose", clientId, pose }`
+ * - `{ type: "mix", action: "set_positional", enabled }`
+ * - `{ type: "mix", action: "list_clients" }` (optional)
+ *
+ * Acks: `{ type: "mix_ack", action, ok: true, ... }` or `{ ok: false, error }`.
+ * Ignores `ping` / chat strings (voice-control readiness). No TTS during smoke.
+ */
+import {
+  MIX_REQUIRES_VOICE_PLUS_DATA,
+  createMixGroup,
+  defineAgent,
+  sendToClient,
+  setClientPose,
+  setPositionalMixing,
+} from "@voicethere/agent";
+
+/** E2E fixture marker — rewritten before each fixture upload when needed. */
+export const FIXTURE_MARKER = "mix-smoke-fixture-a";
+
+export type MixPose = {
+  position: { x: number; y: number; z: number };
+  orientation: { x: number; y: number; z: number; w: number };
+};
+
+export type MixCommand =
+  | { type: "mix"; action: "whoami" }
+  | {
+      type: "mix";
+      action: "create_group";
+      groupId: string;
+      clientIds: string[];
+    }
+  | { type: "mix"; action: "set_pose"; clientId: string; pose: MixPose }
+  | { type: "mix"; action: "set_positional"; enabled: boolean }
+  | { type: "mix"; action: "list_clients" };
+
+export type MixAck =
+  | {
+      type: "mix_ack";
+      action: string;
+      ok: true;
+      sessionId?: string;
+      clientIds?: string[];
+    }
+  | { type: "mix_ack"; action: string; ok: false; error: string };
+
+const connectedSessions = new Set<string>();
+let mixAvailableForChild = false;
+
+export function isMixCommand(message: unknown): message is MixCommand {
+  if (!message || typeof message !== "object") return false;
+  const record = message as { type?: unknown; action?: unknown };
+  if (record.type !== "mix" || typeof record.action !== "string") {
+    return false;
+  }
+  switch (record.action) {
+    case "whoami":
+    case "list_clients":
+      return true;
+    case "create_group": {
+      const group = message as {
+        groupId?: unknown;
+        clientIds?: unknown;
+      };
+      return (
+        typeof group.groupId === "string" &&
+        group.groupId.trim().length > 0 &&
+        Array.isArray(group.clientIds) &&
+        group.clientIds.every((id) => typeof id === "string")
+      );
+    }
+    case "set_pose": {
+      const poseMsg = message as { clientId?: unknown; pose?: unknown };
+      return (
+        typeof poseMsg.clientId === "string" &&
+        poseMsg.clientId.trim().length > 0 &&
+        isMixPose(poseMsg.pose)
+      );
+    }
+    case "set_positional": {
+      const positional = message as { enabled?: unknown };
+      return typeof positional.enabled === "boolean";
+    }
+    default:
+      return false;
+  }
+}
+
+export function isMixPose(value: unknown): value is MixPose {
+  if (!value || typeof value !== "object") return false;
+  const pose = value as MixPose;
+  return isVec3(pose.position) && isQuat(pose.orientation);
+}
+
+function isVec3(value: unknown): value is { x: number; y: number; z: number } {
+  if (!value || typeof value !== "object") return false;
+  const v = value as { x?: unknown; y?: unknown; z?: unknown };
+  return (
+    typeof v.x === "number" &&
+    typeof v.y === "number" &&
+    typeof v.z === "number"
+  );
+}
+
+function isQuat(
+  value: unknown,
+): value is { x: number; y: number; z: number; w: number } {
+  if (!value || typeof value !== "object") return false;
+  const q = value as { x?: unknown; y?: unknown; z?: unknown; w?: unknown };
+  return (
+    typeof q.x === "number" &&
+    typeof q.y === "number" &&
+    typeof q.z === "number" &&
+    typeof q.w === "number"
+  );
+}
+
+function ackOk(
+  sessionId: string,
+  action: string,
+  extra?: { sessionId?: string; clientIds?: string[] },
+): void {
+  sendToClient(sessionId, {
+    type: "mix_ack",
+    action,
+    ok: true,
+    ...extra,
+  } satisfies MixAck);
+}
+
+function ackError(sessionId: string, action: string, error: string): void {
+  sendToClient(sessionId, {
+    type: "mix_ack",
+    action,
+    ok: false,
+    error,
+  } satisfies MixAck);
+}
+
+function requireMix(sessionId: string, action: string): boolean {
+  if (mixAvailableForChild) {
+    return true;
+  }
+  ackError(sessionId, action, MIX_REQUIRES_VOICE_PLUS_DATA);
+  return false;
+}
+
+async function handleMixCommand(
+  sessionId: string,
+  command: MixCommand,
+): Promise<void> {
+  const { action } = command;
+
+  switch (action) {
+    case "whoami":
+      ackOk(sessionId, action, { sessionId });
+      return;
+    case "list_clients":
+      ackOk(sessionId, action, {
+        clientIds: [...connectedSessions],
+      });
+      return;
+    case "create_group": {
+      if (!requireMix(sessionId, action)) return;
+      const result = await createMixGroup({
+        id: command.groupId,
+        clientIds: command.clientIds,
+      });
+      if (!result.ok) {
+        ackError(sessionId, action, result.reason ?? "create_group failed");
+        return;
+      }
+      ackOk(sessionId, action);
+      return;
+    }
+    case "set_pose": {
+      if (!requireMix(sessionId, action)) return;
+      const result = await setClientPose(command.clientId, command.pose);
+      if (!result.ok) {
+        ackError(sessionId, action, result.reason ?? "set_pose failed");
+        return;
+      }
+      ackOk(sessionId, action);
+      return;
+    }
+    case "set_positional": {
+      if (!requireMix(sessionId, action)) return;
+      const result = await setPositionalMixing(command.enabled);
+      if (!result.ok) {
+        ackError(sessionId, action, result.reason ?? "set_positional failed");
+        return;
+      }
+      ackOk(sessionId, action);
+      return;
+    }
+    default:
+      ackError(sessionId, "unknown", "unsupported mix action");
+  }
+}
+
+defineAgent({
+  onClientJoin(ctx) {
+    mixAvailableForChild = ctx.mixAvailable;
+    connectedSessions.add(ctx.sessionId);
+  },
+
+  onClientLeave({ sessionId }) {
+    connectedSessions.delete(sessionId);
+  },
+
+  onDataChannelMessage(ctx) {
+    if (isMixCommand(ctx.message)) {
+      void handleMixCommand(ctx.sessionId, ctx.message);
+      return;
+    }
+    // Ignore ping / chat — voice-control readiness only.
+  },
+});

package/templates/positional-tts/agent.ts

@@ -0,0 +1,115 @@
+/**
+ * Positional TTS template — each listener hears TTS orbiting their own pose.
+ *
+ * Requires voice or Voice+Data runner mode ({@link isTtsPoseAvailable}). Mix group
+ * APIs still require Voice+Data ({@link isMixAvailable}).
+ *
+ * Build:
+ *   npx @voicethere/agent build --entry templates/positional-tts/agent.ts
+ */
+import {
+  agentLog,
+  clearTtsPose,
+  defineAgent,
+  isTtsPoseAvailable,
+  parseChatText,
+  setPositionalMixing,
+  setTtsPose,
+  speak,
+} from "@voicethere/agent";
+
+import { orbitTtsPose } from "./orbit.js";
+
+const ORBIT_INTERVAL_MS = 50;
+
+const orbitTimers = new Map<string, ReturnType<typeof setInterval>>();
+const sessionStartTimes = new Map<string, number>();
+
+function clearOrbit(sessionId: string): void {
+  const timer = orbitTimers.get(sessionId);
+  if (timer) {
+    clearInterval(timer);
+    orbitTimers.delete(sessionId);
+  }
+  sessionStartTimes.delete(sessionId);
+}
+
+async function safeMixCall(
+  sessionId: string,
+  label: string,
+  fn: () => Promise<{ ok: boolean; reason?: string }>,
+): Promise<void> {
+  try {
+    const result = await fn();
+    if (!result.ok) {
+      agentLog(
+        "warn",
+        `positional-tts ${label} failed: ${result.reason}`,
+        sessionId,
+      );
+    }
+  } catch (err) {
+    agentLog(
+      "warn",
+      `positional-tts ${label} error: ${String(err)}`,
+      sessionId,
+    );
+  }
+}
+
+defineAgent({
+  onAgentStart() {
+    orbitTimers.clear();
+    sessionStartTimes.clear();
+  },
+
+  onSessionStart(ctx) {
+    const { sessionId } = ctx;
+
+    if (!isTtsPoseAvailable(ctx)) {
+      agentLog(
+        "warn",
+        "positional-tts requires voice or Voice+Data — TTS pose APIs unavailable",
+        sessionId,
+      );
+      return;
+    }
+
+    void (async () => {
+      await safeMixCall(sessionId, "setPositionalMixing", () =>
+        setPositionalMixing(true),
+      );
+
+      const startMs = Date.now();
+      sessionStartTimes.set(sessionId, startMs);
+
+      const timer = setInterval(() => {
+        const started = sessionStartTimes.get(sessionId);
+        if (!started) return;
+        const elapsedSec = (Date.now() - started) / 1000;
+        void safeMixCall(sessionId, "setTtsPose", () =>
+          setTtsPose(sessionId, orbitTtsPose(elapsedSec)),
+        );
+      }, ORBIT_INTERVAL_MS);
+
+      orbitTimers.set(sessionId, timer);
+      speak(sessionId, "I'll circle around you.");
+    })();
+  },
+
+  onUserSpeechFinal({ sessionId, text }) {
+    speak(sessionId, `You said: ${text}`);
+  },
+
+  onDataChannelMessage(ctx) {
+    const text = parseChatText(ctx.message);
+    if (!text) return;
+    speak(ctx.sessionId, `You said: ${text}`);
+  },
+
+  onSessionEnd({ sessionId }) {
+    clearOrbit(sessionId);
+    void safeMixCall(sessionId, "clearTtsPose", () => clearTtsPose(sessionId));
+    agentLog("info", `positional-tts session_end ${sessionId}`, sessionId);
+  },
+});

package/templates/positional-tts/orbit.ts

@@ -0,0 +1,11 @@
+import type { MixPose } from "@voicethere/agent";
+
+/** Circle in the XZ plane around the listener origin (Y-up, look −Z). */
+export function orbitTtsPose(elapsedSec: number, radius = 2): MixPose {
+  const x = Math.cos(elapsedSec) * radius;
+  const z = Math.sin(elapsedSec) * radius;
+  return {
+    position: { x, y: 0, z },
+    orientation: { x: 0, y: 0, z: 0, w: 1 },
+  };
+}