@voicethere/agent 0.8.3 → 0.9.0

package/dist/templates/world-sync-binary/agent.js

@@ -178,6 +178,12 @@ function isSttControlAckMessage(value) {
   const msg = value;
   return msg.type === "stt_control_ack" && typeof msg.requestId === "string";
 }
+function isVoiceLanguageControlAckMessage(value) {
+  if (!value || typeof value !== "object")
+    return false;
+  const msg = value;
+  return msg.type === "voice_language_control_ack" && typeof msg.requestId === "string";
+}
 function isWebhookMessage(value) {
   if (!value || typeof value !== "object")
     return false;
@@ -220,7 +226,7 @@ function isParentMessage(value) {
   if (!value || typeof value !== "object")
     return false;
   const msg = value;
-  return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "conversation_history_control_ack" || msg.type === "play_ack" || msg.type === "play_status_ack" || msg.type === "play_stop_ack" || msg.type === "play_pose_ack" || msg.type === "mix_control_ack" || msg.type === "stt_control_ack" || msg.type === "webhook";
+  return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "conversation_history_control_ack" || msg.type === "play_ack" || msg.type === "play_status_ack" || msg.type === "play_stop_ack" || msg.type === "play_pose_ack" || msg.type === "mix_control_ack" || msg.type === "stt_control_ack" || msg.type === "voice_language_control_ack" || msg.type === "webhook";
 }
 function isSessionScopedParentMessage(value) {
   return isParentMessage(value) && !isWebhookMessage(value);
@@ -247,6 +253,7 @@ var pendingPlayStopAcks = /* @__PURE__ */ new Map();
 var pendingPlayPoseAcks = /* @__PURE__ */ new Map();
 var pendingMixAcks = /* @__PURE__ */ new Map();
 var pendingSttAcks = /* @__PURE__ */ new Map();
+var pendingVoiceLanguageAcks = /* @__PURE__ */ new Map();
 function handleRecordingControlAck(message) {
   const pending = pendingRecordingAcks.get(message.requestId);
   if (!pending)
@@ -349,6 +356,24 @@ function handleSttControlAck(message) {
     requestId: message.requestId
   });
 }
+function handleVoiceLanguageControlAck(message) {
+  const pending = pendingVoiceLanguageAcks.get(message.requestId);
+  if (!pending)
+    return;
+  clearTimeout(pending.timer);
+  pendingVoiceLanguageAcks.delete(message.requestId);
+  pending.resolve({
+    ok: message.ok,
+    reason: message.reason,
+    requestId: message.requestId,
+    ...message.language ? { language: message.language } : {},
+    ...message.voice ? { voice: message.voice } : {},
+    ...message.stt ? { stt: message.stt } : {},
+    ...message.scope ? { scope: message.scope } : {},
+    ...message.sttProvider ? { sttProvider: message.sttProvider } : {},
+    ...message.ttsProvider ? { ttsProvider: message.ttsProvider } : {}
+  });
+}
 function clearPendingRecordingAcksForSession(sessionId, reason) {
   for (const [requestId, pending] of pendingRecordingAcks) {
     if (pending.sessionId !== sessionId)
@@ -414,7 +439,7 @@ function allowOutboundForSession(sessionId) {
 }
 function sendParentMessage(message) {
   const msgType = message && typeof message === "object" && "type" in message ? message.type : void 0;
-  if (msgType === "mix_control" || msgType === "stt_control") {
+  if (msgType === "mix_control" || msgType === "stt_control" || msgType === "voice_language_control") {
     process.send?.(message);
     return;
   }
@@ -624,6 +649,10 @@ function defineAgent(handlers) {
       handleSttControlAck(message);
       return;
     }
+    if (isVoiceLanguageControlAckMessage(message)) {
+      handleVoiceLanguageControlAck(message);
+      return;
+    }
     if (isWebhookMessage(message)) {
       void agentStartReady.then(() => handleWebhookMessage(message, handlers));
       return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicethere/agent",
-  "version": "0.8.3",
+  "version": "0.9.0",
   "description": "VoiceThere customer agent SDK — IPC types and runtime helpers for sandboxed child bundles",
   "type": "module",
   "exports": {

package/templates/README.md

@@ -19,7 +19,7 @@ import {
 
 | Kind        | Dashboard create                                                                                                                                                                 | Prebuilt seed bundle                 | Typical consumer        |
 | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
-| **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `world-sync`, `world-sync-binary`, `game-sync`, `voice-showcase`, `recording-consent`, `positional-tts`, `spatial-showcase`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
+| **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `language-switch`, `world-sync`, `world-sync-binary`, `game-sync`, `voice-showcase`, `recording-consent`, `positional-tts`, `spatial-showcase`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
 | **e2e**     | No                                                                                                                                                                               | No — build from sources at test time | `voicethere/e2e` smokes |
 
 Product templates always set `seedOnCreate: true`. CI fails if a product template is missing its prebuilt bundle after `npm run build`.
@@ -64,6 +64,7 @@ Each folder has its own README. Summary:
 | `echo` | `echo/` | Voice + chat echo |
 | `echo-dc` | `echo-dc/` | Data-channel echo, no TTS |
 | `voice-starter` | `voice-starter/` | Every speech event |
+| `language-switch` | `language-switch/` | Separate `setVoiceLanguage` calls for STT, TTS, and vendor |
 | `world-sync` | `world-sync/` | JSON pose broadcast |
 | `world-sync-binary` | `world-sync-binary/` | Binary pose `ArrayBuffer` |
 | `game-sync` | `game-sync/` | Authoritative sim, Redis + binary snapshots |

package/templates/language-switch/README.md

@@ -0,0 +1,21 @@
+# language-switch
+
+Spoken-language detection tells the agent which ISO 639-1 code it heard. It does **not** change the TTS voice or the STT model. This template changes the two sides separately with `setVoiceLanguage`:
+
+- `scope: "tts"` switches only the speaking voice
+- `scope: "stt"` switches only the listening model
+
+A Sherpa language with no STT id (`it`, `pt`, `nl`, `pl`, `hi`) fails the STT call and still switches TTS.
+
+Chat commands change one vendor while the session stays connected. API keys are project secrets on the running deploy, not arguments:
+
+- `/tts sherpa de` — Sherpa TTS only
+- `/stt sherpa de` — Sherpa STT only
+- `/tts elevenlabs` — ElevenLabs TTS, current STT stays
+- `/stt deepgram de` — Deepgram STT, current voice stays
+
+English finals are echoed as `you said: …` after a short wait, so a language event for the same utterance can cancel the echo. The revealing utterance is not spoken back as an English transcript.
+
+`voice` and `stt` are Sherpa catalog ids (`de`, `en-lessac`, `en-small`), listed in the spoken-language docs. Vendor ids are `local-sherpa`, `openai`, `deepgram`, `assemblyai`, `google`, `elevenlabs`, and `cartesia`.
+
+Entry: `templates/language-switch/agent.ts`.

package/templates/language-switch/agent.ts

@@ -0,0 +1,180 @@
+/**
+ * Change STT and TTS separately, including the vendor, while the call stays up.
+ *
+ * The runner reports `user_language` and does not change either side. This
+ * agent switches the Sherpa voice and the Sherpa STT model as two calls, so
+ * one side can fail without blocking the other. Chat commands change a single
+ * vendor mid-conversation:
+ *
+ * - `/tts sherpa de` — Sherpa TTS only
+ * - `/stt sherpa de` — Sherpa STT only
+ * - `/tts elevenlabs` — ElevenLabs TTS, current STT stays
+ * - `/stt deepgram de` — Deepgram STT, current voice stays
+ *
+ * Vendor API keys are project secrets already on the running deploy. Do not
+ * put keys in this file. See /docs/spoken-language.
+ */
+import {
+  agentLog,
+  defineAgent,
+  parseChatText,
+  setVoiceLanguage,
+  speak,
+  type VoiceLanguageResult,
+} from "@voicethere/agent";
+
+const ECHO_PREFIX = "you said:";
+const ECHO_WAIT_MS = 4000;
+
+const REPLIES: Record<string, string> = {
+  de: "Guten Tag. Ich antworte jetzt auf Deutsch.",
+  es: "Hola. Ahora respondo en español.",
+  fr: "Bonjour. Je réponds maintenant en français.",
+  it: "Ciao. Adesso rispondo in italiano.",
+  pt: "Olá. Agora respondo em português.",
+  nl: "Hallo. Ik antwoord nu in het Nederlands.",
+  pl: "Dzień dobry. Teraz odpowiadam po polsku.",
+  ru: "Здравствуйте. Теперь я отвечаю по-русски.",
+};
+
+type SessionLanguage = {
+  language: string;
+  echoTimer: ReturnType<typeof setTimeout> | undefined;
+  suppressNextFinal: boolean;
+};
+
+const sessions = new Map<string, SessionLanguage>();
+
+function stateFor(sessionId: string): SessionLanguage {
+  const existing = sessions.get(sessionId);
+  if (existing) return existing;
+  const created: SessionLanguage = {
+    language: "en",
+    echoTimer: undefined,
+    suppressNextFinal: false,
+  };
+  sessions.set(sessionId, created);
+  return created;
+}
+
+function replyFor(language: string): string {
+  return REPLIES[language] ?? `Continuing in ${language}.`;
+}
+
+function logSwitch(sessionId: string, side: string, result: VoiceLanguageResult): void {
+  if (!result.ok) {
+    agentLog("warn", `setVoiceLanguage ${side} failed: ${result.reason ?? "unknown"}`);
+    return;
+  }
+  const provider = side === "tts" ? result.ttsProvider : result.sttProvider;
+  const model = side === "tts" ? result.voice : result.stt;
+  agentLog(
+    "info",
+    `voice ${side} ${sessionId} ${result.language ?? ""} provider=${provider ?? "unchanged"} model=${model ?? "unchanged"}`,
+  );
+}
+
+defineAgent({
+  onSessionStart({ sessionId }) {
+    stateFor(sessionId);
+    agentLog("info", `language-switch session_start ${sessionId}`);
+  },
+
+  async onUserLanguage({ sessionId, language }) {
+    const state = stateFor(sessionId);
+    if (!language || language === state.language) return;
+
+    if (state.echoTimer) {
+      clearTimeout(state.echoTimer);
+      state.echoTimer = undefined;
+    } else {
+      state.suppressNextFinal = true;
+    }
+
+    const tts = await setVoiceLanguage(sessionId, {
+      scope: "tts",
+      language,
+      voice: language,
+    });
+    logSwitch(sessionId, "tts", tts);
+
+    const stt = await setVoiceLanguage(sessionId, {
+      scope: "stt",
+      language,
+      stt: language,
+    });
+    logSwitch(sessionId, "stt", stt);
+
+    if (!tts.ok) {
+      state.suppressNextFinal = false;
+      return;
+    }
+
+    state.language = language;
+    speak(sessionId, replyFor(language));
+  },
+
+  async onDataChannelMessage(ctx) {
+    const text = parseChatText(ctx.message);
+    if (!text) return;
+    const [command, vendor, language] = text.trim().split(/\s+/);
+    if (command === "/tts" && vendor === "sherpa" && language) {
+      const result = await setVoiceLanguage(ctx.sessionId, {
+        scope: "tts",
+        language,
+        ttsVendor: { provider: "local-sherpa", voice: language },
+      });
+      logSwitch(ctx.sessionId, "tts", result);
+      if (result.ok) speak(ctx.sessionId, replyFor(language));
+      return;
+    }
+    if (command === "/stt" && vendor === "sherpa" && language) {
+      const result = await setVoiceLanguage(ctx.sessionId, {
+        scope: "stt",
+        language,
+        sttVendor: { provider: "local-sherpa", model: language },
+      });
+      logSwitch(ctx.sessionId, "stt", result);
+      return;
+    }
+    if (command === "/tts" && vendor === "elevenlabs") {
+      const result = await setVoiceLanguage(ctx.sessionId, {
+        scope: "tts",
+        ttsVendor: { provider: "elevenlabs", model: "eleven_multilingual_v2" },
+      });
+      logSwitch(ctx.sessionId, "tts", result);
+      if (result.ok) speak(ctx.sessionId, "Speaking with ElevenLabs.");
+      return;
+    }
+    if (command === "/stt" && vendor === "deepgram") {
+      const result = await setVoiceLanguage(ctx.sessionId, {
+        scope: "stt",
+        sttVendor: {
+          provider: "deepgram",
+          model: "nova-3",
+          language: language || "en",
+        },
+      });
+      logSwitch(ctx.sessionId, "stt", result);
+    }
+  },
+
+  onUserSpeechFinal({ sessionId, text }) {
+    const state = stateFor(sessionId);
+    if (state.suppressNextFinal) {
+      state.suppressNextFinal = false;
+      return;
+    }
+    if (state.echoTimer) clearTimeout(state.echoTimer);
+    state.echoTimer = setTimeout(() => {
+      state.echoTimer = undefined;
+      speak(sessionId, `${ECHO_PREFIX} ${text}`.trim());
+    }, ECHO_WAIT_MS);
+  },
+
+  onSessionEnd({ sessionId }) {
+    const state = sessions.get(sessionId);
+    if (state?.echoTimer) clearTimeout(state.echoTimer);
+    sessions.delete(sessionId);
+  },
+});

package/templates/voice-starter/agent.ts

@@ -179,7 +179,9 @@ defineAgent({
   },
 
   onUserLanguage({ sessionId, language }) {
-    // Route prompts / TTS voice from the detected ISO 639-1 code
+    // Detection does not change TTS or STT. Call setVoiceLanguage with
+    // scope "tts" or "stt" — or pass sttVendor / ttsVendor to change vendor.
+    // See templates/language-switch/agent.ts.
     agentLog("info", `user_language ${sessionId} ${language}`);
   },