@voicethere/agent 0.9.0 → 0.9.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicethere/agent",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "description": "VoiceThere customer agent SDK — IPC types and runtime helpers for sandboxed child bundles",
   "type": "module",
   "exports": {

package/templates/README.md

@@ -17,10 +17,10 @@ import {
 
 ## Product vs e2e
 
-| Kind        | Dashboard create                                                                                                                                                                 | Prebuilt seed bundle                 | Typical consumer        |
-| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
+| Kind        | Dashboard create                                                                                                                                                                                                       | Prebuilt seed bundle                 | Typical consumer        |
+| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
 | **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `language-switch`, `world-sync`, `world-sync-binary`, `game-sync`, `voice-showcase`, `recording-consent`, `positional-tts`, `spatial-showcase`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
-| **e2e**     | No                                                                                                                                                                               | No — build from sources at test time | `voicethere/e2e` smokes |
+| **e2e**     | No                                                                                                                                                                                                                     | No — build from sources at test time | `voicethere/e2e` smokes |
 
 Product templates always set `seedOnCreate: true`. CI fails if a product template is missing its prebuilt bundle after `npm run build`.
 
@@ -47,11 +47,11 @@ npm run build:templates
 
 Three product templates cover positional / world sync, from JSON to Redis:
 
-| Id                   | Channel                         | State                         |
-| -------------------- | ------------------------------- | ----------------------------- |
-| `world-sync`         | `onDataChannelMessage` (JSON)   | One agent, in-memory, no Redis |
-| `world-sync-binary`  | `onDataChannelBinary` + `broadCastBinaryToClients` (`ArrayBuffer`) | One agent, in-memory, no Redis |
-| `game-sync`          | JSON control + binary world snapshots | Redis when `AGENT_REDIS_URL` is set |
+| Id                  | Channel                                                            | State                               |
+| ------------------- | ------------------------------------------------------------------ | ----------------------------------- |
+| `world-sync`        | `onDataChannelMessage` (JSON)                                      | One agent, in-memory, no Redis      |
+| `world-sync-binary` | `onDataChannelBinary` + `broadCastBinaryToClients` (`ArrayBuffer`) | One agent, in-memory, no Redis      |
+| `game-sync`         | JSON control + binary world snapshots                              | Redis when `AGENT_REDIS_URL` is set |
 
 See each folder README for the wire format.
 
@@ -59,33 +59,33 @@ See each folder README for the wire format.
 
 Each folder has its own README. Summary:
 
-| Id | Folder | Summary |
-| -- | ------ | ------- |
-| `echo` | `echo/` | Voice + chat echo |
-| `echo-dc` | `echo-dc/` | Data-channel echo, no TTS |
-| `voice-starter` | `voice-starter/` | Every speech event |
-| `language-switch` | `language-switch/` | Separate `setVoiceLanguage` calls for STT, TTS, and vendor |
-| `world-sync` | `world-sync/` | JSON pose broadcast |
-| `world-sync-binary` | `world-sync-binary/` | Binary pose `ArrayBuffer` |
-| `game-sync` | `game-sync/` | Authoritative sim, Redis + binary snapshots |
-| `voice-showcase` | `voice-showcase/` | Conversational landing demo |
-| `recording-consent` | `recording-consent/` | Recording consent flow |
-| `positional-tts` | `positional-tts/` | Orbiting TTS |
-| `spatial-showcase` | `spatial-showcase/` | Orbit / soundboard / proximity |
-| `webhooks` | `webhooks/` | Inbound HMAC webhooks |
-| `webhooks-redis` | `webhooks-redis/` | Webhooks plus Redis counter |
+| Id                  | Folder               | Summary                                                           |
+| ------------------- | -------------------- | ----------------------------------------------------------------- |
+| `echo`              | `echo/`              | Voice + chat echo                                                 |
+| `echo-dc`           | `echo-dc/`           | Data-channel echo, no TTS                                         |
+| `voice-starter`     | `voice-starter/`     | Every speech event                                                |
+| `language-switch`   | `language-switch/`   | Manual `setVoiceLanguage`, or runner auto-switch with a localized echo of the recognized text |
+| `world-sync`        | `world-sync/`        | JSON pose broadcast                                               |
+| `world-sync-binary` | `world-sync-binary/` | Binary pose `ArrayBuffer`                                         |
+| `game-sync`         | `game-sync/`         | Authoritative sim, Redis + binary snapshots                       |
+| `voice-showcase`    | `voice-showcase/`    | Conversational landing demo                                       |
+| `recording-consent` | `recording-consent/` | Recording consent flow                                            |
+| `positional-tts`    | `positional-tts/`    | Orbiting TTS                                                      |
+| `spatial-showcase`  | `spatial-showcase/`  | Orbit / soundboard / proximity                                    |
+| `webhooks`          | `webhooks/`          | Inbound HMAC webhooks                                             |
+| `webhooks-redis`    | `webhooks-redis/`    | Webhooks plus Redis counter                                       |
 
 ## E2e templates
 
 These mirror former `e2e/fixtures/*` sources. E2E resolves entries from the package, builds into ephemeral workdirs, and uploads `dist/agent.js`.
 
-| Id                | Source                                         | Purpose                                       |
-| ----------------- | ---------------------------------------------- | --------------------------------------------- |
-| `echo-smoke`      | `echo-smoke/agent.ts`                          | voice-smoke, agent-smoke, cli-smoke           |
-| `crash`           | `crash/agent.ts`                               | session-errors-smoke, crash-policy smokes     |
-| `game-sync-smoke` | `game-sync-smoke/agent.ts`                     | deploy-smoke, shared-child, idle smokes       |
-| `redis-sync`      | `redis-sync/agent.ts` + `world-layout.ts`      | redis-sync-smoke (binary positions + Redis world blob) |
-| `mix-smoke`       | `mix-smoke/agent.ts`                           | voice-data-mix-smoke                          |
+| Id                | Source                                    | Purpose                                                |
+| ----------------- | ----------------------------------------- | ------------------------------------------------------ |
+| `echo-smoke`      | `echo-smoke/agent.ts`                     | voice-smoke, agent-smoke, cli-smoke                    |
+| `crash`           | `crash/agent.ts`                          | session-errors-smoke, crash-policy smokes              |
+| `game-sync-smoke` | `game-sync-smoke/agent.ts`                | deploy-smoke, shared-child, idle smokes                |
+| `redis-sync`      | `redis-sync/agent.ts` + `world-layout.ts` | redis-sync-smoke (binary positions + Redis world blob) |
+| `mix-smoke`       | `mix-smoke/agent.ts`                      | voice-data-mix-smoke                                   |
 
 **Note:** Product `echo` is not the same as e2e `echo-smoke` — keep both ids.
 

package/templates/language-switch/README.md

@@ -1,12 +1,22 @@
 # language-switch
 
-Spoken-language detection tells the agent which ISO 639-1 code it heard. It does **not** change the TTS voice or the STT model. This template changes the two sides separately with `setVoiceLanguage`:
+Spoken-language detection tells the agent which ISO 639-1 code it heard. It does **not** change the TTS voice or the STT model by itself. This template shows how to react in two ways:
+
+## Mode A — Manual switch (default)
+
+Leave the project **spoken-language auto-switch** setting off. When `onUserLanguage` fires, this agent calls `setVoiceLanguage` separately for each side:
 
 - `scope: "tts"` switches only the speaking voice
 - `scope: "stt"` switches only the listening model
 
 A Sherpa language with no STT id (`it`, `pt`, `nl`, `pl`, `hi`) fails the STT call and still switches TTS.
 
+## Mode B — Runner auto-switch
+
+Enable auto-switch in the project voice settings (runner applies STT/TTS when LID detects a new language). When `session_start.env` includes a truthy `SHERPA_LID_AUTO_SWITCH`, this template **does not** call `setVoiceLanguage`. The runner replays the utterance into the new language's STT, so the next final is the recognized text in the new language. The agent remembers the language from `onUserLanguage` / `onVoiceLanguageChanged` and answers each final right away with a short localized prefix, for example `you said: …` in English and `Du hast gesagt: …` in German. Use `onVoiceLanguageChanged` when you need the committed language after the runner applies the change.
+
+Detection and chat commands are unchanged: `/tts` and `/stt` still call `setVoiceLanguage` for one vendor at a time.
+
 Chat commands change one vendor while the session stays connected. API keys are project secrets on the running deploy, not arguments:
 
 - `/tts sherpa de` — Sherpa TTS only
@@ -14,7 +24,7 @@ Chat commands change one vendor while the session stays connected. API keys are
 - `/tts elevenlabs` — ElevenLabs TTS, current STT stays
 - `/stt deepgram de` — Deepgram STT, current voice stays
 
-English finals are echoed as `you said: …` after a short wait, so a language event for the same utterance can cancel the echo. The revealing utterance is not spoken back as an English transcript.
+In manual mode, English finals are echoed as `you said: …` after a short wait, so a language event for the same utterance can cancel the echo. The revealing utterance is not spoken back as an English transcript.
 
 `voice` and `stt` are Sherpa catalog ids (`de`, `en-lessac`, `en-small`), listed in the spoken-language docs. Vendor ids are `local-sherpa`, `openai`, `deepgram`, `assemblyai`, `google`, `elevenlabs`, and `cartesia`.
 

package/templates/language-switch/agent.ts

@@ -1,10 +1,17 @@
 /**
  * Change STT and TTS separately, including the vendor, while the call stays up.
  *
- * The runner reports `user_language` and does not change either side. This
- * agent switches the Sherpa voice and the Sherpa STT model as two calls, so
- * one side can fail without blocking the other. Chat commands change a single
- * vendor mid-conversation:
+ * Two deployment modes (see README):
+ *
+ * (A) Project auto-switch off — this agent calls `setVoiceLanguage` from
+ *     `onUserLanguage` when LID detects a new language.
+ * (B) Project enables runner auto-switch — STT/TTS are runner-owned. The
+ *     runner replays the utterance into the new language's STT, so the next
+ *     final is the correctly recognized text. The agent never calls
+ *     `setVoiceLanguage`; it remembers the language and answers each final
+ *     right away with a short localized prefix ("Du hast gesagt: …").
+ *
+ * Chat commands change a single vendor mid-conversation:
  *
  * - `/tts sherpa de` — Sherpa TTS only
  * - `/stt sherpa de` — Sherpa STT only
@@ -17,6 +24,7 @@
 import {
   agentLog,
   defineAgent,
+  isRunnerLidAutoSwitchEnabled,
   parseChatText,
   setVoiceLanguage,
   speak,
@@ -26,6 +34,19 @@ import {
 const ECHO_PREFIX = "you said:";
 const ECHO_WAIT_MS = 4000;
 
+/** Echo prefix per language for runner auto-switch mode. */
+const ECHO_PREFIXES: Record<string, string> = {
+  en: "you said:",
+  de: "Du hast gesagt:",
+  es: "Dijiste:",
+  fr: "Tu as dit :",
+  it: "Hai detto:",
+  pt: "Você disse:",
+  nl: "Je zei:",
+  pl: "Powiedziałeś:",
+  ru: "Вы сказали:",
+};
+
 const REPLIES: Record<string, string> = {
   de: "Guten Tag. Ich antworte jetzt auf Deutsch.",
   es: "Hola. Ahora respondo en español.",
@@ -39,17 +60,22 @@ const REPLIES: Record<string, string> = {
 
 type SessionLanguage = {
   language: string;
+  runnerAutoSwitch: boolean;
   echoTimer: ReturnType<typeof setTimeout> | undefined;
   suppressNextFinal: boolean;
 };
 
 const sessions = new Map<string, SessionLanguage>();
 
-function stateFor(sessionId: string): SessionLanguage {
+function stateFor(
+  sessionId: string,
+  env?: Record<string, string>,
+): SessionLanguage {
   const existing = sessions.get(sessionId);
   if (existing) return existing;
   const created: SessionLanguage = {
     language: "en",
+    runnerAutoSwitch: env ? isRunnerLidAutoSwitchEnabled(env) : false,
     echoTimer: undefined,
     suppressNextFinal: false,
   };
@@ -61,9 +87,20 @@ function replyFor(language: string): string {
   return REPLIES[language] ?? `Continuing in ${language}.`;
 }
 
-function logSwitch(sessionId: string, side: string, result: VoiceLanguageResult): void {
+function echoPrefixFor(language: string): string {
+  return ECHO_PREFIXES[language] ?? ECHO_PREFIXES.en!;
+}
+
+function logSwitch(
+  sessionId: string,
+  side: string,
+  result: VoiceLanguageResult,
+): void {
   if (!result.ok) {
-    agentLog("warn", `setVoiceLanguage ${side} failed: ${result.reason ?? "unknown"}`);
+    agentLog(
+      "warn",
+      `setVoiceLanguage ${side} failed: ${result.reason ?? "unknown"}`,
+    );
     return;
   }
   const provider = side === "tts" ? result.ttsProvider : result.sttProvider;
@@ -74,23 +111,49 @@ function logSwitch(sessionId: string, side: string, result: VoiceLanguageResult)
   );
 }
 
+function prepareLanguageTransition(state: SessionLanguage): void {
+  if (state.echoTimer) {
+    clearTimeout(state.echoTimer);
+    state.echoTimer = undefined;
+  } else {
+    state.suppressNextFinal = true;
+  }
+}
+
 defineAgent({
-  onSessionStart({ sessionId }) {
-    stateFor(sessionId);
-    agentLog("info", `language-switch session_start ${sessionId}`);
+  onSessionStart({ sessionId, env }) {
+    const state = stateFor(sessionId, env);
+    if (state.runnerAutoSwitch) {
+      agentLog(
+        "info",
+        `language-switch session_start ${sessionId} runner LID auto-switch owns STT/TTS`,
+      );
+    } else {
+      agentLog(
+        "info",
+        `language-switch session_start ${sessionId} manual setVoiceLanguage`,
+      );
+    }
   },
 
   async onUserLanguage({ sessionId, language }) {
     const state = stateFor(sessionId);
     if (!language || language === state.language) return;
 
-    if (state.echoTimer) {
-      clearTimeout(state.echoTimer);
-      state.echoTimer = undefined;
-    } else {
-      state.suppressNextFinal = true;
+    if (state.runnerAutoSwitch) {
+      // The runner switches STT/TTS and replays the utterance into the new
+      // STT; the next final is the real utterance, so nothing is suppressed
+      // and no fixed sentence is spoken.
+      agentLog(
+        "info",
+        `LID detected ${language}; runner auto-switch applies STT/TTS — agent only remembers the language`,
+      );
+      state.language = language;
+      return;
     }
 
+    prepareLanguageTransition(state);
+
     const tts = await setVoiceLanguage(sessionId, {
       scope: "tts",
       language,
@@ -114,6 +177,18 @@ defineAgent({
     speak(sessionId, replyFor(language));
   },
 
+  async onVoiceLanguageChanged({ sessionId, language }) {
+    const state = stateFor(sessionId);
+    if (!state.runnerAutoSwitch || !language || language === state.language) {
+      return;
+    }
+    agentLog(
+      "info",
+      `runner committed voice language ${language} for ${sessionId}`,
+    );
+    state.language = language;
+  },
+
   async onDataChannelMessage(ctx) {
     const text = parseChatText(ctx.message);
     if (!text) return;
@@ -161,6 +236,12 @@ defineAgent({
 
   onUserSpeechFinal({ sessionId, text }) {
     const state = stateFor(sessionId);
+    if (state.runnerAutoSwitch) {
+      // Reply immediately: the runner already holds the final until LID has
+      // decided, so no extra wait is needed to avoid a wrong-language echo.
+      speak(sessionId, `${echoPrefixFor(state.language)} ${text}`.trim());
+      return;
+    }
     if (state.suppressNextFinal) {
       state.suppressNextFinal = false;
       return;