@voicethere/agent 0.8.3 → 0.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicethere/agent",
-  "version": "0.8.3",
+  "version": "0.9.1",
   "description": "VoiceThere customer agent SDK — IPC types and runtime helpers for sandboxed child bundles",
   "type": "module",
   "exports": {

package/templates/README.md

@@ -17,10 +17,10 @@ import {
 
 ## Product vs e2e
 
-| Kind        | Dashboard create                                                                                                                                                                 | Prebuilt seed bundle                 | Typical consumer        |
-| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
-| **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `world-sync`, `world-sync-binary`, `game-sync`, `voice-showcase`, `recording-consent`, `positional-tts`, `spatial-showcase`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
-| **e2e**     | No                                                                                                                                                                               | No — build from sources at test time | `voicethere/e2e` smokes |
+| Kind        | Dashboard create                                                                                                                                                                                                       | Prebuilt seed bundle                 | Typical consumer        |
+| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
+| **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `language-switch`, `world-sync`, `world-sync-binary`, `game-sync`, `voice-showcase`, `recording-consent`, `positional-tts`, `spatial-showcase`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
+| **e2e**     | No                                                                                                                                                                                                                     | No — build from sources at test time | `voicethere/e2e` smokes |
 
 Product templates always set `seedOnCreate: true`. CI fails if a product template is missing its prebuilt bundle after `npm run build`.
 
@@ -47,11 +47,11 @@ npm run build:templates
 
 Three product templates cover positional / world sync, from JSON to Redis:
 
-| Id                   | Channel                         | State                         |
-| -------------------- | ------------------------------- | ----------------------------- |
-| `world-sync`         | `onDataChannelMessage` (JSON)   | One agent, in-memory, no Redis |
-| `world-sync-binary`  | `onDataChannelBinary` + `broadCastBinaryToClients` (`ArrayBuffer`) | One agent, in-memory, no Redis |
-| `game-sync`          | JSON control + binary world snapshots | Redis when `AGENT_REDIS_URL` is set |
+| Id                  | Channel                                                            | State                               |
+| ------------------- | ------------------------------------------------------------------ | ----------------------------------- |
+| `world-sync`        | `onDataChannelMessage` (JSON)                                      | One agent, in-memory, no Redis      |
+| `world-sync-binary` | `onDataChannelBinary` + `broadCastBinaryToClients` (`ArrayBuffer`) | One agent, in-memory, no Redis      |
+| `game-sync`         | JSON control + binary world snapshots                              | Redis when `AGENT_REDIS_URL` is set |
 
 See each folder README for the wire format.
 
@@ -59,32 +59,33 @@ See each folder README for the wire format.
 
 Each folder has its own README. Summary:
 
-| Id | Folder | Summary |
-| -- | ------ | ------- |
-| `echo` | `echo/` | Voice + chat echo |
-| `echo-dc` | `echo-dc/` | Data-channel echo, no TTS |
-| `voice-starter` | `voice-starter/` | Every speech event |
-| `world-sync` | `world-sync/` | JSON pose broadcast |
-| `world-sync-binary` | `world-sync-binary/` | Binary pose `ArrayBuffer` |
-| `game-sync` | `game-sync/` | Authoritative sim, Redis + binary snapshots |
-| `voice-showcase` | `voice-showcase/` | Conversational landing demo |
-| `recording-consent` | `recording-consent/` | Recording consent flow |
-| `positional-tts` | `positional-tts/` | Orbiting TTS |
-| `spatial-showcase` | `spatial-showcase/` | Orbit / soundboard / proximity |
-| `webhooks` | `webhooks/` | Inbound HMAC webhooks |
-| `webhooks-redis` | `webhooks-redis/` | Webhooks plus Redis counter |
+| Id                  | Folder               | Summary                                                           |
+| ------------------- | -------------------- | ----------------------------------------------------------------- |
+| `echo`              | `echo/`              | Voice + chat echo                                                 |
+| `echo-dc`           | `echo-dc/`           | Data-channel echo, no TTS                                         |
+| `voice-starter`     | `voice-starter/`     | Every speech event                                                |
+| `language-switch`   | `language-switch/`   | Manual `setVoiceLanguage` or runner auto-switch + prompt handlers |
+| `world-sync`        | `world-sync/`        | JSON pose broadcast                                               |
+| `world-sync-binary` | `world-sync-binary/` | Binary pose `ArrayBuffer`                                         |
+| `game-sync`         | `game-sync/`         | Authoritative sim, Redis + binary snapshots                       |
+| `voice-showcase`    | `voice-showcase/`    | Conversational landing demo                                       |
+| `recording-consent` | `recording-consent/` | Recording consent flow                                            |
+| `positional-tts`    | `positional-tts/`    | Orbiting TTS                                                      |
+| `spatial-showcase`  | `spatial-showcase/`  | Orbit / soundboard / proximity                                    |
+| `webhooks`          | `webhooks/`          | Inbound HMAC webhooks                                             |
+| `webhooks-redis`    | `webhooks-redis/`    | Webhooks plus Redis counter                                       |
 
 ## E2e templates
 
 These mirror former `e2e/fixtures/*` sources. E2E resolves entries from the package, builds into ephemeral workdirs, and uploads `dist/agent.js`.
 
-| Id                | Source                                         | Purpose                                       |
-| ----------------- | ---------------------------------------------- | --------------------------------------------- |
-| `echo-smoke`      | `echo-smoke/agent.ts`                          | voice-smoke, agent-smoke, cli-smoke           |
-| `crash`           | `crash/agent.ts`                               | session-errors-smoke, crash-policy smokes     |
-| `game-sync-smoke` | `game-sync-smoke/agent.ts`                     | deploy-smoke, shared-child, idle smokes       |
-| `redis-sync`      | `redis-sync/agent.ts` + `world-layout.ts`      | redis-sync-smoke (binary positions + Redis world blob) |
-| `mix-smoke`       | `mix-smoke/agent.ts`                           | voice-data-mix-smoke                          |
+| Id                | Source                                    | Purpose                                                |
+| ----------------- | ----------------------------------------- | ------------------------------------------------------ |
+| `echo-smoke`      | `echo-smoke/agent.ts`                     | voice-smoke, agent-smoke, cli-smoke                    |
+| `crash`           | `crash/agent.ts`                          | session-errors-smoke, crash-policy smokes              |
+| `game-sync-smoke` | `game-sync-smoke/agent.ts`                | deploy-smoke, shared-child, idle smokes                |
+| `redis-sync`      | `redis-sync/agent.ts` + `world-layout.ts` | redis-sync-smoke (binary positions + Redis world blob) |
+| `mix-smoke`       | `mix-smoke/agent.ts`                      | voice-data-mix-smoke                                   |
 
 **Note:** Product `echo` is not the same as e2e `echo-smoke` — keep both ids.
 

package/templates/language-switch/README.md

@@ -0,0 +1,31 @@
+# language-switch
+
+Spoken-language detection tells the agent which ISO 639-1 code it heard. It does **not** change the TTS voice or the STT model by itself. This template shows how to react in two ways:
+
+## Mode A — Manual switch (default)
+
+Leave the project **spoken-language auto-switch** setting off. When `onUserLanguage` fires, this agent calls `setVoiceLanguage` separately for each side:
+
+- `scope: "tts"` switches only the speaking voice
+- `scope: "stt"` switches only the listening model
+
+A Sherpa language with no STT id (`it`, `pt`, `nl`, `pl`, `hi`) fails the STT call and still switches TTS.
+
+## Mode B — Runner auto-switch
+
+Enable auto-switch in the project voice settings (runner applies STT/TTS when LID detects a new language). When `session_start.env` includes a truthy `SHERPA_LID_AUTO_SWITCH`, this template **does not** call `setVoiceLanguage` from `onUserLanguage` — it logs that the runner owns the switch and updates prompts only. Use `onVoiceLanguageChanged` when you need the committed language after the runner applies the change.
+
+Detection and chat commands are unchanged: `/tts` and `/stt` still call `setVoiceLanguage` for one vendor at a time.
+
+Chat commands change one vendor while the session stays connected. API keys are project secrets on the running deploy, not arguments:
+
+- `/tts sherpa de` — Sherpa TTS only
+- `/stt sherpa de` — Sherpa STT only
+- `/tts elevenlabs` — ElevenLabs TTS, current STT stays
+- `/stt deepgram de` — Deepgram STT, current voice stays
+
+English finals are echoed as `you said: …` after a short wait, so a language event for the same utterance can cancel the echo. The revealing utterance is not spoken back as an English transcript.
+
+`voice` and `stt` are Sherpa catalog ids (`de`, `en-lessac`, `en-small`), listed in the spoken-language docs. Vendor ids are `local-sherpa`, `openai`, `deepgram`, `assemblyai`, `google`, `elevenlabs`, and `cartesia`.
+
+Entry: `templates/language-switch/agent.ts`.

package/templates/language-switch/agent.ts

@@ -0,0 +1,233 @@
+/**
+ * Change STT and TTS separately, including the vendor, while the call stays up.
+ *
+ * Two deployment modes (see README):
+ *
+ * (A) Project auto-switch off — this agent calls `setVoiceLanguage` from
+ *     `onUserLanguage` when LID detects a new language.
+ * (B) Project enables runner auto-switch — STT/TTS are runner-owned; use
+ *     `onUserLanguage` / `onVoiceLanguageChanged` for prompts only.
+ *
+ * Chat commands change a single vendor mid-conversation:
+ *
+ * - `/tts sherpa de` — Sherpa TTS only
+ * - `/stt sherpa de` — Sherpa STT only
+ * - `/tts elevenlabs` — ElevenLabs TTS, current STT stays
+ * - `/stt deepgram de` — Deepgram STT, current voice stays
+ *
+ * Vendor API keys are project secrets already on the running deploy. Do not
+ * put keys in this file. See /docs/spoken-language.
+ */
+import {
+  agentLog,
+  defineAgent,
+  isRunnerLidAutoSwitchEnabled,
+  parseChatText,
+  setVoiceLanguage,
+  speak,
+  type VoiceLanguageResult,
+} from "@voicethere/agent";
+
+const ECHO_PREFIX = "you said:";
+const ECHO_WAIT_MS = 4000;
+
+const REPLIES: Record<string, string> = {
+  de: "Guten Tag. Ich antworte jetzt auf Deutsch.",
+  es: "Hola. Ahora respondo en español.",
+  fr: "Bonjour. Je réponds maintenant en français.",
+  it: "Ciao. Adesso rispondo in italiano.",
+  pt: "Olá. Agora respondo em português.",
+  nl: "Hallo. Ik antwoord nu in het Nederlands.",
+  pl: "Dzień dobry. Teraz odpowiadam po polsku.",
+  ru: "Здравствуйте. Теперь я отвечаю по-русски.",
+};
+
+type SessionLanguage = {
+  language: string;
+  runnerAutoSwitch: boolean;
+  echoTimer: ReturnType<typeof setTimeout> | undefined;
+  suppressNextFinal: boolean;
+};
+
+const sessions = new Map<string, SessionLanguage>();
+
+function stateFor(
+  sessionId: string,
+  env?: Record<string, string>,
+): SessionLanguage {
+  const existing = sessions.get(sessionId);
+  if (existing) return existing;
+  const created: SessionLanguage = {
+    language: "en",
+    runnerAutoSwitch: env ? isRunnerLidAutoSwitchEnabled(env) : false,
+    echoTimer: undefined,
+    suppressNextFinal: false,
+  };
+  sessions.set(sessionId, created);
+  return created;
+}
+
+function replyFor(language: string): string {
+  return REPLIES[language] ?? `Continuing in ${language}.`;
+}
+
+function logSwitch(
+  sessionId: string,
+  side: string,
+  result: VoiceLanguageResult,
+): void {
+  if (!result.ok) {
+    agentLog(
+      "warn",
+      `setVoiceLanguage ${side} failed: ${result.reason ?? "unknown"}`,
+    );
+    return;
+  }
+  const provider = side === "tts" ? result.ttsProvider : result.sttProvider;
+  const model = side === "tts" ? result.voice : result.stt;
+  agentLog(
+    "info",
+    `voice ${side} ${sessionId} ${result.language ?? ""} provider=${provider ?? "unchanged"} model=${model ?? "unchanged"}`,
+  );
+}
+
+function prepareLanguageTransition(state: SessionLanguage): void {
+  if (state.echoTimer) {
+    clearTimeout(state.echoTimer);
+    state.echoTimer = undefined;
+  } else {
+    state.suppressNextFinal = true;
+  }
+}
+
+defineAgent({
+  onSessionStart({ sessionId, env }) {
+    const state = stateFor(sessionId, env);
+    if (state.runnerAutoSwitch) {
+      agentLog(
+        "info",
+        `language-switch session_start ${sessionId} runner LID auto-switch owns STT/TTS`,
+      );
+    } else {
+      agentLog(
+        "info",
+        `language-switch session_start ${sessionId} manual setVoiceLanguage`,
+      );
+    }
+  },
+
+  async onUserLanguage({ sessionId, language }) {
+    const state = stateFor(sessionId);
+    if (!language || language === state.language) return;
+
+    prepareLanguageTransition(state);
+
+    if (state.runnerAutoSwitch) {
+      agentLog(
+        "info",
+        `LID detected ${language}; runner auto-switch applies STT/TTS — agent updates prompts only`,
+      );
+      state.language = language;
+      speak(sessionId, replyFor(language));
+      return;
+    }
+
+    const tts = await setVoiceLanguage(sessionId, {
+      scope: "tts",
+      language,
+      voice: language,
+    });
+    logSwitch(sessionId, "tts", tts);
+
+    const stt = await setVoiceLanguage(sessionId, {
+      scope: "stt",
+      language,
+      stt: language,
+    });
+    logSwitch(sessionId, "stt", stt);
+
+    if (!tts.ok) {
+      state.suppressNextFinal = false;
+      return;
+    }
+
+    state.language = language;
+    speak(sessionId, replyFor(language));
+  },
+
+  async onVoiceLanguageChanged({ sessionId, language }) {
+    const state = stateFor(sessionId);
+    if (!state.runnerAutoSwitch || !language || language === state.language) {
+      return;
+    }
+    agentLog(
+      "info",
+      `runner committed voice language ${language} for ${sessionId}`,
+    );
+    state.language = language;
+  },
+
+  async onDataChannelMessage(ctx) {
+    const text = parseChatText(ctx.message);
+    if (!text) return;
+    const [command, vendor, language] = text.trim().split(/\s+/);
+    if (command === "/tts" && vendor === "sherpa" && language) {
+      const result = await setVoiceLanguage(ctx.sessionId, {
+        scope: "tts",
+        language,
+        ttsVendor: { provider: "local-sherpa", voice: language },
+      });
+      logSwitch(ctx.sessionId, "tts", result);
+      if (result.ok) speak(ctx.sessionId, replyFor(language));
+      return;
+    }
+    if (command === "/stt" && vendor === "sherpa" && language) {
+      const result = await setVoiceLanguage(ctx.sessionId, {
+        scope: "stt",
+        language,
+        sttVendor: { provider: "local-sherpa", model: language },
+      });
+      logSwitch(ctx.sessionId, "stt", result);
+      return;
+    }
+    if (command === "/tts" && vendor === "elevenlabs") {
+      const result = await setVoiceLanguage(ctx.sessionId, {
+        scope: "tts",
+        ttsVendor: { provider: "elevenlabs", model: "eleven_multilingual_v2" },
+      });
+      logSwitch(ctx.sessionId, "tts", result);
+      if (result.ok) speak(ctx.sessionId, "Speaking with ElevenLabs.");
+      return;
+    }
+    if (command === "/stt" && vendor === "deepgram") {
+      const result = await setVoiceLanguage(ctx.sessionId, {
+        scope: "stt",
+        sttVendor: {
+          provider: "deepgram",
+          model: "nova-3",
+          language: language || "en",
+        },
+      });
+      logSwitch(ctx.sessionId, "stt", result);
+    }
+  },
+
+  onUserSpeechFinal({ sessionId, text }) {
+    const state = stateFor(sessionId);
+    if (state.suppressNextFinal) {
+      state.suppressNextFinal = false;
+      return;
+    }
+    if (state.echoTimer) clearTimeout(state.echoTimer);
+    state.echoTimer = setTimeout(() => {
+      state.echoTimer = undefined;
+      speak(sessionId, `${ECHO_PREFIX} ${text}`.trim());
+    }, ECHO_WAIT_MS);
+  },
+
+  onSessionEnd({ sessionId }) {
+    const state = sessions.get(sessionId);
+    if (state?.echoTimer) clearTimeout(state.echoTimer);
+    sessions.delete(sessionId);
+  },
+});

package/templates/voice-starter/agent.ts

@@ -179,7 +179,9 @@ defineAgent({
   },
 
   onUserLanguage({ sessionId, language }) {
-    // Route prompts / TTS voice from the detected ISO 639-1 code
+    // Detection does not change TTS or STT. Call setVoiceLanguage with
+    // scope "tts" or "stt" — or pass sttVendor / ttsVendor to change vendor.
+    // See templates/language-switch/agent.ts.
     agentLog("info", `user_language ${sessionId} ${language}`);
   },