@voicethere/agent 0.9.2 → 0.9.3

package/dist/templates/language-switch/agent.js

@@ -952,6 +952,50 @@ var ECHO_PREFIXES = {
   pl: "Powiedzia\u0142e\u015B:",
   ru: "\u0412\u044B \u0441\u043A\u0430\u0437\u0430\u043B\u0438:"
 };
+var WAIT_MESSAGES = {
+  en: "One moment please, I'm switching to your language.",
+  de: "Einen Moment bitte, ich wechsle zu Ihrer Sprache.",
+  es: "Un momento, por favor, estoy cambiando a su idioma.",
+  fr: "Un instant, je passe dans votre langue.",
+  it: "Un momento, per favore, sto passando alla tua lingua.",
+  pt: "Um momento, por favor, estou mudando para o seu idioma.",
+  nl: "Een moment alstublieft, ik schakel over naar uw taal.",
+  pl: "Chwileczk\u0119, prze\u0142\u0105czam si\u0119 na Tw\xF3j j\u0119zyk.",
+  ru: "\u041E\u0434\u043D\u0443 \u043C\u0438\u043D\u0443\u0442\u0443, \u044F \u043F\u0435\u0440\u0435\u0445\u043E\u0436\u0443 \u043D\u0430 \u0432\u0430\u0448 \u044F\u0437\u044B\u043A."
+};
+var FAILED_MESSAGES = {
+  en: "Sorry, I couldn't switch languages. I'll keep going in English.",
+  de: "Entschuldigung, der Sprachwechsel hat nicht geklappt. Ich bleibe bei Deutsch.",
+  es: "Lo siento, no pude cambiar de idioma. Sigo en espa\xF1ol.",
+  fr: "D\xE9sol\xE9, je n'ai pas pu changer de langue. Je continue en fran\xE7ais.",
+  it: "Mi dispiace, non sono riuscito a cambiare lingua. Continuo in italiano.",
+  pt: "Desculpe, n\xE3o consegui mudar de idioma. Continuo em portugu\xEAs.",
+  nl: "Sorry, het wisselen van taal is niet gelukt. Ik blijf Nederlands spreken.",
+  pl: "Przepraszam, nie uda\u0142o si\u0119 zmieni\u0107 j\u0119zyka. Zostaj\u0119 przy polskim.",
+  ru: "\u0418\u0437\u0432\u0438\u043D\u0438\u0442\u0435, \u043D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0441\u043C\u0435\u043D\u0438\u0442\u044C \u044F\u0437\u044B\u043A. \u041F\u0440\u043E\u0434\u043E\u043B\u0436\u0430\u044E \u043F\u043E-\u0440\u0443\u0441\u0441\u043A\u0438."
+};
+var WAIT_MESSAGES_ENV = "LANGUAGE_SWITCH_WAIT_MESSAGES_JSON";
+var FAILED_MESSAGES_ENV = "LANGUAGE_SWITCH_FAILED_MESSAGES_JSON";
+function parseMessageOverrides(raw) {
+  if (!raw) return {};
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return {};
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
+  const out = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    if (typeof value === "string" && value.trim()) {
+      out[key.trim().toLowerCase()] = value.trim();
+    }
+  }
+  return out;
+}
+function pickMessage(defaults, overrides, language) {
+  return overrides[language] ?? defaults[language] ?? overrides.en ?? defaults.en;
+}
 var REPLIES = {
   de: "Guten Tag. Ich antworte jetzt auf Deutsch.",
   es: "Hola. Ahora respondo en espa\xF1ol.",
@@ -970,7 +1014,11 @@ function stateFor(sessionId, env) {
     language: "en",
     runnerAutoSwitch: env ? isRunnerLidAutoSwitchEnabled(env) : false,
     echoTimer: void 0,
-    suppressNextFinal: false
+    suppressNextFinal: false,
+    pendingSwitch: void 0,
+    switching: false,
+    waitOverrides: parseMessageOverrides(env?.[WAIT_MESSAGES_ENV]),
+    failedOverrides: parseMessageOverrides(env?.[FAILED_MESSAGES_ENV])
   };
   sessions.set(sessionId, created);
   return created;
@@ -996,6 +1044,44 @@ function logSwitch(sessionId, side, result) {
     `voice ${side} ${sessionId} ${result.language ?? ""} provider=${provider ?? "unchanged"} model=${model ?? "unchanged"}`
   );
 }
+var WAIT_SPEECH_TIMEOUT_MS = 8e3;
+function startSwitch(sessionId, state) {
+  const pending = state.pendingSwitch;
+  if (!pending) return;
+  clearTimeout(pending.timer);
+  state.pendingSwitch = void 0;
+  state.switching = true;
+  void runSwitch(sessionId, state, pending.language, pending.previous).catch((error) => {
+    state.suppressNextFinal = false;
+    agentLog("warn", `language-switch ${sessionId} failed: ${String(error)}`);
+  }).finally(() => {
+    state.switching = false;
+  });
+}
+async function runSwitch(sessionId, state, language, previous) {
+  const tts = await setVoiceLanguage(sessionId, {
+    scope: "tts",
+    language,
+    voice: language
+  });
+  logSwitch(sessionId, "tts", tts);
+  const stt = await setVoiceLanguage(sessionId, {
+    scope: "stt",
+    language,
+    stt: language
+  });
+  logSwitch(sessionId, "stt", stt);
+  if (!tts.ok) {
+    state.suppressNextFinal = false;
+    speak(
+      sessionId,
+      pickMessage(FAILED_MESSAGES, state.failedOverrides, previous)
+    );
+    return;
+  }
+  state.language = language;
+  speak(sessionId, replyFor(language));
+}
 function prepareLanguageTransition(state) {
   if (state.echoTimer) {
     clearTimeout(state.echoTimer);
@@ -1030,25 +1116,23 @@ defineAgent({
       state.language = language;
       return;
     }
+    if (state.pendingSwitch || state.switching) return;
     prepareLanguageTransition(state);
-    const tts = await setVoiceLanguage(sessionId, {
-      scope: "tts",
-      language,
-      voice: language
-    });
-    logSwitch(sessionId, "tts", tts);
-    const stt = await setVoiceLanguage(sessionId, {
-      scope: "stt",
-      language,
-      stt: language
-    });
-    logSwitch(sessionId, "stt", stt);
-    if (!tts.ok) {
-      state.suppressNextFinal = false;
-      return;
-    }
-    state.language = language;
-    speak(sessionId, replyFor(language));
+    const previous = state.language;
+    speak(sessionId, pickMessage(WAIT_MESSAGES, state.waitOverrides, previous));
+    const timer = setTimeout(() => {
+      agentLog(
+        "warn",
+        `language-switch ${sessionId} no agent_speaking_end within ${WAIT_SPEECH_TIMEOUT_MS}ms; switching anyway`
+      );
+      startSwitch(sessionId, state);
+    }, WAIT_SPEECH_TIMEOUT_MS);
+    state.pendingSwitch = { language, previous, timer };
+  },
+  onSpeechEvent({ sessionId }, event) {
+    if (event.type !== "agent_speaking_end") return;
+    const state = sessions.get(sessionId);
+    if (state?.pendingSwitch) startSwitch(sessionId, state);
   },
   async onVoiceLanguageChanged({ sessionId, language }) {
     const state = stateFor(sessionId);
@@ -1124,6 +1208,16 @@ defineAgent({
   onSessionEnd({ sessionId }) {
     const state = sessions.get(sessionId);
     if (state?.echoTimer) clearTimeout(state.echoTimer);
+    if (state?.pendingSwitch) clearTimeout(state.pendingSwitch.timer);
     sessions.delete(sessionId);
   }
 });
+export {
+  FAILED_MESSAGES,
+  FAILED_MESSAGES_ENV,
+  WAIT_MESSAGES,
+  WAIT_MESSAGES_ENV,
+  WAIT_SPEECH_TIMEOUT_MS,
+  parseMessageOverrides,
+  pickMessage
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicethere/agent",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "description": "VoiceThere customer agent SDK — IPC types and runtime helpers for sandboxed child bundles",
   "type": "module",
   "exports": {

package/templates/language-switch/README.md

@@ -9,12 +9,23 @@ Leave the project **spoken-language auto-switch** setting off. When `onUserLangu
 - `scope: "tts"` switches only the speaking voice
 - `scope: "stt"` switches only the listening model
 
+The runner plays nothing for a manual switch, and the target voice and STT pools can take several seconds to warm up. So the agent first speaks a short wait message in the language it is leaving, using the current voice (for example `One moment please, I'm switching to your language.` in English, `Einen Moment bitte, ich wechsle zu Ihrer Sprache.` in German). The runner swaps the voice without draining it, so the agent waits for the `agent_speaking_end` speech event (at most 8 seconds, `WAIT_SPEECH_TIMEOUT_MS`) before it calls `setVoiceLanguage` for TTS and STT and replies in the new language. A second language event during the wait or the switch is ignored. The utterance that revealed the language is not echoed back. If the TTS switch fails (a timeout, for instance), the agent speaks a short fallback in the old language and stays in it.
+
+The texts live in the `WAIT_MESSAGES` and `FAILED_MESSAGES` maps in `agent.ts` (en, de, es, fr, it, pt, nl, pl, ru). Override any of them per language with a JSON object in the session env, keyed by ISO 639-1 code:
+
+- `LANGUAGE_SWITCH_WAIT_MESSAGES_JSON` — wait message, for example `{"en":"Hold on, switching languages."}`
+- `LANGUAGE_SWITCH_FAILED_MESSAGES_JSON` — fallback after a failed switch
+
+Invalid JSON or non-string values are ignored and the built-in text is used. A language without a text falls back to English.
+
 A Sherpa language with no STT id (`it`, `pt`, `nl`, `pl`, `hi`) fails the STT call and still switches TTS.
 
 ## Mode B — Runner auto-switch
 
 Enable auto-switch in the project voice settings (runner applies STT/TTS when LID detects a new language). When `session_start.env` includes a truthy `SHERPA_LID_AUTO_SWITCH`, this template **does not** call `setVoiceLanguage`. The runner replays the utterance into the new language's STT, so the next final is the recognized text in the new language. The agent remembers the language from `onUserLanguage` / `onVoiceLanguageChanged` and answers each final right away with a short localized prefix, for example `you said: …` in English and `Du hast gesagt: …` in German. Use `onVoiceLanguageChanged` when you need the committed language after the runner applies the change.
 
+In this mode the runner owns the wait message and the agent speaks no wait or fallback text of its own.
+
 Detection and chat commands are unchanged: `/tts` and `/stt` still call `setVoiceLanguage` for one vendor at a time.
 
 Chat commands change one vendor while the session stays connected. API keys are project secrets on the running deploy, not arguments:

package/templates/language-switch/agent.ts

@@ -4,7 +4,17 @@
  * Two deployment modes (see README):
  *
  * (A) Project auto-switch off — this agent calls `setVoiceLanguage` from
- *     `onUserLanguage` when LID detects a new language.
+ *     `onUserLanguage` when LID detects a new language. The runner plays
+ *     nothing for manual switches and the target pools may be cold for
+ *     several seconds, so the agent first speaks a wait message in the
+ *     language being left (current voice), waits for `agent_speaking_end`
+ *     (at most WAIT_SPEECH_TIMEOUT_MS) so the runner does not swap the voice
+ *     mid-sentence, then switches TTS and STT, then replies in the new
+ *     language. If the switch fails it speaks a short
+ *     fallback in the old language and stays. Texts live in WAIT_MESSAGES /
+ *     FAILED_MESSAGES and can be overridden per language with the session env
+ *     vars LANGUAGE_SWITCH_WAIT_MESSAGES_JSON and
+ *     LANGUAGE_SWITCH_FAILED_MESSAGES_JSON (JSON object, ISO 639-1 -> text).
  * (B) Project enables runner auto-switch — STT/TTS are runner-owned. The
  *     runner replays the utterance into the new language's STT, so the next
  *     final is the correctly recognized text. The agent never calls
@@ -47,6 +57,70 @@ const ECHO_PREFIXES: Record<string, string> = {
   ru: "Вы сказали:",
 };
 
+/** Spoken in the language being left, before a manual switch starts. */
+export const WAIT_MESSAGES: Record<string, string> = {
+  en: "One moment please, I'm switching to your language.",
+  de: "Einen Moment bitte, ich wechsle zu Ihrer Sprache.",
+  es: "Un momento, por favor, estoy cambiando a su idioma.",
+  fr: "Un instant, je passe dans votre langue.",
+  it: "Un momento, per favore, sto passando alla tua lingua.",
+  pt: "Um momento, por favor, estou mudando para o seu idioma.",
+  nl: "Een moment alstublieft, ik schakel over naar uw taal.",
+  pl: "Chwileczkę, przełączam się na Twój język.",
+  ru: "Одну минуту, я перехожу на ваш язык.",
+};
+
+/** Spoken in the old language when the switch fails and the agent stays. */
+export const FAILED_MESSAGES: Record<string, string> = {
+  en: "Sorry, I couldn't switch languages. I'll keep going in English.",
+  de: "Entschuldigung, der Sprachwechsel hat nicht geklappt. Ich bleibe bei Deutsch.",
+  es: "Lo siento, no pude cambiar de idioma. Sigo en español.",
+  fr: "Désolé, je n'ai pas pu changer de langue. Je continue en français.",
+  it: "Mi dispiace, non sono riuscito a cambiare lingua. Continuo in italiano.",
+  pt: "Desculpe, não consegui mudar de idioma. Continuo em português.",
+  nl: "Sorry, het wisselen van taal is niet gelukt. Ik blijf Nederlands spreken.",
+  pl: "Przepraszam, nie udało się zmienić języka. Zostaję przy polskim.",
+  ru: "Извините, не удалось сменить язык. Продолжаю по-русски.",
+};
+
+export const WAIT_MESSAGES_ENV = "LANGUAGE_SWITCH_WAIT_MESSAGES_JSON";
+export const FAILED_MESSAGES_ENV = "LANGUAGE_SWITCH_FAILED_MESSAGES_JSON";
+
+/**
+ * Parse a per-language override from a session env var. Accepts a JSON object
+ * of language code -> non-empty string; anything else yields no overrides.
+ */
+export function parseMessageOverrides(
+  raw: string | undefined,
+): Record<string, string> {
+  if (!raw) return {};
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return {};
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
+  const out: Record<string, string> = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    if (typeof value === "string" && value.trim()) {
+      out[key.trim().toLowerCase()] = value.trim();
+    }
+  }
+  return out;
+}
+
+/** Message for `language`: override, then default, then English default. */
+export function pickMessage(
+  defaults: Record<string, string>,
+  overrides: Record<string, string>,
+  language: string,
+): string {
+  return (
+    overrides[language] ?? defaults[language] ?? overrides.en ?? defaults.en!
+  );
+}
+
 const REPLIES: Record<string, string> = {
   de: "Guten Tag. Ich antworte jetzt auf Deutsch.",
   es: "Hola. Ahora respondo en español.",
@@ -63,6 +137,16 @@ type SessionLanguage = {
   runnerAutoSwitch: boolean;
   echoTimer: ReturnType<typeof setTimeout> | undefined;
   suppressNextFinal: boolean;
+  pendingSwitch:
+    | {
+        language: string;
+        previous: string;
+        timer: ReturnType<typeof setTimeout>;
+      }
+    | undefined;
+  switching: boolean;
+  waitOverrides: Record<string, string>;
+  failedOverrides: Record<string, string>;
 };
 
 const sessions = new Map<string, SessionLanguage>();
@@ -78,6 +162,10 @@ function stateFor(
     runnerAutoSwitch: env ? isRunnerLidAutoSwitchEnabled(env) : false,
     echoTimer: undefined,
     suppressNextFinal: false,
+    pendingSwitch: undefined,
+    switching: false,
+    waitOverrides: parseMessageOverrides(env?.[WAIT_MESSAGES_ENV]),
+    failedOverrides: parseMessageOverrides(env?.[FAILED_MESSAGES_ENV]),
   };
   sessions.set(sessionId, created);
   return created;
@@ -111,6 +199,61 @@ function logSwitch(
   );
 }
 
+/** Upper bound for waiting on the wait message before switching anyway. */
+export const WAIT_SPEECH_TIMEOUT_MS = 8000;
+
+/** Consume the pending switch (once) and run it without blocking handlers. */
+function startSwitch(sessionId: string, state: SessionLanguage): void {
+  const pending = state.pendingSwitch;
+  if (!pending) return;
+  clearTimeout(pending.timer);
+  state.pendingSwitch = undefined;
+  state.switching = true;
+  void runSwitch(sessionId, state, pending.language, pending.previous)
+    .catch((error: unknown) => {
+      state.suppressNextFinal = false;
+      agentLog("warn", `language-switch ${sessionId} failed: ${String(error)}`);
+    })
+    .finally(() => {
+      state.switching = false;
+    });
+}
+
+async function runSwitch(
+  sessionId: string,
+  state: SessionLanguage,
+  language: string,
+  previous: string,
+): Promise<void> {
+  const tts = await setVoiceLanguage(sessionId, {
+    scope: "tts",
+    language,
+    voice: language,
+  });
+  logSwitch(sessionId, "tts", tts);
+
+  const stt = await setVoiceLanguage(sessionId, {
+    scope: "stt",
+    language,
+    stt: language,
+  });
+  logSwitch(sessionId, "stt", stt);
+
+  if (!tts.ok) {
+    state.suppressNextFinal = false;
+    // Switch failed (for example a timeout): the voice is unchanged, so
+    // apologise in the old language and stay.
+    speak(
+      sessionId,
+      pickMessage(FAILED_MESSAGES, state.failedOverrides, previous),
+    );
+    return;
+  }
+
+  state.language = language;
+  speak(sessionId, replyFor(language));
+}
+
 function prepareLanguageTransition(state: SessionLanguage): void {
   if (state.echoTimer) {
     clearTimeout(state.echoTimer);
@@ -152,29 +295,35 @@ defineAgent({
       return;
     }
 
-    prepareLanguageTransition(state);
+    // A switch is already waiting for the wait message or in flight.
+    if (state.pendingSwitch || state.switching) return;
 
-    const tts = await setVoiceLanguage(sessionId, {
-      scope: "tts",
-      language,
-      voice: language,
-    });
-    logSwitch(sessionId, "tts", tts);
+    prepareLanguageTransition(state);
 
-    const stt = await setVoiceLanguage(sessionId, {
-      scope: "stt",
-      language,
-      stt: language,
-    });
-    logSwitch(sessionId, "stt", stt);
+    // The runner plays nothing for manual switches and the target pools may be
+    // cold, so tell the user first, in the language being left.
+    const previous = state.language;
+    speak(sessionId, pickMessage(WAIT_MESSAGES, state.waitOverrides, previous));
 
-    if (!tts.ok) {
-      state.suppressNextFinal = false;
-      return;
-    }
+    // `speak` is fire-and-forget and the runner swaps the TTS without draining
+    // it, so the switch must wait until the wait message has been spoken
+    // (`agent_speaking_end`, delivered to onSpeechEvent). Inbound handlers of
+    // one session run strictly in order, so this handler must return instead
+    // of awaiting that event. The bounded timer covers a missing event.
+    const timer = setTimeout(() => {
+      agentLog(
+        "warn",
+        `language-switch ${sessionId} no agent_speaking_end within ${WAIT_SPEECH_TIMEOUT_MS}ms; switching anyway`,
+      );
+      startSwitch(sessionId, state);
+    }, WAIT_SPEECH_TIMEOUT_MS);
+    state.pendingSwitch = { language, previous, timer };
+  },
 
-    state.language = language;
-    speak(sessionId, replyFor(language));
+  onSpeechEvent({ sessionId }, event) {
+    if (event.type !== "agent_speaking_end") return;
+    const state = sessions.get(sessionId);
+    if (state?.pendingSwitch) startSwitch(sessionId, state);
   },
 
   async onVoiceLanguageChanged({ sessionId, language }) {
@@ -256,6 +405,7 @@ defineAgent({
   onSessionEnd({ sessionId }) {
     const state = sessions.get(sessionId);
     if (state?.echoTimer) clearTimeout(state.echoTimer);
+    if (state?.pendingSwitch) clearTimeout(state.pendingSwitch.timer);
     sessions.delete(sessionId);
   },
 });