@voicethere/agent 0.9.1 → 0.9.3

package/dist/templates/language-switch/agent.js

@@ -941,6 +941,61 @@ function parseChatText(message) {
 // templates/language-switch/agent.ts
 var ECHO_PREFIX = "you said:";
 var ECHO_WAIT_MS = 4e3;
+var ECHO_PREFIXES = {
+  en: "you said:",
+  de: "Du hast gesagt:",
+  es: "Dijiste:",
+  fr: "Tu as dit :",
+  it: "Hai detto:",
+  pt: "Voc\xEA disse:",
+  nl: "Je zei:",
+  pl: "Powiedzia\u0142e\u015B:",
+  ru: "\u0412\u044B \u0441\u043A\u0430\u0437\u0430\u043B\u0438:"
+};
+var WAIT_MESSAGES = {
+  en: "One moment please, I'm switching to your language.",
+  de: "Einen Moment bitte, ich wechsle zu Ihrer Sprache.",
+  es: "Un momento, por favor, estoy cambiando a su idioma.",
+  fr: "Un instant, je passe dans votre langue.",
+  it: "Un momento, per favore, sto passando alla tua lingua.",
+  pt: "Um momento, por favor, estou mudando para o seu idioma.",
+  nl: "Een moment alstublieft, ik schakel over naar uw taal.",
+  pl: "Chwileczk\u0119, prze\u0142\u0105czam si\u0119 na Tw\xF3j j\u0119zyk.",
+  ru: "\u041E\u0434\u043D\u0443 \u043C\u0438\u043D\u0443\u0442\u0443, \u044F \u043F\u0435\u0440\u0435\u0445\u043E\u0436\u0443 \u043D\u0430 \u0432\u0430\u0448 \u044F\u0437\u044B\u043A."
+};
+var FAILED_MESSAGES = {
+  en: "Sorry, I couldn't switch languages. I'll keep going in English.",
+  de: "Entschuldigung, der Sprachwechsel hat nicht geklappt. Ich bleibe bei Deutsch.",
+  es: "Lo siento, no pude cambiar de idioma. Sigo en espa\xF1ol.",
+  fr: "D\xE9sol\xE9, je n'ai pas pu changer de langue. Je continue en fran\xE7ais.",
+  it: "Mi dispiace, non sono riuscito a cambiare lingua. Continuo in italiano.",
+  pt: "Desculpe, n\xE3o consegui mudar de idioma. Continuo em portugu\xEAs.",
+  nl: "Sorry, het wisselen van taal is niet gelukt. Ik blijf Nederlands spreken.",
+  pl: "Przepraszam, nie uda\u0142o si\u0119 zmieni\u0107 j\u0119zyka. Zostaj\u0119 przy polskim.",
+  ru: "\u0418\u0437\u0432\u0438\u043D\u0438\u0442\u0435, \u043D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0441\u043C\u0435\u043D\u0438\u0442\u044C \u044F\u0437\u044B\u043A. \u041F\u0440\u043E\u0434\u043E\u043B\u0436\u0430\u044E \u043F\u043E-\u0440\u0443\u0441\u0441\u043A\u0438."
+};
+var WAIT_MESSAGES_ENV = "LANGUAGE_SWITCH_WAIT_MESSAGES_JSON";
+var FAILED_MESSAGES_ENV = "LANGUAGE_SWITCH_FAILED_MESSAGES_JSON";
+function parseMessageOverrides(raw) {
+  if (!raw) return {};
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return {};
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
+  const out = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    if (typeof value === "string" && value.trim()) {
+      out[key.trim().toLowerCase()] = value.trim();
+    }
+  }
+  return out;
+}
+function pickMessage(defaults, overrides, language) {
+  return overrides[language] ?? defaults[language] ?? overrides.en ?? defaults.en;
+}
 var REPLIES = {
   de: "Guten Tag. Ich antworte jetzt auf Deutsch.",
   es: "Hola. Ahora respondo en espa\xF1ol.",
@@ -959,7 +1014,11 @@ function stateFor(sessionId, env) {
     language: "en",
     runnerAutoSwitch: env ? isRunnerLidAutoSwitchEnabled(env) : false,
     echoTimer: void 0,
-    suppressNextFinal: false
+    suppressNextFinal: false,
+    pendingSwitch: void 0,
+    switching: false,
+    waitOverrides: parseMessageOverrides(env?.[WAIT_MESSAGES_ENV]),
+    failedOverrides: parseMessageOverrides(env?.[FAILED_MESSAGES_ENV])
   };
   sessions.set(sessionId, created);
   return created;
@@ -967,6 +1026,9 @@ function stateFor(sessionId, env) {
 function replyFor(language) {
   return REPLIES[language] ?? `Continuing in ${language}.`;
 }
+function echoPrefixFor(language) {
+  return ECHO_PREFIXES[language] ?? ECHO_PREFIXES.en;
+}
 function logSwitch(sessionId, side, result) {
   if (!result.ok) {
     agentLog(
@@ -982,6 +1044,44 @@ function logSwitch(sessionId, side, result) {
     `voice ${side} ${sessionId} ${result.language ?? ""} provider=${provider ?? "unchanged"} model=${model ?? "unchanged"}`
   );
 }
+var WAIT_SPEECH_TIMEOUT_MS = 8e3;
+function startSwitch(sessionId, state) {
+  const pending = state.pendingSwitch;
+  if (!pending) return;
+  clearTimeout(pending.timer);
+  state.pendingSwitch = void 0;
+  state.switching = true;
+  void runSwitch(sessionId, state, pending.language, pending.previous).catch((error) => {
+    state.suppressNextFinal = false;
+    agentLog("warn", `language-switch ${sessionId} failed: ${String(error)}`);
+  }).finally(() => {
+    state.switching = false;
+  });
+}
+async function runSwitch(sessionId, state, language, previous) {
+  const tts = await setVoiceLanguage(sessionId, {
+    scope: "tts",
+    language,
+    voice: language
+  });
+  logSwitch(sessionId, "tts", tts);
+  const stt = await setVoiceLanguage(sessionId, {
+    scope: "stt",
+    language,
+    stt: language
+  });
+  logSwitch(sessionId, "stt", stt);
+  if (!tts.ok) {
+    state.suppressNextFinal = false;
+    speak(
+      sessionId,
+      pickMessage(FAILED_MESSAGES, state.failedOverrides, previous)
+    );
+    return;
+  }
+  state.language = language;
+  speak(sessionId, replyFor(language));
+}
 function prepareLanguageTransition(state) {
   if (state.echoTimer) {
     clearTimeout(state.echoTimer);
@@ -1008,34 +1108,31 @@ defineAgent({
   async onUserLanguage({ sessionId, language }) {
     const state = stateFor(sessionId);
     if (!language || language === state.language) return;
-    prepareLanguageTransition(state);
     if (state.runnerAutoSwitch) {
       agentLog(
         "info",
-        `LID detected ${language}; runner auto-switch applies STT/TTS \u2014 agent updates prompts only`
+        `LID detected ${language}; runner auto-switch applies STT/TTS \u2014 agent only remembers the language`
       );
       state.language = language;
-      speak(sessionId, replyFor(language));
       return;
     }
-    const tts = await setVoiceLanguage(sessionId, {
-      scope: "tts",
-      language,
-      voice: language
-    });
-    logSwitch(sessionId, "tts", tts);
-    const stt = await setVoiceLanguage(sessionId, {
-      scope: "stt",
-      language,
-      stt: language
-    });
-    logSwitch(sessionId, "stt", stt);
-    if (!tts.ok) {
-      state.suppressNextFinal = false;
-      return;
-    }
-    state.language = language;
-    speak(sessionId, replyFor(language));
+    if (state.pendingSwitch || state.switching) return;
+    prepareLanguageTransition(state);
+    const previous = state.language;
+    speak(sessionId, pickMessage(WAIT_MESSAGES, state.waitOverrides, previous));
+    const timer = setTimeout(() => {
+      agentLog(
+        "warn",
+        `language-switch ${sessionId} no agent_speaking_end within ${WAIT_SPEECH_TIMEOUT_MS}ms; switching anyway`
+      );
+      startSwitch(sessionId, state);
+    }, WAIT_SPEECH_TIMEOUT_MS);
+    state.pendingSwitch = { language, previous, timer };
+  },
+  onSpeechEvent({ sessionId }, event) {
+    if (event.type !== "agent_speaking_end") return;
+    const state = sessions.get(sessionId);
+    if (state?.pendingSwitch) startSwitch(sessionId, state);
   },
   async onVoiceLanguageChanged({ sessionId, language }) {
     const state = stateFor(sessionId);
@@ -1094,6 +1191,10 @@ defineAgent({
   },
   onUserSpeechFinal({ sessionId, text }) {
     const state = stateFor(sessionId);
+    if (state.runnerAutoSwitch) {
+      speak(sessionId, `${echoPrefixFor(state.language)} ${text}`.trim());
+      return;
+    }
     if (state.suppressNextFinal) {
       state.suppressNextFinal = false;
       return;
@@ -1107,6 +1208,16 @@ defineAgent({
   onSessionEnd({ sessionId }) {
     const state = sessions.get(sessionId);
     if (state?.echoTimer) clearTimeout(state.echoTimer);
+    if (state?.pendingSwitch) clearTimeout(state.pendingSwitch.timer);
     sessions.delete(sessionId);
   }
 });
+export {
+  FAILED_MESSAGES,
+  FAILED_MESSAGES_ENV,
+  WAIT_MESSAGES,
+  WAIT_MESSAGES_ENV,
+  WAIT_SPEECH_TIMEOUT_MS,
+  parseMessageOverrides,
+  pickMessage
+};

package/dist/templates/registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/templates/registry.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,KAAK,CAAC;AAE7C,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,mFAAmF;IACnF,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,YAAY,CAAC;IACnB,mFAAmF;IACnF,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,oFAAoF;IACpF,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED,eAAO,MAAM,eAAe,EAAE,SAAS,uBAAuB,EA8LpD,CAAC;AAEX,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;AAMrE,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,EAAE,IAAI,eAAe,CAEnE;AAED,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,uBAAuB,CAMnE;AAED,wBAAgB,yBAAyB,IAAI,uBAAuB,EAAE,CAErE"}
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/templates/registry.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,KAAK,CAAC;AAE7C,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,mFAAmF;IACnF,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,YAAY,CAAC;IACnB,mFAAmF;IACnF,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,oFAAoF;IACpF,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED,eAAO,MAAM,eAAe,EAAE,SAAS,uBAAuB,EAiMpD,CAAC;AAEX,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;AAMrE,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,EAAE,IAAI,eAAe,CAEnE;AAED,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,uBAAuB,CAMnE;AAED,wBAAgB,yBAAyB,IAAI,uBAAuB,EAAE,CAErE"}

package/dist/templates/registry.js

@@ -20,7 +20,7 @@ export const AGENT_TEMPLATES = [
         entry: "language-switch/agent.ts",
         kind: "product",
         seedOnCreate: true,
-        description: "Switches STT and TTS independently from onUserLanguage, and can change one vendor mid-conversation. Detection does not change either side until this agent calls setVoiceLanguage.",
+        description: "Switches STT and TTS independently from onUserLanguage, and can change one vendor mid-conversation. With project auto-switch on, the runner switches and replays the utterance, and the agent answers the recognized text in the new language. With auto-switch off, detection does not change either side until this agent calls setVoiceLanguage.",
         sourceFiles: ["language-switch/agent.ts"],
     },
     {
@@ -45,7 +45,10 @@ export const AGENT_TEMPLATES = [
         kind: "product",
         seedOnCreate: true,
         description: "Single-agent binary world sync — onDataChannelBinary + sendBinaryToClient ArrayBuffer poses, no Redis.",
-        sourceFiles: ["world-sync-binary/agent.ts", "world-sync-binary/protocol.ts"],
+        sourceFiles: [
+            "world-sync-binary/agent.ts",
+            "world-sync-binary/protocol.ts",
+        ],
     },
     {
         id: "game-sync",

package/dist/templates/registry.js.map

@@ -1 +1 @@
-{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/templates/registry.ts"],"names":[],"mappings":"AAgBA,MAAM,CAAC,MAAM,eAAe,GAAuC;IACjE;QACE,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,eAAe;QACtB,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,8EAA8E;QAChF,WAAW,EAAE,CAAC,eAAe,CAAC;KAC/B;IACD;QACE,EAAE,EAAE,SAAS;QACb,KAAK,EAAE,kBAAkB;QACzB,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,qEAAqE;QACvE,WAAW,EAAE,CAAC,kBAAkB,CAAC;KAClC;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,0BAA0B;QACjC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,oLAAoL;QACtL,WAAW,EAAE,CAAC,0BAA0B,CAAC;KAC1C;IACD;QACE,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,wBAAwB;QAC/B,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,uFAAuF;QACzF,WAAW,EAAE,CAAC,wBAAwB,CAAC;KACxC;IACD;QACE,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,qBAAqB;QAC5B,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,wFAAwF;QAC1F,WAAW,EAAE,CAAC,qBAAqB,CAAC;KACrC;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,4BAA4B;QACnC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,wGAAwG;QAC1G,WAAW,EAAE,CAAC,4BAA4B,EAAE,+BAA+B,CAAC;KAC7E;IACD;QACE,EAAE,EAAE,WAAW;QACf,KAAK,EAAE,oBAAoB;QAC3B,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,sGAAsG;QACxG,WAAW,EAAE;YACX,oBAAoB;YACpB,uBAAuB;YACvB,2BAA2B;YAC3B,kBAAkB;YAClB,oBAAoB;SACrB;QACD,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;KACxC;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,yBAAyB;QAChC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,0FAA0F;QAC5F,WAAW,EAAE;YACX,yBAAyB;YACzB,gCAAgC;YAChC,4BAA4B;YAC5B,2BAA2B;YAC3B,2BAA2B;YAC3B,6BAA6B;SAC9B;KACF;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,4BAA4B;QACnC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,mFAAmF;QACrF,WAAW,EAAE;YACX,4BAA4B;YAC5B,mCAAmC;SACpC;KACF;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,yBAAyB;QAChC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,gFAAgF;QAClF,WAAW,EAAE,CAAC,yBAAyB,EAAE,yBAAyB,CAAC;KACpE;IACD;QACE,EAAE,EAAE,kBAAkB;QACtB,KAAK,EAAE,2BAA2B;QAClC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,oHAAoH;QACtH,WAAW,EAAE;YACX,2BAA2B;YAC3B,8BAA8B;YAC9B,4BAA4B;YAC5B,0BAA0B;YAC1B,2BAA2B;YAC3B,mCAAmC;YACnC,0BAA0B;SAC3B;KACF;IACD;QACE,EAAE,EAAE,UAAU;QACd,KAAK,EAAE,mBAAmB;QAC1B,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,sFAAsF;QACxF,WAAW,EAAE,CAAC,mBAAmB,CAAC;KACnC;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,yBAAyB;QAChC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,4EAA4E;QAC9E,WAAW,EAAE,CAAC,yBAAyB,CAAC;QACxC,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;KACxC;IACD;QACE,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,qBAAqB;QAC5B,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,6EAA6E;QAC/E,WAAW,EAAE,CAAC,qBAAqB,CAAC;KACrC;IACD;QACE,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,gBAAgB;QACvB,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,4EAA4E;QAC9E,WAAW,EAAE,CAAC,gBAAgB,CAAC;KAChC;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,0BAA0B;QACjC,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,6EAA6E;QAC/E,WAAW,EAAE,CAAC,0BAA0B,CAAC;KAC1C;IACD;QACE,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,qBAAqB;QAC5B,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,uGAAuG;QACzG,WAAW,EAAE,CAAC,qBAAqB,EAAE,4BAA4B,CAAC;QAClE,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;KACxC;IACD;QACE,EAAE,EAAE,WAAW;QACf,KAAK,EAAE,oBAAoB;QAC3B,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,iFAAiF;QACnF,WAAW,EAAE,CAAC,oBAAoB,CAAC;KACpC;CACO,CAAC;AAIX,MAAM,cAAc,GAAG,IAAI,GAAG,CAC5B,eAAe,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAC3D,CAAC;AAEF,MAAM,UAAU,iBAAiB,CAAC,EAAU;IAC1C,OAAO,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,8BAA8B,EAAE,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;AACrE,CAAC"}
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/templates/registry.ts"],"names":[],"mappings":"AAgBA,MAAM,CAAC,MAAM,eAAe,GAAuC;IACjE;QACE,EAAE,EAAE,MAAM;QACV,KAAK,EAAE,eAAe;QACtB,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,8EAA8E;QAChF,WAAW,EAAE,CAAC,eAAe,CAAC;KAC/B;IACD;QACE,EAAE,EAAE,SAAS;QACb,KAAK,EAAE,kBAAkB;QACzB,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,qEAAqE;QACvE,WAAW,EAAE,CAAC,kBAAkB,CAAC;KAClC;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,0BAA0B;QACjC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,qVAAqV;QACvV,WAAW,EAAE,CAAC,0BAA0B,CAAC;KAC1C;IACD;QACE,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,wBAAwB;QAC/B,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,uFAAuF;QACzF,WAAW,EAAE,CAAC,wBAAwB,CAAC;KACxC;IACD;QACE,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,qBAAqB;QAC5B,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,wFAAwF;QAC1F,WAAW,EAAE,CAAC,qBAAqB,CAAC;KACrC;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,4BAA4B;QACnC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,wGAAwG;QAC1G,WAAW,EAAE;YACX,4BAA4B;YAC5B,+BAA+B;SAChC;KACF;IACD;QACE,EAAE,EAAE,WAAW;QACf,KAAK,EAAE,oBAAoB;QAC3B,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,sGAAsG;QACxG,WAAW,EAAE;YACX,oBAAoB;YACpB,uBAAuB;YACvB,2BAA2B;YAC3B,kBAAkB;YAClB,oBAAoB;SACrB;QACD,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;KACxC;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,yBAAyB;QAChC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,0FAA0F;QAC5F,WAAW,EAAE;YACX,yBAAyB;YACzB,gCAAgC;YAChC,4BAA4B;YAC5B,2BAA2B;YAC3B,2BAA2B;YAC3B,6BAA6B;SAC9B;KACF;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,4BAA4B;QACnC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,mFAAmF;QACrF,WAAW,EAAE;YACX,4BAA4B;YAC5B,mCAAmC;SACpC;KACF;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,yBAAyB;QAChC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,gFAAgF;QAClF,WAAW,EAAE,CAAC,yBAAyB,EAAE,yBAAyB,CAAC;KACpE;IACD;QACE,EAAE,EAAE,kBAAkB;QACtB,KAAK,EAAE,2BAA2B;QAClC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,oHAAoH;QACtH,WAAW,EAAE;YACX,2BAA2B;YAC3B,8BAA8B;YAC9B,4BAA4B;YAC5B,0BAA0B;YAC1B,2BAA2B;YAC3B,mCAAmC;YACnC,0BAA0B;SAC3B;KACF;IACD;QACE,EAAE,EAAE,UAAU;QACd,KAAK,EAAE,mBAAmB;QAC1B,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,sFAAsF;QACxF,WAAW,EAAE,CAAC,mBAAmB,CAAC;KACnC;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,yBAAyB;QAChC,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,IAAI;QAClB,WAAW,EACT,4EAA4E;QAC9E,WAAW,EAAE,CAAC,yBAAyB,CAAC;QACxC,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;KACxC;IACD;QACE,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,qBAAqB;QAC5B,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,6EAA6E;QAC/E,WAAW,EAAE,CAAC,qBAAqB,CAAC;KACrC;IACD;QACE,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,gBAAgB;QACvB,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,4EAA4E;QAC9E,WAAW,EAAE,CAAC,gBAAgB,CAAC;KAChC;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,0BAA0B;QACjC,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,6EAA6E;QAC/E,WAAW,EAAE,CAAC,0BAA0B,CAAC;KAC1C;IACD;QACE,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,qBAAqB;QAC5B,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,uGAAuG;QACzG,WAAW,EAAE,CAAC,qBAAqB,EAAE,4BAA4B,CAAC;QAClE,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;KACxC;IACD;QACE,EAAE,EAAE,WAAW;QACf,KAAK,EAAE,oBAAoB;QAC3B,IAAI,EAAE,KAAK;QACX,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,iFAAiF;QACnF,WAAW,EAAE,CAAC,oBAAoB,CAAC;KACpC;CACO,CAAC;AAIX,MAAM,cAAc,GAAG,IAAI,GAAG,CAC5B,eAAe,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAC3D,CAAC;AAEF,MAAM,UAAU,iBAAiB,CAAC,EAAU;IAC1C,OAAO,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,8BAA8B,EAAE,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;AACrE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voicethere/agent",
-  "version": "0.9.1",
+  "version": "0.9.3",
   "description": "VoiceThere customer agent SDK — IPC types and runtime helpers for sandboxed child bundles",
   "type": "module",
   "exports": {

package/templates/README.md

@@ -64,7 +64,7 @@ Each folder has its own README. Summary:
 | `echo`              | `echo/`              | Voice + chat echo                                                 |
 | `echo-dc`           | `echo-dc/`           | Data-channel echo, no TTS                                         |
 | `voice-starter`     | `voice-starter/`     | Every speech event                                                |
-| `language-switch`   | `language-switch/`   | Manual `setVoiceLanguage` or runner auto-switch + prompt handlers |
+| `language-switch`   | `language-switch/`   | Manual `setVoiceLanguage`, or runner auto-switch with a localized echo of the recognized text |
 | `world-sync`        | `world-sync/`        | JSON pose broadcast                                               |
 | `world-sync-binary` | `world-sync-binary/` | Binary pose `ArrayBuffer`                                         |
 | `game-sync`         | `game-sync/`         | Authoritative sim, Redis + binary snapshots                       |

package/templates/language-switch/README.md

@@ -9,11 +9,22 @@ Leave the project **spoken-language auto-switch** setting off. When `onUserLangu
 - `scope: "tts"` switches only the speaking voice
 - `scope: "stt"` switches only the listening model
 
+The runner plays nothing for a manual switch, and the target voice and STT pools can take several seconds to warm up. So the agent first speaks a short wait message in the language it is leaving, using the current voice (for example `One moment please, I'm switching to your language.` in English, `Einen Moment bitte, ich wechsle zu Ihrer Sprache.` in German). The runner swaps the voice without draining it, so the agent waits for the `agent_speaking_end` speech event (at most 8 seconds, `WAIT_SPEECH_TIMEOUT_MS`) before it calls `setVoiceLanguage` for TTS and STT and replies in the new language. A second language event during the wait or the switch is ignored. The utterance that revealed the language is not echoed back. If the TTS switch fails (a timeout, for instance), the agent speaks a short fallback in the old language and stays in it.
+
+The texts live in the `WAIT_MESSAGES` and `FAILED_MESSAGES` maps in `agent.ts` (en, de, es, fr, it, pt, nl, pl, ru). Override any of them per language with a JSON object in the session env, keyed by ISO 639-1 code:
+
+- `LANGUAGE_SWITCH_WAIT_MESSAGES_JSON` — wait message, for example `{"en":"Hold on, switching languages."}`
+- `LANGUAGE_SWITCH_FAILED_MESSAGES_JSON` — fallback after a failed switch
+
+Invalid JSON or non-string values are ignored and the built-in text is used. A language without a text falls back to English.
+
 A Sherpa language with no STT id (`it`, `pt`, `nl`, `pl`, `hi`) fails the STT call and still switches TTS.
 
 ## Mode B — Runner auto-switch
 
-Enable auto-switch in the project voice settings (runner applies STT/TTS when LID detects a new language). When `session_start.env` includes a truthy `SHERPA_LID_AUTO_SWITCH`, this template **does not** call `setVoiceLanguage` from `onUserLanguage` — it logs that the runner owns the switch and updates prompts only. Use `onVoiceLanguageChanged` when you need the committed language after the runner applies the change.
+Enable auto-switch in the project voice settings (runner applies STT/TTS when LID detects a new language). When `session_start.env` includes a truthy `SHERPA_LID_AUTO_SWITCH`, this template **does not** call `setVoiceLanguage`. The runner replays the utterance into the new language's STT, so the next final is the recognized text in the new language. The agent remembers the language from `onUserLanguage` / `onVoiceLanguageChanged` and answers each final right away with a short localized prefix, for example `you said: …` in English and `Du hast gesagt: …` in German. Use `onVoiceLanguageChanged` when you need the committed language after the runner applies the change.
+
+In this mode the runner owns the wait message and the agent speaks no wait or fallback text of its own.
 
 Detection and chat commands are unchanged: `/tts` and `/stt` still call `setVoiceLanguage` for one vendor at a time.
 
@@ -24,7 +35,7 @@ Chat commands change one vendor while the session stays connected. API keys are
 - `/tts elevenlabs` — ElevenLabs TTS, current STT stays
 - `/stt deepgram de` — Deepgram STT, current voice stays
 
-English finals are echoed as `you said: …` after a short wait, so a language event for the same utterance can cancel the echo. The revealing utterance is not spoken back as an English transcript.
+In manual mode, English finals are echoed as `you said: …` after a short wait, so a language event for the same utterance can cancel the echo. The revealing utterance is not spoken back as an English transcript.
 
 `voice` and `stt` are Sherpa catalog ids (`de`, `en-lessac`, `en-small`), listed in the spoken-language docs. Vendor ids are `local-sherpa`, `openai`, `deepgram`, `assemblyai`, `google`, `elevenlabs`, and `cartesia`.
 

package/templates/language-switch/agent.ts

@@ -4,9 +4,22 @@
  * Two deployment modes (see README):
  *
  * (A) Project auto-switch off — this agent calls `setVoiceLanguage` from
- *     `onUserLanguage` when LID detects a new language.
- * (B) Project enables runner auto-switch — STT/TTS are runner-owned; use
- *     `onUserLanguage` / `onVoiceLanguageChanged` for prompts only.
+ *     `onUserLanguage` when LID detects a new language. The runner plays
+ *     nothing for manual switches and the target pools may be cold for
+ *     several seconds, so the agent first speaks a wait message in the
+ *     language being left (current voice), waits for `agent_speaking_end`
+ *     (at most WAIT_SPEECH_TIMEOUT_MS) so the runner does not swap the voice
+ *     mid-sentence, then switches TTS and STT, then replies in the new
+ *     language. If the switch fails it speaks a short
+ *     fallback in the old language and stays. Texts live in WAIT_MESSAGES /
+ *     FAILED_MESSAGES and can be overridden per language with the session env
+ *     vars LANGUAGE_SWITCH_WAIT_MESSAGES_JSON and
+ *     LANGUAGE_SWITCH_FAILED_MESSAGES_JSON (JSON object, ISO 639-1 -> text).
+ * (B) Project enables runner auto-switch — STT/TTS are runner-owned. The
+ *     runner replays the utterance into the new language's STT, so the next
+ *     final is the correctly recognized text. The agent never calls
+ *     `setVoiceLanguage`; it remembers the language and answers each final
+ *     right away with a short localized prefix ("Du hast gesagt: …").
  *
  * Chat commands change a single vendor mid-conversation:
  *
@@ -31,6 +44,83 @@ import {
 const ECHO_PREFIX = "you said:";
 const ECHO_WAIT_MS = 4000;
 
+/** Echo prefix per language for runner auto-switch mode. */
+const ECHO_PREFIXES: Record<string, string> = {
+  en: "you said:",
+  de: "Du hast gesagt:",
+  es: "Dijiste:",
+  fr: "Tu as dit :",
+  it: "Hai detto:",
+  pt: "Você disse:",
+  nl: "Je zei:",
+  pl: "Powiedziałeś:",
+  ru: "Вы сказали:",
+};
+
+/** Spoken in the language being left, before a manual switch starts. */
+export const WAIT_MESSAGES: Record<string, string> = {
+  en: "One moment please, I'm switching to your language.",
+  de: "Einen Moment bitte, ich wechsle zu Ihrer Sprache.",
+  es: "Un momento, por favor, estoy cambiando a su idioma.",
+  fr: "Un instant, je passe dans votre langue.",
+  it: "Un momento, per favore, sto passando alla tua lingua.",
+  pt: "Um momento, por favor, estou mudando para o seu idioma.",
+  nl: "Een moment alstublieft, ik schakel over naar uw taal.",
+  pl: "Chwileczkę, przełączam się na Twój język.",
+  ru: "Одну минуту, я перехожу на ваш язык.",
+};
+
+/** Spoken in the old language when the switch fails and the agent stays. */
+export const FAILED_MESSAGES: Record<string, string> = {
+  en: "Sorry, I couldn't switch languages. I'll keep going in English.",
+  de: "Entschuldigung, der Sprachwechsel hat nicht geklappt. Ich bleibe bei Deutsch.",
+  es: "Lo siento, no pude cambiar de idioma. Sigo en español.",
+  fr: "Désolé, je n'ai pas pu changer de langue. Je continue en français.",
+  it: "Mi dispiace, non sono riuscito a cambiare lingua. Continuo in italiano.",
+  pt: "Desculpe, não consegui mudar de idioma. Continuo em português.",
+  nl: "Sorry, het wisselen van taal is niet gelukt. Ik blijf Nederlands spreken.",
+  pl: "Przepraszam, nie udało się zmienić języka. Zostaję przy polskim.",
+  ru: "Извините, не удалось сменить язык. Продолжаю по-русски.",
+};
+
+export const WAIT_MESSAGES_ENV = "LANGUAGE_SWITCH_WAIT_MESSAGES_JSON";
+export const FAILED_MESSAGES_ENV = "LANGUAGE_SWITCH_FAILED_MESSAGES_JSON";
+
+/**
+ * Parse a per-language override from a session env var. Accepts a JSON object
+ * of language code -> non-empty string; anything else yields no overrides.
+ */
+export function parseMessageOverrides(
+  raw: string | undefined,
+): Record<string, string> {
+  if (!raw) return {};
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return {};
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
+  const out: Record<string, string> = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    if (typeof value === "string" && value.trim()) {
+      out[key.trim().toLowerCase()] = value.trim();
+    }
+  }
+  return out;
+}
+
+/** Message for `language`: override, then default, then English default. */
+export function pickMessage(
+  defaults: Record<string, string>,
+  overrides: Record<string, string>,
+  language: string,
+): string {
+  return (
+    overrides[language] ?? defaults[language] ?? overrides.en ?? defaults.en!
+  );
+}
+
 const REPLIES: Record<string, string> = {
   de: "Guten Tag. Ich antworte jetzt auf Deutsch.",
   es: "Hola. Ahora respondo en español.",
@@ -47,6 +137,16 @@ type SessionLanguage = {
   runnerAutoSwitch: boolean;
   echoTimer: ReturnType<typeof setTimeout> | undefined;
   suppressNextFinal: boolean;
+  pendingSwitch:
+    | {
+        language: string;
+        previous: string;
+        timer: ReturnType<typeof setTimeout>;
+      }
+    | undefined;
+  switching: boolean;
+  waitOverrides: Record<string, string>;
+  failedOverrides: Record<string, string>;
 };
 
 const sessions = new Map<string, SessionLanguage>();
@@ -62,6 +162,10 @@ function stateFor(
     runnerAutoSwitch: env ? isRunnerLidAutoSwitchEnabled(env) : false,
     echoTimer: undefined,
     suppressNextFinal: false,
+    pendingSwitch: undefined,
+    switching: false,
+    waitOverrides: parseMessageOverrides(env?.[WAIT_MESSAGES_ENV]),
+    failedOverrides: parseMessageOverrides(env?.[FAILED_MESSAGES_ENV]),
   };
   sessions.set(sessionId, created);
   return created;
@@ -71,6 +175,10 @@ function replyFor(language: string): string {
   return REPLIES[language] ?? `Continuing in ${language}.`;
 }
 
+function echoPrefixFor(language: string): string {
+  return ECHO_PREFIXES[language] ?? ECHO_PREFIXES.en!;
+}
+
 function logSwitch(
   sessionId: string,
   side: string,
@@ -91,6 +199,61 @@ function logSwitch(
   );
 }
 
+/** Upper bound for waiting on the wait message before switching anyway. */
+export const WAIT_SPEECH_TIMEOUT_MS = 8000;
+
+/** Consume the pending switch (once) and run it without blocking handlers. */
+function startSwitch(sessionId: string, state: SessionLanguage): void {
+  const pending = state.pendingSwitch;
+  if (!pending) return;
+  clearTimeout(pending.timer);
+  state.pendingSwitch = undefined;
+  state.switching = true;
+  void runSwitch(sessionId, state, pending.language, pending.previous)
+    .catch((error: unknown) => {
+      state.suppressNextFinal = false;
+      agentLog("warn", `language-switch ${sessionId} failed: ${String(error)}`);
+    })
+    .finally(() => {
+      state.switching = false;
+    });
+}
+
+async function runSwitch(
+  sessionId: string,
+  state: SessionLanguage,
+  language: string,
+  previous: string,
+): Promise<void> {
+  const tts = await setVoiceLanguage(sessionId, {
+    scope: "tts",
+    language,
+    voice: language,
+  });
+  logSwitch(sessionId, "tts", tts);
+
+  const stt = await setVoiceLanguage(sessionId, {
+    scope: "stt",
+    language,
+    stt: language,
+  });
+  logSwitch(sessionId, "stt", stt);
+
+  if (!tts.ok) {
+    state.suppressNextFinal = false;
+    // Switch failed (for example a timeout): the voice is unchanged, so
+    // apologise in the old language and stay.
+    speak(
+      sessionId,
+      pickMessage(FAILED_MESSAGES, state.failedOverrides, previous),
+    );
+    return;
+  }
+
+  state.language = language;
+  speak(sessionId, replyFor(language));
+}
+
 function prepareLanguageTransition(state: SessionLanguage): void {
   if (state.echoTimer) {
     clearTimeout(state.echoTimer);
@@ -120,39 +283,47 @@ defineAgent({
     const state = stateFor(sessionId);
     if (!language || language === state.language) return;
 
-    prepareLanguageTransition(state);
-
     if (state.runnerAutoSwitch) {
+      // The runner switches STT/TTS and replays the utterance into the new
+      // STT; the next final is the real utterance, so nothing is suppressed
+      // and no fixed sentence is spoken.
       agentLog(
         "info",
-        `LID detected ${language}; runner auto-switch applies STT/TTS — agent updates prompts only`,
+        `LID detected ${language}; runner auto-switch applies STT/TTS — agent only remembers the language`,
       );
       state.language = language;
-      speak(sessionId, replyFor(language));
       return;
     }
 
-    const tts = await setVoiceLanguage(sessionId, {
-      scope: "tts",
-      language,
-      voice: language,
-    });
-    logSwitch(sessionId, "tts", tts);
+    // A switch is already waiting for the wait message or in flight.
+    if (state.pendingSwitch || state.switching) return;
 
-    const stt = await setVoiceLanguage(sessionId, {
-      scope: "stt",
-      language,
-      stt: language,
-    });
-    logSwitch(sessionId, "stt", stt);
+    prepareLanguageTransition(state);
 
-    if (!tts.ok) {
-      state.suppressNextFinal = false;
-      return;
-    }
+    // The runner plays nothing for manual switches and the target pools may be
+    // cold, so tell the user first, in the language being left.
+    const previous = state.language;
+    speak(sessionId, pickMessage(WAIT_MESSAGES, state.waitOverrides, previous));
 
-    state.language = language;
-    speak(sessionId, replyFor(language));
+    // `speak` is fire-and-forget and the runner swaps the TTS without draining
+    // it, so the switch must wait until the wait message has been spoken
+    // (`agent_speaking_end`, delivered to onSpeechEvent). Inbound handlers of
+    // one session run strictly in order, so this handler must return instead
+    // of awaiting that event. The bounded timer covers a missing event.
+    const timer = setTimeout(() => {
+      agentLog(
+        "warn",
+        `language-switch ${sessionId} no agent_speaking_end within ${WAIT_SPEECH_TIMEOUT_MS}ms; switching anyway`,
+      );
+      startSwitch(sessionId, state);
+    }, WAIT_SPEECH_TIMEOUT_MS);
+    state.pendingSwitch = { language, previous, timer };
+  },
+
+  onSpeechEvent({ sessionId }, event) {
+    if (event.type !== "agent_speaking_end") return;
+    const state = sessions.get(sessionId);
+    if (state?.pendingSwitch) startSwitch(sessionId, state);
   },
 
   async onVoiceLanguageChanged({ sessionId, language }) {
@@ -214,6 +385,12 @@ defineAgent({
 
   onUserSpeechFinal({ sessionId, text }) {
     const state = stateFor(sessionId);
+    if (state.runnerAutoSwitch) {
+      // Reply immediately: the runner already holds the final until LID has
+      // decided, so no extra wait is needed to avoid a wrong-language echo.
+      speak(sessionId, `${echoPrefixFor(state.language)} ${text}`.trim());
+      return;
+    }
     if (state.suppressNextFinal) {
       state.suppressNextFinal = false;
       return;
@@ -228,6 +405,7 @@ defineAgent({
   onSessionEnd({ sessionId }) {
     const state = sessions.get(sessionId);
     if (state?.echoTimer) clearTimeout(state.echoTimer);
+    if (state?.pendingSwitch) clearTimeout(state.pendingSwitch.timer);
     sessions.delete(sessionId);
   },
 });