@webotme/react-native 0.1.1 → 0.1.3

package/src/voice.ts

@@ -0,0 +1,722 @@
+/**
+ * One-function voice chat, the same way `chatWithUs` is one-function text chat.
+ *
+ * ```tsx
+ * const res = await voiceWithUs(); // tap the mic
+ * // user speaks -> device transcribes -> chatWithUs() -> reply spoken aloud
+ * ```
+ *
+ * Like the web widget, everything runs on the device: the phone's own speech
+ * recogniser transcribes the question and the phone's own synthesiser speaks the
+ * answer. Your server only ever sees plain text, exactly as it does today, so
+ * no server change and no API keys are required.
+ *
+ * Both native modules are optional peer dependencies and are resolved on first
+ * use, so an app that only sends text never loads them and never breaks.
+ */
+import { useEffect, useState } from "react";
+import {
+  chatWithUs,
+  type ChatResult,
+} from "./chat";
+import type { WebotMeMessage } from "./types";
+
+/** Mirrors the web widget's voice states (minus wake-word, which is browser-only). */
+export type VoiceState =
+  | "idle"
+  | "listening"
+  | "thinking"
+  | "speaking"
+  | "unsupported"
+  | "denied";
+
+export type VoiceOptions = {
+  /** BCP-47 language for both recognition and speech. Default "en-US". */
+  lang?: string;
+  rate?: number;
+  pitch?: number;
+  speakAllReplies?: boolean;
+  silenceTimeout?: number;
+  onStateChange?: (state: VoiceState) => void;
+  onPartial?: (text: string) => void;
+  onUserQuery?: (text: string) => void;
+  onError?: (message: string) => void;
+  /** Wake words that enable the bot once recognised (e.g. ["hello botName"]). */
+  activateCommands?: string[];
+  /** Shuts down voice once recognised (e.g. ["off botName"]). */
+  shutdownCommands?: string[];
+  /** Name used to substitute {botName} in the activation/shutdown phrases. */
+  botName?: string;
+  onActivated?: (transcript: string) => void;
+  onDeactivated?: (transcript: string) => void;
+};
+
+/** What `voiceWithUs` hands back, mirroring `ChatResult`. */
+export type VoiceResult = {
+  ok: boolean;
+  /** What the user actually said. */
+  transcript: string;
+  /** The bot's reply, or an empty string when it failed. */
+  reply: string;
+  userMessage: WebotMeMessage | null;
+  botMessage: WebotMeMessage | null;
+  /** False when the reply was not spoken (voice replies disabled, or TTS failed). */
+  spoken: boolean;
+  error?: string;
+};
+
+// ── Optional native modules, resolved on first use ──────────────────────────
+
+type SpeechLike = {
+  speak: (text: string, options?: Record<string, unknown>) => void;
+  stop: () => Promise<void> | void;
+  isSpeakingAsync?: () => Promise<boolean>;
+};
+
+type SpeechEvent = { results?: Array<{ transcript?: string } | string> };
+
+function normalizeTranscript(v: unknown): string {
+  return String(v ?? "")
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+type SpeechRecognitionLike = {
+  start: (options?: Record<string, unknown>) => Promise<void> | void;
+  stop: () => Promise<void> | void;
+  abort: () => Promise<void> | void;
+  requestPermissionsAsync?: () => Promise<{ granted?: boolean; status?: string }>;
+  isRecognitionAvailable?: () => Promise<boolean>;
+  addListener?: (
+    event: string,
+    listener: (event: any) => void,
+  ) => { remove: () => void };
+};
+
+let speechCache: SpeechLike | null | undefined;
+let recognitionCache: SpeechRecognitionLike | null | undefined;
+
+const REBUILD_HINT =
+  "Then rebuild the app — a JS reload is not enough, the native module has to be compiled in: npx expo run:android / run:ios, or eas build.";
+
+const MISSING_STT = `Voice input is not available in this build. Install it with "npx expo install expo-speech-recognition". ${REBUILD_HINT}`;
+const MISSING_TTS = `Voice replies are not available in this build. Install them with "npx expo install expo-speech". ${REBUILD_HINT}`;
+
+/**
+ * Both packages call `requireNativeModule` at module scope, which throws when
+ * the native side is missing — for example after adding the package but before
+ * rebuilding, or inside a build made before the package existed. Asking the
+ * native registry first lets us report "unsupported" instead of throwing on
+ * every mount.
+ */
+function hasNativeModule(name: string): boolean {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { NativeModules } = require("react-native");
+    if (NativeModules?.[name]) return true;
+    // TurboModules are not always mirrored into NativeModules, so fall back to
+    // a probe that returns null instead of throwing when the module is absent.
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { requireOptionalNativeModule } = require("expo");
+    return Boolean(requireOptionalNativeModule?.(name));
+  } catch {
+    return false;
+  }
+}
+
+/** `expo-speech`, or null when the app has not installed it. */
+function loadSpeech(): SpeechLike | null {
+  if (speechCache !== undefined) return speechCache;
+  if (!hasNativeModule("ExpoSpeech")) {
+    speechCache = null;
+    return speechCache;
+  }
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const mod = require("expo-speech");
+    const candidate = (mod?.Speech ?? mod?.default) as SpeechLike | undefined;
+    speechCache = typeof candidate?.speak === "function" ? candidate : null;
+  } catch {
+    speechCache = null;
+  }
+  return speechCache;
+}
+
+/** `expo-speech-recognition`, or null when the app has not installed it. */
+function loadRecognition(): SpeechRecognitionLike | null {
+  if (recognitionCache !== undefined) return recognitionCache;
+  if (!hasNativeModule("ExpoSpeechRecognition")) {
+    recognitionCache = null;
+    return recognitionCache;
+  }
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const mod = require("expo-speech-recognition");
+    const candidate = (mod?.ExpoSpeechRecognitionModule ??
+      mod?.default ??
+      mod) as SpeechRecognitionLike | undefined;
+    recognitionCache =
+      candidate && typeof candidate.start === "function" ? candidate : null;
+  } catch {
+    recognitionCache = null;
+  }
+  return recognitionCache;
+}
+
+function readTranscript(event: SpeechEvent | undefined): string {
+  const first = event?.results?.[0];
+  if (typeof first === "string") return first.trim();
+  if (first && typeof first.transcript === "string") return first.transcript.trim();
+  return "";
+}
+
+// ── Engine ──────────────────────────────────────────────────────────────────
+
+type Listener = () => void;
+
+type VoiceStateShape = {
+  state: VoiceState;
+  supported: boolean;
+  transcript: string;
+  error: string | null;
+  voiceReplies: boolean;
+};
+
+class VoiceEngine {
+  private options: VoiceOptions = {};
+  private listeners = new Set<Listener>();
+  private subscriptions: Array<{ remove: () => void }> = [];
+  private state: VoiceStateShape;
+
+  private transcript = "";
+  private settleTimer: ReturnType<typeof setTimeout> | null = null;
+  private onFinal: ((text: string) => void) | null = null;
+  private session: Promise<VoiceResult> | null = null;
+  private resolveSession: ((result: VoiceResult) => void) | null = null;
+  private speaking = false;
+  private continuous = false;
+  private loopStopped = false;
+  private activated = true;
+  private commandsConfigured = false;
+
+  constructor() {
+    this.state = {
+      state: "idle",
+      supported: false,
+      transcript: "",
+      error: null,
+      voiceReplies: false,
+    };
+  }
+
+  private patch(next: Partial<VoiceStateShape> & { state?: VoiceState }) {
+    const previous = this.state.state;
+    this.state = { ...this.state, ...next };
+    if (next.state && next.state !== previous) {
+      this.options.onStateChange?.(next.state);
+    }
+    for (const listener of this.listeners) listener();
+  }
+
+  subscribe(listener: Listener) {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+
+  getState(): VoiceStateShape {
+    return this.state;
+  }
+
+  configure(options: VoiceOptions) {
+    this.options = { ...this.options, ...options };
+    if (typeof options.speakAllReplies === "boolean") {
+      this.patch({ voiceReplies: options.speakAllReplies });
+    }
+    if (Array.isArray(options.activateCommands) && options.activateCommands.length > 0) {
+      // Lock voice until an activation phrase is heard, but only the first time
+      // the commands are applied so re-configuring never re-locks a live bot.
+      if (!this.commandsConfigured) this.activated = false;
+      this.commandsConfigured = true;
+    } else if (options.activateCommands !== undefined) {
+      // Commands were explicitly cleared -> fall back to always-available voice.
+      this.commandsConfigured = false;
+      this.activated = true;
+    }
+    this.patch({ supported: this.isSupported() });
+  }
+
+  setVoiceReplies(enabled: boolean) {
+    this.patch({ voiceReplies: enabled });
+    if (!enabled) void this.stopSpeaking();
+  }
+
+  isSupported() {
+    try {
+      return loadRecognition() !== null && loadSpeech() !== null;
+    } catch {
+      return false;
+    }
+  }
+
+  private async ensureMicPermission(): Promise<boolean> {
+    const recognition = loadRecognition();
+    if (!recognition?.requestPermissionsAsync) return true;
+    try {
+      const result = await recognition.requestPermissionsAsync();
+      return result?.granted !== false;
+    } catch {
+      return true;
+    }
+  }
+
+  private attachListeners() {
+    if (this.subscriptions.length > 0) return;
+    const recognition = loadRecognition();
+    if (!recognition?.addListener) return;
+
+    const push = (event: any) => {
+      const text = readTranscript(event);
+      if (!text) return;
+      this.transcript = text;
+      this.patch({ transcript: text });
+      this.options.onPartial?.(text);
+      this.scheduleSettle();
+    };
+
+    const fail = (event: any) => {
+      const message: string =
+        event?.error?.message ?? event?.message ?? event?.error ?? "Voice input failed.";
+      this.failRecognition(message);
+    };
+
+    this.subscriptions = [
+      recognition.addListener("result", push),
+      recognition.addListener("end", () => {
+        // The recogniser finalises on its own after a short pause; settle
+        // whatever it heard so a dropped event can never hang the promise.
+        this.settle();
+      }),
+      recognition.addListener("error", fail),
+    ];
+  }
+
+  private scheduleSettle() {
+    if (this.settleTimer) clearTimeout(this.settleTimer);
+    this.settleTimer = setTimeout(
+      () => this.settle(),
+      this.options.silenceTimeout ?? 2200,
+    );
+  }
+
+  private settle() {
+    if (this.settleTimer) {
+      clearTimeout(this.settleTimer);
+      this.settleTimer = null;
+    }
+    const text = this.transcript.trim();
+    if (!text) return;
+    void loadRecognition()?.stop();
+    this.onFinal?.(text);
+  }
+
+  /** Listen once, then resolve with the transcript and the spoken reply. */
+  listen(options?: { continuous?: boolean }): Promise<VoiceResult> {
+    if (this.session) return this.session;
+
+    if (typeof options?.continuous === "boolean") {
+      this.continuous = options.continuous;
+    }
+
+    if (!this.isSupported()) {
+      const error = loadRecognition()
+        ? MISSING_TTS
+        : MISSING_STT;
+      this.patch({ state: "unsupported", supported: false, error });
+      this.options.onError?.(error);
+      return Promise.resolve(this.failure(error));
+    }
+
+    this.session = new Promise<VoiceResult>((resolve) => {
+      this.resolveSession = resolve;
+      this.onFinal = (text) => {
+        this.onFinal = null;
+        this.patch({ state: "thinking" });
+        this.askBot(text);
+      };
+    });
+
+    void this.beginListening();
+    return this.session;
+  }
+
+  private async beginListening() {
+    const recognition = loadRecognition();
+    if (!recognition) return;
+
+    await this.ensureMicPermission();
+    this.attachListeners();
+    this.transcript = "";
+    this.patch({ state: "listening", transcript: "", error: null });
+
+    try {
+      await recognition.start({
+        lang: this.options.lang ?? "en-US",
+        interimResults: true,
+        continuous: false,
+        // Keep audio on the phone where the OS supports it.
+        requiresOnDeviceRecognition: undefined,
+      });
+    } catch (err) {
+      const message =
+        err instanceof Error ? err.message : "Could not start voice input.";
+      // If continuous loop fails to restart, exit loop gracefully.
+      if (this.continuous) {
+        this.continuous = false;
+        this.loopStopped = true;
+      }
+      this.fail(message);
+    }
+  }
+
+  /** [a-z0-9] tokens that make up the bot's name (used for phrase matching). */
+  private static botNameTokens(botName: string): string[] {
+    return String(botName || "assistant")
+      .toLowerCase()
+      .replace(/[^a-z0-9\s]/g, " ")
+      .split(/\s+/)
+      .filter(Boolean);
+  }
+
+  /** Remove every mention of the bot's name from a lowercase phrase. */
+  private static stripName(text: string, botName: string): string {
+    const tokens = VoiceEngine.botNameTokens(botName);
+    if (tokens.length === 0) return text;
+    let clean = " " + (text || "").toLowerCase() + " ";
+    for (const token of tokens) {
+      if (!token) continue;
+      clean = clean.replace(
+        new RegExp(`(^|[^a-z0-9])${token}(?=$|[^a-z0-9])`, "g"),
+        "$1 ",
+      );
+    }
+    return clean.replace(/\s+/g, " ").trim();
+  }
+
+  /**
+   * True when the spoken phrase matches one of the commands. The bot name is
+   * matched both inside the phrase and without it, so "hello rover", "hello",
+   * and "hey rover" all activate when the command is "hello {botName}".
+   */
+  private static matchesAny(
+    normalized: string,
+    commands: string[] | undefined,
+    botName: string,
+  ): boolean {
+    if (!Array.isArray(commands) || commands.length === 0) return false;
+    const stripped = VoiceEngine.stripName(normalized, botName);
+    return commands.some((cmd) => {
+      const raw = cmd
+        .toLowerCase()
+        .replace(/{botname}/g, botName.toLowerCase())
+        .replace(/\s+/g, " ")
+        .trim();
+      if (!raw) return false;
+      if (normalized.includes(raw)) return true;
+      const rawNoName = VoiceEngine.stripName(raw, botName);
+      if (!rawNoName) return false;
+      return stripped.includes(rawNoName) || stripped === rawNoName;
+    });
+  }
+
+  private async askBot(transcript: string) {
+    const normalized = transcript.toLowerCase().replace(/[^a-z0-9\s]/gi, "").trim();
+    const botName = this.options.botName || "assistant";
+    const wantsActivation = VoiceEngine.matchesAny(normalized, this.options.activateCommands, botName);
+    const wantsShutdown = VoiceEngine.matchesAny(normalized, this.options.shutdownCommands, botName);
+
+    if (wantsShutdown) {
+      this.activated = false;
+      this.options.onDeactivated?.(transcript);
+      this.continuous = false;
+      this.loopStopped = true;
+      this.stop();
+      this.finishSession({
+        ok: true,
+        transcript,
+        reply: "",
+        userMessage: null,
+        botMessage: null,
+        spoken: false,
+      });
+      return;
+    }
+
+    if (this.options.activateCommands && this.options.activateCommands.length > 0) {
+      if (!this.activated) {
+        if (wantsActivation) {
+          this.activated = true;
+          this.options.onActivated?.(transcript);
+        } else {
+          // Ignore everything before activation; start loop again if continuous.
+          if (this.continuous && !this.loopStopped) {
+            this.continueLoop();
+          } else {
+            this.finishSession({
+              ok: true,
+              transcript,
+              reply: "",
+              userMessage: null,
+              botMessage: null,
+              spoken: false,
+            });
+          }
+          return;
+        }
+      }
+    }
+
+    // Send the phrase without the bare bot-name mention (like the web engine),
+    // so "hello rover" reaches the bot as just "hello".
+    const forwardText = VoiceEngine.stripName(transcript, botName) || transcript;
+
+    let result: ChatResult;
+    try {
+      result = await chatWithUs(forwardText);
+    } catch (err) {
+      // chatWithUs already reports its own failures, so this is a real crash.
+      this.fail(
+        err instanceof Error ? err.message : "Could not reach the bot.",
+        "denied",
+      );
+      return;
+    }
+    this.options.onUserQuery?.(transcript);
+
+    if (!result.ok) {
+      this.fail(result.error ?? "Could not reach the bot.");
+      return;
+    }
+
+    const spoken = await this.speakReply(result.reply);
+    if (this.continuous && !this.loopStopped) {
+      this.continueLoop();
+    } else {
+      this.finishSession({
+        ok: true,
+        transcript,
+        reply: result.reply,
+        userMessage: result.userMessage,
+        botMessage: result.botMessage,
+        spoken,
+      });
+    }
+  }
+
+  private continueLoop() {
+    this.loopStopped = false;
+    this.transcript = "";
+    this.patch({ transcript: "", error: null });
+    void this.beginListening();
+  }
+
+  /** Speak a reply and resolve when playback ends. */
+  private speakReply(text: string): Promise<boolean> {
+    const speech = loadSpeech();
+    const trimmed = text.trim();
+    if (!speech || !trimmed) return Promise.resolve(false);
+
+    return new Promise<boolean>((resolve) => {
+      this.speaking = true;
+      this.patch({ state: "speaking" });
+
+      let settled = false;
+      const finish = (ok: boolean) => {
+        if (settled) return;
+        settled = true;
+        this.speaking = false;
+        if (this.state.state === "speaking") this.patch({ state: "idle" });
+        resolve(ok);
+      };
+
+      // iOS never fires an end event for empty or whitespace-only text.
+      try {
+        speech.speak(trimmed, {
+          language: this.options.lang ?? "en-US",
+          rate: this.options.rate ?? 0.9,
+          pitch: this.options.pitch ?? 1,
+          onDone: () => finish(true),
+          onStopped: () => finish(false),
+          onError: () => finish(false),
+        });
+      } catch {
+        finish(false);
+        return;
+      }
+
+      // Safety net: never let a missing end event hang the promise.
+      setTimeout(() => finish(true), Math.max(8000, trimmed.length * 120));
+    });
+  }
+
+  /** Speak any text, e.g. to read a reply the user typed. */
+  speak(text: string): Promise<boolean> {
+    return this.speakReply(text);
+  }
+
+  async stopSpeaking() {
+    const speech = loadSpeech();
+    if (!speech) return;
+    this.speaking = false;
+    try {
+      await speech.stop();
+    } catch {
+      /* nothing playing */
+    }
+    if (this.state.state === "speaking") this.patch({ state: "idle" });
+  }
+
+  /** Stop listening and speaking; a pending `voiceWithUs()` promise resolves. */
+  stop() {
+    if (this.settleTimer) {
+      clearTimeout(this.settleTimer);
+      this.settleTimer = null;
+    }
+    const recognition = loadRecognition();
+    void recognition?.abort?.();
+    void recognition?.stop?.();
+    void this.stopSpeaking();
+    this.continuous = false;
+    this.loopStopped = true;
+    if (this.state.state === "listening" || this.state.state === "thinking" || this.state.state === "speaking") {
+      this.patch({ state: "idle" });
+    }
+    // Never leave a caller hanging on `await voiceWithUs()`.
+    if (this.resolveSession) {
+      this.finishSession({
+        ok: true,
+        transcript: this.transcript.trim(),
+        reply: "",
+        userMessage: null,
+        botMessage: null,
+        spoken: false,
+      });
+    }
+  }
+
+  /** A recogniser failure mid-session: report it and go back to idle. */
+  private failRecognition(message: string) {
+    this.patch({ state: "idle", error: message });
+    this.options.onError?.(message);
+    this.finishSession(this.failure(message, this.transcript.trim()));
+  }
+
+  /** Voice could not start at all (permission denied, module missing, no mic). */
+  private fail(message: string, state: VoiceState = "denied") {
+    this.patch({ state, error: message });
+    this.options.onError?.(message);
+    this.finishSession(this.failure(message, this.transcript.trim()));
+  }
+
+  private failure(error: string, transcript = ""): VoiceResult {
+    return {
+      ok: false,
+      transcript,
+      reply: "",
+      userMessage: null,
+      botMessage: null,
+      spoken: false,
+      error,
+    };
+  }
+
+  private finishSession(result: VoiceResult) {
+    const resolve = this.resolveSession;
+    this.resolveSession = null;
+    this.onFinal = null;
+    this.session = null;
+    this.loopStopped = true;
+    this.continuous = false;
+    if (this.state.state !== "unsupported") this.patch({ state: "idle" });
+    resolve?.(result);
+  }
+}
+
+const engine = new VoiceEngine();
+
+/**
+ * Point the voice engine at a language and wire the callbacks. Optional — every
+ * setting has a sensible default, so a bare `voiceWithUs()` works too.
+ *
+ * ```tsx
+ * configureVoice({
+ *   lang: "en-US",
+ *   speakAllReplies: true,
+ *   onPartial: (text) => setText(text),
+ * });
+ * ```
+ */
+export function configureVoice(options: VoiceOptions) {
+  engine.configure(options);
+}
+
+/** Read every bot reply aloud, not only the ones that were spoken. */
+export function setVoiceReplies(enabled: boolean) {
+  engine.setVoiceReplies(enabled);
+}
+
+/** Stop listening and stop talking. */
+export function stopVoice() {
+  engine.stop();
+}
+
+/**
+ * Ask out loud and hear the answer.
+ *
+ * ```tsx
+ * const res = await voiceWithUs();
+ * if (res.ok) console.log(res.transcript, "->", res.reply);
+ * ```
+ */
+export function voiceWithUs(options?: { continuous?: boolean }): Promise<VoiceResult> {
+  return engine.listen(options);
+}
+
+/** Say a line with the device synthesiser, e.g. to read a typed reply. */
+export function speakWithUs(text: string): Promise<boolean> {
+  return engine.speak(text);
+}
+
+/** Read a reply aloud only when `configureVoice({ speakAllReplies: true })`. */
+export function speakReplyIfEnabled(reply: string): Promise<boolean> {
+  return engine.getState().voiceReplies ? engine.speak(reply) : Promise.resolve(false);
+}
+
+/** True when both native modules are installed. */
+export function isVoiceSupported(): boolean {
+  return engine.isSupported();
+}
+
+/** Live voice state for rendering, the same way `useChat` exposes chat state. */
+export function useVoiceChat() {
+  const [state, setState] = useState<VoiceStateShape>(() => engine.getState());
+
+  useEffect(() => {
+    setState(engine.getState());
+    return engine.subscribe(() => setState(engine.getState()));
+  }, []);
+
+  return {
+    ...state,
+    listening: state.state === "listening",
+    thinking: state.state === "thinking",
+    speaking: state.state === "speaking",
+    start: (options?: { continuous?: boolean }) => voiceWithUs(options),
+    speak: speakWithUs,
+    stop: stopVoice,
+    setVoiceReplies,
+  };
+}