@webotme/react-native 0.1.1 → 0.1.2

package/src/voice.ts

@@ -0,0 +1,592 @@
+/**
+ * One-function voice chat, the same way `chatWithUs` is one-function text chat.
+ *
+ * ```tsx
+ * const res = await voiceWithUs(); // tap the mic
+ * // user speaks -> device transcribes -> chatWithUs() -> reply spoken aloud
+ * ```
+ *
+ * Like the web widget, everything runs on the device: the phone's own speech
+ * recogniser transcribes the question and the phone's own synthesiser speaks the
+ * answer. Your server only ever sees plain text, exactly as it does today, so
+ * no server change and no API keys are required.
+ *
+ * Both native modules are optional peer dependencies and are resolved on first
+ * use, so an app that only sends text never loads them and never breaks.
+ */
+import { useEffect, useState } from "react";
+import {
+  chatWithUs,
+  type ChatResult,
+} from "./chat";
+import type { WebotMeMessage } from "./types";
+
+/** Mirrors the web widget's voice states (minus wake-word, which is browser-only). */
+export type VoiceState =
+  | "idle"
+  | "listening"
+  | "thinking"
+  | "speaking"
+  | "unsupported"
+  | "denied";
+
+export type VoiceOptions = {
+  /** BCP-47 language for both recognition and speech. Default "en-US". */
+  lang?: string;
+  /** Speaking speed, 0.5–2. Default 0.9 (same pace the web widget uses). */
+  rate?: number;
+  /** Speaking pitch, 0.5–2. Default 1. */
+  pitch?: number;
+  /**
+   * Speak every bot reply, including ones the user typed. Default false, so
+   * only answers to spoken questions are read out.
+   */
+  speakAllReplies?: boolean;
+  /** Silence in ms that finalises the spoken query. Default 2200. */
+  silenceTimeout?: number;
+  onStateChange?: (state: VoiceState) => void;
+  /** Interim transcript, updated live while the user speaks. */
+  onPartial?: (text: string) => void;
+  /** Final transcript once the user stops talking. */
+  onUserQuery?: (text: string) => void;
+  onError?: (message: string) => void;
+};
+
+/** What `voiceWithUs` hands back, mirroring `ChatResult`. */
+export type VoiceResult = {
+  ok: boolean;
+  /** What the user actually said. */
+  transcript: string;
+  /** The bot's reply, or an empty string when it failed. */
+  reply: string;
+  userMessage: WebotMeMessage | null;
+  botMessage: WebotMeMessage | null;
+  /** False when the reply was not spoken (voice replies disabled, or TTS failed). */
+  spoken: boolean;
+  error?: string;
+};
+
+// ── Optional native modules, resolved on first use ──────────────────────────
+
+type SpeechLike = {
+  speak: (text: string, options?: Record<string, unknown>) => void;
+  stop: () => Promise<void> | void;
+  isSpeakingAsync?: () => Promise<boolean>;
+};
+
+type SpeechEvent = { results?: Array<{ transcript?: string } | string> };
+
+type SpeechRecognitionLike = {
+  start: (options?: Record<string, unknown>) => Promise<void> | void;
+  stop: () => Promise<void> | void;
+  abort: () => Promise<void> | void;
+  requestPermissionsAsync?: () => Promise<{ granted?: boolean; status?: string }>;
+  isRecognitionAvailable?: () => Promise<boolean>;
+  addListener?: (
+    event: string,
+    listener: (event: any) => void,
+  ) => { remove: () => void };
+};
+
+let speechCache: SpeechLike | null | undefined;
+let recognitionCache: SpeechRecognitionLike | null | undefined;
+
+const REBUILD_HINT =
+  "Then rebuild the app — a JS reload is not enough, the native module has to be compiled in: npx expo run:android / run:ios, or eas build.";
+
+const MISSING_STT = `Voice input is not available in this build. Install it with "npx expo install expo-speech-recognition". ${REBUILD_HINT}`;
+const MISSING_TTS = `Voice replies are not available in this build. Install them with "npx expo install expo-speech". ${REBUILD_HINT}`;
+
+/**
+ * Both packages call `requireNativeModule` at module scope, which throws when
+ * the native side is missing — for example after adding the package but before
+ * rebuilding, or inside a build made before the package existed. Asking the
+ * native registry first lets us report "unsupported" instead of throwing on
+ * every mount.
+ */
+function hasNativeModule(name: string): boolean {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { NativeModules } = require("react-native");
+    if (NativeModules?.[name]) return true;
+    // TurboModules are not always mirrored into NativeModules, so fall back to
+    // a probe that returns null instead of throwing when the module is absent.
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { requireOptionalNativeModule } = require("expo");
+    return Boolean(requireOptionalNativeModule?.(name));
+  } catch {
+    return false;
+  }
+}
+
+/** `expo-speech`, or null when the app has not installed it. */
+function loadSpeech(): SpeechLike | null {
+  if (speechCache !== undefined) return speechCache;
+  if (!hasNativeModule("ExpoSpeech")) {
+    speechCache = null;
+    return speechCache;
+  }
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const mod = require("expo-speech");
+    const candidate = (mod?.Speech ?? mod?.default) as SpeechLike | undefined;
+    speechCache = typeof candidate?.speak === "function" ? candidate : null;
+  } catch {
+    speechCache = null;
+  }
+  return speechCache;
+}
+
+/** `expo-speech-recognition`, or null when the app has not installed it. */
+function loadRecognition(): SpeechRecognitionLike | null {
+  if (recognitionCache !== undefined) return recognitionCache;
+  if (!hasNativeModule("ExpoSpeechRecognition")) {
+    recognitionCache = null;
+    return recognitionCache;
+  }
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const mod = require("expo-speech-recognition");
+    const candidate = (mod?.ExpoSpeechRecognitionModule ??
+      mod?.default ??
+      mod) as SpeechRecognitionLike | undefined;
+    recognitionCache =
+      candidate && typeof candidate.start === "function" ? candidate : null;
+  } catch {
+    recognitionCache = null;
+  }
+  return recognitionCache;
+}
+
+function readTranscript(event: SpeechEvent | undefined): string {
+  const first = event?.results?.[0];
+  if (typeof first === "string") return first.trim();
+  if (first && typeof first.transcript === "string") return first.transcript.trim();
+  return "";
+}
+
+// ── Engine ──────────────────────────────────────────────────────────────────
+
+type Listener = () => void;
+
+type VoiceStateShape = {
+  state: VoiceState;
+  supported: boolean;
+  transcript: string;
+  error: string | null;
+  voiceReplies: boolean;
+};
+
+class VoiceEngine {
+  private options: VoiceOptions = {};
+  private listeners = new Set<Listener>();
+  private subscriptions: Array<{ remove: () => void }> = [];
+  private state: VoiceStateShape;
+
+  private transcript = "";
+  private settleTimer: ReturnType<typeof setTimeout> | null = null;
+  private onFinal: ((text: string) => void) | null = null;
+  private session: Promise<VoiceResult> | null = null;
+  private resolveSession: ((result: VoiceResult) => void) | null = null;
+  private speaking = false;
+  private continuous = false;
+  private loopStopped = false;
+
+  constructor() {
+    this.state = {
+      state: "idle",
+      supported: false,
+      transcript: "",
+      error: null,
+      voiceReplies: false,
+    };
+  }
+
+  private patch(next: Partial<VoiceStateShape> & { state?: VoiceState }) {
+    const previous = this.state.state;
+    this.state = { ...this.state, ...next };
+    if (next.state && next.state !== previous) {
+      this.options.onStateChange?.(next.state);
+    }
+    for (const listener of this.listeners) listener();
+  }
+
+  subscribe(listener: Listener) {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+
+  getState(): VoiceStateShape {
+    return this.state;
+  }
+
+  configure(options: VoiceOptions) {
+    this.options = { ...this.options, ...options };
+    if (typeof options.speakAllReplies === "boolean") {
+      this.patch({ voiceReplies: options.speakAllReplies });
+    }
+    this.patch({ supported: this.isSupported() });
+  }
+
+  setVoiceReplies(enabled: boolean) {
+    this.patch({ voiceReplies: enabled });
+    if (!enabled) void this.stopSpeaking();
+  }
+
+  isSupported() {
+    try {
+      return loadRecognition() !== null && loadSpeech() !== null;
+    } catch {
+      return false;
+    }
+  }
+
+  private async ensureMicPermission(): Promise<boolean> {
+    const recognition = loadRecognition();
+    if (!recognition?.requestPermissionsAsync) return true;
+    try {
+      const result = await recognition.requestPermissionsAsync();
+      return result?.granted !== false;
+    } catch {
+      return true;
+    }
+  }
+
+  private attachListeners() {
+    if (this.subscriptions.length > 0) return;
+    const recognition = loadRecognition();
+    if (!recognition?.addListener) return;
+
+    const push = (event: any) => {
+      const text = readTranscript(event);
+      if (!text) return;
+      this.transcript = text;
+      this.patch({ transcript: text });
+      this.options.onPartial?.(text);
+      this.scheduleSettle();
+    };
+
+    const fail = (event: any) => {
+      const message: string =
+        event?.error?.message ?? event?.message ?? event?.error ?? "Voice input failed.";
+      this.failRecognition(message);
+    };
+
+    this.subscriptions = [
+      recognition.addListener("result", push),
+      recognition.addListener("end", () => {
+        // The recogniser finalises on its own after a short pause; settle
+        // whatever it heard so a dropped event can never hang the promise.
+        this.settle();
+      }),
+      recognition.addListener("error", fail),
+    ];
+  }
+
+  private scheduleSettle() {
+    if (this.settleTimer) clearTimeout(this.settleTimer);
+    this.settleTimer = setTimeout(
+      () => this.settle(),
+      this.options.silenceTimeout ?? 2200,
+    );
+  }
+
+  private settle() {
+    if (this.settleTimer) {
+      clearTimeout(this.settleTimer);
+      this.settleTimer = null;
+    }
+    const text = this.transcript.trim();
+    if (!text) return;
+    void loadRecognition()?.stop();
+    this.onFinal?.(text);
+  }
+
+  /** Listen once, then resolve with the transcript and the spoken reply. */
+  listen(options?: { continuous?: boolean }): Promise<VoiceResult> {
+    if (this.session) return this.session;
+
+    if (typeof options?.continuous === "boolean") {
+      this.continuous = options.continuous;
+    }
+
+    if (!this.isSupported()) {
+      const error = loadRecognition()
+        ? MISSING_TTS
+        : MISSING_STT;
+      this.patch({ state: "unsupported", supported: false, error });
+      this.options.onError?.(error);
+      return Promise.resolve(this.failure(error));
+    }
+
+    this.session = new Promise<VoiceResult>((resolve) => {
+      this.resolveSession = resolve;
+      this.onFinal = (text) => {
+        this.onFinal = null;
+        this.patch({ state: "thinking" });
+        this.askBot(text);
+      };
+    });
+
+    void this.beginListening();
+    return this.session;
+  }
+
+  private async beginListening() {
+    const recognition = loadRecognition();
+    if (!recognition) return;
+
+    await this.ensureMicPermission();
+    this.attachListeners();
+    this.transcript = "";
+    this.patch({ state: "listening", transcript: "", error: null });
+
+    try {
+      await recognition.start({
+        lang: this.options.lang ?? "en-US",
+        interimResults: true,
+        continuous: false,
+        // Keep audio on the phone where the OS supports it.
+        requiresOnDeviceRecognition: undefined,
+      });
+    } catch (err) {
+      const message =
+        err instanceof Error ? err.message : "Could not start voice input.";
+      // If continuous loop fails to restart, exit loop gracefully.
+      if (this.continuous) {
+        this.continuous = false;
+        this.loopStopped = true;
+      }
+      this.fail(message);
+    }
+  }
+
+  private async askBot(transcript: string) {
+    let result: ChatResult;
+    try {
+      result = await chatWithUs(transcript);
+    } catch (err) {
+      // chatWithUs already reports its own failures, so this is a real crash.
+      this.fail(
+        err instanceof Error ? err.message : "Could not reach the bot.",
+        "denied",
+      );
+      return;
+    }
+    this.options.onUserQuery?.(transcript);
+
+    if (!result.ok) {
+      this.fail(result.error ?? "Could not reach the bot.");
+      return;
+    }
+
+    const spoken = await this.speakReply(result.reply);
+    if (this.continuous && !this.loopStopped) {
+      this.continueLoop();
+    } else {
+      this.finishSession({
+        ok: true,
+        transcript,
+        reply: result.reply,
+        userMessage: result.userMessage,
+        botMessage: result.botMessage,
+        spoken,
+      });
+    }
+  }
+
+  private continueLoop() {
+    this.loopStopped = false;
+    this.transcript = "";
+    this.patch({ transcript: "", error: null });
+    void this.beginListening();
+  }
+
+  /** Speak a reply and resolve when playback ends. */
+  private speakReply(text: string): Promise<boolean> {
+    const speech = loadSpeech();
+    const trimmed = text.trim();
+    if (!speech || !trimmed) return Promise.resolve(false);
+
+    return new Promise<boolean>((resolve) => {
+      this.speaking = true;
+      this.patch({ state: "speaking" });
+
+      let settled = false;
+      const finish = (ok: boolean) => {
+        if (settled) return;
+        settled = true;
+        this.speaking = false;
+        if (this.state.state === "speaking") this.patch({ state: "idle" });
+        resolve(ok);
+      };
+
+      // iOS never fires an end event for empty or whitespace-only text.
+      try {
+        speech.speak(trimmed, {
+          language: this.options.lang ?? "en-US",
+          rate: this.options.rate ?? 0.9,
+          pitch: this.options.pitch ?? 1,
+          onDone: () => finish(true),
+          onStopped: () => finish(false),
+          onError: () => finish(false),
+        });
+      } catch {
+        finish(false);
+        return;
+      }
+
+      // Safety net: never let a missing end event hang the promise.
+      setTimeout(() => finish(true), Math.max(8000, trimmed.length * 120));
+    });
+  }
+
+  /** Speak any text, e.g. to read a reply the user typed. */
+  speak(text: string): Promise<boolean> {
+    return this.speakReply(text);
+  }
+
+  async stopSpeaking() {
+    const speech = loadSpeech();
+    if (!speech) return;
+    this.speaking = false;
+    try {
+      await speech.stop();
+    } catch {
+      /* nothing playing */
+    }
+    if (this.state.state === "speaking") this.patch({ state: "idle" });
+  }
+
+  /** Stop listening and speaking without settling the promise. */
+  stop() {
+    if (this.settleTimer) {
+      clearTimeout(this.settleTimer);
+      this.settleTimer = null;
+    }
+    const recognition = loadRecognition();
+    void recognition?.abort?.();
+    void recognition?.stop?.();
+    void this.stopSpeaking();
+    this.continuous = false;
+    this.loopStopped = true;
+    if (this.state.state === "listening" || this.state.state === "thinking" || this.state.state === "speaking") {
+      this.patch({ state: "idle" });
+    }
+  }
+
+  /** A recogniser failure mid-session: report it and go back to idle. */
+  private failRecognition(message: string) {
+    this.patch({ state: "idle", error: message });
+    this.options.onError?.(message);
+    this.finishSession(this.failure(message, this.transcript.trim()));
+  }
+
+  /** Voice could not start at all (permission denied, module missing, no mic). */
+  private fail(message: string, state: VoiceState = "denied") {
+    this.patch({ state, error: message });
+    this.options.onError?.(message);
+    this.finishSession(this.failure(message, this.transcript.trim()));
+  }
+
+  private failure(error: string, transcript = ""): VoiceResult {
+    return {
+      ok: false,
+      transcript,
+      reply: "",
+      userMessage: null,
+      botMessage: null,
+      spoken: false,
+      error,
+    };
+  }
+
+  private finishSession(result: VoiceResult) {
+    const resolve = this.resolveSession;
+    this.resolveSession = null;
+    this.onFinal = null;
+    this.session = null;
+    this.loopStopped = true;
+    this.continuous = false;
+    if (this.state.state !== "unsupported") this.patch({ state: "idle" });
+    resolve?.(result);
+  }
+}
+
+const engine = new VoiceEngine();
+
+/**
+ * Point the voice engine at a language and wire the callbacks. Optional — every
+ * setting has a sensible default, so a bare `voiceWithUs()` works too.
+ *
+ * ```tsx
+ * configureVoice({
+ *   lang: "en-US",
+ *   speakAllReplies: true,
+ *   onPartial: (text) => setText(text),
+ * });
+ * ```
+ */
+export function configureVoice(options: VoiceOptions) {
+  engine.configure(options);
+}
+
+/** Read every bot reply aloud, not only the ones that were spoken. */
+export function setVoiceReplies(enabled: boolean) {
+  engine.setVoiceReplies(enabled);
+}
+
+/** Stop listening and stop talking. */
+export function stopVoice() {
+  engine.stop();
+}
+
+/**
+ * Ask out loud and hear the answer.
+ *
+ * ```tsx
+ * const res = await voiceWithUs();
+ * if (res.ok) console.log(res.transcript, "->", res.reply);
+ * ```
+ */
+export function voiceWithUs(options?: { continuous?: boolean }): Promise<VoiceResult> {
+  return engine.listen(options);
+}
+
+/** Say a line with the device synthesiser, e.g. to read a typed reply. */
+export function speakWithUs(text: string): Promise<boolean> {
+  return engine.speak(text);
+}
+
+/** Read a reply aloud only when `configureVoice({ speakAllReplies: true })`. */
+export function speakReplyIfEnabled(reply: string): Promise<boolean> {
+  return engine.getState().voiceReplies ? engine.speak(reply) : Promise.resolve(false);
+}
+
+/** True when both native modules are installed. */
+export function isVoiceSupported(): boolean {
+  return engine.isSupported();
+}
+
+/** Live voice state for rendering, the same way `useChat` exposes chat state. */
+export function useVoiceChat() {
+  const [state, setState] = useState<VoiceStateShape>(() => engine.getState());
+
+  useEffect(() => {
+    setState(engine.getState());
+    return engine.subscribe(() => setState(engine.getState()));
+  }, []);
+
+  return {
+    ...state,
+    listening: state.state === "listening",
+    thinking: state.state === "thinking",
+    speaking: state.state === "speaking",
+    start: (options?: { continuous?: boolean }) => voiceWithUs(options),
+    speak: speakWithUs,
+    stop: stopVoice,
+    setVoiceReplies,
+  };
+}

package/src/widget.tsx

@@ -16,8 +16,7 @@ import {
   KeyboardAvoidingView,
   Platform,
 } from "react-native";
-import { Audio } from "expo-av";
-import { WebView } from "react-native-webview";
+import { loadWebView } from "./lazy-webview";
 import { useWebotMe } from "./provider";
 import { AutoFlowExecutor } from "./AutoFlowExecutor";
 import type { WebotMeTheme, WebotMeAction, WebotMePendingWorkflow, WebotMeAutoTask, WebotMeScreenContext } from "./types";
@@ -342,6 +341,8 @@ export function WebotMeWidget({
     );
   }
 
+  const WebView = loadWebView();
+
   const web = (
     <WebView
       source={{ uri: `${chatUrl}?embed=1&bot=${botId}` }}