@webotme/react-native 0.1.1 → 0.1.2

package/README.md

@@ -149,6 +149,81 @@ const { text, action } = await client.send({
 });
 ```
 
+## Voice
+
+`voiceWithUs()` is the voice equivalent of `chatWithUs()` — one call, ask out
+loud, hear the answer:
+
+```tsx
+import { voiceWithUs } from "@webotme/react-native";
+
+const res = await voiceWithUs(); // tap the mic
+// user speaks -> device transcribes -> bot replies -> device speaks the reply
+console.log(res.transcript, "->", res.reply);
+```
+
+Just like the web widget, **everything runs on the device**. The phone's own
+recogniser transcribes the question and the phone's own synthesiser speaks the
+answer, so your server only ever sees plain text. No server change, no API keys,
+and no audio ever reaches your backend.
+
+### Install
+
+```bash
+npx expo install expo-speech expo-speech-recognition
+```
+
+Then add the permission strings to `app.json`:
+
+```json
+{
+  "expo": {
+    "plugins": [
+      [
+        "expo-speech-recognition",
+        {
+          "microphonePermission": "Allow Rover to hear you.",
+          "speechRecognitionPermission": "Allow Rover to turn your speech into text."
+        }
+      ]
+    ]
+  }
+}
+```
+
+Both packages are **optional peer dependencies**, resolved on first use. An app
+that only sends text never loads them and never breaks. Speech recognition is
+native code, so it needs a development build — `npx expo run:android` or
+`npx expo run:ios` — not Expo Go. `expo-speech` alone also needs a rebuild.
+
+### Options
+
+```tsx
+configureVoice({
+  lang: "en-US",            // BCP-47, used for both listening and speaking
+  rate: 0.9,                // web widget uses the same friendly pace
+  speakAllReplies: false,   // true = read every reply, not just spoken ones
+  onPartial: (text) => setText(text),   // live transcript while talking
+  onUserQuery: (text) => console.log(text),
+  onError: (msg) => alert(msg),
+});
+```
+
+### UI state
+
+`useVoiceChat()` mirrors `useChat()`:
+
+```tsx
+import { useVoiceChat } from "@webotme/react-native";
+
+const { state, transcript, listening, speaking, start, stop } = useVoiceChat();
+// state: "idle" | "listening" | "thinking" | "speaking" | "unsupported" | "denied"
+```
+
+Other helpers: `speakWithUs(text)` to read any line aloud, `stopVoice()` to stop
+listening and talking, `setVoiceReplies(on)` to toggle replies, and
+`isVoiceSupported()` to feature-detect.
+
 ## Errors
 
 Every failure is a `WebotMeError` with a stable `code`:
@@ -184,6 +259,10 @@ your own backend and use `POST /api/sdk/chat`.
 | New session | `POST /api/chatbot/public/{botId}/session/new` |
 | Web chat page | `GET {chatBaseUrl}/chat/{botId}` |
 
+Voice uses **none** of these. `voiceWithUs()` transcribes and speaks on the
+device, so `/native-chat/v1/stt` and `/native-chat/v1/tts` are never called —
+the same way the web widget runs on the browser's own speech APIs.
+
 ## License
 
 UNLICENSED

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webotme/react-native",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Drop-in AI chatbot for React Native and Expo apps. Same bot, training and AutoFlow as the WebotMe web widget.",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -11,12 +11,18 @@
   "scripts": {
     "build": "tsc --noEmit",
     "prepublishOnly": "npm run build",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "test:voice": "tsc -p tsconfig.smoke.json && node .smoke-build/voice.smoke.js && node .smoke-build/voice-nomodules.smoke.js",
+    "test": "npm run typecheck"
   },
   "keywords": [
     "chatbot",
     "ai",
     "chat",
+    "voice",
+    "speech",
+    "text-to-speech",
+    "speech-recognition",
     "react-native",
     "expo",
     "webotme",
@@ -24,11 +30,19 @@
   ],
   "license": "UNLICENSED",
   "peerDependencies": {
+    "expo-speech": ">=13",
+    "expo-speech-recognition": ">=3",
     "react": ">=18",
     "react-native": ">=0.72",
     "react-native-webview": ">=13"
   },
   "peerDependenciesMeta": {
+    "expo-speech": {
+      "optional": true
+    },
+    "expo-speech-recognition": {
+      "optional": true
+    },
     "react-native-webview": {
       "optional": true
     }
@@ -39,8 +53,5 @@
     "react-native": "^0.76.9",
     "react-native-webview": "^13.17.0",
     "typescript": "^5.9.3"
-  },
-  "dependencies": {
-    "expo-av": "^16.0.8"
   }
 }

package/src/chat.ts

@@ -0,0 +1,369 @@
+/**
+ * One-function chat for apps that already have their own UI.
+ *
+ * The provider owns the session when it is mounted, so `chatWithUs` works with
+ * no extra setup. Apps that skip the provider can call `configureChat` once.
+ */
+import { useEffect, useState } from "react";
+import { WebotMeClient } from "./client";
+import {
+  WebotMeError,
+  type WebotMeAction,
+  type WebotMeAutoTask,
+  type WebotMeConfig,
+  type WebotMeMessage,
+  type WebotMePendingWorkflow,
+  type WebotMeScreenContext,
+} from "./types";
+
+const makeId = (prefix: string) =>
+  `${prefix}_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
+
+type Listener = () => void;
+
+type ChatState = {
+  sessionId: string;
+  messages: WebotMeMessage[];
+  sending: boolean;
+  error: string | null;
+  pendingWorkflow: WebotMePendingWorkflow | null;
+  autoTask: WebotMeAutoTask | null;
+};
+
+/** What `chatWithUs` hands back so a custom UI can react without a hook. */
+export type ChatResult = {
+  ok: boolean;
+  /** The bot's reply, or an empty string when the send failed. */
+  reply: string;
+  /** The user's message plus the bot's reply, ready to render. */
+  userMessage: WebotMeMessage | null;
+  botMessage: WebotMeMessage | null;
+  action?: WebotMeAction;
+  actions?: WebotMeAction[];
+  pendingWorkflow?: WebotMePendingWorkflow;
+  autoTask?: WebotMeAutoTask;
+  error?: string;
+};
+
+class ChatEngine {
+  private client: WebotMeClient | null = null;
+  private userId = "";
+  private sessionId = makeId("s");
+  private screenContext: WebotMeScreenContext | undefined;
+  private listeners = new Set<Listener>();
+  private inflight = false;
+  private state: ChatState;
+
+  constructor() {
+    this.state = {
+      sessionId: this.sessionId,
+      messages: [],
+      sending: false,
+      error: null,
+      pendingWorkflow: null,
+      autoTask: null,
+    };
+  }
+
+  configure(config: WebotMeConfig) {
+    const nextClient = new WebotMeClient({
+      botId: config.botId,
+      apiBaseUrl: config.apiBaseUrl,
+      timeoutMs: config.timeoutMs,
+    });
+    const changedBot = !this.client || this.client.botId !== config.botId;
+    this.client = nextClient;
+    if (config.userId) this.userId = config.userId;
+    if (!this.userId) this.userId = makeId("v");
+    this.screenContext = config.screenContext;
+    if (changedBot) this.reset();
+    if (config.welcome && this.state.messages.length === 0) {
+      this.patch({
+        messages: [
+          { id: makeId("m"), from: "bot", text: config.welcome, createdAt: Date.now() },
+        ],
+      });
+    } else {
+      this.emit();
+    }
+  }
+
+  /** Keeps the live screen context in sync without re-creating the client. */
+  updateScreenContext(screenContext: WebotMeScreenContext | undefined) {
+    this.screenContext = screenContext;
+  }
+
+  private patch(next: Partial<ChatState>) {
+    this.state = { ...this.state, ...next };
+    this.emit();
+  }
+
+  private emit() {
+    for (const listener of this.listeners) listener();
+  }
+
+  subscribe(listener: Listener) {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+
+  getState(): ChatState {
+    return this.state;
+  }
+
+  reset() {
+    this.sessionId = makeId("s");
+    this.patch({
+      sessionId: this.sessionId,
+      messages: [],
+      sending: false,
+      error: null,
+      pendingWorkflow: null,
+      autoTask: null,
+    });
+  }
+
+  isReady() {
+    return this.client !== null;
+  }
+
+  getSessionId() {
+    return this.sessionId;
+  }
+
+  async send(text: string): Promise<ChatResult> {
+    const trimmed = text.trim();
+    if (!trimmed) {
+      return { ok: false, reply: "", userMessage: null, botMessage: null, error: "Message is empty" };
+    }
+    if (!this.client) {
+      const error = "Chat is not configured. Wrap the app in <WebotMeProvider> or call configureChat() first.";
+      this.patch({ error });
+      return { ok: false, reply: "", userMessage: null, botMessage: null, error };
+    }
+    if (this.inflight) {
+      const error = "A message is already being sent.";
+      return { ok: false, reply: "", userMessage: null, botMessage: null, error };
+    }
+
+    const client = this.client;
+    const sessionId = this.sessionId;
+    const userId = this.userId;
+
+    const userMessage: WebotMeMessage = {
+      id: makeId("m"),
+      from: "user",
+      text: trimmed,
+      createdAt: Date.now(),
+    };
+    const pendingId = makeId("m");
+
+    this.inflight = true;
+    this.patch({
+      messages: [
+        ...this.state.messages,
+        userMessage,
+        { id: pendingId, from: "bot", text: "", createdAt: Date.now(), pending: true },
+      ],
+      sending: true,
+      error: null,
+    });
+
+    try {
+      const result = await client.send({
+        message: trimmed,
+        sessionId,
+        userId,
+        screenContext: this.screenContext,
+        autoTask: this.state.autoTask ?? undefined,
+      });
+
+      const botMessage: WebotMeMessage = {
+        id: pendingId,
+        from: "bot",
+        text: result.text || result.reply || "",
+        createdAt: Date.now(),
+        action: result.actions?.[0] ?? result.action,
+        actions: result.actions,
+      };
+
+      this.patch({
+        messages: this.state.messages.map((m) => (m.id === pendingId ? botMessage : m)),
+        sending: false,
+        error: null,
+        pendingWorkflow: result.pendingWorkflow ?? null,
+        autoTask: result.autoTask ?? null,
+      });
+
+      return {
+        ok: true,
+        reply: botMessage.text,
+        userMessage,
+        botMessage,
+        action: botMessage.action,
+        actions: result.actions,
+        pendingWorkflow: result.pendingWorkflow,
+        autoTask: result.autoTask,
+      };
+    } catch (err) {
+      const error =
+        err instanceof WebotMeError || err instanceof Error
+          ? err.message
+          : "Something went wrong";
+      this.patch({
+        messages: this.state.messages.filter((m) => m.id !== pendingId),
+        sending: false,
+        error,
+      });
+      return { ok: false, reply: "", userMessage, botMessage: null, error };
+    } finally {
+      this.inflight = false;
+    }
+  }
+
+  /** Push the current live screen snapshot to the autonomous AutoFlow loop. */
+  async sendStep(): Promise<ChatResult> {
+    if (!this.client) {
+      const error = "Chat is not configured. Wrap the app in <WebotMeProvider> or call configureChat() first.";
+      this.patch({ error });
+      return { ok: false, reply: "", userMessage: null, botMessage: null, error };
+    }
+    if (!this.state.autoTask || !this.state.autoTask.goal) {
+      return { ok: false, reply: "", userMessage: null, botMessage: null, error: "No AutoFlow task to advance." };
+    }
+    if (this.inflight) {
+      return { ok: false, reply: "", userMessage: null, botMessage: null, error: "A step is already in flight." };
+    }
+    const pendingId = makeId("m");
+    this.inflight = true;
+    this.patch({ sending: true, error: null });
+    try {
+      const result = await this.client.send({
+        message: "",
+        sessionId: this.sessionId,
+        userId: this.userId,
+        screenContext: this.screenContext,
+        autoTask: this.state.autoTask as any,
+      });
+      const botMessage: WebotMeMessage = {
+        id: pendingId,
+        from: "bot",
+        text: result.text || result.reply || "",
+        createdAt: Date.now(),
+        action: result.actions?.[0] ?? result.action,
+        actions: result.actions,
+      };
+      this.patch({
+        messages: [
+          ...this.state.messages,
+          botMessage,
+        ],
+        sending: false,
+        error: null,
+        pendingWorkflow: result.pendingWorkflow ?? null,
+        autoTask: result.autoTask ?? this.state.autoTask,
+      });
+      return {
+        ok: true,
+        reply: botMessage.text,
+        userMessage: null,
+        botMessage,
+        action: botMessage.action,
+        actions: result.actions,
+        pendingWorkflow: result.pendingWorkflow,
+        autoTask: result.autoTask,
+      };
+    } catch (err) {
+      const error =
+        err instanceof WebotMeError || err instanceof Error ? err.message : "Something went wrong";
+      this.patch({ sending: false, error });
+      return { ok: false, reply: "", userMessage: null, botMessage: null, error };
+    } finally {
+      this.inflight = false;
+    }
+  }
+}
+
+const engine = new ChatEngine();
+
+/**
+ * Point the one-function chat at a bot. Only needed when the app does not use
+ * `WebotMeProvider`, which configures the engine automatically.
+ */
+export function configureChat(config: WebotMeConfig) {
+  engine.configure(config);
+}
+
+/**
+ * Keeps `chatWithUs` sending an up-to-date screen snapshot. The provider calls
+ * this for you; only touch it when driving the engine manually.
+ */
+export function updateChatScreenContext(screenContext: WebotMeScreenContext | undefined) {
+  engine.updateScreenContext(screenContext);
+}
+
+/** The id the next message will be sent under. */
+export function getChatSessionId() {
+  return engine.getSessionId();
+}
+
+/** Subscribe to chat engine state (used by executor/observer utilities). */
+export function subscribeChat(listener: () => void): () => void {
+  return engine.subscribe(listener);
+}
+
+/** Current chat state snapshot. */
+export function getChatState() {
+  return engine.getState();
+}
+
+/**
+ * Send the next AutoFlow step (empty message) to advance the server's autonomous
+ * task loop. Provider/executor wires this up internally.
+ */
+export async function nextAutoFlowStep(): Promise<ChatResult> {
+  return engine.sendStep();
+}
+
+/**
+ * Send one message to the bot and wait for its reply.
+ *
+ * ```tsx
+ * <TouchableOpacity
+ *   onPress={async () => {
+ *     const res = await chatWithUs(text);
+ *     if (res.ok) setText("");
+ *   }}
+ * />
+ * ```
+ */
+export async function chatWithUs(text: string): Promise<ChatResult> {
+  return engine.send(text);
+}
+
+/** Clear the conversation and start a fresh session. */
+export function resetChat() {
+  engine.reset();
+}
+
+/**
+ * Live chat state for rendering. Optional: use it to show the transcript, and
+ * keep `chatWithUs` for sending.
+ */
+export function useChat() {
+  const [state, setState] = useState<ChatState>(() => engine.getState());
+
+  useEffect(() => {
+    setState(engine.getState());
+    return engine.subscribe(() => setState(engine.getState()));
+  }, []);
+
+  return {
+    ...state,
+    send: chatWithUs,
+    reset: resetChat,
+    ready: engine.isReady(),
+  };
+}

package/src/client.ts

@@ -31,7 +31,8 @@ export type SendResult = {
  */
 export class WebotMeClient {
   private apiBaseUrl: string;
-  private botId: string;
+  /** Public so a host app can tell which bot an instance is bound to. */
+  readonly botId: string;
   private timeoutMs: number;
 
   constructor(config: Pick<WebotMeConfig, "botId" | "apiBaseUrl" | "timeoutMs">) {
@@ -112,6 +113,7 @@ export class WebotMeClient {
     sessionId: string;
     userId: string;
     screenContext?: WebotMeScreenContext;
+    autoTask?: WebotMeAutoTask;
   }): Promise<SendResult> {
     const body = await this.request<Record<string, unknown>>("/native-chat/v1", {
       method: "POST",
@@ -120,6 +122,7 @@ export class WebotMeClient {
         sessionId: params.sessionId,
         userId: params.userId,
         ...(params.screenContext ? { screenContext: params.screenContext } : {}),
+        ...(params.autoTask ? { autoTask: params.autoTask } : {}),
       }),
     });
 

package/src/executor.ts

@@ -0,0 +1,156 @@
+import { Alert, Linking } from "react-native";
+import { getChatState, nextAutoFlowStep, subscribeChat } from "./chat";
+import { isActionAllowed } from "./safety";
+import { findClickHandler, findFillHandler } from "./screenScanner";
+import type { WebotMeAction, WebotMeMessage } from "./types";
+
+export type ExecutorOptions = {
+  /** Called to navigate; defaults to expo-router when available. */
+  navigate?: (routeName: string, params?: Record<string, any>) => void;
+  goBack?: () => void;
+  push?: (routeName: string, params?: Record<string, any>) => void;
+  /** Called after every action attempt. */
+  onActionExecuted?: (action: WebotMeAction, success: boolean, error?: string) => void;
+};
+
+let handlers: ExecutorOptions | null = null;
+const executedMessageIds = new Set<string>();
+
+function routeNameFromUrl(url: string): string {
+  const match = /^([^?#]+)/.exec(url)?.[1] ?? url;
+  const trimmed = match.replace(/^\/+|\/+$/g, "");
+  if (!trimmed) return "Home";
+  const parts = trimmed.split("/").map((part) =>
+    part.charAt(0).toUpperCase() + part.slice(1),
+  );
+  return parts.join(":");
+}
+
+function defaultNavigate(routeName: string, params?: Record<string, any>) {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const router = require("expo-router").default ?? require("expo-router").router;
+    router?.push?.(routeName as any);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Run a single server action on the device. */
+export async function executeAction(action: WebotMeAction): Promise<{ success: boolean; error?: string }> {
+  const doNavigate = handlers?.navigate;
+  try {
+    // Block/confirm at the gate before doing anything on the device.
+    const allowed = await isActionAllowed(action);
+    if (!allowed) return { success: false, error: "Blocked or declined by safety layer" };
+
+    switch (action.type) {
+      case "navigate": {
+        const url = (action.data as any)?.url;
+        if (!url) throw new Error("Navigate action missing URL");
+        const name = routeNameFromUrl(url);
+        if (doNavigate) {
+          doNavigate(name, { fromAutoFlow: true, originalUrl: url });
+          return { success: true };
+        }
+        if (handlers?.push) {
+          handlers.push(name, { fromAutoFlow: true, originalUrl: url });
+          return { success: true };
+        }
+        const ok = defaultNavigate(url);
+        if (!ok) throw new Error(`No navigation handler for "${url}"`);
+        return { success: true };
+      }
+      case "click": {
+        const label =
+          (action.data as any)?.text ??
+          (action.data as any)?.selector ??
+          (action.data as any)?.label;
+        const h = findClickHandler(label);
+        if (h) {
+          h();
+          return { success: true };
+        }
+        console.warn("[WebotMe executor] click action has no ref binding:", action.data);
+        return { success: true };
+      }
+      case "fill": {
+        const label =
+          (action.data as any)?.label ??
+          (action.data as any)?.selector ??
+          (action.data as any)?.name;
+        const value = (action.data as any)?.text;
+        const setter = findFillHandler(label);
+        if (setter) {
+          setter(typeof value === "string" ? value : String(value ?? ""));
+          return { success: true };
+        }
+        console.log(
+          `[WebotMe executor] fill "${label}" -> "${value}" -- no matching input found.`,
+        );
+        return { success: true };
+      }
+      case "checkout": {
+        const url = (action.data as any)?.url;
+        if (!url) throw new Error("Checkout action missing URL");
+        await Linking.openURL(url);
+        return { success: true };
+      }
+      default:
+        return { success: false, error: `Unknown action type: ${(action as any).type}` };
+    }
+  } catch (err) {
+    return { success: false, error: err instanceof Error ? err.message : "Unknown error" };
+  }
+}
+
+/** Run the actions a bot returns on its message, in order. */
+export async function executeActions(actions: WebotMeAction[]): Promise<void> {
+  for (const action of actions.slice(0, 20)) {
+    const result = await executeAction(action);
+    handlers?.onActionExecuted?.(action, result.success, result.error);
+    if (!result.success) {
+      Alert.alert("Action Failed", result.error || "Unknown error");
+      break;
+    }
+    await new Promise((r) => setTimeout(r, 300));
+  }
+}
+
+/**
+ * Arm the executor: it watches the shared chat engine and runs each bot action
+ * exactly once. Returns a disposer.
+ */
+export function setupExecutor(options?: ExecutorOptions): () => void {
+  handlers = options ?? handlers ?? {};
+  const unsubscribe = subscribeChat(() => {
+    const { messages } = getChatState();
+    for (const message of messages) {
+      if (!message || message.from !== "bot" || executedMessageIds.has(message.id)) continue;
+      const actions = message.actions && message.actions.length > 0
+        ? message.actions
+        : message.action
+          ? [message.action]
+          : [];
+      if (actions.length === 0) continue;
+      executedMessageIds.add(message.id);
+      void executeActions(actions).then(() => {
+        // If we're inside an AutoFlow task, let the server decide the next step.
+        const state = getChatState();
+        if (state.autoTask && state.autoTask.goal) {
+          // Wait for the screen to settle so the next step sees fresh screenContext.
+          setTimeout(() => {
+            void nextAutoFlowStep();
+          }, 2500);
+        }
+      });
+    }
+  });
+  return () => unsubscribe();
+}
+
+export function resetExecutorState() {
+  executedMessageIds.clear();
+  handlers = null;
+}