@webotme/react-native 0.1.1 → 0.1.2

package/src/screenScanner.tsx

@@ -0,0 +1,376 @@
+import { useEffect, useState } from "react";
+import { updateChatScreenContext } from "./chat";
+import { getCurrentRoute } from "./navigationAdapter";
+import { getCapturedRootFiber } from "./screenHook";
+import type {
+  WebotMeField,
+  WebotMeButton,
+  WebotMeLink,
+  WebotMeList,
+  WebotMeScreenContext,
+} from "./types";
+
+type ReactNodeLike = React.ReactNode;
+
+let lastTree: ReactNodeLike = null;
+let scanTimer: ReturnType<typeof setTimeout> | null = null;
+let lastJson = "";
+
+/** Provider registers whatever it renders so the scanner can read the tree. */
+export function registerScreenTree(node: ReactNodeLike) {
+  lastTree = node;
+}
+
+// ---------------------------------------------------------------------------
+// Fiber-tree helpers (automatic screen capture from the DevTools hook root)
+// ---------------------------------------------------------------------------
+
+const DEBUG_UI_NAMES = ["ChatBot", "ChatBotIcon", "ChatBotWrapper"];
+
+/** Walk live fibers from the captured React root, skipping debug UI that the
+ * Chat modal/bar own (otherwise the assistant starts reading its own logs). */
+function walkAllFibers(): any[] {
+  const root = getCapturedRootFiber();
+  const out: any[] = [];
+  if (!root) return out;
+
+  const stack: any[] = [root];
+  let guard = 0;
+  while (stack.length && guard < 2000) {
+    guard++;
+    const fiber = stack.pop();
+    const name = fiberTypeName(fiber);
+    if (DEBUG_UI_NAMES.includes(name)) {
+      // Skip the whole debug/Modal subtree.
+      continue;
+    }
+    out.push(fiber);
+    let child = fiber?.child;
+    while (child) {
+      // Don't even visit the ChatBotIcon/ChatBot children subtree.
+      if (!DEBUG_UI_NAMES.includes(fiberTypeName(child))) {
+        stack.push(child);
+      } else {
+        // still push its siblings
+      }
+      child = child.sibling;
+    }
+  }
+  return out;
+}
+
+function fiberTypeName(fiber: any): string {
+  const t = fiber?.type;
+  if (typeof t === "string") return t;
+  if (!t) return "";
+  if (t.displayName) return t.displayName;
+  if (t.name) return t.name;
+  if (t.type?.name) return t.type.name;
+  if (t.render?.name) return t.render.name;
+  if (t.render?.displayName) return t.render.displayName;
+  return "";
+}
+
+/** Text visible under this fiber (shallow across children). */
+function deepText(fiber: any, depth: number): string {
+  if (!fiber || depth > 4) return "";
+  const parts: string[] = [];
+  const p = fiber.memoizedProps;
+  if (p) {
+    const kids = Array.isArray(p.children) ? p.children : [p.children];
+    for (const k of kids) {
+      if (typeof k === "string" && k.trim()) parts.push(k.trim());
+    }
+  }
+  let c = fiber.child;
+  while (c) {
+    const t = deepText(c, depth + 1);
+    if (t) parts.push(t);
+    c = c.sibling;
+  }
+  return parts.join(" ");
+}
+
+function normalizeForMatch(v: unknown): string {
+  return String(v ?? "").trim().toLowerCase().replace(/[^a-z0-9]/g, "");
+}
+
+function textMatches(target: unknown, wanted: unknown): boolean {
+  const t = normalizeForMatch(target);
+  const w = normalizeForMatch(wanted);
+  if (!w) return false;
+  return t.includes(w) || w.includes(t);
+}
+
+/** Find a pressable component whose visible label matches, return its onPress. */
+export function findClickHandler(label: unknown): (() => any) | null {
+  const wanted = normalizeForMatch(label);
+  if (!wanted) return null;
+  for (const fiber of walkAllFibers()) {
+    const p = fiber?.memoizedProps;
+    if (!p || typeof p.onPress !== "function") continue;
+    const candidates: unknown[] = [
+      p.accessibilityLabel,
+      p.label,
+      p.title,
+      p.options?.title,
+      deepText(fiber, 0),
+    ];
+    if (candidates.some((c) => textMatches(c, wanted))) {
+      return () => p.onPress();
+    }
+  }
+  return null;
+}
+
+/** Find a text input whose label/placeholder/name matches, return a setter. */
+export function findFillHandler(label: unknown): ((value: string) => any) | null {
+  const wanted = normalizeForMatch(label);
+  if (!wanted) return null;
+  for (const fiber of walkAllFibers()) {
+    const p = fiber?.memoizedProps;
+    if (!p) continue;
+    const typeName = fiberTypeName(fiber);
+    const isInput =
+      /^TextInput|^Input|TextArea/i.test(typeName) ||
+      typeof p.onChangeText === "function" ||
+      p.placeholder !== undefined ||
+      p.keyboardType !== undefined;
+    if (!isInput) continue;
+    const candidates: unknown[] = [
+      p.label,
+      p.name,
+      p.options?.title,
+      p.placeholder,
+      p.accessibilityLabel,
+    ];
+    if (candidates.some((c) => textMatches(c, wanted))) {
+      return (value: string) => p.onChangeText?.(value);
+    }
+  }
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Screen map builder
+// ---------------------------------------------------------------------------
+
+function visit(
+  node: ReactNodeLike,
+  ctx: WebotMeScreenContext,
+  depth: number,
+  budget: { count: number },
+): void {
+  if (depth > 14 || budget.count > 600) return;
+  if (node == null || typeof node === "boolean") return;
+
+  if (typeof node === "string") {
+    const text = node.trim();
+    if (text.length > 1 && (ctx.visibleText?.length ?? 0) < 40) {
+      ctx.visibleText!.push(text);
+    }
+    return;
+  }
+  if (typeof node === "number") return;
+
+  if (Array.isArray(node)) {
+    for (const n of node) visit(n, ctx, depth + 1, budget);
+    return;
+  }
+
+  const element = node as { type?: any; props?: Record<string, any> };
+  if (typeof element !== "object" || !element.props) return;
+
+  budget.count++;
+  const typeName =
+    typeof element.type === "string"
+      ? element.type
+      : element.type?.displayName || element.type?.name || "";
+
+  const label =
+    element.props.accessibilityLabel ??
+    element.props.title ??
+    element.props.label ??
+    element.props.options?.title ??
+    (typeof element.props.children === "string"
+      ? element.props.children
+      : undefined);
+
+  const isButton =
+    element.props.onPress ||
+    /Pressable|Touchable|Button|IconButton/i.test(typeName);
+  const isInput =
+    element.props.placeholder !== undefined ||
+    element.props.keyboardType !== undefined ||
+    /TextInput|Input|TextArea/i.test(typeName);
+  const isLink =
+    /Link/i.test(typeName) ||
+    /Screen$/i.test(typeName) ||
+    element.props.href ||
+    element.props.route;
+  const isList =
+    /FlatList|SectionList|List$/i.test(typeName) ||
+    Array.isArray(element.props.data);
+
+  if (isButton && label) {
+    ctx.buttons!.push({
+      text: label,
+      accessibilityLabel: element.props.accessibilityLabel,
+      disabled: element.props.disabled,
+    });
+  }
+
+  if (isInput) {
+    ctx.forms!.push({
+      label: label ?? element.props.name,
+      placeholder: element.props.placeholder,
+      name: element.props.name,
+      type: element.props.inputMode || element.props.keyboardType,
+      value: element.props.value,
+      required: element.props.required,
+    });
+  }
+
+  if (isLink) {
+    ctx.links!.push({
+      text: label,
+      route: element.props.route ?? element.props.href ?? element.props.name,
+    });
+  }
+
+  if (isList) {
+    const items = Array.isArray(element.props.data)
+      ? element.props.data.map((d: any) => d?.name ?? d?.title ?? String(d))
+      : undefined;
+    ctx.lists!.push({ label, items });
+  }
+
+  visit(element.props.children, ctx, depth + 1, budget);
+}
+
+/** A lightweight map of the current screen for the LLM. Prefers the live
+ * fiber tree; falls back to the element props passed to the provider. */
+export function scanScreen(): WebotMeScreenContext {
+  const ctx: WebotMeScreenContext = {
+    name: getCurrentRoute() || "App",
+    visibleText: [],
+    buttons: [],
+    forms: [],
+    links: [],
+    lists: [],
+    alerts: [],
+  };
+
+  const fibers = walkAllFibers();
+  if (fibers.length > 0) {
+    for (const fiber of fibers) {
+      const p = fiber?.memoizedProps;
+      if (!p) continue;
+      const name = fiberTypeName(fiber);
+
+      const isButton =
+        typeof p.onPress === "function" ||
+        /Pressable|Touchable|^(Icon)?Button$|IconButton/i.test(name);
+      const isInput =
+        /^TextInput|^Input|TextArea/i.test(name) ||
+        typeof p.onChangeText === "function" ||
+        p.placeholder !== undefined ||
+        p.keyboardType !== undefined;
+      const isLink = /Link/i.test(name) || /Screen$/i.test(name) || p.href || p.route;
+      const isList = /FlatList|SectionList|List$/i.test(name) || Array.isArray(p.data);
+
+      const label =
+        p.accessibilityLabel ??
+        p.label ??
+        p.title ??
+        p.options?.title ??
+        (typeof p.children === "string" ? p.children : undefined);
+
+      if (isButton && (label || deepText(fiber, 0))) {
+        ctx.buttons!.push({
+          text: label ?? deepText(fiber, 0),
+          accessibilityLabel: p.accessibilityLabel,
+          disabled: p.disabled,
+        });
+      }
+      if (isInput) {
+        ctx.forms!.push({
+          label: label ?? p.name,
+          placeholder: p.placeholder,
+          name: p.name,
+          type: p.inputMode || p.keyboardType,
+          value: p.value,
+          required: p.required,
+        });
+      }
+      if (isLink) {
+        ctx.links!.push({
+          text: label ?? p.name,
+          route: p.route ?? p.href ?? p.name,
+        });
+      }
+      if (isList) {
+        const items = Array.isArray(p.data)
+          ? p.data.map((d: any) => d?.name ?? d?.title ?? String(d))
+          : undefined;
+        ctx.lists!.push({ label: label ?? undefined, items });
+      }
+
+      if (/Text/i.test(name)) {
+        const kids = Array.isArray(p.children) ? p.children : [p.children];
+        const pieces: string[] = [];
+        for (const k of kids) {
+          if (typeof k === "string" && k.trim()) pieces.push(k.trim());
+        }
+        if (pieces.length) ctx.visibleText!.push(pieces.join(" "));
+      }
+    }
+    ctx.visibleText = Array.from(new Set(ctx.visibleText ?? []));
+    updateChatScreenContext(ctx);
+    return ctx;
+  }
+
+  visit(lastTree, ctx, 0, { count: 0 });
+  ctx.visibleText = Array.from(new Set(ctx.visibleText ?? []));
+  updateChatScreenContext(ctx);
+  return ctx;
+}
+
+/**
+ * Turn the scanner on. It keeps the chat engine's screenContext fresh.
+ * Returns a stop function.
+ */
+export function setupFiberScanner(options?: {
+  intervalMs?: number;
+  onScan?: (ctx: WebotMeScreenContext) => void;
+}): () => void {
+  stopFiberScanner();
+  const interval = options?.intervalMs ?? 2000;
+  scanScreen();
+  scanTimer = setInterval(() => {
+    const ctx = scanScreen();
+    const json = JSON.stringify(ctx);
+    if (json !== lastJson) {
+      lastJson = json;
+      options?.onScan?.(ctx);
+    }
+  }, interval) as unknown as ReturnType<typeof setTimeout>;
+  return stopFiberScanner;
+}
+
+export function stopFiberScanner() {
+  if (scanTimer) {
+    clearTimeout(scanTimer as unknown as ReturnType<typeof setTimeout>);
+    scanTimer = null;
+  }
+}
+
+/** Live snapshot for debug UIs and previews. */
+export function useFiberScanner() {
+  const [context, setContext] = useState<WebotMeScreenContext | null>(null);
+  useEffect(() => {
+    const timer = setInterval(() => setContext(scanScreen()), 2000);
+    return () => clearInterval(timer);
+  }, []);
+  return context;
+}

package/src/types.ts

@@ -57,6 +57,8 @@ export type WebotMeMessage = {
   createdAt: number;
   /** Present when the server attached a native action (AutoFlow). */
   action?: WebotMeAction;
+  /** Optional list of actions returned for this bot turn. */
+  actions?: WebotMeAction[];
   pending?: boolean;
   failed?: boolean;
 };