@graineai/inapp-react-native 0.18.0 → 0.20.0

package/INTEGRATION.md

@@ -389,6 +389,74 @@ nowhere. Save is what pushes the catalogue to the agent.
 
 ---
 
+## Step 4b — Show them where, and draw them things
+
+### Pointing at something on screen
+
+"Tap Continue" is useless if the customer cannot find Continue.
+
+```tsx
+const { highlighted } = useGraineHighlight("continue_button");
+
+<Pressable style={[styles.btn, highlighted && styles.glow]}>
+  <Text>Continue</Text>
+</Pressable>
+```
+
+That is the whole integration. Registering the name tells the agent the element
+exists and can be pointed at; the boolean tells you when it is being pointed at.
+
+**The SDK does not draw the highlight**, deliberately — it does not own your
+design language, and a ring that looks right in one app looks broken in the
+next. You get a boolean and style it the way the rest of your app would.
+
+Highlights expire after six seconds. One that stays lit because the
+conversation moved on is worse than none: the customer keeps looking at the
+wrong thing.
+
+Declare `highlight_element` once on the agent, in the dashboard, alongside your
+other actions. **You do not implement it** — the SDK answers it, checks the name
+against what is actually registered on this screen, and refuses with the list of
+real targets if the agent invents one.
+
+### Components, not just sentences
+
+Sometimes a card, a set of choices or a three-field form serves the customer
+better than another sentence. The agent decides, and sends one:
+
+```tsx
+const { widget, submit, dismiss } = useGraineWidget();
+
+if (widget?.kind === "choices") {
+  return (
+    <Choices
+      options={widget.data.options}
+      onPick={(o) => submit({ chosen: o.id }, `I'll go with ${o.label}.`)}
+    />
+  );
+}
+```
+
+`submit` takes **two** things and both matter: the structured answer your
+backend wants, and a sentence for the conversation. Send only the first and the
+customer taps a card, sees a tick, and the agent carries on as though nothing
+happened.
+
+`widget.fallbackText` is what the agent would have said instead. Render it if
+you get a `kind` you do not handle — a component you cannot draw should still
+move the conversation forward.
+
+`dismiss()` is local only. A customer closing a card has not answered the
+question, so the agent is not told; it will ask again in words, which is right.
+
+<Callout>
+  Works on both transports. On `rtc` these travel the control channel beside
+  screen context and actions, for the same reason: they must arrive whole and in
+  order.
+</Callout>
+
+---
+
 ## Step 5 — Product events, without interrupting
 
 ```tsx
@@ -705,6 +773,8 @@ and resamples in its output stage, which is where the artefacts came from.
 | `useGraineIdentify()` | who the customer is |
 | `useGraineScreen()` | what is on the screen |
 | `useGraineAction()` | one handler per declared action |
+| `useGraineHighlight()` | let the agent point at an element on this screen |
+| `useGraineWidget()` | the component the agent drew, and how to answer it |
 | `useGraineTrack()` | product events that inform but do not interrupt |
 | `useGraineEvents()` | conversation lifecycle |
 | `useGraineVoice()` | only if you bring your own native audio instead of the WebView |

package/dist/client.d.ts

@@ -26,6 +26,7 @@ export declare class GraineInAppClient {
     private connecting;
     private listeners;
     private actions;
+    private highlightables;
     private screen;
     private screenSeq;
     private screenSince;
@@ -62,6 +63,9 @@ export declare class GraineInAppClient {
         data?: Record<string, unknown>;
     }>;
     getAvailableActions(): string[];
+    registerHighlightable(name: string): () => void;
+    getHighlightables(): string[];
+    highlight(name: string | null, ttlMs?: number): void;
     registerAction(name: string, handler: ActionHandler): () => void;
     get availableActions(): string[];
     setScreen(context: ScreenContext | null): void;

package/dist/client.js

@@ -10,6 +10,7 @@ export class GraineInAppClient {
         this.connecting = null;
         this.listeners = new Map();
         this.actions = new Map();
+        this.highlightables = new Set();
         this.screen = null;
         this.screenSeq = 0;
         this.screenSince = 0;
@@ -274,6 +275,22 @@ export class GraineInAppClient {
     getAvailableActions() {
         return [...this.actions.keys()];
     }
+    registerHighlightable(name) {
+        if (!name)
+            return () => { };
+        this.highlightables.add(name);
+        this.scheduleScreen();
+        return () => {
+            this.highlightables.delete(name);
+            this.scheduleScreen();
+        };
+    }
+    getHighlightables() {
+        return [...this.highlightables];
+    }
+    highlight(name, ttlMs = 6000) {
+        this.emit("highlight", { name, ttlMs });
+    }
     registerAction(name, handler) {
         this.actions.set(name, handler);
         this.scheduleScreen();
@@ -336,6 +353,7 @@ export class GraineInAppClient {
                 ...(this.screen ? { idle_ms: Date.now() - this.screenSince } : {}),
                 ...(Object.keys(this.identity).length ? { user: this.identity } : {}),
                 ...(this.recentEvents.length ? { recent_events: this.recentEvents } : {}),
+                ...(this.highlightables.size ? { highlightable: [...this.highlightables] } : {}),
             }),
             available_actions: (availableActions ?? this.availableActions).filter((a) => this.actions.has(a)),
             seq: ++this.screenSeq,
@@ -424,6 +442,7 @@ export class GraineInAppClient {
         return this.agentSpeaking;
     }
     submitWidget(widgetId, data, summary) {
+        this.emit("widget_result", { widget_id: widgetId, data, summary });
         this.send({ type: "widget_result", widget_id: widgetId, data });
         if (summary)
             this.say(summary);

package/dist/react-native/index.d.ts

@@ -50,6 +50,12 @@ export type GraineEvent = {
     type: "error";
     message: string;
 };
+export interface GraineWidget {
+    id: string;
+    kind: string;
+    data: Record<string, unknown>;
+    fallbackText: string;
+}
 export interface Caption {
     role: "agent" | "customer";
     text: string;
@@ -86,5 +92,13 @@ export declare function useGraineVoice(adapter: AudioAdapter | null, options?: V
     stop: () => Promise<void>;
 };
 export declare function useGraineScreen(context: ScreenContext | null): void;
+export declare function useGraineHighlight(name: string): {
+    highlighted: boolean;
+};
+export declare function useGraineWidget(): {
+    widget: GraineWidget | null;
+    submit: (data: Record<string, unknown>, summary: string) => void;
+    dismiss: () => void;
+};
 export declare function useGraineAction(name: string, handler: ActionHandler): void;
 export {};

package/dist/react-native/index.js

@@ -148,6 +148,20 @@ export function GraineProvider({ children, autoConnect = true, onProactive, endO
                 ? { ...prev, text: prev.text + text }
                 : { role: "agent", text, live: true })),
         ];
+        offs.push(client.registerAction("highlight_element", async (args) => {
+            const name = String(args?.name || args?.element || args?.target || "");
+            const known = client.getHighlightables();
+            if (!name || !known.includes(name)) {
+                return {
+                    status: "refused",
+                    message: known.length
+                        ? `There is nothing called "${name}" on this screen. I can point at: ${known.join(", ")}.`
+                        : "There is nothing on this screen I can point at.",
+                };
+            }
+            client.highlight(name, Number(args?.ttl_ms) || 6000);
+            return { status: "ok", message: `Highlighted ${name} for the customer.` };
+        }));
         if (autoConnect) {
             setConnecting(true);
             client.connect().catch((err) => {
@@ -296,6 +310,49 @@ export function useGraineScreen(context) {
         return () => client.setScreen(null);
     }, [client, serialised]);
 }
+export function useGraineHighlight(name) {
+    const { client } = useGraine();
+    const [highlighted, setHighlighted] = useState(false);
+    useEffect(() => client.registerHighlightable(name), [client, name]);
+    useEffect(() => {
+        let timer = null;
+        const off = client.on("highlight", (e) => {
+            const on = e?.name === name;
+            setHighlighted(on);
+            if (timer) {
+                clearTimeout(timer);
+                timer = null;
+            }
+            if (on)
+                timer = setTimeout(() => setHighlighted(false), Number(e?.ttlMs) || 6000);
+        });
+        return () => { if (timer)
+            clearTimeout(timer); off(); };
+    }, [client, name]);
+    return { highlighted };
+}
+export function useGraineWidget() {
+    const { client } = useGraine();
+    const [widget, setWidget] = useState(null);
+    useEffect(() => client.on("widget", (w) => {
+        if (!w?.widget_id)
+            return;
+        setWidget({
+            id: String(w.widget_id),
+            kind: String(w.kind || ""),
+            data: (w.data || {}),
+            fallbackText: String(w.fallback_text || ""),
+        });
+    }), [client]);
+    const submit = useCallback((data, summary) => {
+        if (!widget)
+            return;
+        client.submitWidget(widget.id, data, summary);
+        setWidget(null);
+    }, [client, widget]);
+    const dismiss = useCallback(() => setWidget(null), []);
+    return { widget, submit, dismiss };
+}
 export function useGraineAction(name, handler) {
     const { client } = useGraine();
     const handlerRef = useRef(handler);

package/dist/react-native/voice-launcher.js

@@ -66,9 +66,11 @@ export function GraineVoiceLauncher({ webView: WebView, autoStart = false, trans
         });
     }, [client, post]);
     useEffect(() => {
-        if (ready)
-            pushContext();
-    }, [ready, pushContext]);
+        if (!ready)
+            return;
+        pushContext();
+        return client.on("screen", () => pushContext());
+    }, [client, ready, pushContext]);
     const beginCall = useCallback(async () => {
         if (client.isConnected) {
             console.warn("[graine] The provider has its own connection open while GraineVoiceLauncher " +
@@ -93,6 +95,18 @@ export function GraineVoiceLauncher({ webView: WebView, autoStart = false, trans
             post({ type: "graine:app-event", name, data: rest });
         });
     }, [client, connected, post, pushContext]);
+    useEffect(() => {
+        if (!connected)
+            return;
+        return client.on("widget_result", (r) => {
+            post({
+                type: "graine:widget-result",
+                widgetId: r?.widget_id,
+                data: r?.data ?? {},
+                summary: r?.summary ?? "",
+            });
+        });
+    }, [client, connected, post]);
     const onMessage = useCallback(async (e) => {
         let msg;
         try {
@@ -143,6 +157,10 @@ export function GraineVoiceLauncher({ webView: WebView, autoStart = false, trans
             case "graine:mic-denied":
                 onMicDenied?.(String(msg.reason || "denied"));
                 return;
+            case "graine:widget": {
+                client.emit?.("widget", msg.widget || {});
+                return;
+            }
             case "graine:app-action": {
                 const action = msg.action || {};
                 const result = await client.executeAction(action.name, action.arguments || {});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graineai/inapp-react-native",
-  "version": "0.18.0",
+  "version": "0.20.0",
   "type": "module",
   "description": "Graine in-app agent for React Native \u2014 an agent that sees the screen your customer is on and can act on it.",
   "license": "MIT",