@graineai/inapp-react-native 0.27.5 → 0.30.0

package/INTEGRATION.md

@@ -558,6 +558,53 @@ your funnel.
 
 ---
 
+## Captions and hanging up
+
+`GraineVoiceLauncher`'s render prop gives you the whole transcript and the CC
+switch, so a caption box is a `map` and nothing else. **0.28.0.**
+
+```tsx
+<GraineVoiceLauncher webView={WebView} active={open} autoStart>
+  {(api) => (
+    <>
+      {api.captionsOn && api.captions.map((c, i) => (
+        <Text key={i} style={{ opacity: c.live ? 0.7 : 1 }}>
+          {c.role === "agent" ? "Ring" : "You"}: {c.text}
+        </Text>
+      ))}
+      <Button title="CC" onPress={() => api.setCaptionsOn(!api.captionsOn)} />
+      <Button title="End" onPress={api.end} />
+    </>
+  )}
+</GraineVoiceLauncher>
+```
+
+| On `api` | |
+|---|---|
+| `captions` | Every turn so far, oldest first, capped at `CAPTION_HISTORY` (12) |
+| `caption` | The turn being said now, or the last one. `null` before any |
+| `captionsOn` / `setCaptionsOn` | The CC switch. Defaults **on** |
+| `clearCaptions()` | Forget the transcript without ending the call |
+| `end()` | Hang up |
+
+A turn arrives many times as its words come in — each with fuller text and
+`live: true` — then once more settled with `live: false`. The launcher folds
+those into one entry, so `captions` holds turns, not keystrokes. Render `live`
+ones as provisional if you want; they are the same line about to be finished.
+
+The transcript is **cleared when a call starts and when you set `active={false}`**.
+A call's words outliving its call is a privacy question, not a feature — if you
+need them afterwards, copy them in `onEnded`.
+
+`onCaption` still fires for every line and is unchanged; it is now a convenience
+on top of `captions` rather than the only way to see them.
+
+**Hanging up** happens on `api.end()` and again on `active={false}` — both post
+the same idempotent stop frame, so a host that does one, the other, or both
+ends exactly one call. Before 0.27.6 the deactivate path was conditional on the
+launcher's own idea of whether a call was running, which could close the UI and
+leave the call up.
+
 ## Tap capture when your screens sit above the provider
 
 `GraineProvider` wraps its children in `GraineTouchCapture`, so an app whose
@@ -1077,7 +1124,7 @@ told to ignore).
 | `onMicDenied` | `(reason) => void` | `NotAllowedError` is a refusal; `NotFoundError` is a device with no usable microphone. Different sentences for a customer. |
 | `onError` | `(message) => void` | Voice could not be set up at all — key rejected, agent not enabled for apps, no network. Without this the launcher renders nothing and you cannot tell why. |
 | `children` | `(api) => ReactNode` | Receives `{ connected, connecting, muted, start, end, setMuted }`. |
-| `active` | `boolean` | Default `true`. Mount early with `false` so the engine page loads before the customer needs it (it registers nothing while idle); flip to `true` when your UI opens — with `autoStart` that places the call, without it it warms the registration so the tap is instant. Back to `false` ends a live call. (0.27.5) |
+| `active` | `boolean` | Default `true`. Mount early with `false` so the engine page loads before the customer needs it (it registers nothing while idle); flip to `true` when your UI opens — with `autoStart` that places the call, without it it warms the registration so the tap is instant. Back to `false` ends a live call. (0.27.5; 0.27.6 makes that hangup unconditional) |
 
 **The launcher owns the connection.** It asks for the microphone, resolves the
 session, holds the socket and relays your screen, actions and events onto it. So

package/dist/friction.d.ts

@@ -20,6 +20,7 @@ export interface FrictionScreenRules {
     repeatedErrors?: RepeatedErrorsRule | false;
     backAndForth?: BackAndForthRule | false;
     rageTaps?: RageTapsRule | false;
+    incompleteSubmit?: IncompleteSubmitRule | false;
     screenError?: boolean;
     message?: string;
 }
@@ -34,10 +35,13 @@ export interface FrictionConfig extends FrictionScreenRules {
     nudge?: FrictionNudge;
     screens?: Record<string, FrictionScreenRules>;
 }
+export interface IncompleteSubmitRule {
+    taps: string[];
+}
 export declare const DEFAULT_FRICTION: Readonly<Required<Omit<FrictionConfig, "message" | "ignore">>>;
 export declare function resolveFriction(...layers: Array<FrictionConfig | null | undefined>): Required<FrictionConfig>;
 export declare function rulesFor(config: Required<FrictionConfig>, screenId: string | null | undefined): Required<FrictionScreenRules>;
-export type FrictionKind = "stalled_on_step" | "repeated_field_error" | "looping_between_screens" | "rage_taps" | "screen_error";
+export type FrictionKind = "stalled_on_step" | "repeated_field_error" | "looping_between_screens" | "rage_taps" | "incomplete_submit" | "screen_error";
 export interface FrictionSignal {
     name: FrictionKind;
     screen: string;

package/dist/friction.js

@@ -3,6 +3,9 @@ const DEFAULT_RULES = {
     repeatedErrors: { count: 2, windowMs: 60000 },
     backAndForth: { visits: 3, windowMs: 45000 },
     rageTaps: { taps: 4, windowMs: 1500 },
+    incompleteSubmit: {
+        taps: ["submit", "next", "continue", "save", "proceed", "apply", "confirm", "done", "pay", "verify"],
+    },
 };
 export const DEFAULT_FRICTION = Object.freeze({
     enabled: true,
@@ -25,8 +28,19 @@ function rule(raw, shape) {
         return undefined;
     const out = {};
     for (const [k, fallback] of Object.entries(shape)) {
-        const v = num(raw[k]);
-        out[k] = v ?? fallback;
+        const given = raw[k];
+        if (typeof fallback === "number") {
+            out[k] = num(given) ?? fallback;
+        }
+        else if (Array.isArray(fallback)) {
+            out[k] = Array.isArray(given) && given.length ? given.map((v) => String(v)) : fallback;
+        }
+        else if (typeof fallback === "boolean") {
+            out[k] = typeof given === "boolean" ? given : fallback;
+        }
+        else {
+            out[k] = given ?? fallback;
+        }
     }
     return out;
 }
@@ -49,6 +63,9 @@ function screenRules(raw, base) {
     const rage = rule(r.rageTaps, base.rageTaps || DEFAULT_RULES.rageTaps);
     if (rage !== undefined)
         out.rageTaps = rage;
+    const inc = rule(r.incompleteSubmit, base.incompleteSubmit || DEFAULT_RULES.incompleteSubmit);
+    if (inc !== undefined)
+        out.incompleteSubmit = inc;
     if (typeof r.screenError === "boolean")
         out.screenError = r.screenError;
     if (typeof r.message === "string" && r.message.trim())
@@ -100,6 +117,7 @@ export function rulesFor(config, screenId) {
         ignore: over.ignore ?? false,
         idle: over.idle ?? config.idle,
         repeatedErrors: over.repeatedErrors ?? config.repeatedErrors,
+        incompleteSubmit: over.incompleteSubmit ?? config.incompleteSubmit,
         backAndForth: over.backAndForth ?? config.backAndForth,
         rageTaps: over.rageTaps ?? config.rageTaps,
         screenError: over.screenError ?? config.screenError,
@@ -109,6 +127,7 @@ export function rulesFor(config, screenId) {
 function fill(template, vars) {
     return template.replace(/\{(\w+)\}/g, (_, k) => (vars[k] === undefined || vars[k] === null ? "" : String(vars[k])));
 }
+const outstanding = (fields) => (fields ?? []).filter((f) => f.error || (!f.optional && (f.status === "invalid" || f.status === "empty")));
 const stuckField = (fields) => (fields ?? []).find((f) => f.status === "invalid" || f.error) ?? (fields ?? []).find((f) => f.status === "empty");
 export class FrictionDetector {
     constructor(config) {
@@ -213,10 +232,26 @@ export class FrictionDetector {
     }
     onTrack(name, now) {
         const rules = this.active();
-        if (!rules || !rules.rageTaps || !this.screen)
+        if (!rules || !this.screen)
             return [];
         if (!name.startsWith("tap:"))
             return [];
+        if (!rules.rageTaps && !rules.incompleteSubmit)
+            return [];
+        const label0 = name.slice(4);
+        if (rules.incompleteSubmit) {
+            const lower = label0.toLowerCase();
+            const isSubmit = rules.incompleteSubmit.taps.some((t) => lower.includes(t.toLowerCase()));
+            const left = outstanding(this.screen.fields);
+            if (isSubmit && left.length) {
+                const names = left.map((f) => f.label || f.name).filter(Boolean);
+                const s = this.signal("incomplete_submit", { tap: label0, count: left.length, fields: names.join(", "), first: names[0] ?? "" }, "The customer pressed \"{tap}\" on the \"{title}\" screen with {count} still to fill: {fields}. Tell them what is missing before they hit the error.", now);
+                if (s)
+                    return [s];
+            }
+        }
+        if (!rules.rageTaps)
+            return [];
         const { taps, windowMs } = rules.rageTaps;
         const key = `${this.screen.screen}:${name}`;
         const times = (this.taps.get(key) ?? []).filter((t) => now - t <= windowMs);
@@ -225,7 +260,7 @@ export class FrictionDetector {
         if (times.length < taps)
             return [];
         this.taps.set(key, []);
-        const label = name.slice(4);
+        const label = label0;
         const s = this.signal("rage_taps", { tap: label, count: times.length, seconds: (windowMs / 1000).toFixed(1) }, label === "screen"
             ? `The customer tapped the "{title}" screen {count} times in {seconds}s — something on it may not be responding or not doing what they expect.`
             : `The customer tapped "{tap}" {count} times in {seconds}s on the "{title}" screen — it may not be responding or not doing what they expect.`, now);

package/dist/index.d.ts

@@ -7,9 +7,9 @@ export { addMaskRule, maskDeep, maskString } from "./mask.js";
 export { GraineProvider, useGraineAgent, useGraineScreen, useGraineAction, useGraineVoice, useGraineIdentify, useGraineEvents, useGraineField, useGraineClient, setDefaultClient, useGraineTrack, useGraineTap, useGraineHighlight, useGraineWidget, useGraineReady, useGraineInit, type GraineEvent, type GraineProviderProps, type Turn, type Caption, } from "./react-native/index.js";
 export { requestMicrophonePermission } from "./react-native/permissions.js";
 export { GraineFab, GraineTouchCapture, TAP_MAX_MOVE_PX, TAP_MAX_MS, type GraineFabProps, type GraineTouchCaptureProps, } from "./react-native/fab.js";
-export { UNNAMED_TAP } from "./protocol.js";
+export { UNNAMED_TAP, resolveWidgetImages } from "./protocol.js";
 export { resolveNativeAppearance, type NativeAppearance, type NativeAppearanceFallback, } from "./react-native/appearance.js";
 export { RtcVoiceSessionController, RtcVoiceError, type RtcVoiceState, type RtcVoiceErrorCode, type RtcVoiceSession, type RtcVoiceOptions, } from "./react-native/voice-rtc.js";
 export { GraineAgentBar, GraineLauncher, type GraineAgentBarProps, type GraineLauncherProps, type GraineBarTheme, } from "./react-native/ui.js";
-export { GraineVoiceLauncher, type GraineVoiceLauncherProps, type GraineVoiceApi, } from "./react-native/voice-launcher.js";
+export { GraineVoiceLauncher, CAPTION_HISTORY, type GraineVoiceLauncherProps, type GraineVoiceApi, } from "./react-native/voice-launcher.js";
 export { LauncherVisibilityTracker, activeRouteName, type LauncherInset, type LauncherContinuity, type LauncherDelayPolicy, type LauncherGroup, type LauncherVisibility, type VisibilityDecision, } from "./react-native/navigation.js";

package/dist/index.js

@@ -6,9 +6,9 @@ export { addMaskRule, maskDeep, maskString } from "./mask.js";
 export { GraineProvider, useGraineAgent, useGraineScreen, useGraineAction, useGraineVoice, useGraineIdentify, useGraineEvents, useGraineField, useGraineClient, setDefaultClient, useGraineTrack, useGraineTap, useGraineHighlight, useGraineWidget, useGraineReady, useGraineInit, } from "./react-native/index.js";
 export { requestMicrophonePermission } from "./react-native/permissions.js";
 export { GraineFab, GraineTouchCapture, TAP_MAX_MOVE_PX, TAP_MAX_MS, } from "./react-native/fab.js";
-export { UNNAMED_TAP } from "./protocol.js";
+export { UNNAMED_TAP, resolveWidgetImages } from "./protocol.js";
 export { resolveNativeAppearance, } from "./react-native/appearance.js";
 export { RtcVoiceSessionController, RtcVoiceError, } from "./react-native/voice-rtc.js";
 export { GraineAgentBar, GraineLauncher, } from "./react-native/ui.js";
-export { GraineVoiceLauncher, } from "./react-native/voice-launcher.js";
+export { GraineVoiceLauncher, CAPTION_HISTORY, } from "./react-native/voice-launcher.js";
 export { LauncherVisibilityTracker, activeRouteName, } from "./react-native/navigation.js";

package/dist/protocol.d.ts

@@ -4,6 +4,7 @@ export interface ScreenField {
     value?: string | number | null;
     status?: "empty" | "filled" | "invalid" | "pending" | "locked";
     error?: string;
+    optional?: boolean;
 }
 export interface ScreenContext {
     screen: string;
@@ -12,6 +13,7 @@ export interface ScreenContext {
         name: string;
         step?: number;
         of?: number;
+        next?: string;
     };
     fields?: ScreenField[];
     error?: string;
@@ -59,3 +61,4 @@ export type ActionHandler = (args: Record<string, any>) => AppActionResult | voi
 export declare const ACTION_DEADLINE_MS = 6000;
 export declare const CONTEXT_THROTTLE_MS = 400;
 export declare const UNNAMED_TAP = "tap:screen";
+export declare function resolveWidgetImages(data: Record<string, unknown>, baseUrl: string): Record<string, unknown>;

package/dist/protocol.js

@@ -1,3 +1,31 @@
 export const ACTION_DEADLINE_MS = 6000;
 export const CONTEXT_THROTTLE_MS = 400;
 export const UNNAMED_TAP = "tap:screen";
+const IMAGE_KEY = /^(image|imageurl|image_url|icon|iconurl|icon_url|logo|logourl|logo_url|photo|photourl|photo_url|thumbnail|thumbnailurl|thumbnail_url)$/i;
+function resolveOne(value, base) {
+    if (typeof value !== "string" || !value.startsWith("/") || value.startsWith("//"))
+        return value;
+    return `${base.replace(/\/+$/, "")}${value}`;
+}
+export function resolveWidgetImages(data, baseUrl) {
+    if (!data || typeof data !== "object" || !baseUrl)
+        return data;
+    const out = {};
+    for (const [k, v] of Object.entries(data)) {
+        if (IMAGE_KEY.test(k)) {
+            out[k] = resolveOne(v, baseUrl);
+        }
+        else if (Array.isArray(v)) {
+            out[k] = v.map((row) => row && typeof row === "object" && !Array.isArray(row)
+                ? Object.fromEntries(Object.entries(row).map(([rk, rv]) => [
+                    rk,
+                    IMAGE_KEY.test(rk) ? resolveOne(rv, baseUrl) : rv,
+                ]))
+                : row);
+        }
+        else {
+            out[k] = v;
+        }
+    }
+    return out;
+}

package/dist/react-native/index.js

@@ -3,6 +3,7 @@ import { createContext, useCallback, useContext, useEffect, useMemo, useRef, use
 import { AppState } from "react-native";
 import { GraineInAppClient } from "../client.js";
 import { VoiceSession } from "../voice.js";
+import { resolveWidgetImages } from "../protocol.js";
 import { GraineFab, GraineTouchCapture } from "./fab.js";
 export { GraineFab, GraineTouchCapture } from "./fab.js";
 import { LauncherVisibilityTracker, activeRouteName, } from "./navigation.js";
@@ -395,7 +396,7 @@ export function useGraineWidget() {
         setWidget({
             id: String(w.widget_id),
             kind: String(w.kind || ""),
-            data: (w.data || {}),
+            data: resolveWidgetImages((w.data || {}), client.opts.baseUrl),
             fallbackText: String(w.fallback_text || ""),
         });
     }), [client]);

package/dist/react-native/voice-launcher.d.ts

@@ -1,4 +1,5 @@
 import React from "react";
+import { type Caption } from "./index.js";
 export interface GraineVoiceLauncherProps {
     transport?: "ws" | "rtc";
     onTransport?: (info: {
@@ -32,5 +33,11 @@ export interface GraineVoiceApi {
     start: () => void;
     end: () => void;
     setMuted: (m: boolean) => void;
+    captions: Caption[];
+    caption: Caption | null;
+    captionsOn: boolean;
+    setCaptionsOn: (on: boolean) => void;
+    clearCaptions: () => void;
 }
+export declare const CAPTION_HISTORY = 12;
 export declare function GraineVoiceLauncher({ webView: WebView, autoStart, active, transport, onTransport, onCaption, onCallState, onMicDenied, onError, onEnded, children, }: GraineVoiceLauncherProps): React.JSX.Element;

package/dist/react-native/voice-launcher.js

@@ -3,6 +3,7 @@ import { useCallback, useEffect, useRef, useState } from "react";
 import { AppState, View } from "react-native";
 import { useGraineAgent } from "./index.js";
 import { requestMicrophonePermission } from "./permissions.js";
+export const CAPTION_HISTORY = 12;
 export function GraineVoiceLauncher({ webView: WebView, autoStart = false, active = true, transport, onTransport, onCaption, onCallState, onMicDenied, onError, onEnded, children, }) {
     const { client, appearance } = useGraineAgent();
     const ref = useRef(null);
@@ -14,6 +15,22 @@ export function GraineVoiceLauncher({ webView: WebView, autoStart = false, activ
     const [connecting, setConnecting] = useState(false);
     const [muted, setMutedState] = useState(false);
     const [ready, setReady] = useState(false);
+    const [captions, setCaptions] = useState([]);
+    const [captionsOn, setCaptionsOn] = useState(true);
+    const foldCaption = useCallback((next) => {
+        setCaptions((prev) => {
+            const tail = prev[prev.length - 1];
+            if (!next.text) {
+                if (tail && tail.live && tail.role === next.role) {
+                    return [...prev.slice(0, -1), { ...tail, live: false }];
+                }
+                return prev;
+            }
+            const continues = tail && tail.live && tail.role === next.role;
+            const merged = continues ? [...prev.slice(0, -1), next] : [...prev, next];
+            return merged.length > CAPTION_HISTORY ? merged.slice(-CAPTION_HISTORY) : merged;
+        });
+    }, []);
     const post = useCallback((msg) => {
         ref.current?.injectJavaScript(`window.postMessage(${JSON.stringify(msg)}, "*"); true;`);
     }, []);
@@ -132,9 +149,16 @@ export function GraineVoiceLauncher({ webView: WebView, autoStart = false, activ
             case "graine:muted":
                 setMutedState(!!msg.muted);
                 return;
-            case "graine:caption":
-                onCaption?.({ role: msg.role, text: msg.text, live: !!msg.live });
+            case "graine:caption": {
+                const line = {
+                    role: msg.role === "agent" ? "agent" : "customer",
+                    text: String(msg.text ?? ""),
+                    live: !!msg.live,
+                };
+                foldCaption(line);
+                onCaption?.(line);
                 return;
+            }
             case "graine:ended":
                 setConnected(false);
                 setConnecting(false);
@@ -161,15 +185,21 @@ export function GraineVoiceLauncher({ webView: WebView, autoStart = false, activ
                 return;
             }
         }
-    }, [autoStart, beginCall, client, onCaption, onCallState, onMicDenied, onTransport, onError, onEnded, post, pushContext]);
+    }, [autoStart, beginCall, client, onCaption, onCallState, onMicDenied, onTransport, onError, onEnded, post, pushContext, foldCaption]);
     const pendingStartRef = useRef(false);
     const api = {
         connected,
         connecting,
         muted,
         ready: !!voiceAgentId && ready,
+        captions,
+        caption: captions.length ? captions[captions.length - 1] : null,
+        captionsOn,
+        setCaptionsOn,
+        clearCaptions: useCallback(() => setCaptions([]), []),
         start: useCallback(() => {
             setConnecting(true);
+            setCaptions([]);
             if (!ready) {
                 pendingStartRef.current = true;
                 return;
@@ -191,6 +221,8 @@ export function GraineVoiceLauncher({ webView: WebView, autoStart = false, activ
             if (autoStart) {
                 if (startedRef.current)
                     return;
+                setConnecting(true);
+                setCaptions([]);
                 if (ready) {
                     startedRef.current = true;
                     void beginCall();
@@ -205,10 +237,13 @@ export function GraineVoiceLauncher({ webView: WebView, autoStart = false, activ
         }
         if (first)
             return;
-        if (startedRef.current || pendingStartRef.current)
-            post({ type: "graine:end-call" });
+        post({ type: "graine:end-call" });
         startedRef.current = false;
         pendingStartRef.current = false;
+        setConnected(false);
+        setConnecting(false);
+        setCaptions([]);
+        onCallState?.({ connected: false, connecting: false });
     }, [active]);
     useEffect(() => {
         let last = AppState.currentState;

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@graineai/inapp-react-native",
-  "version": "0.27.5",
+  "version": "0.30.0",
   "type": "module",
-  "description": "Graine in-app agent for React Native — an agent that sees the screen your customer is on and can act on it.",
+  "description": "Graine in-app agent for React Native \u2014 an agent that sees the screen your customer is on and can act on it.",
   "license": "MIT",
   "private": false,
   "main": "dist/index.js",