@vanira/sdk-react-native 0.0.2

package/src/presets/liveVision/LiveVisionCameraHost.tsx

@@ -0,0 +1,317 @@
+import React, {useEffect, useRef, useState} from 'react';
+import {Platform, Pressable, StyleSheet, Text, View} from 'react-native';
+import {
+  Camera,
+  useCameraDevice,
+  useCameraPermission,
+  type Camera as CameraRef,
+} from 'react-native-vision-camera';
+import type {PresetClient} from '../types';
+import {captureAndSendLiveVisionFrame} from './liveVisionFrameLoop';
+import {
+  getLiveVisionSession,
+  parseLiveVisionArgs,
+  stopLiveVision,
+  subscribeLiveVision,
+  type LiveVisionSession,
+} from './liveVisionSession';
+
+type Props = {
+  client: PresetClient | null;
+};
+
+function toFileUri(path: string): string {
+  if (path.startsWith('file://')) {
+    return path;
+  }
+  return Platform.OS === 'android' ? `file://${path}` : path;
+}
+
+async function captureCameraFrame(
+  camera: CameraRef,
+): Promise<string | null> {
+  if (Platform.OS === 'android' && typeof camera.takeSnapshot === 'function') {
+    const snapshot = await camera.takeSnapshot({quality: 75});
+    return snapshot?.path ? toFileUri(snapshot.path) : null;
+  }
+
+  const photo = await camera.takePhoto({
+    flash: 'off',
+    enableShutterSound: false,
+  });
+  return photo?.path ? toFileUri(photo.path) : null;
+}
+
+/**
+ * Live vision camera host — web parity via a dedicated camera pipeline
+ * (hidden video + canvas on web; Vision Camera takePhoto/snapshot on RN).
+ *
+ * Does NOT use RTCView + ViewShot (black frames on GL surfaces).
+ */
+export function LiveVisionCameraHost({client}: Props) {
+  const [session, setSession] = useState<LiveVisionSession | null>(
+    getLiveVisionSession(),
+  );
+  const [cameraReady, setCameraReady] = useState(false);
+  const cameraRef = useRef<CameraRef>(null);
+  const frameSeqRef = useRef(0);
+  const captureTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const autoStopTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+  const runningSessionIdRef = useRef<string | null>(null);
+  const uploadingRef = useRef(false);
+  const startedSessionRef = useRef<string | null>(null);
+
+  const {hasPermission, requestPermission} = useCameraPermission();
+
+  useEffect(
+    () =>
+      subscribeLiveVision(() => {
+        setSession(getLiveVisionSession());
+      }),
+    [],
+  );
+
+  useEffect(() => {
+    if (!client) {
+      stopLiveVision(null);
+    }
+  }, [client]);
+
+  useEffect(() => {
+    if (!session) {
+      startedSessionRef.current = null;
+      runningSessionIdRef.current = null;
+      frameSeqRef.current = 0;
+      setCameraReady(false);
+      return;
+    }
+
+    if (!hasPermission) {
+      requestPermission().then(granted => {
+        if (!granted) {
+          console.warn('[LiveVision] Camera permission denied');
+          stopLiveVision(session.client);
+        }
+      });
+    }
+  }, [session, hasPermission, requestPermission]);
+
+  const parsed = session ? parseLiveVisionArgs(session.args) : null;
+  const device = useCameraDevice(
+    parsed?.facingMode === 'user' ? 'front' : 'back',
+  );
+
+  useEffect(() => {
+    if (!session || !client || !device || !hasPermission || !cameraReady) {
+      return;
+    }
+
+    const activeSessionId = session.sessionId;
+    if (startedSessionRef.current === activeSessionId) {
+      return;
+    }
+
+    const {
+      scope,
+      targetFps,
+      reason,
+      durationSec,
+      showPreview,
+    } = parseLiveVisionArgs(session.args);
+    const activeClient = session.client;
+
+    startedSessionRef.current = activeSessionId;
+    runningSessionIdRef.current = activeSessionId;
+    frameSeqRef.current = 0;
+
+    activeClient.sendEvent?.('client_live_camera_started', {
+      data: {
+        session_id: activeSessionId,
+        scope,
+        ...(scope === 'timed' && durationSec > 0
+          ? {duration_sec: durationSec}
+          : {}),
+        reason,
+        target_fps: targetFps,
+      },
+    });
+    console.log(
+      `[LiveVision] Session started: ${activeSessionId} (${scope}, ${targetFps} fps, preview=${showPreview})`,
+    );
+
+    const intervalMs = Math.max(250, Math.round(1000 / targetFps));
+    const warmup = setTimeout(() => {
+      captureTimerRef.current = setInterval(async () => {
+        const camera = cameraRef.current;
+        if (
+          uploadingRef.current ||
+          !camera ||
+          runningSessionIdRef.current !== activeSessionId
+        ) {
+          return;
+        }
+
+        uploadingRef.current = true;
+        try {
+          const frameUri = await captureCameraFrame(camera);
+          if (!frameUri || runningSessionIdRef.current !== activeSessionId) {
+            return;
+          }
+
+          frameSeqRef.current = await captureAndSendLiveVisionFrame(
+            activeClient,
+            activeSessionId,
+            frameUri,
+            reason,
+            frameSeqRef.current,
+            () => getLiveVisionSession()?.sessionId === activeSessionId,
+          );
+        } catch (err) {
+          console.warn('[LiveVision] Frame capture failed:', err);
+        } finally {
+          uploadingRef.current = false;
+        }
+      }, intervalMs);
+    }, 800);
+
+    if (scope === 'timed' && durationSec > 0) {
+      autoStopTimerRef.current = setTimeout(() => {
+        console.log(`[LiveVision] Timed session expired (${durationSec}s)`);
+        stopLiveVision(activeClient);
+      }, durationSec * 1000);
+    }
+
+    return () => {
+      clearTimeout(warmup);
+      if (captureTimerRef.current) {
+        clearInterval(captureTimerRef.current);
+        captureTimerRef.current = null;
+      }
+      if (autoStopTimerRef.current) {
+        clearTimeout(autoStopTimerRef.current);
+        autoStopTimerRef.current = null;
+      }
+      if (runningSessionIdRef.current === activeSessionId) {
+        runningSessionIdRef.current = null;
+      }
+      if (startedSessionRef.current === activeSessionId) {
+        startedSessionRef.current = null;
+      }
+    };
+  }, [session, client, device, hasPermission, cameraReady]);
+
+  if (!session || !device || !hasPermission) {
+    return null;
+  }
+
+  const showPreview = parsed?.showPreview ?? true;
+  const mirror = parsed?.facingMode === 'user';
+
+  if (!showPreview) {
+    return (
+      <View style={styles.hiddenHost}>
+        <Camera
+          ref={cameraRef}
+          style={styles.offscreenCamera}
+          device={device}
+          isActive={true}
+          photo={true}
+          video={Platform.OS === 'android'}
+          isMirrored={mirror}
+          onInitialized={() => setCameraReady(true)}
+        />
+      </View>
+    );
+  }
+
+  return (
+    <View style={styles.pipHost} pointerEvents="box-none">
+      <View style={styles.pipCard}>
+        <Camera
+          ref={cameraRef}
+          style={styles.pipCamera}
+          device={device}
+          isActive={true}
+          photo={true}
+          video={Platform.OS === 'android'}
+          isMirrored={mirror}
+          onInitialized={() => setCameraReady(true)}
+        />
+        <View style={styles.pipBadge}>
+          <Text style={styles.pipBadgeText}>LIVE</Text>
+        </View>
+        <Pressable
+          style={styles.pipStop}
+          onPress={() => stopLiveVision(client)}
+          hitSlop={8}>
+          <Text style={styles.pipStopText}>Stop</Text>
+        </Pressable>
+      </View>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  hiddenHost: {
+    position: 'absolute',
+    width: 640,
+    height: 480,
+    left: -2000,
+    top: 0,
+    opacity: 0.01,
+    overflow: 'hidden',
+  },
+  offscreenCamera: {
+    width: 640,
+    height: 480,
+  },
+  pipHost: {
+    ...StyleSheet.absoluteFillObject,
+    zIndex: 1000001,
+    elevation: 1000001,
+    justifyContent: 'flex-end',
+    alignItems: 'flex-end',
+    padding: 20,
+  },
+  pipCard: {
+    width: 180,
+    height: 135,
+    borderRadius: 16,
+    overflow: 'hidden',
+    borderWidth: 2,
+    borderColor: '#f87171',
+    backgroundColor: '#000',
+  },
+  pipCamera: {
+    flex: 1,
+  },
+  pipBadge: {
+    position: 'absolute',
+    top: 8,
+    left: 8,
+    backgroundColor: '#dc2626',
+    borderRadius: 6,
+    paddingHorizontal: 6,
+    paddingVertical: 2,
+  },
+  pipBadgeText: {
+    color: '#fff',
+    fontSize: 10,
+    fontWeight: '800',
+    letterSpacing: 0.8,
+  },
+  pipStop: {
+    position: 'absolute',
+    bottom: 8,
+    right: 8,
+    backgroundColor: 'rgba(15, 23, 42, 0.85)',
+    borderRadius: 8,
+    paddingHorizontal: 10,
+    paddingVertical: 4,
+  },
+  pipStopText: {
+    color: '#f8fafc',
+    fontSize: 11,
+    fontWeight: '700',
+  },
+});

package/src/presets/liveVision/LiveVisionHandler.tsx

@@ -0,0 +1,26 @@
+import {useEffect} from 'react';
+import type {PresetClient, PresetContext} from '../types';
+import {beginLiveVision} from './liveVisionSession';
+
+type Props = {
+  client: PresetClient | null;
+  ctx: PresetContext;
+  /** Must NOT stop the live-vision session — only clears active tool_call UI. */
+  onReleaseToolCall?: () => void;
+};
+
+/** Headless open handler — camera runs in LiveVisionPipOverlay until close/disconnect. */
+export function LiveVisionHandler({client, ctx, onReleaseToolCall}: Props) {
+  useEffect(() => {
+    if (!client) {
+      return;
+    }
+
+    const merged = {...ctx.args, ...ctx.clientFields};
+    beginLiveVision(client, merged);
+    onReleaseToolCall?.();
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [client, ctx.toolCallId]);
+
+  return null;
+}

package/src/presets/liveVision/LiveVisionPipOverlay.tsx

@@ -0,0 +1,7 @@
+import type {PresetClient} from '../types';
+import {LiveVisionCameraHost} from './LiveVisionCameraHost';
+
+/** Root overlay for active live-vision sessions (PiP or offscreen capture). */
+export function LiveVisionPipOverlay({client}: {client: PresetClient | null}) {
+  return <LiveVisionCameraHost client={client} />;
+}

package/src/presets/liveVision/liveVisionFrameLoop.ts

@@ -0,0 +1,38 @@
+import type {PresetClient} from '../types';
+import {
+  sendLiveVisionFrameEvent,
+  uploadLiveVisionFrame,
+} from './liveVisionUpload';
+
+/**
+ * Mirrors web WidgetPresetRenderer.captureAndSendFrame:
+ * raw /media/upload then client_media_frame only (no client_media_update).
+ */
+export async function captureAndSendLiveVisionFrame(
+  client: PresetClient,
+  sessionId: string,
+  frameUri: string,
+  reason: string,
+  frameSeq: number,
+  isSessionActive: () => boolean,
+): Promise<number> {
+  if (!isSessionActive()) {
+    return frameSeq;
+  }
+
+  const {media_id, url} = await uploadLiveVisionFrame(
+    client,
+    sessionId,
+    frameUri,
+    reason,
+  );
+
+  if (!isSessionActive()) {
+    return frameSeq;
+  }
+
+  const nextSeq = frameSeq + 1;
+  sendLiveVisionFrameEvent(client, sessionId, nextSeq, media_id, url, reason);
+  console.log(`[LiveVision] Frame #${nextSeq} sent (media_id: ${media_id})`);
+  return nextSeq;
+}

package/src/presets/liveVision/liveVisionSession.ts

@@ -0,0 +1,75 @@
+import type {PresetClient} from '../types';
+
+export type LiveVisionSession = {
+  sessionId: string;
+  client: PresetClient;
+  args: Record<string, unknown>;
+};
+
+let activeSession: LiveVisionSession | null = null;
+const listeners = new Set<() => void>();
+
+function notify(): void {
+  listeners.forEach(listener => listener());
+}
+
+export function subscribeLiveVision(listener: () => void): () => void {
+  listeners.add(listener);
+  return () => listeners.delete(listener);
+}
+
+export function getLiveVisionSession(): LiveVisionSession | null {
+  return activeSession;
+}
+
+export function beginLiveVision(
+  client: PresetClient,
+  args: Record<string, unknown>,
+): string {
+  stopLiveVision(client);
+
+  const sessionId = `lv_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
+  activeSession = {sessionId, client, args};
+  notify();
+  return sessionId;
+}
+
+export function stopLiveVision(client: PresetClient | null): void {
+  if (!activeSession) {
+    return;
+  }
+
+  const sessionId = activeSession.sessionId;
+  const notifyClient = client ?? activeSession.client;
+  activeSession = null;
+  notify();
+
+  if (notifyClient?.sendEvent) {
+    notifyClient.sendEvent('client_live_camera_stopped', {
+      data: {session_id: sessionId},
+    });
+  }
+  console.log(`[LiveVision] Session stopped: ${sessionId}`);
+}
+
+export function parseLiveVisionArgs(args: Record<string, unknown>) {
+  const scope = String(args.scope ?? 'until_call_end');
+  const maxFps = Number(args.max_fps ?? 3);
+  const targetFps = Math.min(Number(args.target_fps ?? 1), maxFps);
+  const maxWidth = Number(args.max_width ?? 640);
+  const reason = String(args.reason ?? 'camera_capture');
+  const durationSec = Number(args.duration_sec ?? 0);
+  const facingMode = String(args.facing_mode ?? 'environment');
+  const showPreview =
+    args.show_preview !== false && args.show_preview !== 'false';
+
+  return {
+    scope,
+    targetFps,
+    maxWidth,
+    reason,
+    durationSec,
+    facingMode,
+    showPreview,
+  };
+}

package/src/presets/liveVision/liveVisionUpload.ts

@@ -0,0 +1,62 @@
+import type {PresetClient} from '../types';
+
+export function resolveUploadBase(serverUrl: string): string {
+  if (!serverUrl) {
+    return '';
+  }
+  try {
+    return new URL(serverUrl).origin;
+  } catch {
+    return serverUrl.replace(/\/webrtc.*$/, '').replace(/\/$/, '');
+  }
+}
+
+export async function uploadLiveVisionFrame(
+  client: PresetClient,
+  sessionId: string,
+  frameUri: string,
+  reason: string,
+): Promise<{media_id: string; url: string}> {
+  const uploadBase = resolveUploadBase(client.serverUrl ?? '');
+  const callId = client.callId ?? '';
+  const uploadUrl = `${uploadBase}/media/upload`;
+
+  const form = new FormData();
+  form.append('file', {
+    uri: frameUri,
+    type: 'image/jpeg',
+    name: `frame_${sessionId}.jpg`,
+  } as unknown as Blob);
+  form.append('call_id', callId);
+  form.append('reason', reason);
+
+  const res = await fetch(uploadUrl, {method: 'POST', body: form});
+  if (!res.ok) {
+    throw new Error(`Upload HTTP ${res.status}`);
+  }
+  const data = (await res.json()) as {media_id?: string; url?: string};
+  if (!data.media_id || !data.url) {
+    throw new Error('Upload response missing media_id/url');
+  }
+  return {media_id: data.media_id, url: data.url};
+}
+
+export function sendLiveVisionFrameEvent(
+  client: PresetClient,
+  sessionId: string,
+  frameSeq: number,
+  media_id: string,
+  media_url: string,
+  reason: string,
+): void {
+  client.sendEvent?.('client_media_frame', {
+    data: {
+      session_id: sessionId,
+      frame_seq: frameSeq,
+      media_id,
+      media_url,
+      content_type: 'image/jpeg',
+      reason,
+    },
+  });
+}

package/src/presets/navigation/internalRouteRegistry.ts

@@ -0,0 +1,25 @@
+import type {InternalRouteHandler} from './navigationTypes';
+
+let internalRouteHandler: InternalRouteHandler | null = null;
+
+/**
+ * Register SPA / in-app router handler — mirrors web `vanira:navigate` listener
+ * that sets `detail.handled = true`.
+ *
+ * Kept free of react-native imports so consumer apps can import this symbol
+ * without pulling RN types through the SDK package graph.
+ */
+export function registerInternalRouteHandler(
+  handler: InternalRouteHandler | null,
+): () => void {
+  internalRouteHandler = handler;
+  return () => {
+    if (internalRouteHandler === handler) {
+      internalRouteHandler = null;
+    }
+  };
+}
+
+export function getInternalRouteHandler(): InternalRouteHandler | null {
+  return internalRouteHandler;
+}

package/src/presets/navigation/navigationBridge.ts

@@ -0,0 +1,76 @@
+import {Linking} from 'react-native';
+import {getInternalRouteHandler} from './internalRouteRegistry';
+import type {NavigationEvent} from './navigationTypes';
+
+export type {NavigationEvent, InternalRouteHandler} from './navigationTypes';
+export {registerInternalRouteHandler} from './internalRouteRegistry';
+
+let lastNavigation: NavigationEvent | null = null;
+const listeners = new Set<(event: NavigationEvent) => void>();
+
+function notify(event: NavigationEvent): void {
+  lastNavigation = event;
+  listeners.forEach(listener => listener(event));
+}
+
+export function subscribeNavigation(
+  listener: (event: NavigationEvent) => void,
+): () => void {
+  listeners.add(listener);
+  if (lastNavigation) {
+    listener(lastNavigation);
+  }
+  return () => listeners.delete(listener);
+}
+
+export function getLastNavigation(): NavigationEvent | null {
+  return lastNavigation;
+}
+
+function isExternalUrl(url: string): boolean {
+  return /^https?:\/\//i.test(url);
+}
+
+/**
+ * RN equivalent of web WidgetPresetRenderer.renderNavigate:
+ * - external → Linking.openURL
+ * - internal → internalRouteHandler must return true or navigation fails
+ */
+export async function navigateToTarget(
+  targetUrl: string,
+): Promise<NavigationEvent> {
+  const url = String(targetUrl || '/').trim() || '/';
+  const external = isExternalUrl(url);
+
+  if (external) {
+    const canOpen = await Linking.canOpenURL(url);
+    if (canOpen) {
+      await Linking.openURL(url);
+    }
+    const event: NavigationEvent = {
+      url,
+      external: true,
+      opened: canOpen,
+      handled: canOpen,
+      at: Date.now(),
+    };
+    notify(event);
+    return event;
+  }
+
+  const internalRouteHandler = getInternalRouteHandler();
+  let handled = false;
+  if (internalRouteHandler) {
+    handled = Boolean(await Promise.resolve(internalRouteHandler(url)));
+  }
+
+  const event: NavigationEvent = {
+    url,
+    external: false,
+    opened: handled,
+    handled,
+    at: Date.now(),
+  };
+  notify(event);
+  return event;
+}

package/src/presets/navigation/navigationTypes.ts

@@ -0,0 +1,12 @@
+export type NavigationEvent = {
+  url: string;
+  external: boolean;
+  opened: boolean;
+  /** Internal routes require an app handler (web: vanira:navigate detail.handled). */
+  handled: boolean;
+  at: number;
+};
+
+export type InternalRouteHandler = (
+  path: string,
+) => boolean | Promise<boolean>;

package/src/presets/parseToolCall.ts

@@ -0,0 +1,60 @@
+import type {ParsedToolCall, PresetId} from './types';
+import {TOOL_NAME_TO_PRESET_ID, isVaniraPresetId} from './registry';
+
+/** Same normalization as PresetRenderer.tsx + WidgetPresetRenderer.handle(). */
+export function parseToolCall(toolCall: unknown): ParsedToolCall | null {
+  if (!toolCall || typeof toolCall !== 'object') {
+    return null;
+  }
+  let data =
+    (toolCall as {data?: Record<string, unknown>}).data ??
+    (toolCall as Record<string, unknown>);
+
+  // Unwrap nested tool_call envelopes from some server payloads.
+  const nested = data.tool_call;
+  if (nested && typeof nested === 'object') {
+    data = nested as Record<string, unknown>;
+  }
+
+  let args: Record<string, unknown> = {};
+  const rawArgs = data.arguments ?? data.args;
+  if (typeof rawArgs === 'string') {
+    try {
+      args = JSON.parse(rawArgs) as Record<string, unknown>;
+    } catch {
+      args = {};
+    }
+  } else if (rawArgs && typeof rawArgs === 'object') {
+    args = rawArgs as Record<string, unknown>;
+  }
+
+  const clientFields = (data.client_fields ??
+    data.clientFields ??
+    {}) as Record<string, unknown>;
+  const fn = data.function;
+  const functionName =
+    fn && typeof fn === 'object'
+      ? String((fn as Record<string, unknown>).name ?? '')
+      : '';
+  const name = String(
+    data.name ?? data.tool_name ?? data.ref_code ?? functionName ?? '',
+  );
+  const presetIdRaw =
+    clientFields.preset_id ?? args.preset_id ?? TOOL_NAME_TO_PRESET_ID[name];
+  const presetId =
+    typeof presetIdRaw === 'string' && isVaniraPresetId(presetIdRaw)
+      ? (presetIdRaw as PresetId)
+      : undefined;
+  const toolCallId = String(
+    data.tool_call_id ?? data.call_id ?? '',
+  );
+
+  return {
+    raw: toolCall,
+    name,
+    toolCallId,
+    arguments: args,
+    clientFields,
+    presetId,
+  };
+}