@vanira/sdk-react-native 0.0.2

package/src/presets/upload/pickUploadFile.ts

@@ -0,0 +1,256 @@
+import type {Asset} from 'react-native-image-picker';
+import {launchImageLibrary} from 'react-native-image-picker';
+import {
+  acceptRequiresDocumentPicker,
+  mimeMatchesAccept,
+  normalizeUploadMime,
+  WEB_DEFAULT_UPLOAD_ACCEPT,
+} from './uploadFormats';
+import {ensureGalleryPermission, type RNUploadFile} from './uploadUtils';
+
+type DocumentPickerModule = {
+  pick: (opts: {
+    type: unknown[];
+    allowMultiSelection?: boolean;
+  }) => Promise<
+    Array<{
+      uri: string;
+      name?: string | null;
+      type?: string | null;
+      size?: number | null;
+    }>
+  >;
+  types: {
+    images: string;
+    pdf: string;
+    plainText: string;
+    csv: string | string[];
+  };
+  isErrorWithCode?: (err: unknown, code: string) => boolean;
+  errorCodes?: {OPERATION_CANCELED?: string};
+};
+
+function loadDocumentPicker(): DocumentPickerModule | null {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const mod = require('@react-native-documents/picker');
+    const pick = mod?.pick ?? mod?.default?.pick;
+    const types = mod?.types ?? mod?.default?.types;
+    if (typeof pick === 'function' && types) {
+      return {pick, types, isErrorWithCode: mod.isErrorWithCode, errorCodes: mod.errorCodes};
+    }
+  } catch {
+    /* host app may not link document picker */
+  }
+  return null;
+}
+
+function acceptToDocumentPickerTypes(
+  accept: string,
+  docTypes: DocumentPickerModule['types'],
+): unknown[] {
+  const lower = accept.toLowerCase();
+  const result: unknown[] = [];
+
+  const add = (value: unknown) => {
+    if (Array.isArray(value)) {
+      value.forEach(v => {
+        if (!result.includes(v)) {
+          result.push(v);
+        }
+      });
+      return;
+    }
+    if (!result.includes(value)) {
+      result.push(value);
+    }
+  };
+
+  if (
+    lower.includes('image/*') ||
+    lower.includes('image/jpeg') ||
+    lower.includes('image/jpg') ||
+    lower.includes('image/png') ||
+    lower.includes('image/gif') ||
+    lower.includes('image/webp') ||
+    lower.includes('image/bmp')
+  ) {
+    add(docTypes.images);
+  }
+  if (lower.includes('pdf') || lower.includes('application/pdf')) {
+    add(docTypes.pdf);
+  }
+  if (lower.includes('text/plain')) {
+    add(docTypes.plainText);
+  }
+  if (lower.includes('text/csv') || lower.includes('csv')) {
+    add(docTypes.csv);
+  }
+
+  return result.length ? result : [docTypes.images, docTypes.pdf, docTypes.plainText, docTypes.csv];
+}
+
+function assetToUploadFile(
+  asset: Asset,
+  accept: string,
+): RNUploadFile | null {
+  if (!asset.uri) {
+    return null;
+  }
+
+  const name =
+    asset.fileName ||
+    (asset.uri.split('/').pop()?.split('?')[0] ?? `upload_${Date.now()}.jpg`);
+  const type = normalizeUploadMime(name, asset.type);
+
+  if (!mimeMatchesAccept(type, accept)) {
+    return null;
+  }
+
+  return {
+    uri: asset.uri,
+    type,
+    name,
+    size: asset.fileSize,
+  };
+}
+
+function uriToUploadFile(
+  uri: string,
+  name: string,
+  rawType: string | null | undefined,
+  size: number | null | undefined,
+  accept: string,
+): RNUploadFile | null {
+  const type = normalizeUploadMime(name, rawType);
+  if (!mimeMatchesAccept(type, accept)) {
+    return null;
+  }
+  return {uri, type, name, size: size ?? undefined};
+}
+
+async function pickFromDocumentLibrary(
+  accept: string,
+): Promise<RNUploadFile | null> {
+  const docPicker = loadDocumentPicker();
+  if (!docPicker) {
+    return null;
+  }
+
+  try {
+    const results = await docPicker.pick({
+      type: acceptToDocumentPickerTypes(accept, docPicker.types),
+      allowMultiSelection: false,
+    });
+    const file = results[0];
+    if (!file?.uri) {
+      return null;
+    }
+    const name = file.name || `upload_${Date.now()}`;
+    return uriToUploadFile(file.uri, name, file.type, file.size, accept);
+  } catch (err: unknown) {
+    const canceled =
+      docPicker.isErrorWithCode?.(
+        err,
+        docPicker.errorCodes?.OPERATION_CANCELED ?? 'OPERATION_CANCELED',
+      ) ?? false;
+    if (canceled) {
+      return null;
+    }
+    throw err;
+  }
+}
+
+async function pickFromGallery(accept: string): Promise<RNUploadFile | null> {
+  const galleryOk = await ensureGalleryPermission();
+  if (!galleryOk) {
+    throw new Error('GALLERY_PERMISSION_DENIED');
+  }
+
+  const pickerResult = await launchImageLibrary({
+    mediaType: pickerMediaTypeFromAcceptLocal(accept),
+    selectionLimit: 1,
+    includeBase64: false,
+  });
+
+  if (pickerResult.didCancel) {
+    return null;
+  }
+  if (pickerResult.errorCode) {
+    throw new Error(pickerResult.errorMessage ?? pickerResult.errorCode);
+  }
+
+  const asset = pickerResult.assets?.[0];
+  if (!asset) {
+    return null;
+  }
+
+  const file = assetToUploadFile(asset, accept);
+  if (!file) {
+    throw new Error('UNSUPPORTED_FILE_TYPE');
+  }
+  return file;
+}
+
+function pickerMediaTypeFromAcceptLocal(
+  accept: string,
+): 'photo' | 'mixed' {
+  const lower = accept.toLowerCase();
+  return lower.includes('video/') ? 'mixed' : 'photo';
+}
+
+export type PickUploadFileResult =
+  | {status: 'ok'; file: RNUploadFile}
+  | {status: 'cancelled'}
+  | {status: 'permission_denied'}
+  | {status: 'unsupported_type'}
+  | {status: 'error'; message: string};
+
+/**
+ * Pick a file for vanira_upload — mirrors web `<input accept="...">`.
+ * Uses document picker when PDF/text are allowed; gallery for image-only accept.
+ */
+export async function pickUploadFile(
+  accept: string = WEB_DEFAULT_UPLOAD_ACCEPT,
+): Promise<PickUploadFileResult> {
+  try {
+    const useDocumentPicker = acceptRequiresDocumentPicker(accept);
+
+    if (useDocumentPicker) {
+      const docFile = await pickFromDocumentLibrary(accept);
+      if (docFile) {
+        if (!mimeMatchesAccept(docFile.type, accept)) {
+          return {status: 'unsupported_type'};
+        }
+        return {status: 'ok', file: docFile};
+      }
+
+      // Document picker unavailable — fall back to gallery for images-only subset
+      if (!loadDocumentPicker()) {
+        const galleryFile = await pickFromGallery(accept);
+        if (galleryFile) {
+          return {status: 'ok', file: galleryFile};
+        }
+        return {status: 'cancelled'};
+      }
+
+      return {status: 'cancelled'};
+    }
+
+    const galleryFile = await pickFromGallery(accept);
+    if (!galleryFile) {
+      return {status: 'cancelled'};
+    }
+    return {status: 'ok', file: galleryFile};
+  } catch (err: unknown) {
+    if (err instanceof Error && err.message === 'GALLERY_PERMISSION_DENIED') {
+      return {status: 'permission_denied'};
+    }
+    if (err instanceof Error && err.message === 'UNSUPPORTED_FILE_TYPE') {
+      return {status: 'unsupported_type'};
+    }
+    const message =
+      err instanceof Error ? err.message : 'Could not open file picker.';
+    return {status: 'error', message};
+  }
+}

package/src/presets/upload/uploadFormats.ts

@@ -0,0 +1,163 @@
+/**
+ * Upload accept + MIME parity with web WidgetPresetRenderer.renderUpload().
+ * Default accept: image/*, application/pdf, text/plain, text/csv
+ */
+
+/** Same default as web vanira_upload preset. */
+export const WEB_DEFAULT_UPLOAD_ACCEPT =
+  'image/*,application/pdf,text/plain,text/csv';
+
+/** MIME types accepted by VaniraAI.uploadMedia (web + RN). */
+export const SUPPORTED_UPLOAD_MIME_TYPES = [
+  'image/jpeg',
+  'image/png',
+  'image/gif',
+  'image/webp',
+  'image/bmp',
+  'application/pdf',
+  'text/plain',
+  'text/csv',
+] as const;
+
+export type SupportedUploadMime = (typeof SUPPORTED_UPLOAD_MIME_TYPES)[number];
+
+const EXT_TO_MIME: Record<string, SupportedUploadMime | string> = {
+  jpg: 'image/jpeg',
+  jpeg: 'image/jpeg',
+  png: 'image/png',
+  gif: 'image/gif',
+  webp: 'image/webp',
+  bmp: 'image/bmp',
+  heic: 'image/jpeg',
+  heif: 'image/jpeg',
+  pdf: 'application/pdf',
+  txt: 'text/plain',
+  csv: 'text/csv',
+};
+
+const MIME_ALIASES: Record<string, SupportedUploadMime> = {
+  'image/jpg': 'image/jpeg',
+  'image/pjpeg': 'image/jpeg',
+  'text/comma-separated-values': 'text/csv',
+};
+
+/** Normalize picker / filesystem MIME (jpg → jpeg, extension fallback). */
+export function normalizeUploadMime(
+  fileName: string,
+  rawType?: string | null,
+): string {
+  const trimmed = (rawType ?? '').trim().toLowerCase();
+  if (trimmed && trimmed in MIME_ALIASES) {
+    return MIME_ALIASES[trimmed];
+  }
+  if (
+    trimmed &&
+    trimmed !== 'application/octet-stream' &&
+    !trimmed.endsWith('*')
+  ) {
+    return trimmed;
+  }
+
+  const ext = fileName.split('.').pop()?.toLowerCase();
+  if (ext && ext in EXT_TO_MIME) {
+    return EXT_TO_MIME[ext];
+  }
+
+  return trimmed || 'application/octet-stream';
+}
+
+export function isSupportedUploadMime(mime: string): boolean {
+  const normalized = normalizeUploadMime('', mime);
+  return (SUPPORTED_UPLOAD_MIME_TYPES as readonly string[]).includes(
+    normalized,
+  );
+}
+
+export function mimeMatchesAccept(mime: string, accept: string): boolean {
+  const normalized = normalizeUploadMime('', mime);
+  const tokens = accept
+    .split(',')
+    .map(t => t.trim().toLowerCase())
+    .filter(Boolean);
+
+  if (!tokens.length) {
+    return normalized.startsWith('image/');
+  }
+
+  return tokens.some(token => {
+    if (token === 'image/*') {
+      return normalized.startsWith('image/');
+    }
+    if (token.endsWith('/*')) {
+      const prefix = token.slice(0, -1);
+      return normalized.startsWith(prefix);
+    }
+    const canonical = normalizeUploadMime('', token);
+    return normalized === canonical || normalized === token;
+  });
+}
+
+/** True when accept includes PDF or text types (needs document picker on RN). */
+export function acceptRequiresDocumentPicker(accept: string): boolean {
+  const lower = accept.toLowerCase();
+  return (
+    lower.includes('pdf') ||
+    lower.includes('text/plain') ||
+    lower.includes('text/csv') ||
+    lower.includes('application/') ||
+    /\.(pdf|txt|csv)\b/.test(lower)
+  );
+}
+
+export function formatAcceptHint(accept?: string): string {
+  const value = accept?.trim() || WEB_DEFAULT_UPLOAD_ACCEPT;
+  const lower = value.toLowerCase();
+
+  if (value === WEB_DEFAULT_UPLOAD_ACCEPT || lower.includes('text/csv')) {
+    return 'JPEG, JPG, PNG, GIF, WebP, BMP, PDF, TXT, CSV · Max 50 MB';
+  }
+
+  const parts: string[] = [];
+  if (lower.includes('image')) {
+    parts.push('JPEG, JPG, PNG, GIF, WebP, BMP');
+  }
+  if (lower.includes('pdf')) {
+    parts.push('PDF');
+  }
+  if (lower.includes('text/plain') || lower.includes('.txt')) {
+    parts.push('TXT');
+  }
+  if (lower.includes('csv')) {
+    parts.push('CSV');
+  }
+
+  return parts.length
+    ? `${parts.join(', ')} · Max 50 MB`
+    : 'JPEG, JPG, PNG · Max 50 MB';
+}
+
+export function pickerMediaTypeFromAccept(
+  accept?: string,
+): 'photo' | 'mixed' {
+  const lower = (accept ?? WEB_DEFAULT_UPLOAD_ACCEPT).toLowerCase();
+  if (
+    lower.includes('video/') ||
+    (lower.includes('image/*') && !acceptRequiresDocumentPicker(accept ?? ''))
+  ) {
+    return lower.includes('video/') ? 'mixed' : 'photo';
+  }
+  return 'photo';
+}
+
+export function filePreviewIcon(mime: string): string {
+  if (mime.startsWith('image/')) {
+    return '🖼️';
+  }
+  if (mime.includes('pdf')) {
+    return '📄';
+  }
+  if (mime.startsWith('text/')) {
+    return '📝';
+  }
+  return '📁';
+}

package/src/presets/upload/uploadUtils.ts

@@ -0,0 +1,68 @@
+import {Platform} from 'react-native';
+import type {Asset} from 'react-native-image-picker';
+import {PERMISSIONS, RESULTS, request} from 'react-native-permissions';
+import {
+  formatAcceptHint,
+  mimeMatchesAccept,
+  normalizeUploadMime,
+  pickerMediaTypeFromAccept,
+  WEB_DEFAULT_UPLOAD_ACCEPT,
+} from './uploadFormats';
+
+export type RNUploadFile = {
+  uri: string;
+  type: string;
+  name: string;
+  size?: number;
+};
+
+export {
+  formatAcceptHint,
+  mimeMatchesAccept,
+  normalizeUploadMime,
+  pickerMediaTypeFromAccept,
+  WEB_DEFAULT_UPLOAD_ACCEPT,
+};
+
+export async function ensureGalleryPermission(): Promise<boolean> {
+  const permission =
+    Platform.OS === 'ios'
+      ? PERMISSIONS.IOS.PHOTO_LIBRARY
+      : Platform.OS === 'android'
+        ? Number(Platform.Version) >= 33
+          ? PERMISSIONS.ANDROID.READ_MEDIA_IMAGES
+          : PERMISSIONS.ANDROID.READ_EXTERNAL_STORAGE
+        : null;
+
+  if (!permission) {
+    return true;
+  }
+
+  const result = await request(permission);
+  return result === RESULTS.GRANTED || result === RESULTS.LIMITED;
+}
+
+export function assetToUploadFile(
+  asset: Asset,
+  accept: string = WEB_DEFAULT_UPLOAD_ACCEPT,
+): RNUploadFile | null {
+  if (!asset.uri) {
+    return null;
+  }
+
+  const name =
+    asset.fileName ||
+    (asset.uri.split('/').pop()?.split('?')[0] ?? `upload_${Date.now()}.jpg`);
+  const type = normalizeUploadMime(name, asset.type);
+
+  if (!mimeMatchesAccept(type, accept)) {
+    return null;
+  }
+
+  return {
+    uri: asset.uri,
+    type,
+    name,
+    size: asset.fileSize,
+  };
+}

package/src/react/PresetRenderer.tsx

@@ -0,0 +1,144 @@
+import React, { useEffect, useRef } from 'react';
+import { WidgetPresetRenderer } from '../ui/presets/WidgetPresetRenderer';
+
+export interface BaseClient {
+    sendToolResult: (toolCallId: string, result: any) => void;
+    sendToolError: (toolCallId: string, error: string) => void;
+    sendContextUpdate?: (data: any) => void;
+    triggerActionInterrupt?: () => void;
+    sendActionTrigger?: (actionName: string, data: Record<string, any>) => void;
+}
+
+export interface PresetRendererProps {
+    client: BaseClient | null;
+    toolCall: any | null;
+    registry?: Record<string, any>;
+    /** Fired immediately if the toolCall is NOT a preset */
+    onCustomTool?: (toolCall: any) => void;
+}
+
+export const PresetRenderer: React.FC<PresetRendererProps> = ({ 
+    client, 
+    toolCall,
+    onCustomTool 
+}) => {
+    const rendererRef = useRef<WidgetPresetRenderer | null>(null);
+
+    useEffect(() => {
+        rendererRef.current = new WidgetPresetRenderer(null as any);
+        return () => {
+            rendererRef.current?.dismiss();
+        };
+    }, []);
+
+    useEffect(() => {
+        if (!toolCall || !rendererRef.current) return;
+
+        const data = toolCall.data || toolCall;
+        
+        let args: Record<string, any> = {};
+        const rawArgs = data.arguments || data.args;
+        if (typeof rawArgs === 'string') {
+            try { args = JSON.parse(rawArgs); } catch (_) { args = {}; }
+        } else if (rawArgs && typeof rawArgs === 'object') {
+            args = rawArgs;
+        }
+
+        const clientFields = data.client_fields || {};
+        const presetId = clientFields?.preset_id || args?.preset_id;
+        const toolCallId = data.tool_call_id || data.call_id || '';
+
+        // If it's a known preset supported by vanilla WidgetPresetRenderer
+        const knownPresets = [
+            'vanira_form',
+            'vanira_calendar',
+            'vanira_navigate',
+            'vanira_upload',
+            'vanira_camera',
+            'vanira_highlight_element',
+            'vanira_type_text',
+            'vanira_erase_text'
+        ];
+
+        if (presetId && knownPresets.includes(presetId)) {
+            // Auto-acknowledge so the AI stays silent while user fills the form (skip for highlight, type_text, and erase_text)
+            const isNonInteractive = presetId === 'vanira_highlight_element' || presetId === 'vanira_type_text' || presetId === 'vanira_erase_text';
+            if (!isNonInteractive && client && client.sendContextUpdate && toolCallId) {
+                client.sendToolResult(toolCallId, {
+                    status: "popup_shown",
+                    message: `The UI preset ${presetId} is now visible. I am waiting for the user to complete it. STAY SILENT.`
+                });
+                client.sendContextUpdate({
+                    ui_state: "waiting_for_preset_data",
+                    tool_status: "active_waiting",
+                    visible_preset: presetId
+                });
+            }
+
+            // Handle with vanilla renderer
+            rendererRef.current.handle(
+                toolCall,
+                (result) => {
+                    console.log(`🎯 [PresetRenderer Wrapper] Preset ${presetId} completed! Result:`, result);
+                    if (client) {
+                        const payload = typeof result === 'string' ? { response: result } : result;
+
+                        // Skip client action trigger if client_media_update already happened (payload has media_id)
+                        if (payload && (payload.media_id || payload.url)) {
+                            console.log('[PresetRenderer Wrapper] Skipping sendActionTrigger because media was already updated.');
+                            return;
+                        }
+
+                        // If it's a typing or erase preset, don't send back anything to the AI
+                        if (presetId === 'vanira_type_text' || presetId === 'vanira_erase_text') {
+                            console.log(`[PresetRenderer Wrapper] Skipping sendActionTrigger/sendToolResult for preset: ${presetId}`);
+                            return;
+                        }
+
+                        // 1. Wake up the AI FIRST to clear its buffers
+                        if (client.triggerActionInterrupt) {
+                            console.log(`⚡ [PresetRenderer Wrapper] Triggering action interrupt to wake AI...`);
+                            client.triggerActionInterrupt();
+                        }
+
+                        // 2. Send the action trigger with dynamic naming based on the tool
+                        if (client.sendActionTrigger) {
+                            const actionName = clientFields.action_name || (presetId === 'vanira_calendar' ? 'calendar_slot_selected' : `user_submitted_${data.name}`);
+                            console.log(`📤 [PresetRenderer Wrapper] Submitting preset data via sendActionTrigger (${actionName})...`);
+                            client.sendActionTrigger(actionName, payload);
+                        } else {
+                            // Fallback for older clients
+                            console.log(`📤 [PresetRenderer Wrapper] Submitting tool result for ID: ${toolCallId}`);
+                            client.sendToolResult(toolCallId, payload);
+                        }
+                    }
+                },
+                (reason) => {
+                    console.log(`❌ [PresetRenderer Wrapper] Preset ${presetId} cancelled:`, reason);
+                    if (client) {
+                        // If it's a typing or erase preset, don't send back anything to the AI
+                        if (presetId === 'vanira_type_text' || presetId === 'vanira_erase_text') {
+                            console.log(`[PresetRenderer Wrapper] Skipping sendToolError/sendToolResult onCancel for preset: ${presetId}`);
+                            return;
+                        }
+
+                        if (typeof client.sendToolError === 'function') {
+                            client.sendToolError(toolCallId, reason || 'User cancelled the action');
+                        } else {
+                            // Fallback: send a tool result with cancelled status so the AI isn't left hanging
+                            try {
+                                client.sendToolResult(toolCallId, { status: 'cancelled', reason: reason || 'User dismissed' });
+                            } catch (_) { /* silent */ }
+                        }
+                    }
+                },
+                client
+            );
+        } else if (onCustomTool) {
+            // Not a preset. Hand execution back to the developer!
+            onCustomTool(toolCall);
+        }
+    }, [toolCall, client, onCustomTool]);
+
+    return null;
+};

package/src/react/index.ts

@@ -0,0 +1 @@
+export * from './PresetRenderer';

package/src/runtime/browserRuntime.ts

@@ -0,0 +1,54 @@
+/**
+ * Browser Runtime
+ *
+ * Pre-wires all browser adapters into a VaniraRuntime.
+ * Import this module only in browser/web environments — it references
+ * browser globals (HTMLAudioElement, navigator.mediaDevices, RTCPeerConnection).
+ *
+ * Usage:
+ *   import { browserRuntime, createBrowserClient } from 'vanira-sdk/runtime/browser';
+ *
+ *   const client = createBrowserClient({ agentId: '...', apiKey: '...' });
+ */
+
+import type { WebRTCClientConfig } from '../types';
+
+import { WebRTCClient } from '../core/WebRTCClient';
+import { VaniraAI } from '../core/VaniraAI';
+
+import {
+    browserDefaultBundle,
+    browserCapabilities,
+} from './runtimeBundles';
+
+// Re-export bundle + capabilities (single source of truth in runtimeBundles.ts)
+export { browserCapabilities };
+export const browserRuntime = browserDefaultBundle;
+
+// ─── Factory functions ────────────────────────────────────────────────────────
+
+/**
+ * Create a WebRTCClient pre-wired for the browser runtime.
+ * Adapter fields (audioAdapter, mediaAdapter, peerAdapter, dataChannelAdapter)
+ * are set automatically — do not set them manually.
+ */
+export function createBrowserClient(
+    config: Omit<WebRTCClientConfig, 'audioAdapter' | 'mediaAdapter' | 'peerAdapter' | 'dataChannelAdapter'>
+): WebRTCClient {
+    return new WebRTCClient({
+        ...config,
+        runtime: browserRuntime,
+    });
+}
+
+/**
+ * Create a VaniraAI instance pre-wired for the browser runtime.
+ */
+export function createBrowserAI(
+    config: Omit<WebRTCClientConfig, 'audioAdapter' | 'mediaAdapter' | 'peerAdapter' | 'dataChannelAdapter'>
+): VaniraAI {
+    return new VaniraAI({
+        ...config,
+        runtime: browserRuntime,
+    });
+}