@vanira/sdk-react-native 0.0.2

package/src/presets/liveScreen/LiveScreenCaptureHost.tsx

@@ -0,0 +1,312 @@
+import React, {useEffect, useRef, useState} from 'react';
+import {Pressable, StyleSheet, Text, View} from 'react-native';
+import {RTCView, mediaDevices, type MediaStream} from 'react-native-webrtc';
+import {captureRef} from 'react-native-view-shot';
+import type {PresetClient} from '../types';
+import {captureAndSendStreamingFrame} from '../streaming/mediaFrameUpload';
+import {
+  getLiveScreenSession,
+  parseLiveScreenArgs,
+  stopLiveScreen,
+  subscribeLiveScreen,
+  type LiveScreenSession,
+} from './liveScreenSession';
+
+type Props = {
+  client: PresetClient | null;
+};
+
+/**
+ * Screen share host via react-native-webrtc getDisplayMedia.
+ * Frame capture uses view-shot on the RTCView preview (best-effort on mobile).
+ */
+export function LiveScreenCaptureHost({client}: Props) {
+  const [session, setSession] = useState<LiveScreenSession | null>(
+    getLiveScreenSession(),
+  );
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const [streamError, setStreamError] = useState<string | null>(null);
+  const previewRef = useRef<View>(null);
+  const frameSeqRef = useRef(0);
+  const captureTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const autoStopTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+  const runningSessionIdRef = useRef<string | null>(null);
+  const uploadingRef = useRef(false);
+  const startedSessionRef = useRef<string | null>(null);
+
+  useEffect(
+    () =>
+      subscribeLiveScreen(() => {
+        setSession(getLiveScreenSession());
+      }),
+    [],
+  );
+
+  useEffect(() => {
+    if (!client) {
+      stopLiveScreen(null);
+    }
+  }, [client]);
+
+  useEffect(() => {
+    if (!session || !client) {
+      if (stream) {
+        stream.getTracks().forEach(track => track.stop());
+        setStream(null);
+      }
+      runningSessionIdRef.current = null;
+      startedSessionRef.current = null;
+      if (captureTimerRef.current) {
+        clearInterval(captureTimerRef.current);
+        captureTimerRef.current = null;
+      }
+      if (autoStopTimerRef.current) {
+        clearTimeout(autoStopTimerRef.current);
+        autoStopTimerRef.current = null;
+      }
+      return;
+    }
+
+    if (runningSessionIdRef.current === session.sessionId) {
+      return;
+    }
+    runningSessionIdRef.current = session.sessionId;
+    setStreamError(null);
+
+    const parsed = parseLiveScreenArgs(session.args);
+    let activeStream: MediaStream | null = null;
+    let cancelled = false;
+
+    (async () => {
+      try {
+        activeStream = await mediaDevices.getDisplayMedia();
+        if (cancelled || runningSessionIdRef.current !== session.sessionId) {
+          activeStream.getTracks().forEach(track => track.stop());
+          return;
+        }
+
+        setStream(activeStream);
+        const track = activeStream.getVideoTracks()[0];
+        if (track) {
+          track.onended = () => {
+            console.log('[LiveScreen] User stopped sharing via system UI');
+            stopLiveScreen(client);
+          };
+        }
+
+        if (startedSessionRef.current !== session.sessionId) {
+          startedSessionRef.current = session.sessionId;
+          client.sendEvent?.('client_live_screen_started', {
+            data: {
+              session_id: session.sessionId,
+              scope: parsed.scope,
+              ...(parsed.scope === 'timed' && parsed.durationSec > 0
+                ? {duration_sec: parsed.durationSec}
+                : {}),
+              display_surface: 'mobile',
+              reason: parsed.reason,
+              target_fps: parsed.targetFps,
+            },
+          });
+          console.log(
+            `[LiveScreen] Session started: ${session.sessionId} (${parsed.targetFps} fps)`,
+          );
+        }
+
+        const intervalMs = Math.round(1000 / parsed.targetFps);
+        captureTimerRef.current = setInterval(async () => {
+          if (
+            uploadingRef.current ||
+            !previewRef.current ||
+            runningSessionIdRef.current !== session.sessionId
+          ) {
+            return;
+          }
+          uploadingRef.current = true;
+          try {
+            const uri = await captureRef(previewRef, {
+              format: 'jpg',
+              quality: 0.75,
+              result: 'tmpfile',
+            });
+            if (!uri) {
+              return;
+            }
+            frameSeqRef.current = await captureAndSendStreamingFrame(
+              client,
+              session.sessionId,
+              uri,
+              parsed.reason,
+              frameSeqRef.current,
+              () => runningSessionIdRef.current === session.sessionId,
+              {source: 'screen'},
+            );
+          } catch (err) {
+            console.warn('[LiveScreen] Frame capture failed:', err);
+          } finally {
+            uploadingRef.current = false;
+          }
+        }, intervalMs);
+
+        if (parsed.scope === 'timed' && parsed.durationSec > 0) {
+          autoStopTimerRef.current = setTimeout(() => {
+            console.log(
+              `[LiveScreen] Timed session expired (${parsed.durationSec}s)`,
+            );
+            stopLiveScreen(client);
+          }, parsed.durationSec * 1000);
+        }
+      } catch (err: unknown) {
+        const message =
+          err instanceof Error ? err.message : 'Screen share permission denied';
+        console.warn('[LiveScreen] getDisplayMedia failed:', message);
+        setStreamError(message);
+        client.sendEvent?.('client_live_screen_denied', {
+          data: {session_id: session.sessionId, reason: message},
+        });
+        stopLiveScreen(client);
+      }
+    })();
+
+    return () => {
+      cancelled = true;
+      if (captureTimerRef.current) {
+        clearInterval(captureTimerRef.current);
+        captureTimerRef.current = null;
+      }
+      if (autoStopTimerRef.current) {
+        clearTimeout(autoStopTimerRef.current);
+        autoStopTimerRef.current = null;
+      }
+      if (activeStream) {
+        activeStream.getTracks().forEach(track => track.stop());
+      }
+    };
+  }, [session, client]);
+
+  if (!session) {
+    return null;
+  }
+
+  const parsed = parseLiveScreenArgs(session.args);
+  const showPreview = parsed.showPreview;
+
+  if (streamError) {
+    return null;
+  }
+
+  if (!showPreview && stream) {
+    return (
+      <View
+        ref={previewRef}
+        collapsable={false}
+        style={styles.offscreen}
+        pointerEvents="none">
+        <RTCView
+          streamURL={stream.toURL()}
+          style={styles.offscreenRtc}
+          objectFit="contain"
+        />
+      </View>
+    );
+  }
+
+  if (!showPreview) {
+    return null;
+  }
+
+  return (
+    <View style={styles.pipContainer} pointerEvents="box-none">
+      <View ref={previewRef} collapsable={false} style={styles.pipFrame}>
+        {stream ? (
+          <RTCView
+            streamURL={stream.toURL()}
+            style={styles.rtc}
+            objectFit="contain"
+          />
+        ) : (
+          <View style={styles.loading}>
+            <Text style={styles.loadingText}>Starting screen share…</Text>
+          </View>
+        )}
+        <View style={styles.badge}>
+          <View style={styles.dot} />
+          <Text style={styles.badgeText}>SCREEN</Text>
+        </View>
+        <Pressable
+          style={styles.stopBtn}
+          onPress={() => stopLiveScreen(client)}
+          hitSlop={8}>
+          <Text style={styles.stopText}>✕</Text>
+        </Pressable>
+      </View>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  pipContainer: {
+    position: 'absolute',
+    bottom: 24,
+    right: 16,
+    zIndex: 999998,
+    elevation: 999998,
+  },
+  pipFrame: {
+    width: 220,
+    height: 124,
+    borderRadius: 16,
+    overflow: 'hidden',
+    backgroundColor: '#0f172a',
+    borderWidth: 1,
+    borderColor: 'rgba(96,165,250,0.25)',
+  },
+  rtc: {flex: 1, backgroundColor: '#020617'},
+  offscreen: {
+    position: 'absolute',
+    width: 320,
+    height: 180,
+    opacity: 0.01,
+    left: -400,
+    top: 0,
+  },
+  offscreenRtc: {flex: 1},
+  loading: {flex: 1, alignItems: 'center', justifyContent: 'center'},
+  loadingText: {color: '#94a3b8', fontSize: 11},
+  badge: {
+    position: 'absolute',
+    top: 10,
+    left: 10,
+    flexDirection: 'row',
+    alignItems: 'center',
+    gap: 6,
+    paddingHorizontal: 8,
+    paddingVertical: 3,
+    borderRadius: 6,
+    backgroundColor: 'rgba(15,23,42,0.8)',
+  },
+  dot: {
+    width: 6,
+    height: 6,
+    borderRadius: 3,
+    backgroundColor: '#3b82f6',
+  },
+  badgeText: {
+    color: '#f8fafc',
+    fontSize: 10,
+    fontWeight: '700',
+    letterSpacing: 0.5,
+  },
+  stopBtn: {
+    position: 'absolute',
+    top: 10,
+    right: 10,
+    width: 22,
+    height: 22,
+    borderRadius: 11,
+    backgroundColor: 'rgba(15,23,42,0.8)',
+    alignItems: 'center',
+    justifyContent: 'center',
+  },
+  stopText: {color: '#fff', fontSize: 10, fontWeight: '700'},
+});

package/src/presets/liveScreen/LiveScreenHandler.tsx

@@ -0,0 +1,25 @@
+import {useEffect} from 'react';
+import type {PresetClient, PresetContext} from '../types';
+import {beginLiveScreen} from './liveScreenSession';
+
+type Props = {
+  client: PresetClient | null;
+  ctx: PresetContext;
+  onReleaseToolCall?: () => void;
+};
+
+/** Headless open handler — screen share runs in LiveScreenPipOverlay until close/disconnect. */
+export function LiveScreenHandler({client, ctx, onReleaseToolCall}: Props) {
+  useEffect(() => {
+    if (!client) {
+      return;
+    }
+
+    const merged = {...ctx.args, ...ctx.clientFields};
+    beginLiveScreen(client, merged);
+    onReleaseToolCall?.();
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [client, ctx.toolCallId]);
+
+  return null;
+}

package/src/presets/liveScreen/LiveScreenPipOverlay.tsx

@@ -0,0 +1,6 @@
+import type {PresetClient} from '../types';
+import {LiveScreenCaptureHost} from './LiveScreenCaptureHost';
+
+export function LiveScreenPipOverlay({client}: {client: PresetClient | null}) {
+  return <LiveScreenCaptureHost client={client} />;
+}

package/src/presets/liveScreen/liveScreenSession.ts

@@ -0,0 +1,73 @@
+import type {PresetClient} from '../types';
+
+export type LiveScreenSession = {
+  sessionId: string;
+  client: PresetClient;
+  args: Record<string, unknown>;
+};
+
+let activeSession: LiveScreenSession | null = null;
+const listeners = new Set<() => void>();
+
+function notify(): void {
+  listeners.forEach(listener => listener());
+}
+
+export function subscribeLiveScreen(listener: () => void): () => void {
+  listeners.add(listener);
+  return () => listeners.delete(listener);
+}
+
+export function getLiveScreenSession(): LiveScreenSession | null {
+  return activeSession;
+}
+
+export function beginLiveScreen(
+  client: PresetClient,
+  args: Record<string, unknown>,
+): string {
+  stopLiveScreen(client);
+
+  const sessionId = `ls_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
+  activeSession = {sessionId, client, args};
+  notify();
+  return sessionId;
+}
+
+export function stopLiveScreen(client: PresetClient | null): void {
+  if (!activeSession) {
+    return;
+  }
+
+  const sessionId = activeSession.sessionId;
+  const notifyClient = client ?? activeSession.client;
+  activeSession = null;
+  notify();
+
+  if (notifyClient?.sendEvent) {
+    notifyClient.sendEvent('client_live_screen_stopped', {
+      data: {session_id: sessionId},
+    });
+  }
+  console.log(`[LiveScreen] Session stopped: ${sessionId}`);
+}
+
+export function parseLiveScreenArgs(args: Record<string, unknown>) {
+  const scope = String(args.scope ?? 'until_call_end');
+  const maxFps = Number(args.max_fps ?? 3);
+  const targetFps = Math.min(Number(args.target_fps ?? 1), maxFps);
+  const maxWidth = Number(args.max_width ?? 1280);
+  const reason = String(args.reason ?? 'screen_capture');
+  const durationSec = Number(args.duration_sec ?? 0);
+  const showPreview =
+    args.show_preview !== false && args.show_preview !== 'false';
+
+  return {
+    scope,
+    targetFps,
+    maxWidth,
+    reason,
+    durationSec,
+    showPreview,
+  };
+}

package/src/presets/liveVision/CloseLiveVisionHandler.tsx

@@ -0,0 +1,29 @@
+import {useEffect} from 'react';
+import type {PresetClient, PresetContext} from '../types';
+import {stopLiveVision} from './liveVisionSession';
+
+type Props = {
+  client: PresetClient | null;
+  ctx: PresetContext;
+  onComplete: (payload: Record<string, unknown>) => void;
+  onDismiss?: () => void;
+};
+
+export function CloseLiveVisionHandler({
+  client,
+  ctx,
+  onComplete,
+  onDismiss,
+}: Props) {
+  useEffect(() => {
+    stopLiveVision(client);
+    onComplete({
+      status: 'success',
+      preset_id: ctx.presetId,
+      message: 'Live camera stopped',
+    });
+    onDismiss?.();
+  }, [client, ctx.presetId, onComplete, onDismiss]);
+
+  return null;
+}