@trustchex/react-native-sdk 1.381.0 → 1.409.0

package/src/Shared/Services/WebRTCService.ts

@@ -0,0 +1,636 @@
+import {
+  RTCPeerConnection,
+  RTCIceCandidate,
+  RTCSessionDescription,
+  mediaDevices,
+  MediaStream,
+} from 'react-native-webrtc';
+import {
+  SignalingClient,
+  type SignalingMessage,
+} from '../Libs/SignalingClient';
+
+export interface WebRTCConfig {
+  baseUrl: string;
+  sessionId: string;
+  identificationId?: string;
+  onRemoteStream: (stream: MediaStream) => void;
+  onConnectionStateChange: (state: string) => void;
+  onCommand?: (command: any) => void;
+  onSessionEnded?: (state: string) => void;
+  onSnapshotRequest?: (requestId: string) => void;
+  onLocalStreamUpdate?: (stream: MediaStream) => void;
+}
+
+export class WebRTCService {
+  private peerConnection: RTCPeerConnection | null = null;
+  private localStream: MediaStream | null = null;
+  private remoteStream: MediaStream | null = null;
+  private readonly signalingClient: SignalingClient;
+  private readonly onRemoteStream: (stream: MediaStream) => void;
+  private readonly onConnectionStateChange: (state: string) => void;
+  private readonly onCommand?: (command: any) => void;
+  private readonly onSnapshotRequest?: (requestId: string) => void;
+  private readonly onLocalStreamUpdate?: (stream: MediaStream) => void;
+  private readonly baseUrl: string;
+  private readonly sessionId: string;
+  private readonly identificationId?: string;
+  private isFlashOn: boolean = false;
+  private pendingCandidates: RTCIceCandidate[] = [];
+  private hasRemoteDescription = false;
+  private isProcessingOffer = false;
+  private lastProcessedOfferSdp: string | null = null;
+  private tracksAdded = false;
+  private pendingOffer: SignalingMessage | null = null;
+  private isSwitchingCamera = false;
+  private lastSwitchTime = 0;
+  private facingMode: 'user' | 'environment' = 'user';
+  private videoTrackEndedHandler: (() => void) | null = null;
+
+  constructor(config: WebRTCConfig) {
+    this.onRemoteStream = config.onRemoteStream;
+    this.onConnectionStateChange = config.onConnectionStateChange;
+    this.onCommand = config.onCommand;
+    this.onSnapshotRequest = config.onSnapshotRequest;
+    this.onLocalStreamUpdate = config.onLocalStreamUpdate;
+    this.baseUrl = config.baseUrl;
+    this.sessionId = config.sessionId;
+    this.identificationId = config.identificationId;
+
+    this.signalingClient = new SignalingClient(
+      config.baseUrl,
+      config.sessionId,
+      this.handleSignalingMessage.bind(this),
+      config.identificationId,
+      config.onSessionEnded
+    );
+  }
+
+  private async fetchIceServers(): Promise<object[]> {
+    try {
+      const params = this.identificationId
+        ? `?identificationId=${this.identificationId}`
+        : '';
+      const response = await fetch(
+        `${this.baseUrl}/api/app/mobile/video-sessions/${this.sessionId}/ice-servers${params}`
+      );
+      if (!response.ok) return [];
+      const data = await response.json();
+      return Array.isArray(data.iceServers) ? data.iceServers : [];
+    } catch {
+      return [];
+    }
+  }
+
+  public async initialize(): Promise<MediaStream> {
+    console.log('[WebRTCService] Initializing...');
+
+    try {
+      // Fetch ICE servers before getUserMedia so peer connection is ready sooner
+      const iceServers = await this.fetchIceServers();
+
+      // Initialize PeerConnection before getUserMedia so signaling can start
+      // immediately — avoids missing the agent's offer while camera init blocks.
+      this.peerConnection = new RTCPeerConnection({
+        iceServers,
+        iceCandidatePoolSize: 2,
+      });
+
+      // Set up event handlers before signaling connects.
+      // Use addEventListener instead of property assignment for reliable
+      // delivery in react-native-webrtc (property may be overwritten).
+      (this.peerConnection as any).addEventListener('track', (event: any) => {
+        if (event.streams && event.streams[0]) {
+          this.remoteStream = event.streams[0];
+          this.onRemoteStream(this.remoteStream);
+        } else if (event.track) {
+          const existingTracks = this.remoteStream
+            ? this.remoteStream.getTracks()
+            : [];
+          this.remoteStream = new MediaStream([...existingTracks, event.track]);
+          this.onRemoteStream(this.remoteStream);
+        }
+      });
+
+      (this.peerConnection as any).addEventListener(
+        'icecandidate',
+        (event: any) => {
+          if (event.candidate) {
+            this.signalingClient.send(
+              'ice-candidate',
+              event.candidate.toJSON()
+            );
+          }
+        }
+      );
+
+      // react-native-webrtc often does not fire connectionstatechange or sets
+      // connectionState to undefined. Use iceconnectionstatechange which is
+      // reliably implemented across all versions.
+      (this.peerConnection as any).addEventListener(
+        'iceconnectionstatechange',
+        () => {
+          if (!this.peerConnection) return;
+          const iceState: string =
+            (this.peerConnection as any).iceConnectionState || 'new';
+          // Map ICE states to the connection states the UI expects
+          const mapped =
+            iceState === 'connected' || iceState === 'completed'
+              ? 'connected'
+              : iceState === 'failed'
+                ? 'failed'
+                : iceState === 'disconnected'
+                  ? 'disconnected'
+                  : iceState === 'closed'
+                    ? 'closed'
+                    : iceState;
+          this.onConnectionStateChange(mapped);
+          if (mapped === 'connected') {
+            this.sendTorchAvailability();
+          }
+        }
+      );
+
+      // Connect signaling now — peer connection + handlers are ready.
+      // Any offer that arrives before local tracks are added is buffered in
+      // pendingOffer and processed once tracks are attached below.
+      this.signalingClient.connect();
+
+      // Get local stream. Avoid strict min constraints that hang on some Android
+      // devices when the camera cannot satisfy the combination.
+      const stream = await mediaDevices.getUserMedia({
+        audio: true,
+        video: {
+          facingMode: 'user',
+          frameRate: { ideal: 30 },
+        },
+      });
+
+      this.localStream = stream;
+
+      // Add local tracks
+      stream.getTracks().forEach((track) => {
+        this.peerConnection?.addTrack(track, stream);
+      });
+      this.tracksAdded = true;
+
+      // If the agent's offer arrived while getUserMedia was pending, process it now.
+      if (this.pendingOffer) {
+        const buffered = this.pendingOffer;
+        this.pendingOffer = null;
+        await this.handleSignalingMessage(buffered);
+      }
+
+      this.sendTorchAvailability();
+
+      return stream;
+    } catch (error) {
+      console.error('[WebRTCService] Initialization failed:', error);
+      throw error;
+    }
+  }
+
+  public async handleSignalingMessage(message: SignalingMessage) {
+    if (!this.peerConnection) return;
+
+    try {
+      switch (message.type) {
+        case 'offer': {
+          // Buffer the offer until local tracks have been added so the answer
+          // includes the mobile's audio/video tracks.
+          if (!this.tracksAdded) {
+            console.log(
+              '[WebRTCService] Offer received before tracks ready, buffering'
+            );
+            this.pendingOffer = message;
+            return;
+          }
+
+          // Skip if we're already processing an offer
+          if (this.isProcessingOffer) {
+            console.log(
+              '[WebRTCService] Skipping offer - already processing one'
+            );
+            return;
+          }
+
+          const offerSdp: string | undefined = message.payload?.sdp;
+          if (offerSdp && this.lastProcessedOfferSdp === offerSdp) {
+            console.log('[WebRTCService] Skipping offer - duplicate SDP');
+            return;
+          }
+
+          this.isProcessingOffer = true;
+          console.log('[WebRTCService] Received offer');
+
+          try {
+            await this.peerConnection.setRemoteDescription(
+              new RTCSessionDescription(message.payload)
+            );
+            this.hasRemoteDescription = true;
+            // Add any pending candidates
+            await this.addPendingCandidates();
+            const answer = await this.peerConnection.createAnswer();
+            await this.peerConnection.setLocalDescription(answer);
+            await this.signalingClient.send('answer', {
+              type: answer.type,
+              sdp: answer.sdp,
+            });
+            this.lastProcessedOfferSdp = offerSdp ?? null;
+          } finally {
+            this.isProcessingOffer = false;
+          }
+          break;
+        }
+
+        case 'answer':
+          console.log('[WebRTCService] Received answer');
+          await this.peerConnection.setRemoteDescription(
+            new RTCSessionDescription(message.payload)
+          );
+          this.hasRemoteDescription = true;
+          // Add any pending candidates
+          await this.addPendingCandidates();
+          break;
+
+        case 'ice-candidate':
+          if (message.payload) {
+            const candidate = new RTCIceCandidate(message.payload);
+            if (this.hasRemoteDescription) {
+              console.log('[WebRTCService] Adding ICE candidate');
+              await this.peerConnection.addIceCandidate(candidate);
+            } else {
+              this.pendingCandidates.push(candidate);
+            }
+          }
+          break;
+
+        case 'command':
+          this.handleCommand(message.payload);
+          break;
+      }
+    } catch (error) {
+      console.error('[WebRTCService] Error handling signaling:', error);
+    }
+  }
+
+  private handleCommand(payload: any) {
+    console.log('[WebRTCService] Received command:', payload);
+
+    if (this.onCommand) {
+      this.onCommand(payload);
+    }
+
+    switch (payload.type) {
+      case 'switchCamera':
+        this.switchCamera();
+        break;
+      case 'toggleFlash':
+        this.toggleFlash();
+        break;
+      case 'captureSnapshot':
+        this.captureSnapshot(payload.requestId);
+        break;
+      case 'takeSnapshot':
+        this.captureSnapshot(payload.requestId);
+        break;
+    }
+  }
+
+  private async addPendingCandidates() {
+    if (this.pendingCandidates.length > 0 && this.peerConnection) {
+      console.log(
+        `[WebRTCService] Adding ${this.pendingCandidates.length} pending ICE candidates`
+      );
+      for (const candidate of this.pendingCandidates) {
+        try {
+          await this.peerConnection.addIceCandidate(candidate);
+        } catch (error) {
+          console.error(
+            '[WebRTCService] Error adding pending candidate:',
+            error
+          );
+        }
+      }
+      this.pendingCandidates = [];
+    }
+  }
+
+  public cleanup() {
+    this.localStream?.getTracks().forEach((t) => t.stop());
+    this.peerConnection?.close();
+    this.signalingClient.disconnect();
+    this.localStream = null;
+    this.remoteStream = null;
+    this.peerConnection = null;
+    this.pendingCandidates = [];
+    this.hasRemoteDescription = false;
+    this.lastProcessedOfferSdp = null;
+    this.isProcessingOffer = false;
+    this.tracksAdded = false;
+    this.pendingOffer = null;
+  }
+
+  public async switchCamera() {
+    // Debounce: ignore if already switching or within 2s cooldown
+    const now = Date.now();
+    if (this.isSwitchingCamera || now - this.lastSwitchTime < 2000) {
+      console.warn('[WebRTCService] switchCamera ignored — cooldown');
+      return;
+    }
+
+    if (!this.localStream || !this.peerConnection) return;
+
+    const videoTrack = this.localStream.getVideoTracks()[0];
+    if (!videoTrack) return;
+
+    this.isSwitchingCamera = true;
+    this.lastSwitchTime = now;
+
+    try {
+      const newFacing = this.facingMode === 'user' ? 'environment' : 'user';
+
+      // First, remove the video track from the sender (set to null).
+      // This signals to the encoder to stop, which releases the camera HAL.
+      const senders = (this.peerConnection as any).getSenders?.() as any[];
+      let videoSender: any = null;
+      if (senders) {
+        for (const sender of senders) {
+          if (sender.track?.kind === 'video') {
+            videoSender = sender;
+            await sender.replaceTrack(null);
+            break;
+          }
+        }
+      }
+
+      // Stop the old track to fully release the camera hardware.
+      videoTrack.stop();
+
+      // Small delay to let the camera HAL fully release on Android.
+      await new Promise((res) => setTimeout(res, 200));
+
+      // Acquire a new stream from the target camera. Now that the old camera
+      // is fully released, there's no concurrent camera conflict.
+      const newStream = await mediaDevices.getUserMedia({
+        audio: false,
+        video: {
+          facingMode: newFacing,
+          frameRate: { ideal: 30 },
+        },
+      });
+
+      const newVideoTrack = newStream.getVideoTracks()[0];
+      if (!newVideoTrack) {
+        console.warn(
+          '[WebRTCService] switchCamera: no video track from getUserMedia'
+        );
+        return;
+      }
+
+      // Replace the null track in the sender with the new camera track.
+      // This properly notifies the WebRTC encoder to start encoding from
+      // the new camera — unlike _switchCamera() which only changes the
+      // hardware source without informing the encoder/remote side.
+      if (videoSender) {
+        await videoSender.replaceTrack(newVideoTrack);
+      }
+
+      // Build a new MediaStream with existing audio + new video.
+      const audioTracks = this.localStream.getAudioTracks();
+      this.localStream = new MediaStream([...audioTracks, newVideoTrack]);
+      this.facingMode = newFacing;
+
+      // Listen for the track ending unexpectedly (Android camera HAL power
+      // management can kill the rear camera mid-call). Re-acquire on ended.
+      if (this.videoTrackEndedHandler) {
+        (videoTrack as any).removeEventListener?.(
+          'ended',
+          this.videoTrackEndedHandler
+        );
+      }
+      this.videoTrackEndedHandler = () => {
+        console.warn(
+          '[WebRTCService] video track ended unexpectedly — re-acquiring'
+        );
+        this.reacquireVideoTrack();
+      };
+      (newVideoTrack as any).addEventListener?.(
+        'ended',
+        this.videoTrackEndedHandler
+      );
+
+      // Notify the screen to update its local stream reference and remount RTCView
+      this.onLocalStreamUpdate?.(this.localStream);
+    } catch (error) {
+      console.warn('[WebRTCService] switchCamera failed:', error);
+    } finally {
+      this.isSwitchingCamera = false;
+    }
+  }
+
+  private async reacquireVideoTrack() {
+    if (!this.localStream || !this.peerConnection) return;
+    try {
+      const newStream = await mediaDevices.getUserMedia({
+        audio: false,
+        video: { facingMode: this.facingMode, frameRate: { ideal: 30 } },
+      });
+      const newVideoTrack = newStream.getVideoTracks()[0];
+      if (!newVideoTrack) return;
+
+      const senders = (this.peerConnection as any).getSenders?.() as any[];
+      if (senders) {
+        for (const sender of senders) {
+          if (sender.track?.kind === 'video' || sender.track === null) {
+            await sender.replaceTrack(newVideoTrack);
+            break;
+          }
+        }
+      }
+
+      if (this.videoTrackEndedHandler) {
+        (newVideoTrack as any).addEventListener?.(
+          'ended',
+          this.videoTrackEndedHandler
+        );
+      }
+
+      const audioTracks = this.localStream.getAudioTracks();
+      this.localStream = new MediaStream([...audioTracks, newVideoTrack]);
+      this.onLocalStreamUpdate?.(this.localStream);
+    } catch (error) {
+      console.warn('[WebRTCService] reacquireVideoTrack failed:', error);
+    }
+  }
+
+  public async toggleFlash() {
+    const videoTrack = this.localStream?.getVideoTracks()[0];
+    if (!videoTrack) {
+      console.warn('[WebRTCService] No video track available for flash toggle');
+      return;
+    }
+
+    this.isFlashOn = !this.isFlashOn;
+
+    // applyConstraints is the only reliable method for torch on Android.
+    // Retry up to 3 times with 300ms delay to handle "Camera switch already in progress" race.
+    let success = false;
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        await (videoTrack as any).applyConstraints({
+          advanced: [{ torch: this.isFlashOn }],
+        });
+        success = true;
+        break;
+      } catch (error: any) {
+        const msg: string = error?.message || '';
+        if (msg.includes('Camera switch') && attempt < 2) {
+          await new Promise((res) => setTimeout(res, 300));
+          continue;
+        }
+        console.warn('[WebRTCService] Failed to toggle flash:', error);
+        this.isFlashOn = !this.isFlashOn; // revert
+        this.signalingClient.send('command', {
+          type: 'torch_availability',
+          available: false,
+        });
+        return;
+      }
+    }
+
+    if (!success) {
+      this.isFlashOn = !this.isFlashOn; // revert
+      this.signalingClient.send('command', {
+        type: 'torch_availability',
+        available: false,
+      });
+      return;
+    }
+
+    // Confirm flash is available now that we know it works
+    this.signalingClient.send('command', {
+      type: 'torch_availability',
+      available: true,
+    });
+
+    this.signalingClient.send('command', {
+      type: 'flash_state',
+      isFlashOn: this.isFlashOn,
+    });
+  }
+
+  private sendTorchAvailability() {
+    const videoTrack = this.localStream?.getVideoTracks()[0];
+    if (!videoTrack) {
+      this.signalingClient.send('command', {
+        type: 'torch_availability',
+        available: false,
+      });
+      return;
+    }
+
+    let capabilities: any;
+
+    try {
+      capabilities = (videoTrack as any).getCapabilities?.();
+    } catch {
+      capabilities = undefined;
+    }
+
+    // Only trust getCapabilities().torch — _setTorch exists on every Android
+    // track regardless of whether the camera actually has a flash.
+    const available = !!capabilities?.torch;
+
+    this.signalingClient.send('command', {
+      type: 'torch_availability',
+      available,
+    });
+  }
+
+  /**
+   * Handle snapshot capture request. Captures a frame from the local video track directly.
+   */
+  private async captureSnapshot(requestId?: string) {
+    console.log('[WebRTCService] Snapshot requested, requestId:', requestId);
+
+    if (this.onSnapshotRequest && requestId) {
+      // Delegate to parent component which has access to the RTCView
+      this.onSnapshotRequest(requestId);
+      return;
+    }
+
+    // Capture directly from local video track via react-native-webrtc captureFrame()
+    const videoTrack = this.localStream?.getVideoTracks()[0];
+    if (!videoTrack) {
+      console.error('[WebRTCService] No local video track for snapshot');
+      return;
+    }
+
+    try {
+      const frame = await (videoTrack as any).captureFrame();
+      const base64 = frame?.data;
+      if (!base64) {
+        console.error('[WebRTCService] captureFrame returned no data');
+        return;
+      }
+      await this.uploadSnapshot(requestId, base64);
+    } catch (error) {
+      console.error('[WebRTCService] captureFrame failed:', error);
+    }
+  }
+
+  /**
+   * Upload a captured snapshot image (called by parent component after capturing RTCView).
+   */
+  public async uploadSnapshot(
+    requestId: string,
+    base64Image: string
+  ): Promise<void> {
+    console.log('[WebRTCService] Uploading snapshot, requestId:', requestId);
+
+    try {
+      const formData = new FormData();
+      // Convert base64 to Blob via fetch data URL (works in React Native)
+      const dataResponse = await fetch(`data:image/jpeg;base64,${base64Image}`);
+      const blob = await dataResponse.blob();
+      formData.append('image', blob);
+
+      const uploadUrl = `${this.baseUrl}/api/app/mobile/video-sessions/${this.sessionId}/snapshots?identificationId=${this.identificationId}`;
+      const response = await fetch(uploadUrl, {
+        method: 'POST',
+        body: formData,
+      });
+
+      if (response.ok) {
+        const result = await response.json();
+        console.log('[WebRTCService] Snapshot uploaded:', result);
+        await this.signalingClient.send('command', {
+          type: 'snapshotReady',
+          snapshotId: result.snapshotId,
+          url: result.url,
+          createdAt: new Date().toISOString(),
+        });
+      } else {
+        const errorText = await response.text();
+        console.error(
+          '[WebRTCService] Snapshot upload failed:',
+          response.status,
+          errorText
+        );
+        await this.signalingClient.send('command', {
+          type: 'snapshotError',
+          requestId,
+          error: `Upload failed: ${response.status}`,
+        });
+      }
+    } catch (error) {
+      console.error('[WebRTCService] Snapshot upload error:', error);
+      await this.signalingClient.send('command', {
+        type: 'snapshotError',
+        requestId,
+        error: String(error),
+      });
+    }
+  }
+}

package/src/Shared/Types/analytics.types.ts

@@ -82,6 +82,8 @@ export enum AnalyticsEventName {
   IDENTITY_DOCUMENT_EID_SCAN_COMPLETED = 'identity_document_eid_scan_completed',
   LIVENESS_CHECK_STARTED = 'liveness_check_started',
   LIVENESS_CHECK_COMPLETED = 'liveness_check_completed',
+  VIDEO_CALL_STARTED = 'video_call_started',
+  VIDEO_CALL_COMPLETED = 'video_call_completed',
 
   // NFC Scan Events (used by trackNFCScan* helpers)
   NFC_SCAN_STARTED = 'nfc_scan_started',

package/src/Shared/Types/identificationInfo.ts

@@ -11,6 +11,9 @@ export interface IdentificationInfo {
   scannedDocument?: ScannedIdentityDocument;
   livenessDetection?: LivenessDetection;
   locale: string;
+  authToken?: string;
+  videoSessionId?: string;
+  mediaUploadedDuringVideoCall?: boolean;
 }
 
 export interface ScannedIdentityDocument {
@@ -39,7 +42,8 @@ export interface WorkflowStep {
     | 'IDENTITY_DOCUMENT_SCAN'
     | 'IDENTITY_DOCUMENT_EID_SCAN'
     | 'LIVENESS_CHECK'
-    | 'AML_CHECK';
+    | 'AML_CHECK'
+    | 'VIDEO_CALL';
   data?: {
     contracts?: {
       en: {

package/src/Translation/Resources/en.ts

@@ -158,6 +158,31 @@ export default {
   'identityDocumentCamera.documentTooLarge': 'Too close. Move back.',
   'identityDocumentCamera.holdSteady': 'Keep device steady',
   'identityDocumentCamera.centerDocument': 'Center document in the frame',
+  'videoCallScreen.guideHeader': 'Get Ready for Video Call',
+  'videoCallScreen.guideText': 'Before you begin, please note the following:',
+  'videoCallScreen.guidePoint1':
+    'Ensure you are in a quiet environment with good lighting',
+  'videoCallScreen.guidePoint2':
+    'Have your identity document ready if requested',
+  'videoCallScreen.guidePoint3': 'Make sure your internet connection is stable',
+  'videoCallScreen.guidePoint4':
+    'The agent will guide you through the verification process',
+  'videoCallScreen.waitingForAgent': 'Waiting for agent...',
+  'videoCallScreen.queuePosition': 'Queue position: {position}',
+  'videoCallScreen.new': 'Initializing connection...',
+  'videoCallScreen.connecting': 'Connecting...',
+  'videoCallScreen.connected': 'Connected',
+  'videoCallScreen.disconnected': 'Disconnected',
+  'videoCallScreen.failed': 'Connection failed',
+  'videoCallScreen.closed': 'Connection closed',
+  'videoCallScreen.permissions_denied': 'Camera permissions required',
+  'videoCallScreen.leaveQueue': 'Leave Queue',
+  'videoCallScreen.closeSession': 'Close Session',
+  'videoCallScreen.closeSessionHint':
+    'You can close this session and start a new verification',
+  'videoCallScreen.agentInstructions': 'Instructions from Agent',
+  'videoCallScreen.callNotCompleted':
+    'Video call was not completed. This step is required to proceed.',
   'navigationManager.skipStepWarning':
     'Completing this step is recommended for successful verification. Skip anyway?',
   'navigationManager.skipStepLabel': 'Skip This Step',